├── .coveragerc
├── .flake8
├── .github
└── workflows
│ ├── publish_pypi.yml
│ └── run_tests.yml
├── .gitignore
├── LICENSE
├── MANIFEST.in
├── README.md
├── doc
└── images
│ ├── switch_versions.png
│ └── transfer_accounts.png
├── requirements.dev.txt
├── requirements.test.txt
├── requirements.txt
├── setup.py
├── sf_git.test.conf
├── sf_git
├── __init__.py
├── cache.py
├── cli.py
├── commands.py
├── config.py
├── git_utils.py
├── models.py
├── rest_utils.py
├── sf_git.conf
├── snowsight_auth.py
└── worksheets_utils.py
└── tests
├── conftest.py
├── data
├── Benchmarking_Tutorials
│ ├── .[Tutorial]_Sample_queries_on_TPC-DS_data_metadata.json
│ ├── .[Tutorial]_Sample_queries_on_TPC-H_data_metadata.json
│ ├── .[Tutorial]_TPC-DS_100TB_Complete_Query_Test_metadata.json
│ ├── .[Tutorial]_TPC-DS_10TB_Complete_Query_Test_metadata.json
│ ├── [Tutorial]_Sample_queries_on_TPC-DS_data.sql
│ ├── [Tutorial]_Sample_queries_on_TPC-H_data.sql
│ ├── [Tutorial]_TPC-DS_100TB_Complete_Query_Test.sql
│ └── [Tutorial]_TPC-DS_10TB_Complete_Query_Test.sql
└── Getting_Started_Tutorials
│ ├── .[Template]_Adding_a_user_and_granting_roles_metadata.json
│ ├── .[Tutorial]_Using_Python_to_load_and_query_sample_data_metadata.json
│ ├── .[Tutorial]_Using_SQL_to_load_and_query_sample_data_metadata.json
│ ├── [Template]_Adding_a_user_and_granting_roles.sql
│ ├── [Tutorial]_Using_Python_to_load_and_query_sample_data.py
│ └── [Tutorial]_Using_SQL_to_load_and_query_sample_data.sql
├── fixtures
├── contents.json
├── entities.json
├── folders.json
├── worksheets.json
└── worksheets_update.json
├── test_cache.py
├── test_commands.py
├── test_config.py
└── test_worksheets_utils.py
/.coveragerc:
--------------------------------------------------------------------------------
1 | [run]
2 | source = sf_git
3 | omit =
4 | *tests*
5 | sf_git/cli.py
6 | sf_git/snowsight_auth.py
7 | sf_git/rest_utils.py
8 | disable_warnings = no-data-collected
9 | branch = True
10 |
11 | [html]
12 | directory = coverage_html_report
--------------------------------------------------------------------------------
/.flake8:
--------------------------------------------------------------------------------
1 | [flake8]
2 | ignore = D203
3 | exclude =
4 | .git,
5 | __pycache__,
6 | old,
7 | build,
8 | dist
9 | max-complexity = 10
10 | max-line-length = 80
11 |
--------------------------------------------------------------------------------
/.github/workflows/publish_pypi.yml:
--------------------------------------------------------------------------------
1 | name: Publish to PyPI and TestPyPi
2 |
3 | on:
4 | push:
5 | branches:
6 | - releases
7 | workflow_dispatch:
8 |
9 | jobs:
10 | build:
11 | name: Build distribution
12 | if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' # only publish to PyPI on tag pushes
13 | runs-on: ubuntu-latest
14 | environment: production
15 |
16 | steps:
17 | - uses: actions/checkout@v4
18 | - name: Set up Python
19 | uses: actions/setup-python@v4
20 | with:
21 | python-version: "3.10"
22 | - name: Install pypa/build
23 | run: >-
24 | python3 -m
25 | pip install
26 | setuptools-scm wheel
27 | - name: Build a binary wheel and a source tarball
28 | run: python3 setup.py sdist bdist_wheel
29 | - name: Store the distribution packages
30 | uses: actions/upload-artifact@v3
31 | with:
32 | name: python-package-distributions
33 | path: dist/
34 |
35 | # publish-to-pypi:
36 | # name: >-
37 | # Publish to PyPI
38 | # if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' # only publish to PyPI on tag pushes
39 | # needs:
40 | # - build
41 | # runs-on: ubuntu-latest
42 | # environment:
43 | # name: production
44 | # url: https://pypi.org/p/sf_git
45 | # permissions:
46 | # id-token: write # IMPORTANT: mandatory for trusted publishing
47 | #
48 | # steps:
49 | # - name: Download all the dists
50 | # uses: actions/download-artifact@v3
51 | # with:
52 | # name: python-package-distributions
53 | # path: dist/
54 | # - name: Publish to PyPI
55 | # uses: pypa/gh-action-pypi-publish@release/v1
56 |
57 | # github-release:
58 | # name: >-
59 | # Sign the with Sigstore
60 | # and upload them to GitHub Release
61 | # if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' # only publish to PyPI on tag pushes
62 | # needs:
63 | # - publish-to-pypi
64 | # runs-on: ubuntu-latest
65 | # environment: production
66 | #
67 | # permissions:
68 | # contents: write # IMPORTANT: mandatory for making GitHub Releases
69 | # id-token: write # IMPORTANT: mandatory for sigstore
70 | #
71 | # steps:
72 | # - name: Download all the dists
73 | # uses: actions/download-artifact@v3
74 | # with:
75 | # name: python-package-distributions
76 | # path: dist/
77 | # - name: Sign the dists with Sigstore
78 | # uses: sigstore/gh-action-sigstore-python@v1.2.3
79 | # with:
80 | # inputs: >-
81 | # ./dist/*.tar.gz
82 | # ./dist/*.whl
83 | # - name: Create GitHub Release
84 | # env:
85 | # GITHUB_TOKEN: ${{ github.token }}
86 | # run: >-
87 | # gh release create
88 | # '${{ github.ref_name }}'
89 | # --repo '${{ github.repository }}'
90 | # --notes ""
91 | # - name: Upload artifact signatures to GitHub Release
92 | # env:
93 | # GITHUB_TOKEN: ${{ github.token }}
94 | # # Upload to GitHub Release using the `gh` CLI.
95 | # # `dist/` contains the built packages, and the
96 | # # sigstore-produced signatures and certificates.
97 | # run: >-
98 | # gh release upload
99 | # '${{ github.ref_name }}' dist/**
100 | # --repo '${{ github.repository }}'
101 |
102 | publish-to-testpypi:
103 | name: Publish to TestPyPI
104 | needs:
105 | - build
106 | runs-on: ubuntu-latest
107 |
108 | environment:
109 | name: test
110 | url: https://test.pypi.org/p/sf_git
111 |
112 | permissions:
113 | id-token: write # IMPORTANT: mandatory for trusted publishing
114 |
115 | steps:
116 | - name: Download all the dists
117 | uses: actions/download-artifact@v3
118 | with:
119 | name: python-package-distributions
120 | path: dist/
121 | - name: Publish to TestPyPI
122 | uses: pypa/gh-action-pypi-publish@release/v1
123 | with:
124 | repository-url: https://test.pypi.org/legacy/
125 |
126 |
--------------------------------------------------------------------------------
/.github/workflows/run_tests.yml:
--------------------------------------------------------------------------------
1 | name: Run Unit Test via Pytest
2 |
3 | on:
4 | push:
5 | branches:
6 | - main
7 | paths:
8 | - 'sf_git/**.py'
9 | workflow_dispatch:
10 |
11 | jobs:
12 | build:
13 | runs-on: ubuntu-latest
14 | environment: pytest
15 | strategy:
16 | matrix:
17 | python-version: ["3.10"]
18 |
19 | steps:
20 | - uses: actions/checkout@v3
21 | - name: Set up Python ${{ matrix.python-version }}
22 | uses: actions/setup-python@v4
23 | with:
24 | python-version: ${{ matrix.python-version }}
25 | - name: Install dependencies
26 | run: |
27 | python -m pip install --upgrade pip
28 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
29 | if [ -f requirements.test.txt ]; then pip install -r requirements.test.txt; fi
30 | - name: set git default branch to main
31 | run: |
32 | git config --global init.defaultBranch main
33 | - name: Lint with Ruff
34 | run: |
35 | pip install flake8
36 | flake8 sf_git
37 | continue-on-error: true
38 | - name: Test with pytest
39 | run: |
40 | coverage run -m pytest tests -v -s
41 | - name: Generate Coverage Report
42 | run: |
43 | REPORT="$(coverage report -m | tail -1)"
44 | COV_TOTAL="$(echo $REPORT | tail -c 5 | sed 's/ //g' | sed 's/%//g')"
45 | echo "COVERAGE=$COV_TOTAL" >> $GITHUB_ENV
46 | # - name: Upload coverage data
47 | # uses: actions/upload-artifact@v3
48 | # with:
49 | # name: "coverage-data"
50 | # path: .coverage.*
51 | # if-no-files-found: ignore
52 | # - name: Extract branch name
53 | # shell: bash
54 | # run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
55 | # id: extract_branch
56 | - name: Format branch name for cov badge
57 | shell: bash
58 | run: |
59 | BRANCH_NAME="$(echo ${GITHUB_REF_NAME} | sed 's/\//_/g')"
60 | echo "BRANCH_NAME=${BRANCH_NAME}" >> $GITHUB_ENV
61 | - name: Create badge
62 | uses: schneegans/dynamic-badges-action@v1.7.0
63 | with:
64 | auth: ${{ secrets.GIST_TOKEN }}
65 | gistID: e2c293d7db07bee70d2845387cb133ff
66 | filename: sf_git_${{ env.BRANCH_NAME }}_cov_badge.json
67 | label: sf_git coverage
68 | message: ${{ env.COVERAGE }}%
69 | minColorRange: 50
70 | maxColorRange: 90
71 | valColorRange: ${{ env.COVERAGE }}
72 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | build/
12 | develop-eggs/
13 | dist/
14 | downloads/
15 | eggs/
16 | .eggs/
17 | lib/
18 | lib64/
19 | parts/
20 | sdist/
21 | var/
22 | wheels/
23 | share/python-wheels/
24 | *.egg-info/
25 | .installed.cfg
26 | *.egg
27 | MANIFEST
28 | version.txt
29 |
30 | # PyInstaller
31 | # Usually these files are written by a python script from a template
32 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
33 | *.manifest
34 | *.spec
35 |
36 | # Installer logs
37 | pip-log.txt
38 | pip-delete-this-directory.txt
39 |
40 | # Unit test / coverage reports
41 | htmlcov/
42 | .tox/
43 | .nox/
44 | .coverage
45 | .coverage.*
46 | .cache
47 | nosetests.xml
48 | coverage.xml
49 | *.cover
50 | *.py,cover
51 | .hypothesis/
52 | .pytest_cache/
53 | cover/
54 |
55 | # Translations
56 | *.mo
57 | *.pot
58 |
59 | # Django stuff:
60 | *.log
61 | local_settings.py
62 | db.sqlite3
63 | db.sqlite3-journal
64 |
65 | # Flask stuff:
66 | instance/
67 | .webassets-cache
68 |
69 | # Scrapy stuff:
70 | .scrapy
71 |
72 | # Sphinx documentation
73 | docs/_build/
74 |
75 | # PyBuilder
76 | .pybuilder/
77 | target/
78 |
79 | # Jupyter Notebook
80 | .ipynb_checkpoints
81 |
82 | # IPython
83 | profile_default/
84 | ipython_config.py
85 |
86 | # pyenv
87 | # For a library or package, you might want to ignore these files since the code is
88 | # intended to run in multiple environments; otherwise, check them in:
89 | # .python-version
90 |
91 | # pipenv
92 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
93 | # However, in case of collaboration, if having platform-specific dependencies or dependencies
94 | # having no cross-platform support, pipenv may install dependencies that don't work, or not
95 | # install all needed dependencies.
96 | #Pipfile.lock
97 |
98 | # poetry
99 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
100 | # This is especially recommended for binary packages to ensure reproducibility, and is more
101 | # commonly ignored for libraries.
102 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
103 | #poetry.lock
104 |
105 | # pdm
106 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
107 | #pdm.lock
108 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
109 | # in version control.
110 | # https://pdm.fming.dev/#use-with-ide
111 | .pdm.toml
112 |
113 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
114 | __pypackages__/
115 |
116 | # Celery stuff
117 | celerybeat-schedule
118 | celerybeat.pid
119 |
120 | # SageMath parsed files
121 | *.sage.py
122 |
123 | # Environments
124 | .env
125 | .venv
126 | env/
127 | venv/
128 | ENV/
129 | env.bak/
130 | venv.bak/
131 |
132 | # Spyder project settings
133 | .spyderproject
134 | .spyproject
135 |
136 | # Rope project settings
137 | .ropeproject
138 |
139 | # mkdocs documentation
140 | /site
141 |
142 | # mypy
143 | .mypy_cache/
144 | .dmypy.json
145 | dmypy.json
146 |
147 | # Pyre type checker
148 | .pyre/
149 |
150 | # pytype static type analyzer
151 | .pytype/
152 |
153 | # Cython debug symbols
154 | cython_debug/
155 |
156 | # PyCharm
157 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
158 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
159 | # and can be added to the global gitignore or merged into this file. For a more nuclear
160 | # option (not recommended) you can uncomment the following to ignore the entire idea folder.
161 | .idea/
162 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2024 tdambrin
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
1 | include sf_git.conf
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | 
6 | [](mailto:thomas.dambrin@gmail.com?subject=[GitHub]%20Snowflake%20Git%20Versioning)
7 | [](https://opensource.org/licenses/MIT)
8 | 
9 |
10 |
11 |
12 | # Snowflake Git
13 |
14 | 🆕 Git now supported in Snowflake, see the [official documentation](https://docs.snowflake.com/en/developer-guide/git/git-overview).
15 |
16 | # Worksheet Versioning
17 |
18 | Inspired by a snowflake developers maintained [repository](https://github.com/Snowflake-Labs/sfsnowsightextensions).
19 |
20 | ### Git integration
21 |
22 | The extension is designed to **apply git versioning on worksheets while developing on Snowsight, fully taking advantage of its functionalities**.\
23 | The following workflow is advised :
24 | 1. [Start session] Upload worksheet from local branch to a user Snowsight workspace
25 | 2. Test, update and validate on Snowsight
26 | 3. [End session] Update local branch with modified Snowsight worksheets
27 |
28 | ## Install
29 |
30 | Entry points are accessible through a CLI once the package is installed.
31 | To install it, please follow the following steps :
32 |
33 | ```bash
34 | # [Optional: Python virtual environement]
35 | $ pyenv virtualenv 3.10.4 sf
36 | $ pyenv activate sf
37 |
38 | # [Mandatory: Pip install]
39 | $ pip install -U pip
40 | $ pip install sf_git==1.4.2
41 |
42 | # [Check your installation]
43 | $ sfgit --version
44 | # [Should result in:]
45 | # sfgit, version 1.4.2
46 |
47 |
48 | # [Check your installation]
49 | $ sfgit --help
50 |
51 | # [Should result in:]
52 | # Usage: sfgit [OPTIONS] COMMAND [ARGS]...
53 | #
54 | # Options:
55 | # --help Show this message and exit.
56 | #
57 | # Commands:
58 | # commit Commit Snowsight worksheets to Git repository.
59 | # config Configure sfgit for easier version control.
60 | # fetch Fetch worksheets from user Snowsight account and store them in...
61 | # init Initialize a git repository and set it as the sfgit versioning...
62 | # push Upload locally stored worksheets to Snowsight user workspace.
63 | ```
64 |
65 | Commands have been created to **import/export (respectively fetch/push) snowsight worksheets to/from local**.
66 |
67 | ## Configure your git
68 |
69 | > **Warning**
70 | > A git repository is necessary to manage worksheets. You can either use an existing one
71 | > or create a new one.
72 |
73 | To apply versioning to your worksheets, you need to **configure Git information**
74 | through the config command.
75 |
76 | First, set git repository to be used:
77 |
78 | ```bash
79 | # if you want to use an existing git repository
80 | $ sfgit config --git-repo
81 |
82 | # if you want to create a new one
83 | $ sfgit init -p
84 | ```
85 |
86 | Then, set a location to save your worksheets within this git repository:
87 | ```bash
88 | $ sfgit config --save-dir
89 | ```
90 |
91 | ## Authenticate
92 | Currently, only authentication mode supported is the credentials (PWD) mode.
93 |
94 | > :warning: The single sign-on (SSO) will be fixed.
95 |
96 | Commands requiring Snowsight authentication all have options to provide at command time.
97 | If you don't want to manually input them everytime, you can set them at Python/Virtual environement level with :
98 |
99 |
100 | ```bash
101 | $ sfgit config --account
102 | $ sfgit config --username
103 | $ sfgit config --password # unnecessary for SSO authentication mode
104 | ```
105 |
106 | ### Account ID
107 |
108 | > [!WARNING]
109 | > `The account ID to be configured is in the .. format.`
110 |
111 | If you are unsure about how to retrieve it for your snowflake account, you can run this query:
112 |
113 | ```sql
114 | SHOW REGIONS;
115 |
116 | WITH
117 | SF_REGIONS AS (SELECT * FROM TABLE(RESULT_SCAN(LAST_QUERY_ID()))),
118 | INFOS AS (SELECT CURRENT_REGION() AS CR, CURRENT_ACCOUNT() AS CA)
119 | SELECT CONCAT(
120 | LOWER(INFOS.CA),
121 | '.',
122 | SF_REGIONS."region",
123 | '.',
124 | SF_REGIONS."cloud"
125 | ) AS account_id
126 | FROM INFOS LEFT JOIN SF_REGIONS ON INFOS.CR = SF_REGIONS."snowflake_region";
127 |
128 | ```
129 |
130 | Unfortunately, the _region_ is not always the same in the _SHOW REGIONS_ ouput. Please check and adapt the format comforming to the official [documentation](https://docs.snowflake.com/en/user-guide/admin-account-identifier#non-vps-account-locator-formats-by-cloud-platform-and-region).
131 | For example, eastus2 for Azure should actually be east-us-2.
132 |
133 |
134 | ## Use
135 |
136 | **Import user worksheet locally** :
137 | ```bash
138 | $ sfgit fetch --auth-mode PWD
139 | ```
140 |
141 | **Import user worksheet locally (using command time args)** :
142 | ```bash
143 | $ sfgit fetch --username tdambrin --account-id my_account.west-europe.azure -p mysecret -am PWD
144 | ```
145 |
146 | **See what changed for only your worksheets in the git** :
147 | ```bash
148 | $ sfgit diff
149 | ```
150 |
151 | **Commit you worksheets** (or through git commands for more flexibility) :
152 | ```bash
153 | $ sfgit commit --branch master -m "Initial worksheet commit"
154 | ```
155 |
156 | **Export user worksheets to Snowsight**
157 | ```bash
158 | $ sfgit push --auth-mode PWD --branch master
159 | ```
160 |
161 | ## Be creative
162 |
163 | Use the package to fit your use case, versioning is a way to do many things.
164 |
165 | ### Switch versions
166 |
167 |
168 |

169 |
170 |
171 |
172 | ### Transfer worksheets to another account
173 |
174 | 
175 |
176 | ## Policies
177 | Feedbacks and contributions are greatly appreciated. This package was made to ease every day life for Snowflake
178 | developers and promote version control as much as possible.
179 |
180 | For questions, please feel free to reach out [by email](mailto:thomas.dambrin@gmail.com?subject=[GitHub]%20Snowflake%20Git%20Versioning).
181 |
--------------------------------------------------------------------------------
/doc/images/switch_versions.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/tdambrin/sf_git/36ab52b6b1451817badfdbe8a8de47f7ecd4c41b/doc/images/switch_versions.png
--------------------------------------------------------------------------------
/doc/images/transfer_accounts.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/tdambrin/sf_git/36ab52b6b1451817badfdbe8a8de47f7ecd4c41b/doc/images/transfer_accounts.png
--------------------------------------------------------------------------------
/requirements.dev.txt:
--------------------------------------------------------------------------------
1 | black
2 | isort
3 | flake8>
--------------------------------------------------------------------------------
/requirements.test.txt:
--------------------------------------------------------------------------------
1 | pytest
2 | pytest-mock
3 | pytest-ordering
4 | requests-mock
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | pip>=21.3.1
2 | setuptools>=56
3 | pandas
4 | coverage
5 | python-dotenv
6 | click==7.1.2
7 | deepdiff
8 | gitpython>=3.1
9 | requests
10 | urllib3
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | from setuptools import find_packages, setup
2 |
3 | description = """
4 | Python package to manipulate Snowsight worksheets
5 | and easily apply Git versioning on it.
6 | """
7 |
8 | with open("README.md", "r") as f:
9 | long_description = f.read()
10 |
11 | with open("requirements.txt", "r") as f:
12 | requirements = f.read()
13 |
14 | setup(
15 | name="sf_git",
16 | version="1.4.2",
17 | setup_requires=["setuptools_scm"],
18 | author="Thomas Dambrin",
19 | author_email="thomas.dambrin@gmail.com",
20 | url="https://github.com/tdambrin/sf_git",
21 | license="MIT",
22 | description=description,
23 | packages=find_packages(),
24 | install_requires=requirements,
25 | keywords=["python", "snowflake", "git"],
26 | classifiers=[
27 | "Intended Audience :: Developers",
28 | "Programming Language :: Python :: 3",
29 | "Operating System :: MacOS :: MacOS X",
30 | "Operating System :: Unix",
31 | "Operating System :: Microsoft :: Windows",
32 | ],
33 | # include_package_data: to install data from MANIFEST.in
34 | include_package_data=True,
35 | zip_safe=False,
36 | # all functions @cli.command() decorated in sf_git/cli.py
37 | entry_points={"console_scripts": ["sfgit = sf_git.cli:cli"]},
38 | scripts=[],
39 | long_description=long_description,
40 | long_description_content_type="text/markdown",
41 | )
42 |
--------------------------------------------------------------------------------
/sf_git.test.conf:
--------------------------------------------------------------------------------
1 | SNOWFLAKE_VERSIONING_REPO='./tmp/sf_git_test'
2 | WORKSHEETS_PATH='./tmp/sf_git_test/data'
3 | SF_ACCOUNT_ID='fake.west-europe.azure'
4 | SF_LOGIN_NAME='fake'
5 | SF_PWD='fake'
6 |
--------------------------------------------------------------------------------
/sf_git/__init__.py:
--------------------------------------------------------------------------------
1 | from pathlib import Path
2 |
3 | from dotenv import load_dotenv
4 |
5 | __version__ = "1.4.2"
6 |
7 | HERE = Path(__file__).parent
8 | DOTENV_PATH: Path = HERE / "sf_git.conf"
9 |
10 | if DOTENV_PATH.is_file():
11 | print(f"loading dotenv from {DOTENV_PATH}")
12 | load_dotenv(dotenv_path=DOTENV_PATH)
13 |
--------------------------------------------------------------------------------
/sf_git/cache.py:
--------------------------------------------------------------------------------
1 | import json
2 | import os
3 | import re
4 | from pathlib import Path
5 | from typing import List, Optional, Union
6 | import git
7 |
8 | import sf_git.config as config
9 | from sf_git.models import Worksheet, WorksheetError
10 | from sf_git.git_utils import get_tracked_files, get_blobs_content
11 |
12 |
13 | def save_worksheets_to_cache(worksheets: List[Worksheet]):
14 | """
15 | Save worksheets to cache. Git is not involved here.
16 |
17 | For each worksheet, two files are created/overriden:
18 | - ._metadata.json (worksheet info)
19 | - .sql or .py (worksheet content)
20 |
21 | :param worksheets: list of worksheets to save
22 | """
23 |
24 | print(f"[Worksheets] Saving to {config.GLOBAL_CONFIG.worksheets_path}")
25 | if not os.path.exists(config.GLOBAL_CONFIG.worksheets_path):
26 | os.makedirs(config.GLOBAL_CONFIG.worksheets_path, exist_ok=True)
27 |
28 | for ws in worksheets:
29 | ws_name = re.sub(r"[ :/]", "_", ws.name)
30 | extension = "py" if ws.content_type == "python" else "sql"
31 | if ws.folder_name:
32 | folder_name = re.sub(r"[ :/]", "_", ws.folder_name)
33 | file_name = f"{folder_name}/{ws_name}.{extension}"
34 | worksheet_metadata_file_name = (
35 | f"{folder_name}/.{ws_name}_metadata.json"
36 | )
37 |
38 | # create folder if not exists
39 | if not os.path.exists(
40 | config.GLOBAL_CONFIG.worksheets_path / folder_name
41 | ):
42 | os.mkdir(config.GLOBAL_CONFIG.worksheets_path / folder_name)
43 | else:
44 | file_name = f"{ws_name}.{extension}"
45 | worksheet_metadata_file_name = f".{ws_name}_metadata.json"
46 |
47 | with open(config.GLOBAL_CONFIG.worksheets_path / file_name, "w", encoding="utf-8") as f:
48 | f.write(ws.content)
49 | ws_metadata = {
50 | "name": ws.name,
51 | "_id": ws._id,
52 | "folder_name": ws.folder_name,
53 | "folder_id": ws.folder_id,
54 | "content_type": ws.content_type,
55 | }
56 | with open(
57 | config.GLOBAL_CONFIG.worksheets_path
58 | / worksheet_metadata_file_name,
59 | "w",
60 | encoding="utf-8",
61 | ) as f:
62 | f.write(json.dumps(ws_metadata))
63 | print("[Worksheets] Saved")
64 |
65 |
66 | def load_worksheets_from_cache(
67 | repo: git.Repo,
68 | branch_name: Optional[str] = None,
69 | only_folder: Optional[Union[str, Path]] = None,
70 | ) -> List[Worksheet]:
71 | """
72 | Load worksheets from cache.
73 |
74 | :param repo: Git repository as it only considers tracked files
75 | :param branch_name: name of git branch to get files from
76 | :param only_folder: to get only worksheets in that folder
77 |
78 | :return: list of tracked worksheet objects
79 | """
80 |
81 | print(f"[Worksheets] Loading from {config.GLOBAL_CONFIG.worksheets_path}")
82 | if not os.path.exists(config.GLOBAL_CONFIG.worksheets_path):
83 | raise WorksheetError(
84 | "Could not retrieve worksheets from cache. "
85 | f"The folder {config.GLOBAL_CONFIG.worksheets_path} does not exist"
86 | )
87 |
88 | tracked_files = [
89 | f
90 | for f in get_tracked_files(
91 | repo, config.GLOBAL_CONFIG.worksheets_path, branch_name
92 | )
93 | ]
94 |
95 | # filter on worksheet files
96 | ws_metadata_files = [
97 | f for f in tracked_files if f.name.endswith("_metadata.json")
98 | ]
99 | if len(ws_metadata_files) == 0:
100 | return []
101 |
102 | # map to worksheet objects
103 | worksheets = []
104 | metadata_contents = get_blobs_content(ws_metadata_files)
105 |
106 | for wsf in metadata_contents.values():
107 | ws_metadata = json.loads(wsf)
108 | if only_folder and ws_metadata["folder_name"] != only_folder:
109 | continue
110 | current_ws = Worksheet(
111 | ws_metadata["_id"],
112 | ws_metadata["name"],
113 | ws_metadata["folder_id"],
114 | ws_metadata["folder_name"],
115 | content_type=ws_metadata.get("content_type", "sql"),
116 | )
117 | extension = "py" if current_ws.content_type == "python" else "sql"
118 | content_filename = re.sub(
119 | r"[ :/]", "_", f"{ws_metadata['name']}.{extension}"
120 | )
121 |
122 | try:
123 | content_blob = next(
124 | f for f in tracked_files if f.name == content_filename
125 | )
126 | except StopIteration:
127 | tracked_files = [f.name for f in tracked_files]
128 | print(
129 | f"{content_filename} not found in {tracked_files}"
130 | )
131 | return []
132 |
133 | ws_content_as_dict = get_blobs_content([content_blob])
134 | ws_content = list(ws_content_as_dict.values())[0]
135 | current_ws.content = ws_content
136 | worksheets.append(current_ws)
137 |
138 | return worksheets
139 |
--------------------------------------------------------------------------------
/sf_git/cli.py:
--------------------------------------------------------------------------------
1 | import click
2 |
3 | import sf_git.config as config
4 | import sf_git.commands
5 |
6 |
7 | @click.group()
8 | def cli():
9 | pass
10 |
11 |
12 | @click.group()
13 | @click.version_option(sf_git.__version__)
14 | @click.pass_context
15 | def cli(ctx):
16 | pass
17 |
18 |
19 | @click.command("init")
20 | @click.option(
21 | "--path",
22 | "-p",
23 | type=str,
24 | help="Absolute path of Git repository to be created.",
25 | default=".",
26 | show_default=True,
27 | )
28 | @click.option(
29 | "--mkdir",
30 | help="""(Flag) If provided, the repository directory will be created
31 | if it doesn't exist""",
32 | is_flag=True,
33 | default=False,
34 | show_default=True,
35 | )
36 | def init(path: str, mkdir: bool):
37 | """
38 | Initialize a git repository and set it as the sfgit versioning repository.
39 | """
40 |
41 | sf_git.commands.init_repo_procedure(path=path, mkdir=mkdir)
42 |
43 |
44 | @click.command("config")
45 | @click.option(
46 | "--get",
47 | help="If provided, print the current value for following config option",
48 | type=str,
49 |
50 | )
51 | @click.option(
52 | "--git-repo",
53 | "-r",
54 | type=str,
55 | help="Absolute path of Git repository used for versioning",
56 | )
57 | @click.option(
58 | "--save-dir",
59 | "-d",
60 | type=str,
61 | help="""Absolute path of directory to save worksheets in.
62 | Must be included in fit repository. Created if does not exist.
63 | """,
64 | )
65 | @click.option(
66 | "--account",
67 | "-a",
68 | type=str,
69 | help="""
70 | Default account for Snowsight authentication.
71 | """,
72 | )
73 | @click.option(
74 | "--username",
75 | "-u",
76 | type=str,
77 | help="""
78 | Default username for Snowsight authentication.
79 | """,
80 | )
81 | @click.option(
82 | "--password",
83 | "-p",
84 | type=str,
85 | help="""
86 | Default password for Snowsight authentication.
87 | """,
88 | )
89 | def config_repo(
90 | get: str,
91 | git_repo: str,
92 | save_dir: str,
93 | account: str,
94 | username: str,
95 | password: str,
96 | ):
97 | """
98 | Configure sfgit for easier version control.
99 | Git_repo configuration is mandatory.
100 | """
101 |
102 | if get:
103 | sf_git.commands.get_config_repo_procedure(get, click.echo)
104 | else:
105 | sf_git.commands.set_config_repo_procedure(
106 | git_repo=git_repo,
107 | save_dir=save_dir,
108 | account=account,
109 | username=username,
110 | password=password,
111 | logger=click.echo,
112 | )
113 |
114 |
115 | @click.command("fetch")
116 | @click.option(
117 | "--username",
118 | "-u",
119 | type=str,
120 | help="Snowflake user",
121 | )
122 | @click.option("--account-id", "-a", type=str, help="Snowflake Account Id")
123 | @click.option(
124 | "--auth-mode",
125 | "-am",
126 | type=str,
127 | help="Authentication Mode. Currently supports PWD (Default) and SSO.",
128 | default="PWD",
129 | show_default=True,
130 | )
131 | @click.option("--password", "-p", type=str, help="Snowflake password")
132 | @click.option(
133 | "--store/--no-store",
134 | "-l",
135 | help="(Flag) Whether to store the worksheets or just display them.",
136 | default=True,
137 | show_default=True,
138 | )
139 | @click.option(
140 | "--only-folder",
141 | "-only",
142 | type=str,
143 | help="Only fetch worksheets with given folder name",
144 | )
145 | def fetch_worksheets(
146 | username: str,
147 | account_id: str,
148 | auth_mode: str,
149 | password: str,
150 | store: bool,
151 | only_folder: str,
152 | ):
153 | """
154 | Fetch worksheets from user Snowsight account and store them in cache.
155 | """
156 |
157 | username = username or config.GLOBAL_CONFIG.sf_login_name
158 | account_id = account_id or config.GLOBAL_CONFIG.sf_account_id
159 | password = password or config.GLOBAL_CONFIG.sf_pwd
160 |
161 | sf_git.commands.fetch_worksheets_procedure(
162 | username=username,
163 | account_id=account_id,
164 | auth_mode=auth_mode,
165 | password=password,
166 | store=store,
167 | only_folder=only_folder,
168 | logger=click.echo,
169 | )
170 |
171 |
172 | @click.command("commit")
173 | @click.option(
174 | "--branch",
175 | "-b",
176 | type=str,
177 | help="Branch to commit to. Default is current.",
178 | )
179 | @click.option("--message", "-m", type=str, help="Commit message")
180 | def commit(
181 | branch: str,
182 | message: str,
183 | ):
184 | """
185 | Commit Snowsight worksheets to Git repository.
186 | """
187 |
188 | sf_git.commands.commit_procedure(
189 | branch=branch, message=message, logger=click.echo
190 | )
191 |
192 |
193 | @click.command("push")
194 | @click.option("--username", "-u", type=str, help="Snowflake user")
195 | @click.option(
196 | "--account-id",
197 | "-a",
198 | type=str,
199 | help="Snowflake Account Id",
200 | )
201 | @click.option(
202 | "--auth-mode",
203 | "-am",
204 | type=str,
205 | help="Authentication Mode. Currently supports PWD (Default) and SSO.",
206 | default="PWD",
207 | show_default=True,
208 | )
209 | @click.option("--password", "-p", type=str, help="Snowflake password")
210 | @click.option(
211 | "--branch",
212 | "-b",
213 | type=str,
214 | help="Branch to commit to. Default is current.",
215 | )
216 | @click.option(
217 | "--only-folder",
218 | "-only",
219 | type=str,
220 | help="Only push worksheets with given folder name",
221 | )
222 | def push_worksheets(
223 | username: str,
224 | account_id: str,
225 | auth_mode: str,
226 | password: str,
227 | branch: str,
228 | only_folder: str,
229 | ):
230 | """
231 | Upload locally stored worksheets to Snowsight user workspace.
232 | The local directory containing worksheets data is named {username}.
233 | More flexibility to come.
234 | """
235 |
236 | username = username or config.GLOBAL_CONFIG.sf_login_name
237 | account_id = account_id or config.GLOBAL_CONFIG.sf_account_id
238 | password = password or config.GLOBAL_CONFIG.sf_pwd
239 |
240 | sf_git.commands.push_worksheets_procedure(
241 | username=username,
242 | account_id=account_id,
243 | auth_mode=auth_mode,
244 | password=password,
245 | branch=branch,
246 | only_folder=only_folder,
247 | logger=click.echo,
248 | )
249 |
250 |
251 | @click.command("diff")
252 | def diff():
253 | """
254 | Displays unstaged changes on worksheets
255 | """
256 |
257 | sf_git.commands.diff_procedure(logger=click.echo)
258 |
259 |
260 | @click.group()
261 | @click.version_option(sf_git.__version__)
262 | @click.pass_context
263 | def cli(ctx):
264 | pass
265 |
266 |
267 | cli.add_command(init)
268 | cli.add_command(config_repo)
269 | cli.add_command(fetch_worksheets)
270 | cli.add_command(commit)
271 | cli.add_command(push_worksheets)
272 | cli.add_command(diff)
273 |
274 | if __name__ == "__main__":
275 | cli()
276 |
--------------------------------------------------------------------------------
/sf_git/commands.py:
--------------------------------------------------------------------------------
1 | import os
2 | import platform
3 | from typing import Callable, List
4 | from pathlib import Path, WindowsPath
5 | import git
6 | import dotenv
7 |
8 | from click import UsageError
9 |
10 | import sf_git.config as config
11 | from sf_git.cache import load_worksheets_from_cache
12 | from sf_git.snowsight_auth import authenticate_to_snowsight
13 | from sf_git.models import AuthenticationMode, SnowflakeGitError, Worksheet
14 | from sf_git.worksheets_utils import get_worksheets as sf_get_worksheets
15 | from sf_git.worksheets_utils import (
16 | print_worksheets,
17 | upload_to_snowsight,
18 | )
19 | from sf_git.git_utils import diff
20 | from sf_git import DOTENV_PATH
21 |
22 |
23 | def init_repo_procedure(path: str, mkdir: bool = False) -> str:
24 | """
25 | Initialize a git repository.
26 |
27 | :param path: absolute or relative path to git repository root
28 | :param mkdir: create the repository if it doesn't exist
29 |
30 | :return: new value of versioning repo in sf_git config
31 | """
32 | p_constructor = WindowsPath if platform.system() == "Windows" else Path
33 | abs_path = p_constructor(path).absolute()
34 | if not os.path.exists(abs_path) and not mkdir:
35 | raise UsageError(
36 | f"[Init] {abs_path} does not exist."
37 | "\nPlease provide a path towards an existing directory"
38 | " or add the --mkdir option to create it."
39 | )
40 |
41 | # create repository
42 | git.Repo.init(path, mkdir=mkdir)
43 |
44 | # set as sf_git versioning repository
45 | dotenv.set_key(
46 | DOTENV_PATH,
47 | key_to_set="SNOWFLAKE_VERSIONING_REPO",
48 | value_to_set=str(abs_path),
49 | )
50 |
51 | return dotenv.get_key(DOTENV_PATH, "SNOWFLAKE_VERSIONING_REPO")
52 |
53 |
54 | def get_config_repo_procedure(key: str, logger: Callable = print) -> str:
55 | """
56 | Get sf_git config value.
57 |
58 | :param key: key to retrieve value for
59 | :param logger: logging function e.g. print
60 |
61 | :returns: sf_git config value
62 | """
63 |
64 | dotenv_config = dotenv.dotenv_values(DOTENV_PATH)
65 | if key not in dotenv_config.keys():
66 | raise UsageError(
67 | f"[Config] {key} does not exist.\n"
68 | "Config values to be retrieved are "
69 | f"{list(dotenv_config.keys())}"
70 | )
71 |
72 | logger(dotenv_config[key])
73 |
74 | return dotenv_config[key]
75 |
76 |
77 | def set_config_repo_procedure(
78 | git_repo: str = None,
79 | save_dir: str = None,
80 | account: str = None,
81 | username: str = None,
82 | password: str = None,
83 | logger: Callable = print,
84 | ) -> dict:
85 | """
86 | Set sf_git config values.
87 |
88 | :param git_repo: if provided, set the git repository path
89 | :param save_dir: if provided, set the worksheet directory path
90 | :param account: if provided, set the account id
91 | :param username: if provided, set the user login name
92 | :param password: if provided, set the user password
93 | :param logger: logging function e.g. print
94 |
95 | :returns: dict with newly set keys and their values
96 | """
97 |
98 | updates = {}
99 |
100 | # check repositories
101 | if git_repo:
102 | repo_path = Path(git_repo).absolute()
103 | if not os.path.exists(repo_path):
104 | raise UsageError(
105 | f"[Config] {git_repo} does not exist."
106 | "Please provide path towards an existing git repository"
107 | " or create a new one with sfgit init."
108 | )
109 |
110 | if save_dir:
111 | save_dir = Path(save_dir).absolute()
112 | # get git_repo
113 | repo_path = (
114 | Path(git_repo).absolute()
115 | if git_repo
116 | else config.GLOBAL_CONFIG.repo_path
117 | )
118 | if repo_path not in save_dir.parents and repo_path != save_dir:
119 | raise UsageError(
120 | "[Config] "
121 | f"{save_dir} is not a subdirectory of {repo_path}.\n"
122 | "Please provide a saving directory within the git repository."
123 | )
124 |
125 | if git_repo:
126 | dotenv.set_key(
127 | DOTENV_PATH,
128 | key_to_set="SNOWFLAKE_VERSIONING_REPO",
129 | value_to_set=str(git_repo),
130 | )
131 | logger(f"Set SNOWFLAKE_VERSIONING_REPO to {str(git_repo)}")
132 | updates["SNOWFLAKE_VERSIONING_REPO"] = dotenv.get_key(
133 | DOTENV_PATH, "SNOWFLAKE_VERSIONING_REPO"
134 | )
135 |
136 | if save_dir:
137 | dotenv.set_key(
138 | DOTENV_PATH,
139 | key_to_set="WORKSHEETS_PATH",
140 | value_to_set=str(save_dir),
141 | )
142 | logger(f"Set WORKSHEETS_PATH to {str(save_dir)}")
143 | updates["WORKSHEETS_PATH"] = dotenv.get_key(
144 | DOTENV_PATH, "WORKSHEETS_PATH"
145 | )
146 |
147 | if account:
148 | dotenv.set_key(
149 | DOTENV_PATH,
150 | key_to_set="SF_ACCOUNT_ID",
151 | value_to_set=account,
152 | )
153 | logger(f"Set SF_ACCOUNT_ID to {account}")
154 | updates["SF_ACCOUNT_ID"] = dotenv.get_key(DOTENV_PATH, "SF_ACCOUNT_ID")
155 |
156 | if username:
157 | dotenv.set_key(
158 | DOTENV_PATH,
159 | key_to_set="SF_LOGIN_NAME",
160 | value_to_set=username,
161 | )
162 | logger(f"Set SF_LOGIN_NAME to {username}")
163 | updates["SF_LOGIN_NAME"] = dotenv.get_key(DOTENV_PATH, "SF_LOGIN_NAME")
164 |
165 | if password:
166 | dotenv.set_key(
167 | DOTENV_PATH,
168 | key_to_set="SF_PWD",
169 | value_to_set=password,
170 | )
171 | logger("Set SF_PWD to provided password.")
172 | updates["SF_PWD"] = "*" * len(password)
173 |
174 | return updates
175 |
176 |
177 | def fetch_worksheets_procedure(
178 | username: str,
179 | account_id: str,
180 | auth_mode: str = None,
181 | password: str = None,
182 | only_folder: str = None,
183 | store: bool = True,
184 | logger: Callable = print,
185 | ) -> List[Worksheet]:
186 | """
187 | Fetch worksheets from Snowsight.
188 |
189 | :param username: username to authenticate
190 | :param account_id: account id to authenticate
191 | :param auth_mode: authentication mode, supported are PWD (default) and SSO
192 | :param password: password to authenticate (not required for SSO)
193 | :param only_folder: name of folder if only fetch a specific folder from Snowsight
194 | :param store: (flag) save worksheets locally in configured worksheet directory
195 | :param logger: logging function e.g. print
196 |
197 | :returns: list of fetched worksheets
198 | """ # noqa: E501
199 |
200 | # Get auth parameters
201 | logger(" ## Authenticating to Snowsight ##")
202 |
203 | if auth_mode:
204 | if auth_mode == "SSO":
205 | auth_mode = AuthenticationMode.SSO
206 | password = None
207 | elif auth_mode == "PWD":
208 | auth_mode = AuthenticationMode.PWD
209 | if not password:
210 | raise UsageError(
211 | "No password provided for PWD authentication mode."
212 | "Please provide one."
213 | )
214 | else:
215 | raise UsageError(f"{auth_mode} is not supported.")
216 | else: # default
217 | auth_mode = AuthenticationMode.PWD
218 | if not password:
219 | raise UsageError(
220 | "No password provided for PWD authentication mode."
221 | "Please provide one."
222 | )
223 | logger(f" ## Password authentication with username={username} ##")
224 | auth_context = authenticate_to_snowsight(
225 | account_id, username, password, auth_mode=auth_mode
226 | )
227 |
228 | if auth_context.snowsight_token != "":
229 | logger(f" ## Authenticated as {auth_context.username}##")
230 | else:
231 | logger(" ## Authentication failed ##")
232 | exit(1)
233 |
234 | logger(" ## Getting worksheets ##")
235 | worksheets = sf_get_worksheets(
236 | auth_context, store_to_cache=store, only_folder=only_folder
237 | )
238 |
239 | logger("## Got worksheets ##")
240 | print_worksheets(worksheets, logger=logger)
241 |
242 | if store and worksheets:
243 | worksheet_path = dotenv.get_key(DOTENV_PATH, "WORKSHEETS_PATH")
244 | logger(f"## Worksheets saved to {worksheet_path} ##")
245 |
246 | return worksheets
247 |
248 |
249 | def commit_procedure(branch: str, message: str, logger: Callable) -> str:
250 | """
251 | Commits all worksheets in worksheet directory
252 |
253 | :param branch: name of branch to commit to, defaults to active one
254 | :param message: commit message
255 | :param logger: logging function e.g. print
256 |
257 | :returns: commit sha
258 | """
259 | # Get git repo
260 | try:
261 | repo = git.Repo(config.GLOBAL_CONFIG.repo_path)
262 | except git.InvalidGitRepositoryError as exc:
263 | raise SnowflakeGitError(
264 | "Could not find Git Repository here : "
265 | f"{config.GLOBAL_CONFIG.repo_path}"
266 | ) from exc
267 |
268 | # Get git branch
269 | if branch:
270 | available_branches = {b.name: b for b in repo.branches}
271 | if branch not in available_branches.keys():
272 | raise UsageError(f"Could not find branch {branch}")
273 | branch = available_branches[branch]
274 |
275 | # and checkout if necessary
276 | if repo.active_branch.name != branch.name:
277 | repo.head.reference = branch
278 | else:
279 | branch = repo.active_branch
280 |
281 | # Add worksheets to staged files
282 | repo.index.add(config.GLOBAL_CONFIG.worksheets_path)
283 |
284 | # Commit
285 | if message:
286 | commit_message = message
287 | else:
288 | commit_message = "[UPDATE] Snowflake worksheets"
289 |
290 | c = repo.index.commit(message=commit_message)
291 |
292 | logger(f"## Committed worksheets to branch {branch.name}")
293 |
294 | return c.hexsha
295 |
296 |
297 | def push_worksheets_procedure(
298 | username: str,
299 | account_id: str,
300 | auth_mode: str = None,
301 | password: str = None,
302 | branch: str = None,
303 | only_folder: str = None,
304 | logger: Callable = print,
305 | ) -> dict:
306 | """
307 | Push committed worksheet to Snowsight.
308 |
309 | :param username: username to authenticate
310 | :param account_id: account id to authenticate
311 | :param auth_mode: authentication mode, supported are PWD (default) and SSO
312 | :param password: password to authenticate (not required for SSO)
313 | :param only_folder: name of folder if only push a specific folder to Snowsight
314 | :param branch: branch to get worksheets from
315 | :param logger: logging function e.g. print
316 |
317 | :returns: upload report with success and errors per worksheet
318 | """ # noqa: E501
319 |
320 | # Get auth parameters
321 | logger(" ## Authenticating to Snowsight ##")
322 | if not username:
323 | raise SnowflakeGitError("No username to authenticate with.")
324 | if not account_id:
325 | raise SnowflakeGitError("No account to authenticate with.")
326 |
327 | if auth_mode:
328 | if auth_mode == "SSO":
329 | auth_mode = AuthenticationMode.SSO
330 | password = None
331 | elif auth_mode == "PWD":
332 | auth_mode = AuthenticationMode.PWD
333 | if not password:
334 | raise UsageError(
335 | "No password provided for PWD authentication mode."
336 | "Please provide one."
337 | )
338 | else:
339 | raise UsageError(f"{auth_mode} is not supported.")
340 | else: # default
341 | auth_mode = AuthenticationMode.PWD
342 | if not password:
343 | raise UsageError(
344 | "No password provided for PWD authentication mode."
345 | "Please provide one."
346 | )
347 |
348 | auth_context = authenticate_to_snowsight(
349 | account_id, username, password, auth_mode=auth_mode
350 | )
351 |
352 | if auth_context.snowsight_token != "":
353 | logger(f" ## Authenticated as {auth_context.login_name}##")
354 | else:
355 | logger(" ## Authentication failed ##")
356 | exit(1)
357 |
358 | # Get file content from git utils
359 | try:
360 | repo = git.Repo(config.GLOBAL_CONFIG.repo_path)
361 | except git.InvalidGitRepositoryError as exc:
362 | raise SnowflakeGitError(
363 | "Could not find Git Repository here : "
364 | f"{config.GLOBAL_CONFIG.repo_path}"
365 | ) from exc
366 |
367 | logger(f" ## Getting worksheets from cache for user {username} ##")
368 | worksheets = load_worksheets_from_cache(
369 | repo=repo,
370 | branch_name=branch,
371 | only_folder=only_folder,
372 | )
373 |
374 | logger("## Got worksheets ##")
375 | print_worksheets(worksheets, logger=logger)
376 |
377 | logger("## Uploading to SnowSight ##")
378 | upload_report = upload_to_snowsight(auth_context, worksheets)
379 | worksheet_errors = upload_report["errors"]
380 | logger("## Uploaded to SnowSight ##")
381 |
382 | if worksheet_errors:
383 | logger("Errors happened for the following worksheets :")
384 | for err in worksheet_errors:
385 | logger(
386 | f"Name : {err['name']} "
387 | f"| Error type : {err['error'].snowsight_error}"
388 | )
389 |
390 | return upload_report
391 |
392 |
393 | def diff_procedure(logger: Callable = print) -> str:
394 | """
395 | Displays unstaged changes on worksheets for configured repository and worksheets path.
396 |
397 | :param logger: logging function e.g. print
398 |
399 | :returns: diff output as a string
400 | """ # noqa: E501
401 |
402 | # Get configuration
403 | try:
404 | repo = git.Repo(config.GLOBAL_CONFIG.repo_path)
405 | except git.InvalidGitRepositoryError as exc:
406 | raise SnowflakeGitError(
407 | "Could not find Git Repository here : "
408 | f"{config.GLOBAL_CONFIG.repo_path}"
409 | ) from exc
410 |
411 | worksheets_path = config.GLOBAL_CONFIG.worksheets_path
412 | if not os.path.exists(worksheets_path):
413 | raise SnowflakeGitError(
414 | "Worksheets path is not set or the folder doesn't exist. "
415 | "Please set it or create it (manually or with sfgit fetch"
416 | )
417 |
418 | diff_output = diff(
419 | repo, subdirectory=worksheets_path, file_extensions=["py", "sql"]
420 | )
421 | logger(diff_output)
422 |
423 | return diff_output
424 |
--------------------------------------------------------------------------------
/sf_git/config.py:
--------------------------------------------------------------------------------
1 | import os
2 | from pathlib import Path, PosixPath, WindowsPath
3 | import platform
4 | from dataclasses import dataclass
5 | from typing import Union
6 |
7 |
8 | @dataclass
9 | class Config:
10 | """Package configuration, git, Snowflake and Filesystem"""
11 |
12 | sf_main_app_url: str = "https://app.snowflake.com"
13 | repo_path: Union[PosixPath, WindowsPath] = None
14 | worksheets_path: Union[PosixPath, WindowsPath] = None
15 | sf_account_id: str = None
16 | sf_login_name: str = None
17 | sf_pwd: str = None
18 |
19 | def __post_init__(self):
20 | # make paths windows if necessary
21 | if platform.system() == "Windows":
22 | if self.repo_path:
23 | self.repo_path = WindowsPath(self.repo_path)
24 | if self.worksheets_path:
25 | self.worksheets_path = WindowsPath(self.worksheets_path)
26 | self.repo_path = self.repo_path.absolute()
27 | self.worksheets_path = self.worksheets_path.absolute()
28 |
29 |
30 | GLOBAL_CONFIG = Config(
31 | repo_path=Path(os.environ["SNOWFLAKE_VERSIONING_REPO"]).absolute(),
32 | worksheets_path=Path(
33 | os.environ.get("WORKSHEETS_PATH") or "/tmp/snowflake_worksheets"
34 | ).absolute(),
35 | sf_account_id=os.environ.get("SF_ACCOUNT_ID"),
36 | sf_login_name=os.environ.get("SF_LOGIN_NAME"),
37 | sf_pwd=os.environ.get("SF_PWD"),
38 | )
39 |
--------------------------------------------------------------------------------
/sf_git/git_utils.py:
--------------------------------------------------------------------------------
1 | from pathlib import Path
2 | from typing import List, Type, Union, Dict, Optional
3 | import re
4 |
5 | import git
6 | from git.objects.blob import Blob
7 | from git.objects.tree import Tree
8 | from git.repo.base import Repo
9 |
10 | from sf_git.models import SnowflakeGitError
11 |
12 |
13 | def get_tracked_files(
14 | repo: Repo, folder: Path, branch_name: Optional[str] = None
15 | ) -> List[Union[Type[Blob], Type[Tree]]]:
16 | """
17 | Get all git tracked files in a folder inside a git repo
18 |
19 | :param repo: git repository tracking files
20 | :param folder: name of folder inside git repo to only load from
21 | :param branch_name: branch to consider, default is active branch
22 |
23 | :returns: list of blobs (file) and tree (dir)
24 | """
25 |
26 | # check that folder is in git
27 | repo_wd = Path(repo.working_dir)
28 | if repo_wd not in folder.parents:
29 | return []
30 |
31 | # retrieve branch, active one by default
32 | if branch_name is not None:
33 | available_branches = {b.name: b for b in repo.branches}
34 | if branch_name not in available_branches.keys():
35 | raise SnowflakeGitError(
36 | f"Unable to retrieve branch {branch_name}"
37 | f" in Repository {repo.working_dir}."
38 | "Please check that the branch name is correct"
39 | )
40 | branch = available_branches[branch_name]
41 | else:
42 | branch = repo.active_branch
43 |
44 | # get to folder by folder names
45 | repo_objects = branch.commit.tree.traverse()
46 | try:
47 | folder_tree = next(
48 | obj for obj in repo_objects if obj.abspath == str(folder)
49 | )
50 | except StopIteration as exc:
51 | raise SnowflakeGitError(
52 | f"Unable to retrieve folder {str(folder)}"
53 | f" in Repository {repo.working_dir} and branch {branch.name}."
54 | "Please check that the files you are looking for are committed"
55 | ) from exc
56 |
57 | # get files
58 | tracked_files = folder_tree.traverse()
59 | return tracked_files
60 |
61 |
62 | def get_blobs_content(blobs: List[Blob]) -> Dict[str, bytes]:
63 | """
64 | Get blob contents and return as {blob_name: content}.
65 | Allow to get file content from git trees easily.
66 | """
67 |
68 | contents = {
69 | b.name: b.data_stream.read() for b in blobs if isinstance(b, Blob)
70 | }
71 | return contents
72 |
73 |
74 | def diff(
75 | repo: git.Repo,
76 | subdirectory: Union[str, Path] = None,
77 | file_extensions: Union[str, List[str]] = None,
78 | ) -> str:
79 | """
80 | Get git diff output with subdirectory and file extension filters
81 |
82 | :param repo: git repository
83 | :param subdirectory: only on files within this subdirectory
84 | :param file_extensions: only match files with these extensions
85 |
86 | :returns: str, git diff output
87 | """
88 | # Check input
89 | if subdirectory and isinstance(subdirectory, str):
90 | subdirectory = Path(subdirectory)
91 | if file_extensions and isinstance(file_extensions, str):
92 | file_extensions = [file_extensions]
93 |
94 | # Get blobs
95 | search_path = (
96 | subdirectory
97 | if subdirectory
98 | else Path(repo.git.rev_parse("--show-toplevel"))
99 | )
100 | globs = []
101 | if not file_extensions:
102 | globs.extend(search_path.glob("**/*"))
103 | else:
104 | for extension in file_extensions:
105 | globs.extend(list(search_path.glob(f"**/*.{extension}")))
106 |
107 | # Get git diff output
108 | if not globs:
109 | return ""
110 |
111 | diff_output = repo.git.diff(globs)
112 |
113 | # Add coloring
114 | diff_output = re.sub(
115 | r"^(\++)(.*?)$",
116 | "\033[32m\\1\\2\033[0m",
117 | diff_output,
118 | flags=re.MULTILINE,
119 | )
120 | diff_output = re.sub(
121 | r"^(-+)(.*?)$",
122 | "\033[31m\\1\\2\033[0m",
123 | diff_output,
124 | flags=re.MULTILINE,
125 | )
126 |
127 | return diff_output
128 |
--------------------------------------------------------------------------------
/sf_git/models.py:
--------------------------------------------------------------------------------
1 | from dataclasses import dataclass, field
2 | from enum import Enum
3 | from typing import Any
4 |
5 |
6 | class AuthenticationMode(Enum):
7 | SSO = "SSO"
8 | PWD = "PWD"
9 |
10 |
11 | class AuthenticationError(Exception):
12 | def __init__(self, *args):
13 | if args:
14 | self.message = args[0]
15 | else:
16 | self.message = None
17 |
18 | def __str__(self):
19 | if self.message:
20 | return f"AuthenticationError: {self.message}"
21 | else:
22 | return "AuthenticationError: failed with no more information."
23 |
24 |
25 | @dataclass
26 | class AuthenticationContext:
27 | """To store authentication result information"""
28 |
29 | account: str = ""
30 | account_name: str = ""
31 | account_url: str = ""
32 | auth_code_challenge: str = ""
33 | auth_code_challenge_method: str = ""
34 | auth_originator: str = ""
35 | auth_redirect_uri: str = ""
36 | auth_session_token: str = ""
37 | app_server_url: str = ""
38 | client_id: str = ""
39 | cookies: Any = None
40 | csrf: str = ""
41 | login_name: str = ""
42 | main_app_url: str = ""
43 | master_token: str = ""
44 | oauth_nonce: str = ""
45 | organization_id: str = ""
46 | region: str = ""
47 | server_version: str = ""
48 | snowsight_token: dict = field(default_factory=dict)
49 | username: str = ""
50 | window_id: str = ""
51 |
52 |
53 | class SnowsightError(Enum):
54 | PERMISSION = "PERMISSION"
55 | UNKNOWN = "UNKNOWN"
56 |
57 |
58 | class WorksheetError(Exception):
59 | def __init__(self, *args):
60 | if args:
61 | self.message = args[0]
62 | if len(args) > 1:
63 | self.snowsight_error = SnowsightError.PERMISSION
64 | else:
65 | self.snowsight_error = SnowsightError.UNKNOWN
66 | else:
67 | self.message = None
68 | self.snowsight_error = SnowsightError.UNKNOWN
69 |
70 | def __str__(self):
71 | if self.message:
72 | return (
73 | f"WorksheetError: {self.message}."
74 | f" Error type is {self.snowsight_error}"
75 | )
76 | else:
77 | return "WorksheetError: no more information provided"
78 |
79 |
80 | class Worksheet:
81 | def __init__(
82 | self,
83 | _id: str,
84 | name: str,
85 | folder_id: str,
86 | folder_name: str,
87 | content: str = "To be filled",
88 | content_type: str = "sql",
89 | ):
90 | self._id = _id
91 | self.name = name
92 | self.folder_id = folder_id
93 | self.folder_name = folder_name
94 | self.content = content
95 | self.content_type = content_type
96 |
97 | def to_dict(self):
98 | return {
99 | "_id": self._id,
100 | "name": self.name,
101 | "folder_id": self.folder_id,
102 | "folder_name": self.folder_name,
103 | "content": self.content,
104 | "content_type": self.content_type,
105 | }
106 |
107 |
108 | class Folder:
109 | def __init__(self, _id: str, name: str, worksheets=None):
110 | if worksheets is None:
111 | worksheets = []
112 | self._id = _id
113 | self.name = name
114 | self.worksheets = worksheets
115 |
116 | def to_dict(self):
117 | return {
118 | "_id": self._id,
119 | "name": self.name,
120 | "worksheets": self.worksheets,
121 | }
122 |
123 |
124 | class SnowflakeGitError(Exception):
125 | """
126 | Custom Exception for sf_git specific raised exceptions
127 | """
128 |
129 | def __init__(self, *args):
130 | if args:
131 | self.message = args[0]
132 | else:
133 | self.message = "Not provided"
134 |
135 | def __str__(self):
136 | if self.message:
137 | return f"SnowflakeGitError: {self.message}."
138 |
139 | return "SnowflakeGitError: no more information provided"
140 |
--------------------------------------------------------------------------------
/sf_git/rest_utils.py:
--------------------------------------------------------------------------------
1 | import logging
2 | import platform
3 | import re
4 | import socket
5 | import subprocess
6 | import time
7 |
8 | import requests
9 |
10 |
11 | def api_post(
12 | base_url: str,
13 | rest_api_url: str,
14 | accept_header: str,
15 | request_body: str = None,
16 | request_type_header: str = None,
17 | snowflake_context: str = None,
18 | referer: str = None,
19 | csrf: str = None,
20 | cookies: requests.cookies.RequestsCookieJar = None, # noqa
21 | allow_redirect: bool = False,
22 | as_obj: bool = False,
23 | ):
24 | start_time = time.time()
25 |
26 | if not base_url.endswith("/"):
27 | base_url += "/"
28 |
29 | try:
30 | session = requests.Session()
31 | session.verify = False
32 | session.max_redirects = 20
33 |
34 | headers = requests.models.CaseInsensitiveDict()
35 | if referer:
36 | headers["Referer"] = referer
37 | if snowflake_context:
38 | headers["x-snowflake-context"] = snowflake_context
39 | if cookies:
40 | session.cookies.update(cookies)
41 | if csrf:
42 | headers["X-CSRF-Token"] = csrf
43 |
44 | headers["Accept"] = accept_header
45 |
46 | content_type = request_type_header
47 | headers["Content-Type"] = content_type
48 |
49 | response = session.post(
50 | base_url + rest_api_url,
51 | headers=headers,
52 | data=request_body,
53 | timeout=60,
54 | allow_redirects=allow_redirect,
55 | )
56 |
57 | if response.status_code < 400:
58 | result_string = response.text or ""
59 | logging.info(
60 | f"POST {base_url}/{rest_api_url} returned "
61 | f"{response.status_code}"
62 | f"({response.reason})\nRequest Headers:\n{headers}\nCookies:\n"
63 | f"{len(result_string)}:\n{result_string}"
64 | )
65 | if as_obj:
66 | return response
67 | return result_string
68 | else:
69 | result_string = response.text or ""
70 | cookies_list = response.headers.get("Set-Cookie")
71 | if result_string:
72 | logging.error(
73 | f"POST {base_url}/{rest_api_url}"
74 | f" returned {response.status_code}"
75 | f" ({response.reason})\n"
76 | f"Request Headers:\n{headers}\n"
77 | f"Cookies:\n{cookies_list}\n"
78 | f"Request:\n{request_body}\n"
79 | f"Response Length {len(result_string)}"
80 | )
81 | else:
82 | logging.error(
83 | f"POST {base_url}/{rest_api_url} "
84 | f"returned {response.status_code}\n"
85 | f"Request Headers:\n{headers}\nCookies:\n{cookies_list}\n"
86 | f"Request:\n{request_body}"
87 | )
88 | if response.status_code in (401, 403):
89 | logging.warning(
90 | f"POST {base_url}/{rest_api_url} "
91 | f"returned {response.status_code} ({response.reason}),"
92 | f" Request:\n{request_body}"
93 | )
94 |
95 | response.raise_for_status()
96 | except requests.exceptions.RequestException as ex:
97 | logging.error(
98 | f"POST {base_url}/{rest_api_url} "
99 | f"threw {ex.__class__.__name__} ({ex})"
100 | )
101 | logging.error(ex)
102 |
103 | logging.error(
104 | f"POST {base_url}/{rest_api_url} "
105 | f"threw {ex.__class__.__name__} ({ex})"
106 | )
107 |
108 | return ""
109 | finally:
110 | elapsed_time = time.time() - start_time
111 | logging.info(f"POST {base_url}/{rest_api_url} took {elapsed_time:.2f}")
112 |
113 |
114 | def api_get(
115 | base_url: str,
116 | rest_api_url: str,
117 | accept_header: str,
118 | snowflake_context: str = None,
119 | referer: str = None,
120 | csrf: str = None,
121 | cookies: requests.cookies.RequestsCookieJar = None, # noqa
122 | allow_redirect: bool = True,
123 | as_obj: bool = False,
124 | ):
125 | start_time = time.time()
126 |
127 | if not base_url.endswith("/"):
128 | base_url += "/"
129 | try:
130 | session = requests.Session()
131 | session.verify = False
132 | session.max_redirects = 20
133 |
134 | headers = requests.models.CaseInsensitiveDict()
135 | if referer:
136 | headers["Referer"] = referer
137 | if snowflake_context:
138 | headers["x-snowflake-context"] = snowflake_context
139 | if cookies:
140 | session.cookies.update(cookies)
141 | if csrf:
142 | headers["X-CSRF-Token"] = csrf
143 |
144 | headers["Accept"] = accept_header
145 |
146 | response = session.get(
147 | base_url + rest_api_url,
148 | headers=headers,
149 | timeout=60,
150 | allow_redirects=allow_redirect,
151 | )
152 |
153 | if response.status_code < 400:
154 | result_string = response.text or ""
155 | logging.info(
156 | f"POST {base_url}/{rest_api_url} returned "
157 | f"{response.status_code}"
158 | f"({response.reason})\nRequest Headers:\n{headers}\nCookies:\n"
159 | f"{len(result_string)}:\n{result_string}"
160 | )
161 | if as_obj:
162 | return response
163 | return result_string
164 | else:
165 | result_string = response.text or ""
166 | cookies_list = response.headers.get("Set-Cookie")
167 | if result_string:
168 | logging.error(
169 | f"GET {base_url}/{rest_api_url}"
170 | f" returned {response.status_code}"
171 | f" ({response.reason})\n"
172 | f"Request Headers:\n{headers}\n"
173 | f"Cookies:\n{cookies_list}\n"
174 | f"Response Length {len(result_string)}"
175 | )
176 | else:
177 | logging.error(
178 | f"GET {base_url}/{rest_api_url} "
179 | f"returned {response.status_code}\n"
180 | f"Request Headers:\n{headers}\nCookies:\n{cookies_list}\n"
181 | )
182 | if response.status_code in (401, 403):
183 | logging.warning(
184 | f"GET {base_url}/{rest_api_url} "
185 | f"returned {response.status_code} ({response.reason}),"
186 | )
187 |
188 | response.raise_for_status()
189 | except requests.exceptions.RequestException as ex:
190 | logging.error(
191 | f"GET {base_url}/{rest_api_url} "
192 | f"threw {ex.__class__.__name__} ({ex})"
193 | )
194 | logging.error(ex)
195 |
196 | logging.error(
197 | f"GET {base_url}/{rest_api_url} "
198 | f"threw {ex.__class__.__name__} ({ex})"
199 | )
200 |
201 | return ""
202 | finally:
203 | elapsed_time = time.time() - start_time
204 | logging.info(f"GET {base_url}/{rest_api_url} took {elapsed_time:.2f}")
205 |
206 |
207 | def random_unused_port() -> int:
208 | sock = socket.socket()
209 | sock.bind(("", 0))
210 | return sock.getsockname()[1]
211 |
212 |
213 | def start_browser(url: str):
214 | if platform.system() == "Windows":
215 | subprocess.Popen(
216 | ["cmd", "/c", "start", url.replace("&", "^&")],
217 | creationflags=subprocess.CREATE_NO_WINDOW,
218 | )
219 | elif platform.system() == "Linux":
220 | subprocess.Popen(["xdg-open", url])
221 | elif platform.system() == "Darwin":
222 | subprocess.Popen(["open", url])
223 | else:
224 | raise NotImplementedError(f"Unsupported platform: {platform.system()}")
225 |
226 |
227 | def agg_cookies_from_responses(
228 | response_flow: requests.Response,
229 | ) -> requests.cookies.RequestsCookieJar:
230 | if not response_flow.history:
231 | return response_flow.cookies
232 |
233 | agg_cookies: requests.cookies.RequestsCookieJar = response_flow.history[
234 | 0
235 | ].cookies
236 | for resp in response_flow.history[1:]:
237 | agg_cookies.update(resp.cookies)
238 | agg_cookies.update(response_flow.cookies)
239 | return agg_cookies
240 |
--------------------------------------------------------------------------------
/sf_git/sf_git.conf:
--------------------------------------------------------------------------------
1 | SNOWFLAKE_VERSIONING_REPO='/tmp/trash'
2 | WORKSHEETS_PATH='/tmp/trash/worksheets'
3 | SF_ACCOUNT_ID='tofill.west-europe.azure'
4 | SF_LOGIN_NAME='TOFILL'
5 |
--------------------------------------------------------------------------------
/sf_git/snowsight_auth.py:
--------------------------------------------------------------------------------
1 | import json
2 | import os
3 | import re
4 | import socket
5 | import uuid
6 | from typing import Any, Dict
7 | from urllib import parse
8 | from urllib.parse import urlparse
9 |
10 | import requests
11 | import urllib3
12 |
13 | import sf_git.config as config
14 | from sf_git.models import (
15 | AuthenticationContext,
16 | AuthenticationError,
17 | AuthenticationMode,
18 | )
19 | from sf_git.rest_utils import (
20 | agg_cookies_from_responses,
21 | api_get,
22 | api_post,
23 | random_unused_port,
24 | start_browser,
25 | )
26 |
27 | urllib3.disable_warnings()
28 |
29 |
30 | def authenticate_to_snowsight(
31 | account_name: str,
32 | login_name: str,
33 | password: str,
34 | auth_mode: AuthenticationMode = AuthenticationMode.PWD,
35 | ) -> AuthenticationContext:
36 | auth_context = AuthenticationContext()
37 | auth_context.main_app_url = config.GLOBAL_CONFIG.sf_main_app_url
38 | auth_context.account_name = account_name
39 | auth_context.login_name = login_name
40 |
41 | # Get App Server Url and Account Url
42 | app_endpoint = get_account_app_endpoint(account_name)
43 | if not app_endpoint["valid"]:
44 | raise AuthenticationError(
45 | f"No valid endpoint for account {account_name}"
46 | )
47 |
48 | auth_context.account = app_endpoint["account"]
49 | auth_context.account_url = app_endpoint["url"]
50 | auth_context.app_server_url = app_endpoint["appServerUrl"]
51 | auth_context.region = app_endpoint["region"]
52 |
53 | # Get unverified csrf
54 | bootstrap_response = get_csrf_from_boostrap_cookie(auth_context)
55 | csrf_cookie = next(
56 | (c for c in bootstrap_response.cookies if c.name.startswith("csrf-")),
57 | None,
58 | )
59 | if not csrf_cookie:
60 | raise AuthenticationError("Could not get csrf from bootstrap endpoint")
61 | auth_context.csrf = csrf_cookie.value
62 | auth_context.cookies = bootstrap_response.cookies
63 |
64 | # Get client ID
65 | client_id_responses = oauth_start_get_snowsight_client_id_in_deployment(
66 | auth_context=auth_context
67 | )
68 | parsed = urlparse(client_id_responses.url)
69 | parsed_query = parse.parse_qs(parsed.query)
70 | auth_context.cookies = agg_cookies_from_responses(
71 | client_id_responses
72 | ) # client_id_responses.history[-1].cookies
73 | state = json.loads(parsed_query["state"][0])
74 | auth_context.oauth_nonce = state["oauthNonce"]
75 | auth_context.auth_originator = state.get("originator")
76 | auth_context.window_id = state.get("windowId")
77 | auth_context.auth_code_challenge = parsed_query["code_challenge"][0]
78 | auth_context.auth_code_challenge_method = parsed_query[
79 | "code_challenge_method"
80 | ][0]
81 | auth_context.auth_redirect_uri = parsed_query["redirect_uri"][0]
82 | auth_context.client_id = parsed_query["client_id"][0]
83 |
84 | if not auth_context.organization_id:
85 | auth_context.organization_id = os.environ.get("SF_ORGANIZATION_ID")
86 |
87 | # Get master token
88 | if auth_mode == AuthenticationMode.PWD:
89 | auth_response = get_token_from_credentials(
90 | auth_context=auth_context,
91 | login_name=login_name,
92 | password=password,
93 | )
94 | auth_response_details = json.loads(auth_response.text)
95 | if not auth_response_details["success"]:
96 | raise AuthenticationError("Invalid credentials")
97 |
98 | auth_context.master_token = auth_response_details["data"][
99 | "masterToken"
100 | ]
101 | auth_context.auth_session_token = auth_response_details["data"][
102 | "token"
103 | ]
104 | auth_context.server_version = auth_response_details["data"].get(
105 | "serverVersion"
106 | )
107 |
108 | oauth_response = oauth_get_master_token_from_credentials(
109 | auth_context=auth_context, login_name=login_name, password=password
110 | )
111 | oauth_response_details = json.loads(oauth_response.text)
112 | if not oauth_response_details["success"]:
113 | raise AuthenticationError("Invalid credentials")
114 | auth_context.master_token = oauth_response_details["data"][
115 | "masterToken"
116 | ]
117 |
118 | elif auth_mode == AuthenticationMode.SSO:
119 | redirect_port = random_unused_port()
120 | sso_link_response = json.loads(
121 | get_sso_login_link(
122 | auth_context.account_url,
123 | account_name.split(".")[0],
124 | login_name,
125 | redirect_port,
126 | )
127 | )
128 | idp_url = sso_link_response["data"]["ssoUrl"]
129 | proof_key = sso_link_response["data"]["proofKey"]
130 | with socket.socket() as s:
131 | s.bind(("localhost", redirect_port))
132 | start_browser(idp_url)
133 | s.listen(5)
134 | conn, addr = s.accept()
135 | sso_token = ""
136 | while True:
137 | data = conn.recv(8188)
138 | if not data:
139 | break
140 | str_data = data.decode("utf-8")
141 | if "token=" in str_data:
142 | str_data = str_data.split("token=")[1]
143 | if " HTTP" in str_data: # end of token
144 | str_data = str_data.split(" HTTP")[0]
145 | sso_token += str_data
146 | break
147 | sso_token += str_data
148 | s.close()
149 | if sso_token == "":
150 | raise AuthenticationError("Could not retrieve SSO token")
151 |
152 | auth_context.master_token = get_master_token_from_sso_token(
153 | auth_context.account_url,
154 | auth_context.account_name,
155 | auth_context.login_name,
156 | sso_token,
157 | proof_key,
158 | )
159 |
160 | # Get oauth redirect
161 | oauth_response = oauth_authorize_get_oauth_redirect_from_oauth_token(
162 | auth_context=auth_context
163 | )
164 | if (
165 | oauth_response["code"] == "390302"
166 | or oauth_response["message"] == "Invalid consent request."
167 | ):
168 | print(oauth_response)
169 | raise AuthenticationError(
170 | "Could not get redirect from master token "
171 | f"for account url {auth_context.account_url} "
172 | f"and client_id {auth_context.client_id}. "
173 | "Please check your credentials."
174 | )
175 |
176 | # Finalize authentication
177 | finalized_response = oauth_complete_get_auth_token_from_redirect(
178 | auth_context=auth_context, url=oauth_response["data"]["redirectUrl"]
179 | )
180 | if finalized_response.status_code != 200:
181 | raise AuthenticationError(
182 | "Could not finalize authentication "
183 | f"for account url {auth_context.account_url} "
184 | f"with oauth_nonce_cookie {auth_context.oauth_nonce}"
185 | )
186 | auth_context.cookies = agg_cookies_from_responses(finalized_response)
187 | for cookie in finalized_response.cookies:
188 | if cookie.name.startswith("user"):
189 | auth_context.snowsight_token[cookie.name] = cookie.value
190 | if not auth_context.snowsight_token:
191 | raise AuthenticationError(
192 | "Finalized Authentication but didnt find a token"
193 | " in final response : {status_code : "
194 | f"{finalized_response.status_code}, url : "
195 | f"{finalized_response.url}, cookies : "
196 | f"{finalized_response.cookies}"
197 | )
198 |
199 | # handle different username
200 | params_page = finalized_response.content.decode("utf-8")
201 | match = re.search("(?i)var params = ({.*})", params_page, re.IGNORECASE)
202 | if match is not None:
203 | params = match.group(0).replace("var params = ", "")
204 | params = json.loads(params)
205 | username = params["user"].get("username")
206 | if username:
207 | auth_context.username = username
208 | else:
209 | auth_context.username = login_name
210 | else:
211 | auth_context.username = login_name
212 |
213 | return auth_context
214 |
215 |
216 | def get_account_app_endpoint(account_name: str) -> Dict[str, Any]:
217 | main_app_url = config.GLOBAL_CONFIG.sf_main_app_url
218 | response = api_post(
219 | main_app_url,
220 | f"v0/validate-snowflake-url?url={account_name}",
221 | "*/*",
222 | )
223 |
224 | return json.loads(response)
225 |
226 |
227 | def get_csrf_from_boostrap_cookie(
228 | auth_context: AuthenticationContext,
229 | ) -> requests.Response:
230 | response = api_get(
231 | base_url=auth_context.main_app_url,
232 | rest_api_url="bootstrap",
233 | accept_header="*/*",
234 | allow_redirect=True,
235 | as_obj=True,
236 | cookies=auth_context.cookies,
237 | )
238 | return response
239 |
240 |
241 | def oauth_start_get_snowsight_client_id_in_deployment(
242 | auth_context: AuthenticationContext,
243 | ) -> requests.Response:
244 | state_params = (
245 | '{"csrf":'
246 | f'"{auth_context.csrf}","url":"{auth_context.account_url}","windowId":"{uuid.uuid4()}","browserUrl":"{auth_context.main_app_url}"}}' # noqa: E501
247 | )
248 |
249 | rest_api_url = (
250 | "start-oauth/snowflake?accountUrl="
251 | f"{auth_context.account_url}"
252 | f"&&state={parse.quote_plus(state_params)}"
253 | )
254 |
255 | response = api_get(
256 | base_url=auth_context.app_server_url,
257 | rest_api_url=rest_api_url,
258 | accept_header="text/html",
259 | csrf=auth_context.csrf,
260 | allow_redirect=True,
261 | cookies=auth_context.cookies,
262 | as_obj=True,
263 | )
264 |
265 | if response.status_code == 200:
266 | return response
267 |
268 | raise AuthenticationError(
269 | f"No client id could be retrieved for account {auth_context.account_url}"
270 | )
271 |
272 |
273 | def get_token_from_credentials(
274 | auth_context: AuthenticationContext,
275 | login_name: str,
276 | password: str,
277 | ) -> requests.Response:
278 | request_body = {
279 | "data": {
280 | "ACCOUNT_NAME": auth_context.account_name.split(".")[0],
281 | "LOGIN_NAME": login_name,
282 | "PASSWORD": password,
283 | }
284 | }
285 |
286 | auth_response = api_post(
287 | base_url=auth_context.account_url,
288 | rest_api_url="session/v1/login-request",
289 | request_type_header="application/json",
290 | accept_header="application/json",
291 | request_body=json.dumps(request_body),
292 | cookies=auth_context.cookies,
293 | as_obj=True,
294 | )
295 | return auth_response
296 |
297 |
298 | def oauth_get_master_token_from_credentials(
299 | auth_context: AuthenticationContext,
300 | login_name: str,
301 | password: str,
302 | ) -> requests.Response:
303 | state_params = (
304 | f'{{"csrf":"{auth_context.csrf}"'
305 | f',"url":"{auth_context.account_url}","browserUrl":"{auth_context.main_app_url}"'
306 | f',"originator":"{auth_context.auth_originator}","oauthNonce":"{auth_context.oauth_nonce}"}}'
307 | )
308 |
309 | request_body = {
310 | "data": {
311 | "ACCOUNT_NAME": auth_context.account_name.split(".")[0],
312 | "LOGIN_NAME": login_name,
313 | "clientId": auth_context.client_id,
314 | "redirectURI": auth_context.auth_redirect_uri,
315 | "responseType": "code",
316 | "state": state_params,
317 | "scope": "refresh_token",
318 | "codeChallenge": auth_context.auth_code_challenge,
319 | "codeChallengeMethod": auth_context.auth_code_challenge_method
320 | or "S256",
321 | "CLIENT_APP_ID": "Snowflake UI",
322 | "CLIENT_APP_VERSION": 20241007103851,
323 | "PASSWORD": password,
324 | }
325 | }
326 | return api_post(
327 | base_url=auth_context.account_url,
328 | rest_api_url="session/authenticate-request",
329 | accept_header="application/json",
330 | request_body=json.dumps(request_body),
331 | request_type_header="application/json",
332 | cookies=auth_context.cookies,
333 | as_obj=True,
334 | allow_redirect=True,
335 | )
336 |
337 |
338 | def oauth_authorize_get_oauth_redirect_from_oauth_token(
339 | auth_context: AuthenticationContext,
340 | ) -> Dict[str, Any]:
341 | state_params = (
342 | f'{{"csrf":"{auth_context.csrf}"'
343 | f',"url":"{auth_context.account_url}","windowId":"{auth_context.window_id}","browserUrl":"{auth_context.main_app_url}"' # noqa: E501
344 | f',"originator":"{auth_context.auth_originator}","oauthNonce":"{auth_context.oauth_nonce}"}}'
345 | )
346 | request_body = {
347 | "masterToken": auth_context.master_token,
348 | "clientId": auth_context.client_id,
349 | "redirectURI": auth_context.auth_redirect_uri,
350 | "responseType": "code",
351 | "state": state_params,
352 | "scope": "refresh_token",
353 | "codeChallenge": auth_context.auth_code_challenge,
354 | "codeChallengeMethod": auth_context.auth_code_challenge_method
355 | or "S256",
356 | }
357 |
358 | response_content = api_post(
359 | base_url=auth_context.account_url,
360 | rest_api_url="oauth/authorization-request",
361 | accept_header="application/json",
362 | request_body=json.dumps(request_body),
363 | request_type_header="application/json",
364 | csrf=auth_context.csrf,
365 | cookies=auth_context.cookies,
366 | allow_redirect=True,
367 | )
368 |
369 | return json.loads(response_content)
370 |
371 |
372 | def oauth_complete_get_auth_token_from_redirect(
373 | auth_context: AuthenticationContext,
374 | url: str,
375 | ) -> requests.Response:
376 | headers = {
377 | "Accept": "*/*",
378 | "Referer": f"{auth_context.main_app_url}/",
379 | }
380 |
381 | response = requests.get(
382 | url,
383 | headers=headers,
384 | allow_redirects=True,
385 | timeout=10,
386 | cookies=auth_context.cookies,
387 | )
388 |
389 | return response
390 |
391 |
392 | def get_sso_login_link(
393 | account_url: str,
394 | account_name: str,
395 | login_name: str,
396 | redirect_port: int,
397 | ):
398 | request_json_template = {
399 | "data": {
400 | "ACCOUNT_NAME": account_name,
401 | "LOGIN_NAME": login_name,
402 | "AUTHENTICATOR": "externalbrowser",
403 | "BROWSER_MODE_REDIRECT_PORT": redirect_port,
404 | }
405 | }
406 |
407 | request_body = json.dumps(request_json_template)
408 |
409 | response = api_post(
410 | account_url,
411 | "session/authenticator-request",
412 | "application/json",
413 | request_body,
414 | "application/json",
415 | )
416 |
417 | return response
418 |
419 |
420 | def get_master_token_from_sso_token(
421 | account_url: str,
422 | account_name: str,
423 | login_name: str,
424 | sso_token: str,
425 | proof_key: str,
426 | ) -> str:
427 | request_json_template = {
428 | "data": {
429 | "ACCOUNT_NAME": account_name,
430 | "LOGIN_NAME": login_name,
431 | "AUTHENTICATOR": "externalbrowser",
432 | "TOKEN": sso_token,
433 | "PROOF_KEY": proof_key,
434 | }
435 | }
436 |
437 | request_body = json.dumps(request_json_template)
438 |
439 | response = api_post(
440 | account_url,
441 | "session/v1/login-request",
442 | "application/json",
443 | request_body,
444 | "application/json",
445 | )
446 |
447 | parsed = json.loads(response)
448 | if not parsed["success"]:
449 | raise AuthenticationError(
450 | "Could not retrieve master token from sso_token for user"
451 | f"{login_name} and account {account_name}"
452 | )
453 | return parsed["data"]["masterToken"]
454 |
--------------------------------------------------------------------------------
/sf_git/worksheets_utils.py:
--------------------------------------------------------------------------------
1 | import json
2 | from urllib import parse
3 | from typing import Callable, List, Optional
4 | import pandas as pd
5 | import requests
6 |
7 | from sf_git.cache import save_worksheets_to_cache
8 | from sf_git.models import (
9 | AuthenticationContext,
10 | Folder,
11 | SnowsightError,
12 | Worksheet,
13 | WorksheetError,
14 | )
15 |
16 |
17 | def get_worksheets(
18 | auth_context: AuthenticationContext,
19 | store_to_cache: Optional[bool] = False,
20 | only_folder: Optional[str] = None,
21 | ) -> List[Worksheet]:
22 | """
23 | Get list of worksheets available for authenticated user
24 | """
25 |
26 | optionsparams = (
27 | '{"sort":{"col":"viewed","dir":"desc"},"limit":500,"owner":null,'
28 | '"types":["query", "folder"],"showNeverViewed":"if-invited"}'
29 | )
30 |
31 | request_json_template = {
32 | "options": optionsparams,
33 | "location": "worksheets",
34 | }
35 | req_body = parse.urlencode(request_json_template)
36 |
37 | sf_context_name = auth_context.username
38 | snowflake_context = f"{sf_context_name}::{auth_context.account_url}"
39 |
40 | res = requests.post(
41 | f"{auth_context.app_server_url}/v0/organizations/"
42 | f"{auth_context.organization_id}/entities/list",
43 | data=req_body,
44 | headers={
45 | "Accept": "application/json",
46 | "Content-Type": "application/x-www-form-urlencoded",
47 | "X-Snowflake-Context": snowflake_context,
48 | "Referer": auth_context.main_app_url,
49 | },
50 | cookies=auth_context.snowsight_token,
51 | timeout=90,
52 | )
53 |
54 | if res.status_code != 200:
55 | raise WorksheetError(
56 | "Failed to get worksheet list\n" f"\t Reason is {res.text}"
57 | )
58 | res_data = json.loads(res.text)
59 | entities = res_data["entities"]
60 | contents = res_data["models"].get("queries")
61 | if contents is None:
62 | return []
63 |
64 | worksheets = []
65 | for worksheet in entities:
66 | if (
67 | only_folder is not None
68 | and worksheet["info"]["folderName"] != only_folder
69 | ):
70 | continue
71 | if worksheet["entityType"] == "query":
72 | current_ws = Worksheet(
73 | worksheet["entityId"],
74 | worksheet["info"]["name"],
75 | worksheet["info"]["folderId"],
76 | worksheet["info"]["folderName"],
77 | content_type=worksheet["info"]["queryLanguage"],
78 | )
79 | worksheets.append(current_ws)
80 |
81 | for i, _ in enumerate(worksheets):
82 | content = contents[worksheets[i]._id]
83 | if "query" in content.keys():
84 | worksheets[i].content = content["query"]
85 | elif "drafts" in content.keys():
86 | draft_ids = [draft_id for draft_id in content["drafts"].keys()]
87 | if len(draft_ids) > 0:
88 | if len(draft_ids) > 1: # keys are timestamps
89 | last_update_id = str(
90 | max([int(draft_id) for draft_id in draft_ids])
91 | )
92 | else:
93 | last_update_id = draft_ids[0]
94 | worksheets[i].content = content["drafts"][last_update_id][
95 | "query"
96 | ]
97 | else:
98 | worksheets[i].content = ""
99 | else:
100 | worksheets[i].content = ""
101 |
102 | if store_to_cache:
103 | save_worksheets_to_cache(worksheets)
104 |
105 | return worksheets
106 |
107 |
108 | def print_worksheets(
109 | worksheets: List[Worksheet], n=10, logger: Callable = print
110 | ):
111 | """
112 | Log worksheets as a dataframe.
113 |
114 | :param worksheets: worksheets to log
115 | :param n: maximum number of worksheets to print (from head)
116 | :param logger: logging function e.g. print
117 | """
118 | worksheets_df = pd.DataFrame([ws.to_dict() for ws in worksheets])
119 | if worksheets_df.empty:
120 | logger("No worksheet")
121 | else:
122 | logger(worksheets_df.head(n))
123 |
124 |
125 | def write_worksheet(
126 | auth_context: AuthenticationContext, worksheet: Worksheet
127 | ) -> Optional[WorksheetError]:
128 | """Write local worksheet to Snowsight worksheet."""
129 |
130 | version = 1
131 | request_json_template = {
132 | "action": "saveDraft",
133 | "id": worksheet._id,
134 | "projectId": f"{worksheet._id}@{version}",
135 | "query": worksheet.content,
136 | "version": version,
137 | "modifiedTime": "2024-10-11T20:15:28.558Z",
138 | "appSessionId": "9054529735682",
139 | }
140 |
141 | req_body = parse.urlencode(request_json_template)
142 |
143 | sf_context_name = auth_context.username
144 | snowflake_context = f"{sf_context_name}::{auth_context.account_url}"
145 | res = requests.post(
146 | f"{auth_context.app_server_url}/v0/queries",
147 | data=req_body,
148 | headers={
149 | "Accept": "application/json",
150 | "Content-Type": "application/x-www-form-urlencoded",
151 | "X-Snowflake-Context": snowflake_context,
152 | "Referer": auth_context.main_app_url,
153 | "X-CSRF-Token": auth_context.csrf,
154 | "X-Snowflake-Role": "ACCOUNTADMIN",
155 | "X-Snowflake-Page-Source": "worksheet"
156 | },
157 | cookies=auth_context.cookies,
158 | timeout=90,
159 | )
160 |
161 | if res.status_code != 200:
162 | error_type = (
163 | SnowsightError.PERMISSION
164 | if res.status_code == 403
165 | else SnowsightError.UNKNOWN
166 | )
167 | return WorksheetError(
168 | f"Failed to write worksheet {worksheet.name}\n"
169 | f"\t Reason is {res.text}",
170 | error_type,
171 | )
172 |
173 |
174 | def create_worksheet(
175 | auth_context: AuthenticationContext,
176 | worksheet_name: str,
177 | folder_id: str = None,
178 | ) -> str:
179 | """Create empty worksheet on Snowsight user workspace.
180 |
181 | :returns: str new worksheet id.
182 | """
183 |
184 | request_json_template = {
185 | "action": "create",
186 | "orgId": auth_context.organization_id,
187 | "name": worksheet_name,
188 | }
189 |
190 | if folder_id:
191 | request_json_template["folderId"] = folder_id
192 |
193 | req_body = parse.urlencode(request_json_template)
194 |
195 | sf_context_name = auth_context.username
196 | snowflake_context = f"{sf_context_name}::{auth_context.account_url}"
197 |
198 | res = requests.post(
199 | f"{auth_context.app_server_url}/v0/queries",
200 | data=req_body,
201 | headers={
202 | "Accept": "application/json",
203 | "Content-Type": "application/x-www-form-urlencoded",
204 | "X-Snowflake-Context": snowflake_context,
205 | "Referer": auth_context.main_app_url,
206 | },
207 | cookies=auth_context.snowsight_token,
208 | timeout=90,
209 | )
210 |
211 | if res.status_code != 200:
212 | raise WorksheetError(
213 | f"Failed to create worksheet {worksheet_name}\n"
214 | f"\t Reason is {res.text}"
215 | )
216 | response_data = json.loads(res.text)
217 | return response_data["pid"]
218 |
219 |
220 | def get_folders(auth_context: AuthenticationContext) -> List[Folder]:
221 | """
222 | Get list of folders on authenticated user workspace
223 | """
224 |
225 | optionsparams = (
226 | '{"sort":{"col":"viewed","dir":"desc"},"limit":500,"owner":null,'
227 | '"types":["query", "folder"],"showNeverViewed":"if-invited"}'
228 | )
229 |
230 | request_json_template = {
231 | "options": optionsparams,
232 | "location": "worksheets",
233 | }
234 | req_body = parse.urlencode(request_json_template)
235 |
236 | sf_context_name = auth_context.username
237 | snowflake_context = f"{sf_context_name}::{auth_context.account_url}"
238 |
239 | res = requests.post(
240 | f"{auth_context.app_server_url}/v0/organizations/"
241 | f"{auth_context.organization_id}/entities/list",
242 | data=req_body,
243 | headers={
244 | "Accept": "application/json",
245 | "Content-Type": "application/x-www-form-urlencoded",
246 | "X-Snowflake-Context": snowflake_context,
247 | "Referer": auth_context.main_app_url,
248 | },
249 | cookies=auth_context.snowsight_token,
250 | timeout=90,
251 | )
252 |
253 | if res.status_code != 200:
254 | raise WorksheetError(
255 | "Failed to get folder list\n" f"\t Reason is {res.text}"
256 | )
257 | res_data = json.loads(res.text)
258 | entities = res_data["entities"]
259 |
260 | folders = []
261 | for entity in entities:
262 | if entity["entityType"] == "folder":
263 | current_folder = Folder(
264 | entity["entityId"],
265 | entity["info"]["name"],
266 | )
267 | folders.append(current_folder)
268 | return folders
269 |
270 |
271 | def print_folders(folders: List[Folder], n=10):
272 | folders_df = pd.DataFrame([f.to_dict() for f in folders])
273 | print(folders_df.head(n))
274 |
275 |
276 | def create_folder(
277 | auth_context: AuthenticationContext, folder_name: str
278 | ) -> str:
279 | """Create empty folder on Snowsight user workspace.
280 |
281 | :returns: str new folder id.
282 | """
283 |
284 | request_json_template = {
285 | "orgId": auth_context.organization_id,
286 | "name": folder_name,
287 | "type": "list",
288 | "visibility": "private",
289 | }
290 |
291 | req_body = parse.urlencode(request_json_template)
292 |
293 | sf_context_name = auth_context.username
294 | snowflake_context = f"{sf_context_name}::{auth_context.account_url}"
295 |
296 | res = requests.post(
297 | f"{auth_context.app_server_url}/v0/folders",
298 | data=req_body,
299 | headers={
300 | "Accept": "*/*",
301 | "Content-Type": "application/x-www-form-urlencoded",
302 | "X-Snowflake-Context": snowflake_context,
303 | "Referer": auth_context.main_app_url,
304 | },
305 | cookies=auth_context.snowsight_token,
306 | timeout=90,
307 | )
308 |
309 | if res.status_code != 200:
310 | raise WorksheetError(
311 | f"Failed to create folder {folder_name}\n"
312 | f"\t Reason is {res.text}"
313 | )
314 | response_data = json.loads(res.text)
315 | return response_data["createdFolderId"]
316 |
317 |
318 | def upload_to_snowsight(
319 | auth_context: AuthenticationContext, worksheets: List[Worksheet]
320 | ) -> dict[str, List[dict]]:
321 | """
322 | Upload worksheets to Snowsight user workspace
323 | keeping folder architecture.
324 |
325 | :param auth_context: Authentication info for Snowsight
326 | :param worksheets: list of worksheets to upload
327 |
328 | :returns: upload report with {'completed': list, 'errors': list}
329 | """
330 |
331 | upload_report = {"completed": [], "errors": []}
332 |
333 | ss_folders = get_folders(auth_context)
334 | ss_folders = {folder.name: folder for folder in ss_folders}
335 |
336 | ss_worksheets = get_worksheets(auth_context)
337 | ss_worksheets = {ws.name: ws for ws in ss_worksheets}
338 |
339 | print(
340 | " ## Writing local worksheet to SnowSight"
341 | f" for user {auth_context.username} ##"
342 | )
343 | for ws in worksheets:
344 | # folder management
345 | if ws.folder_name:
346 | if ws.folder_name in ss_folders.keys():
347 | folder_id = ss_folders[ws.folder_name]._id
348 | else:
349 | print(f"creating folder {ws.folder_name}")
350 | folder_id = create_folder(auth_context, ws.folder_name)
351 | new_folder = Folder(
352 | folder_id,
353 | ws.folder_name,
354 | )
355 | ss_folders[ws.folder_name] = new_folder
356 | else:
357 | folder_id = None
358 |
359 | # worksheet management
360 | if ws.name not in ss_worksheets.keys():
361 | print(f"creating worksheet {ws.name}")
362 | worksheet_id = create_worksheet(auth_context, ws.name, folder_id)
363 | update_content = True
364 | else:
365 | worksheet_id = ss_worksheets[ws.name]._id
366 | update_content = ws.content != ss_worksheets[ws.name].content
367 |
368 | # content management
369 | if ws.content and update_content:
370 | print(f"updating worksheet {ws.name}")
371 | err = write_worksheet(
372 | auth_context,
373 | Worksheet(
374 | worksheet_id,
375 | ws.name,
376 | folder_id,
377 | ws.folder_name,
378 | ws.content,
379 | ),
380 | )
381 | if err is not None:
382 | upload_report["errors"].append({"name": ws.name, "error": err})
383 | else:
384 | upload_report["completed"].append({"name": ws.name})
385 |
386 | print(" ## SnowSight updated ##")
387 | return upload_report
388 |
--------------------------------------------------------------------------------
/tests/conftest.py:
--------------------------------------------------------------------------------
1 | import pytest
2 | import shutil
3 | from pathlib import Path
4 | import datetime
5 | from zoneinfo import ZoneInfo
6 | from git import Repo, Actor
7 | from dotenv import dotenv_values
8 |
9 | import sf_git.models
10 | from sf_git.config import Config
11 |
12 | PACKAGE_ROOT = Path(__file__).parent.parent
13 | TEST_CONF = dotenv_values(PACKAGE_ROOT / "sf_git.test.conf")
14 |
15 | TESTING_FOLDER = Path(__file__).parent.absolute()
16 | REPO_ROOT_PATH = Path(TEST_CONF["SNOWFLAKE_VERSIONING_REPO"]).absolute()
17 | REPO_DATA_PATH = Path(TEST_CONF["WORKSHEETS_PATH"]).absolute()
18 |
19 |
20 | @pytest.fixture(scope="session", name="testing_folder")
21 | def testing_folder():
22 | return TESTING_FOLDER
23 |
24 |
25 | @pytest.fixture(scope="session", name="fixture_dotenv_path")
26 | def dotenv_path():
27 | return REPO_ROOT_PATH / "sf_git.test.conf"
28 |
29 |
30 | @pytest.fixture(scope="session", name="repo_root_path")
31 | def repo_root_path():
32 | return REPO_ROOT_PATH
33 |
34 |
35 | @pytest.fixture(scope="session", autouse=True)
36 | def setup_and_teardown():
37 | """
38 | Setup test envs and teardown
39 | """
40 |
41 | # ---- SETUP -----
42 | # Copy the content from `tests/assets/test_data`.
43 | if REPO_ROOT_PATH.is_dir():
44 | shutil.rmtree(REPO_ROOT_PATH)
45 |
46 | shutil.copytree(
47 | TESTING_FOLDER / "data", REPO_DATA_PATH, dirs_exist_ok=True
48 | )
49 | shutil.copy(PACKAGE_ROOT / "sf_git.test.conf", REPO_ROOT_PATH)
50 |
51 | # ---- RUN TESTS -----
52 | yield
53 |
54 | # ---- TEARDOWN -----
55 | shutil.rmtree(REPO_ROOT_PATH)
56 |
57 |
58 | @pytest.fixture(autouse=True, name="test_config")
59 | def mock_config(monkeypatch):
60 | import sf_git.config as config
61 |
62 | global_config = Config(
63 | repo_path=REPO_ROOT_PATH,
64 | worksheets_path=REPO_DATA_PATH,
65 | sf_account_id=TEST_CONF["SF_ACCOUNT_ID"],
66 | sf_login_name=TEST_CONF["SF_LOGIN_NAME"],
67 | sf_pwd=TEST_CONF["SF_PWD"],
68 | )
69 | monkeypatch.setattr(config, "GLOBAL_CONFIG", global_config)
70 |
71 | return global_config
72 |
73 |
74 | @pytest.fixture(scope="session", name="repo")
75 | def repo() -> Repo:
76 | """
77 | Create a git repository with sample data and history.
78 | """
79 |
80 | author = Actor("An author", "author@example.com")
81 | committer = Actor("A committer", "committer@example.com")
82 | commit_date = datetime.datetime(
83 | 2024, 1, 16, tzinfo=ZoneInfo("Europe/Paris")
84 | )
85 |
86 | # Init repo
87 | repo = Repo.init(REPO_ROOT_PATH)
88 |
89 | # Add some commits
90 | repo.index.add([REPO_DATA_PATH])
91 | repo.index.commit(
92 | "Initial skeleton",
93 | author=author,
94 | committer=committer,
95 | author_date=commit_date,
96 | commit_date=commit_date,
97 | )
98 |
99 | return repo
100 |
101 |
102 | @pytest.fixture(scope="session", name="auth_context")
103 | def auth_context() -> sf_git.models.AuthenticationContext:
104 | import sf_git.config as config
105 |
106 | context = sf_git.models.AuthenticationContext(
107 | app_server_url="https://test_snowflake.com",
108 | account_url="https://account.test_snowflake.com",
109 | account="fake.snowflake.account",
110 | account_name=config.GLOBAL_CONFIG.sf_account_id,
111 | client_id="fake",
112 | main_app_url="https://test_snowflake.com",
113 | master_token="fake",
114 | organization_id="fake.organization",
115 | region="west-europe",
116 | snowsight_token={"fake": "fake"},
117 | login_name=config.GLOBAL_CONFIG.sf_login_name,
118 | username=config.GLOBAL_CONFIG.sf_login_name,
119 | )
120 |
121 | return context
122 |
--------------------------------------------------------------------------------
/tests/data/Benchmarking_Tutorials/.[Tutorial]_Sample_queries_on_TPC-DS_data_metadata.json:
--------------------------------------------------------------------------------
1 | {"name": "[Tutorial] Sample queries on TPC-DS data", "_id": "5QwzKJlvL21", "folder_name": "Benchmarking Tutorials", "folder_id": "3nHNavVT", "content_type": "sql"}
--------------------------------------------------------------------------------
/tests/data/Benchmarking_Tutorials/.[Tutorial]_Sample_queries_on_TPC-H_data_metadata.json:
--------------------------------------------------------------------------------
1 | {"name": "[Tutorial] Sample queries on TPC-H data", "_id": "1AX6lOXnDRj", "folder_name": "Benchmarking Tutorials", "folder_id": "3nHNavVT", "content_type": "sql"}
--------------------------------------------------------------------------------
/tests/data/Benchmarking_Tutorials/.[Tutorial]_TPC-DS_100TB_Complete_Query_Test_metadata.json:
--------------------------------------------------------------------------------
1 | {"name": "[Tutorial] TPC-DS 100TB Complete Query Test", "_id": "1vN2lXM6eHu", "folder_name": "Benchmarking Tutorials", "folder_id": "3nHNavVT", "content_type": "sql"}
--------------------------------------------------------------------------------
/tests/data/Benchmarking_Tutorials/.[Tutorial]_TPC-DS_10TB_Complete_Query_Test_metadata.json:
--------------------------------------------------------------------------------
1 | {"name": "[Tutorial] TPC-DS 10TB Complete Query Test", "_id": "5S4d9X4vnRH", "folder_name": "Benchmarking Tutorials", "folder_id": "3nHNavVT", "content_type": "sql"}
--------------------------------------------------------------------------------
/tests/data/Benchmarking_Tutorials/[Tutorial]_Sample_queries_on_TPC-H_data.sql:
--------------------------------------------------------------------------------
1 | /* Tutorial 1: Sample queries on TPC-H data
2 |
3 | Prerequisites
4 | This tutorial requires the Snowflake provided
5 | snowflake_sample_data database. If you don't
6 | have this database already in your account
7 | please add it by following these instructions:
8 | https://docs.snowflake.net/manuals/user-guide/sample-data-using.html
9 |
10 | Pricing Summary Report Query (Q1)
11 | This query demonstrates basic SQL
12 | functionality using the included TPC-H sample
13 | data. The results are the amount of business
14 | that was billed, shipped, and returned.
15 |
16 | Business Question: The Pricing Summary Report
17 | Query provides a summary pricing report for
18 | all line items shipped as of a given date. The
19 | date is within 60 - 120 days of the greatest
20 | ship date contained in the database. The query
21 | lists totals for extended price, discounted
22 | extended price, discounted extended price plus
23 | tax, average quantity, average extended price,
24 | and average discount. These aggregates are
25 | grouped by RETURNFLAG and LINESTATUS, and
26 | listed in ascending order of RETURNFLAG and
27 | LINESTATUS. A count of the number of line items
28 | in each group is included.
29 | */
30 |
31 | use schema snowflake_sample_data.tpch_sf1;
32 | -- or tpch_sf100, tpch_sf1000
33 |
34 | SELECT
35 | l_returnflag,
36 | l_linestatus,
37 | sum(l_quantity) as sum_qty,
38 | sum(l_extendedprice) as sum_base_price,
39 | sum(l_extendedprice * (1-l_discount))
40 | as sum_disc_price,
41 | sum(l_extendedprice * (1-l_discount) *
42 | (1+l_tax)) as sum_charge,
43 | avg(l_quantity) as avg_qty,
44 | avg(l_extendedprice) as avg_price,
45 | avg(l_discount) as avg_disc,
46 | count(*) as count_order
47 | FROM
48 | lineitem
49 | WHERE
50 | l_shipdate <= dateadd(day, -90, to_date('1998-12-01'))
51 | GROUP BY
52 | l_returnflag,
53 | l_linestatus
54 | ORDER BY
55 | l_returnflag,
56 | l_linestatus;
57 |
58 |
--------------------------------------------------------------------------------
/tests/data/Getting_Started_Tutorials/.[Template]_Adding_a_user_and_granting_roles_metadata.json:
--------------------------------------------------------------------------------
1 | {"name": "[Template] Adding a user and granting roles", "_id": "4ux48jcFhEw", "folder_name": "Getting Started Tutorials", "folder_id": "Ga8xhnsR", "content_type": "sql"}
--------------------------------------------------------------------------------
/tests/data/Getting_Started_Tutorials/.[Tutorial]_Using_Python_to_load_and_query_sample_data_metadata.json:
--------------------------------------------------------------------------------
1 | {"name": "[Tutorial] Using Python to load and query sample data", "_id": "vMjWswGje9", "folder_name": "Getting Started Tutorials", "folder_id": "Ga8xhnsR", "content_type": "python"}
--------------------------------------------------------------------------------
/tests/data/Getting_Started_Tutorials/.[Tutorial]_Using_SQL_to_load_and_query_sample_data_metadata.json:
--------------------------------------------------------------------------------
1 | {"name": "[Tutorial] Using SQL to load and query sample data", "_id": "Je8VP5w1VD", "folder_name": "Getting Started Tutorials", "folder_id": "Ga8xhnsR", "content_type": "sql"}
--------------------------------------------------------------------------------
/tests/data/Getting_Started_Tutorials/[Template]_Adding_a_user_and_granting_roles.sql:
--------------------------------------------------------------------------------
1 | /*--
2 | In this Worksheet we will walk through creating a User in Snowflake.
3 |
4 | For the User we will provide grants to a defined default role and default warehouse
5 | and then walk through viewing all other users and roles in our account.
6 |
7 | To conclude, we will drop the created User.
8 | --*/
9 |
10 |
11 | -------------------------------------------------------------------------------------------
12 | -- Step 1: To start, we first must set our Role context
13 | -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role
14 | -- System-Defined Roles: https://docs.snowflake.com/en/user-guide/security-access-control-overview#system-defined-roles
15 | -------------------------------------------------------------------------------------------
16 |
17 | --> To run a single query, place your cursor in the query editor and select the Run button (⌘-Return).
18 | --> To run the entire worksheet, select 'Run All' from the dropdown next to the Run button (⌘-Shift-Return).
19 |
20 | ---> set our Role context
21 | USE ROLE USERADMIN;
22 |
23 | -------------------------------------------------------------------------------------------
24 | -- Step 2: Create our User
25 | -- CREATE USER: https://docs.snowflake.com/en/sql-reference/sql/create-user
26 | -------------------------------------------------------------------------------------------
27 |
28 | ---> now let's create a User using various available parameters.
29 | -- NOTE: please fill out each section below before executing the query
30 |
31 | CREATE OR REPLACE USER -- adjust user name
32 | PASSWORD = '' -- add a secure password
33 | LOGIN_NAME = '' -- add a login name
34 | FIRST_NAME = '' -- add user's first name
35 | LAST_NAME = '' -- add user's last name
36 | EMAIL = '' -- add user's email
37 | MUST_CHANGE_PASSWORD = true -- ensures a password reset on first login
38 | DEFAULT_WAREHOUSE = COMPUTE_WH; -- set default warehouse to COMPUTE_WH
39 |
40 |
41 | /*--
42 | With the User created, send the following information in a secure manner
43 | to whomever the User is created for, so that they can access this Snowflake account:
44 | --> Snowflake Account URL: This is the Snowflake account link that they'll need to login. You can find this link at the top of your browser:(ex: https://app.snowflake.com/xxxxxxx/xxxxxxxx/)
45 | --> LOGIN_NAME: from above
46 | --> PASSWORD: from above
47 | --*/
48 |
49 | -------------------------------------------------------------------------------------------
50 | -- Step 3: Grant access to a Role and Warehouse for our User
51 | -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role
52 | -- GRANT ROLE: https://docs.snowflake.com/en/sql-reference/sql/grant-role
53 | -- GRANT : https://docs.snowflake.com/en/sql-reference/sql/grant-privilege
54 | -------------------------------------------------------------------------------------------
55 |
56 | ---> with the User created, let's use our SECURITYADMIN role to grant the SYSADMIN role and COMPUTE_WH warehouse to it
57 | USE ROLE SECURITYADMIN;
58 |
59 | /*--
60 | • Granting a role to another role creates a “parent-child” relationship between the roles (also referred to as a role hierarchy).
61 | • Granting a role to a user enables the user to perform all operations allowed by the role (through the access privileges granted to the role).
62 |
63 | NOTE: The SYSADMIN role has privileges to create warehouses, databases, and database objects in an account and grant those privileges to other roles.
64 | Only grant this role to Users who should have these privileges. You can view other system-defined roles in the documentation below:
65 | • https://docs.snowflake.com/en/user-guide/security-access-control-overview#label-access-control-overview-roles-system
66 | --*/
67 |
68 | -- grant role SYSADMIN to our User
69 | GRANT ROLE SYSADMIN TO USER ;
70 |
71 |
72 | -- grant usage on the COMPUTE_WH warehouse to our SYSADMIN role
73 | GRANT USAGE ON WAREHOUSE COMPUTE_WH TO ROLE SYSADMIN;
74 |
75 |
76 | -------------------------------------------------------------------------------------------
77 | -- Step 4: Explore all Users and Roles in our Account
78 | -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role
79 | -- SHOW USERS: https://docs.snowflake.com/en/sql-reference/sql/show-users
80 | -- SHOW ROLES: https://docs.snowflake.com/en/sql-reference/sql/show-roles
81 | -------------------------------------------------------------------------------------------
82 |
83 | ---> let's now explore all users and roles in our account using our ACCOUNTADMIN role
84 | USE ROLE ACCOUNTADMIN;
85 |
86 | -- show all users in account
87 | SHOW USERS;
88 |
89 | -- show all roles in account
90 | SHOW ROLES;
91 |
92 | -------------------------------------------------------------------------------------------
93 | -- Step 5: Drop our created Users
94 | -- DROP USER: https://docs.snowflake.com/en/sql-reference/sql/drop-user
95 | -------------------------------------------------------------------------------------------
96 |
97 | ---> to drop the user, we could execute the following command
98 | DROP USER ;
99 |
--------------------------------------------------------------------------------
/tests/data/Getting_Started_Tutorials/[Tutorial]_Using_Python_to_load_and_query_sample_data.py:
--------------------------------------------------------------------------------
1 | ###################################################################################################
2 | # Tasty Bytes is a fictitious, global food truck network, that is on a mission to serve unique food
3 | # options with high quality items in a safe, convenient and cost effective way. In order to drive
4 | # forward on their mission, Tasty Bytes is beginning to leverage the Snowflake Data Cloud.
5 |
6 | # In this Python Worksheet, we will walk through the end to end process required to load a CSV file
7 | # containing Menu specific data that is currently hosted in Blob Storage.
8 |
9 | # Please click Run and see details below on what each step it doing. The final output will return
10 | # a Dataframe
11 | ###################################################################################################
12 |
13 | ###################################################################################################
14 | # Step 1 - To start, we must first import our Snowpark Package and a few Functions and Types
15 | ###################################################################################################
16 | ## Note: You can add more packages by selecting them using the Packages control and then importing them.
17 |
18 | import snowflake.snowpark as snowpark
19 | from snowflake.snowpark.functions import col
20 | from snowflake.snowpark.types import StructField, StructType, IntegerType, StringType, VariantType
21 |
22 | ###################################################################################################
23 | # Step 2 - Let's now define our Session and Create our Database, Schema and Blob Stage
24 | ###################################################################################################
25 |
26 | def main(session: snowpark.Session):
27 | # Use SQL to create our Tasty Bytes Database
28 | session.sql('CREATE OR REPLACE DATABASE tasty_bytes_sample_data;').collect()
29 |
30 | # Use SQL to create our Raw POS (Point-of-Sale) Schema
31 | session.sql('CREATE OR REPLACE SCHEMA tasty_bytes_sample_data.raw_pos;').collect()
32 |
33 | # Use SQL to create our Blob Stage
34 | session.sql('CREATE OR REPLACE STAGE tasty_bytes_sample_data.public.blob_stage url = "s3://sfquickstarts/tastybytes/" file_format = (type = csv);').collect()
35 |
36 | ###################################################################################################
37 | # Step 3 - Now, we will define the Schema for our CSV file
38 | ###################################################################################################
39 |
40 | # Define our Menu Schema
41 | menu_schema = StructType([StructField("menu_id",IntegerType()),\
42 | StructField("menu_type_id",IntegerType()),\
43 | StructField("menu_type",StringType()),\
44 | StructField("truck_brand_name",StringType()),\
45 | StructField("menu_item_id",IntegerType()),\
46 | StructField("menu_item_name",StringType()),\
47 | StructField("item_category",StringType()),\
48 | StructField("item_subcategory",StringType()),\
49 | StructField("cost_of_goods_usd",IntegerType()),\
50 | StructField("sale_price_usd",IntegerType()),\
51 | StructField("menu_item_health_metrics_obj",VariantType())])
52 |
53 | ###################################################################################################
54 | # Step 4 - Using the Schema, let's Create a Dataframe from the Menu file and Save it as a Table
55 | ###################################################################################################
56 |
57 | # Create a Dataframe from our Menu file from our Blob Stage
58 | df_blob_stage_read = session.read.schema(menu_schema).csv('@tasty_bytes_sample_data.public.blob_stage/raw_pos/menu/')
59 |
60 | # Save our Dataframe as a Menu table in our Tasty Bytes Database and Raw POS Schema
61 | df_blob_stage_read.write.mode("overwrite").save_as_table("tasty_bytes_sample_data.raw_pos.menu")
62 |
63 | ###################################################################################################
64 | # Step 5 - With the table saved, let's create a new, filtered Dataframe and Return the results
65 | ###################################################################################################
66 |
67 | # Create a new Dataframe reading from our Menu table and filtering for the Freezing Point brand
68 | df_menu_freezing_point = session.table("tasty_bytes_sample_data.raw_pos.menu").filter(col("truck_brand_name") == 'Freezing Point')
69 |
70 | # return our Dataframe
71 | return df_menu_freezing_point
--------------------------------------------------------------------------------
/tests/data/Getting_Started_Tutorials/[Tutorial]_Using_SQL_to_load_and_query_sample_data.sql:
--------------------------------------------------------------------------------
1 | /*--
2 | Tasty Bytes is a fictitious, global food truck network, that is on a mission to serve unique food options with high
3 | quality items in a safe, convenient and cost effective way. In order to drive forward on their mission, Tasty Bytes
4 | is beginning to leverage the Snowflake Data Cloud.
5 |
6 | Within this Worksheet, we will walk through the end to end process required to load a CSV file containing Menu specific data
7 | that is currently hosted in Blob Storage.
8 | --*/
9 |
10 | -------------------------------------------------------------------------------------------
11 | -- Step 1: To start, let's set the Role and Warehouse context
12 | -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role
13 | -- USE WAREHOUSE: https://docs.snowflake.com/en/sql-reference/sql/use-warehouse
14 | -------------------------------------------------------------------------------------------
15 |
16 | /*--
17 | - To run a single query, place your cursor in the query editor and select the Run button (⌘-Return).
18 | - To run the entire worksheet, select 'Run All' from the dropdown next to the Run button (⌘-Shift-Return).
19 | --*/
20 |
21 | ---> set the Role
22 | USE ROLE accountadmin;
23 |
24 | ---> set the Warehouse
25 | USE WAREHOUSE compute_wh;
26 |
27 | -------------------------------------------------------------------------------------------
28 | -- Step 2: With context in place, let's now create a Database, Schema, and Table
29 | -- CREATE DATABASE: https://docs.snowflake.com/en/sql-reference/sql/create-database
30 | -- CREATE SCHEMA: https://docs.snowflake.com/en/sql-reference/sql/create-schema
31 | -- CREATE TABLE: https://docs.snowflake.com/en/sql-reference/sql/create-table
32 | -------------------------------------------------------------------------------------------
33 |
34 | ---> create the Tasty Bytes Database
35 | CREATE OR REPLACE DATABASE tasty_bytes_sample_data;
36 |
37 | ---> create the Raw POS (Point-of-Sale) Schema
38 | CREATE OR REPLACE SCHEMA tasty_bytes_sample_data.raw_pos;
39 |
40 | ---> create the Raw Menu Table
41 | CREATE OR REPLACE TABLE tasty_bytes_sample_data.raw_pos.menu
42 | (
43 | menu_id NUMBER(19,0),
44 | menu_type_id NUMBER(38,0),
45 | menu_type VARCHAR(16777216),
46 | truck_brand_name VARCHAR(16777216),
47 | menu_item_id NUMBER(38,0),
48 | menu_item_name VARCHAR(16777216),
49 | item_category VARCHAR(16777216),
50 | item_subcategory VARCHAR(16777216),
51 | cost_of_goods_usd NUMBER(38,4),
52 | sale_price_usd NUMBER(38,4),
53 | menu_item_health_metrics_obj VARIANT
54 | );
55 |
56 | ---> confirm the empty Menu table exists
57 | SELECT * FROM tasty_bytes_sample_data.raw_pos.menu;
58 |
59 |
60 | -------------------------------------------------------------------------------------------
61 | -- Step 3: To connect to the Blob Storage, let's create a Stage
62 | -- Creating an S3 Stage: https://docs.snowflake.com/en/user-guide/data-load-s3-create-stage
63 | -------------------------------------------------------------------------------------------
64 |
65 | ---> create the Stage referencing the Blob location and CSV File Format
66 | CREATE OR REPLACE STAGE tasty_bytes_sample_data.public.blob_stage
67 | url = 's3://sfquickstarts/tastybytes/'
68 | file_format = (type = csv);
69 |
70 | ---> query the Stage to find the Menu CSV file
71 | LIST @tasty_bytes_sample_data.public.blob_stage/raw_pos/menu/;
72 |
73 |
74 | -------------------------------------------------------------------------------------------
75 | -- Step 4: Now let's Load the Menu CSV file from the Stage
76 | -- COPY INTO : https://docs.snowflake.com/en/sql-reference/sql/copy-into-table
77 | -------------------------------------------------------------------------------------------
78 |
79 | ---> copy the Menu file into the Menu table
80 | COPY INTO tasty_bytes_sample_data.raw_pos.menu
81 | FROM @tasty_bytes_sample_data.public.blob_stage/raw_pos/menu/;
82 |
83 |
84 | -------------------------------------------------------------------------------------------
85 | -- Step 5: Query the Menu table
86 | -- SELECT: https://docs.snowflake.com/en/sql-reference/sql/select
87 | -- TOP : https://docs.snowflake.com/en/sql-reference/constructs/top_n
88 | -- FLATTEN: https://docs.snowflake.com/en/sql-reference/functions/flatten
89 | -------------------------------------------------------------------------------------------
90 |
91 | ---> how many rows are in the table?
92 | SELECT COUNT(*) AS row_count FROM tasty_bytes_sample_data.raw_pos.menu;
93 |
94 | ---> what do the top 10 rows look like?
95 | SELECT TOP 10 * FROM tasty_bytes_sample_data.raw_pos.menu;
96 |
97 | ---> what menu items does the Freezing Point brand sell?
98 | SELECT
99 | menu_item_name
100 | FROM tasty_bytes_sample_data.raw_pos.menu
101 | WHERE truck_brand_name = 'Freezing Point';
102 |
103 | ---> what is the profit on Mango Sticky Rice?
104 | SELECT
105 | menu_item_name,
106 | (sale_price_usd - cost_of_goods_usd) AS profit_usd
107 | FROM tasty_bytes_sample_data.raw_pos.menu
108 | WHERE 1=1
109 | AND truck_brand_name = 'Freezing Point'
110 | AND menu_item_name = 'Mango Sticky Rice';
111 |
112 | ---> to finish, let's extract the Mango Sticky Rice ingredients from the semi-structured column
113 | SELECT
114 | m.menu_item_name,
115 | obj.value:"ingredients" AS ingredients
116 | FROM tasty_bytes_sample_data.raw_pos.menu m,
117 | LATERAL FLATTEN (input => m.menu_item_health_metrics_obj:menu_item_health_metrics) obj
118 | WHERE 1=1
119 | AND truck_brand_name = 'Freezing Point'
120 | AND menu_item_name = 'Mango Sticky Rice';
--------------------------------------------------------------------------------
/tests/fixtures/contents.json:
--------------------------------------------------------------------------------
1 | {
2 | "4ux48jcFhEw": {
3 | "snowflakeRequestId": null,
4 | "snowflakeQueryId": "",
5 | "query": "/*--\nIn this Worksheet we will walk through creating a User in Snowflake.\n\nFor the User we will provide grants to a defined default role and default warehouse\nand then walk through viewing all other users and roles in our account.\n\nTo conclude, we will drop the created User.\n--*/\n\n\n-------------------------------------------------------------------------------------------\n -- Step 1: To start, we first must set our Role context\n -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role\n -- System-Defined Roles: https://docs.snowflake.com/en/user-guide/security-access-control-overview#system-defined-roles\n-------------------------------------------------------------------------------------------\n\n--> To run a single query, place your cursor in the query editor and select the Run button (⌘-Return).\n--> To run the entire worksheet, select 'Run All' from the dropdown next to the Run button (⌘-Shift-Return).\n\n---> set our Role context\n USE ROLE USERADMIN;\n\n-------------------------------------------------------------------------------------------\n -- Step 2: Create our User\n -- CREATE USER: https://docs.snowflake.com/en/sql-reference/sql/create-user\n-------------------------------------------------------------------------------------------\n\n---> now let's create a User using various available parameters.\n -- NOTE: please fill out each section below before executing the query\n\nCREATE OR REPLACE USER -- adjust user name\nPASSWORD = '' -- add a secure password\nLOGIN_NAME = '' -- add a login name\nFIRST_NAME = '' -- add user's first name\nLAST_NAME = '' -- add user's last name\nEMAIL = '' -- add user's email \nMUST_CHANGE_PASSWORD = true -- ensures a password reset on first login\nDEFAULT_WAREHOUSE = COMPUTE_WH; -- set default warehouse to COMPUTE_WH\n\n \n/*--\nWith the User created, send the following information in a secure manner\nto whomever the User is created for, so that they can access this Snowflake account:\n --> Snowflake Account URL: This is the Snowflake account link that they'll need to login. You can find this link at the top of your browser:(ex: https://app.snowflake.com/xxxxxxx/xxxxxxxx/)\n --> LOGIN_NAME: from above\n --> PASSWORD: from above\n--*/\n\n-------------------------------------------------------------------------------------------\n -- Step 3: Grant access to a Role and Warehouse for our User\n -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role\n -- GRANT ROLE: https://docs.snowflake.com/en/sql-reference/sql/grant-role\n -- GRANT : https://docs.snowflake.com/en/sql-reference/sql/grant-privilege\n-------------------------------------------------------------------------------------------\n\n---> with the User created, let's use our SECURITYADMIN role to grant the SYSADMIN role and COMPUTE_WH warehouse to it\nUSE ROLE SECURITYADMIN;\n\n /*--\n • Granting a role to another role creates a “parent-child” relationship between the roles (also referred to as a role hierarchy).\n • Granting a role to a user enables the user to perform all operations allowed by the role (through the access privileges granted to the role).\n\n NOTE: The SYSADMIN role has privileges to create warehouses, databases, and database objects in an account and grant those privileges to other roles.\n Only grant this role to Users who should have these privileges. You can view other system-defined roles in the documentation below:\n • https://docs.snowflake.com/en/user-guide/security-access-control-overview#label-access-control-overview-roles-system\n --*/\n\n-- grant role SYSADMIN to our User\nGRANT ROLE SYSADMIN TO USER ;\n\n\n-- grant usage on the COMPUTE_WH warehouse to our SYSADMIN role\nGRANT USAGE ON WAREHOUSE COMPUTE_WH TO ROLE SYSADMIN;\n\n\n-------------------------------------------------------------------------------------------\n -- Step 4: Explore all Users and Roles in our Account\n -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role\n -- SHOW USERS: https://docs.snowflake.com/en/sql-reference/sql/show-users\n -- SHOW ROLES: https://docs.snowflake.com/en/sql-reference/sql/show-roles\n-------------------------------------------------------------------------------------------\n\n---> let's now explore all users and roles in our account using our ACCOUNTADMIN role\nUSE ROLE ACCOUNTADMIN;\n\n-- show all users in account\nSHOW USERS;\n\n-- show all roles in account\nSHOW ROLES;\n\n-------------------------------------------------------------------------------------------\n -- Step 5: Drop our created Users\n -- DROP USER: https://docs.snowflake.com/en/sql-reference/sql/drop-user\n-------------------------------------------------------------------------------------------\n\n---> to drop the user, we could execute the following command\nDROP USER ;\n",
6 | "queryContext": {
7 | "role": "ACCOUNTADMIN",
8 | "warehouse": "COMPUTE_WH",
9 | "database": "",
10 | "schema": "",
11 | "secondaryRoles": ""
12 | },
13 | "drafts": {
14 | "147276799089": {
15 | "query": "/*--\nIn this Worksheet we will walk through creating a User in Snowflake.\n\nFor the User we will provide grants to a defined default role and default warehouse\nand then walk through viewing all other users and roles in our account.\n\nTo conclude, we will drop the created User.\n--*/\n\n\n-------------------------------------------------------------------------------------------\n -- Step 1: To start, we first must set our Role context\n -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role\n -- System-Defined Roles: https://docs.snowflake.com/en/user-guide/security-access-control-overview#system-defined-roles\n-------------------------------------------------------------------------------------------\n\n--> To run a single query, place your cursor in the query editor and select the Run button (⌘-Return).\n--> To run the entire worksheet, select 'Run All' from the dropdown next to the Run button (⌘-Shift-Return).\n\n---> set our Role context\n USE ROLE USERADMIN;\n\n-------------------------------------------------------------------------------------------\n -- Step 2: Create our User\n -- CREATE USER: https://docs.snowflake.com/en/sql-reference/sql/create-user\n-------------------------------------------------------------------------------------------\n\n---> now let's create a User using various available parameters.\n -- NOTE: please fill out each section below before executing the query\n\nCREATE OR REPLACE USER -- adjust user name\nPASSWORD = '' -- add a secure password\nLOGIN_NAME = '' -- add a login name\nFIRST_NAME = '' -- add user's first name\nLAST_NAME = '' -- add user's last name\nEMAIL = '' -- add user's email \nMUST_CHANGE_PASSWORD = true -- ensures a password reset on first login\nDEFAULT_WAREHOUSE = COMPUTE_WH; -- set default warehouse to COMPUTE_WH\n\n \n/*--\nWith the User created, send the following information in a secure manner\nto whomever the User is created for, so that they can access this Snowflake account:\n --> Snowflake Account URL: This is the Snowflake account link that they'll need to login. You can find this link at the top of your browser:(ex: https://app.snowflake.com/xxxxxxx/xxxxxxxx/)\n --> LOGIN_NAME: from above\n --> PASSWORD: from above\n--*/\n\n-------------------------------------------------------------------------------------------\n -- Step 3: Grant access to a Role and Warehouse for our User\n -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role\n -- GRANT ROLE: https://docs.snowflake.com/en/sql-reference/sql/grant-role\n -- GRANT : https://docs.snowflake.com/en/sql-reference/sql/grant-privilege\n-------------------------------------------------------------------------------------------\n\n---> with the User created, let's use our SECURITYADMIN role to grant the SYSADMIN role and COMPUTE_WH warehouse to it\nUSE ROLE SECURITYADMIN;\n\n /*--\n • Granting a role to another role creates a “parent-child” relationship between the roles (also referred to as a role hierarchy).\n • Granting a role to a user enables the user to perform all operations allowed by the role (through the access privileges granted to the role).\n\n NOTE: The SYSADMIN role has privileges to create warehouses, databases, and database objects in an account and grant those privileges to other roles.\n Only grant this role to Users who should have these privileges. You can view other system-defined roles in the documentation below:\n • https://docs.snowflake.com/en/user-guide/security-access-control-overview#label-access-control-overview-roles-system\n --*/\n\n-- grant role SYSADMIN to our User\nGRANT ROLE SYSADMIN TO USER ;\n\n\n-- grant usage on the COMPUTE_WH warehouse to our SYSADMIN role\nGRANT USAGE ON WAREHOUSE COMPUTE_WH TO ROLE SYSADMIN;\n\n\n-------------------------------------------------------------------------------------------\n -- Step 4: Explore all Users and Roles in our Account\n -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role\n -- SHOW USERS: https://docs.snowflake.com/en/sql-reference/sql/show-users\n -- SHOW ROLES: https://docs.snowflake.com/en/sql-reference/sql/show-roles\n-------------------------------------------------------------------------------------------\n\n---> let's now explore all users and roles in our account using our ACCOUNTADMIN role\nUSE ROLE ACCOUNTADMIN;\n\n-- show all users in account\nSHOW USERS;\n\n-- show all roles in account\nSHOW ROLES;\n\n-------------------------------------------------------------------------------------------\n -- Step 5: Drop our created Users\n -- DROP USER: https://docs.snowflake.com/en/sql-reference/sql/drop-user\n-------------------------------------------------------------------------------------------\n\n---> to drop the user, we could execute the following command\nDROP USER ;\n",
16 | "paramRefs": [],
17 | "queryRange": null,
18 | "executionContext": {
19 | "role": "ACCOUNTADMIN",
20 | "warehouse": "COMPUTE_WH",
21 | "database": "",
22 | "schema": "",
23 | "secondaryRoles": ""
24 | },
25 | "queryLanguage": "sql",
26 | "appSessionId": 0
27 | }
28 | },
29 | "draftUpdates": {
30 | "147276799089": 1705628729507
31 | },
32 | "transforms": [],
33 | "queryLanguage": "sql",
34 | "pid": "4ux48jcFhEw",
35 | "name": "[Template] Adding a user and granting roles",
36 | "orgId": "mocked",
37 | "ownerId": "mocked",
38 | "folderId": "Ga8xhnsR",
39 | "visibility": "private",
40 | "modified": "2024-01-19T01:45:29.514886Z",
41 | "version": 2,
42 | "isParamQuery": false,
43 | "projectType": "query",
44 | "executionContext": {
45 | "role": "ACCOUNTADMIN",
46 | "warehouse": "COMPUTE_WH",
47 | "database": "",
48 | "schema": "",
49 | "secondaryRoles": ""
50 | },
51 | "editable": true,
52 | "runnable": true,
53 | "resultsViewable": true,
54 | "url": "mocked",
55 | "slug": "mocked",
56 | "members": [
57 | {
58 | "memberType": "user",
59 | "userId": "mocked",
60 | "memberId": "mocked",
61 | "role": "owner",
62 | "hasRole": true
63 | }
64 | ],
65 | "hasRequiredRole": true
66 | },
67 | "1EDjP07epDt": {
68 | "draftUpdates": {
69 | "147276799089": 1706161972435
70 | },
71 | "drafts": {
72 | "147276799089": {
73 | "appSessionId": 0,
74 | "executionContext": {
75 | "database": "",
76 | "role": "ACCOUNTADMIN",
77 | "schema": "",
78 | "secondaryRoles": "",
79 | "warehouse": ""
80 | },
81 | "paramRefs": [],
82 | "query": "SELECT COUNT(*) from pytests;",
83 | "queryLanguage": "sql",
84 | "queryRange": null
85 | }
86 | },
87 | "editable": true,
88 | "executionContext": {
89 | "database": "",
90 | "role": "ACCOUNTADMIN",
91 | "schema": "",
92 | "secondaryRoles": "",
93 | "warehouse": ""
94 | },
95 | "folderId": "YPegqcHt",
96 | "hasRequiredRole": true,
97 | "isParamQuery": false,
98 | "members": [
99 | {
100 | "hasRole": true,
101 | "memberId": "fake",
102 | "memberType": "user",
103 | "role": "owner",
104 | "userId": "fake"
105 | }
106 | ],
107 | "modified": "2024-01-25T05:52:52.471755Z",
108 | "name": "[Tutorial] Sample queries on TPC-DS data",
109 | "orgId": "fake",
110 | "ownerId": "fake",
111 | "pid": "1EDjP07epDt",
112 | "projectType": "query",
113 | "queryContext": {
114 | "database": "",
115 | "role": "ACCOUNTADMIN",
116 | "schema": "",
117 | "secondaryRoles": "",
118 | "warehouse": ""
119 | },
120 | "queryLanguage": "sql",
121 | "resultsViewable": true,
122 | "runnable": true,
123 | "slug": "fake",
124 | "snowflakeQueryId": "",
125 | "snowflakeRequestId": "",
126 | "startDate": "2024-01-25T04:25:49.861694096Z",
127 | "transforms": [],
128 | "url": "/fake#query",
129 | "version": 1,
130 | "visibility": "private"
131 | }
132 | }
--------------------------------------------------------------------------------
/tests/fixtures/entities.json:
--------------------------------------------------------------------------------
1 | [{
2 | "entityId": "4ux48jcFhEw",
3 | "entityType": "query",
4 | "info": {
5 | "name": "[Template] Adding a user and granting roles",
6 | "slug": "mocked",
7 | "version": 0,
8 | "content": "",
9 | "dashboardRows": [],
10 | "folderId": "Ga8xhnsR",
11 | "folderName": "Getting Started Tutorials",
12 | "folderType": "list",
13 | "folderUrl": "/mocked",
14 | "folderSlug": "getting-started-tutorials-fGa8xhnsR",
15 | "visibility": "private",
16 | "ownerId": 0,
17 | "modified": "2024-01-19T01:45:29.514886Z",
18 | "created": "2024-01-14T08:27:39.503542Z",
19 | "viewed": "2024-01-17T01:03:59.460044Z",
20 | "queryLanguage": "sql",
21 | "role": "ACCOUNTADMIN",
22 | "url": "/mocked"
23 | },
24 | "match": null
25 | },
26 | {
27 | "entityId": "Ga8xhnsR",
28 | "entityType": "folder",
29 | "info": {
30 | "name": "Getting Started Tutorials",
31 | "slug": "getting-started-tutorials-fGa8xhnsR",
32 | "version": 0,
33 | "content": "",
34 | "dashboardRows": [],
35 | "folderId": null,
36 | "folderName": "",
37 | "folderType": "list",
38 | "visibility": "private",
39 | "ownerId": 0,
40 | "modified": "2024-01-23T00:56:39.139336Z",
41 | "created": "2024-01-23T00:56:39.139336Z",
42 | "viewed": null,
43 | "queryLanguage": "",
44 | "role": "ACCOUNTADMIN",
45 | "url": "/mocked"
46 | },
47 | "match": null
48 | },
49 | {
50 | "entityId": "1EDjP07epDt",
51 | "entityType": "query",
52 | "info": {
53 | "content": "",
54 | "created": "2024-01-25T04:25:49.858888Z",
55 | "dashboardRows": [],
56 | "folderId": "YPegqcHt",
57 | "folderName": "Benchmarking Tutorials",
58 | "folderSlug": "benchmarking-tutorials-fYPegqcHt",
59 | "folderType": "list",
60 | "folderUrl": "/fake/#/benchmarking-tutorials-fYPegqcHt",
61 | "modified": "2024-01-25T05:52:52.471755Z",
62 | "name": "[Tutorial] Sample queries on TPC-DS data",
63 | "ownerId": 0,
64 | "queryLanguage": "sql",
65 | "role": "ACCOUNTADMIN",
66 | "slug": "fake",
67 | "url": "/fake#query",
68 | "version": 0,
69 | "viewed": "2024-01-25T05:03:28.247018Z",
70 | "visibility": "private"
71 | },
72 | "match": null
73 | },
74 | {
75 | "entityId": "YPegqcHt",
76 | "entityType": "folder",
77 | "info": {
78 | "content": "",
79 | "created": "2024-01-25T04:25:49.140711Z",
80 | "dashboardRows": [],
81 | "folderId": null,
82 | "folderName": "",
83 | "folderType": "list",
84 | "modified": "2024-01-25T04:25:49.140711Z",
85 | "name": "Benchmarking Tutorials",
86 | "ownerId": 0,
87 | "queryLanguage": "",
88 | "role": "ACCOUNTADMIN",
89 | "slug": "benchmarking-tutorials-fYPegqcHt",
90 | "url": "/fake/#/benchmarking-tutorials-fYPegqcHt",
91 | "version": 0,
92 | "viewed": null,
93 | "visibility": "private"
94 | },
95 | "match": null
96 | }
97 | ]
--------------------------------------------------------------------------------
/tests/fixtures/folders.json:
--------------------------------------------------------------------------------
1 | [{"_id": "Ga8xhnsR", "name": "Getting Started Tutorials", "worksheets": []}]
--------------------------------------------------------------------------------
/tests/fixtures/worksheets.json:
--------------------------------------------------------------------------------
1 | [{"_id": "4ux48jcFhEw", "name": "[Template] Adding a user and granting roles", "folder_id": "Ga8xhnsR", "folder_name": "Getting Started Tutorials", "content": "/*--\nIn this Worksheet we will walk through creating a User in Snowflake.\n\nFor the User we will provide grants to a defined default role and default warehouse\nand then walk through viewing all other users and roles in our account.\n\nTo conclude, we will drop the created User.\n--*/\n\n\n-------------------------------------------------------------------------------------------\n -- Step 1: To start, we first must set our Role context\n -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role\n -- System-Defined Roles: https://docs.snowflake.com/en/user-guide/security-access-control-overview#system-defined-roles\n-------------------------------------------------------------------------------------------\n\n--> To run a single query, place your cursor in the query editor and select the Run button (\u2318-Return).\n--> To run the entire worksheet, select 'Run All' from the dropdown next to the Run button (\u2318-Shift-Return).\n\n---> set our Role context\n USE ROLE USERADMIN;\n\n-------------------------------------------------------------------------------------------\n -- Step 2: Create our User\n -- CREATE USER: https://docs.snowflake.com/en/sql-reference/sql/create-user\n-------------------------------------------------------------------------------------------\n\n---> now let's create a User using various available parameters.\n -- NOTE: please fill out each section below before executing the query\n\nCREATE OR REPLACE USER -- adjust user name\nPASSWORD = '' -- add a secure password\nLOGIN_NAME = '' -- add a login name\nFIRST_NAME = '' -- add user's first name\nLAST_NAME = '' -- add user's last name\nEMAIL = '' -- add user's email \nMUST_CHANGE_PASSWORD = true -- ensures a password reset on first login\nDEFAULT_WAREHOUSE = COMPUTE_WH; -- set default warehouse to COMPUTE_WH\n\n \n/*--\nWith the User created, send the following information in a secure manner\nto whomever the User is created for, so that they can access this Snowflake account:\n --> Snowflake Account URL: This is the Snowflake account link that they'll need to login. You can find this link at the top of your browser:(ex: https://app.snowflake.com/xxxxxxx/xxxxxxxx/)\n --> LOGIN_NAME: from above\n --> PASSWORD: from above\n--*/\n\n-------------------------------------------------------------------------------------------\n -- Step 3: Grant access to a Role and Warehouse for our User\n -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role\n -- GRANT ROLE: https://docs.snowflake.com/en/sql-reference/sql/grant-role\n -- GRANT : https://docs.snowflake.com/en/sql-reference/sql/grant-privilege\n-------------------------------------------------------------------------------------------\n\n---> with the User created, let's use our SECURITYADMIN role to grant the SYSADMIN role and COMPUTE_WH warehouse to it\nUSE ROLE SECURITYADMIN;\n\n /*--\n \u2022 Granting a role to another role creates a \u201cparent-child\u201d relationship between the roles (also referred to as a role hierarchy).\n \u2022 Granting a role to a user enables the user to perform all operations allowed by the role (through the access privileges granted to the role).\n\n NOTE: The SYSADMIN role has privileges to create warehouses, databases, and database objects in an account and grant those privileges to other roles.\n Only grant this role to Users who should have these privileges. You can view other system-defined roles in the documentation below:\n \u2022 https://docs.snowflake.com/en/user-guide/security-access-control-overview#label-access-control-overview-roles-system\n --*/\n\n-- grant role SYSADMIN to our User\nGRANT ROLE SYSADMIN TO USER ;\n\n\n-- grant usage on the COMPUTE_WH warehouse to our SYSADMIN role\nGRANT USAGE ON WAREHOUSE COMPUTE_WH TO ROLE SYSADMIN;\n\n\n-------------------------------------------------------------------------------------------\n -- Step 4: Explore all Users and Roles in our Account\n -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role\n -- SHOW USERS: https://docs.snowflake.com/en/sql-reference/sql/show-users\n -- SHOW ROLES: https://docs.snowflake.com/en/sql-reference/sql/show-roles\n-------------------------------------------------------------------------------------------\n\n---> let's now explore all users and roles in our account using our ACCOUNTADMIN role\nUSE ROLE ACCOUNTADMIN;\n\n-- show all users in account\nSHOW USERS;\n\n-- show all roles in account\nSHOW ROLES;\n\n-------------------------------------------------------------------------------------------\n -- Step 5: Drop our created Users\n -- DROP USER: https://docs.snowflake.com/en/sql-reference/sql/drop-user\n-------------------------------------------------------------------------------------------\n\n---> to drop the user, we could execute the following command\nDROP USER ;\n", "content_type": "sql"}]
--------------------------------------------------------------------------------
/tests/fixtures/worksheets_update.json:
--------------------------------------------------------------------------------
1 | [{"_id": "4ux48jcFhEw", "name": "[Template] Adding a user and granting roles", "folder_id": "Ga8xhnsR", "folder_name": "Getting Started Tutorials", "content": "// Updated", "content_type": "sql"}]
--------------------------------------------------------------------------------
/tests/test_cache.py:
--------------------------------------------------------------------------------
1 | import pytest
2 |
3 | import sf_git.cache as cache
4 | from sf_git.models import SnowflakeGitError, Worksheet
5 | import sf_git.config as config
6 |
7 |
8 | def test_load_ws_after_init(repo):
9 |
10 | worksheets = cache.load_worksheets_from_cache(
11 | repo,
12 | branch_name='main',
13 | )
14 |
15 | assert isinstance(worksheets, list)
16 | assert len(worksheets) == 7
17 |
18 |
19 | def test_load_ws_wrong_branch(repo):
20 |
21 | with pytest.raises(SnowflakeGitError):
22 | cache.load_worksheets_from_cache(
23 | repo,
24 | branch_name='non_existing'
25 | )
26 |
27 |
28 | def test_load_ws_from_existing_folder(repo):
29 |
30 | # Todo: only_folder param is the SF one, not intuitive
31 | worksheets = cache.load_worksheets_from_cache(
32 | repo,
33 | only_folder='Benchmarking Tutorials'
34 | )
35 |
36 | assert isinstance(worksheets, list)
37 | assert len(worksheets) == 4
38 |
39 |
40 | def test_load_ws_from_non_existing_folder(repo):
41 | worksheets = cache.load_worksheets_from_cache(
42 | repo,
43 | only_folder='Non existing'
44 | )
45 |
46 | assert isinstance(worksheets, list)
47 | assert len(worksheets) == 0
48 |
49 |
50 | @pytest.mark.parametrize(
51 | "worksheet",
52 | [Worksheet(
53 | _id="worksheet_id_01",
54 | name="test_worksheet_01",
55 | folder_id="folder_id_01",
56 | folder_name="Benchmarking_Tutorials",
57 | content_type="sql",
58 | content="SELECT count(*) from PASSING_TESTS"
59 | )]
60 | )
61 | def test_save_sql_ws_to_cache_existing_folder(worksheet):
62 |
63 | cache.save_worksheets_to_cache([worksheet])
64 |
65 | expected_file_path = config.GLOBAL_CONFIG.worksheets_path / "Benchmarking_Tutorials" / f"{worksheet.name}.sql"
66 |
67 | assert expected_file_path.is_file()
68 | with open(expected_file_path, "r") as f:
69 | assert f.read() == worksheet.content
70 |
71 |
72 | @pytest.mark.parametrize(
73 | "worksheet",
74 | [Worksheet(
75 | _id="worksheet_id_02",
76 | name="test_worksheet_02",
77 | folder_id="folder_id_01",
78 | folder_name="Benchmarking_Tutorials",
79 | content_type="python",
80 | content="import sf_git; print(sf_git.__version__)"
81 | )]
82 | )
83 | def test_save_py_ws_to_cache_existing_folder(
84 | worksheet
85 | ):
86 |
87 | cache.save_worksheets_to_cache([worksheet])
88 |
89 | expected_file_path = config.GLOBAL_CONFIG.worksheets_path / "Benchmarking_Tutorials" / f"{worksheet.name}.py"
90 |
91 | assert expected_file_path.is_file()
92 | with open(expected_file_path, "r") as f:
93 | assert f.read() == worksheet.content
94 |
95 |
96 | @pytest.mark.parametrize(
97 | "worksheet",
98 | [Worksheet(
99 | _id="worksheet_id_03",
100 | name="test_worksheet_03",
101 | folder_id="folder_id_02",
102 | folder_name="new_folder_for_test",
103 | content_type="sql",
104 | content="SELECT count(*) from PASSING_TESTS"
105 | )]
106 | )
107 | def test_save_sql_ws_to_cache_new_folder(worksheet):
108 |
109 | cache.save_worksheets_to_cache([worksheet])
110 |
111 | expected_file_path = config.GLOBAL_CONFIG.worksheets_path / "new_folder_for_test" / f"{worksheet.name}.sql"
112 |
113 | assert expected_file_path.is_file()
114 | with open(expected_file_path, "r") as f:
115 | assert f.read() == worksheet.content
116 |
117 |
118 | @pytest.mark.parametrize(
119 | "worksheet",
120 | [Worksheet(
121 | _id="worksheet_id_04",
122 | name="test_worksheet_04",
123 | folder_id="folder_id_02",
124 | folder_name="new_folder_for_test",
125 | content_type="scala",
126 | content='println("Coming soon");'
127 | )]
128 | )
129 | def test_save_unsupported_extension_ws_to_cache_new_folder(worksheet):
130 |
131 | cache.save_worksheets_to_cache([worksheet])
132 |
133 | # Todo: defaults to sql as of now
134 | expected_file_path = config.GLOBAL_CONFIG.worksheets_path / "new_folder_for_test" / f"{worksheet.name}.scala"
135 |
136 | assert not expected_file_path.is_file()
137 |
138 |
139 | @pytest.mark.run(after='test_save_py_ws_to_cache_existing_folder')
140 | def test_load_only_tracked_files(
141 | repo,
142 | ):
143 |
144 | worksheets = cache.load_worksheets_from_cache(
145 | repo,
146 | only_folder="Benchmarking Tutorials"
147 | )
148 |
149 | assert isinstance(worksheets, list)
150 | assert 'test_worksheet_02' not in [ws.name for ws in worksheets]
151 |
152 |
153 | @pytest.mark.run(after='test_save_sql_ws_to_cache_new_folder')
154 | def test_load_only_tracked_files(
155 | repo,
156 | ):
157 |
158 | worksheets = cache.load_worksheets_from_cache(
159 | repo,
160 | only_folder="new_folder_for_test"
161 | )
162 |
163 | assert isinstance(worksheets, list)
164 | assert len(worksheets) == 0
165 |
--------------------------------------------------------------------------------
/tests/test_commands.py:
--------------------------------------------------------------------------------
1 | import json
2 | import shutil
3 |
4 | import dotenv
5 | import pytest
6 | from click import UsageError
7 | from git.repo import Repo
8 |
9 | import sf_git.commands
10 | from sf_git.models import Worksheet
11 | import sf_git.config as config
12 |
13 |
14 | @pytest.fixture
15 | def worksheets(testing_folder):
16 | with open(testing_folder / "fixtures" / "worksheets.json", "r") as f:
17 | worksheets_as_dict = json.load(f)
18 | return [Worksheet(**w) for w in worksheets_as_dict]
19 |
20 |
21 | @pytest.fixture
22 | def mock_authenticate_to_snowsight(auth_context, monkeypatch):
23 | def get_fake_auth_context(account_id, username, password, auth_mode=None):
24 | return auth_context
25 |
26 | monkeypatch.setattr(
27 | sf_git.commands, "authenticate_to_snowsight", get_fake_auth_context
28 | )
29 |
30 |
31 | @pytest.fixture
32 | def mock_sf_get_worksheets(worksheets, monkeypatch):
33 | def get_fake_worksheets(auth_context, store_to_cache, only_folder):
34 | return worksheets
35 |
36 | monkeypatch.setattr(
37 | sf_git.commands, "sf_get_worksheets", get_fake_worksheets
38 | )
39 |
40 |
41 | @pytest.fixture
42 | def mock_load_worksheets_from_cache(worksheets, monkeypatch):
43 | def get_fixture_worksheets(repo, branch_name, only_folder):
44 | return worksheets
45 |
46 | monkeypatch.setattr(
47 | sf_git.commands, "load_worksheets_from_cache", get_fixture_worksheets
48 | )
49 |
50 |
51 | @pytest.fixture
52 | def no_print(monkeypatch):
53 | def do_nothing(worksheets, logger):
54 | pass
55 |
56 | monkeypatch.setattr(sf_git.commands, "print_worksheets", do_nothing)
57 |
58 |
59 | @pytest.fixture
60 | def no_upload(monkeypatch):
61 | def successful_upload(auth_context, worksheets):
62 | return {
63 | "completed": [worksheet.name for worksheet in worksheets],
64 | "errors": [],
65 | }
66 |
67 | monkeypatch.setattr(
68 | sf_git.commands, "upload_to_snowsight", successful_upload
69 | )
70 |
71 |
72 | @pytest.fixture
73 | def alternative_repo_path(testing_folder):
74 | return testing_folder / "tmp" / "sf_git_test_alternative"
75 |
76 |
77 | @pytest.fixture(name="alternative_repo")
78 | def setup_and_teardown(alternative_repo_path):
79 | """
80 | Setup alternative git repo and teardown
81 | """
82 |
83 | # ---- SETUP -----
84 | Repo.init(alternative_repo_path, mkdir=True)
85 |
86 | # ---- RUN TESTS -----
87 | yield
88 |
89 | # ---- TEARDOWN -----
90 | shutil.rmtree(alternative_repo_path)
91 |
92 |
93 | def test_fetch_worksheets_when_pwd_auth_with_pwd(
94 | mock_authenticate_to_snowsight,
95 | mock_sf_get_worksheets,
96 | no_print,
97 | ):
98 | worksheets = sf_git.commands.fetch_worksheets_procedure(
99 | username=config.GLOBAL_CONFIG.sf_login_name,
100 | account_id=config.GLOBAL_CONFIG.sf_account_id,
101 | auth_mode="PWD",
102 | password=config.GLOBAL_CONFIG.sf_pwd,
103 | store=False,
104 | only_folder="",
105 | logger=lambda x: None,
106 | )
107 |
108 | assert isinstance(worksheets, list)
109 | assert len(worksheets) == 1
110 |
111 |
112 | def test_fetch_worksheets_when_pwd_auth_without_pwd(
113 | mock_authenticate_to_snowsight,
114 | mock_sf_get_worksheets,
115 | no_print,
116 | ):
117 | with pytest.raises(UsageError):
118 | sf_git.commands.fetch_worksheets_procedure(
119 | username=config.GLOBAL_CONFIG.sf_login_name,
120 | account_id=config.GLOBAL_CONFIG.sf_account_id,
121 | auth_mode="PWD",
122 | password=None,
123 | store=False,
124 | only_folder="",
125 | logger=lambda x: None,
126 | )
127 |
128 |
129 | def test_push_worksheets_when_pwd_auth_without_pwd(
130 | mock_authenticate_to_snowsight,
131 | mock_load_worksheets_from_cache,
132 | no_upload,
133 | no_print,
134 | ):
135 | with pytest.raises(UsageError):
136 | sf_git.commands.push_worksheets_procedure(
137 | username=config.GLOBAL_CONFIG.sf_login_name,
138 | account_id=config.GLOBAL_CONFIG.sf_account_id,
139 | auth_mode="PWD",
140 | password=None,
141 | only_folder="",
142 | logger=lambda x: None,
143 | )
144 |
145 |
146 | def test_push_worksheets_when_pwd_auth_with_pwd(
147 | repo,
148 | mock_authenticate_to_snowsight,
149 | mock_load_worksheets_from_cache,
150 | no_upload,
151 | no_print,
152 | ):
153 | report = sf_git.commands.push_worksheets_procedure(
154 | username=config.GLOBAL_CONFIG.sf_login_name,
155 | account_id=config.GLOBAL_CONFIG.sf_account_id,
156 | auth_mode="PWD",
157 | password=config.GLOBAL_CONFIG.sf_pwd,
158 | only_folder="",
159 | logger=lambda x: None,
160 | )
161 |
162 | assert isinstance(report, dict)
163 | assert len(report["completed"]) == 1
164 | assert len(report["errors"]) == 0
165 |
166 |
167 | @pytest.mark.parametrize(
168 | "key",
169 | [
170 | "SNOWFLAKE_VERSIONING_REPO",
171 | "WORKSHEETS_PATH",
172 | "SF_ACCOUNT_ID",
173 | "SF_LOGIN_NAME",
174 | "SF_PWD",
175 | ],
176 | )
177 | def test_get_config_repo_procedure_when_key_exists(
178 | key,
179 | fixture_dotenv_path,
180 | monkeypatch,
181 | ):
182 | dotenv_config = dotenv.dotenv_values(fixture_dotenv_path)
183 | monkeypatch.setattr(sf_git.commands, "DOTENV_PATH", fixture_dotenv_path)
184 |
185 | value = sf_git.commands.get_config_repo_procedure(
186 | key, logger=lambda x: None
187 | )
188 |
189 | assert value == dotenv_config[key]
190 |
191 |
192 | def test_get_config_repo_procedure_when_key_doesnt_exists(
193 | fixture_dotenv_path,
194 | monkeypatch,
195 | ):
196 | monkeypatch.setattr(sf_git.commands, "DOTENV_PATH", fixture_dotenv_path)
197 |
198 | with pytest.raises(UsageError):
199 | sf_git.commands.get_config_repo_procedure(
200 | "NOT_EXISTING", logger=lambda x: None
201 | )
202 |
203 |
204 | @pytest.mark.parametrize(
205 | "params, expected",
206 | [
207 | (
208 | {"username": "new_login", "password": "new_password"},
209 | {"SF_LOGIN_NAME": "new_login", "SF_PWD": "************"},
210 | ),
211 | ],
212 | )
213 | def test_set_config_repo_procedure_when_key_exists(
214 | params,
215 | expected,
216 | fixture_dotenv_path,
217 | repo,
218 | monkeypatch,
219 | ):
220 | dotenv_config = dotenv.dotenv_values(fixture_dotenv_path)
221 | monkeypatch.setattr(sf_git.commands, "DOTENV_PATH", fixture_dotenv_path)
222 |
223 | updates = sf_git.commands.set_config_repo_procedure(**params)
224 |
225 | assert updates == expected
226 |
227 |
228 | def test_set_config_procedure_when_valid_repo(
229 | alternative_repo_path,
230 | fixture_dotenv_path,
231 | repo,
232 | alternative_repo,
233 | monkeypatch,
234 | ):
235 | params = {"git_repo": str(alternative_repo_path)}
236 | expected = {"SNOWFLAKE_VERSIONING_REPO": str(alternative_repo_path)}
237 |
238 | monkeypatch.setattr(sf_git.commands, "DOTENV_PATH", fixture_dotenv_path)
239 |
240 | updates = sf_git.commands.set_config_repo_procedure(**params)
241 |
242 | assert updates == expected
243 |
244 |
245 | def test_set_config_procedure_when_invalid_repo(
246 | fixture_dotenv_path,
247 | repo,
248 | testing_folder,
249 | monkeypatch,
250 | ):
251 | params = {"git_repo": str(testing_folder / "not_initialized")}
252 |
253 | monkeypatch.setattr(sf_git.commands, "DOTENV_PATH", fixture_dotenv_path)
254 |
255 | with pytest.raises(UsageError):
256 | sf_git.commands.set_config_repo_procedure(**params)
257 |
258 |
259 | def test_set_config_repo_ws_path_when_invalid(
260 | alternative_repo_path,
261 | alternative_repo,
262 | fixture_dotenv_path,
263 | repo,
264 | testing_folder,
265 | monkeypatch,
266 | ):
267 | params = {
268 | "git_repo": str(alternative_repo_path),
269 | "save_dir": str(testing_folder / "not_a_subdirectory"),
270 | }
271 |
272 | monkeypatch.setattr(sf_git.commands, "DOTENV_PATH", fixture_dotenv_path)
273 |
274 | with pytest.raises(UsageError):
275 | sf_git.commands.set_config_repo_procedure(**params)
276 |
277 |
278 | def test_set_config_repo_ws_path_when_valid(
279 | alternative_repo_path,
280 | alternative_repo,
281 | fixture_dotenv_path,
282 | repo,
283 | testing_folder,
284 | monkeypatch,
285 | ):
286 | params = {
287 | "git_repo": str(alternative_repo_path),
288 | "save_dir": str(alternative_repo_path / "a_subdirectory"),
289 | }
290 | expected = {
291 | "SNOWFLAKE_VERSIONING_REPO": str(alternative_repo_path),
292 | "WORKSHEETS_PATH": str(alternative_repo_path / "a_subdirectory"),
293 | }
294 |
295 | monkeypatch.setattr(sf_git.commands, "DOTENV_PATH", fixture_dotenv_path)
296 |
297 | updates = sf_git.commands.set_config_repo_procedure(**params)
298 |
299 | assert isinstance(updates, dict)
300 | assert updates == expected
301 |
302 |
303 | def test_set_config_ws_path_when_valid(
304 | fixture_dotenv_path,
305 | repo,
306 | repo_root_path,
307 | testing_folder,
308 | monkeypatch,
309 | ):
310 | params = {"save_dir": str(repo_root_path / "a_subdirectory")}
311 | expected = {"WORKSHEETS_PATH": str(repo_root_path / "a_subdirectory")}
312 |
313 | monkeypatch.setattr(sf_git.commands, "DOTENV_PATH", fixture_dotenv_path)
314 |
315 | updates = sf_git.commands.set_config_repo_procedure(**params)
316 |
317 | assert isinstance(updates, dict)
318 | assert updates == expected
319 |
--------------------------------------------------------------------------------
/tests/test_config.py:
--------------------------------------------------------------------------------
1 | from git import Repo
2 | from sf_git import DOTENV_PATH
3 | from sf_git.config import GLOBAL_CONFIG
4 |
5 |
6 | def test_repo(repo: Repo):
7 |
8 | assert not repo.bare
9 | assert len(repo.head.reference.log()) == 1
10 |
11 |
12 | def test_dotenv_path():
13 | from dotenv import dotenv_values
14 | values = dotenv_values(DOTENV_PATH)
15 | assert values['SF_LOGIN_NAME'] == GLOBAL_CONFIG.sf_login_name
16 |
--------------------------------------------------------------------------------
/tests/test_worksheets_utils.py:
--------------------------------------------------------------------------------
1 | import pytest
2 | import json
3 | import requests_mock
4 | import re
5 |
6 | import sf_git.models
7 | import sf_git.worksheets_utils as worksheets_utils
8 |
9 |
10 | @pytest.fixture
11 | def mock_api():
12 | with requests_mock.Mocker() as m:
13 | yield m
14 |
15 |
16 | @pytest.fixture
17 | def get_worksheets_api_response_with_worksheets(monkeypatch, testing_folder):
18 | with open(testing_folder / "fixtures" / "contents.json") as f:
19 | contents = json.loads(f.read())
20 |
21 | with open(testing_folder / "fixtures" / "entities.json") as f:
22 | entities = json.loads(f.read())
23 |
24 | mock_response_body = {
25 | "entities": entities,
26 | "models": {"queries": contents},
27 | }
28 |
29 | return json.dumps(mock_response_body)
30 |
31 |
32 | @pytest.fixture()
33 | def get_worksheets_api_response_without_worksheets(
34 | monkeypatch, testing_folder
35 | ):
36 | mock_response_body = {"entities": [], "models": {"queries": {}}}
37 |
38 | return json.dumps(mock_response_body)
39 |
40 |
41 | @pytest.fixture()
42 | def mock_get_folders(monkeypatch, testing_folder):
43 | def mocked_folders(auth_context):
44 | with open(testing_folder / "fixtures" / "folders.json", "r") as f:
45 | folders_as_dict = json.load(f)
46 | return [sf_git.models.Folder(**f) for f in folders_as_dict]
47 |
48 | monkeypatch.setattr(worksheets_utils, "get_folders", mocked_folders)
49 |
50 |
51 | @pytest.fixture()
52 | def mock_get_worksheets(monkeypatch, testing_folder):
53 | def mocked_worksheets(auth_context):
54 | with open(testing_folder / "fixtures" / "worksheets.json", "r") as f:
55 | worksheets_as_dict = json.load(f)
56 | return [sf_git.models.Worksheet(**w) for w in worksheets_as_dict]
57 |
58 | monkeypatch.setattr(worksheets_utils, "get_worksheets", mocked_worksheets)
59 |
60 |
61 | @pytest.fixture()
62 | def mock_all_write_to_snowsight(monkeypatch):
63 | monkeypatch.setattr(
64 | worksheets_utils,
65 | "create_folder",
66 | lambda auth_context, folder_name: None,
67 | )
68 | monkeypatch.setattr(
69 | worksheets_utils,
70 | "create_worksheet",
71 | lambda auth_context, worksheet_name, folder_id: None,
72 | )
73 | monkeypatch.setattr(
74 | worksheets_utils,
75 | "write_worksheet",
76 | lambda auth_context, worksheet: None,
77 | )
78 |
79 |
80 | def test_get_worksheets_when_two(
81 | mock_api, get_worksheets_api_response_with_worksheets, auth_context
82 | ):
83 | mock_api.post(
84 | re.compile(auth_context.app_server_url),
85 | text=get_worksheets_api_response_with_worksheets,
86 | status_code=200,
87 | )
88 |
89 | worksheets = worksheets_utils.get_worksheets(
90 | auth_context=auth_context,
91 | store_to_cache=False,
92 | )
93 |
94 | assert isinstance(worksheets, list)
95 | assert len(worksheets) == 2
96 |
97 |
98 | def test_get_folders_when_two(
99 | mock_api, get_worksheets_api_response_with_worksheets, auth_context
100 | ):
101 | mock_api.post(
102 | re.compile(auth_context.app_server_url),
103 | text=get_worksheets_api_response_with_worksheets,
104 | status_code=200,
105 | )
106 |
107 | folders = worksheets_utils.get_folders(
108 | auth_context=auth_context,
109 | )
110 |
111 | assert isinstance(folders, list)
112 | assert len(folders) == 2
113 |
114 |
115 | def test_get_worksheets_when_none(
116 | mock_api, get_worksheets_api_response_without_worksheets, auth_context
117 | ):
118 | mock_api.post(
119 | re.compile(auth_context.app_server_url),
120 | text=get_worksheets_api_response_without_worksheets,
121 | status_code=200,
122 | )
123 |
124 | worksheets = worksheets_utils.get_worksheets(
125 | auth_context=auth_context,
126 | store_to_cache=False,
127 | )
128 |
129 | assert isinstance(worksheets, list)
130 | assert len(worksheets) == 0
131 |
132 |
133 | def test_get_folders_when_none(
134 | mock_api, get_worksheets_api_response_without_worksheets, auth_context
135 | ):
136 | mock_api.post(
137 | re.compile(auth_context.app_server_url),
138 | text=get_worksheets_api_response_without_worksheets,
139 | status_code=200,
140 | )
141 |
142 | folders = worksheets_utils.get_folders(
143 | auth_context=auth_context,
144 | )
145 |
146 | assert isinstance(folders, list)
147 | assert len(folders) == 0
148 |
149 |
150 | def test_upload_snowsight_when_one_to_update(
151 | monkeypatch,
152 | mock_get_folders,
153 | mock_get_worksheets,
154 | mock_all_write_to_snowsight,
155 | testing_folder,
156 | auth_context,
157 | ):
158 | with open(
159 | testing_folder / "fixtures" / "worksheets_update.json", "r"
160 | ) as f:
161 | worksheets_as_dicts = json.load(f)
162 |
163 | worksheets = [sf_git.models.Worksheet(**w) for w in worksheets_as_dicts]
164 | upload_report = worksheets_utils.upload_to_snowsight(
165 | auth_context, worksheets
166 | )
167 |
168 | assert isinstance(upload_report, dict)
169 | assert set(upload_report.keys()) == {"completed", "errors"}
170 | assert len(upload_report["completed"]) == len(worksheets)
171 | assert len(upload_report["errors"]) == 0
172 |
173 |
174 | def test_upload_snowsight_when_none_to_update(
175 | monkeypatch,
176 | mock_get_folders,
177 | mock_get_worksheets,
178 | mock_all_write_to_snowsight,
179 | testing_folder,
180 | auth_context,
181 | ):
182 | with open(
183 | testing_folder / "fixtures" / "worksheets.json", "r"
184 | ) as f:
185 | worksheets_as_dicts = json.load(f)
186 |
187 | worksheets = [sf_git.models.Worksheet(**w) for w in worksheets_as_dicts]
188 | upload_report = worksheets_utils.upload_to_snowsight(
189 | auth_context, worksheets
190 | )
191 |
192 | assert isinstance(upload_report, dict)
193 | assert set(upload_report.keys()) == {"completed", "errors"}
194 | assert len(upload_report["completed"]) == 0
195 | assert len(upload_report["errors"]) == 0
196 |
--------------------------------------------------------------------------------