├── .docstr.yaml ├── .flake8 ├── .github └── workflows │ ├── docstr-coverage.yml │ ├── flake8.yml │ ├── python-publish.yml │ └── readthedocs.yml ├── .gitignore ├── LICENSE.txt ├── MANIFEST.in ├── docs ├── CONTRIBUTING.md ├── PyColumn.md ├── PyColumns.md ├── PyMeasure.md ├── PyMeasures.md ├── PyObject.md ├── PyObjects.md ├── PyPartition.md ├── PyPartitions.md ├── PyTable.md ├── PyTables.md ├── README.md ├── Tabular.md ├── best_practice_analyzer.md ├── document.md ├── logic_utils.md ├── pbi_helper.md ├── query.md ├── refresh.md ├── tabular_editor.md ├── tabular_tracing.md └── tmdl.md ├── mkdocs.yml ├── pyproject.toml ├── pytabular ├── __init__.py ├── best_practice_analyzer.py ├── column.py ├── culture.py ├── currency.py ├── dll │ ├── Microsoft.AnalysisServices.AdomdClient.dll │ ├── Microsoft.AnalysisServices.Core.dll │ ├── Microsoft.AnalysisServices.Tabular.Json.dll │ ├── Microsoft.AnalysisServices.Tabular.dll │ └── Microsoft.AnalysisServices.dll ├── document.py ├── logic_utils.py ├── measure.py ├── object.py ├── partition.py ├── pbi_helper.py ├── pytabular.py ├── query.py ├── refresh.py ├── relationship.py ├── table.py ├── tabular_editor.py ├── tabular_tracing.py └── tmdl.py ├── test ├── __init__.py ├── adventureworks │ ├── AdventureWorks Sales.pbix │ └── AdventureWorks Sales.xlsx ├── config.py ├── conftest.py ├── dfvaltest.dax ├── singlevaltest.dax ├── test_10logic_utils.py ├── test_11document.py ├── test_12tmdl.py ├── test_1sanity.py ├── test_2object.py ├── test_3tabular.py ├── test_4measure.py ├── test_5column.py ├── test_6table.py ├── test_7tabular_tracing.py ├── test_8bpa.py └── test_9custom.py └── tox.ini /.docstr.yaml: -------------------------------------------------------------------------------- 1 | paths: ["pytabular"] 2 | verbose: 3 3 | skip_init: True 4 | fail_under: 100 -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | extend-ignore=E203, D107 3 | max-line-length=100 4 | docstring-convention=google -------------------------------------------------------------------------------- /.github/workflows/docstr-coverage.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions 2 | 3 | name: 100% docstring coverage 4 | 5 | # Controls when the workflow will run 6 | on: 7 | pull_request: 8 | branches: [ master ] 9 | push: 10 | branches: [ master ] 11 | 12 | # Allows you to run this workflow manually from the Actions tab 13 | workflow_dispatch: 14 | 15 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 16 | jobs: 17 | # This workflow contains a single job called "build" 18 | build: 19 | # The type of runner that the job will run on 20 | runs-on: ubuntu-latest 21 | 22 | # Steps represent a sequence of tasks that will be executed as part of the job 23 | steps: 24 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 25 | - uses: actions/checkout@v2 26 | with: 27 | fetch-depth: 0 28 | - uses: actions/setup-python@v2 29 | with: 30 | python-version: '3.10' 31 | - run: pip install --upgrade pip 32 | - run: pip install docstr-coverage==2.2.0 33 | - run: docstr-coverage --skip-init -------------------------------------------------------------------------------- /.github/workflows/flake8.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions 2 | 3 | name: flake8 with pep8-naming and flake8-docstrings 4 | 5 | # Controls when the workflow will run 6 | on: 7 | pull_request: 8 | branches: [ master ] 9 | push: 10 | branches: [ master ] 11 | 12 | # Allows you to run this workflow manually from the Actions tab 13 | workflow_dispatch: 14 | 15 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 16 | jobs: 17 | # This workflow contains a single job called "build" 18 | build: 19 | # The type of runner that the job will run on 20 | runs-on: ubuntu-latest 21 | 22 | # Steps represent a sequence of tasks that will be executed as part of the job 23 | steps: 24 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 25 | - uses: actions/checkout@v2 26 | with: 27 | fetch-depth: 0 28 | - uses: actions/setup-python@v2 29 | - run: pip install --upgrade pip 30 | - run: pip install flake8 31 | - run: pip install pep8-naming 32 | - run: pip install flake8-docstrings 33 | - run: python3 -m flake8 --count -------------------------------------------------------------------------------- /.github/workflows/python-publish.yml: -------------------------------------------------------------------------------- 1 | # This workflow will upload a Python Package using Twine when a release is created 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries 3 | 4 | # This workflow uses actions that are not certified by GitHub. 5 | # They are provided by a third-party and are governed by 6 | # separate terms of service, privacy policy, and support 7 | # documentation. 8 | 9 | name: Upload python package to pypi 10 | 11 | on: 12 | release: 13 | types: [published] 14 | workflow_dispatch: 15 | 16 | permissions: 17 | contents: read 18 | 19 | jobs: 20 | deploy: 21 | 22 | runs-on: ubuntu-latest 23 | environment: pypi 24 | steps: 25 | - uses: actions/checkout@v3 26 | - name: Set up Python 27 | uses: actions/setup-python@v3 28 | with: 29 | python-version: '3.x' 30 | - name: Install dependencies 31 | run: | 32 | python -m pip install --upgrade pip 33 | pip install build 34 | - name: Build package 35 | run: python -m build 36 | - name: Publish package 37 | uses: pypa/gh-action-pypi-publish@release/v1 38 | with: 39 | user: __token__ 40 | password: ${{ secrets.PYPI_API_TOKEN }} 41 | -------------------------------------------------------------------------------- /.github/workflows/readthedocs.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions 2 | 3 | name: Generate and deploy docs 4 | 5 | # Controls when the workflow will run 6 | on: 7 | # Triggers the workflow on push or pull request events but only for the master branch 8 | push: 9 | branches: [ master ] 10 | 11 | # Allows you to run this workflow manually from the Actions tab 12 | workflow_dispatch: 13 | 14 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 15 | jobs: 16 | # This workflow contains a single job called "build" 17 | build: 18 | # The type of runner that the job will run on 19 | runs-on: ubuntu-latest 20 | 21 | # Steps represent a sequence of tasks that will be executed as part of the job 22 | steps: 23 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 24 | - uses: actions/checkout@v2 25 | with: 26 | fetch-depth: 0 27 | - uses: actions/setup-python@v2 28 | - run: pip install --upgrade pip 29 | - run: pip install mkdocstrings[python] 30 | - run: pip install mkdocs-material 31 | - run: pip install mkdocs mkdocs-gen-files 32 | - run: git config user.name 'github-actions[bot]' && git config user.email 'github-actions[bot]@users.noreply.github.com' 33 | - name: Publish docs 34 | run: mkdocs gh-deploy 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # C extensions 2 | *.so 3 | 4 | # Distribution / packaging 5 | .Python 6 | build/ 7 | develop-eggs/ 8 | dist/ 9 | downloads/ 10 | eggs/ 11 | .eggs/ 12 | lib/ 13 | lib64/ 14 | parts/ 15 | sdist/ 16 | var/ 17 | wheels/ 18 | share/python-wheels/ 19 | *.egg-info/ 20 | .installed.cfg 21 | *.egg 22 | MANIFEST 23 | 24 | # PyInstaller 25 | # Usually these files are written by a python script from a template 26 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 27 | *.manifest 28 | *.spec 29 | 30 | # Installer logs 31 | pip-log.txt 32 | pip-delete-this-directory.txt 33 | 34 | # Unit test / coverage reports 35 | htmlcov/ 36 | .tox/ 37 | .nox/ 38 | .coverage 39 | .coverage.* 40 | .cache 41 | nosetests.xml 42 | coverage.xml 43 | *.cover 44 | *.py,cover 45 | .hypothesis/ 46 | .pytest_cache/ 47 | cover/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | db.sqlite3 57 | db.sqlite3-journal 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | .pybuilder/ 71 | target/ 72 | 73 | # Jupyter Notebook 74 | .ipynb_checkpoints 75 | 76 | # IPython 77 | profile_default/ 78 | ipython_config.py 79 | 80 | # pyenv 81 | # For a library or package, you might want to ignore these files since the code is 82 | # intended to run in multiple environments; otherwise, check them in: 83 | # .python-version 84 | 85 | # pipenv 86 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 87 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 88 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 89 | # install all needed dependencies. 90 | #Pipfile.lock 91 | 92 | # poetry 93 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 94 | # This is especially recommended for binary packages to ensure reproducibility, and is more 95 | # commonly ignored for libraries. 96 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 97 | #poetry.lock 98 | 99 | # pdm 100 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 101 | #pdm.lock 102 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 103 | # in version control. 104 | # https://pdm.fming.dev/#use-with-ide 105 | .pdm.toml 106 | 107 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 108 | __pypackages__/ 109 | 110 | # Celery stuff 111 | celerybeat-schedule 112 | celerybeat.pid 113 | 114 | # SageMath parsed files 115 | *.sage.py 116 | 117 | # Environments 118 | .env 119 | .venv 120 | env/ 121 | venv/ 122 | ENV/ 123 | env.bak/ 124 | venv.bak/ 125 | 126 | # Spyder project settings 127 | .spyderproject 128 | .spyproject 129 | 130 | # Rope project settings 131 | .ropeproject 132 | 133 | # mkdocs documentation 134 | /site 135 | 136 | # Output for model documentation 137 | /model-docs 138 | 139 | # mypy 140 | .mypy_cache/ 141 | .dmypy.json 142 | dmypy.json 143 | 144 | # Pyre type checker 145 | .pyre/ 146 | 147 | # pytype static type analyzer 148 | .pytype/ 149 | 150 | # Cython debug symbols 151 | cython_debug/ 152 | 153 | # PyCharm 154 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 155 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 156 | # and can be added to the global gitignore or merged into this file. For a more nuclear 157 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 158 | #.idea/ 159 | 160 | # Vim swap files 161 | *.swp 162 | *.swo 163 | *.swn 164 | 165 | # Project-specific 166 | __pycache__ 167 | /.vscode 168 | /daxqueries 169 | /pytabular/localsecret.py 170 | /TE2 171 | setup.py 172 | notes.txt 173 | adhoc.py 174 | *.bim 175 | *.csv 176 | docs/_config.yml# Byte-compiled / optimized / DLL files 177 | *.py[cod] 178 | *$py.class 179 | /Best_Practice_Analyzer 180 | /Tabular_Editor_2 181 | *.tmdl 182 | 183 | # Test files for new functionality 184 | test-notebook.ipynb 185 | test.py 186 | testing.py -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Curtis Stallings 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include pytabular/dll/Microsoft.AnalysisServices.AdomdClient.dll 2 | include pytabular/dll/Microsoft.AnalysisServices.Core.dll 3 | include pytabular/dll/Microsoft.AnalysisServices.dll 4 | include pytabular/dll/Microsoft.AnalysisServices.Tabular.dll 5 | include pytabular/dll/Microsoft.AnalysisServices.Tabular.Json.dll -------------------------------------------------------------------------------- /docs/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | ## Goal 4 | - Make **Python** a first class citizen for interacting with **Tabular models**. 5 | 6 | ## Info 7 | 8 | - Use `tox` for local testing. 9 | - Will run docstring coverage, linter, and python versions needed to support w/ pytest. 10 | 11 | ## Misc 12 | - Work will be distributed under a MIT license. -------------------------------------------------------------------------------- /docs/PyColumn.md: -------------------------------------------------------------------------------- 1 | :::pytabular.column.PyColumn -------------------------------------------------------------------------------- /docs/PyColumns.md: -------------------------------------------------------------------------------- 1 | :::pytabular.column.PyColumns 2 | :::pytabular.object.PyObjects 3 | options: 4 | show_root_toc_entry: false 5 | members: 6 | - find -------------------------------------------------------------------------------- /docs/PyMeasure.md: -------------------------------------------------------------------------------- 1 | :::pytabular.measure.PyMeasure -------------------------------------------------------------------------------- /docs/PyMeasures.md: -------------------------------------------------------------------------------- 1 | :::pytabular.measure.PyMeasures 2 | :::pytabular.object.PyObjects 3 | options: 4 | show_root_toc_entry: false 5 | members: 6 | - find -------------------------------------------------------------------------------- /docs/PyObject.md: -------------------------------------------------------------------------------- 1 | :::pytabular.object.PyObject -------------------------------------------------------------------------------- /docs/PyObjects.md: -------------------------------------------------------------------------------- 1 | :::pytabular.object.PyObjects -------------------------------------------------------------------------------- /docs/PyPartition.md: -------------------------------------------------------------------------------- 1 | :::pytabular.partition.PyPartition -------------------------------------------------------------------------------- /docs/PyPartitions.md: -------------------------------------------------------------------------------- 1 | :::pytabular.partition.PyPartitions 2 | :::pytabular.object.PyObjects 3 | options: 4 | show_root_toc_entry: false 5 | members: 6 | - find -------------------------------------------------------------------------------- /docs/PyTable.md: -------------------------------------------------------------------------------- 1 | :::pytabular.table.PyTable -------------------------------------------------------------------------------- /docs/PyTables.md: -------------------------------------------------------------------------------- 1 | :::pytabular.table.PyTables 2 | :::pytabular.object.PyObjects 3 | options: 4 | show_root_toc_entry: false 5 | members: 6 | - find -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # PyTabular 2 | 3 | [![PyPI version](https://badge.fury.io/py/python-tabular.svg)](https://badge.fury.io/py/python-tabular) 4 | [![Downloads](https://pepy.tech/badge/python-tabular)](https://pepy.tech/project/python-tabular) 5 | [![readthedocs](https://github.com/Curts0/PyTabular/actions/workflows/readthedocs.yml/badge.svg)](https://github.com/Curts0/PyTabular/actions/workflows/readthedocs.yml) 6 | [![pages-build-deployment](https://github.com/Curts0/PyTabular/actions/workflows/pages/pages-build-deployment/badge.svg)](https://github.com/Curts0/PyTabular/actions/workflows/pages/pages-build-deployment) 7 | [![flake8](https://github.com/Curts0/PyTabular/actions/workflows/flake8.yml/badge.svg?branch=master)](https://github.com/Curts0/PyTabular/actions/workflows/flake8.yml) 8 | [![docstr-coverage](https://github.com/Curts0/PyTabular/actions/workflows/docstr-coverage.yml/badge.svg)](https://github.com/Curts0/PyTabular/actions/workflows/docstr-coverage.yml) 9 | [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit) 10 | 11 | ### What is it? 12 | 13 | [PyTabular](https://github.com/Curts0/PyTabular) (**python-tabular** in [pypi](https://pypi.org/project/python-tabular/)) is a python package that allows for programmatic execution on your tabular models! This is possible thanks to [Pythonnet](https://pythonnet.github.io/) and Microsoft's [.Net APIs on Azure Analysis Services](https://docs.microsoft.com/en-us/dotnet/api/microsoft.analysisservices?view=analysisservices-dotnet). Currently, this build is tested and working on **Windows Operating System only**. Help is needed to expand this for another os. See the [Documentation Here](https://curts0.github.io/PyTabular/). PyTabular is still considered alpha. Please send bugs my way! Preferably in the issues section in Github. 14 | 15 | ### Getting Started 16 | 17 | See the [Pypi project](https://pypi.org/project/python-tabular/) for available versions. 18 | 19 | !!! DANGER "Before 0.3.5" 20 | 21 | **To become PEP8 compliant with naming conventions, serious name changes were made in 0.3.5.** Install v. 0.3.4 or lower to get the older naming conventions. 22 | 23 | ```powershell title="Install Example" 24 | python3 -m pip install python-tabular 25 | 26 | #install specific version 27 | python3 -m pip install python-tabular==0.3.4 28 | ``` 29 | 30 | In your python environment, import pytabular and call the main Tabular Class. The only parameter needed is a solid connection string. 31 | 32 | ```python title="Connecting to Model" 33 | import pytabular 34 | model = pytabular.Tabular(CONNECTION_STR) # (1) 35 | ``` 36 | 37 | 1. That's it. A solid connection string. 38 | 39 | You may have noticed some logging into your console. I'm a big fan of logging, if you don't want any just get the logger and disable it. 40 | 41 | ```python title="Logging Example" 42 | import pytabular 43 | pytabular.logger.disabled = True 44 | ``` 45 | 46 | You can query your models with the `query()` method from your tabular class. For Dax Queries, it will need the full Dax syntax. See [EVALUATE example](https://dax.guide/st/evaluate/). This will return a [Pandas DataFrame](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html). If you are looking to return a single value, see below. Simply wrap your query in the curly brackets. The method will take that single-cell table and just return the individual value. You can also query your DMV. See below for example. 47 | 48 | ```python title="Query Examples" 49 | #Run basic queries 50 | DAX_QUERY = "EVALUATE TOPN(100, 'Table1')" 51 | model.query(DAX_QUERY) # (1) 52 | 53 | #or... 54 | DMV_QUERY = "select * from $SYSTEM.DISCOVER_TRACE_EVENT_CATEGORIES" 55 | model.query(DMV_QUERY) # (2) 56 | 57 | #or... 58 | SINGLE_VALUE_QUERY_EX = "EVALUATE {1}" 59 | model.query(SINGLE_VALUE_QUERY_EX) # (3) 60 | 61 | #or... 62 | FILE_PATH = 'C:\\FILEPATHEXAMPLE\\file.dax' 63 | model.query(FILE_PATH) # (4) 64 | ``` 65 | 66 | 1. Returns a `pd.DataFrame()`. 67 | 2. Returns a `pd.DataFrame()`. 68 | 3. This will return a single value. Example, `1` or `'string'`. 69 | 4. This will return the same logic as above, single values if possible else will return `pd.DataFrame()`. Supply any file type. 70 | 71 | You can also explore your tables, partitions, columns, etc. via the attributes of your `Tabular` class. 72 | 73 | ```python title="Usage Examples" 74 | model.Tables['Table Name'].refresh() # (1) 75 | 76 | #or 77 | model.Tables['Table Name'].Partitions['Partition Name'].refresh() # (2) 78 | 79 | #or 80 | model.Tables['Table Name'].Partitions[4].last_refresh() # (3) 81 | 82 | #or 83 | model.Tables['Table Name'].row_count() # (4) 84 | 85 | #or 86 | model.Tables['Table Name'].Columns['Column Name'].distinct_count() # (5) 87 | ``` 88 | 89 | 1. Refresh a specific table. The `.Tables` is your attribute to gain access to your `PyTables` class. From that, you can iterate into specific `PyTable` classes. 90 | 2. Refresh a specific partition. 91 | 3. Get the last refresh time of a specific partition. Notice this time that instead of the partition name, an `int` was used to index into the specific `PyPartition`. 92 | 4. Get the row count of a table. 93 | 5. Get a distinct count of a column. 94 | 95 | Use the `refresh()` method to handle refreshes on your model. This is synchronous. Should be flexible enough to handle a variety of inputs. See [PyTabular Docs for Refreshing Tables and Partitions](https://curts0.github.io/PyTabular/refresh). The most basic way to refresh is to input the table name string. The method will search for the table and output exception if unable to find it. For partitions, you will need a key, and value combination. Example, `{'Table1':'Partition1'}`. You can also take the key-value pair and iterate through a group of partitions. Example, `{'Table1':['Partition1','Partition2']}`. Rather than providing a string, you can also input the actual class. See below for those examples. You can access them from the built-in attributes `self.Tables`, `self.Partitions`. 96 | 97 | ```python title="Refresh Examples" 98 | model.refresh('Table Name') # (1) 99 | 100 | model.refresh(['Table1','Table2','Table3']) # (2) 101 | 102 | #or... 103 | model.refresh() # (3) 104 | 105 | #or... 106 | model.refresh() # (4) 107 | 108 | #or... 109 | model.refresh({'Table Name':'Partition Name'}) # (5) 110 | 111 | #or... 112 | model.refresh( 113 | [ 114 | { 115 | :, 116 | 'Table Name':['Partition1','Partition2'] 117 | }, 118 | 'Table Name', 119 | 'Table Name2' 120 | ] 121 | ) # (6) 122 | 123 | #or... 124 | model.Tables['Table Name'].refresh() # (7) 125 | 126 | #or... 127 | model.Tables['Table Name'].Partitions['Partition Name'].refresh() # (8) 128 | 129 | #or... 130 | model.refresh(['Table1','Table2'], trace = None) # (9) 131 | ``` 132 | 133 | 1. Basic refresh of a specific table by table name string. 134 | 2. Basic refresh of a group of tables by table name strings. An example is with a list, but as long as it's iterable you should be fine. 135 | 3. Refresh a table by passing the `PyTable` class. 136 | 4. Refresh a partition by passing the `PyPartition` class. 137 | 5. Refresh a specific partition by passing a dictionary with the table name as the key and the partition name as the value. 138 | 6. Get crazy. Pass all kinds of weird combinations. 139 | 7. Basic refresh from a `PyTable` class. 140 | 8. Basic refresh from a `PyPartition` class. 141 | 9. By default, a `RefreshTrace` is started during refresh. It can be disabled by setting `trace = None`. 142 | 143 | ### Use Cases 144 | 145 | #### If a blank table, then refresh the table. 146 | 147 | This will use the function [find_zero_rows](https://curts0.github.io/PyTabular/PyTables/#pytabular.table.PyTables.find_zero_rows) and the method [refresh](https://curts0.github.io/PyTabular/PyTables/#pytabular.table.PyTables.refresh) from the Tabular class. 148 | 149 | ```python 150 | import pytabular 151 | model = pytabular.Tabular(CONNECTION_STR) 152 | tables = model.Tables.find_zero_rows() 153 | if len(tables) > 0: 154 | tables.refresh() 155 | ``` 156 | 157 | Maybe you only want to check a subset of tables? Like `find()` tables with 'fact' in the name, then check if any facts are blank. 158 | 159 | ```python 160 | import pytabular 161 | model = pytabular.Tabular(CONNECTION_STR) 162 | tables = model.Tables.find('fact').find_zero_rows() 163 | if len(tables) > 0: 164 | tables.refresh() 165 | ``` 166 | 167 | #### Sneak in a refresh. 168 | 169 | This will use the method [is_process](https://curts0.github.io/PyTabular/Tabular/#pytabular.pytabular.Tabular.is_process) and the method [refresh](https://curts0.github.io/PyTabular/Tabular/#pytabular.pytabular.Tabular.refresh) from the Tabular class. It will check the DMV to see if any jobs are currently running and classified as processing. 170 | 171 | ```python 172 | import pytabular 173 | model = pytabular.Tabular(CONNECTION_STR) 174 | if model.is_process(): 175 | #do what you want if there is a refresh happening 176 | else: 177 | model.refresh(TABLES_OR_PARTITIONS_TO_REFRESH) 178 | ``` 179 | 180 | #### Show refresh times in the model. 181 | 182 | This will use the function [last_refresh](https://curts0.github.io/PyTabular/PyTables/#pytabular.table.PyTables.last_refresh) and the method [create_table](https://curts0.github.io/PyTabular/Tabular/#pytabular.pytabular.Tabular.create_table) from the Tabular class. It will search through the model for all tables and partitions and pull the 'RefreshedTime' property from it. It will return results into a pandas data frame, which will then be converted into an M expression used for a new table. 183 | 184 | ```python 185 | import pytabular 186 | model = pytabular.Tabular(CONNECTION_STR) 187 | df = model.Tables.last_refresh() 188 | model.create_table(df, 'Refresh Times') 189 | ``` 190 | 191 | #### If BPA Violation, then reverts deployment. 192 | 193 | This uses a few things. First the [BPA Class](https://curts0.github.io/PyTabular/best_practice_analyzer/), then the [TE2 Class](https://curts0.github.io/PyTabular/tabular_editor/), and will finish with the [analyze_bpa](https://curts0.github.io/PyTabular/Tabular/#pytabular.pytabular.Tabular.analyze_bpa) method. Did not want to reinvent the wheel with the amazing work done with Tabular Editor and its BPA capabilities. 194 | 195 | ```python 196 | import pytabular 197 | model = pytabular.Tabular(CONNECTION_STR) 198 | # Feel free to input your TE2 File path or this will download for you. 199 | te2 = pytabular.TabularEditor() 200 | # Feel free to input your own BPA file or this will download for you from: 201 | # https://raw.githubusercontent.com/microsoft/Analysis-Services/master/BestPracticeRules/BPARules.json 202 | bpa = pytabular.BPA() 203 | results = model.analyze_bpa(te2.exe,bpa.location) 204 | 205 | if len(results) > 0: 206 | #Revert deployment here! 207 | ``` 208 | 209 | #### Loop through and query DAX files 210 | 211 | Let's say you have multiple DAX queries you would like to store and run through as checks. The [query](https://curts0.github.io/PyTabular/query/#pytabular.query.Connection.query) method on the Tabular class can also take file paths. It can be any file type as it's just checking `os.path.isfile()`. But would suggest `.dax` or `.txt`. It will read the file and use that as the new `query_str` argument. 212 | 213 | ```python 214 | import pytabular 215 | model = pytabular.Tabular(CONNECTION_STR) 216 | LIST_OF_FILE_PATHS = [ 217 | 'C:\\FilePath\\file1.dax', 218 | 'C:\\FilePath\\file1.txt', 219 | 'C:\\FilePath\\file2.dax', 220 | 'C:\\FilePath\\file2.txt' 221 | ] 222 | for file_path in LIST_OF_FILE_PATHS: 223 | model.query(file_path) 224 | ``` 225 | 226 | #### Advanced Refreshing with Pre and Post Checks 227 | 228 | Maybe you are introducing new logic to a fact table, and you need to ensure that a measure checking last month's values never changes. To do that you can take advantage of the `RefreshCheck` and `RefreshCheckCollection` classes. But using those you can build out something that would first check the results of the measure, then refresh, then check the results of the measure after the refresh, and lastly perform your desired check. In this case, the `pre` value matches the `post` value. When refreshing, if your pre does not equal post, it would fail and give an assertion error in your logging. 229 | 230 | ```python 231 | from pytabular import Tabular 232 | from pytabular.refresh import RefreshCheck, RefreshCheckCollection 233 | 234 | model = Tabular(CONNECTION_STR) 235 | 236 | # This is our custom check that we want to run after refresh. 237 | # Does the pre refresh value match the post refresh value. 238 | def sum_of_sales_assertion(pre, post): 239 | return pre == post 240 | 241 | # This is where we put it all together into the `RefreshCheck` class. Give it a name, give it a query to run, and give it the assertion you want to make. 242 | sum_of_last_month_sales = RefreshCheck( 243 | 'Last Month Sales', 244 | lambda: model.query("EVALUATE {[Last Month Sales]}") 245 | ,sum_of_sales_assertion 246 | ) 247 | 248 | # Here we are adding it to a `RefreshCheckCollection` because you can have more than on `Refresh_Check` to run. 249 | all_refresh_check = RefreshCheckCollection([sum_of_last_month_sales]) 250 | 251 | model.Refresh( 252 | 'Fact Table Name', 253 | refresh_checks = RefreshCheckCollection([sum_of_last_month_sales]) 254 | 255 | ) 256 | ``` 257 | 258 | #### Query as Another User 259 | 260 | There are plenty of tools that allow you to query as an 'Effective User' inheriting their security when querying. This is an extremely valuable concept built natively into the .Net APIs. My only gripe is they were all UI based. This allows you to programmatically connect as an effective user and query in Python. You could easily loop through all your users to run tests on their security. 261 | 262 | ```python 263 | import pytabular as p 264 | 265 | #Connect to your model like usual... 266 | model = p.Tabular(CONNECTION_STR) 267 | 268 | #This will be the query I run... 269 | query_str = ''' 270 | EVALUATE 271 | SUMMARIZE( 272 | 'Product Dimension', 273 | 'Product Dimension'[Product Name], 274 | "Total Product Sales", [Total Sales] 275 | ) 276 | ''' 277 | #This will be the user I want to query as... 278 | user_email = 'user1@company.com' 279 | 280 | #Base line, to query as the user connecting to the model. 281 | model.query(query_str) 282 | 283 | #Option 1, Connect via connection class... 284 | user1 = p.Connection(model.Server, effective_user = user_email) 285 | user1.query(query_str) 286 | 287 | #Option 2, Just add Effective_User 288 | model.query(query_str, effective_user = user_email) 289 | 290 | #PyTabular will do it's best to handle multiple accounts... 291 | #So you won't have to reconnect on every query 292 | ``` 293 | 294 | #### Refresh Related Tables 295 | 296 | Ever need to refresh related tables of a Fact? Now should be a lot easier. 297 | 298 | ```python 299 | import pytabular as p 300 | 301 | #Connect to model 302 | model = p.Tabular(CONNECTION_STR) 303 | 304 | #Get related tables 305 | tables = model.Tables[TABLE_NAME].related() 306 | 307 | #Now just refresh like usual... 308 | tables.refresh() 309 | ``` 310 | 311 | ## Documenting a Model 312 | 313 | The Tabular model contains a lot of information (meta-data) that can be used to generate documentation if filled in. Currently, the markdown files are generated with the Docusaurs heading in place, but this will be changed in the future to support multiple documentation platforms. 314 | 315 | **Tip**: With Tabular Editor 2 (Free) or 3 (Paid) you can easily add Descriptions, Translations (Cultures) and other additional information that can later be used for generating the documentation. 316 | 317 | Args: 318 | 319 | - **model**: Tabular 320 | - **friendly_name**: Default > No Value 321 | 322 | To specify the location of the docs, just supply the location of where you want to store the files (=`save_location`). 323 | 324 | - **save_location**: Default > docs 325 | 326 | Each page in the generation process has its own specific name, with these arguments you can rename them to your liking. 327 | 328 | - **general_page_url**: Default > index.md 329 | - **measure_page_url**: Default > measures.md 330 | - **table_folder**: Default > tables 331 | - **column_page_url**: Default > 4-columns.md 332 | 333 | **Folder structure** 334 | 335 | ``` 336 | adventure-works > Model Name 337 | └─── index.md > General Information 338 | └─── measures.md > Page with all measures in the model. 339 | └─── tables 340 | │ └─── index.md > Overview page with all tables in the model and a summary per table. 341 | | └─── technical_table_name.md > Details of a specific table with all columns and attributes. 342 | | └─── ......md 343 | | └─── ......md 344 | ``` 345 | 346 | ### Documenting a Model 347 | 348 | The simplest way to document a tabular model is to connect to the model, initialize the documentation and execute `save_documentation()`. 349 | 350 | ```python 351 | import pytabular 352 | 353 | # Connect to a Tabular Model Model 354 | model = pytabular.Tabular(CONNECTION_STR) 355 | 356 | # Initiate the Docs 357 | docs = pytabular.ModelDocumenter(model) 358 | 359 | # Generate the pages. 360 | docs.generate_documentation_pages() 361 | 362 | # Save docs to the default location 363 | docs.save_documentation() 364 | ``` 365 | 366 | ### Documenting a Model with Cultures 367 | 368 | Some model creators choose to add cultures to a tabular model for different kinds of reasons. We can leverage those cultures to use the translation names instead of the original object names. To enable this, you can set translations to `True` and specify the culture you want to use (e.g. `'en-US'`). 369 | 370 | ```python 371 | import pytabular 372 | 373 | # Connect to a Tabular Model Model 374 | model = pytabular.Tabular(CONNECTION_STR) 375 | 376 | # Initiate the Docs 377 | docs = pytabular.ModelDocumenter(model) 378 | 379 | # Set the translation for documentation to an available culture. 380 | # By setting the Tranlsations to `True` it will check if it exists and if it does, 381 | # it will start using the translations for the docs 382 | docs.set_translations( 383 | enable_translations = True, 384 | culture = 'en-US' 385 | ) 386 | 387 | # Generate the pages. 388 | docs.generate_documentation_pages() 389 | 390 | # Save docs to the default location 391 | docs.save_documentation() 392 | ``` 393 | 394 | ### Documenting a Power BI > Local Model. 395 | 396 | The Local model doesn't have a "name", only an Id. So we need to Supply a "Friendly Name", which will be used to store the markdown files. 397 | 398 | ```python 399 | import pytabular 400 | 401 | # Connect to a Tabular Model Model 402 | model = pytabular.Tabular(CONNECTION_STR) 403 | 404 | # Initiate the Docs and set a friendly name to store the markdown files. 405 | docs = pytabular.ModelDocumenter( 406 | model = model, 407 | friendly_name = "Adventure Works" 408 | ) 409 | 410 | # Generate the pages. 411 | docs.generate_documentation_pages() 412 | 413 | # Save docs to the default location 414 | docs.save_documentation() 415 | ``` 416 | 417 | ### Contributing 418 | 419 | See [contributing.md](CONTRIBUTING.md) 420 | -------------------------------------------------------------------------------- /docs/Tabular.md: -------------------------------------------------------------------------------- 1 | :::pytabular.pytabular.Tabular -------------------------------------------------------------------------------- /docs/best_practice_analyzer.md: -------------------------------------------------------------------------------- 1 | :::pytabular.best_practice_analyzer -------------------------------------------------------------------------------- /docs/document.md: -------------------------------------------------------------------------------- 1 | :::pytabular.document -------------------------------------------------------------------------------- /docs/logic_utils.md: -------------------------------------------------------------------------------- 1 | :::pytabular.logic_utils -------------------------------------------------------------------------------- /docs/pbi_helper.md: -------------------------------------------------------------------------------- 1 | :::pytabular.pbi_helper -------------------------------------------------------------------------------- /docs/query.md: -------------------------------------------------------------------------------- 1 | :::pytabular.query -------------------------------------------------------------------------------- /docs/refresh.md: -------------------------------------------------------------------------------- 1 | :::pytabular.refresh -------------------------------------------------------------------------------- /docs/tabular_editor.md: -------------------------------------------------------------------------------- 1 | :::pytabular.tabular_editor -------------------------------------------------------------------------------- /docs/tabular_tracing.md: -------------------------------------------------------------------------------- 1 | :::pytabular.tabular_tracing -------------------------------------------------------------------------------- /docs/tmdl.md: -------------------------------------------------------------------------------- 1 | :::pytabular.tmdl.Tmdl -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: PyTabular 2 | site_description: "Connect to your Tabular models in Python!" 3 | site_url: https://curts0.github.io/PyTabular/ 4 | docs_dir: docs 5 | repo_name: Curts0/PyTabular 6 | repo_url: https://github.com/Curts0/PyTabular 7 | nav: 8 | - Home: README.md 9 | - Main Tabular Class: Tabular.md 10 | - Query Model: query.md 11 | - Refresh Model: refresh.md 12 | - PyObject Reference: 13 | - PyObjects: PyObjects.md 14 | - PyObject: PyObject.md 15 | - PyTables: PyTables.md 16 | - PyTable: PyTable.md 17 | - PyColumns: PyColumns.md 18 | - PyColumn: PyColumn.md 19 | - PyPartitions: PyPartitions.md 20 | - PyPartition: PyPartition.md 21 | - PyMeasures: PyMeasures.md 22 | - PyMeasure: PyMeasure.md 23 | - Misc. File Reference: 24 | - tabular_editor: tabular_editor.md 25 | - best_practice_analyzer: best_practice_analyzer.md 26 | - pbi_helper: pbi_helper.md 27 | - logic_utils: logic_utils.md 28 | - tmdl: tmdl.md 29 | - Running Traces: tabular_tracing.md 30 | - Documenting Model: document.md 31 | - Contributing: CONTRIBUTING.md 32 | 33 | markdown_extensions: 34 | - pymdownx.highlight: 35 | anchor_linenums: true 36 | - pymdownx.inlinehilite 37 | - pymdownx.snippets 38 | - pymdownx.superfences 39 | - pymdownx.tabbed 40 | - admonition 41 | theme: 42 | name: "material" 43 | features: 44 | - content.code.annotate 45 | plugins: 46 | - search 47 | - mkdocstrings: 48 | handlers: 49 | python: 50 | import: 51 | - https://docs.python.org/3/objects.inv 52 | - https://pandas.pydata.org/docs/objects.inv 53 | options: 54 | docstring_style: google 55 | show_submodules: true 56 | show_root_full_path: false 57 | members_order: source -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=61.0"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "python_tabular" 7 | version = "0.5.7" 8 | authors = [ 9 | { name="Curtis Stallings", email="curtisrstallings@gmail.com" }, 10 | ] 11 | dependencies = [ 12 | "pythonnet>=3.0.3", 13 | "clr-loader>=0.2.6", 14 | "xmltodict==0.13.0", 15 | "pandas>=1.4.3", 16 | "requests>=2.28.1", 17 | "rich>=12.5.1" 18 | ] 19 | description = "Connect to your tabular model and perform operations programmatically" 20 | readme = "README.md" 21 | requires-python = ">=3.8" 22 | classifiers = [ 23 | "Programming Language :: Python :: 3.8", 24 | "Programming Language :: Python :: 3.9", 25 | "Programming Language :: Python :: 3.10", 26 | "Programming Language :: Python :: 3.11", 27 | "Programming Language :: Python :: 3.12", 28 | "Programming Language :: Python :: 3.13", 29 | "Development Status :: 5 - Production/Stable", 30 | "Operating System :: Microsoft", 31 | "License :: OSI Approved :: MIT License" 32 | ] 33 | 34 | [project.urls] 35 | "Homepage" = "https://github.com/Curts0/PyTabular" 36 | "Bug Tracker" = "https://github.com/Curts0/PyTabular/issues" 37 | 38 | [tool.setuptools] 39 | packages.find.where = ["."] 40 | packages.find.include = ["pytabular"] 41 | include-package-data = true 42 | 43 | [tool.setuptools.package-data] 44 | "*" = ["*.dll"] 45 | 46 | [tool.pytest.ini_options] 47 | filterwarnings = ["ignore::DeprecationWarning"] 48 | addopts = "-vv" 49 | 50 | [tool.coverage.report] 51 | exclude_lines = [ 52 | "pragma: no cover" 53 | ] 54 | [tool.coverage.run] 55 | include = [ 56 | "pytabular/*" 57 | ] 58 | -------------------------------------------------------------------------------- /pytabular/__init__.py: -------------------------------------------------------------------------------- 1 | """Welcome to PyTabular. 2 | 3 | __init__.py will start to setup the basics. 4 | It will setup logging and make sure Pythonnet is good to go. 5 | Then it will begin to import specifics of the module. 6 | """ 7 | 8 | # flake8: noqa 9 | import logging 10 | import os 11 | import sys 12 | import platform 13 | from rich.logging import RichHandler 14 | from rich.theme import Theme 15 | from rich.console import Console 16 | from rich import pretty 17 | 18 | pretty.install() 19 | console = Console(theme=Theme({"logging.level.warning": "bold reverse red"})) 20 | logging.basicConfig( 21 | level=logging.DEBUG, 22 | format="%(message)s", 23 | datefmt="[%H:%M:%S]", 24 | handlers=[RichHandler(console=console)], 25 | ) 26 | logger = logging.getLogger("PyTabular") 27 | logger.setLevel(logging.INFO) 28 | logger.info("Logging configured...") 29 | logger.info("To update logging:") 30 | logger.info(">>> import logging") 31 | logger.info(">>> pytabular.logger.setLevel(level=logging.INFO)") 32 | logger.info("See https://docs.python.org/3/library/logging.html#logging-levels") 33 | 34 | 35 | logger.info(f"Python Version::{sys.version}") 36 | logger.info(f"Python Location::{sys.exec_prefix}") 37 | logger.info(f"Package Location:: {__file__}") 38 | logger.info(f"Working Directory:: {os.getcwd()}") 39 | logger.info(f"Platform:: {sys.platform}-{platform.release()}") 40 | 41 | dll = os.path.join(os.path.dirname(__file__), "dll") 42 | sys.path.append(dll) 43 | sys.path.append(os.path.dirname(__file__)) 44 | 45 | logger.info("Beginning CLR references...") 46 | import clr 47 | 48 | logger.info("Adding Reference Microsoft.AnalysisServices.AdomdClient") 49 | clr.AddReference("Microsoft.AnalysisServices.AdomdClient") 50 | logger.info("Adding Reference Microsoft.AnalysisServices.Tabular") 51 | clr.AddReference("Microsoft.AnalysisServices.Tabular") 52 | logger.info("Adding Reference Microsoft.AnalysisServices") 53 | clr.AddReference("Microsoft.AnalysisServices") 54 | 55 | logger.info("Importing specifics in module...") 56 | from .pytabular import Tabular 57 | 58 | from .logic_utils import ( 59 | pd_dataframe_to_m_expression, 60 | pandas_datatype_to_tabular_datatype, 61 | ) 62 | from .tabular_tracing import BaseTrace, RefreshTrace, QueryMonitor 63 | from .tabular_editor import TabularEditor 64 | from .best_practice_analyzer import BPA 65 | from .query import Connection 66 | from .pbi_helper import find_local_pbi_instances 67 | from .document import ModelDocumenter 68 | from .tmdl import Tmdl 69 | 70 | 71 | logger.info("Import successful...") 72 | -------------------------------------------------------------------------------- /pytabular/best_practice_analyzer.py: -------------------------------------------------------------------------------- 1 | """This is currently just a POC. Handle all BPA related items. 2 | 3 | You can call the `BPA()` class to download or specify your own BPA file. 4 | It is used with tabular_editor.py to run BPA. 5 | I did not want to re-invent the wheel, so just letting TE2 work it's magic. 6 | """ 7 | 8 | import logging 9 | import requests as r 10 | import atexit 11 | import json 12 | import os 13 | from pytabular.logic_utils import remove_folder_and_contents 14 | 15 | 16 | logger = logging.getLogger("PyTabular") 17 | 18 | 19 | def download_bpa_file( 20 | download_location: str = ( 21 | "https://raw.githubusercontent.com/microsoft/Analysis-Services/master/BestPracticeRules/BPARules.json" # noqa: E501 22 | ), 23 | folder: str = "Best_Practice_Analyzer", 24 | auto_remove: bool = True, 25 | verify: bool = False, 26 | ) -> str: 27 | """Download a BPA file from local or web. 28 | 29 | Runs a request.get() to retrieve the json file from web. 30 | Will return and store in directory. 31 | Will also register the removal of the new directory and file when exiting program. 32 | 33 | Args: 34 | download_location (str, optional): Defaults to 35 | [BPA]'https://raw.githubusercontent.com/microsoft/Analysis-Services/master/BestPracticeRules/BPARules.json'. 36 | folder (str, optional): New folder string. 37 | Defaults to 'Best_Practice_Analyzer'. 38 | auto_remove (bool, optional): Auto Remove when script exits. Defaults to True. 39 | verify (bool, optional): Passthrough argument for `r.get`. Need to update later. 40 | 41 | Returns: 42 | str: File path for the newly downloaded BPA. 43 | """ 44 | logger.info(f"Downloading BPA from {download_location}") 45 | folder_location = os.path.join(os.getcwd(), folder) 46 | if os.path.exists(folder_location) is False: 47 | os.makedirs(folder_location) 48 | response = r.get(download_location, verify=verify) 49 | file_location = os.path.join(folder_location, download_location.split("/")[-1]) 50 | with open(file_location, "w", encoding="utf-8") as bpa: 51 | json.dump(response.json(), bpa, ensure_ascii=False, indent=4) 52 | if auto_remove: 53 | logger.debug(f"Registering removal on termination... For {folder_location}") 54 | atexit.register(remove_folder_and_contents, folder_location) 55 | return file_location 56 | 57 | 58 | class BPA: 59 | """Setting BPA Class for future work...""" 60 | 61 | def __init__( 62 | self, file_path: str = "Default", verify_download: bool = True 63 | ) -> None: 64 | """BPA class to be used with the TE2 class. 65 | 66 | You can create the BPA class without any arguments. 67 | This doesn't do much right now... 68 | BPA().location is where the file path is stored. 69 | 70 | Args: 71 | file_path (str, optional): See `Download_BPA_File()`. Defaults to "Default". 72 | verify_download (bool, optional): Passthrough argument for `r.get`. 73 | Need to update later. 74 | """ 75 | logger.debug(f"Initializing BPA Class:: {file_path}") 76 | if file_path == "Default": 77 | self.location: str = download_bpa_file(verify=verify_download) 78 | else: 79 | self.location: str = file_path 80 | pass 81 | -------------------------------------------------------------------------------- /pytabular/column.py: -------------------------------------------------------------------------------- 1 | """`column.py` houses the main `PyColumn` and `PyColumns` class. 2 | 3 | Once connected to your model, interacting with column(s) will be done through these classes. 4 | """ 5 | 6 | import logging 7 | import pandas as pd 8 | from pytabular.object import PyObject, PyObjects 9 | from Microsoft.AnalysisServices.Tabular import ColumnType 10 | 11 | logger = logging.getLogger("PyTabular") 12 | 13 | 14 | class PyColumn(PyObject): 15 | """The main class to work with your columns. 16 | 17 | Notice the `PyObject` magic method `__getattr__()` will search in `self._object` 18 | if it is unable to find it in the default attributes. 19 | This let's you also easily check the default .Net properties. 20 | See methods for extra functionality. 21 | """ 22 | 23 | def __init__(self, object, table) -> None: 24 | """Init that connects your column to parent table. 25 | 26 | It will also build custom rows for your `rich` 27 | display table. 28 | 29 | Args: 30 | object (Column): .Net column object. 31 | table (Table): .Net table object. 32 | """ 33 | super().__init__(object) 34 | self.Table = table 35 | self._display.add_row( 36 | "Description", str(self._object.Description), end_section=True 37 | ) 38 | self._display.add_row("DataType", str(self._object.DataType)) 39 | self._display.add_row("EncodingHint", str(self._object.EncodingHint)) 40 | self._display.add_row("IsAvailableInMDX", str(self._object.IsAvailableInMDX)) 41 | self._display.add_row("IsHidden", str(self._object.IsHidden)) 42 | self._display.add_row("IsKey", str(self._object.IsKey)) 43 | self._display.add_row("IsNullable", str(self._object.IsNullable)) 44 | self._display.add_row("State", str(self._object.State)) 45 | self._display.add_row("DisplayFolder", str(self._object.DisplayFolder)) 46 | 47 | def get_dependencies(self) -> pd.DataFrame: 48 | """Returns the dependant columns of a measure.""" 49 | dmv_query = f"select * from $SYSTEM.DISCOVER_CALC_DEPENDENCY where [OBJECT] = \ 50 | '{self.Name}' and [TABLE] = '{self.Table.Name}'" 51 | return self.Table.Model.query(dmv_query) 52 | 53 | def get_sample_values(self, top_n: int = 3) -> pd.DataFrame: 54 | """Get sample values of column.""" 55 | column_to_sample = f"'{self.Table.Name}'[{self.Name}]" 56 | try: 57 | # adding temporary try except. TOPNSKIP will not work for directquery mode. 58 | # Need an efficient way to identify if query is direct query or not. 59 | dax_query = f"""EVALUATE 60 | TOPNSKIP( 61 | {top_n}, 62 | 0, 63 | FILTER( 64 | VALUES({column_to_sample}), 65 | NOT ISBLANK({column_to_sample}) 66 | && LEN({column_to_sample}) > 0 67 | ), 68 | 1 69 | ) 70 | ORDER BY {column_to_sample} 71 | """ 72 | return self.Table.Model.query(dax_query) 73 | except Exception: 74 | # This is really tech debt anyways and should be replaced... 75 | dax_query = f""" 76 | EVALUATE 77 | TOPN( 78 | {top_n}, 79 | FILTER( 80 | VALUES({column_to_sample}), 81 | NOT ISBLANK({column_to_sample}) && LEN({column_to_sample}) > 0 82 | ) 83 | ) 84 | """ 85 | return self.Table.Model.query(dax_query) 86 | 87 | def distinct_count(self, no_blank=False) -> int: 88 | """Get the `DISTINCTCOUNT` of a column. 89 | 90 | Args: 91 | no_blank (bool, optional): If `True`, will call `DISTINCTCOUNTNOBLANK`. 92 | Defaults to `False`. 93 | 94 | Returns: 95 | int: Number of Distinct Count from column. 96 | If `no_blank == True` then will return number of distinct count no blanks. 97 | """ 98 | func = "DISTINCTCOUNT" 99 | if no_blank: 100 | func += "NOBLANK" 101 | return self.Table.Model.Adomd.query( 102 | f"EVALUATE {{{func}('{self.Table.Name}'[{self.Name}])}}" 103 | ) 104 | 105 | def values(self) -> pd.DataFrame: 106 | """Get single column DataFrame of values in column. 107 | 108 | Similar to `get_sample_values()` but will return **all**. 109 | 110 | Returns: 111 | pd.DataFrame: Single column DataFrame of values. 112 | """ 113 | return self.Table.Model.Adomd.query( 114 | f"EVALUATE VALUES('{self.Table.Name}'[{self.Name}])" 115 | ) 116 | 117 | 118 | class PyColumns(PyObjects): 119 | """Groups together multiple `PyColumn()`. 120 | 121 | See `PyObjects` class for what more it can do. 122 | You can interact with `PyColumns` straight from model. For ex: `model.Columns`. 123 | Or through individual tables `model.Tables[TABLE_NAME].Columns`. 124 | You can even filter down with `.Find()`. 125 | For example find all columns with `Key` in name. 126 | `model.Columns.Find('Key')`. 127 | """ 128 | 129 | def __init__(self, objects) -> None: 130 | """Init extends through to the `PyObjects()` init.""" 131 | super().__init__(objects) 132 | 133 | def query_all(self, query_function: str = "COUNTROWS(VALUES(_))") -> pd.DataFrame: 134 | """This will dynamically all columns in `PyColumns()` class. 135 | 136 | It will replace the `_` with the column to run 137 | whatever the given `query_function` value is. 138 | 139 | Args: 140 | query_function (str, optional): Default is `COUNTROWS(VALUES(_))`. 141 | The `_` gets replaced with the column in question. 142 | Method will take whatever DAX query is given. 143 | 144 | Returns: 145 | pd.DataFrame: Returns dataframe with results. 146 | """ 147 | logger.info("Beginning execution of querying every column...") 148 | logger.debug(f"Function to be run: {query_function}") 149 | logger.debug("Dynamically creating DAX query...") 150 | query_str = "EVALUATE UNION(\n" 151 | columns = [column for column in self] 152 | for column in columns: 153 | if column.Type != ColumnType.RowNumber: 154 | table_name = column.Table.get_Name() 155 | column_name = column.get_Name() 156 | dax_identifier = f"'{table_name}'[{column_name}]" 157 | query_str += f"ROW(\"Table\",\"{table_name}\",\ 158 | \"Column\",\"{column_name}\",\"{query_function}\",\ 159 | {query_function.replace('_',dax_identifier)}),\n" # noqa: E231, E261 160 | query_str = f"{query_str[:-2]})" 161 | return self[0].Table.Model.query(query_str) 162 | -------------------------------------------------------------------------------- /pytabular/culture.py: -------------------------------------------------------------------------------- 1 | """`culture.py` is used to house the `PyCulture`, and `PyCultures` classes.""" 2 | 3 | import logging 4 | from pytabular.object import PyObject, PyObjects 5 | from typing import List 6 | 7 | logger = logging.getLogger("PyTabular") 8 | 9 | 10 | class PyCulture(PyObject): 11 | """Main class to interact with cultures in model.""" 12 | 13 | def __init__(self, object, model) -> None: 14 | """Mostly extends from `PyObject`. But will add rows to `rich`.""" 15 | super().__init__(object) 16 | self.Model = model 17 | self._display.add_row("Culture Name", self._object.Name) 18 | self.ObjectTranslations = self.set_translation() 19 | 20 | def set_translation(self) -> List[dict]: 21 | """Based on the culture, it creates a list of dicts with available translations. 22 | 23 | The model object doesn't have a Parent object. So that will stay 24 | empty. 25 | 26 | Returns: 27 | List[dict]: Translations per object. 28 | """ 29 | return [ 30 | { 31 | "object_translation": translation.Value, 32 | "object_name": translation.Object.Name, 33 | "object_parent_name": ( 34 | translation.Object.Parent.Name if translation.Object.Parent else "" 35 | ), 36 | "object_type": str(translation.Property), 37 | } 38 | for translation in self._object.ObjectTranslations 39 | ] 40 | 41 | def get_translation( 42 | self, object_name: str, object_parent_name: str, object_type: str = "Caption" 43 | ) -> dict: 44 | """Get Translation makes it possible to seach a specific translation of an object. 45 | 46 | By default it will search for the "Caption" object type, due to fact that a 47 | Display folder and Description can also have translations. 48 | 49 | Args: 50 | object_name (str): Object name that you want to translate. 51 | object_parent_name (str): Parent Object name that you want to translate. 52 | object_type (str, optional): The Display Folders can also have translations. 53 | Defaults to "Caption" > Object translation. 54 | 55 | Returns: 56 | dict: With translation of the object. 57 | """ 58 | try: 59 | translations = [ 60 | d 61 | for d in self.ObjectTranslations 62 | if d["object_name"] == object_name 63 | and d["object_type"] == object_type 64 | and d["object_parent_name"] == object_parent_name 65 | ] 66 | return translations[0] 67 | except Exception: 68 | return {"object_translation": object_name} 69 | 70 | 71 | class PyCultures(PyObjects): 72 | """Houses grouping of `PyCulture`.""" 73 | 74 | def __init__(self, objects) -> None: 75 | """Extends `PyObjects` class.""" 76 | super().__init__(objects) 77 | -------------------------------------------------------------------------------- /pytabular/currency.py: -------------------------------------------------------------------------------- 1 | """List of [unicode currencies](https://www.unicode.org/charts/PDF/U20A0.pdf). 2 | 3 | Used to strip formatted values from DAX queries. 4 | For example `$(12,345.67)` will output -> `-12345.67`. 5 | See `logic_utils.clean_formatting` for more. 6 | """ 7 | 8 | unicode_list = [ 9 | "\u0024", 10 | "\u00a2", 11 | "\u00a3", 12 | "\u00a4", 13 | "\u00a5", 14 | "\u0192", 15 | "\u058f", 16 | "\u060b", 17 | "\u09f2", 18 | "\u09f3", 19 | "\u0af1", 20 | "\u0bf9", 21 | "\u0e3f", 22 | "\u17db", 23 | "\u2133", 24 | "\u5143", 25 | "\u5186", 26 | "\u5706", 27 | "\u5713", 28 | "\ufdfc", 29 | # "\u1E2FF", 30 | "\u0024", 31 | "\u20a0", 32 | "\u20a1", 33 | "\u20a2", 34 | "\u20a3", 35 | "\u20a4", 36 | "\u20a5", 37 | "\u20a6", 38 | "\u20a7", 39 | "\u20a8", 40 | "\u20a9", 41 | "\u20aa", 42 | "\u20ab", 43 | "\u20ac", 44 | "\u20ad", 45 | "\u20ae", 46 | "\u20af", 47 | "\u20b0", 48 | "\u20b1", 49 | "\u20b2", 50 | "\u20b3", 51 | "\u20b4", 52 | "\u20b5", 53 | "\u20b6", 54 | "\u20b7", 55 | "\u20b8", 56 | "\u20b9", 57 | "\u20ba", 58 | "\u20bb", 59 | "\u20bc", 60 | "\u20bd", 61 | "\u20be", 62 | "\u20bf", 63 | "\u20c0", 64 | ] 65 | 66 | unicodes: str.maketrans = str.maketrans(dict.fromkeys(unicode_list, "")) 67 | -------------------------------------------------------------------------------- /pytabular/dll/Microsoft.AnalysisServices.AdomdClient.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Curts0/PyTabular/5d43df51f0d113b9c083968bedf12d81ebdc6594/pytabular/dll/Microsoft.AnalysisServices.AdomdClient.dll -------------------------------------------------------------------------------- /pytabular/dll/Microsoft.AnalysisServices.Core.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Curts0/PyTabular/5d43df51f0d113b9c083968bedf12d81ebdc6594/pytabular/dll/Microsoft.AnalysisServices.Core.dll -------------------------------------------------------------------------------- /pytabular/dll/Microsoft.AnalysisServices.Tabular.Json.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Curts0/PyTabular/5d43df51f0d113b9c083968bedf12d81ebdc6594/pytabular/dll/Microsoft.AnalysisServices.Tabular.Json.dll -------------------------------------------------------------------------------- /pytabular/dll/Microsoft.AnalysisServices.Tabular.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Curts0/PyTabular/5d43df51f0d113b9c083968bedf12d81ebdc6594/pytabular/dll/Microsoft.AnalysisServices.Tabular.dll -------------------------------------------------------------------------------- /pytabular/dll/Microsoft.AnalysisServices.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Curts0/PyTabular/5d43df51f0d113b9c083968bedf12d81ebdc6594/pytabular/dll/Microsoft.AnalysisServices.dll -------------------------------------------------------------------------------- /pytabular/document.py: -------------------------------------------------------------------------------- 1 | """`document.py` is where a specific part of pytabular start. 2 | 3 | This module can generate pages in markdown for use in Docusaurus. 4 | """ 5 | 6 | import logging 7 | 8 | from pathlib import Path 9 | 10 | from pytabular.table import PyTable 11 | from pytabular.column import PyColumn 12 | from pytabular.culture import PyCulture 13 | from pytabular.measure import PyMeasure 14 | from pytabular.pytabular import Tabular 15 | from typing import List, Dict 16 | 17 | logger = logging.getLogger("PyTabular") 18 | 19 | 20 | class ModelDocumenter: 21 | """The ModelDocumenter class can generate documentation. 22 | 23 | This is based on the tabular object model and it will generate it suitable for docusaurus. 24 | TODO: Add a General Pages template with Roles and RLS Expressions. 25 | TODO: Create a Sub Page per table for all columns, instead of one big page? 26 | TODO: Add Depencies per Measure with correct links. 27 | """ 28 | 29 | def __init__( 30 | self, 31 | model: Tabular, 32 | friendly_name: str = str(), 33 | save_location: str = "docs", 34 | general_page_url: str = "index.md", 35 | measure_page_url: str = "measures.md", 36 | roles_page_url: str = "roles.md", 37 | table_folder: str = "tables", 38 | # table_page_url: str = "3-tables.md", 39 | # column_page_url: str = "4-columns.md", 40 | ): 41 | """Init will set attributes based on arguments given. 42 | 43 | See `generate_documentation_pages()` and `save_documentation()` 44 | for info on how to execute and retrieve documentation. 45 | 46 | Args: 47 | model (Tabular): Main `Tabular()` class to pull metadata from for documentation. 48 | friendly_name (str, optional): Replaces the model name to a friendly string, 49 | so it can be used in an URL. Defaults to `str()`. 50 | save_location (str, optional): The save location where the files will be stored. 51 | Defaults to "docs". 52 | general_page_url (str, optional): Name of the `md` file for general information. 53 | Defaults to "index.md". 54 | measure_page_url (str, optional): Name of the `md` file for measures. 55 | Defaults to "measures.md". 56 | table_folder (str, optional): Name of the folder where columns info is stored. 57 | Defaults to "table_folder". 58 | roles_page_url (str, optional): Name of the `md` file for roles. 59 | Defaults to "roles.md". 60 | """ 61 | self.model = model 62 | self.model_name = friendly_name or model.Catalog or model.Database.Name 63 | self.friendly_name: str = str() 64 | self.save_path: Path 65 | self.save_location: str = save_location 66 | 67 | # Translation information 68 | self.culture_include: bool = False 69 | self.culture_selected: str = "en-US" 70 | self.culture_object: PyCulture 71 | 72 | # Documentation Parts 73 | self.general_page: str = str() 74 | self.general_page_url: str = general_page_url 75 | 76 | self.measure_page: str = str() 77 | self.measure_page_url: str = measure_page_url 78 | 79 | self.roles_page: str = str() 80 | self.roles_page_url: str = roles_page_url 81 | 82 | self.table_page: str = str() 83 | self.table_folder: str = table_folder 84 | 85 | # Generate an url friendly name for the model / folder 86 | self.friendly_name: str = self.set_url_friendly_name(self.model_name) 87 | 88 | # Initialize Save path so checks can be run against it. 89 | self.save_path = self.set_save_path() 90 | 91 | def create_object_reference(self, object: str, object_parent: str) -> str: 92 | """Create a Custom ID for link sections in the docs. 93 | 94 | This is based on the technical names in the model, 95 | so not the once in the translations. This makes it 96 | possible to link based on dependencies. 97 | (Scope is only Docusaurus) 98 | 99 | Args: 100 | object (str): Object Name 101 | object_parent (str): Object Parent (e.g. Table) 102 | 103 | Returns: 104 | str: String that can be used for custom linking 105 | """ 106 | url_reference = f"{object_parent}-{object}".replace(" ", "") 107 | return f"{{#{url_reference}}}" 108 | 109 | def generate_documentation_pages(self) -> None: 110 | """Generate Documentation for each specific part of the model.""" 111 | self.measure_page = self.generate_markdown_measure_page() 112 | self.table_page = self.generate_markdown_table_page() 113 | self.general_page = self.generate_general_info_file() 114 | 115 | def get_object_caption(self, object_name: str, object_parent: str) -> str: 116 | """Retrieves the caption of an object, based on the translations in the culture. 117 | 118 | If no culture is present, the object_name is returned. 119 | 120 | Args: 121 | object_name (str): Object Name 122 | object_parent (str): Object Parent Name 123 | 124 | Returns: 125 | str: Translated object. 126 | """ 127 | if self.culture_include: 128 | return str( 129 | self.culture_object.get_translation( 130 | object_name=object_name, object_parent_name=object_parent 131 | ).get("object_translation") 132 | ) 133 | 134 | return object_name 135 | 136 | def set_translations( 137 | self, enable_translations: bool = False, culture: str = "en-US" 138 | ) -> None: 139 | """Set translations to active or inactive, depending on the needs of the users. 140 | 141 | Args: 142 | enable_translations (bool, optional): Flag to enable or disable translations. 143 | Defaults to False. 144 | culture (str, optional): Set culture that needs to be used in the docs. 145 | Defaults to "en-US". 146 | """ 147 | logger.info(f"Using Translations set to > {enable_translations}") 148 | 149 | if enable_translations: 150 | try: 151 | self.culture_object = self.model.Cultures[culture] 152 | self.culture_selected = culture 153 | self.culture_include = enable_translations 154 | except IndexError: 155 | self.culture_include = False 156 | logger.warn( 157 | "Culture not found, reverting back to orginal setting > False" 158 | ) 159 | else: 160 | logger.info(f"Setting culture to {self.culture_selected}") 161 | 162 | else: 163 | self.culture_include = enable_translations 164 | 165 | def set_save_path(self) -> Path: 166 | """Set the location of the documentation. 167 | 168 | Returns: 169 | Path: Path where the docs are saved. 170 | """ 171 | return Path(f"{self.save_location}/{self.friendly_name}") 172 | 173 | def save_page(self, content: str, page_name: str, keep_file: bool = False) -> None: 174 | """Save the content of the documentation to a file. 175 | 176 | Based on the class setup. 177 | - Save Location 178 | - Model Friendly Name 179 | - Page to be written 180 | 181 | Args: 182 | content (str): File content to write to file. 183 | page_name (str): Name of the file that will be used. 184 | keep_file (bool): The file will only be overwritten if 185 | the keep_file is set to False. 186 | 187 | Returns: 188 | None 189 | """ 190 | target_file = self.save_path / page_name 191 | 192 | if target_file.parent.exists() is False: 193 | target_file.parent.mkdir(parents=True, exist_ok=True) 194 | 195 | if keep_file and target_file.exists(): 196 | logger.info(f"{page_name} already exists -> file will not overwritten.") 197 | else: 198 | logger.info(f"Results are written to -> {page_name}.") 199 | 200 | with target_file.open("w", encoding="utf-8") as f: 201 | f.write(content) 202 | f.close() 203 | 204 | def save_documentation(self) -> None: 205 | """Generate documentation of the model, based on the meta-data in the model definitions. 206 | 207 | This first checks if the folder exists, 208 | and then starts to export the files that are needed 209 | for the documentatation. 210 | - General Information Page -> Free format page to create. 211 | - Measure Page -> Describes the measures in the model. 212 | - Tables Page -> Describes the tables in the model. 213 | - Columns Page -> Describes all columns in the model per table. 214 | - Roles Page -> Describes the roles in the model. 215 | 216 | Args: 217 | self (ModelDocumenter): Model object for documentation. 218 | 219 | Returns: 220 | None 221 | """ 222 | if self.save_path.exists(): 223 | logger.info( 224 | f"Path exists -> Generating documentation for {self.friendly_name}" 225 | ) 226 | else: 227 | logger.info( 228 | f"Path does not exist -> Creating directory for {self.friendly_name}" 229 | ) 230 | self.save_path.mkdir(parents=True, exist_ok=True) 231 | 232 | if self.general_page: 233 | self.save_page( 234 | content=self.general_page, 235 | keep_file=True, 236 | page_name=self.general_page_url, 237 | ) 238 | 239 | if self.measure_page: 240 | self.save_page( 241 | content=self.measure_page, 242 | keep_file=False, 243 | page_name=self.measure_page_url, 244 | ) 245 | 246 | for table in self.create_markdown_for_table_and_column(): 247 | table = table.items() 248 | page_name, page_content = list(table)[0] 249 | self.save_page( 250 | content=page_content, 251 | keep_file=False, 252 | page_name=f"{self.table_folder}/{page_name}", 253 | ) 254 | 255 | if self.roles_page: 256 | self.save_page( 257 | content=self.roles_page, keep_file=False, page_name=self.roles_page_url 258 | ) 259 | 260 | def create_markdown_for_measure(self, object: PyMeasure) -> str: 261 | """Create Markdown for a specific measure. 262 | 263 | That can later on be used for generating the whole measure page. 264 | 265 | Args: 266 | object (PyMeasure): The measure to document. 267 | 268 | Returns: 269 | str: Markdown section for specific Measure 270 | """ 271 | object_caption = ( 272 | self.get_object_caption( 273 | object_name=object.Name, object_parent=object.Parent.Name 274 | ) 275 | or object.Name 276 | ) 277 | 278 | obj_description = (object.Description or "No Description available").replace( 279 | "\\n", "" 280 | ) 281 | 282 | obj_description = obj_description.replace("<>", "not equal to ") 283 | 284 | object_properties = [ 285 | {"Measure Name": object.Name}, 286 | {"Display Folder": object.DisplayFolder}, 287 | {"Format String": object.FormatString}, 288 | {"Is Hidden": "Yes" if object.IsHidden else "No"}, 289 | ] 290 | 291 | obj_text = [ 292 | f"### {object_caption}", 293 | "**Description**:", 294 | f"> {obj_description}", 295 | "", 296 | "" f"{self.generate_object_properties(object_properties)}" "", 297 | f'```dax title="Technical: {object.Name}"', 298 | f"{object.Expression}", 299 | "```", 300 | "---", 301 | ] 302 | return "\n".join(obj_text) 303 | 304 | def generate_markdown_measure_page(self) -> str: 305 | """This function generates the meausure documation page. 306 | 307 | Returns: 308 | str: The full markdown text that is needed 309 | make it compatible with Docusaurus. 310 | """ 311 | prev_display_folder = "" 312 | markdown_template = [ 313 | "---", 314 | "sidebar_position: 1", 315 | "title: Measures", 316 | "description: This page contains all measures for " 317 | f"the {self.model.Name} model, including the description, " 318 | "format string, and other technical details.", 319 | "---", 320 | "", 321 | f"# Measures for {self.model.Name}", 322 | ] 323 | 324 | measures = sorted( 325 | self.model.Measures, key=lambda x: x.DisplayFolder, reverse=False 326 | ) 327 | 328 | for measure in measures: 329 | logger.debug(f"Creating docs for {measure.Name}") 330 | display_folder = measure.DisplayFolder or "Other" 331 | display_folder = display_folder.split("\\")[0] 332 | 333 | if prev_display_folder != display_folder: 334 | markdown_template.append(f"""## {display_folder}""") 335 | prev_display_folder = display_folder 336 | 337 | markdown_template.append(self.create_markdown_for_measure(measure)) 338 | 339 | return "\n".join(markdown_template) 340 | 341 | def create_markdown_for_table_and_column(self) -> list: 342 | """Create Pages for Tables and Columns. 343 | 344 | Based on the model this functions creates a general 345 | overview pages for all tables and then with per 346 | table a page with all column details. 347 | 348 | Returns: 349 | list: List of dicts per page. 350 | 351 | Example: 352 | ``` 353 | { 354 | "Overview": "Content", 355 | "Table1": "Content", 356 | "Table2": "Content", 357 | } 358 | 359 | """ 360 | obj_content = [{"index.md": self.generate_markdown_table_page()}] 361 | 362 | for idx, table in enumerate(self.model.Tables): 363 | obj_caption = ( 364 | self.get_object_caption( 365 | object_name=table.Name, object_parent=table.Parent.Name 366 | ) 367 | or table.Name 368 | ) 369 | 370 | obj_caption = obj_caption.replace("[", "").replace("]", "") 371 | 372 | key = f"{self.set_url_friendly_name(obj_caption)}.md" 373 | value = self.generate_markdown_column_page( 374 | object=table, object_caption=obj_caption, page_index=idx + 2 375 | ) 376 | 377 | obj_content.append({key: value}) 378 | 379 | return obj_content 380 | 381 | def create_markdown_for_table(self, object: PyTable) -> str: 382 | """This functions returns the markdown for a table. 383 | 384 | Args: 385 | object (PyTable): Based on the PyTabular Package. 386 | 387 | Returns: 388 | str: Will be appended to the page text. 389 | """ 390 | object_caption = ( 391 | self.get_object_caption( 392 | object_name=object.Name, object_parent=object.Parent.Name 393 | ) 394 | or object.Name 395 | ) 396 | 397 | obj_description = (object.Description or "No Description available").replace( 398 | "\\n", "" 399 | ) 400 | 401 | object_properties = [ 402 | {"Measures (#)": len(object.Measures)}, 403 | {"Columns (#)": len(object.Columns)}, 404 | {"Partiton (#)": len(object.Partitions)}, 405 | {"Data Category": object.DataCategory or "Regular Table"}, 406 | {"Is Hidden": object.IsHidden}, 407 | {"Table Type": object.Partitions[0].ObjectType}, 408 | {"Source Type": object.Partitions[0].SourceType}, 409 | ] 410 | 411 | obj_description = obj_description.replace("<>", "not equal to ") 412 | 413 | partition_type = "" 414 | partition_source = "" 415 | 416 | logger.debug(f"{object_caption} => {str(object.Partitions[0].SourceType)}") 417 | 418 | if str(object.Partitions[0].SourceType) == "Calculated": 419 | partition_type = "dax" 420 | partition_source = object.Partitions[0].Source.Expression 421 | elif str(object.Partitions[0].SourceType) == "M": 422 | partition_type = "powerquery" 423 | partition_source = object.Partitions[0].Source.Expression 424 | elif str(object.Partitions[0].SourceType) == "CalculationGroup": 425 | partition_type = "" 426 | partition_source = "" 427 | else: 428 | partition_type = "sql" 429 | partition_source = object.Partitions[0].Source.Query 430 | 431 | obj_text = [ 432 | f"### {object_caption}", 433 | "**Description**: ", 434 | f"> {obj_description}", 435 | "", 436 | f"{self.generate_object_properties(object_properties)}", 437 | "", 438 | f'```{partition_type} title="Table Source: {object.Name}"', 439 | f"{partition_source}", 440 | "```", 441 | "---", 442 | ] 443 | 444 | return "\n".join(obj_text) 445 | 446 | def generate_markdown_table_page(self) -> str: 447 | """This function generates the markdown for table documentation. 448 | 449 | Returns: 450 | str: Will be appended to the page text. 451 | """ 452 | markdown_template = [ 453 | "---", 454 | "sidebar_position: 2", 455 | "sidebar_label: Tables", 456 | "description: This page contains all columns with " 457 | f"tables for {self.model.Name}, including the description, " 458 | "and technical details.", 459 | "---", 460 | "", 461 | f"# Tables {self.model.Name}", 462 | ] 463 | 464 | markdown_template.extend( 465 | self.create_markdown_for_table(table) for table in self.model.Tables 466 | ) 467 | return "\n".join(markdown_template) 468 | 469 | def create_markdown_for_column(self, object: PyColumn) -> str: 470 | """Generates the Markdown for a specifc column. 471 | 472 | If a columns is calculated, then it also shows the expression for 473 | that column in DAX. 474 | 475 | Args: 476 | object (PyColumn): Needs PyColumn objects input 477 | 478 | Returns: 479 | str: Will be appended to the page text. 480 | """ 481 | object_caption = ( 482 | self.get_object_caption( 483 | object_name=object.Name, object_parent=object.Parent.Name 484 | ) 485 | or object.Name 486 | ) 487 | 488 | obj_description = ( 489 | object.Description.replace("\\n", "") or "No Description available" 490 | ) 491 | 492 | obj_description = obj_description.replace("<>", "not equal to ") 493 | 494 | obj_heading = f"""{object_caption}""" 495 | 496 | object_properties = [ 497 | {"Column Name": object.Name}, 498 | {"Object Type": object.ObjectType}, 499 | {"Type": object.Type}, 500 | {"Is Available In Excel": object.IsAvailableInMDX}, 501 | {"Is Hidden": object.IsHidden}, 502 | {"Data Category": object.DataCategory}, 503 | {"Data Type": object.DataType}, 504 | {"Display Folder": object.DisplayFolder}, 505 | ] 506 | 507 | obj_text = [ 508 | f"### {obj_heading}", 509 | "**Description**:", 510 | f"> {obj_description}", 511 | "", 512 | f"{self.generate_object_properties(object_properties)}", 513 | ] 514 | 515 | if str(object.Type) == "Calculated": 516 | obj_text.extend( 517 | ( 518 | f'```dax title="Technical: {object.Name}"', 519 | f"{object.Expression}", 520 | "```", 521 | ) 522 | ) 523 | obj_text.append("---") 524 | 525 | return "\n".join(obj_text) 526 | 527 | def generate_markdown_column_page( 528 | self, object: PyTable, object_caption: str, page_index: int = 2 529 | ) -> str: 530 | """This function generates the markdown for the colums documentation. 531 | 532 | Returns: 533 | str: Will be appended to the page text. 534 | """ 535 | markdown_template = [ 536 | "---", 537 | f"sidebar_position: {page_index}", 538 | f"sidebar_label: {object_caption}", 539 | f"title: {object_caption}", 540 | f"description: This page contains all columns with " 541 | f"Columns for {self.model.Name} " 542 | "including the description, format string, and other technical details.", 543 | "---", 544 | "", 545 | ] 546 | 547 | markdown_template.extend( 548 | self.create_markdown_for_column(column) 549 | for column in object.Columns 550 | if "RowNumber" not in column.Name 551 | ) 552 | return "\n".join(markdown_template) 553 | 554 | def generate_general_info_file(self) -> str: 555 | """Index.md file for the model. 556 | 557 | Basic text for an introduction page. 558 | 559 | Returns: 560 | str: Markdown str for info page 561 | """ 562 | return "\n".join( 563 | [ 564 | "---", 565 | "sidebar_position: 1", 566 | f"title: {self.model_name}", 567 | "description: This page contains all measures for the Model model," 568 | "including the description," 569 | "format string, and other technical details.", 570 | "---", 571 | "", 572 | "## General information", 573 | "### Business Owners", 574 | "", 575 | "## Information Sources", 576 | ] 577 | ) 578 | 579 | @staticmethod 580 | def generate_object_properties(properties: List[Dict[str, str]]) -> str: 581 | """Generate the section for object properties. 582 | 583 | You can select your own properties to display 584 | by providing a the properties in a list of 585 | dicts. 586 | 587 | Args: 588 | properties (dict): The ones you want to show. 589 | 590 | Returns: 591 | str: HTML used in the markdown. 592 | 593 | Example: 594 | ```python 595 | [ 596 | { "Display Folder": "Sales Order Information" }, 597 | { "Is Hidden": "False" }, 598 | { "Format String": "#.###,## } 599 | ] 600 | ``` 601 | Returns: 602 | ``` 603 |
604 |
Display Folder
605 |
Sales Order Information
606 | 607 |
Is Hidden
608 |
False
609 | 610 |
Format String
611 |
#.###,##
612 |
613 | ``` 614 | """ 615 | obj_text = ["
"] 616 | 617 | for obj_prop in properties: 618 | for caption, text in obj_prop.items(): 619 | save_text = str(text).replace("\\", " > ") 620 | obj_text.extend( 621 | (f"
{caption}
", f"
{save_text}
", "") 622 | ) 623 | obj_text.extend(("
", "")) 624 | return "\n".join(obj_text) 625 | 626 | @staticmethod 627 | def set_url_friendly_name(page_name: str) -> str: 628 | """Replaces the model name to a friendly string, so it can be used in an URL. 629 | 630 | Returns: 631 | str: Friendly model name used in url for docs. 632 | """ 633 | # return (self.model_name).replace(" ", "-").replace("_", "-").lower() 634 | return ( 635 | page_name.replace(" ", "-") 636 | .replace("_", "-") 637 | .lower() 638 | .replace("[", "") 639 | .replace("]", "") 640 | ) 641 | -------------------------------------------------------------------------------- /pytabular/logic_utils.py: -------------------------------------------------------------------------------- 1 | """`logic_utils` used to store multiple functions that are used in many different files.""" 2 | 3 | import logging 4 | import datetime 5 | import os 6 | from typing import Dict, List 7 | import pandas as pd 8 | 9 | from pytabular.currency import unicodes 10 | 11 | from Microsoft.AnalysisServices.Tabular import DataType 12 | from Microsoft.AnalysisServices.AdomdClient import AdomdDataReader 13 | 14 | logger = logging.getLogger("PyTabular") 15 | 16 | 17 | def ticks_to_datetime(ticks: int) -> datetime.datetime: 18 | """Converts a C# system datetime tick into a python datatime. 19 | 20 | Args: 21 | ticks (int): C# DateTime Tick. 22 | 23 | Returns: 24 | datetime.datetime: datetime of tick. 25 | """ 26 | return datetime.datetime(1, 1, 1) + datetime.timedelta(microseconds=ticks // 10) 27 | 28 | 29 | def pandas_datatype_to_tabular_datatype(df: pd.DataFrame) -> Dict: 30 | """Takes dataframe columns and gets respective tabular column datatype. 31 | 32 | Args: 33 | df (pd.DataFrame): Pandas DataFrame 34 | 35 | Returns: 36 | Dict: dictionary with results. 37 | 38 | Example: 39 | ``` 40 | { 41 | 'col1': , 42 | 'col2': , 43 | 'col3': 44 | } 45 | ``` 46 | """ 47 | logger.info("Getting DF Column Dtypes to Tabular Dtypes...") 48 | tabular_datatype_mapping_key = { 49 | "b": DataType.Boolean, 50 | "i": DataType.Int64, 51 | "u": DataType.Int64, 52 | "f": DataType.Double, 53 | "c": DataType.Double, 54 | "m": DataType.DateTime, 55 | "M": DataType.DateTime, 56 | "O": DataType.String, 57 | "S": DataType.String, 58 | "U": DataType.String, 59 | "V": DataType.String, 60 | } 61 | return { 62 | column: tabular_datatype_mapping_key[df[column].dtype.kind] 63 | for column in df.columns 64 | } 65 | 66 | 67 | def pd_dataframe_to_m_expression(df: pd.DataFrame) -> str: 68 | """This will take a pandas dataframe and convert to an m expression. 69 | 70 | Args: 71 | df (pd.DataFrame): Pandas DataFrame 72 | 73 | Returns: 74 | str: Currently only returning string values in your tabular model. 75 | 76 | Example: 77 | ``` 78 | col1 col2 79 | 0 1 3 80 | 1 2 4 81 | ``` 82 | converts to 83 | ``` 84 | Source=#table({"col1","col2"}, 85 | { 86 | {"1","3"},{"2","4"} 87 | }) 88 | in 89 | Source 90 | ``` 91 | """ 92 | 93 | def m_list_expression_generator(list_of_strings: List[str]) -> str: 94 | """Takes a python list of strings and converts to power query m expression list format. 95 | 96 | Ex: `["item1","item2","item3"]` --> `{"item1","item2","item3"}` 97 | Codepoint reference --> `\u007b` == `{` and `\u007d` == `}` 98 | """ 99 | string_components = ",".join( 100 | [f'"{string_value}"' for string_value in list_of_strings] 101 | ) 102 | return f"\u007b{string_components}\u007d" 103 | 104 | logger.debug(f"Executing m_list_generator()... for {df.columns}") 105 | columns = m_list_expression_generator(df.columns) 106 | expression_str = f"let\nSource=#table({columns},\n" 107 | logger.debug( 108 | f"Iterating through rows to build expression... df has {len(df)} rows..." 109 | ) 110 | expression_list_rows = [] 111 | for _, row in df.iterrows(): 112 | expression_list_rows += [m_list_expression_generator(row.to_list())] 113 | expression_str += f"\u007b\n{','.join(expression_list_rows)}\n\u007d)\nin\nSource" 114 | return expression_str 115 | 116 | 117 | def remove_folder_and_contents(folder_location): 118 | """Internal used in tabular_editor.py and best_practice_analyzer.py. 119 | 120 | Args: 121 | folder_location (str): Folder path to remove directory and contents. 122 | """ 123 | import shutil 124 | 125 | if os.path.exists(folder_location): 126 | logger.info(f"Removing Dir and Contents -> {folder_location}") 127 | shutil.rmtree(folder_location) 128 | 129 | 130 | def remove_file(file_path): 131 | """Just `os.remove()` but wanted a `logger.info()` with it.""" 132 | logger.info(f"Removing file - {file_path}") 133 | os.remove(file_path) 134 | pass 135 | 136 | 137 | def remove_suffix(input_string, suffix): 138 | """Adding for <3.9 compatiblity. 139 | 140 | Args: 141 | input_string (str): input string to remove suffix from. 142 | suffix (str): suffix to be removed. 143 | """ 144 | # [Stackoverflow Answer](https://stackoverflow.com/questions/66683630/removesuffix-returns-error-str-object-has-no-attribute-removesuffix) # noqa: E501 145 | output = ( 146 | input_string[: -len(suffix)] 147 | if suffix and input_string.endswith(suffix) 148 | else input_string 149 | ) 150 | return output 151 | 152 | 153 | def get_sub_list(lst: list, n: int) -> list: 154 | """Nest list by n amount. 155 | 156 | Args: 157 | lst (list): List to nest. 158 | n (int): Amount to nest list. 159 | 160 | Returns: 161 | list: Nested list. 162 | Example: 163 | `get_sub_list([1,2,3,4,5,6],2) == [[1,2],[3,4],[5,6]]` 164 | """ 165 | return [lst[i : i + n] for i in range(0, len(lst), n)] 166 | 167 | 168 | def clean_formatting(query_result: str, sep: str = ",") -> float: 169 | """Attempts to clean DAX formatting. 170 | 171 | For example, `$(12,345.67)` will output -> `-12345.67`. 172 | 173 | Args: 174 | query_result (str): The value given from `get_value_to_df` 175 | sep (str, optional): The thousands separator. Defaults to ",". 176 | 177 | Returns: 178 | float: Value of DAX query cell. 179 | """ 180 | multiplier = 1 181 | if "(" in query_result and ")" in query_result: 182 | multiplier = -1 183 | query_result = query_result.replace("(", "").replace(")", "") 184 | 185 | query_result = query_result.replace(sep, "") 186 | 187 | return float(query_result.translate(unicodes)) * multiplier 188 | 189 | 190 | def get_value_to_df(query: AdomdDataReader, index: int): 191 | """Gets the values from the AdomdDataReader to convert to python df. 192 | 193 | Lots of room for improvement on this one. 194 | 195 | Args: 196 | query (AdomdDataReader): The AdomdDataReader .Net object. 197 | index (int): Index of the value to perform the logic on. 198 | """ 199 | # TODO: Clean this up 200 | if ( 201 | query.GetDataTypeName((index)) in ("Decimal") 202 | and query.GetValue(index) is not None 203 | ): 204 | return query.GetValue(index).ToDouble(query.GetValue(index)) 205 | elif query.GetDataTypeName((index)) in ( 206 | "String" 207 | ): 208 | try: 209 | return clean_formatting(query.GetValue(index)) 210 | except Exception: 211 | return query.GetValue(index) 212 | else: 213 | return query.GetValue(index) 214 | 215 | 216 | def dataframe_to_dict(df: pd.DataFrame) -> List[dict]: 217 | """Convert to Dataframe to dictionary and alter columns names with it. 218 | 219 | Will convert the underscores (_) to spaces, 220 | and all strings are converted to Title Case. 221 | 222 | Args: 223 | df (pd.DataFrame): Original table that needs to be converted 224 | to a list with dicts. 225 | 226 | Returns: 227 | list of dictionaries. 228 | """ 229 | list_of_dicts = df.to_dict("records") 230 | return [ 231 | {k.replace("_", " ").title(): v for k, v in dict.items()} 232 | for dict in list_of_dicts 233 | ] 234 | 235 | 236 | def dict_to_markdown_table(list_of_dicts: list, columns_to_include: list = None) -> str: 237 | """Generate a Markdown table based on a list of dictionaries. 238 | 239 | Args: 240 | list_of_dicts (list): List of Dictionaries that need to be converted 241 | to a markdown table. 242 | columns_to_include (list): Default = None, and all colums are included. 243 | If a list is supplied, those columns will be included. 244 | 245 | Returns: 246 | String that will represent a table in Markdown. 247 | 248 | Example: 249 | ```python 250 | columns = ['Referenced Object Type', 'Referenced Table', 'Referenced Object'] 251 | dict_to_markdown_table(dependancies, columns) 252 | ``` 253 | Returns: 254 | ``` 255 | | Referenced Object Type | Referenced Table | Referenced Object | 256 | | ---------------------- | ---------------- | ------------------------------- | 257 | | TABLE | Cases | Cases | 258 | | COLUMN | Cases | IsClosed | 259 | | CALC_COLUMN | Cases | Resolution Time (Working Hours) | 260 | ``` 261 | """ 262 | keys = set().union(*[set(d.keys()) for d in list_of_dicts]) 263 | 264 | if columns_to_include is not None: 265 | keys = list(keys.intersection(columns_to_include)) 266 | 267 | table_header = f"| {' | '.join(map(str, keys))} |" 268 | table_header_separator = "|-----" * len(keys) + "|" 269 | markdown_table = [table_header, table_header_separator] 270 | 271 | for row in list_of_dicts: 272 | table_row = f"| {' | '.join(str(row.get(key, '')) for key in keys)} |" 273 | markdown_table.append(table_row) 274 | return "\n".join(markdown_table) 275 | -------------------------------------------------------------------------------- /pytabular/measure.py: -------------------------------------------------------------------------------- 1 | """`measure.py` houses the main `PyMeasure` and `PyMeasures` class. 2 | 3 | Once connected to your model, interacting with measure(s) 4 | will be done through these classes. 5 | """ 6 | 7 | import logging 8 | import pandas as pd 9 | from pytabular.object import PyObject, PyObjects 10 | from Microsoft.AnalysisServices.Tabular import Measure, Table 11 | 12 | 13 | logger = logging.getLogger("PyTabular") 14 | 15 | 16 | class PyMeasure(PyObject): 17 | """Main class for interacting with measures. 18 | 19 | See methods for available functionality. 20 | """ 21 | 22 | def __init__(self, object, table) -> None: 23 | """Connects measure to parent `PyTable`. 24 | 25 | It will also add some custom rows for the `rich` 26 | table display. 27 | 28 | Args: 29 | object (object.PyObject): The .Net measure object. 30 | table (table.PyTable): The parent `PyTable`. 31 | """ 32 | super().__init__(object) 33 | self.Table = table 34 | self._display.add_row("Expression", self._object.Expression, end_section=True) 35 | self._display.add_row("DisplayFolder", self._object.DisplayFolder) 36 | self._display.add_row("IsHidden", str(self._object.IsHidden)) 37 | self._display.add_row("FormatString", self._object.FormatString) 38 | 39 | def get_dependencies(self) -> pd.DataFrame: 40 | """Get the dependant objects of a measure. 41 | 42 | Returns: 43 | pd.DataFrame: The Return Value is a Pandas dataframe 44 | which displays all the dependancies 45 | of the object. 46 | 47 | """ 48 | dmv_query = f"select * from $SYSTEM.DISCOVER_CALC_DEPENDENCY where \ 49 | [OBJECT] = '{self.Name}' and [TABLE] = '{self.Table.Name}'" 50 | return self.Table.Model.query(dmv_query) 51 | 52 | 53 | class PyMeasures(PyObjects): 54 | """Groups together multiple measures. 55 | 56 | See `PyObjects` class for what more it can do. 57 | You can interact with `PyMeasures` straight from model. For ex: `model.Measures`. 58 | Or through individual tables `model.Tables[TABLE_NAME].Measures`. 59 | You can even filter down with `.find()`. 60 | For example find all measures with `ratio` in name. 61 | `model.Measures.find('ratio')`. 62 | """ 63 | 64 | def __init__(self, objects, parent=None) -> None: 65 | """Extends init from `PyObjects`.""" 66 | super().__init__(objects, parent) 67 | 68 | def __call__(self, *args, **kwargs): 69 | """Made `PyMeasures` just sends args through to `add_measure`.""" 70 | return self.add_measure(*args, **kwargs) 71 | 72 | def add_measure( 73 | self, name: str, expression: str, auto_save: bool = True, **kwargs 74 | ) -> PyMeasure: 75 | """Add or replace measures from `PyMeasures` class. 76 | 77 | Required is just `name` and `expression`. 78 | But you can pass through any properties you wish to update as a kwarg. 79 | This method is also used when calling the class, 80 | so you can create a new measure that way. 81 | kwargs will be set via the `settr` built in function. 82 | Anything in the .Net Measures properties should be viable. 83 | [Measure Class](https://learn.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.measure?#properties) # noqa: E501 84 | 85 | Example: 86 | ``` 87 | expr = "SUM('Orders'[Amount])" 88 | model.Measures.add_measure("Orders Total", expr) 89 | ``` 90 | 91 | ``` 92 | expr = "SUM('Orders'[Amount])" 93 | model.Measures.add_measure("Orders Total", expr, Folder = 'Measures') 94 | ``` 95 | 96 | ``` 97 | expr = "SUM('Orders'[Amount])" 98 | model.Tables['Sales'].Measures('Total Sales', expr, Folder = 'Measures') 99 | ``` 100 | 101 | Args: 102 | name (str): Name of the measure. Brackets ARE NOT required. 103 | expression (str): DAX expression for the measure. 104 | auto_save (bool, optional): Automatically save changes after measure creations. 105 | Defaults to `True` 106 | """ 107 | if isinstance(self.parent._object, Table): 108 | table = self.parent 109 | model = self.parent.Model 110 | else: 111 | table = self.parent.Tables._first_visible_object() 112 | model = self.parent 113 | 114 | logger.debug(f"Creating measure in {table.Name}") 115 | 116 | new = True 117 | 118 | try: 119 | logger.debug(f"Measure {name} exists... Overwriting...") 120 | new_measure = self.parent.Measures[name]._object 121 | new = False 122 | except IndexError: 123 | logger.debug(f"Creating new measure {name}") 124 | new_measure = Measure() 125 | 126 | new_measure.set_Name(name) 127 | new_measure.set_Expression(expression) 128 | 129 | for key, value in kwargs.items(): 130 | logger.debug(f"Setting '{key}'='{value}' for {new_measure.Name}") 131 | setattr(new_measure, key, value) 132 | 133 | if new: 134 | measures = table.get_Measures() 135 | measures.Add(new_measure) 136 | if auto_save: 137 | model.save_changes() 138 | return model.Measures[new_measure.Name] 139 | else: 140 | return True 141 | -------------------------------------------------------------------------------- /pytabular/object.py: -------------------------------------------------------------------------------- 1 | """`object.py` stores the main parent classes `PyObject` and `PyObjects`. 2 | 3 | These classes are used with the others (Tables, Columns, Measures, Partitions, etc.). 4 | """ 5 | 6 | from __future__ import annotations 7 | from abc import ABC 8 | from rich.console import Console 9 | from rich.table import Table 10 | from collections.abc import Iterable 11 | 12 | 13 | class PyObject(ABC): 14 | """The main parent class for your (Tables, Columns, Measures, Partitions, etc.). 15 | 16 | Notice the magic methods. `__rich_repr__()` starts the baseline for displaying your model. 17 | It uses the amazing `rich` python package and 18 | builds your display from the `self._display`. 19 | `__getattr__()` will check in `self._object`, if unable to find anything in `self`. 20 | This will let you pull properties from the main .Net class. 21 | """ 22 | 23 | def __init__(self, object) -> None: 24 | """Init to create your PyObject. 25 | 26 | This will take the `object` and 27 | set as an attribute to the `self._object`. 28 | You can use that if you want to interact directly with the .Net object. 29 | It will also begin to build out a default `rich` table display. 30 | 31 | Args: 32 | object (.Net object): A .Net object. 33 | """ 34 | self._object = object 35 | self._display = Table(title=self.Name) 36 | self._display.add_column( 37 | "Properties", justify="right", style="cyan", no_wrap=True 38 | ) 39 | self._display.add_column("", justify="left", style="magenta", no_wrap=False) 40 | 41 | self._display.add_row("Name", self.Name) 42 | self._display.add_row("ObjectType", str(self.ObjectType)) 43 | if str(self.ObjectType) not in "Model": 44 | self._display.add_row("ParentName", self.Parent.Name) 45 | self._display.add_row( 46 | "ParentObjectType", 47 | str(self.Parent.ObjectType), 48 | end_section=True, 49 | ) 50 | 51 | def __rich_repr__(self) -> str: 52 | """See [Rich Repr](https://rich.readthedocs.io/en/stable/pretty.html#rich-repr-protocol).""" 53 | Console().print(self._display) 54 | 55 | def __getattr__(self, attr): 56 | """Searches in `self._object`.""" 57 | return getattr(self._object, attr) 58 | 59 | 60 | class PyObjects: 61 | """The main parent class for grouping your (Tables, Columns, Measures, Partitions, etc.). 62 | 63 | Notice the magic methods. `__rich_repr__()` starts the baseline for displaying your model. 64 | It uses the amazing `rich` python package and 65 | builds your display from the `self._display`. 66 | Still building out the magic methods to give `PyObjects` more flexibility. 67 | """ 68 | 69 | def __init__(self, objects: list[PyObject], parent=None) -> None: 70 | """Initialization of `PyObjects`. 71 | 72 | Takes the objects in something that is iterable. 73 | Then will build a default `rich` table display. 74 | 75 | Args: 76 | objects(list[PyObject]): .Net objects. 77 | parent: Parent Object. Defaults to `None`. 78 | """ 79 | self._objects = objects 80 | self.parent = parent 81 | self._display = Table(title=str(self.__class__.mro()[0])) 82 | for index, obj in enumerate(self._objects): 83 | self._display.add_row(str(index), obj.Name) 84 | 85 | def __rich_repr__(self) -> str: 86 | """See [Rich Repr](https://rich.readthedocs.io/en/stable/pretty.html#rich-repr-protocol).""" 87 | Console().print(self._display) 88 | 89 | def __getitem__(self, object): 90 | """Get item from `PyObjects`. 91 | 92 | Checks if item is str or int. 93 | If string will iterate through and try to find matching name. 94 | Otherwise, will call into `self._objects[int]` to retrieve item. 95 | """ 96 | if isinstance(object, str): 97 | return [pyobject for pyobject in self._objects if object == pyobject.Name][ 98 | -1 99 | ] 100 | elif isinstance(object, slice): 101 | cls = type(self) 102 | return cls(self._objects[object]) 103 | else: 104 | return self._objects[object] 105 | 106 | def __iter__(self): 107 | """Iterate through `PyObjects`.""" 108 | yield from self._objects 109 | 110 | def __len__(self) -> int: 111 | """Get length of `PyObjects`. 112 | 113 | Returns: 114 | int: Number of PyObject in PyObjects 115 | """ 116 | return len(self._objects) 117 | 118 | def __iadd__(self, obj): 119 | """Add a `PyObject` or `PyObjects` to your current `PyObjects` class. 120 | 121 | This is useful for building out a custom `PyObjects` class to work with. 122 | """ 123 | if isinstance(obj, Iterable): 124 | self._objects.__iadd__(obj._objects) 125 | else: 126 | self._objects.__iadd__([obj]) 127 | 128 | self.__init__(self._objects) 129 | return self 130 | 131 | def _first_visible_object(self): 132 | """Does what the method is called. Get's first `object.IsHidden is False`.""" 133 | for object in self: 134 | if object.IsHidden is False: 135 | return object 136 | return None 137 | 138 | def find(self, object_str: str): 139 | """Finds any or all `PyObject` inside of `PyObjects` that match the `object_str`. 140 | 141 | It is case insensitive. 142 | 143 | Args: 144 | object_str (str): str to lookup in `PyObjects` 145 | 146 | Returns: 147 | PyObjects (object.PyObjects): Returns a `PyObjects` class with all `PyObject` 148 | where the `PyObject.Name` matches `object_str`. 149 | """ 150 | items = [ 151 | object 152 | for object in self._objects 153 | if object_str.lower() in object.Name.lower() 154 | ] 155 | return self.__class__.mro()[0](items) 156 | 157 | def get(self, object_str: str, alt_result: str = "") -> str: 158 | """Gets the object based on str. 159 | 160 | If the object isnt found, then an alternate result 161 | can be supplied as an argument. 162 | 163 | Args: 164 | object_str (str): str to lookup object 165 | alt_result (str): str to return when value isn't found. 166 | 167 | Returns: 168 | str: Result of the lookup, or the alternate result. 169 | """ 170 | try: 171 | return self.__getitem__(object_str) 172 | except Exception as e: 173 | Console().print(e) 174 | 175 | return alt_result 176 | -------------------------------------------------------------------------------- /pytabular/partition.py: -------------------------------------------------------------------------------- 1 | """`partition.py` houses the main `PyPartition` and `PyPartitions` class. 2 | 3 | Once connected to your model, interacting with partition(s) will be done through these classes. 4 | """ 5 | 6 | import logging 7 | 8 | from pytabular.object import PyObject, PyObjects 9 | from logic_utils import ticks_to_datetime 10 | import pandas as pd 11 | from datetime import datetime 12 | 13 | logger = logging.getLogger("PyTabular") 14 | 15 | 16 | class PyPartition(PyObject): 17 | """Main class for interacting with partitions. 18 | 19 | See methods for available uses. 20 | """ 21 | 22 | def __init__(self, object, table) -> None: 23 | """Extends from `PyObject` class. 24 | 25 | Adds a few custom rows to `rich` table for the partition. 26 | 27 | Args: 28 | object (Partition): .Net Partition object. 29 | table (PyTable): Parent table of the partition in question. 30 | """ 31 | super().__init__(object) 32 | self.Table = table 33 | self._display.add_row("Mode", str(self._object.Mode)) 34 | self._display.add_row("State", str(self._object.State)) 35 | self._display.add_row( 36 | "SourceType", str(self._object.SourceType), end_section=True 37 | ) 38 | self._display.add_row( 39 | "RefreshedTime", self.last_refresh().strftime("%m/%d/%Y, %H:%M:%S") 40 | ) 41 | 42 | def last_refresh(self) -> datetime: 43 | """Queries `RefreshedTime` attribute in the partition. 44 | 45 | Converts from C# Ticks to Python datetime. 46 | 47 | Returns: 48 | datetime.datetime: Last Refreshed time of Partition in datetime format 49 | """ 50 | return ticks_to_datetime(self.RefreshedTime.Ticks) 51 | 52 | def refresh(self, *args, **kwargs) -> pd.DataFrame: 53 | """Same method from Model Refresh. 54 | 55 | You can pass through any extra parameters. For example: 56 | `Tabular().Tables['Table Name'].Partitions[0].refresh()` 57 | 58 | Returns: 59 | pd.DataFrame: Returns pandas dataframe with some refresh details 60 | """ 61 | return self.Table.Model.refresh(self, *args, **kwargs) 62 | 63 | 64 | class PyPartitions(PyObjects): 65 | """Groups together multiple partitions. 66 | 67 | See `PyObjects` class for what more it can do. 68 | You can interact with `PyPartitions` straight from model. 69 | For ex: `model.Partitions`. 70 | Or through individual tables `model.Tables[TABLE_NAME].Partitions`. 71 | You can even filter down with `.find()`. For example find partitions with `prev-year` in name. 72 | `model.Partitions.find('prev-year')`. 73 | """ 74 | 75 | def __init__(self, objects) -> None: 76 | """Extends through to `PyObjects`.""" 77 | super().__init__(objects) 78 | 79 | def refresh(self, *args, **kwargs): 80 | """Refreshes all `PyPartition`(s) in class.""" 81 | model = self[0].Table.Model 82 | return model.refresh(self, *args, **kwargs) 83 | -------------------------------------------------------------------------------- /pytabular/pbi_helper.py: -------------------------------------------------------------------------------- 1 | """`pbi_helper.py` was reverse engineered from DaxStudio `PowerBiHelper.cs`. 2 | 3 | So all credit and genius should go to DaxStudio. 4 | I just wanted it in python... 5 | The main function is `find_local_pbi_instances()`. 6 | It will find any open PBIX files on your computer and spit out a connection string for you. 7 | """ 8 | 9 | import pytabular as p 10 | import subprocess 11 | 12 | 13 | def get_msmdsrv() -> list: 14 | """Runs powershel command to retrieve the ProcessId. 15 | 16 | Uses `Get-CimInstance` where `Name == 'msmdsrv.exe'`. 17 | 18 | Returns: 19 | list: returns ProcessId(s) in list. 20 | Formatted to account for multiple PBIX files open at the same time. 21 | """ 22 | p.logger.debug("Retrieving msmdsrv.exe(s)") 23 | 24 | try: 25 | msmdsrv = subprocess.check_output( 26 | [ 27 | "powershell", 28 | """Get-CimInstance -ClassName Win32_Process \ 29 | -Property * -Filter "Name = 'msmdsrv.exe'" | \ 30 | Select-Object -Property ProcessId -ExpandProperty ProcessId""", 31 | ] 32 | ) 33 | 34 | msmdsrv_id = msmdsrv.decode().strip().splitlines() 35 | p.logger.debug(f"ProcessId for msmdsrv.exe {msmdsrv_id}") 36 | return msmdsrv_id 37 | 38 | except subprocess.CalledProcessError as e: 39 | p.logger.error( 40 | f"command '{e.cmd}' return with error (code {e.returncode}): {e.output}" 41 | ) 42 | p.logger.warn( 43 | "Check if powershell is availabe in the PATH environment variables." 44 | ) 45 | raise RuntimeError( 46 | f"command '{e.cmd}' return with error (code {e.returncode}): {e.output}" 47 | ) from e 48 | 49 | 50 | def get_port_number(msmdsrv: str) -> str: 51 | """Gets the local port number of given msmdsrv ProcessId. Via PowerShell. 52 | 53 | Args: 54 | msmdsrv (str): A ProcessId returned from `get_msmdsrv()`. 55 | 56 | Returns: 57 | str: `LocalPort` returned for specific ProcessId. 58 | """ 59 | port = subprocess.check_output( 60 | [ 61 | "powershell", 62 | f"Get-NetTCPConnection -State Listen \ 63 | -OwningProcess {msmdsrv} | Select-Object \ 64 | -Property LocalPort -First 1 -ExpandProperty LocalPort", 65 | ] 66 | ) 67 | port_number = port.decode().strip() 68 | p.logger.debug(f"Listening port - {port_number} for msmdsrv.exe - {msmdsrv}") 69 | return port_number 70 | 71 | 72 | def get_parent_id(msmdsrv: str) -> str: 73 | """Gets ParentProcessId via PowerShell from the msmdsrv ProcessId. 74 | 75 | Args: 76 | msmdsrv (str): A ProcessId returned from `get_msmdsrv()`. 77 | 78 | Returns: 79 | str: Returns ParentProcessId in `str` format. 80 | """ 81 | parent = subprocess.check_output( 82 | [ 83 | "powershell", 84 | f'Get-CimInstance -ClassName Win32_Process -Property * \ 85 | -Filter "ProcessId = {msmdsrv}" | \ 86 | Select-Object -Property ParentProcessId -ExpandProperty ParentProcessId', 87 | ] 88 | ) 89 | parent_id = parent.decode().strip() 90 | p.logger.debug(f"ProcessId - {parent_id} for parent of msmdsrv.exe - {msmdsrv}") 91 | return parent_id 92 | 93 | 94 | def get_parent_title(parent_id: str) -> str: 95 | """Takes the ParentProcessId and gets the name of the PBIX file. 96 | 97 | Args: 98 | parent_id (str): Takes ParentProcessId which can be retrieved from `get_parent_id(msmdsrv)` 99 | 100 | Returns: 101 | str: Returns str of title of PBIX file. 102 | """ 103 | pbi_title_suffixes: list = [ 104 | " \u002D Power BI Desktop", # Dash Punctuation - minus hyphen 105 | " \u2212 Power BI Desktop", # Math Symbol - minus sign 106 | " \u2011 Power BI Desktop", # Dash Punctuation - non-breaking hyphen 107 | " \u2013 Power BI Desktop", # Dash Punctuation - en dash 108 | " \u2014 Power BI Desktop", # Dash Punctuation - em dash 109 | " \u2015 Power BI Desktop", # Dash Punctuation - horizontal bar 110 | ] 111 | title = subprocess.check_output( 112 | ["powershell", f"""(Get-Process -Id {parent_id}).MainWindowTitle"""] 113 | ) 114 | title_name = title.decode().strip() 115 | for suffix in pbi_title_suffixes: 116 | title_name = title_name.replace(suffix, "") 117 | p.logger.debug(f"Title - {title_name} for {parent_id}") 118 | return title_name 119 | 120 | 121 | def create_connection_str(port_number: str) -> str: 122 | """This takes the port number and adds to connection string. 123 | 124 | This is pretty bland right now, may improve later. 125 | 126 | Args: 127 | port_number (str): port number retrieved from `get_port_number(msmdsrv)`. 128 | 129 | Returns: 130 | str: port number as string. 131 | """ 132 | connection_str = f"Data Source=localhost:{port_number}" 133 | p.logger.debug(f"Local Connection Str - {connection_str}") 134 | return connection_str 135 | 136 | 137 | def find_local_pbi_instances() -> list: 138 | """The real genius is from Dax Studio. 139 | 140 | I just wanted it in python not C#, so reverse engineered what DaxStudio did. 141 | It will run some powershell scripts to pull the appropriate info. 142 | Then will spit out a list with tuples inside. 143 | You can use the connection string to connect to your model with pytabular. 144 | [Dax Studio](https://github.com/DaxStudio/DaxStudio/blob/master/src/DaxStudio.UI/Utils/PowerBIHelper.cs). 145 | 146 | Returns: 147 | list: EX `[('PBI File Name1','localhost:{port}'),('PBI File Name2','localhost:{port}')]` 148 | """ # noqa: E501 149 | instances = get_msmdsrv() 150 | pbi_instances = [] 151 | for instance in instances: 152 | p.logger.debug(f"Building connection for {instance}") 153 | port = get_port_number(instance) 154 | parent = get_parent_id(instance) 155 | title = get_parent_title(parent) 156 | connect_str = create_connection_str(port) 157 | pbi_instances += [(title, connect_str)] 158 | return pbi_instances 159 | -------------------------------------------------------------------------------- /pytabular/pytabular.py: -------------------------------------------------------------------------------- 1 | """`pytabular.py` is where it all starts. 2 | 3 | Main class is `Tabular()`. Use that for connecting with your models. 4 | """ 5 | 6 | import logging 7 | 8 | from Microsoft.AnalysisServices.Tabular import ( 9 | Server, 10 | ColumnType, 11 | Table, 12 | DataColumn, 13 | Partition, 14 | MPartitionSource, 15 | ) 16 | 17 | from typing import List, Union 18 | from collections import namedtuple 19 | import pandas as pd 20 | import os 21 | import subprocess 22 | import atexit 23 | from logic_utils import ( 24 | pd_dataframe_to_m_expression, 25 | pandas_datatype_to_tabular_datatype, 26 | remove_suffix, 27 | remove_file, 28 | ) 29 | 30 | from pytabular.table import PyTable, PyTables 31 | from pytabular.partition import PyPartitions 32 | from pytabular.column import PyColumns 33 | from pytabular.measure import PyMeasures 34 | from pytabular.culture import PyCultures, PyCulture 35 | from pytabular.relationship import PyRelationship, PyRelationships 36 | from pytabular.object import PyObject 37 | from pytabular.refresh import PyRefresh 38 | from pytabular.query import Connection 39 | 40 | logger = logging.getLogger("PyTabular") 41 | 42 | 43 | class Tabular(PyObject): 44 | """This is the Tabular Class to perform operations. 45 | 46 | This is the main class to work with in PyTabular. 47 | You can connect to the other classes via the supplied attributes. 48 | 49 | Args: 50 | connection_str (str): Need a valid connection string: 51 | [link](https://learn.microsoft.com/en-us/analysis-services/instances/connection-string-properties-analysis-services) 52 | 53 | Attributes: 54 | Adomd (Connection): For querying. 55 | This is the `Connection` class. 56 | Tables (PyTables): See `PyTables` for more information. 57 | Iterate through your tables in your model. 58 | Columns (PyColumns): See `PyColumns` for more information. 59 | Partitions (PyPartitions): See `PyPartitions` for more information. 60 | Measures (PyMeasures): See `PyMeasures` for more information. 61 | PyRefresh (PyRefresh): See `PyRefresh` for more information. 62 | """ 63 | 64 | def __init__(self, connection_str: str): 65 | """Connect to model. Just supply a solid connection string.""" 66 | # Connecting to model... 67 | logger.debug("Initializing Tabular Class") 68 | self.Server = Server() 69 | self.Server.Connect(connection_str) 70 | logger.info(f"Connected to Server - {self.Server.Name}") 71 | self.Catalog = self.Server.ConnectionInfo.Catalog 72 | logger.debug(f"Received Catalog - {self.Catalog}") 73 | try: 74 | self.Database = [ 75 | database 76 | for database in self.Server.Databases.GetEnumerator() 77 | if database.Name == self.Catalog or self.Catalog is None 78 | ][0] 79 | except Exception: 80 | err_msg = f"Unable to find Database... {self.Catalog}" 81 | logger.error(err_msg) 82 | raise Exception(err_msg) 83 | logger.info(f"Connected to Database - {self.Database.Name}") 84 | self.CompatibilityLevel: int = self.Database.CompatibilityLevel 85 | self.CompatibilityMode: int = self.Database.CompatibilityMode.value__ 86 | self.Model = self.Database.Model 87 | logger.info(f"Connected to Model - {self.Model.Name}") 88 | self.Adomd: Connection = Connection(self.Server) 89 | self.effective_users: dict = {} 90 | self.PyRefresh: PyRefresh = PyRefresh 91 | 92 | # Build PyObjects 93 | self.reload_model_info() 94 | 95 | # Run subclass init 96 | super().__init__(self.Model) 97 | 98 | # Building rich table display for repr 99 | self._display.add_row( 100 | "EstimatedSize", 101 | f"{round(self.Database.EstimatedSize / 1000000000, 2)} GB", 102 | end_section=True, 103 | ) 104 | self._display.add_row("# of Tables", str(len(self.Tables))) 105 | self._display.add_row("# of Partitions", str(len(self.Partitions))) 106 | self._display.add_row("# of Columns", str(len(self.Columns))) 107 | self._display.add_row( 108 | "# of Measures", str(len(self.Measures)), end_section=True 109 | ) 110 | self._display.add_row("Database", self.Database.Name) 111 | self._display.add_row("Server", self.Server.Name) 112 | 113 | # Finished and registering disconnect 114 | logger.debug("Class Initialization Completed") 115 | logger.debug("Registering Disconnect on Termination...") 116 | atexit.register(self.disconnect) 117 | 118 | def reload_model_info(self) -> bool: 119 | """Reload your model info into the `Tabular` class. 120 | 121 | Should be called after any model changes. 122 | Called in `save_changes()` and `__init__()`. 123 | 124 | Returns: 125 | bool: True if successful 126 | """ 127 | self.Database.Refresh() 128 | 129 | self.Tables = PyTables( 130 | [PyTable(table, self) for table in self.Model.Tables.GetEnumerator()] 131 | ) 132 | self.Relationships = PyRelationships( 133 | [ 134 | PyRelationship(relationship, self) 135 | for relationship in self.Model.Relationships.GetEnumerator() 136 | ] 137 | ) 138 | self.Partitions = PyPartitions( 139 | [partition for table in self.Tables for partition in table.Partitions] 140 | ) 141 | self.Columns = PyColumns( 142 | [column for table in self.Tables for column in table.Columns] 143 | ) 144 | self.Measures = PyMeasures( 145 | [measure for table in self.Tables for measure in table.Measures], self 146 | ) 147 | 148 | self.Cultures = PyCultures( 149 | [ 150 | PyCulture(culture, self) 151 | for culture in self.Model.Cultures.GetEnumerator() 152 | ] 153 | ) 154 | return True 155 | 156 | def is_process(self) -> bool: 157 | """Run method to check if Processing is occurring. 158 | 159 | Will query DMV `$SYSTEM.DISCOVER_JOBS` 160 | to see if any processing is happening. 161 | 162 | Returns: 163 | bool: True if DMV shows Process, False if not. 164 | """ 165 | _jobs_df = self.query("select * from $SYSTEM.DISCOVER_JOBS") 166 | return len(_jobs_df[_jobs_df["JOB_DESCRIPTION"] == "Process"]) > 0 167 | 168 | def disconnect(self) -> None: 169 | """Disconnects from Model.""" 170 | logger.info(f"Disconnecting from - {self.Server.Name}") 171 | atexit.unregister(self.disconnect) 172 | return self.Server.Disconnect() 173 | 174 | def reconnect(self) -> None: 175 | """Reconnects to Model.""" 176 | logger.info(f"Reconnecting to {self.Server.Name}") 177 | return self.Server.Reconnect() 178 | 179 | def refresh(self, *args, **kwargs) -> pd.DataFrame: 180 | """`PyRefresh` class to handle refreshes of model. 181 | 182 | See the `PyRefresh()` class for more details on what you can do with this. 183 | """ 184 | return self.PyRefresh(self, *args, **kwargs).run() 185 | 186 | def save_changes(self): 187 | """Called after refreshes or any model changes. 188 | 189 | Currently will return a named tuple of all changes detected. 190 | A ton of room for improvement on what gets returned here. 191 | """ 192 | if self.Server.Connected is False: 193 | self.reconnect() 194 | 195 | def property_changes(property_changes_var): 196 | """Returns any property changes.""" 197 | property_change = namedtuple( 198 | "property_change", 199 | "new_value object original_value property_name property_type", 200 | ) 201 | return [ 202 | property_change( 203 | change.NewValue, 204 | change.Object, 205 | change.OriginalValue, 206 | change.PropertyName, 207 | change.PropertyType, 208 | ) 209 | for change in property_changes_var.GetEnumerator() 210 | ] 211 | 212 | logger.info("Executing save_changes()...") 213 | model_save_results = self.Model.SaveChanges() 214 | if isinstance(model_save_results.Impact, type(None)): 215 | logger.warning(f"No changes detected on save for {self.Server.Name}") 216 | return None 217 | else: 218 | property_changes_var = model_save_results.Impact.PropertyChanges 219 | added_objects = model_save_results.Impact.AddedObjects 220 | added_subtree_roots = model_save_results.Impact.AddedSubtreeRoots 221 | removed_objects = model_save_results.Impact.RemovedObjects 222 | removed_subtree_roots = model_save_results.Impact.RemovedSubtreeRoots 223 | xmla_results = model_save_results.XmlaResults 224 | changes = namedtuple( 225 | "changes", 226 | "property_changes added_objects added_subtree_roots \ 227 | removed_objects removed_subtree_Roots xmla_results", 228 | ) 229 | [ 230 | property_changes(property_changes_var), 231 | added_objects, 232 | added_subtree_roots, 233 | removed_objects, 234 | removed_subtree_roots, 235 | xmla_results, 236 | ] 237 | self.reload_model_info() 238 | return changes( 239 | property_changes(property_changes_var), 240 | added_objects, 241 | added_subtree_roots, 242 | removed_objects, 243 | removed_subtree_roots, 244 | xmla_results, 245 | ) 246 | 247 | def backup_table(self, table_str: str) -> bool: 248 | """This will be removed. 249 | 250 | Used in conjunction with `revert_table()`. 251 | """ 252 | logger.info("Backup Beginning...") 253 | logger.debug(f"Cloning {table_str}") 254 | table = self.Model.Tables.Find(table_str).Clone() 255 | logger.info("Beginning Renames") 256 | 257 | def rename(items): 258 | """Iterates through items and requests rename.""" 259 | for item in items: 260 | item.RequestRename(f"{item.Name}_backup") 261 | logger.debug(f"Renamed - {item.Name}") 262 | 263 | logger.info("Renaming Columns") 264 | rename(table.Columns.GetEnumerator()) 265 | logger.info("Renaming Partitions") 266 | rename(table.Partitions.GetEnumerator()) 267 | logger.info("Renaming Measures") 268 | rename(table.Measures.GetEnumerator()) 269 | logger.info("Renaming Hierarchies") 270 | rename(table.Hierarchies.GetEnumerator()) 271 | logger.info("Renaming Table") 272 | table.RequestRename(f"{table.Name}_backup") 273 | logger.info("Adding Table to Model as backup") 274 | self.Model.Tables.Add(table) 275 | logger.info("Finding Necessary Relationships... Cloning...") 276 | relationships = [ 277 | relationship.Clone() 278 | for relationship in self.Model.Relationships.GetEnumerator() 279 | if relationship.ToTable.Name == remove_suffix(table.Name, "_backup") 280 | or relationship.FromTable.Name == remove_suffix(table.Name, "_backup") 281 | ] 282 | logger.info("Renaming Relationships") 283 | rename(relationships) 284 | logger.info("Switching Relationships to Clone Table & Column") 285 | for relationship in relationships: 286 | logger.debug(f"Renaming - {relationship.Name}") 287 | if relationship.ToTable.Name == remove_suffix(table.Name, "_backup"): 288 | relationship.set_ToColumn( 289 | table.Columns.find(f"{relationship.ToColumn.Name}_backup") 290 | ) 291 | elif relationship.FromTable.Name == remove_suffix(table.Name, "_backup"): 292 | relationship.set_FromColumn( 293 | table.Columns.find(f"{relationship.FromColumn.Name}_backup") 294 | ) 295 | logger.debug(f"Adding {relationship.Name} to {self.Model.Name}") 296 | self.Model.Relationships.Add(relationship) 297 | 298 | def clone_role_permissions(): 299 | """Clones the role permissions for table.""" 300 | logger.info("Beginning to handle roles and permissions for table...") 301 | logger.debug("Finding Roles...") 302 | roles = [ 303 | role 304 | for role in self.Model.Roles.GetEnumerator() 305 | for tablepermission in role.TablePermissions.GetEnumerator() 306 | if tablepermission.Name == table_str 307 | ] 308 | for role in roles: 309 | logger.debug(f"Role {role.Name} matched, looking into it...") 310 | logger.debug("Searching for table specific permissions") 311 | tablepermissions = [ 312 | table.Clone() 313 | for table in role.TablePermissions.GetEnumerator() 314 | if table.Name == table_str 315 | ] 316 | for tablepermission in tablepermissions: 317 | logger.debug( 318 | f"{tablepermission.Name} found... switching table to clone" 319 | ) 320 | tablepermission.set_Table(table) 321 | for column in tablepermission.ColumnPermissions.GetEnumerator(): 322 | logger.debug( 323 | f"Column - {column.Name} copying permissions to clone..." 324 | ) 325 | column.set_Column( 326 | self.Model.Tables.find(table.Name).Columns.find( 327 | f"{column.Name}_backup" 328 | ) 329 | ) 330 | logger.debug(f"Adding {tablepermission.Name} to {role.Name}") 331 | role.TablePermissions.Add(tablepermission) 332 | return True 333 | 334 | clone_role_permissions() 335 | logger.info(f"Refreshing Clone... {table.Name}") 336 | self.reload_model_info() 337 | self.refresh(table.Name, default_row_count_check=False) 338 | logger.info(f"Updating Model {self.Model.Name}") 339 | self.save_changes() 340 | return True 341 | 342 | def revert_table(self, table_str: str) -> bool: 343 | """This will be removed. 344 | 345 | This is used in conjunction with `backup_table()`. 346 | """ 347 | logger.info(f"Beginning Revert for {table_str}") 348 | logger.debug(f"Finding original {table_str}") 349 | main = self.Tables.find(table_str)[0]._object 350 | logger.debug(f"Finding backup {table_str}") 351 | backup = self.Tables.find(f"{table_str}_backup")[0]._object 352 | logger.debug("Finding original relationships") 353 | main_relationships = [ 354 | relationship 355 | for relationship in self.Model.Relationships.GetEnumerator() 356 | if relationship.ToTable.Name == main.Name 357 | or relationship.FromTable.Name == main.Name 358 | ] 359 | logger.debug("Finding backup relationships") 360 | backup_relationships = [ 361 | relationship 362 | for relationship in self.Model.Relationships.GetEnumerator() 363 | if relationship.ToTable.Name == backup.Name 364 | or relationship.FromTable.Name == backup.Name 365 | ] 366 | 367 | def remove_role_permissions(): 368 | """Removes role permissions from table.""" 369 | logger.debug( 370 | f"Finding table and column permission in roles to remove from {table_str}" 371 | ) 372 | roles = [ 373 | role 374 | for role in self.Model.Roles.GetEnumerator() 375 | for tablepermission in role.TablePermissions.GetEnumerator() 376 | if tablepermission.Name == table_str 377 | ] 378 | for role in roles: 379 | logger.debug(f"Role {role.Name} Found") 380 | tablepermissions = [ 381 | table 382 | for table in role.TablePermissions.GetEnumerator() 383 | if table.Name == table_str 384 | ] 385 | for tablepermission in tablepermissions: 386 | logger.debug(f"Removing {tablepermission.Name} from {role.Name}") 387 | role.TablePermissions.Remove(tablepermission) 388 | 389 | for relationship in main_relationships: 390 | logger.debug("Cleaning relationships...") 391 | if relationship.ToTable.Name == main.Name: 392 | logger.debug(f"Removing {relationship.Name}") 393 | self.Model.Relationships.Remove(relationship) 394 | elif relationship.FromTable.Name == main.Name: 395 | logger.debug(f"Removing {relationship.Name}") 396 | self.Model.Relationships.Remove(relationship) 397 | logger.debug(f"Removing Original Table {main.Name}") 398 | self.Model.Tables.Remove(main) 399 | remove_role_permissions() 400 | 401 | def dename(items): 402 | """Denames all items.""" 403 | for item in items: 404 | logger.debug(f"Removing Suffix for {item.Name}") 405 | item.RequestRename(remove_suffix(item.Name, "_backup")) 406 | logger.debug(f"Saving Changes... for {item.Name}") 407 | self.save_changes() 408 | 409 | logger.info("Name changes for Columns...") 410 | dename( 411 | [ 412 | column 413 | for column in backup.Columns.GetEnumerator() 414 | if column.Type != ColumnType.RowNumber 415 | ] 416 | ) 417 | logger.info("Name changes for Partitions...") 418 | dename(backup.Partitions.GetEnumerator()) 419 | logger.info("Name changes for Measures...") 420 | dename(backup.Measures.GetEnumerator()) 421 | logger.info("Name changes for Hierarchies...") 422 | dename(backup.Hierarchies.GetEnumerator()) 423 | logger.info("Name changes for Relationships...") 424 | dename(backup_relationships) 425 | logger.info("Name changes for Backup Table...") 426 | backup.RequestRename(remove_suffix(backup.Name, "_backup")) 427 | self.save_changes() 428 | return True 429 | 430 | def query( 431 | self, query_str: str, effective_user: str = None 432 | ) -> Union[pd.DataFrame, str, int]: 433 | """Executes a query on model. 434 | 435 | See `Connection().query()` for details on execution. 436 | 437 | Args: 438 | query_str (str): Query string to execute. 439 | effective_user (str, optional): Pass through an effective user 440 | if desired. It will create and store a new `Connection()` class if need, 441 | which will help with speed if looping through multiple users in a row. 442 | Defaults to None. 443 | 444 | Returns: 445 | Union[pd.DataFrame, str, int]: Depending on query, will return DataFrame 446 | or single value. 447 | Example: 448 | ```python 449 | model.query("EVALUATE {1}") 450 | 451 | model.query("EVALUATE TOPN(5, 'Customer')") 452 | 453 | model.query( 454 | "EVALUATE VALUES('Sales Region'[Region])", 455 | effective_user = "user@company.com" 456 | ) 457 | ``` 458 | """ 459 | if effective_user is None: 460 | return self.Adomd.query(query_str) 461 | 462 | try: 463 | # This needs a public model with effective users to properly test 464 | conn = self.effective_users[effective_user] 465 | logger.debug(f"Effective user found querying as... {effective_user}") 466 | except Exception: 467 | logger.info(f"Creating new connection with {effective_user}") 468 | conn = Connection(self.Server, effective_user=effective_user) 469 | self.effective_users[effective_user] = conn 470 | 471 | return conn.query(query_str) 472 | 473 | def analyze_bpa( 474 | self, tabular_editor_exe: str, best_practice_analyzer: str 475 | ) -> List[str]: 476 | """Takes your Tabular Model and performs TE2s BPA. Runs through Command line. 477 | 478 | Nothing fancy hear. Really just a simple wrapper so you could 479 | call BPA in the same python script. 480 | 481 | Args: 482 | tabular_editor_exe (str): TE2 Exe File path. 483 | Feel free to use class TE2().EXE_Path or provide your own. 484 | best_practice_analyzer (str): BPA json file path. 485 | Feel free to use class BPA().Location or provide your own. 486 | 487 | Returns: 488 | List[str]: Assuming no failure, 489 | will return list of BPA violations. 490 | Else will return error from command line. 491 | """ 492 | logger.debug("Beginning request to talk with TE2 & Find BPA...") 493 | bim_file_location = f"{os.getcwd()}\\Model.bim" 494 | atexit.register(remove_file, bim_file_location) 495 | cmd = f'"{tabular_editor_exe}" "Provider=MSOLAP;\ 496 | {self.Adomd.ConnectionString}" {self.Database.Name} -B "{bim_file_location}" \ 497 | -A {best_practice_analyzer} -V/?' 498 | logger.debug("Command Generated") 499 | logger.debug("Submitting Command...") 500 | sp = subprocess.Popen( 501 | cmd, 502 | shell=True, 503 | stdout=subprocess.PIPE, 504 | stderr=subprocess.PIPE, 505 | universal_newlines=True, 506 | ) 507 | raw_output, error = sp.communicate() 508 | if len(error) > 0: 509 | return error 510 | else: 511 | return [ 512 | output for output in raw_output.split("\n") if "violates rule" in output 513 | ] 514 | 515 | def create_table(self, df: pd.DataFrame, table_name: str) -> bool: 516 | """Creates table from pd.DataFrame to a table in your model. 517 | 518 | It will convert the dataframe to M-Partition logic via the M query table constructor. 519 | Then will run a refresh and update model. 520 | Has some obvious limitations right now, because 521 | the datframe values are hard coded into M-Partition, 522 | which means you could hit limits with the size of your table. 523 | 524 | Args: 525 | df (pd.DataFrame): DataFrame to add to model. 526 | table_name (str): Name of the table. 527 | 528 | Returns: 529 | bool: True if successful 530 | """ 531 | logger.debug(f"Beginning to create table for {table_name}...") 532 | new_table = Table() 533 | new_table.RequestRename(table_name) 534 | logger.debug("Sorting through columns...") 535 | df_column_names = df.columns 536 | dtype_conversion = pandas_datatype_to_tabular_datatype(df) 537 | for df_column_name in df_column_names: 538 | logger.debug(f"Adding {df_column_name} to Table...") 539 | column = DataColumn() 540 | column.RequestRename(df_column_name) 541 | column.set_SourceColumn(df_column_name) 542 | column.set_DataType(dtype_conversion[df_column_name]) 543 | new_table.Columns.Add(column) 544 | logger.debug("Expression String Created...") 545 | logger.debug("Creating MPartition...") 546 | partition = Partition() 547 | partition.set_Source(MPartitionSource()) 548 | logger.debug("Setting MPartition Expression...") 549 | partition.Source.set_Expression(pd_dataframe_to_m_expression(df)) 550 | logger.debug( 551 | f"Adding partition: {partition.Name} to {self.Server.Name}\ 552 | ::{self.Database.Name}::{self.Model.Name}" 553 | ) 554 | new_table.Partitions.Add(partition) 555 | logger.debug( 556 | f"Adding table: {new_table.Name} to {self.Server.Name}\ 557 | ::{self.Database.Name}::{self.Model.Name}" 558 | ) 559 | self.Model.Tables.Add(new_table) 560 | self.save_changes() 561 | self.reload_model_info() 562 | self.refresh(new_table.Name) 563 | return True 564 | -------------------------------------------------------------------------------- /pytabular/query.py: -------------------------------------------------------------------------------- 1 | """`query.py` houses a custom `Connection` class that uses the .Net AdomdConnection. 2 | 3 | `Connection` is created automatically when connecting to your model. 4 | 5 | Example: 6 | ```python title="query from model" 7 | import pytabular as p 8 | model = p.Tabular(CONNECTION_STR) 9 | model.query("EVALUATE {1}") 10 | ``` 11 | 12 | ```python title="pass an effective user" 13 | model.query( 14 | "EVALUATE {1}", 15 | effective_user = "user@company.com" 16 | ) 17 | ``` 18 | """ 19 | 20 | import logging 21 | import os 22 | from typing import Union 23 | from pytabular.logic_utils import get_value_to_df 24 | import pandas as pd 25 | from Microsoft.AnalysisServices.AdomdClient import AdomdCommand, AdomdConnection 26 | 27 | 28 | logger = logging.getLogger("PyTabular") 29 | 30 | 31 | class Connection(AdomdConnection): 32 | """Connection class creates an AdomdConnection. 33 | 34 | Mainly used for the `query()` method. The `query()` 35 | method in the Tabular class is just a wrapper for this class. 36 | But you can pass through your `effective_user` more efficiently, 37 | so use that instead. 38 | """ 39 | 40 | def __init__(self, server, effective_user=None) -> None: 41 | """Init creates the connection. 42 | 43 | Args: 44 | server (Server): The server that you are connecting to. 45 | effective_user (str, optional): Pass through an effective user 46 | to query as somebody else. Defaults to None. 47 | """ 48 | super().__init__() 49 | if server.ConnectionInfo.Password is None: 50 | connection_string = server.ConnectionString 51 | else: 52 | connection_string = ( 53 | f"{server.ConnectionString};Password='{server.ConnectionInfo.Password}'" 54 | ) 55 | logger.debug(f"ADOMD Connection: {connection_string}") 56 | if effective_user is not None: 57 | connection_string += f";EffectiveUserName={effective_user}" 58 | self.ConnectionString = connection_string 59 | 60 | def query(self, query_str: str) -> Union[pd.DataFrame, str, int]: 61 | """Executes query on Model and returns results in Pandas DataFrame. 62 | 63 | Iterates through results of `AdomdCommmand().ExecuteReader()` 64 | in the .Net library. If result is a single value, it will 65 | return that single value instead of DataFrame. 66 | 67 | Args: 68 | query_str (str): Dax Query. Note, needs full syntax 69 | (ex: `EVALUATE`). See [DAX](https://docs.microsoft.com/en-us/dax/dax-queries). 70 | Will check if query string is a file. 71 | If it is, then it will perform a query 72 | on whatever is read from the file. 73 | It is also possible to query DMV. 74 | For example. 75 | `query("select * from $SYSTEM.DISCOVER_TRACE_EVENT_CATEGORIES")`. 76 | 77 | Returns: 78 | pd.DataFrame: Returns dataframe with results. 79 | """ 80 | try: 81 | is_file = os.path.isfile(query_str) 82 | except Exception: 83 | is_file = False 84 | 85 | if is_file: 86 | logger.debug( 87 | f"File path detected, reading file... -> {query_str}", 88 | ) 89 | with open(query_str, "r") as file: 90 | query_str = str(file.read()) 91 | 92 | if str(self.get_State()) != "Open": 93 | # Works for now, need to update to handle different types of conneciton properties 94 | # https://learn.microsoft.com/en-us/dotnet/api/system.data.connectionstate?view=net-7.0 95 | logger.info("Checking initial Adomd Connection...") 96 | self.Open() 97 | logger.info(f"Connected! Session ID - {self.SessionID}") 98 | 99 | logger.debug("Querying Model...") 100 | logger.debug(query_str) 101 | query = AdomdCommand(query_str, self).ExecuteReader() 102 | column_headers = [ 103 | (index, query.GetName(index)) for index in range(0, query.FieldCount) 104 | ] 105 | results = list() 106 | while query.Read(): 107 | results.append( 108 | [ 109 | get_value_to_df(query, index) 110 | for index in range(0, len(column_headers)) 111 | ] 112 | ) 113 | 114 | query.Close() 115 | logger.debug("Data retrieved... reading...") 116 | df = pd.DataFrame(results, columns=[value for _, value in column_headers]) 117 | if len(df) == 1 and len(df.columns) == 1: 118 | return df.iloc[0][df.columns[0]] 119 | return df 120 | -------------------------------------------------------------------------------- /pytabular/refresh.py: -------------------------------------------------------------------------------- 1 | """`refresh.py` is the main file to handle all the components of refreshing your model. 2 | 3 | You have may ways to interact with refreshing. 4 | 5 | Example: 6 | ```python title="refresh from model" 7 | import pytabular as p 8 | model = p.Tabular(CONNECTION_STR) 9 | model.refresh('Table Name') 10 | ``` 11 | 12 | ```python title="refresh from PyTables" 13 | model.Tables.find("fact").refresh() # (1) 14 | ``` 15 | 16 | 1. Refresh all tables with 'fact' in the name. 17 | 18 | ```python title="refresh from PyTable" 19 | model.Tables['Sales'].refresh() 20 | ``` 21 | 22 | ```python title="refresh from PyPartitions" 23 | model.Tables['Large Sales Fact'].Partitions.refresh() 24 | ``` 25 | 26 | ```python title="refresh from PyPartition" 27 | model.Tables['Sales'].Partitions['Last Fiscal Year'].refresh() 28 | ``` 29 | """ 30 | 31 | from tabular_tracing import RefreshTrace, BaseTrace 32 | import logging 33 | from Microsoft.AnalysisServices.Tabular import ( 34 | RefreshType, 35 | Table, 36 | Partition, 37 | ) 38 | import pandas as pd 39 | from logic_utils import ticks_to_datetime 40 | from typing import Union, Dict, Any 41 | from pytabular.table import PyTable, PyTables 42 | from pytabular.partition import PyPartition 43 | from abc import ABC 44 | 45 | logger = logging.getLogger("PyTabular") 46 | 47 | 48 | class RefreshCheck(ABC): 49 | """`RefreshCheck` is an test you run after your refreshes. 50 | 51 | It will run the given `function` before and after refreshes, 52 | then run the assertion of before and after. 53 | The default given in a refresh is to check row count. 54 | It will check row count before, and row count after. 55 | Then fail if row count after is zero. 56 | """ 57 | 58 | def __init__(self, name: str, function, assertion=None) -> None: 59 | """Sets the necessary components to perform a refresh check. 60 | 61 | Args: 62 | name (str): Name of refresh check. 63 | function (Callable): Function to run on pre and post checks. 64 | For example, a dax query. readme has examples of this. 65 | assertion (Callable, optional): A function that can be run. 66 | Supply the assertion function with 2 arguments. The first one 67 | for your 'pre' results from the `function` argument. The second 68 | for your `post` results from the`function` argument. 69 | Return `True` or `False` depending on the comparison of the two arguments 70 | to determine a pass or fail status of your refresh. 71 | Defaults to None. 72 | """ 73 | super().__init__() 74 | self._name = name 75 | self._function = function 76 | self._assertion = assertion 77 | self._pre = None 78 | self._post = None 79 | 80 | def __repr__(self) -> str: 81 | """`__repre__` that returns details on `RefreshCheck`.""" 82 | return f"{self.name} - {self.pre} - {self.post} - {str(self.function)}" 83 | 84 | @property 85 | def name(self): 86 | """Get your custom name of refresh check.""" 87 | return self._name 88 | 89 | @name.setter 90 | def name(self, name): 91 | self._name = name 92 | 93 | @name.deleter 94 | def name(self): 95 | del self._name 96 | 97 | @property 98 | def function(self): 99 | """Get the function that is used to run a pre and post check.""" 100 | return self._function 101 | 102 | @function.setter 103 | def function(self, func): 104 | self._function = func 105 | 106 | @function.deleter 107 | def function(self): 108 | del self._function 109 | 110 | @property 111 | def pre(self): 112 | """Get the pre value that is the result from the pre refresh check.""" 113 | return self._pre 114 | 115 | @pre.setter 116 | def pre(self, pre): 117 | self._pre = pre 118 | 119 | @pre.deleter 120 | def pre(self): 121 | del self._pre 122 | 123 | @property 124 | def post(self): 125 | """Get the post value that is the result from the post refresh check.""" 126 | return self._post 127 | 128 | @post.setter 129 | def post(self, post): 130 | self._post = post 131 | 132 | @post.deleter 133 | def post(self): 134 | del self._post 135 | 136 | @property 137 | def assertion(self): 138 | """Get the assertion that is the result from the post refresh check.""" 139 | return self._assertion 140 | 141 | @assertion.setter 142 | def assertion(self, assertion): 143 | self._assertion = assertion 144 | 145 | @assertion.deleter 146 | def assertion(self): 147 | del self._assertion 148 | 149 | def _check(self, stage: str): 150 | """Runs the given function and stores results. 151 | 152 | Stored in either `self.pre` or `self.post` depending on `stage`. 153 | 154 | Args: 155 | stage (str): Either 'Pre' or 'Post' 156 | 157 | Returns: 158 | object: Returns the results of the pre or post check. 159 | """ 160 | logger.debug(f"Running {stage}-Check for {self.name}") 161 | results = self.function() 162 | if stage == "Pre": 163 | self.pre = results 164 | else: 165 | self.post = results 166 | logger.info(f"{stage}-Check results for {self.name} - {results}") 167 | return results 168 | 169 | def pre_check(self): 170 | """Runs `self._check("Pre")`.""" 171 | self._check("Pre") 172 | pass 173 | 174 | def post_check(self): 175 | """Runs `self._check("Post")` then `self.assertion_run()`.""" 176 | self._check("Post") 177 | self.assertion_run() 178 | pass 179 | 180 | def assertion_run(self): 181 | """Runs the given self.assertion function with `self.pre` and `self.post`. 182 | 183 | So, `self.assertion_run(self.pre, self.post)`. 184 | """ 185 | if self.assertion is None: 186 | logger.debug("Skipping assertion none given") 187 | else: 188 | test = self.assertion(self.pre, self.post) 189 | assert_str = f"Test {self.name} - {test} - Pre Results - {self.pre} | Post Results {self.post}" # noqa: E501 190 | if test: 191 | logger.info(assert_str) 192 | else: 193 | logger.critical(assert_str) 194 | assert ( 195 | test 196 | ), f"Test failed! Pre Results - {self.pre} | Post Results {self.post}" 197 | 198 | 199 | class RefreshCheckCollection: 200 | """Groups together your `RefreshChecks`. 201 | 202 | Used to handle multiple types of checks in a single refresh. 203 | """ 204 | 205 | def __init__(self, refresh_checks: RefreshCheck = []) -> None: 206 | """Init to supply RefreshChecks. 207 | 208 | Args: 209 | refresh_checks (RefreshCheck, optional): Defaults to []. 210 | """ 211 | self._refreshchecks = refresh_checks 212 | pass 213 | 214 | def __iter__(self): 215 | """Basic iteration through the different `RefreshCheck`(s).""" 216 | for refresh_check in self._refreshchecks: 217 | yield refresh_check 218 | 219 | def add_refresh_check(self, refresh_check: RefreshCheck): 220 | """Add a RefreshCheck. 221 | 222 | Supply the `RefreshCheck` to add. 223 | 224 | Args: 225 | refresh_check (RefreshCheck): `RefreshCheck` class. 226 | """ 227 | self._refreshchecks.append(refresh_check) 228 | 229 | def remove_refresh_check(self, refresh_check: RefreshCheck): 230 | """Remove a RefreshCheck. 231 | 232 | Supply the `RefreshCheck` to remove. 233 | 234 | Args: 235 | refresh_check (RefreshCheck): `RefreshCheck` class. 236 | """ 237 | self._refreshchecks.remove(refresh_check) 238 | 239 | def clear_refresh_checks(self): 240 | """Clear Refresh Checks.""" 241 | self._refreshchecks.clear() 242 | 243 | 244 | class PyRefresh: 245 | """PyRefresh Class to handle refreshes of model.""" 246 | 247 | def __init__( 248 | self, 249 | model, 250 | object: Union[str, PyTable, PyPartition, Dict[str, Any]], 251 | trace: BaseTrace = RefreshTrace, 252 | refresh_checks: RefreshCheckCollection = RefreshCheckCollection(), 253 | default_row_count_check: bool = True, 254 | refresh_type: RefreshType = RefreshType.Full, 255 | ) -> None: 256 | """Init when a refresh is requested. 257 | 258 | Runs through requested tables and partitions 259 | to make sure they are in model. 260 | Then will run pre checks on the requested objects. 261 | 262 | Args: 263 | model (Tabular): Tabular model. 264 | object (Union[str, PyTable, PyPartition, Dict[str, Any]]): The objects 265 | that you are wanting to refresh. Can be a `PyTable`, `PyPartition`, 266 | `TABLE_NAME` string, or a dict with `{TABLE_REFERENCE:PARTITION_REFERENCE}` 267 | trace (BaseTrace, optional): Defaults to RefreshTrace. 268 | refresh_checks (RefreshCheckCollection, optional): Defaults to RefreshCheckCollection(). 269 | default_row_count_check (bool, optional): Defaults to True. 270 | refresh_type (RefreshType, optional): Defaults to RefreshType.Full. 271 | """ 272 | self.model = model 273 | self.object = object 274 | self.trace = trace 275 | self.default_row_count_check = default_row_count_check 276 | self.refresh_type = refresh_type 277 | self._objects_to_refresh = [] 278 | self._request_refresh(self.object) 279 | self._checks = refresh_checks 280 | self._pre_checks() 281 | logger.info("Refresh Request Completed!") 282 | pass 283 | 284 | def _pre_checks(self): 285 | """Checks if any `BaseTrace` classes are needed from `tabular_tracing.py`. 286 | 287 | Then checks if any `RefreshChecks` are needed, along with the default `row_count` check. 288 | """ 289 | logger.debug("Running Pre-checks") 290 | if self.trace is not None: 291 | logger.debug("Getting Trace") 292 | self.trace = self._get_trace() 293 | if self.default_row_count_check: 294 | logger.debug( 295 | f"Running default row count check - {self.default_row_count_check}" 296 | ) 297 | tables = [ 298 | table 299 | for refresh_dict in self._objects_to_refresh 300 | for table in refresh_dict.keys() 301 | ] 302 | 303 | def row_count_assertion(pre, post): 304 | """Checks if table refreshed zero rows.""" 305 | post = 0 if post is None else post 306 | return post > 0 307 | 308 | for table in set(tables): 309 | check = RefreshCheck( 310 | f"{table.Name} Row Count", table.row_count, row_count_assertion 311 | ) 312 | self._checks.add_refresh_check(check) 313 | for check in self._checks: 314 | check.pre_check() 315 | pass 316 | 317 | def _post_checks(self): 318 | """If traces are running it Stops and Drops it. 319 | 320 | Runs through any `post_checks()` in `RefreshChecks`. 321 | """ 322 | if self.trace is not None: 323 | self.trace.stop() 324 | self.trace.drop() 325 | for check in self._checks: 326 | check.post_check() 327 | # self._checks.remove_refresh_check(check) 328 | self._checks.clear_refresh_checks() 329 | pass 330 | 331 | def _get_trace(self) -> BaseTrace: 332 | """Creates Trace and creates it in model.""" 333 | return self.trace(self.model) 334 | 335 | def _find_table(self, table_str: str) -> Table: 336 | """Finds table in `PyTables` class.""" 337 | try: 338 | result = self.model.Tables[table_str] 339 | except Exception: 340 | raise Exception(f"Unable to find table! from {table_str}") 341 | logger.debug(f"Found table {result.Name}") 342 | return result 343 | 344 | def _find_partition(self, table: Table, partition_str: str) -> Partition: 345 | """Finds partition in `PyPartitions` class.""" 346 | try: 347 | result = table.Partitions[partition_str] 348 | except Exception: 349 | raise Exception(f"Unable to find partition! {table.Name}|{partition_str}") 350 | logger.debug(f"Found partition {result.Table.Name}|{result.Name}") 351 | return result 352 | 353 | def _refresh_table(self, table: PyTable) -> None: 354 | """Runs .Net `RequestRefresh()` on table.""" 355 | logging.info(f"Requesting refresh for {table.Name}") 356 | self._objects_to_refresh += [ 357 | {table: [partition for partition in table.Partitions]} 358 | ] 359 | table.RequestRefresh(self.refresh_type) 360 | 361 | def _refresh_partition(self, partition: PyPartition) -> None: 362 | """Runs .Net `RequestRefresh()` on partition.""" 363 | logging.info(f"Requesting refresh for {partition.Table.Name}|{partition.Name}") 364 | self._objects_to_refresh += [{partition.Table: [partition]}] 365 | partition.RequestRefresh(self.refresh_type) 366 | 367 | def _refresh_dict(self, partition_dict: Dict) -> None: 368 | """Handles refreshes if argument given was a dictionary.""" 369 | for table in partition_dict.keys(): 370 | table_object = self._find_table(table) if isinstance(table, str) else table 371 | 372 | def handle_partitions(object): 373 | """Figures out if partition argument given is a str or an actual `PyPartition`. 374 | 375 | Then will run `self._refresh_partition()` appropriately. 376 | """ 377 | if isinstance(object, str): 378 | self._refresh_partition(self._find_partition(table_object, object)) 379 | elif isinstance(object, PyPartition): 380 | self._refresh_partition(object) 381 | else: 382 | [handle_partitions(obj) for obj in object] 383 | 384 | handle_partitions(partition_dict[table]) 385 | 386 | def _request_refresh(self, object): 387 | """Base method to parse through argument and figure out what needs to be refreshed. 388 | 389 | Someone please make this better... 390 | """ 391 | logger.debug(f"Requesting Refresh for {object}") 392 | if isinstance(object, str): 393 | self._refresh_table(self._find_table(object)) 394 | elif isinstance(object, PyTables): 395 | [self._refresh_table(table) for table in object] 396 | elif isinstance(object, Dict): 397 | self._refresh_dict(object) 398 | elif isinstance(object, PyTable): 399 | self._refresh_table(object) 400 | elif isinstance(object, PyPartition): 401 | self._refresh_partition(object) 402 | else: 403 | [self._request_refresh(obj) for obj in object] 404 | 405 | def _refresh_report(self, property_changes) -> pd.DataFrame: 406 | """Builds a DataFrame that displays details on the refresh. 407 | 408 | Args: 409 | Property_Changes: Which is returned from `model.save_changes()` 410 | 411 | Returns: 412 | pd.DataFrame: DataFrame of refresh details. 413 | """ 414 | logger.debug("Running Refresh Report...") 415 | refresh_data = [] 416 | for property_change in property_changes: 417 | if ( 418 | isinstance(property_change.object, Partition) 419 | and property_change.property_name == "RefreshedTime" 420 | ): 421 | table, partition, refreshed_time = ( 422 | property_change.object.Table.Name, 423 | property_change.object.Name, 424 | ticks_to_datetime(property_change.new_value.Ticks), 425 | ) 426 | logger.info( 427 | f'{table} - {partition} Refreshed! - {refreshed_time.strftime("%m/%d/%Y, %H:%M:%S")}' # noqa: E501 428 | ) 429 | refresh_data += [[table, partition, refreshed_time]] 430 | return pd.DataFrame( 431 | refresh_data, columns=["Table", "Partition", "Refreshed Time"] 432 | ) 433 | 434 | def run(self) -> pd.DataFrame: 435 | """When ready, execute to start the refresh process. 436 | 437 | First checks if connected and reconnects if needed. 438 | Then starts the trace if needed. 439 | Next will execute `save_changes()` 440 | and run the post checks after that. 441 | Last will return a `pd.DataFrame` of refresh results. 442 | """ 443 | if self.model.Server.Connected is False: 444 | logger.info(f"{self.Server.Name} - Reconnecting...") 445 | self.model.reconnect() 446 | 447 | if self.trace is not None: 448 | self.trace.start() 449 | 450 | save_changes = self.model.save_changes() 451 | 452 | self._post_checks() 453 | 454 | return self._refresh_report(save_changes.property_changes) 455 | -------------------------------------------------------------------------------- /pytabular/relationship.py: -------------------------------------------------------------------------------- 1 | """`relationship.py` houses the main `PyRelationship` and `PyRelationships` class. 2 | 3 | Once connected to your model, interacting with relationship(s) 4 | will be done through these classes. 5 | """ 6 | 7 | import logging 8 | from pytabular.object import PyObject, PyObjects 9 | from pytabular.table import PyTable, PyTables 10 | 11 | from Microsoft.AnalysisServices.Tabular import ( 12 | CrossFilteringBehavior, 13 | SecurityFilteringBehavior, 14 | ) 15 | 16 | from typing import Union 17 | 18 | logger = logging.getLogger("PyTabular") 19 | 20 | 21 | class PyRelationship(PyObject): 22 | """The main class for interacting with relationships in your model.""" 23 | 24 | def __init__(self, object, model) -> None: 25 | """Init extends to `PyObject`. 26 | 27 | It also extends a few unique rows to `rich` table. 28 | A few easy access attributes have been added. 29 | For example, see `self.From_Table` or `self.To_Column` 30 | 31 | Args: 32 | object (_type_): _description_ 33 | model (_type_): _description_ 34 | """ 35 | super().__init__(object) 36 | self.Model = model 37 | self.CrossFilteringBehavior = CrossFilteringBehavior( 38 | self.CrossFilteringBehavior.value__ 39 | ).ToString() 40 | self.SecurityFilteringBehavior = SecurityFilteringBehavior( 41 | self.SecurityFilteringBehavior.value__ 42 | ).ToString() 43 | self.To_Table = self.Model.Tables[self.ToTable.Name] 44 | self.To_Column = self.To_Table.Columns[self.ToColumn.Name] 45 | self.From_Table = self.Model.Tables[self.FromTable.Name] 46 | self.From_Column = self.From_Table.Columns[self.FromColumn.Name] 47 | self._display.add_row("Is Active", str(self.IsActive)) 48 | self._display.add_row("Cross Filtering Behavior", self.CrossFilteringBehavior) 49 | self._display.add_row( 50 | "Security Filtering Behavior", self.SecurityFilteringBehavior 51 | ) 52 | self._display.add_row( 53 | "From", f"'{self.From_Table.Name}'[{self.From_Column.Name}]" 54 | ) 55 | self._display.add_row("To", f"'{self.To_Table.Name}'[{self.To_Column.Name}]") 56 | 57 | 58 | class PyRelationships(PyObjects): 59 | """Groups together multiple relationships. 60 | 61 | See `PyObjects` class for what more it can do. 62 | You can interact with `PyRelationships` straight from model. 63 | For ex: `model.Relationships`. 64 | """ 65 | 66 | def __init__(self, objects) -> None: 67 | """Init just extends from PyObjects.""" 68 | super().__init__(objects) 69 | 70 | def related(self, object: Union[PyTable, str]) -> PyTables: 71 | """Finds related tables of a given table. 72 | 73 | Args: 74 | object (Union[PyTable, str]): `PyTable` or str of table name to find related tables for. 75 | 76 | Returns: 77 | PyTables: Returns `PyTables` class of the tables in question. 78 | """ 79 | table_to_find = object if isinstance(object, str) else object.Name 80 | to_tables = [ 81 | rel.To_Table 82 | for rel in self._objects 83 | if rel.From_Table.Name == table_to_find 84 | ] 85 | from_tables = [ 86 | rel.From_Table 87 | for rel in self._objects 88 | if rel.To_Table.Name == table_to_find 89 | ] 90 | to_tables += from_tables 91 | return PyTables(to_tables) 92 | -------------------------------------------------------------------------------- /pytabular/table.py: -------------------------------------------------------------------------------- 1 | """`table.py` houses the main `PyTable` and `PyTables` class. 2 | 3 | Once connected to your model, interacting with table(s) will be done through these classes. 4 | """ 5 | 6 | import logging 7 | import pandas as pd 8 | from pytabular.partition import PyPartition, PyPartitions 9 | from pytabular.column import PyColumn, PyColumns 10 | from pytabular.measure import PyMeasure, PyMeasures 11 | from pytabular.object import PyObjects, PyObject 12 | from logic_utils import ticks_to_datetime 13 | from datetime import datetime 14 | 15 | logger = logging.getLogger("PyTabular") 16 | 17 | 18 | class PyTable(PyObject): 19 | """The main PyTable class to interact with the tables in model. 20 | 21 | Attributes: 22 | Name (str): Name of table. 23 | IsHidden (bool): Is the table hidden. 24 | Description (str): The description of the table. 25 | Model (Tabular): The parent `Tabular()` class. 26 | Partitions (PyPartitions): The `PyPartitions()` in the table. 27 | Columns (PyColumns): The `PyColumns()` in the table. 28 | Measures (PyMeasures): The `PyMeasures()` in the table. 29 | 30 | Example: 31 | ```python title="Passing through PyTable to PyPartition" 32 | 33 | model.Tables[0].Partitions['Last Year'].refresh() # (1) 34 | ``` 35 | 36 | 1. This shows the ability to travel through your model 37 | to a specific partition and then running a refresh 38 | for that specific partition. 39 | `model` -> `PyTables` -> `PyTable` (1st index) -> `PyPartitions` 40 | -> `PyPartition` (.Name == 'Last Year') -> `.refresh()` 41 | """ 42 | 43 | def __init__(self, object, model) -> None: 44 | """Init extends from `PyObject` class. 45 | 46 | Also adds a few specific rows to the `rich` 47 | table. 48 | 49 | Args: 50 | object (Table): The actual .Net table. 51 | model (Tabular): The model that the table is in. 52 | """ 53 | super().__init__(object) 54 | self.Model = model 55 | self.Partitions: PyPartitions = PyPartitions( 56 | [ 57 | PyPartition(partition, self) 58 | for partition in self._object.Partitions.GetEnumerator() 59 | ] 60 | ) 61 | self.Columns: PyColumns = PyColumns( 62 | [PyColumn(column, self) for column in self._object.Columns.GetEnumerator()] 63 | ) 64 | self.Measures: PyMeasures = PyMeasures( 65 | [ 66 | PyMeasure(measure, self) 67 | for measure in self._object.Measures.GetEnumerator() 68 | ], 69 | self, 70 | ) 71 | self._display.add_row("# of Partitions", str(len(self.Partitions))) 72 | self._display.add_row("# of Columns", str(len(self.Columns))) 73 | self._display.add_row( 74 | "# of Measures", str(len(self.Measures)), end_section=True 75 | ) 76 | self._display.add_row("Description", self._object.Description, end_section=True) 77 | self._display.add_row("DataCategory", str(self._object.DataCategory)) 78 | self._display.add_row("IsHidden", str(self._object.IsHidden)) 79 | self._display.add_row("IsPrivate", str(self._object.IsPrivate)) 80 | self._display.add_row( 81 | "ModifiedTime", 82 | ticks_to_datetime(self._object.ModifiedTime.Ticks).strftime( 83 | "%m/%d/%Y, %H:%M:%S" 84 | ), 85 | ) 86 | 87 | def row_count(self) -> int: 88 | """Method to return count of rows. 89 | 90 | Simple Dax Query: `EVALUATE {COUNTROWS('Table Name')}`. 91 | 92 | Returns: 93 | int: Number of rows using `COUNTROWS`. 94 | 95 | Example: 96 | ```python 97 | model.Tables['Table Name'].row_count() 98 | ``` 99 | """ 100 | return self.Model.Adomd.query(f"EVALUATE {{COUNTROWS('{self.Name}')}}") 101 | 102 | def refresh(self, *args, **kwargs) -> pd.DataFrame: 103 | """Use this to refresh the PyTable. 104 | 105 | Returns: 106 | pd.DataFrame: Returns pandas dataframe with some refresh details. 107 | 108 | Example: 109 | ```python 110 | model.Tables['Table Name'].refresh() 111 | 112 | model.Tables['Table Name'].refresh(trace = None) # (1) 113 | ``` 114 | 115 | 1. You can pass through arguments to `PyRefresh`, like removing trace. 116 | """ 117 | return self.Model.refresh(self, *args, **kwargs) 118 | 119 | def last_refresh(self) -> datetime: 120 | """Will query each partition for the last refresh time. 121 | 122 | Then will select the max value to return. 123 | 124 | Returns: 125 | datetime: Last refresh time in datetime format 126 | """ 127 | partition_refreshes = [ 128 | partition.last_refresh() for partition in self.Partitions 129 | ] 130 | return max(partition_refreshes) 131 | 132 | def related(self): 133 | """Returns tables with a relationship with the table in question.""" 134 | return self.Model.Relationships.related(self) 135 | 136 | 137 | class PyTables(PyObjects): 138 | """Groups together multiple tables. 139 | 140 | You can interact with `PyTables` straight from model. 141 | You can even filter down with `.find()`. 142 | """ 143 | 144 | def __init__(self, objects) -> None: 145 | """Init just extends from the main `PyObjects` class.""" 146 | super().__init__(objects) 147 | 148 | def refresh(self, *args, **kwargs): 149 | """Refreshes all `PyTable`(s) in class.""" 150 | model = self._objects[0].Model 151 | return model.refresh(self, *args, **kwargs) 152 | 153 | def query_all(self, query_function: str = "COUNTROWS(_)") -> pd.DataFrame: 154 | """Dynamically query all tables. 155 | 156 | It will replace the `_` with the `query_function` arg 157 | to build out the query to run. 158 | 159 | Args: 160 | query_function (str, optional): Dax query is 161 | dynamically building a query with the 162 | `UNION` & `ROW` DAX Functions. Defaults to 'COUNTROWS(_)'. 163 | 164 | Returns: 165 | pd.DataFrame: Returns dataframe with results 166 | 167 | Example: 168 | ```python 169 | model.Tables.find('fact').query_all() # (1) 170 | ``` 171 | 172 | 1. Because `.find()` will return the `PyObjects` you are searching in, 173 | another `PyTables` is returned, but reduced to just 174 | the `PyTable`(s) with the 'fact' in the name. Then will 175 | get the # of rows for each table. 176 | """ 177 | logger.info("Querying every table in PyTables...") 178 | logger.debug(f"Function to be run: {query_function}") 179 | logger.debug("Dynamically creating DAX query...") 180 | query_str = "EVALUATE UNION(\n" 181 | for table in self: 182 | table_name = table.get_Name() 183 | dax_table_identifier = f"'{table_name}'" 184 | query_str += f"ROW(\"Table\",\"{table_name}\",\"{query_function}\",\ 185 | {query_function.replace('_',dax_table_identifier)}),\n" # noqa: E231, E261 186 | query_str = f"{query_str[:-2]})" 187 | return self[0].Model.query(query_str) 188 | 189 | def find_zero_rows(self) -> "PyTables": 190 | """Returns PyTables class of tables with zero rows queried. 191 | 192 | Returns: 193 | PyTables: A subset of the `PyTables` that contains zero rows. 194 | """ 195 | query_function: str = "COUNTROWS(_)" 196 | df = self.query_all(query_function) 197 | 198 | table_names = df[df[f"[{query_function}]"].isna()]["[Table]"].to_list() 199 | logger.debug(f"Found {table_names}") 200 | tables = [self[name] for name in table_names] 201 | return self.__class__(tables) 202 | 203 | def last_refresh(self, group_partition: bool = True) -> pd.DataFrame: 204 | """Returns `pd.DataFrame` of tables with their latest refresh time. 205 | 206 | Optional 'group_partition' variable, default is True. 207 | If False an extra column will be include to 208 | have the last refresh time to the grain of the partition 209 | Example to add to model 210 | `model.Create_Table(p.Table_Last_Refresh_Times(model),'RefreshTimes')`. 211 | 212 | Args: 213 | group_partition (bool, optional): Whether or not you want 214 | the grain of the dataframe to be by table or by partition. 215 | Defaults to True. 216 | 217 | Returns: 218 | pd.DataFrame: pd dataframe with the RefreshedTime property 219 | If group_partition == True and the table has 220 | multiple partitions, then df.groupby(by["tables"]).max() 221 | """ 222 | data = { 223 | "Tables": [ 224 | partition.Table.Name for table in self for partition in table.Partitions 225 | ], 226 | "Partitions": [ 227 | partition.Name for table in self for partition in table.Partitions 228 | ], 229 | "RefreshedTime": [ 230 | partition.last_refresh() 231 | for table in self 232 | for partition in table.Partitions 233 | ], 234 | } 235 | df = pd.DataFrame(data) 236 | if group_partition: 237 | logger.debug("Grouping together to grain of Table") 238 | return ( 239 | df[["Tables", "RefreshedTime"]] 240 | .groupby(by=["Tables"]) 241 | .max() 242 | .reset_index(drop=False) 243 | ) 244 | else: 245 | logger.debug("Returning DF") 246 | return df 247 | -------------------------------------------------------------------------------- /pytabular/tabular_editor.py: -------------------------------------------------------------------------------- 1 | """This has a `Tabular_Editor` class which will download TE2 from a default location. 2 | 3 | Or you can input your own location. 4 | """ 5 | 6 | import logging 7 | import os 8 | import requests as r 9 | import zipfile as z 10 | import atexit 11 | from logic_utils import remove_folder_and_contents 12 | 13 | logger = logging.getLogger("PyTabular") 14 | 15 | 16 | def download_tabular_editor( 17 | download_location: str = ( 18 | "https://github.com/TabularEditor/TabularEditor/releases/download/2.16.7/TabularEditor.Portable.zip" # noqa: E501 19 | ), 20 | folder: str = "Tabular_Editor_2", 21 | auto_remove=True, 22 | verify=False, 23 | ) -> str: 24 | """Runs a request.get() to retrieve the zip file from web. 25 | 26 | Will unzip response and store in directory. 27 | Will also register the removal of the new directory 28 | and files when exiting program. 29 | 30 | Args: 31 | download_location (str, optional): File path for zip of Tabular Editor 2. 32 | See code args for default download url. 33 | folder (str, optional): New Folder Location. Defaults to "Tabular_Editor_2". 34 | auto_remove (bool, optional): Boolean to determine auto 35 | removal of files once script exits. Defaults to True. 36 | verify (bool, optional): Passthrough argument for `r.get`. Need to update later. 37 | 38 | Returns: 39 | str: File path of TabularEditor.exe 40 | """ 41 | logger.info("Downloading Tabular Editor 2...") 42 | logger.info(f"From... {download_location}") 43 | folder_location = os.path.join(os.getcwd(), folder) 44 | response = r.get(download_location, verify=verify) 45 | file_location = f"{os.getcwd()}\\{download_location.split('/')[-1]}" 46 | with open(file_location, "wb") as te2_zip: 47 | te2_zip.write(response.content) 48 | with z.ZipFile(file_location) as zipper: 49 | zipper.extractall(path=folder_location) 50 | logger.debug("Removing Zip File...") 51 | os.remove(file_location) 52 | logger.info(f"Tabular Editor Downloaded and Extracted to {folder_location}") 53 | if auto_remove: 54 | logger.debug(f"Registering removal on termination... For {folder_location}") 55 | atexit.register(remove_folder_and_contents, folder_location) 56 | return f"{folder_location}\\TabularEditor.exe" 57 | 58 | 59 | class TabularEditor: 60 | """Setting Tabular_Editor Class for future work. 61 | 62 | Mainly runs `download_tabular_editor()` 63 | """ 64 | 65 | def __init__( 66 | self, exe_file_path: str = "Default", verify_download: bool = True 67 | ) -> None: 68 | """Init for `TabularEditor()` class. 69 | 70 | This is mostly a placeholder right now. 71 | But useful if you want an easy way to download TE2. 72 | 73 | Args: 74 | exe_file_path (str, optional): File path where TE2 lives. Defaults to "Default". 75 | If "Default", it will run `download_tabular_editor()` 76 | and download from github. 77 | verify_download (bool, optional): Passthrough argument for `r.get`. 78 | Need to update later. 79 | """ 80 | logger.debug(f"Initializing Tabular Editor Class:: {exe_file_path}") 81 | if exe_file_path == "Default": 82 | self.exe: str = download_tabular_editor(verify=verify_download) 83 | else: 84 | self.exe: str = exe_file_path 85 | pass 86 | -------------------------------------------------------------------------------- /pytabular/tabular_tracing.py: -------------------------------------------------------------------------------- 1 | """`tabular_tracing.py` handles all tracing capabilities in your model. 2 | 3 | It also includes some pre built traces to make life easier. 4 | Feel free to build your own. 5 | 6 | Example: 7 | ```python title="Monitor Queries" 8 | import pytabular as p 9 | import logging as l 10 | model = p.Tabular(CONNECTION_STR) 11 | query_trace = p.QueryMonitor(model) 12 | query_trace.start() # (1) 13 | 14 | ### 15 | 16 | p.logger.setLevel(l.DEBUG) # (2) 17 | 18 | ### 19 | 20 | query_trace.stop() 21 | query_trace.drop() # (3) 22 | ``` 23 | 24 | 1. You will now start to see query traces on your model get outputed to your console. 25 | 2. If you want to see the FULL query then set logging to DEBUG. 26 | 3. You can drop on your own, or will get dropped on script exit. 27 | """ 28 | 29 | import logging 30 | import random 31 | import xmltodict 32 | from typing import List, Callable 33 | from Microsoft.AnalysisServices.Tabular import Trace, TraceEvent, TraceEventHandler 34 | from Microsoft.AnalysisServices import ( 35 | TraceColumn, 36 | TraceEventClass, 37 | TraceEventSubclass, 38 | ) 39 | import atexit 40 | 41 | logger = logging.getLogger("PyTabular") 42 | 43 | 44 | class BaseTrace: 45 | """Generates trace to be run on Server. 46 | 47 | This is the base class to customize the type of Trace you are looking for. 48 | It's recommended to use the out of the box traces built. 49 | It's on the roadmap to have an intuitive way to build traces for users. 50 | """ 51 | 52 | def __init__( 53 | self, 54 | tabular_class, 55 | trace_events: List[TraceEvent], 56 | trace_event_columns: List[TraceColumn], 57 | handler: Callable, 58 | ) -> None: 59 | """This will `build()`, `add()`, and `update()` the trace to model. 60 | 61 | It will also register the dropping on the trace on exiting python. 62 | 63 | Args: 64 | tabular_class (Tabular): The model you want the trace for. 65 | trace_events (List[TraceEvent]): The TraceEvents you want have in your trace. 66 | From Microsoft.AnalysisServices.TraceEventClass. 67 | trace_event_columns (List[TraceColumn]): The trace event columns you want in your trace. 68 | From Microsoft.AnalysisServices.TraceColumn. 69 | handler (Callable): The `handler` is a function that will take in two args. 70 | The first arg is `source` and it is currently unused. 71 | The second is arg is `args` and here 72 | is where you can access the results of the trace. 73 | """ 74 | logger.debug("Trace Base Class initializing...") 75 | self.Name = "PyTabular_" + "".join( 76 | random.SystemRandom().choices( 77 | [str(x) for x in [y for y in range(0, 10)]], k=10 78 | ) 79 | ) 80 | self.ID = self.Name.replace("PyTabular_", "") 81 | self.Trace = Trace(self.Name, self.ID) 82 | logger.debug(f"Trace {self.Trace.Name} created...") 83 | self.tabular_class = tabular_class 84 | self.Event_Categories = self._query_dmv_for_event_categories() 85 | 86 | self.trace_events = trace_events 87 | self.trace_event_columns = trace_event_columns 88 | self.handler = handler 89 | 90 | self.build() 91 | self.add() 92 | self.update() 93 | atexit.register(self.drop) 94 | 95 | def build(self) -> bool: 96 | """Run on init. 97 | 98 | This will take the inputed arguments for the class 99 | and attempt to build the Trace. 100 | 101 | Returns: 102 | bool: True if successful 103 | """ 104 | logger.info(f"Building Trace {self.Name}") 105 | te = [TraceEvent(trace_event) for trace_event in self.trace_events] 106 | logger.debug(f"Adding Events to... {self.Trace.Name}") 107 | [self.Trace.get_Events().Add(t) for t in te] 108 | 109 | def add_column(trace_event, trace_event_column): 110 | """Adds the column to trace event.""" 111 | try: 112 | trace_event.Columns.Add(trace_event_column) 113 | except Exception: 114 | logger.warning(f"{trace_event} - {trace_event_column} Skipped") 115 | pass 116 | 117 | logger.debug("Adding Trace Event Columns...") 118 | [ 119 | add_column(trace_event, trace_event_column) 120 | for trace_event_column in self.trace_event_columns 121 | for trace_event in te 122 | if str(trace_event_column.value__) 123 | in self.Event_Categories[str(trace_event.EventID.value__)] 124 | ] 125 | 126 | logger.debug("Adding Handler to Trace...") 127 | self.handler = TraceEventHandler(self.handler) 128 | self.Trace.OnEvent += self.handler 129 | return True 130 | 131 | def add(self) -> int: 132 | """Runs on init. Adds built trace to the Server. 133 | 134 | Returns: 135 | int: Return int of placement in Server.Traces.get_Item(int). 136 | """ 137 | logger.info(f"Adding {self.Name} to {self.tabular_class.Server.Name}") 138 | return self.tabular_class.Server.Traces.Add(self.Trace) 139 | 140 | def update(self) -> None: 141 | """Runs on init. Syncs with Server. 142 | 143 | Returns: 144 | None: Returns None. 145 | Unless unsuccessful then it will return the error from Server. 146 | """ 147 | logger.info(f"Updating {self.Name} in {self.tabular_class.Server.Name}") 148 | if self.tabular_class.Server.Connected is False: 149 | self.tabular_class.reconnect() 150 | 151 | return self.Trace.Update() 152 | 153 | def start(self) -> None: 154 | """Call when you want to start the trace. 155 | 156 | Returns: 157 | None: Returns None. 158 | Unless unsuccessful then it will return the error from Server. 159 | """ 160 | logger.info(f"Starting {self.Name} in {self.tabular_class.Server.Name}") 161 | return self.Trace.Start() 162 | 163 | def stop(self) -> None: 164 | """Call when you want to stop the trace. 165 | 166 | Returns: 167 | None: Returns None. 168 | Unless unsuccessful then it will return the error from Server. 169 | """ 170 | logger.info(f"Stopping {self.Name} in {self.tabular_class.Server.Name}") 171 | return self.Trace.Stop() 172 | 173 | def drop(self) -> None: 174 | """Call when you want to drop the trace. 175 | 176 | Returns: 177 | None: Returns None. Unless unsuccessful, 178 | then it will return the error from Server. 179 | """ 180 | logger.info(f"Dropping {self.Name} in {self.tabular_class.Server.Name}") 181 | atexit.unregister(self.drop) 182 | return self.Trace.Drop() 183 | 184 | def _query_dmv_for_event_categories(self): 185 | """Internal use. Called during the building process of a refresh. 186 | 187 | It is used to locate allowed columns for event categories. 188 | This is done by executing a `Tabular().Query()` 189 | on the `DISCOVER_EVENT_CATEGORIES` table in the DMV. 190 | Then the function will parse the results, 191 | as it is xml inside of rows. 192 | """ 193 | event_categories = {} 194 | events = [] 195 | logger.debug("Querying DMV for columns rules...") 196 | logger.debug("select * from $SYSTEM.DISCOVER_TRACE_EVENT_CATEGORIES") 197 | df = self.tabular_class.query( 198 | "select * from $SYSTEM.DISCOVER_TRACE_EVENT_CATEGORIES" 199 | ) 200 | for index, row in df.iterrows(): 201 | xml_data = xmltodict.parse(row.Data) 202 | if isinstance(xml_data["EVENTCATEGORY"]["EVENTLIST"]["EVENT"], list): 203 | events += [ 204 | event for event in xml_data["EVENTCATEGORY"]["EVENTLIST"]["EVENT"] 205 | ] 206 | else: 207 | events += [xml_data["EVENTCATEGORY"]["EVENTLIST"]["EVENT"]] 208 | for event in events: 209 | event_categories[event["ID"]] = [ 210 | column["ID"] for column in event["EVENTCOLUMNLIST"]["EVENTCOLUMN"] 211 | ] 212 | return event_categories 213 | 214 | 215 | def _refresh_handler(source, args): 216 | """Default handler called when `RefreshTrace()` is used. 217 | 218 | It will log various steps of the refresh process. 219 | Mostly will output the current # of rows read. 220 | Will output `logger.warning()` if refresh produces zero rows, 221 | or if a switching dictionary event occurs. 222 | The rest of the EventSubclass' will output the raw text. 223 | For example, TabularSequencePoint, TabularRefresh, Process, 224 | Vertipaq, CompressSegment, TabularCommit, RelationshipBuildPrepare, 225 | AnalyzeEncodeData, ReadData. If there is anything else not prebuilt 226 | out for logging, it will dump the arguments int `logger.debug()`. 227 | """ 228 | text_data = args.TextData.replace("", "").replace("", "") 229 | 230 | if ( 231 | args.EventClass == TraceEventClass.ProgressReportCurrent 232 | and args.EventSubclass == TraceEventSubclass.ReadData 233 | ): 234 | table_name = args.ObjectPath.split(".")[-2] 235 | part_name = args.ObjectPath.split(".")[-1] 236 | logger.info( 237 | f"{args.ProgressTotal} row read from '{table_name}' - '{part_name}' " 238 | ) 239 | 240 | elif ( 241 | args.EventClass == TraceEventClass.ProgressReportEnd 242 | and args.EventSubclass == TraceEventSubclass.ReadData 243 | ): 244 | if args.ProgressTotal == 0: 245 | logger.warning( 246 | f"{' - '.join(args.ObjectPath.split('.')[-2:])} QUERIED {args.ProgressTotal} ROWS!" 247 | ) 248 | else: 249 | table_partition = "::".join(args.ObjectPath.split(".")[-2:]) 250 | logger.info( 251 | f"Finished Reading {table_partition} for {args.ProgressTotal} Rows!" 252 | ) 253 | 254 | elif args.EventSubclass == TraceEventSubclass.SwitchingDictionary: 255 | logger.warning(f"{text_data}") 256 | 257 | elif ( 258 | args.EventClass == TraceEventClass.ProgressReportBegin 259 | and args.EventSubclass 260 | in [ 261 | TraceEventSubclass.TabularSequencePoint, 262 | TraceEventSubclass.TabularRefresh, 263 | TraceEventSubclass.Process, 264 | TraceEventSubclass.VertiPaq, 265 | TraceEventSubclass.CompressSegment, 266 | TraceEventSubclass.TabularCommit, 267 | TraceEventSubclass.RelationshipBuildPrepare, 268 | TraceEventSubclass.AnalyzeEncodeData, 269 | TraceEventSubclass.ReadData, 270 | ] 271 | ): 272 | logger.info(f"{text_data}") 273 | 274 | elif ( 275 | args.EventClass == TraceEventClass.ProgressReportEnd 276 | and args.EventSubclass 277 | in [ 278 | TraceEventSubclass.TabularSequencePoint, 279 | TraceEventSubclass.TabularRefresh, 280 | TraceEventSubclass.Process, 281 | TraceEventSubclass.VertiPaq, 282 | TraceEventSubclass.CompressSegment, 283 | TraceEventSubclass.TabularCommit, 284 | TraceEventSubclass.RelationshipBuildPrepare, 285 | TraceEventSubclass.AnalyzeEncodeData, 286 | ] 287 | ): 288 | logger.info(f"{text_data}") 289 | 290 | else: 291 | logger.debug(f"{args.EventClass}::{args.EventSubclass}::{text_data}") 292 | 293 | 294 | class RefreshTrace(BaseTrace): 295 | """Subclass of `BaseTrace()`. Usefull for monitoring refreshes. 296 | 297 | This is the default trace that is run on refreshes. 298 | It will output all the various details into `logger()`. 299 | See `_refresh_handler()` for more details on what gets 300 | put into `logger()`. 301 | """ 302 | 303 | def __init__( 304 | self, 305 | tabular_class, 306 | trace_events: List[TraceEvent] = [ 307 | TraceEventClass.ProgressReportBegin, 308 | TraceEventClass.ProgressReportCurrent, 309 | TraceEventClass.ProgressReportEnd, 310 | TraceEventClass.ProgressReportError, 311 | ], 312 | trace_event_columns: List[TraceColumn] = [ 313 | TraceColumn.EventSubclass, 314 | TraceColumn.CurrentTime, 315 | TraceColumn.ObjectName, 316 | TraceColumn.ObjectPath, 317 | TraceColumn.DatabaseName, 318 | TraceColumn.SessionID, 319 | TraceColumn.TextData, 320 | TraceColumn.EventClass, 321 | TraceColumn.ProgressTotal, 322 | ], 323 | handler: Callable = _refresh_handler, 324 | ) -> None: 325 | """Init will extend through `BaseTrace()`. But pass through specific params. 326 | 327 | Args: 328 | tabular_class (Tabular): This is your `Tabular()` class. 329 | trace_events (List[TraceEvent], optional): Defaults to [ 330 | TraceEventClass.ProgressReportBegin, 331 | TraceEventClass.ProgressReportCurrent, TraceEventClass.ProgressReportEnd, 332 | TraceEventClass.ProgressReportError, ]. 333 | trace_event_columns (List[TraceColumn], optional): Defaults to 334 | [ TraceColumn.EventSubclass, TraceColumn.CurrentTime, 335 | TraceColumn.ObjectName, TraceColumn.ObjectPath, TraceColumn.DatabaseName, 336 | TraceColumn.SessionID, TraceColumn.TextData, TraceColumn.EventClass, 337 | TraceColumn.ProgressTotal, ]. 338 | handler (Callable, optional): _description_. Defaults to _refresh_handler. 339 | """ 340 | super().__init__(tabular_class, trace_events, trace_event_columns, handler) 341 | 342 | 343 | def _query_monitor_handler(source, args): 344 | """Default function used with the `Query_Monitor()` trace. 345 | 346 | Will return query type, user (effective user), application, start time, 347 | end time, and total seconds of query in `logger.info()`. 348 | To see full query set logger to debug `logger.setLevel(logging.DEBUG)`. 349 | """ 350 | total_secs = args.Duration / 1000 351 | domain_site = args.NTUserName.find("\\") 352 | if domain_site > 0: 353 | user = args.NTUserName[domain_site + 1 :] 354 | else: 355 | user = args.NTUserName 356 | logger.info(f"{args.EventSubclass} by {user} in {args.ApplicationName}") 357 | logger.info( 358 | f"From {args.StartTime} to {args.EndTime} for {str(total_secs)} seconds" 359 | ) 360 | if args.Severity == 3: 361 | logger.error(f"Query failure... {str(args.Error)}") 362 | logger.error(f"{args.TextData}") 363 | logger.debug(f"{args.TextData}") 364 | 365 | 366 | class QueryMonitor(BaseTrace): 367 | """Subclass of `BaseTrace()`. Usefull for monitoring queries. 368 | 369 | The default handler for `QueryMonitor()` shows full query in `logger.debug()`. 370 | So you will need to set your logger to `debug()` if you would like to see them. 371 | Otherwise, will show basic info on who/what is querying. 372 | """ 373 | 374 | def __init__( 375 | self, 376 | tabular_class, 377 | trace_events: List[TraceEvent] = [TraceEventClass.QueryEnd], 378 | trace_event_columns: List[TraceColumn] = [ 379 | TraceColumn.EventSubclass, 380 | TraceColumn.StartTime, 381 | TraceColumn.EndTime, 382 | TraceColumn.Duration, 383 | TraceColumn.Severity, 384 | TraceColumn.Error, 385 | TraceColumn.NTUserName, 386 | TraceColumn.DatabaseName, 387 | TraceColumn.ApplicationName, 388 | TraceColumn.TextData, 389 | ], 390 | handler: Callable = _query_monitor_handler, 391 | ) -> None: 392 | """Init will extend through to BaseTrace, but pass through specific params. 393 | 394 | Args: 395 | tabular_class (Tabular): This is your `Tabular()` class. 396 | All that will need to provided to successfully init. 397 | trace_events (List[TraceEvent], optional): Defaults to [TraceEventClass.QueryEnd]. 398 | trace_event_columns (List[TraceColumn], optional): Defaults to 399 | [ TraceColumn.EventSubclass, TraceColumn.StartTime, TraceColumn.EndTime, 400 | TraceColumn.Duration, TraceColumn.Severity, TraceColumn.Error, 401 | TraceColumn.NTUserName, TraceColumn.DatabaseName, TraceColumn.ApplicationName, 402 | TraceColumn.TextData, ]. 403 | handler (Callable, optional): Defaults to `_query_monitor_handler()`. 404 | """ 405 | super().__init__(tabular_class, trace_events, trace_event_columns, handler) 406 | logger.info("Query text lives in DEBUG, adjust logging to see query text.") 407 | -------------------------------------------------------------------------------- /pytabular/tmdl.py: -------------------------------------------------------------------------------- 1 | """See [TMDL Scripting].(https://learn.microsoft.com/en-us/analysis-services/tmdl/tmdl-overview). 2 | 3 | Run `Tmdl(model).save_to_folder()` to save tmdl of model. 4 | Run `Tmdl(model).execute()` to execute a specific set of tmdl scripts. 5 | """ 6 | 7 | from Microsoft.AnalysisServices.Tabular import TmdlSerializer 8 | 9 | 10 | class Tmdl: 11 | """Specify the specific model you want to use for scripting. 12 | 13 | Args: 14 | model (Tabular): Initialize with Tabular model. 15 | """ 16 | def __init__(self, model): 17 | self.model = model 18 | 19 | def save_to_folder(self, path: str = "tmdl"): 20 | """Runs `SerializeModelToFolder` from .net library. 21 | 22 | Args: 23 | path (str, optional): directory where to save tmdl structure. 24 | Defaults to "tmdl". 25 | """ 26 | TmdlSerializer.SerializeModelToFolder(self.model._object, path) 27 | return True 28 | 29 | def execute(self, path: str = "tmdl", auto_save: bool = True): 30 | """Runs `DeserializeModelFromFolder` from .net library. 31 | 32 | Args: 33 | path (str, optional): directory to look for tmdl scripts. 34 | Defaults to "tmdl". 35 | auto_save (bool, optional): You can set to false 36 | if you want to precheck a few things, but will need to 37 | run `model.save_changes()`. Setting to `True` will go ahead 38 | and execute `model.save_changes()` Defaults to True. 39 | """ 40 | model_object = TmdlSerializer.DeserializeModelFromFolder(path) 41 | model_object.CopyTo(self.model._object) 42 | if auto_save: 43 | return self.model.save_changes() 44 | else: 45 | return True 46 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- 1 | """Init for pytest.""" 2 | -------------------------------------------------------------------------------- /test/adventureworks/AdventureWorks Sales.pbix: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Curts0/PyTabular/5d43df51f0d113b9c083968bedf12d81ebdc6594/test/adventureworks/AdventureWorks Sales.pbix -------------------------------------------------------------------------------- /test/adventureworks/AdventureWorks Sales.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Curts0/PyTabular/5d43df51f0d113b9c083968bedf12d81ebdc6594/test/adventureworks/AdventureWorks Sales.xlsx -------------------------------------------------------------------------------- /test/config.py: -------------------------------------------------------------------------------- 1 | """Custom configurations for pytest.""" 2 | 3 | import pytabular as p 4 | import os 5 | import pandas as pd 6 | import pytest 7 | import subprocess 8 | from time import sleep 9 | 10 | 11 | def get_test_path(): 12 | """Gets working test path.""" 13 | cwd = os.getcwd() 14 | if os.path.basename(cwd) == "test": 15 | return cwd 16 | elif "test" in os.listdir(): 17 | return cwd + "\\test" 18 | else: 19 | raise BaseException("Unable to find test path...") 20 | 21 | 22 | adventureworks_path = f'"{get_test_path()}\\adventureworks\\AdventureWorks Sales.pbix"' 23 | 24 | 25 | def find_local_pbi(): 26 | """Finds local pbix file if exists. Otherwise, will open AdventureWorks.""" 27 | attempt = p.find_local_pbi_instances() 28 | if len(attempt) > 0: 29 | return attempt[0] 30 | else: 31 | p.logger.info(f"Opening {adventureworks_path}") 32 | subprocess.run(["powershell", f"Start-Process {adventureworks_path}"]) 33 | # Got to be a better way to wait and ensure the PBIX file is open? 34 | p.logger.warning( 35 | "sleep(30)... Need a better way to wait until PBIX is loaded..." 36 | ) 37 | sleep(30) 38 | return p.find_local_pbi_instances()[0] 39 | 40 | 41 | LOCAL_FILE = find_local_pbi() 42 | 43 | p.logger.info(f"Connecting to... {LOCAL_FILE[0]} - {LOCAL_FILE[1]}") 44 | local_pbix = p.Tabular(LOCAL_FILE[1]) 45 | 46 | 47 | p.logger.info("Generating test data...") 48 | testingtablename = "PyTestTable" 49 | testingtabledf = pd.DataFrame(data={"col1": [1, 2, 3], "col2": ["four", "five", "six"]}) 50 | testing_parameters = [ 51 | pytest.param(local_pbix, id=local_pbix.Name), 52 | ] 53 | -------------------------------------------------------------------------------- /test/conftest.py: -------------------------------------------------------------------------------- 1 | """pytest conftest.py. 2 | 3 | Has a `TestingStorage()` class to pass parameters from one test to another. 4 | This file also has functions that give instructions on start and finish of pytest. 5 | See `config.py` for more testing configurations. 6 | """ 7 | 8 | from test.config import ( 9 | local_pbix, 10 | testingtablename, 11 | testingtabledf, 12 | ) 13 | import pytabular as p 14 | 15 | 16 | class TestStorage: 17 | """Class to pass parameters from on pytest to another.""" 18 | 19 | query_trace = None 20 | documentation_class = None 21 | 22 | 23 | def pytest_report_header(config): 24 | """Pytest header name.""" 25 | return "PyTabular Local Testing" 26 | 27 | 28 | def remove_testing_table(model): 29 | """Function to remove the testing table.""" 30 | table_check = [ 31 | table 32 | for table in model.Model.Tables.GetEnumerator() 33 | if testingtablename in table.Name 34 | ] 35 | for table in table_check: 36 | p.logger.info(f"Removing table {table.Name} from {model.Server.Name}") 37 | model.Model.Tables.Remove(table) 38 | model.SaveChanges() 39 | 40 | 41 | def pytest_generate_tests(metafunc): 42 | """Creates the model param for most tests. 43 | 44 | Will update to use a semantic model via online connection 45 | vs. connecting to a local model. 46 | """ 47 | if "model" in metafunc.fixturenames: 48 | metafunc.parametrize("model", [local_pbix], ids=[local_pbix.Name]) 49 | 50 | 51 | def pytest_sessionstart(session): 52 | """Run at pytest start. 53 | 54 | Removes testing table if exists, and creates a new one. 55 | """ 56 | p.logger.info("Executing pytest setup...") 57 | remove_testing_table(local_pbix) 58 | local_pbix.create_table(testingtabledf, testingtablename) 59 | return True 60 | 61 | 62 | def pytest_sessionfinish(session, exitstatus): 63 | """On pytest finish it will remove testing table.""" 64 | p.logger.info("Executing pytest cleanup...") 65 | remove_testing_table(local_pbix) 66 | # p.logger.info("Finding and closing PBIX file...") 67 | # subprocess.run(["powershell", "Stop-Process -Name PBIDesktop"]) 68 | return True 69 | -------------------------------------------------------------------------------- /test/dfvaltest.dax: -------------------------------------------------------------------------------- 1 | EVALUATE 2 | { ( 1, 2 ), ( 3, 4 ) } -------------------------------------------------------------------------------- /test/singlevaltest.dax: -------------------------------------------------------------------------------- 1 | EVALUATE 2 | { 1 } -------------------------------------------------------------------------------- /test/test_10logic_utils.py: -------------------------------------------------------------------------------- 1 | """pytest for the table.py file. Covers the PyTable and PyTables classes.""" 2 | 3 | import pytest 4 | from pytabular import logic_utils 5 | import pandas as pd 6 | import os 7 | 8 | sublists = [pytest.param(range(0, 50), x, id=f"Sublist {str(x)}") for x in range(1, 5)] 9 | 10 | 11 | @pytest.mark.parametrize("lst, n", sublists) 12 | def test_sub_list(lst, n): 13 | """Tests `get_sub_list()` from `logic_utils.py`.""" 14 | result = logic_utils.get_sub_list(lst, n) 15 | assert isinstance(result, list) 16 | 17 | 18 | suffix_list = [ 19 | "helloworld_backup", 20 | "helloworld_testing", 21 | "helloworld_suffix", 22 | "nosuffix", 23 | ] 24 | 25 | 26 | @pytest.mark.parametrize( 27 | "file_name, suffix", 28 | [ 29 | pytest.param( 30 | suffix, suffix[suffix.find("_") + 1 :], id=suffix[suffix.find("_") + 1 :] 31 | ) 32 | for suffix in suffix_list 33 | ], 34 | ) 35 | def test_remove_suffix(file_name, suffix): 36 | """Tests `remove_suffix()` function. 37 | 38 | Note this exists so lower python versions can stay compatible. 39 | """ 40 | result = logic_utils.remove_suffix(file_name, suffix) 41 | assert suffix not in result 42 | 43 | 44 | dfs = [ 45 | pytest.param(pd.DataFrame({"column_1": [0, 1, 2, 3, 4, 5]}), id="DataFrame1"), 46 | pytest.param(pd.DataFrame({"column_1": ["one", "two", "three"]}), id="DataFrame2"), 47 | ] 48 | 49 | 50 | @pytest.mark.parametrize("df", dfs) 51 | def test_dataframe_to_dict(df): 52 | """Tests `dataframe_to_dict()` function.""" 53 | assert isinstance(logic_utils.dataframe_to_dict(df), list) 54 | 55 | 56 | def test_dict_to_markdown_table(model): 57 | """Tests `dict_to_markdown_table()` function.""" 58 | dependencies = [measure.get_dependencies() for measure in model.Measures] 59 | columns = ["Referenced Object Type", "Referenced Table", "Referenced Object"] 60 | result = logic_utils.dict_to_markdown_table(dependencies, columns) 61 | assert isinstance(result, str) 62 | 63 | 64 | def test_remove_dir(): 65 | """Tests removing a directory.""" 66 | dir = "testing_to_be_deleted" 67 | os.makedirs(dir) 68 | remove = f"{os.getcwd()}\\{dir}" 69 | logic_utils.remove_folder_and_contents(remove) 70 | assert dir not in os.listdir() 71 | 72 | 73 | def test_remove_file(): 74 | """Tests removing a file.""" 75 | file_to_delete = "testing_to_be_deleted.txt" 76 | with open(file_to_delete, "w") as f: 77 | f.write("Delete this file...") 78 | logic_utils.remove_file(f"{os.getcwd()}\\{file_to_delete}") 79 | assert file_to_delete not in os.listdir() 80 | -------------------------------------------------------------------------------- /test/test_11document.py: -------------------------------------------------------------------------------- 1 | """Tests to cover the document.py file.""" 2 | 3 | import pytest 4 | import pytabular as p 5 | import os 6 | from pytabular import logic_utils 7 | from test.conftest import TestStorage 8 | 9 | 10 | def test_basic_document_funcionality(model): 11 | """Tests basic documentation functionality.""" 12 | try: 13 | docs = p.ModelDocumenter(model=model) 14 | docs.generate_documentation_pages() 15 | docs.save_documentation() 16 | TestStorage.documentation_class = docs 17 | except Exception: 18 | pytest.fail("Unsuccessful documentation generation..") 19 | 20 | 21 | def test_basic_documentation_removed(): 22 | """Tests that created documentation gets removed.""" 23 | docs_class = TestStorage.documentation_class 24 | remove = f"{docs_class.save_location}/{docs_class.friendly_name}" 25 | logic_utils.remove_folder_and_contents(remove) 26 | assert os.path.exists(remove) is False 27 | -------------------------------------------------------------------------------- /test/test_12tmdl.py: -------------------------------------------------------------------------------- 1 | """Tests to cover the tmdl.py file.""" 2 | 3 | import pytabular as p 4 | 5 | path = "tmdl_testing" 6 | 7 | 8 | def test_tmdl_save(model): 9 | """Tests basic tmdl save to folder functionality.""" 10 | stf = p.Tmdl(model).save_to_folder(path) 11 | assert stf 12 | 13 | 14 | def test_tmdl_execute(model): 15 | """Tests basic tmdl execute functionality.""" 16 | exec = p.Tmdl(model).execute(path, False) 17 | p.logic_utils.remove_folder_and_contents(path) 18 | assert exec 19 | -------------------------------------------------------------------------------- /test/test_1sanity.py: -------------------------------------------------------------------------------- 1 | """Sanity pytests. 2 | 3 | Does 1 actually equal 1? 4 | Or am I crazy person about to descend into madness. 5 | """ 6 | 7 | from Microsoft.AnalysisServices.Tabular import Database 8 | 9 | 10 | def test_sanity_check(): 11 | """Just in case... I might be crazy.""" 12 | assert 1 == 1 13 | 14 | 15 | def test_connection(model): 16 | """Does a quick check on connection to `Tabular()` class.""" 17 | assert model.Server.Connected 18 | 19 | 20 | def test_database(model): 21 | """Tests that `model.Database` is actually a Database.""" 22 | assert isinstance(model.Database, Database) 23 | -------------------------------------------------------------------------------- /test/test_2object.py: -------------------------------------------------------------------------------- 1 | """pytest for the table.py file. Covers the PyTable and PyTables classes.""" 2 | 3 | import pytest 4 | from pytabular import Tabular 5 | 6 | 7 | def test_rich_repr_model(model): 8 | """Tests successful `__rich_repr()` `on Tabular()` class.""" 9 | try: 10 | model.__rich_repr__() 11 | except Exception: 12 | pytest.fail("__rich_repr__() failed") 13 | 14 | 15 | def test_rich_repr_table(model): 16 | """Tests successful `__rich_repr()` `on PyTable()` class.""" 17 | try: 18 | model.Tables[0].__rich_repr__() 19 | except Exception: 20 | pytest.fail("__rich_repr__() failed") 21 | 22 | 23 | def test_rich_repr_tables(model): 24 | """Tests successful `__rich_repr()` `on PyTables()` class.""" 25 | try: 26 | model.Tables.__rich_repr__() 27 | except Exception: 28 | pytest.fail("__rich_repr__() failed") 29 | 30 | 31 | def test_rich_repr_column(model): 32 | """Tests successful `__rich_repr()` `on PyColumn()` class.""" 33 | try: 34 | model.Columns[0].__rich_repr__() 35 | except Exception: 36 | pytest.fail("__rich_repr__() failed") 37 | 38 | 39 | def test_rich_repr_columns(model): 40 | """Tests successful `__rich_repr()` `on PyColumns()` class.""" 41 | try: 42 | model.Columns.__rich_repr__() 43 | except Exception: 44 | pytest.fail("__rich_repr__() failed") 45 | 46 | 47 | def test_rich_repr_partition(model): 48 | """Tests successful `__rich_repr()` `on PyPartition()` class.""" 49 | try: 50 | model.Partitions[0].__rich_repr__() 51 | except Exception: 52 | pytest.fail("__rich_repr__() failed") 53 | 54 | 55 | def test_rich_repr_partitions(model): 56 | """Tests successful `__rich_repr()` `on PyPartitions()` class.""" 57 | try: 58 | model.Partitions.__rich_repr__() 59 | except Exception: 60 | pytest.fail("__rich_repr__() failed") 61 | 62 | 63 | def test_rich_repr_measure(model): 64 | """Tests successful `__rich_repr()` `on PyMeasure()` class.""" 65 | try: 66 | model.Measures[0].__rich_repr__() 67 | except Exception: 68 | pytest.fail("__rich_repr__() failed") 69 | 70 | 71 | def test_rich_repr_measures(model): 72 | """Tests successful `__rich_repr()` `on PyMeasures()` class.""" 73 | try: 74 | model.Measures.__rich_repr__() 75 | except Exception: 76 | pytest.fail("__rich_repr__() failed") 77 | 78 | 79 | def test_get_attr(model): 80 | """Tests custom get attribute from `PyObject` class.""" 81 | assert isinstance(model.Tables[0].Model, Tabular) 82 | 83 | 84 | def test_iadd_tables(model): 85 | """Tests `__iadd__()` with `PyTables()`.""" 86 | a = model.Tables.find("Sales") 87 | b = model.Tables.find("Date") 88 | a += b 89 | assert len(a.find("Date")) > 0 90 | 91 | 92 | def test_iadd_table(model): 93 | """Tests `__iadd__()` with a `PyTable()`.""" 94 | a = model.Tables.find("Sales") 95 | b = model.Tables.find("Date")[0] 96 | a += b 97 | assert len(a.find("Date")) > 0 98 | 99 | 100 | def test_find_measure(model): 101 | """Tests `find()` with a `PyMeasure()`.""" 102 | a = model.Measures[0].Name 103 | b = model.Measures.find(a) 104 | assert len(b) > 0 105 | -------------------------------------------------------------------------------- /test/test_3tabular.py: -------------------------------------------------------------------------------- 1 | """Bulk of pytests for `Tabular()` class.""" 2 | 3 | import pytest 4 | import pandas as pd 5 | import pytabular as p 6 | from test.config import testingtablename, get_test_path 7 | 8 | 9 | def test_basic_query(model): 10 | """Tests a basic query execution.""" 11 | int_result = model.query("EVALUATE {1}") 12 | text_result = model.query('EVALUATE {"Hello World"}') 13 | assert int_result == 1 and text_result == "Hello World" 14 | 15 | 16 | datatype_queries = [ 17 | ["this is a string", '"this is a string"'], 18 | [1, 1], 19 | [1000.78, "CONVERT(1000.78,CURRENCY)"], 20 | ] 21 | 22 | 23 | def test_datatype_query(model): 24 | """Tests the results of different datatypes.""" 25 | for query in datatype_queries: 26 | result = model.query(f"EVALUATE {{{query[1]}}}") 27 | assert result == query[0] 28 | 29 | 30 | number_queries = ( 31 | pytest.param(("EVALUATE {1}", 1), id="basic"), 32 | pytest.param( 33 | ("""EVALUATE {FORMAT(12345.67, "$#,##0.00;($#,##0.00)")}""", 12345.67), 34 | id="$#,##0.00", 35 | ), 36 | pytest.param( 37 | ("""EVALUATE {FORMAT(12345.00, "$#,##0.00;($#,##0.00)")}""", 12345.00), 38 | id="$#,##0.00 w/o decimal", 39 | ), 40 | pytest.param( 41 | ("""EVALUATE {FORMAT(-12345.67, "$#,##0.00;($#,##0.00)")}""", -12345.67), 42 | id="$#,##0.00 w/ -", 43 | ), 44 | pytest.param( 45 | ("""EVALUATE {FORMAT(12345.67, "#,##0.00;(#,##0.00)")}""", 12345.67), 46 | id="#,##0.00", 47 | ), 48 | ) 49 | 50 | 51 | @pytest.mark.parametrize("number_queries", number_queries) 52 | def test_parsing_query(model, number_queries): 53 | """Assert that the proper query result stripping is happenign.""" 54 | assert model.query(number_queries[0]) == number_queries[1] 55 | 56 | 57 | def test_file_query(model): 58 | """Test `query()` via a file.""" 59 | singlevaltest = get_test_path() + "\\singlevaltest.dax" 60 | dfvaltest = get_test_path() + "\\dfvaltest.dax" 61 | dfdupe = pd.DataFrame({"[Value1]": (1, 3), "[Value2]": (2, 4)}) 62 | assert model.query(singlevaltest) == 1 and model.query(dfvaltest).equals(dfdupe) 63 | 64 | 65 | def test_repr_str(model): 66 | """Testing successful `__repr__()` on model.""" 67 | assert isinstance(model.__repr__(), str) 68 | 69 | 70 | def test_pytables_count(model): 71 | """Testing row count of testingtable.""" 72 | assert model.Tables[testingtablename].row_count() > 0 73 | 74 | 75 | def test_pytables_refresh(model): 76 | """Tests refrshing table of testingtable.""" 77 | assert len(model.Tables[testingtablename].refresh()) > 0 78 | 79 | 80 | def test_pypartition_refresh(model): 81 | """Tests refreshing partition of testingtable.""" 82 | assert len(model.Tables[testingtablename].Partitions[0].refresh()) > 0 83 | 84 | 85 | def test_pypartitions_refresh(model): 86 | """Tests refreshing partitions of testingtable.""" 87 | assert len(model.Tables[testingtablename].Partitions.refresh()) > 0 88 | 89 | 90 | def test_pyobjects_adding(model): 91 | """Tests adding a PyObject.""" 92 | table = model.Tables.find(testingtablename) 93 | table += table 94 | assert len(table) == 2 95 | 96 | 97 | def test_nonetype_decimal_bug(model): 98 | """Tests the pesky nonetype decimal bug.""" 99 | query_str = """ 100 | EVALUATE 101 | { 102 | (1, CONVERT( 1.24, CURRENCY ), "Hello"), 103 | (2, CONVERT( 87661, CURRENCY ), "World"), 104 | (3,,"Test") 105 | } 106 | """ 107 | assert len(model.query(query_str)) == 3 108 | 109 | 110 | def test_table_last_refresh_times(model): 111 | """Really just testing the the function completes successfully and returns df.""" 112 | assert isinstance(model.Tables.last_refresh(), pd.DataFrame) 113 | 114 | 115 | def test_return_zero_row_tables(model): 116 | """Testing that `find_zero_rows()`.""" 117 | assert isinstance(model.Tables.find_zero_rows(), p.pytabular.PyTables) 118 | 119 | 120 | def test_get_dependencies(model): 121 | """Tests execution of `PyMeasure.get_dependencies()`.""" 122 | dependencies = model.Measures[0].get_dependencies() 123 | assert len(dependencies) > 0 124 | 125 | 126 | def test_disconnect(model): 127 | """Tests `Disconnect()` from `Tabular` class.""" 128 | model.disconnect() 129 | assert model.Server.Connected is False 130 | 131 | 132 | def test_reconnect(model): 133 | """Tests `Reconnect()` from `Tabular` class.""" 134 | model.reconnect() 135 | assert model.Server.Connected is True 136 | 137 | 138 | def test_reconnect_savechanges(model): 139 | """This will test the `reconnect()` gets called in `save_changes()`.""" 140 | model.disconnect() 141 | model.save_changes() 142 | assert model.Server.Connected is True 143 | 144 | 145 | def test_is_process(model): 146 | """Checks that `Is_Process()` from `Tabular` class returns bool.""" 147 | assert isinstance(model.is_process(), bool) 148 | 149 | 150 | def test_bad_table(model): 151 | """Checks for unable to find table exception.""" 152 | with pytest.raises(Exception): 153 | model.refresh("badtablename") 154 | 155 | 156 | def test_refresh_dict(model): 157 | """Checks for refreshing dictionary.""" 158 | table = model.Tables[testingtablename] 159 | refresh = model.refresh({table.Name: table.Partitions[0].Name}) 160 | assert isinstance(refresh, pd.DataFrame) 161 | 162 | 163 | def test_refresh_dict_pypartition(model): 164 | """Checks for refreshing dictionary with PyPartition.""" 165 | table = model.Tables[testingtablename] 166 | refresh = model.refresh({table.Name: table.Partitions[0]}) 167 | assert isinstance(refresh, pd.DataFrame) 168 | 169 | 170 | def test_bad_partition(model): 171 | """Checks for refreshing dictionary failure.""" 172 | table = model.Tables[testingtablename] 173 | with pytest.raises(Exception): 174 | model.refresh({table.Name: table.Partitions[0].Name + "fail"}) 175 | -------------------------------------------------------------------------------- /test/test_4measure.py: -------------------------------------------------------------------------------- 1 | """Bulk of pytests for `PyMeasure()` class.""" 2 | 3 | 4 | def test_create_measure(model): 5 | """Test Creating Measure.""" 6 | name = "Test Measure" 7 | expression = "1 + 4" 8 | folder = "Testing" 9 | model.Measures(name, expression, DisplayFolder=folder) 10 | query = model.query(f"EVALUATE {{[{name}]}}") 11 | ans = 5 12 | new_measure = model.Measures[name]._object 13 | new_measure.Parent.Measures.Remove(new_measure) 14 | model.save_changes() 15 | assert query == ans 16 | -------------------------------------------------------------------------------- /test/test_5column.py: -------------------------------------------------------------------------------- 1 | """pytest for the column.py file. Covers the PyColumn and PyColumns classes.""" 2 | 3 | from test.config import testingtablename 4 | import pandas as pd 5 | from numpy import int64 6 | 7 | 8 | def test_values(model): 9 | """Tests for `Values()` of PyColumn class.""" 10 | vals = model.Tables[testingtablename].Columns[1].values() 11 | assert isinstance(vals, pd.DataFrame) or isinstance(vals, int) 12 | 13 | 14 | def test_distinct_count_no_blank(model): 15 | """Tests No_Blank=True for `Distinct_Count()` of PyColumn class.""" 16 | vals = model.Tables[testingtablename].Columns[1].distinct_count(no_blank=True) 17 | assert isinstance(vals, int64) 18 | 19 | 20 | def test_distinct_count_blank(model): 21 | """Tests No_Blank=False for `Distinct_Count()` of PyColumn class.""" 22 | vals = model.Tables[testingtablename].Columns[1].distinct_count(no_blank=False) 23 | assert isinstance(vals, int64) 24 | 25 | 26 | def test_get_sample_values(model): 27 | """Tests for `get_sample_values()` of PyColumn class.""" 28 | sample_vals = model.Tables[testingtablename].Columns[1].get_sample_values() 29 | assert isinstance(sample_vals, pd.DataFrame) or isinstance(sample_vals, int) 30 | 31 | 32 | def test_query_every_column(model): 33 | """Tests `Query_All()` of PyColumns class.""" 34 | assert isinstance(model.Tables[testingtablename].Columns.query_all(), pd.DataFrame) 35 | 36 | 37 | def test_dependencies(model): 38 | """Tests execution of `PyColumn().get_dependencies()`.""" 39 | df = model.Tables[0].Columns[1].get_dependencies() 40 | assert isinstance(df, pd.DataFrame) 41 | -------------------------------------------------------------------------------- /test/test_6table.py: -------------------------------------------------------------------------------- 1 | """pytest for the table.py file. Covers the PyTable and PyTables classes.""" 2 | 3 | from test.config import testingtablename 4 | import pandas as pd 5 | from datetime import datetime 6 | 7 | 8 | def test_row_count(model): 9 | """Tests for `Row_Count()` of PyTable class.""" 10 | assert model.Tables[testingtablename].row_count() > 0 11 | 12 | 13 | def test_refresh(model): 14 | """Tests for `Refresh()` of PyTable class.""" 15 | assert isinstance(model.Tables[testingtablename].refresh(), pd.DataFrame) 16 | 17 | 18 | def test_last_refresh(model): 19 | """Tests for `Last_Refresh()` of PyTable class.""" 20 | assert isinstance(model.Tables[testingtablename].last_refresh(), datetime) 21 | 22 | 23 | def test_related(model): 24 | """Tests for `Related()` of PyTable class.""" 25 | assert isinstance(model.Tables[testingtablename].related(), type(model.Tables)) 26 | 27 | 28 | def test_refresh_pytables(model): 29 | """Tests for `Refresh()` of PyTables class.""" 30 | assert isinstance(model.Tables.find(testingtablename).refresh(), pd.DataFrame) 31 | 32 | 33 | def test_query_all_pytables(model): 34 | """Tests for `Query_All()` of PyTables class.""" 35 | assert isinstance(model.Tables.query_all(), pd.DataFrame) 36 | 37 | 38 | def test_find_zero_rows_pytables(model): 39 | """Tests for `Find_Zero_Rows()` of PyTables class.""" 40 | assert isinstance(model.Tables.find_zero_rows(), type(model.Tables)) 41 | 42 | 43 | def test_last_refresh_pytables_true(model): 44 | """Tests group_partition=True for `Last_Refresh()` of PyTables class.""" 45 | assert isinstance(model.Tables.last_refresh(group_partition=True), pd.DataFrame) 46 | 47 | 48 | def test_last_refresh_pytables_false(model): 49 | """Tests group_partition=False for `Last_Refresh()` of PyTables class.""" 50 | assert isinstance(model.Tables.last_refresh(group_partition=False), pd.DataFrame) 51 | -------------------------------------------------------------------------------- /test/test_7tabular_tracing.py: -------------------------------------------------------------------------------- 1 | """pytest for the table.py file. Covers the PyTable and PyTables classes.""" 2 | 3 | from test.config import testingtablename 4 | import pytabular as p 5 | from test.conftest import TestStorage 6 | 7 | 8 | def test_disconnect_for_trace(model): 9 | """Tests `disconnect()` from `Tabular` class.""" 10 | model.disconnect() 11 | assert model.Server.Connected is False 12 | 13 | 14 | def test_reconnect_update(model): 15 | """This will test the `reconnect()`. 16 | 17 | Gets called in `update()` of the `Base_Trace` class. 18 | """ 19 | model.disconnect() 20 | model.Tables[testingtablename].refresh() 21 | assert model.Server.Connected is True 22 | 23 | 24 | def test_query_monitor_start(model): 25 | """This will test the `QueryMonitor` trace and `start()` it.""" 26 | query_trace = p.QueryMonitor(model) 27 | query_trace.start() 28 | TestStorage.query_trace = query_trace 29 | assert TestStorage.query_trace.Trace.IsStarted 30 | 31 | 32 | def test_query_monitor_stop(model): 33 | """Tests `stop()` of `QueryMonitor` trace.""" 34 | TestStorage.query_trace.stop() 35 | assert TestStorage.query_trace.Trace.IsStarted is False 36 | 37 | 38 | def test_query_monitor_drop(model): 39 | """Tests `drop()` of `QueryMonitor` trace.""" 40 | assert TestStorage.query_trace.drop() is None 41 | -------------------------------------------------------------------------------- /test/test_8bpa.py: -------------------------------------------------------------------------------- 1 | """pytest for bpa.""" 2 | 3 | import pytabular as p 4 | from os import getcwd 5 | 6 | 7 | def test_bpa(model): 8 | """Testing execution of `model.analyze_bpa()`.""" 9 | te2 = p.TabularEditor(verify_download=False).exe 10 | bpa = p.BPA(verify_download=False).location 11 | assert isinstance(model.analyze_bpa(te2, bpa), list) 12 | 13 | 14 | def test_te2_custom_file_path(model): 15 | """Testing TE2 Class.""" 16 | assert isinstance(p.TabularEditor(getcwd()), p.TabularEditor) 17 | 18 | 19 | def test_bpa_custom_file_path(model): 20 | """Testing BPA Class.""" 21 | assert isinstance(p.BPA(getcwd()), p.BPA) 22 | -------------------------------------------------------------------------------- /test/test_9custom.py: -------------------------------------------------------------------------------- 1 | """These are tests I have for pretty janky features of PyTabular. 2 | 3 | These were designed selfishly for my own uses. 4 | So seperating out... To one day sunset and remove. 5 | """ 6 | 7 | from test.config import testingtablename 8 | 9 | 10 | def test_backingup_table(model): 11 | """Tests model.backup_table().""" 12 | model.backup_table(testingtablename) 13 | assert ( 14 | len( 15 | [ 16 | table 17 | for table in model.Model.Tables.GetEnumerator() 18 | if f"{testingtablename}_backup" == table.Name 19 | ] 20 | ) 21 | == 1 22 | ) 23 | 24 | 25 | def test_revert_table2(model): 26 | """Tests model.revert_table().""" 27 | model.revert_table(testingtablename) 28 | assert ( 29 | len( 30 | [ 31 | table 32 | for table in model.Model.Tables.GetEnumerator() 33 | if f"{testingtablename}" == table.Name 34 | ] 35 | ) 36 | == 1 37 | ) 38 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | env_list = docstring, linter, mkdocs, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13 3 | 4 | [testenv] 5 | description = "run unit tests" 6 | skip_install = True 7 | ; Currently managing these dependencies in two spots... 8 | ; pyproject.toml & tox.ini 9 | ; need to figure that out 10 | setenv = 11 | VIRTUALENV_NO_SETUPTOOLS=1 12 | deps = 13 | pythonnet>=3.0.3 14 | clr-loader>=0.2.6 15 | xmltodict==0.13.0 16 | pandas>=1.4.3 17 | requests>=2.28.1 18 | rich>=12.5.1 19 | pytest 20 | commands = 21 | pytest 22 | 23 | [testenv:docstring] 24 | description = check docstring coverage 25 | skip_install=True 26 | deps = 27 | docstr-coverage==2.3.2 28 | commands = docstr-coverage --skip-init 29 | 30 | [testenv:linter] 31 | description = run linters 32 | skip_install = True 33 | deps = flake8 34 | pep8-naming 35 | flake8-docstrings 36 | commands = flake8 pytabular test --count 37 | 38 | [testenv:mkdocs] 39 | description = check doc creation 40 | skip_install = True 41 | deps = 42 | mkdocstrings[python] 43 | mkdocs-material 44 | mkdocs 45 | mkdocs-gen-files 46 | commands = mkdocs build --------------------------------------------------------------------------------