├── .coveragerc ├── .flake8 ├── .github ├── dependabot.yaml └── workflows │ ├── publish.yaml │ └── tests.yaml ├── .gitignore ├── .pre-commit-config.yaml ├── .vscode └── settings.json ├── LICENSE ├── README.md ├── metrics_layer ├── __init__.py ├── cli │ ├── __init__.py │ ├── cli_commands.py │ └── seeding.py ├── core │ ├── __init__.py │ ├── convert │ │ ├── __init__.py │ │ └── convert.py │ ├── exceptions.py │ ├── model │ │ ├── __init__.py │ │ ├── base.py │ │ ├── dashboard.py │ │ ├── definitions.py │ │ ├── field.py │ │ ├── filter.py │ │ ├── join.py │ │ ├── join_graph.py │ │ ├── model.py │ │ ├── project.py │ │ ├── set.py │ │ ├── topic.py │ │ ├── view.py │ │ └── week_start_day_types.py │ ├── parse │ │ ├── .gitignore │ │ ├── __init__.py │ │ ├── connections.py │ │ ├── github_repo.py │ │ ├── manifest.py │ │ ├── project_dumper.py │ │ ├── project_loader.py │ │ ├── project_reader_base.py │ │ ├── project_reader_metricflow.py │ │ └── project_reader_metrics_layer.py │ ├── query │ │ ├── __init__.py │ │ └── query.py │ ├── sql │ │ ├── __init__.py │ │ ├── arbitrary_merge_resolve.py │ │ ├── merged_query_resolve.py │ │ ├── query_arbitrary_merged_queries.py │ │ ├── query_base.py │ │ ├── query_cumulative_metric.py │ │ ├── query_design.py │ │ ├── query_dialect.py │ │ ├── query_errors.py │ │ ├── query_filter.py │ │ ├── query_funnel.py │ │ ├── query_generator.py │ │ ├── query_merged_results.py │ │ ├── query_runner.py │ │ ├── resolve.py │ │ └── single_query_resolve.py │ └── utils.py └── integrations │ └── metricflow │ ├── __init__.py │ ├── metricflow_to_zenlytic.py │ └── metricflow_types.py ├── poetry.lock ├── poetry.toml ├── publish.sh ├── pyproject.toml ├── pytest.ini ├── release.sh └── tests ├── .gitkeep ├── __init__.py ├── config ├── dbt │ ├── .gitignore │ ├── analyses │ │ └── .gitkeep │ ├── dashboards │ │ └── sales_dashboard_dbt.yml │ ├── data │ │ └── .gitkeep │ ├── dbt_project.yml │ ├── macros │ │ └── .gitkeep │ ├── models │ │ └── example │ │ │ ├── order_LINES.sql │ │ │ ├── order_lines.yml │ │ │ ├── stg_customers.sql │ │ │ ├── stg_customers.yml │ │ │ ├── stg_discounts.sql │ │ │ ├── stg_discounts.yml │ │ │ ├── stg_order_lines.sql │ │ │ ├── stg_order_lines.yml │ │ │ ├── stg_orders.sql │ │ │ └── stg_orders.yml │ ├── profiles.yml │ ├── snapshots │ │ └── .gitkeep │ ├── tests │ │ └── .gitkeep │ └── zenlytic_project.yml ├── metricflow │ ├── dbt_project.yml │ ├── models │ │ ├── customers.yml │ │ ├── order_items.yml │ │ └── orders.yml │ ├── profiles.yml │ └── zenlytic_project.yml └── metrics_layer_config │ ├── dashboards │ ├── sales_dashboard.yml │ └── sales_dashboard_v2.yml │ ├── data_model │ ├── model_with_all_fields.yml │ └── view_with_all_fields.yml │ ├── dbt_models │ └── customers.sql │ ├── dbt_project.yml │ ├── models │ ├── commerce_test_model.yml │ └── new_model.yml │ ├── profiles │ ├── bq-test-service-account.json │ └── profiles.yml │ ├── topics │ ├── order_lines_only_topic.yml │ ├── order_lines_topic.yml │ ├── order_lines_topic_no_always_filters.yml │ └── recurring_revenue_topic.yml │ ├── views │ ├── accounts.yml │ ├── acquired_accounts.yml │ ├── customer_accounts.yml │ ├── monthly_aggregates.yml │ ├── mrr.yml │ ├── other_db_traffic.yml │ ├── test_clicked_on_page.yml │ ├── test_country_detail.yml │ ├── test_created_workspace.yml │ ├── test_customers.yml │ ├── test_discount_detail.yml │ ├── test_discounts.yml │ ├── test_events.yml │ ├── test_login_events.yml │ ├── test_order_lines.yml │ ├── test_orders.yml │ ├── test_sessions.yml │ ├── test_submitted_form.yml │ └── traffic.yml │ └── zenlytic_project.yml ├── conftest.py ├── integration ├── test_e2e_metricflow_to_zenlytic.py └── test_unit_metricflow_to_zenlytic.py ├── test_access_grants_fields.py ├── test_access_grants_queries.py ├── test_arbitrary_merged_results.py ├── test_cli.py ├── test_config_parse.py ├── test_configuration.py ├── test_cumulative_query.py ├── test_dashboards.py ├── test_field_mappings.py ├── test_funnels.py ├── test_join_query.py ├── test_join_query_raw.py ├── test_listing_functions.py ├── test_merged_results.py ├── test_mql_parse.py ├── test_non_additive_dimensions.py ├── test_project.py ├── test_project_validation.py ├── test_query_running.py ├── test_seeding.py ├── test_set_functions.py ├── test_simple_query.py ├── test_simple_query_raw.py ├── test_sql_query_resolver.py ├── test_subquery_filter_query.py ├── test_symmetric_aggregates.py ├── test_topic_based_query.py ├── test_user_attributes_query.py └── test_window_functions.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | source = metrics_layer/ 4 | 5 | [report] 6 | exclude_lines = 7 | if self.debug: 8 | pragma: no cover 9 | raise NotImplementedError 10 | if __name__ == .__main__.: 11 | ignore_errors = True 12 | omit = 13 | tests/* 14 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = B006,E203,W504,W503,B010,E711,F541 3 | max-line-length = 110 4 | 5 | -------------------------------------------------------------------------------- /.github/dependabot.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | # Maintain dependencies for GitHub Actions 4 | - package-ecosystem: "github-actions" 5 | directory: "/" 6 | schedule: 7 | # Check for updates to GitHub Actions every week 8 | interval: "weekly" -------------------------------------------------------------------------------- /.github/workflows/publish.yaml: -------------------------------------------------------------------------------- 1 | name: Metrics Layer Publish 2 | 3 | on: 4 | release: 5 | types: [published] 6 | 7 | permissions: 8 | contents: read 9 | 10 | jobs: 11 | publish: 12 | runs-on: "ubuntu-latest" 13 | 14 | steps: 15 | - uses: actions/checkout@v4 16 | 17 | - name: Set up Python 3.8 18 | uses: actions/setup-python@v5 19 | with: 20 | python-version: 3.8 21 | 22 | - name: Install dependencies 23 | run: | 24 | python -m pip install --upgrade pip 25 | pip install poetry==1.7.1 26 | pip install ".[all]" 27 | 28 | - name: Publish 29 | env: 30 | PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} 31 | run: | 32 | python -m poetry config pypi-token.pypi $PYPI_TOKEN 33 | poetry publish --build -------------------------------------------------------------------------------- /.github/workflows/tests.yaml: -------------------------------------------------------------------------------- 1 | name: Metrics Layer CI 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | strategy: 9 | matrix: 10 | python-version: ['3.8', '3.9', '3.10', '3.11'] 11 | 12 | steps: 13 | - uses: actions/checkout@v4 14 | 15 | - name: Set up Python ${{ matrix.python-version }} 16 | uses: actions/setup-python@v5 17 | with: 18 | python-version: ${{ matrix.python-version }} 19 | 20 | - name: Install dependencies 21 | run: | 22 | python -m pip install --upgrade pip 23 | pip install flake8 pytest pytest-cov pytest-mock pendulum[test] 24 | pip install ".[all]" 25 | 26 | - name: Lint with flake8 27 | run: | 28 | # stop the build if there are Python syntax errors or undefined names 29 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 30 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 31 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 32 | 33 | - name: Test with pytest 34 | run: | 35 | pytest --cov=metrics_layer/ --cov-report=xml 36 | 37 | - name: Report on code coverage 38 | uses: codecov/codecov-action@v5 39 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | metrics_layer/api/test.db 9 | .env 10 | .user.yml 11 | logs/ 12 | dbt_packages/ 13 | metrics_layer/core/parse/ssh_p8_key* 14 | 15 | # Distribution / packaging 16 | .Python 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | .eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | wheels/ 29 | pip-wheel-metadata/ 30 | share/python-wheels/ 31 | *.egg-info/ 32 | .installed.cfg 33 | *.egg 34 | MANIFEST 35 | 36 | # PyInstaller 37 | # Usually these files are written by a python script from a template 38 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 39 | *.manifest 40 | *.spec 41 | 42 | # Installer logs 43 | pip-log.txt 44 | pip-delete-this-directory.txt 45 | 46 | # Unit test / coverage reports 47 | htmlcov/ 48 | .tox/ 49 | .nox/ 50 | .coverage 51 | .coverage.* 52 | .cache 53 | nosetests.xml 54 | coverage.xml 55 | *.cover 56 | *.py,cover 57 | .hypothesis/ 58 | .pytest_cache/ 59 | 60 | # Translations 61 | *.mo 62 | *.pot 63 | 64 | # Django stuff: 65 | *.log 66 | local_settings.py 67 | db.sqlite3 68 | db.sqlite3-journal 69 | 70 | # Flask stuff: 71 | instance/ 72 | .webassets-cache 73 | 74 | # Scrapy stuff: 75 | .scrapy 76 | 77 | # Sphinx documentation 78 | docs/_build/ 79 | 80 | # PyBuilder 81 | target/ 82 | 83 | # Jupyter Notebook 84 | .ipynb_checkpoints 85 | 86 | # IPython 87 | profile_default/ 88 | ipython_config.py 89 | 90 | # pyenv 91 | .python-version 92 | 93 | # pipenv 94 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 95 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 96 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 97 | # install all needed dependencies. 98 | #Pipfile.lock 99 | 100 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 101 | __pypackages__/ 102 | 103 | # Celery stuff 104 | celerybeat-schedule 105 | celerybeat.pid 106 | 107 | # SageMath parsed files 108 | *.sage.py 109 | 110 | # Environments 111 | .env 112 | .venv 113 | env/ 114 | venv/ 115 | ENV/ 116 | env.bak/ 117 | venv.bak/ 118 | 119 | # Spyder project settings 120 | .spyderproject 121 | .spyproject 122 | 123 | # Rope project settings 124 | .ropeproject 125 | 126 | # mkdocs documentation 127 | /site 128 | 129 | # mypy 130 | .mypy_cache/ 131 | .dmypy.json 132 | dmypy.json 133 | 134 | # Pyre type checker 135 | .pyre/ 136 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # See https://pre-commit.com for more information 2 | # See https://pre-commit.com/hooks.html for more hooks 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v3.3.0 6 | hooks: 7 | - id: check-yaml 8 | - id: check-added-large-files 9 | - id: check-merge-conflict 10 | - id: check-toml 11 | - id: debug-statements 12 | - repo: https://github.com/PyCQA/isort 13 | rev: 5.9.3 14 | hooks: 15 | - id: isort 16 | args: 17 | - '--settings pyproject.toml' 18 | - repo: https://github.com/psf/black 19 | rev: 20.8b1 20 | hooks: 21 | - id: black 22 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.formatOnSave": true, 3 | "python.linting.pylintEnabled": false, 4 | "python.linting.flake8Enabled": true, 5 | "python.linting.flake8Path": "./.venv/bin/flake8", 6 | "python.linting.flake8Args": [ 7 | "--ignore=B006,E203,W504,W503,B010,E711,E402,F541", 8 | "--max-line-length=110" 9 | ], 10 | "python.linting.enabled": true, 11 | "python.formatting.provider": "none", 12 | "python.formatting.blackArgs": [ 13 | "--line-length=110" 14 | ], 15 | "python.formatting.blackPath": "./.venv/bin/black", 16 | "black-formatter.args": ["--config", "pyproject.toml"], 17 | "[python]": { 18 | "editor.defaultFormatter": "ms-python.black-formatter" 19 | } 20 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Metrics Layer 2 | 3 | ![Github Actions](https://github.com/Zenlytic/metrics_layer/actions/workflows/tests.yaml/badge.svg) 4 | [![codecov](https://codecov.io/gh/Zenlytic/metrics_layer/branch/master/graph/badge.svg?token=7JA6PKNV57)](https://codecov.io/gh/Zenlytic/metrics_layer) 5 | [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) 6 | 7 | # What is a Metrics Layer? 8 | 9 | Metrics Layer is an open source project with the goal of making access to metrics consistent throughout an organization. We believe you should be able to access consistent metrics from any tool you use to access data. This metrics layer is designed to work with [Zenlytic](https://zenlytic.com) as a BI tool. 10 | 11 | ## How does it work? 12 | 13 | [Zenlytic](https://zenlytic.com) is the only supported BI tool. The Metrics Layer will read your data model and give you the ability to access those metrics and dimensions in a python client library, or through SQL with a special `MQL` tag. 14 | 15 | Sound interesting? Here's how to set Metrics Layer up with your data model and start querying your metrics in **in under 2 minutes**. 16 | 17 | ## Installation 18 | 19 | Make sure that your data warehouse is one of the supported types. Metrics Layer currently supports Snowflake, BigQuery, Postgres, Druid (only SQL compilation, not running the query), DuckDB (only SQL compilation, not running the query), SQL Server (only SQL compilation, not running the query), and Redshift, and only works with `python >= 3.8` up to `python < 3.11`. 20 | 21 | Install Metrics Layer with the appropriate extra for your warehouse 22 | 23 | For Snowflake run `pip install metrics-layer[snowflake]` 24 | 25 | For BigQuery run `pip install metrics-layer[bigquery]` 26 | 27 | For Redshift run `pip install metrics-layer[redshift]` 28 | 29 | For Postgres run `pip install metrics-layer[postgres]` 30 | 31 | ## Profile set up 32 | 33 | There are several ways to set up a profile, we're going to look at the fastest one here. 34 | 35 | The fastest way to get connected is to pass the necessary information directly into Metrics Layer. Once you've installed the library with the warehouse you need, you should be able to run the code snippet below and start querying. 36 | 37 | You'll pull the repo from Github for this example. For more detail on getting set up, check out the [documentation](https://docs.zenlytic.com)! 38 | 39 | ``` 40 | from metrics_layer import MetricsLayerConnection 41 | 42 | # Give metrics_layer the info to connect to your data model and warehouse 43 | config = { 44 | "location": "https://myusername:myaccesstoken@github.com/myorg/myrepo.git", 45 | "branch": "develop", 46 | "connections": [ 47 | { 48 | "name": "mycompany", # The name of the connection in your data model (you'll see this in model files) 49 | "type": "snowflake", 50 | "account": "2e12ewdq.us-east-1", 51 | "username": "demo_user", 52 | "password": "q23e13erfwefqw", 53 | "database": "ANALYTICS", 54 | "schema": "DEV", # Optional 55 | } 56 | ], 57 | } 58 | conn = MetricsLayerConnection(**config) 59 | 60 | # You're off to the races. Query away! 61 | df = conn.query(metrics=["total_revenue"], dimensions=["channel", "region"]) 62 | ``` 63 | 64 | For more advanced methods of connection and more information about the project check out [the docs](https://docs.zenlytic.com) 65 | -------------------------------------------------------------------------------- /metrics_layer/__init__.py: -------------------------------------------------------------------------------- 1 | from metrics_layer.cli import cli_group # noqa 2 | from metrics_layer.core import MetricsLayerConnection # noqa 3 | 4 | try: 5 | import importlib.metadata as importlib_metadata 6 | except ModuleNotFoundError: 7 | import importlib_metadata 8 | 9 | __version__ = importlib_metadata.version(__name__) 10 | -------------------------------------------------------------------------------- /metrics_layer/cli/__init__.py: -------------------------------------------------------------------------------- 1 | from .cli_commands import * # noqa 2 | -------------------------------------------------------------------------------- /metrics_layer/core/__init__.py: -------------------------------------------------------------------------------- 1 | # from metrics_layer.core.parse import MetricsLayerConfiguration # noqa 2 | from metrics_layer.core.query import MetricsLayerConnection # noqa 3 | -------------------------------------------------------------------------------- /metrics_layer/core/convert/__init__.py: -------------------------------------------------------------------------------- 1 | from .convert import MQLConverter # noqa 2 | -------------------------------------------------------------------------------- /metrics_layer/core/exceptions.py: -------------------------------------------------------------------------------- 1 | class AccessDeniedOrDoesNotExistException(Exception): 2 | def __init__(self, message: str, object_name: str, object_type: str): 3 | self.message = message 4 | self.object_name = object_name 5 | self.object_type = object_type 6 | 7 | def __str__(self): 8 | return self.message 9 | 10 | 11 | class QueryError(Exception): 12 | def __init__(self, message: str): 13 | self.message = message 14 | 15 | def __str__(self): 16 | return self.message 17 | 18 | 19 | class JoinError(QueryError): 20 | pass 21 | -------------------------------------------------------------------------------- /metrics_layer/core/model/__init__.py: -------------------------------------------------------------------------------- 1 | from .definitions import Definitions # noqa 2 | from .project import Project # noqa 3 | -------------------------------------------------------------------------------- /metrics_layer/core/model/base.py: -------------------------------------------------------------------------------- 1 | import difflib 2 | import re 3 | from typing import Iterable 4 | 5 | from metrics_layer.core.exceptions import QueryError 6 | 7 | NAME_REGEX = re.compile(r"([A-Za-z0-9\_]+)") 8 | 9 | 10 | class MetricsLayerBase: 11 | def __init__(self, definition: dict = {}) -> None: 12 | self._definition = definition 13 | 14 | def __getattr__(self, attr: str): 15 | return self._definition.get(attr, None) 16 | 17 | def __repr__(self): 18 | return f"<{self.__class__.__name__} name={self.name}>" 19 | 20 | def to_dict(self): 21 | return {**self._definition} 22 | 23 | @staticmethod 24 | def valid_name(name: str): 25 | match = re.match(NAME_REGEX, name) 26 | if match is None: 27 | return False 28 | return match.group(1) == name 29 | 30 | @staticmethod 31 | def name_error(entity_name: str, name: str): 32 | return ( 33 | f"{entity_name.title()} name: {name} is invalid. Please reference " 34 | "the naming conventions (only letters, numbers, or underscores)" 35 | ) 36 | 37 | @staticmethod 38 | def _raise_query_error_from_cte(field_name: str): 39 | raise QueryError( 40 | f"Field {field_name} is not present in either source query, so it" 41 | " cannot be applied as a filter. Please add it to one of the source queries." 42 | ) 43 | 44 | @staticmethod 45 | def line_col(element): 46 | line = getattr(getattr(element, "lc", None), "line", None) 47 | column = getattr(getattr(element, "lc", None), "col", None) 48 | return line, column 49 | 50 | @staticmethod 51 | def invalid_property_error( 52 | definition: dict, valid_properties: Iterable[str], entity_name: str, name: str, error_func: callable 53 | ): 54 | errors = [] 55 | for key in definition: 56 | if key not in valid_properties: 57 | proposed_property = MetricsLayerBase.propose_property(key, valid_properties) 58 | proposed = f" Did you mean {proposed_property}?" if proposed_property else "" 59 | errors.append( 60 | error_func( 61 | definition[key], 62 | ( 63 | f"Property {key} is present on {entity_name.title()} {name}, but it is not a" 64 | f" valid property.{proposed}" 65 | ), 66 | ) 67 | ) 68 | return errors 69 | 70 | @staticmethod 71 | def field_name_parts(field_name: str): 72 | if field_name.count(".") == 1: 73 | view_name, name = field_name.split(".") 74 | else: 75 | view_name, name = None, field_name 76 | return view_name, name 77 | 78 | @staticmethod 79 | def propose_property(invalid_property_name: str, valid_properties: Iterable[str]) -> str: 80 | closest_match = difflib.get_close_matches(invalid_property_name, valid_properties, n=1) 81 | if closest_match: 82 | return closest_match[0] 83 | else: 84 | return "" 85 | 86 | 87 | class SQLReplacement: 88 | @staticmethod 89 | def fields_to_replace(text: str): 90 | matches = re.finditer(r"\$\{(.*?)\}", text, re.MULTILINE) 91 | return [match.group(1) for match in matches] 92 | -------------------------------------------------------------------------------- /metrics_layer/core/model/dashboard.py: -------------------------------------------------------------------------------- 1 | import json 2 | from typing import TYPE_CHECKING 3 | 4 | from metrics_layer.core.exceptions import ( 5 | AccessDeniedOrDoesNotExistException, 6 | QueryError, 7 | ) 8 | 9 | from .base import MetricsLayerBase 10 | from .filter import Filter 11 | 12 | if TYPE_CHECKING: 13 | from metrics_layer.core.model.project import Project 14 | 15 | 16 | class DashboardLayouts: 17 | grid = "grid" 18 | 19 | 20 | class DashboardElement(MetricsLayerBase): 21 | def __init__(self, definition: dict, dashboard, project) -> None: 22 | self.project: Project = project 23 | self.dashboard = dashboard 24 | self.validate(definition) 25 | super().__init__(definition) 26 | 27 | def get_model(self): 28 | model = self.project.get_model(self.model) 29 | if model is None: 30 | raise QueryError( 31 | f"Could not find model {self.model} referenced in dashboard {self.dashboard.name}." 32 | ) 33 | return model 34 | 35 | def validate(self, definition: dict): 36 | required_keys = ["model"] 37 | for k in required_keys: 38 | if k not in definition: 39 | raise QueryError(f"Dashboard Element missing required key {k}") 40 | 41 | def to_dict(self): 42 | definition = json.loads(json.dumps(self._definition)) 43 | definition["metrics"] = self.metrics 44 | definition["filters"] = self.parsed_filters(json_safe=True) 45 | return definition 46 | 47 | @property 48 | def slice_by(self): 49 | return self._definition.get("slice_by", []) 50 | 51 | @property 52 | def metrics(self): 53 | if "metric" in self._definition: 54 | metric_input = self._definition["metric"] 55 | else: 56 | metric_input = self._definition.get("metrics", []) 57 | return [metric_input] if isinstance(metric_input, str) else metric_input 58 | 59 | def _raw_filters(self): 60 | if self.filters is None: 61 | return [] 62 | return self.filters 63 | 64 | def parsed_filters(self, json_safe=False): 65 | to_add = {"week_start_day": self.get_model().week_start_day, "timezone": self.project.timezone} 66 | return [f for raw in self._raw_filters() for f in Filter({**raw, **to_add}).filter_dict(json_safe)] 67 | 68 | def _error(self, element, error, extra: dict = {}): 69 | return self.dashboard._error(element, error, extra) 70 | 71 | def collect_errors(self): 72 | errors = [] 73 | 74 | try: 75 | self.get_model() 76 | except (AccessDeniedOrDoesNotExistException, QueryError) as e: 77 | errors.append( 78 | self._error(self._definition.get("model"), str(e) + " in dashboard " + self.dashboard.name) 79 | ) 80 | 81 | for field in self.metrics: 82 | if not self._function_executes(self.project.get_field, field): 83 | err_msg = f"Could not find field {field} referenced in dashboard {self.dashboard.name}" 84 | errors.append(self._error(self._definition.get("metrics"), err_msg)) 85 | 86 | for field in self.slice_by: 87 | if not self._function_executes(self.project.get_field, field): 88 | err_msg = f"Could not find field {field} referenced in dashboard {self.dashboard.name}" 89 | errors.append(self._error(self._definition.get("slice_by"), err_msg)) 90 | 91 | for f in self._raw_filters(): 92 | if not self._function_executes(self.project.get_field, f.get("field")): 93 | err_msg = ( 94 | f"Could not find field {f.get('field')} referenced" 95 | f" in a filter in dashboard {self.dashboard.name}" 96 | ) 97 | errors.append(self._error(self._definition.get("filters"), err_msg)) 98 | return errors 99 | 100 | @staticmethod 101 | def _function_executes(func, argument, **kwargs): 102 | try: 103 | func(argument, **kwargs) 104 | return True 105 | except Exception: 106 | return False 107 | 108 | 109 | class Dashboard(MetricsLayerBase): 110 | def __init__(self, definition: dict, project) -> None: 111 | if definition.get("name") is not None: 112 | definition["name"] = definition["name"].lower() 113 | 114 | if definition.get("layout") is None: 115 | definition["layout"] = DashboardLayouts.grid 116 | 117 | self.project: Project = project 118 | self.validate(definition) 119 | super().__init__(definition) 120 | 121 | @property 122 | def label(self): 123 | if self._definition.get("label"): 124 | return self._definition.get("label") 125 | return self.name.replace("_", " ").title() 126 | 127 | def validate(self, definition: dict): 128 | required_keys = ["name", "layout"] 129 | for k in required_keys: 130 | if k not in definition: 131 | raise QueryError(f"Dashboard missing required key {k}") 132 | 133 | def to_dict(self): 134 | definition = json.loads(json.dumps(self._definition)) 135 | definition["elements"] = [e.to_dict() for e in self.elements()] 136 | definition["filters"] = self.parsed_filters(json_safe=True) 137 | return definition 138 | 139 | def _error(self, element, error, extra: dict = {}): 140 | line, column = self.line_col(element) 141 | return {**extra, "dashboard_name": self.name, "message": error, "line": line, "column": column} 142 | 143 | def collect_errors(self): 144 | errors = [] 145 | for f in self._raw_filters(): 146 | try: 147 | self.project.get_field(f["field"]) 148 | except Exception: 149 | err_msg = f"Could not find field {f['field']} referenced in a filter in dashboard {self.name}" 150 | errors.append(self._error(self._definition["filters"], err_msg)) 151 | 152 | for element in self.elements(): 153 | errors.extend(element.collect_errors()) 154 | return errors 155 | 156 | def printable_attributes(self): 157 | to_print = ["name", "label", "description"] 158 | attributes = self.to_dict() 159 | attributes["type"] = "dashboard" 160 | return {key: attributes.get(key) for key in to_print if attributes.get(key) is not None} 161 | 162 | def _raw_filters(self): 163 | if self.filters is None: 164 | return [] 165 | return self.filters 166 | 167 | def parsed_filters(self, json_safe=False): 168 | all_filters = [] 169 | info = {"timezone": self.project.timezone} 170 | week_start_days = set(e.get_model().week_start_day for e in self.elements()) 171 | if len(week_start_days) == 1: 172 | info["week_start_day"] = week_start_days.pop() 173 | else: 174 | info["week_start_day"] = None 175 | for f in self._raw_filters(): 176 | clean_filters = Filter({**f, **info}).filter_dict(json_safe) 177 | for clean_filter in clean_filters: 178 | if clean_filter != {}: 179 | all_filters.append(clean_filter) 180 | return all_filters 181 | 182 | def elements(self): 183 | elements = self._definition.get("elements", []) 184 | return [DashboardElement(e, dashboard=self, project=self.project) for e in elements] 185 | 186 | def _missing_filter_explore_error(self, filter_obj: dict): 187 | return ( 188 | f"Argument 'explore' not found in the the filter {filter_obj} on dashboard " 189 | f"{self.name}. The 'explore' argument is required on filters for the whole dashboard." 190 | ) 191 | -------------------------------------------------------------------------------- /metrics_layer/core/model/definitions.py: -------------------------------------------------------------------------------- 1 | class Definitions: 2 | snowflake = "SNOWFLAKE" 3 | bigquery = "BIGQUERY" 4 | redshift = "REDSHIFT" 5 | postgres = "POSTGRES" 6 | druid = "DRUID" 7 | sql_server = "SQL_SERVER" 8 | duck_db = "DUCK_DB" 9 | databricks = "DATABRICKS" 10 | azure_synapse = "AZURE_SYNAPSE" 11 | trino = "TRINO" 12 | mysql = "MYSQL" 13 | supported_warehouses = [ 14 | snowflake, 15 | bigquery, 16 | redshift, 17 | postgres, 18 | druid, 19 | sql_server, 20 | duck_db, 21 | databricks, 22 | azure_synapse, 23 | trino, 24 | mysql, 25 | ] 26 | symmetric_aggregates_supported_warehouses = [ 27 | snowflake, 28 | redshift, 29 | bigquery, 30 | postgres, 31 | duck_db, 32 | azure_synapse, 33 | sql_server, 34 | ] 35 | no_semicolon_warehouses = [druid, trino] 36 | needs_datetime_cast = [bigquery, trino] 37 | supported_warehouses_text = ", ".join(supported_warehouses) 38 | 39 | does_not_exist = "__DOES_NOT_EXIST__" 40 | canon_date_join_graph_root = "canon_date_core" 41 | 42 | date_format_tz = "%Y-%m-%dT%H:%M:%SZ" 43 | -------------------------------------------------------------------------------- /metrics_layer/core/model/set.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING, Union 2 | 3 | from metrics_layer.core.exceptions import QueryError 4 | 5 | from .base import MetricsLayerBase 6 | 7 | if TYPE_CHECKING: 8 | from metrics_layer.core.model.project import Project 9 | 10 | 11 | class Set(MetricsLayerBase): 12 | valid_properties = ["name", "fields", "view_name"] 13 | 14 | def __init__(self, definition: dict, project) -> None: 15 | self.validate(definition) 16 | 17 | self.project: Project = project 18 | super().__init__(definition) 19 | 20 | def validate(self, definition: dict): 21 | required_keys = ["name", "fields"] 22 | for k in required_keys: 23 | if k not in definition: 24 | raise QueryError(f"Set missing required key {k}") 25 | 26 | def _error(self, element, error, extra: dict = {}): 27 | line, column = self.line_col(element) 28 | error = {**extra, "message": error, "line": line, "column": column} 29 | if self.view_name: 30 | error["view_name"] = self.view_name 31 | return error 32 | 33 | def collect_errors(self): 34 | errors = [] 35 | if not self.valid_name(self.name): 36 | errors.append(self._error(self.name, self.name_error("set", self.name))) 37 | 38 | if "view_name" in self._definition and self.valid_name(self.view_name): 39 | try: 40 | self.project.get_view(self.view_name) 41 | except Exception as e: 42 | errors.append(self._error(self.view_name, f"In the Set {self.name} " + str(e))) 43 | 44 | if not isinstance(self.fields, list): 45 | errors.append( 46 | self._error( 47 | self._definition["fields"], 48 | f"The fields property, {self.fields} must be a list in the Set {self.name}", 49 | ) 50 | ) 51 | else: 52 | try: 53 | field_names = self.field_names() 54 | except Exception: 55 | # These errors are handled in the view in which the set is defined 56 | field_names = [] 57 | 58 | for field in field_names: 59 | try: 60 | self.project.get_field(field) 61 | except Exception as e: 62 | errors.append( 63 | self._error(self._definition["fields"], f"In the Set {self.name} " + str(e)) 64 | ) 65 | 66 | errors.extend( 67 | self.invalid_property_error( 68 | self._definition, self.valid_properties, "set", self.name, error_func=self._error 69 | ) 70 | ) 71 | return errors 72 | 73 | def field_names(self): 74 | all_field_names, names_to_exclude = [], [] 75 | for field_name in self.fields: 76 | # This means we're subtracting a set from the result 77 | if "*" in field_name and "-" in field_name: 78 | clean_name = field_name.replace("*", "").replace("-", "") 79 | fields_to_remove = self._internal_get_fields_from_set(clean_name) 80 | names_to_exclude.extend(fields_to_remove) 81 | 82 | # This means we're expanding a set into this set 83 | elif "*" in field_name: 84 | clean_name = field_name.replace("*", "") 85 | fields_to_add = self._internal_get_fields_from_set(clean_name) 86 | all_field_names.extend(fields_to_add) 87 | 88 | # This means we're removing a field from the result 89 | elif "-" in field_name: 90 | view_name, field_name = self.field_name_parts(field_name.replace("-", "")) 91 | view_name = self._get_view_name(view_name, field_name) 92 | names_to_exclude.append(f"{view_name}.{field_name}") 93 | 94 | # This is just a field that we're adding to the result 95 | else: 96 | view_name, field_name = self.field_name_parts(field_name) 97 | view_name = self._get_view_name(view_name, field_name) 98 | all_field_names.append(f"{view_name}.{field_name}") 99 | 100 | # Perform exclusion 101 | result_field_names = set(f for f in all_field_names if f not in names_to_exclude) 102 | 103 | return sorted(list(result_field_names), key=all_field_names.index) 104 | 105 | def _internal_get_fields_from_set(self, set_name: str): 106 | if set_name == "ALL_FIELDS": 107 | all_fields = self.project.fields( 108 | view_name=self.view_name, show_hidden=True, expand_dimension_groups=True 109 | ) 110 | return [f"{f.view.name}.{f.alias()}" for f in all_fields] 111 | 112 | view_name, set_name = self.field_name_parts(set_name) 113 | _set = self.project.get_set(set_name, view_name=view_name) 114 | if _set is None: 115 | print(f"WARNING: Could not find set with name {set_name}, disregarding those fields") 116 | return [] 117 | return _set.field_names() 118 | 119 | def _get_view_name(self, view_name: Union[None, str], field_name: str): 120 | if view_name: 121 | return view_name 122 | elif view_name is None and self.view_name: 123 | return self.view_name 124 | else: 125 | raise QueryError(f"Cannot find a valid view name for the field {field_name} in set {self.name}") 126 | -------------------------------------------------------------------------------- /metrics_layer/core/model/week_start_day_types.py: -------------------------------------------------------------------------------- 1 | class WeekStartDayTypes: 2 | monday = "monday" 3 | tuesday = "tuesday" 4 | wednesday = "wednesday" 5 | thursday = "thursday" 6 | friday = "friday" 7 | saturday = "saturday" 8 | sunday = "sunday" 9 | default = monday 10 | options = [monday, tuesday, wednesday, thursday, friday, saturday, sunday] 11 | -------------------------------------------------------------------------------- /metrics_layer/core/parse/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore downloaded data model repositories 2 | */ 3 | -------------------------------------------------------------------------------- /metrics_layer/core/parse/__init__.py: -------------------------------------------------------------------------------- 1 | from .project_loader import * # noqa 2 | from .connections import * # noqa 3 | from .github_repo import * # noqa 4 | from .project_dumper import * # noqa 5 | from .project_reader_metricflow import * # noqa 6 | from .project_reader_metrics_layer import * # noqa 7 | -------------------------------------------------------------------------------- /metrics_layer/core/parse/manifest.py: -------------------------------------------------------------------------------- 1 | from metrics_layer.core.exceptions import QueryError 2 | 3 | 4 | class Manifest: 5 | def __init__(self, definition: dict): 6 | self._definition = definition 7 | 8 | def exists(self): 9 | return self._definition is not None and self._definition != {} 10 | 11 | def get_model(self, model_name: str): 12 | return next( 13 | ( 14 | v 15 | for v in self._definition["nodes"].values() 16 | if v["resource_type"] == "model" and v["alias"] == model_name 17 | ), 18 | None, 19 | ) 20 | 21 | def models(self, schema: str = None, table: str = None): 22 | tables = [] 23 | for v in self._definition["nodes"].values(): 24 | is_model = v["resource_type"] == "model" 25 | 26 | # All tables in the whole database 27 | if is_model and schema is None and table is None: 28 | tables.append(self._node_to_table(v)) 29 | # All tables in the schema with not table specified 30 | elif is_model and v["schema"] == schema and table is None: 31 | tables.append(self._node_to_table(v)) 32 | # All tables matching the given table with not schema specified 33 | elif is_model and schema is None and v["alias"] == table: 34 | tables.append(self._node_to_table(v)) 35 | # All tables matching the given table and schema specified 36 | elif is_model and v["schema"] == schema and v["alias"] == table: 37 | tables.append(self._node_to_table(v)) 38 | 39 | return tables 40 | 41 | def _resolve_node(self, name: str): 42 | key = next((k for k in self._definition["nodes"].keys() if name == k.split(".")[-1]), None) 43 | if key is None: 44 | raise QueryError( 45 | f"Could not find the ref {name} in the co-located dbt project." 46 | " Please check the name in your dbt project." 47 | ) 48 | return self._definition["nodes"][key] 49 | 50 | def resolve_name(self, name: str, schema_override=None): 51 | node = self._resolve_node(name) 52 | if schema_override is None: 53 | return self._node_to_table(node) 54 | return f"{schema_override}.{node['alias']}" 55 | 56 | @staticmethod 57 | def _node_to_table(node: dict): 58 | return f"{node['schema']}.{node['alias']}" 59 | -------------------------------------------------------------------------------- /metrics_layer/core/parse/project_dumper.py: -------------------------------------------------------------------------------- 1 | import os 2 | from ruamel.yaml.comments import CommentedMap, CommentedSeq 3 | 4 | from metrics_layer.core.parse.project_reader_base import ProjectReaderBase 5 | 6 | 7 | class ProjectDumper(ProjectReaderBase): 8 | def __init__(self, models: list, model_folder: str, views: list, view_folder: str): 9 | self.models_to_dump = models 10 | self._model_folder = model_folder 11 | self.views_to_dump = views 12 | self._view_folder = view_folder 13 | 14 | def dump(self, path: str): 15 | for model in self.models_to_dump: 16 | if "_file_path" in model: 17 | file_path = os.path.join(path, model["_file_path"]) 18 | directory = os.path.dirname(file_path) 19 | if not os.path.exists(directory): 20 | os.makedirs(directory) 21 | else: 22 | file_name = model["name"] + "_model.yml" 23 | models_folder = os.path.join(path, self._model_folder) 24 | if not os.path.exists(models_folder): 25 | os.mkdir(models_folder) 26 | file_path = os.path.join(models_folder, file_name) 27 | self.dump_yaml_file(self._sort_model(model), file_path) 28 | 29 | for view in self.views_to_dump: 30 | if "_file_path" in view: 31 | file_path = os.path.join(path, view["_file_path"]) 32 | directory = os.path.dirname(file_path) 33 | if not os.path.exists(directory): 34 | os.makedirs(directory) 35 | else: 36 | file_name = view["name"] + "_view.yml" 37 | views_folder = os.path.join(path, self._view_folder) 38 | if not os.path.exists(views_folder): 39 | os.mkdir(views_folder) 40 | file_path = os.path.join(views_folder, file_name) 41 | self.dump_yaml_file(self._sort_view(view), file_path) 42 | 43 | def _sort_view(self, view: dict): 44 | view_key_order = [ 45 | "version", 46 | "type", 47 | "name", 48 | "label", 49 | "description", 50 | "model_name", 51 | "sql_table_name", 52 | "default_date", 53 | "row_label", 54 | "extends", 55 | "extension", 56 | "required_access_grants", 57 | "sets", 58 | "identifiers", 59 | "fields", 60 | ] 61 | extra_keys = [k for k in view.keys() if k not in view_key_order] 62 | new_view = CommentedMap() 63 | for k in view_key_order + extra_keys: 64 | if k in view: 65 | if k == "fields": 66 | new_view[k] = self._sort_fields(view[k]) 67 | else: 68 | new_view[k] = view[k] 69 | return new_view 70 | 71 | def _sort_fields(self, fields: list): 72 | sort_key = ["dimension", "dimension_group", "measure"] 73 | sorted_fields = sorted( 74 | fields, key=lambda x: (sort_key.index(x["field_type"]), -1 if "id" in x["name"] else 0) 75 | ) 76 | result_seq = CommentedSeq([self._sort_field(f) for f in sorted_fields]) 77 | for i in range(1, len(sorted_fields)): 78 | result_seq.yaml_set_comment_before_after_key(i, before="\n") 79 | return result_seq 80 | 81 | def _sort_field(self, field: dict): 82 | field_key_order = [ 83 | "name", 84 | "field_type", 85 | "type", 86 | "datatype", 87 | "hidden", 88 | "primary_key", 89 | "label", 90 | "view_label", 91 | "description", 92 | "required_access_grants", 93 | "value_format_name", 94 | "drill_fields", 95 | "sql_distinct_key", 96 | "tiers", 97 | "timeframes", 98 | "intervals", 99 | "sql_start", 100 | "sql_end", 101 | "sql", 102 | "filters", 103 | "extra", 104 | ] 105 | extra_keys = [k for k in field.keys() if k not in field_key_order] 106 | new_field = CommentedMap() 107 | for k in field_key_order + extra_keys: 108 | if k in field: 109 | new_field[k] = field[k] 110 | return new_field 111 | 112 | def _sort_model(self, model: dict): 113 | model_key_order = [ 114 | "version", 115 | "type", 116 | "name", 117 | "label", 118 | "connection", 119 | "fiscal_month_offset", 120 | "week_start_day", 121 | "access_grants", 122 | ] 123 | extra_keys = [k for k in model.keys() if k not in model_key_order] 124 | new_model = CommentedMap() 125 | for k in model_key_order + extra_keys: 126 | if k in model: 127 | new_model[k] = model[k] 128 | return new_model 129 | -------------------------------------------------------------------------------- /metrics_layer/core/parse/project_loader.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from metrics_layer.core.model.project import Project 4 | from metrics_layer.core.parse.connections import BaseConnection, connection_class_lookup 5 | 6 | from .github_repo import GithubRepo, LocalRepo 7 | from .manifest import Manifest 8 | from .project_reader_base import ProjectReaderBase 9 | from .project_reader_metricflow import MetricflowProjectReader 10 | from .project_reader_metrics_layer import MetricsLayerProjectReader 11 | 12 | 13 | class ConfigError(Exception): 14 | pass 15 | 16 | 17 | class ProjectLoader: 18 | def __init__(self, location: str, branch: str = "master", connections: list = [], **kwargs): 19 | self.kwargs = kwargs 20 | self.repo = self._get_repo(location, branch, kwargs) 21 | self._raw_connections = connections 22 | self._project = None 23 | self._user = None 24 | 25 | def load(self, private_key: str = None): 26 | self._connections = self.load_connections(self._raw_connections) 27 | self._project = self._load_project(private_key) 28 | return self._project 29 | 30 | @property 31 | def zenlytic_project(self): 32 | reader = ProjectReaderBase(repo=self.repo) 33 | return reader.zenlytic_project if reader.zenlytic_project else {} 34 | 35 | @staticmethod 36 | def profiles_path(): 37 | home_dir = os.path.expanduser("~") 38 | profiles_path = os.path.join(home_dir, ".dbt", "profiles.yml") 39 | return profiles_path 40 | 41 | def get_branch_options(self): 42 | return self.repo.branch_options 43 | 44 | def _load_project(self, private_key): 45 | self.repo.fetch(private_key=private_key) 46 | repo_type = self.repo.get_repo_type() 47 | if repo_type == "metricflow": 48 | reader = MetricflowProjectReader(repo=self.repo) 49 | elif repo_type == "metrics_layer": 50 | reader = MetricsLayerProjectReader(self.repo) 51 | else: 52 | raise TypeError(f"Unknown repo type: {repo_type}, valid types are 'metrics_layer', 'metricflow'") 53 | 54 | models, views, dashboards, topics, errors = reader.load() 55 | commit_hash = ( 56 | self.repo.git_repo.head.commit.hexsha 57 | if isinstance(self.repo, GithubRepo) and self.repo.git_repo is not None 58 | else None 59 | ) 60 | self.repo.delete() 61 | 62 | project = Project( 63 | models=models, 64 | views=views, 65 | dashboards=dashboards, 66 | topics=topics, 67 | connection_lookup={c.name: c.type for c in self._connections}, 68 | manifest=Manifest(reader.manifest), 69 | commit_hash=commit_hash, 70 | conversion_errors=errors, 71 | ) 72 | return project 73 | 74 | def _get_repo(self, location: str, branch: str, kwargs: dict): 75 | # Config is passed explicitly: this gets first priority 76 | if location is not None: 77 | return self._get_repo_from_location(location, branch, kwargs) 78 | 79 | # Next look for environment variables 80 | repo = self._get_repo_from_environment(kwargs) 81 | if repo: 82 | return repo 83 | 84 | raise ConfigError( 85 | """ We could not find a valid configuration in the environment. Try following the 86 | documentation (https://docs.zenlytic.com/docs/development_environment/development_environment) 87 | to properly set your environment variables, or pass the configuration explicitly 88 | """ 89 | ) 90 | 91 | @staticmethod 92 | def _get_repo_from_location(location: str, branch: str, kwargs: dict): 93 | if ProjectLoader._is_local(location): 94 | return LocalRepo(repo_path=location, **kwargs) 95 | return GithubRepo(repo_url=location, branch=branch, **kwargs) 96 | 97 | @staticmethod 98 | def _get_repo_from_environment(kwargs: dict): 99 | prefix = "METRICS_LAYER" 100 | location = os.getenv(f"{prefix}_LOCATION") 101 | branch = os.getenv(f"{prefix}_BRANCH", "master") 102 | repo_type = os.getenv(f"{prefix}_REPO_TYPE") 103 | if location is None: 104 | return None 105 | 106 | if ProjectLoader._is_local(location): 107 | return LocalRepo(repo_path=location, repo_type=repo_type, **kwargs) 108 | return GithubRepo(repo_url=location, branch=branch, repo_type=repo_type, **kwargs) 109 | 110 | @staticmethod 111 | def _is_local(location: str): 112 | is_http = "http://" in location.lower() or "https://" in location.lower() 113 | is_ssh = location.lower().startswith("git@") 114 | return not (is_http or is_ssh) 115 | 116 | @staticmethod 117 | def load_connections(connections: list): 118 | results = [] 119 | for connection in connections: 120 | if isinstance(connection, BaseConnection): 121 | connection_class = connection 122 | else: 123 | connection_type = connection["type"].upper() 124 | connection_class = connection_class_lookup[connection_type](**connection) 125 | results.append(connection_class) 126 | return results 127 | 128 | def get_connections_from_profile(profile_name: str, target: str = None): 129 | profile_path = ProjectLoader.profiles_path() 130 | profiles_directory = os.path.dirname(profile_path) 131 | profiles_dict = MetricsLayerProjectReader.read_yaml_if_exists(profile_path) 132 | if profiles_dict is None: 133 | raise ConfigError(f"Could not find dbt profiles.yml at {profile_path}") 134 | 135 | profile = profiles_dict.get(profile_name) 136 | 137 | if profile is None: 138 | raise ConfigError(f"Could not find profile {profile_name} in profiles.yml at {profile_path}") 139 | 140 | if target is None: 141 | target = profile["target"] 142 | 143 | target_dict = profile["outputs"].get(target) 144 | 145 | if target_dict is None: 146 | raise ConfigError( 147 | f"Could not find target {target} in profile {profile_name} in profiles.yml at {profile_path}" 148 | ) 149 | 150 | return [{**target_dict, "directory": profiles_directory, "name": profile_name}] 151 | -------------------------------------------------------------------------------- /metrics_layer/core/parse/project_reader_base.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import ruamel.yaml 4 | 5 | from .github_repo import BaseRepo 6 | 7 | 8 | class ProjectReaderBase: 9 | def __init__(self, repo: BaseRepo, profiles_dir: str = None): 10 | self.repo = repo 11 | self.profiles_dir = profiles_dir 12 | self.version = 1 13 | self.unloaded = True 14 | self.has_dbt_project = False 15 | self.manifest = {} 16 | self._models = [] 17 | self._views = [] 18 | self._dashboards = [] 19 | 20 | @property 21 | def models(self): 22 | if self.unloaded: 23 | self.load() 24 | return self._models 25 | 26 | @property 27 | def views(self): 28 | if self.unloaded: 29 | self.load() 30 | return self._views 31 | 32 | @property 33 | def dashboards(self): 34 | if self.unloaded: 35 | self.load() 36 | return self._dashboards 37 | 38 | @property 39 | def zenlytic_project(self): 40 | return self.read_yaml_if_exists(self.zenlytic_project_path) 41 | 42 | @property 43 | def zenlytic_project_path(self): 44 | zenlytic_project = self.read_yaml_if_exists(os.path.join(self.repo.folder, "zenlytic_project.yml")) 45 | if zenlytic_project: 46 | return os.path.join(self.repo.folder, "zenlytic_project.yml") 47 | return os.path.join(self.dbt_folder, "zenlytic_project.yml") 48 | 49 | @property 50 | def dbt_project(self): 51 | return self.read_yaml_if_exists(os.path.join(self.dbt_folder, "dbt_project.yml")) 52 | 53 | @property 54 | def dbt_folder(self): 55 | return self.repo.dbt_path if self.repo.dbt_path else self.repo.folder 56 | 57 | def get_folders(self, key: str, default: str = None, raise_errors: bool = True): 58 | if not self.zenlytic_project: 59 | return [] 60 | 61 | if key in self.zenlytic_project: 62 | return [self._abs_path(p) for p in self.zenlytic_project[key]] 63 | elif raise_errors: 64 | raise KeyError( 65 | f"Missing required key '{key}' in zenlytic_project.yml \n" 66 | "Learn more about setting these keys here: https://docs.zenlytic.com" 67 | ) 68 | return [] 69 | 70 | def _abs_path(self, path: str): 71 | if not os.path.isabs(path): 72 | path = os.path.join(self.repo.folder, path) 73 | return path 74 | 75 | def search_for_yaml_files(self, folders: list): 76 | file_names = self.repo.search("*.yml", folders) + self.repo.search("*.yaml", folders) 77 | return list(set(file_names)) 78 | 79 | @staticmethod 80 | def read_yaml_if_exists(file_path: str): 81 | if os.path.exists(file_path): 82 | return ProjectReaderBase.read_yaml_file(file_path) 83 | return None 84 | 85 | @staticmethod 86 | def read_yaml_file(path: str): 87 | yaml = ruamel.yaml.YAML(typ="rt") 88 | yaml.version = (1, 1) 89 | with open(path, "r") as f: 90 | yaml_dict = yaml.load(f) 91 | return yaml_dict 92 | 93 | @staticmethod 94 | def repr_str(representer, data): 95 | return representer.represent_str(str(data)) 96 | 97 | @staticmethod 98 | def dump_yaml_file(data: dict, path: str): 99 | yaml = ruamel.yaml.YAML(typ="rt") 100 | filtered_data = {k: v for k, v in data.items() if not k.startswith("_")} 101 | with open(path, "w") as f: 102 | yaml.dump(filtered_data, f) 103 | 104 | def load(self) -> None: 105 | raise NotImplementedError() 106 | -------------------------------------------------------------------------------- /metrics_layer/core/parse/project_reader_metricflow.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from metrics_layer.core.model import Project 4 | from metrics_layer.core.sql.query_design import MetricsLayerDesign 5 | from metrics_layer.integrations.metricflow.metricflow_to_zenlytic import ( 6 | convert_mf_project_to_zenlytic_project, 7 | load_mf_project, 8 | ) 9 | 10 | from .project_reader_base import ProjectReaderBase 11 | 12 | 13 | class MetricflowParsingException(Exception): 14 | pass 15 | 16 | 17 | class MetricflowProjectReader(ProjectReaderBase): 18 | def load(self) -> tuple: 19 | if self.dbt_project is None: 20 | raise MetricflowParsingException( 21 | "No dbt project found. Make sure you have a dbt_project.yml file, and it is accessible to" 22 | " Zenlytic." 23 | ) 24 | 25 | self.project_name = self.dbt_project["name"] 26 | 27 | if self.zenlytic_project and "metricflow-path" in self.zenlytic_project: 28 | metricflow_path = self.zenlytic_project["metricflow-path"] 29 | metricflow_path = os.path.join(self.repo.folder, metricflow_path) 30 | else: 31 | metricflow_path = self.dbt_folder 32 | 33 | metricflow_project = load_mf_project(metricflow_path) 34 | 35 | dbt_profile_name = self.dbt_project.get("profile", self.project_name) 36 | if self.zenlytic_project: 37 | self.profile_name = self.zenlytic_project.get("profile", dbt_profile_name) 38 | else: 39 | self.profile_name = dbt_profile_name 40 | 41 | topic_folders = self.get_folders("topic-paths", raise_errors=False) 42 | model_folders = self.get_folders("model-paths", raise_errors=False) 43 | all_folders = model_folders + topic_folders 44 | 45 | file_names = self.search_for_yaml_files(all_folders) 46 | 47 | models = [] 48 | topics = [] 49 | for fn in file_names: 50 | yaml_dict = self.read_yaml_file(fn) 51 | yaml_dict["_file_path"] = os.path.relpath(fn, start=self.repo.folder) 52 | 53 | yaml_type = yaml_dict.get("type") 54 | if yaml_type == "model": 55 | models.append(yaml_dict) 56 | elif yaml_type == "topic": 57 | topics.append(yaml_dict) 58 | 59 | if len(models) == 1: 60 | model_dict = models[0] 61 | elif len(models) > 1: 62 | raise MetricflowParsingException( 63 | "Multiple models found in model-paths. Only one model is supported with Metricflow." 64 | ) 65 | else: 66 | model_dict = {} 67 | 68 | models, views, errors = convert_mf_project_to_zenlytic_project( 69 | metricflow_project, self.profile_name, self.profile_name, model_dict 70 | ) 71 | if self.zenlytic_project.get("use_default_topics", True): 72 | topics += self.derive_topics(models, views, models[0]["name"]) 73 | 74 | return models, views, [], topics, errors 75 | 76 | def derive_topics(self, models: list, views: list, model_name: str) -> list: 77 | project = Project(models, views) 78 | graph = project.join_graph.graph 79 | sorted_components = project.join_graph._strongly_connected_components(graph) 80 | 81 | design = MetricsLayerDesign( 82 | no_group_by=False, 83 | query_type="SNOWFLAKE", # this doesn't matter 84 | field_lookup={}, 85 | model=None, 86 | project=project, 87 | ) 88 | 89 | topic_views = [] 90 | topics = {} 91 | 92 | for component_set in sorted_components: 93 | subgraph_nodes = project.join_graph._subgraph_nodes_from_components(graph, component_set) 94 | design._join_subgraph = project.join_graph.subgraph(subgraph_nodes) 95 | ordered_components = design.determine_join_order(subgraph_nodes) 96 | 97 | if len(ordered_components) == 0: 98 | ordered_views = subgraph_nodes 99 | else: 100 | ordered_views = [view for components in ordered_components for view in components] 101 | ordered_views = list(sorted(set(ordered_views), key=lambda x: ordered_views.index(x))) 102 | 103 | topic_views.append(ordered_views) 104 | 105 | # make sure all views have been used in a topic 106 | all_views = set(v.name for v in project.views()) 107 | all_topic_views = set([topic_view for topic_view_set in topic_views for topic_view in topic_view_set]) 108 | assert all_views == all_topic_views 109 | 110 | # make sure each topic has at least one view 111 | for tv in topic_views: 112 | assert len(tv) > 0 113 | 114 | topics = [] 115 | for i, topic_view_list in enumerate(topic_views): 116 | base_view = topic_view_list[0] 117 | topic_data = { 118 | "label": f"{base_view.replace('_', ' ').title()}", 119 | "base_view": base_view, 120 | "model_name": model_name, 121 | "views": {}, 122 | } 123 | for view in topic_view_list[1:]: 124 | try: 125 | design._join_subgraph = project.join_graph.subgraph([base_view, view]) 126 | ordered_join_components = design.determine_join_order([base_view, view]) 127 | if len(ordered_join_components) == 1: 128 | join = project.join_graph.get_join(base_view, view) 129 | else: 130 | connecting_view, destination_view = ordered_join_components[-1] 131 | join = project.join_graph.get_join(connecting_view, destination_view) 132 | 133 | join_type = join._definition.get("type") 134 | relationship = join._definition.get("relationship") 135 | sql_on = join._definition.get("sql_on") 136 | 137 | if join_type is not None and relationship is not None and sql_on is not None: 138 | topic_data["views"][view] = { 139 | "join": { 140 | "join_type": join_type, 141 | "relationship": relationship, 142 | "sql_on": sql_on, 143 | } 144 | } 145 | 146 | except Exception as e: 147 | print(f"error_getting_join_for_topic: {base_view} and {view}: {e}") 148 | 149 | topics.append(topic_data) 150 | 151 | return topics 152 | -------------------------------------------------------------------------------- /metrics_layer/core/parse/project_reader_metrics_layer.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from .project_reader_base import ProjectReaderBase 4 | 5 | 6 | class MetricsLayerProjectReader(ProjectReaderBase): 7 | def load(self) -> tuple: 8 | models, views, dashboards, topics = [], [], [], [] 9 | 10 | model_folders = self.get_folders("model-paths") 11 | view_folders = self.get_folders("view-paths") 12 | dashboard_folders = self.get_folders("dashboard-paths", raise_errors=False) 13 | topic_folders = self.get_folders("topic-paths", raise_errors=False) 14 | all_folders = model_folders + view_folders + dashboard_folders + topic_folders 15 | 16 | file_names = self.search_for_yaml_files(all_folders) 17 | 18 | for fn in file_names: 19 | yaml_dict = self.read_yaml_file(fn) 20 | yaml_dict["_file_path"] = os.path.relpath(fn, start=self.repo.folder) 21 | 22 | # Handle keyerror 23 | if "type" not in yaml_dict and "zenlytic_project" not in fn: 24 | print(f"WARNING: file {fn} is missing a type") 25 | 26 | yaml_type = yaml_dict.get("type") 27 | 28 | if yaml_type == "model": 29 | models.append(yaml_dict) 30 | elif yaml_type == "view": 31 | views.append(yaml_dict) 32 | elif yaml_type == "dashboard": 33 | dashboards.append(yaml_dict) 34 | elif yaml_type == "topic": 35 | topics.append(yaml_dict) 36 | elif yaml_type: 37 | print( 38 | f"WARNING: Unknown file type '{yaml_type}' options are 'model', 'view', 'dashboard', " 39 | "or 'topic'" 40 | ) 41 | 42 | return models, views, dashboards, topics, [] 43 | -------------------------------------------------------------------------------- /metrics_layer/core/query/__init__.py: -------------------------------------------------------------------------------- 1 | from .query import MetricsLayerConnection # noqa 2 | -------------------------------------------------------------------------------- /metrics_layer/core/sql/__init__.py: -------------------------------------------------------------------------------- 1 | from .query_runner import QueryRunner # noqa 2 | from .resolve import SQLQueryResolver # noqa 3 | -------------------------------------------------------------------------------- /metrics_layer/core/sql/query_arbitrary_merged_queries.py: -------------------------------------------------------------------------------- 1 | from pypika import AliasedQuery, Criterion, Order 2 | from pypika.terms import LiteralValue 3 | 4 | from metrics_layer.core.model.definitions import Definitions 5 | from metrics_layer.core.model.filter import LiteralValueCriterion 6 | from metrics_layer.core.model.join import ZenlyticJoinType 7 | from metrics_layer.core.sql.query_base import MetricsLayerQueryBase 8 | from metrics_layer.core.sql.query_dialect import NullSorting, query_lookup 9 | 10 | 11 | class MetricsLayerMergedQueries(MetricsLayerQueryBase): 12 | """Resolve the SQL query for multiple, arbitrary merged queries""" 13 | 14 | def __init__(self, definition: dict) -> None: 15 | self.query_lookup = query_lookup 16 | super().__init__(definition) 17 | 18 | def get_query(self, semicolon: bool = True): 19 | # Build the base_cte table from the referenced queries + join them with all dimensions 20 | base_cte_query = self.build_cte_from() 21 | 22 | # Add all columns in the SELECT clause 23 | select = self.get_select_columns() 24 | complete_query = base_cte_query.select(*select) 25 | if self.where: 26 | where = self.get_where_with_aliases( 27 | self.where, 28 | project=self.project, 29 | cte_alias_lookup=self.cte_alias_lookup, 30 | raise_if_not_in_lookup=True, 31 | ) 32 | complete_query = complete_query.where(Criterion.all(where)) 33 | 34 | if self.having: 35 | # These become a where because we're only dealing with aliases, post aggregation 36 | having = self.get_where_with_aliases( 37 | self.having, 38 | project=self.project, 39 | cte_alias_lookup=self.cte_alias_lookup, 40 | raise_if_not_in_lookup=True, 41 | ) 42 | complete_query = complete_query.where(Criterion.all(having)) 43 | 44 | if self.order_by: 45 | for order_clause in self.order_by: 46 | field = self.project.get_field(order_clause["field"]) 47 | order_by_alias = field.alias(with_view=True) 48 | if order_by_alias in self.cte_alias_lookup: 49 | order_by_alias = f"{self.cte_alias_lookup[order_by_alias]}.{order_by_alias}" 50 | else: 51 | self._raise_query_error_from_cte(field.id()) 52 | 53 | order = Order.desc if order_clause.get("sort", "asc").lower() == "desc" else Order.asc 54 | complete_query = complete_query.orderby( 55 | LiteralValue(order_by_alias), order=order, nulls=NullSorting.last 56 | ) 57 | 58 | sql = str(complete_query.limit(self.limit)) 59 | if semicolon: 60 | sql += ";" 61 | return sql 62 | 63 | def build_cte_from(self): 64 | base_cte_query = self._base_query() 65 | for query in self.merged_queries: 66 | base_cte_query = base_cte_query.with_(query["query"], query["cte_alias"]) 67 | 68 | base_cte_alias = self.merged_queries[0]["cte_alias"] 69 | base_cte_query = base_cte_query.from_(AliasedQuery(base_cte_alias)) 70 | 71 | # We're starting on the second one because the first one is the base in the from statement, 72 | # and all other queries are joined to it 73 | for query in self.merged_queries[1:]: 74 | # We have to do this because Redshift doesn't support a full outer join 75 | # of two CTE's without dimensions using 1=1 76 | if self.query_type == Definitions.redshift and len(query["join_fields"]) == 0: 77 | base_cte_query = base_cte_query.join(AliasedQuery(query["cte_alias"])).cross() 78 | else: 79 | criteria = self._build_join_criteria(query["join_fields"], base_cte_alias, query["cte_alias"]) 80 | zenlytic_join_type = query.get("join_type") 81 | if zenlytic_join_type is None: 82 | zenlytic_join_type = ZenlyticJoinType.left_outer 83 | join_type = self.pypika_join_type_lookup(zenlytic_join_type) 84 | base_cte_query = base_cte_query.join(AliasedQuery(query["cte_alias"]), how=join_type).on( 85 | criteria 86 | ) 87 | 88 | return base_cte_query 89 | 90 | def _build_join_criteria(self, join_logic: list, base_query_alias: str, joined_query_alias: str): 91 | # Join logic is a list with {'field': field_in_current_cte, 'source_field': field_in_base_cte} 92 | # No dimensions to join on, the query results must be just one number each 93 | if len(join_logic) == 0: 94 | return LiteralValueCriterion("1=1") 95 | 96 | join_criteria = [] 97 | for logic in join_logic: 98 | joined_field = self.project.get_field(logic["field"]) 99 | base_field = self.project.get_field(logic["source_field"]) 100 | base_alias_and_id = f"{base_query_alias}.{base_field.alias(with_view=True)}" 101 | joined_alias_and_id = f"{joined_query_alias}.{joined_field.alias(with_view=True)}" 102 | # We need to add casting for differing datatypes on dimension groups for BigQuery 103 | if Definitions.bigquery == self.query_type and base_field.datatype != joined_field.datatype: 104 | join_condition = ( 105 | f"CAST({base_alias_and_id} AS TIMESTAMP)=CAST({joined_alias_and_id} AS TIMESTAMP)" 106 | ) 107 | else: 108 | join_condition = f"{base_alias_and_id}={joined_alias_and_id}" 109 | join_criteria.append(join_condition) 110 | 111 | return LiteralValueCriterion(" and ".join(join_criteria)) 112 | 113 | # Code to handle SELECT portion of query 114 | def get_select_columns(self): 115 | self.cte_alias_lookup = {} 116 | select = [] 117 | existing_aliases = [] 118 | for query in self.merged_queries: 119 | # We do not want to include the join fields in the SELECT clause, 120 | # unless they are part of the primary (base) query 121 | for j in query.get("join_fields", []): 122 | field = self.project.get_field(j["field"]) 123 | 124 | alias = field.alias(with_view=True) 125 | existing_aliases.append(alias) 126 | 127 | field_ids = query.get("metrics", []) + query.get("dimensions", []) 128 | for field_id in field_ids: 129 | field = self.project.get_field(field_id) 130 | alias = field.alias(with_view=True) 131 | 132 | if alias not in existing_aliases: 133 | self.cte_alias_lookup[alias] = query["cte_alias"] 134 | select.append(self.sql(f"{query['cte_alias']}.{alias}", alias=alias)) 135 | existing_aliases.append(alias) 136 | 137 | return select 138 | -------------------------------------------------------------------------------- /metrics_layer/core/sql/query_base.py: -------------------------------------------------------------------------------- 1 | import re 2 | from copy import deepcopy 3 | from typing import Union 4 | 5 | import sqlparse 6 | from pypika import JoinType 7 | from pypika.terms import LiteralValue 8 | from sqlparse.tokens import Name, Punctuation 9 | 10 | from metrics_layer.core.model.base import MetricsLayerBase 11 | from metrics_layer.core.model.join import Join, ZenlyticJoinType 12 | from metrics_layer.core.sql.query_filter import MetricsLayerFilter 13 | 14 | 15 | class MetricsLayerQueryBase(MetricsLayerBase): 16 | def _base_query(self): 17 | return self.query_lookup[self.query_type] 18 | 19 | def get_where_with_aliases( 20 | self, filters: list, project, cte_alias_lookup: dict = {}, raise_if_not_in_lookup: bool = False 21 | ): 22 | where = [] 23 | for filter_clause in filters: 24 | filter_clause["query_type"] = self.query_type 25 | f = MetricsLayerFilter( 26 | definition=filter_clause, design=None, filter_type="where", project=project 27 | ) 28 | where.append( 29 | f.sql_query( 30 | alias_query=True, 31 | cte_alias_lookup=cte_alias_lookup, 32 | raise_if_not_in_lookup=raise_if_not_in_lookup, 33 | ) 34 | ) 35 | return where 36 | 37 | @staticmethod 38 | def parse_identifiers_from_clause(clause: str): 39 | if clause is None: 40 | return [] 41 | generator = list(sqlparse.parse(clause)[0].flatten()) 42 | 43 | field_names = [] 44 | for i, token in enumerate(generator): 45 | not_already_added = i == 0 or str(generator[i - 1]) != "." 46 | if token.ttype == Name and not_already_added: 47 | field_names.append(str(token)) 48 | 49 | if token.ttype == Punctuation and str(token) == ".": 50 | if generator[i - 1].ttype == Name and generator[i + 1].ttype == Name: 51 | field_names[-1] += f".{str(generator[i+1])}" 52 | return field_names 53 | 54 | @staticmethod 55 | def get_pypika_join_type(join: Join): 56 | return MetricsLayerQueryBase.pypika_join_type_lookup(join.join_type) 57 | 58 | @staticmethod 59 | def pypika_join_type_lookup(join_type: str): 60 | if join_type == ZenlyticJoinType.left_outer: 61 | return JoinType.left 62 | elif join_type == ZenlyticJoinType.inner: 63 | return JoinType.inner 64 | elif join_type == ZenlyticJoinType.full_outer: 65 | return JoinType.outer 66 | elif join_type == ZenlyticJoinType.cross: 67 | return JoinType.cross 68 | return JoinType.left 69 | 70 | @staticmethod 71 | def sql(sql: str, alias: Union[None, str] = None): 72 | if alias: 73 | return LiteralValue(sql + f" as {alias}") 74 | return LiteralValue(sql) 75 | 76 | @staticmethod 77 | def strip_alias(sql: str): 78 | stripped_sql = deepcopy(sql) 79 | matches = re.findall(r"(?i)\ as\ ", stripped_sql) 80 | if matches: 81 | alias = " AS " 82 | for match in matches: 83 | stripped_sql = stripped_sql.replace(match, alias) 84 | return alias.join(stripped_sql.split(alias)[:-1]) 85 | return sql 86 | 87 | 88 | class QueryKindTypes: 89 | merged = "MERGED" 90 | single = "SINGLE" 91 | -------------------------------------------------------------------------------- /metrics_layer/core/sql/query_errors.py: -------------------------------------------------------------------------------- 1 | class ParseError(Exception): 2 | pass 3 | 4 | 5 | class ArgumentError(Exception): 6 | pass 7 | -------------------------------------------------------------------------------- /metrics_layer/core/utils.py: -------------------------------------------------------------------------------- 1 | import random 2 | import string 3 | import uuid 4 | 5 | 6 | def generate_uuid(db_safe=False): 7 | if db_safe: 8 | return generate_random_password(40) 9 | return str(uuid.uuid4()) 10 | 11 | 12 | def generate_random_password(length): 13 | # Random string with the combination of lower and upper case 14 | letters = string.ascii_letters 15 | result_str = "".join(random.choice(letters) for i in range(length)) 16 | return result_str 17 | 18 | 19 | def flatten_filters(filters: list, return_nesting_depth: bool = False): 20 | nesting_depth = 0 21 | flat_list = [] 22 | 23 | def recurse(filter_obj, return_nesting_depth: bool): 24 | nonlocal nesting_depth 25 | if isinstance(filter_obj, dict): 26 | if "conditions" in filter_obj: 27 | nesting_depth += 1 28 | for f in filter_obj["conditions"]: 29 | recurse(f, return_nesting_depth) 30 | else: 31 | if return_nesting_depth: 32 | filter_obj["nesting_depth"] = nesting_depth 33 | flat_list.append(filter_obj) 34 | elif isinstance(filter_obj, list): 35 | nesting_depth += 1 36 | for item in filter_obj: 37 | recurse(item, return_nesting_depth) 38 | 39 | recurse(filters, return_nesting_depth=return_nesting_depth) 40 | return flat_list 41 | -------------------------------------------------------------------------------- /metrics_layer/integrations/metricflow/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zenlytic/metrics_layer/cd16a318a2be64675f515b36c728955c79cd4bdd/metrics_layer/integrations/metricflow/__init__.py -------------------------------------------------------------------------------- /metrics_layer/integrations/metricflow/metricflow_types.py: -------------------------------------------------------------------------------- 1 | class MetricflowMetricTypes: 2 | simple = "simple" 3 | derived = "derived" 4 | cumulative = "cumulative" 5 | ratio = "ratio" 6 | -------------------------------------------------------------------------------- /poetry.toml: -------------------------------------------------------------------------------- 1 | [virtualenvs] 2 | in-project = true 3 | prefer-active-python = true 4 | create = true 5 | -------------------------------------------------------------------------------- /publish.sh: -------------------------------------------------------------------------------- 1 | poetry publish --build --username $PYPI_USERNAME --password $PYPI_PASSWORD 2 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "metrics_layer" 3 | version = "0.14.4" 4 | description = "The open source metrics layer." 5 | authors = ["Paul Blankley "] 6 | keywords = ["Metrics Layer", "Business Intelligence", "Analytics"] 7 | readme = "README.md" 8 | license = "Apache 2.0" 9 | homepage = "https://github.com/Zenlytic/metrics_layer" 10 | repository = "https://github.com/Zenlytic/metrics_layer" 11 | documentation = "https://docs.zenlytic.com" 12 | 13 | [tool.poetry.dependencies] 14 | python = ">=3.8.1, <3.13" 15 | GitPython = ">=3.1.20" 16 | sqlparse = ">=0.4.1" 17 | PyPika = "^0.48.8" 18 | pandas = "^1.5.2" 19 | numpy = "^1.24.4" 20 | jinja2 = "^3.1.2" 21 | redshift-connector = {version = "^2.0.905", optional = true} 22 | snowflake-connector-python = {version = "^3.5.0", optional = true} 23 | pyarrow = {version = ">=10", optional = true} 24 | google-cloud-bigquery = {version = "^3.13.0", optional = true} 25 | psycopg2-binary = {version = "^2.9.9", optional = true} 26 | SQLAlchemy = {version = "^2.0.21", optional = true} 27 | networkx = "^2.8.2" 28 | click = "^8.0" 29 | colorama = "^0.4.4" 30 | "ruamel.yaml" = "^0.17.20" 31 | pendulum = "^3.0.0" 32 | PyYAML = "^6.0" 33 | 34 | [tool.poetry.dev-dependencies] 35 | pytest = "^6.2.5" 36 | black = "^24.3.0" 37 | flake8 = "^3.9.2" 38 | pre-commit = "^2.15.0" 39 | isort = "^5.9.3" 40 | pytest-cov = "^2.12.1" 41 | pytest-mock = "^3.6.1" 42 | pytest-xdist = "^3.5.0" 43 | pendulum = {version = "^3.0.0", extras = ["test"]} 44 | 45 | 46 | [tool.poetry.extras] 47 | snowflake = ["snowflake-connector-python", "pyarrow"] 48 | bigquery = ["google-cloud-bigquery", "pyarrow"] 49 | redshift = ["redshift-connector"] 50 | postgres = ["psycopg2-binary"] 51 | all = ["snowflake-connector-python", "google-cloud-bigquery", "pyarrow", "redshift-connector", "psycopg2-binary"] 52 | 53 | [tool.black] 54 | line-length = 110 55 | preview = true 56 | 57 | [tool.isort] 58 | profile = "black" 59 | 60 | [build-system] 61 | requires = ["poetry-core>=1.0.0"] 62 | build-backend = "poetry.core.masonry.api" 63 | 64 | [tool.poetry.scripts] 65 | # command_name = module_for_handler : function_for_handler 66 | metrics_layer = 'metrics_layer:cli_group' 67 | ml = 'metrics_layer:cli_group' 68 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | testpaths = tests 3 | markers = 4 | cli 5 | query 6 | project 7 | running 8 | dbt 9 | seeding 10 | validation 11 | filters 12 | primary_dt: mark this as a primary datetime test (included by default). It depends on a certain datetime to run. The primary tests test the bare minimum number of datetimes, usually just the current datetime. 13 | -------------------------------------------------------------------------------- /release.sh: -------------------------------------------------------------------------------- 1 | release="$1" 2 | 3 | if [[ -n $release ]]; then 4 | echo "Creating git tag with version $release" 5 | git commit --allow-empty -m "Release $release" 6 | git tag -a $release -m "Version $release" 7 | git push --tags 8 | else 9 | echo "Failed to create git tag. Pass a version to this script" 10 | exit 1 11 | fi 12 | -------------------------------------------------------------------------------- /tests/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zenlytic/metrics_layer/cd16a318a2be64675f515b36c728955c79cd4bdd/tests/.gitkeep -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zenlytic/metrics_layer/cd16a318a2be64675f515b36c728955c79cd4bdd/tests/__init__.py -------------------------------------------------------------------------------- /tests/config/dbt/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | profiles.yml 6 | .user.yml -------------------------------------------------------------------------------- /tests/config/dbt/analyses/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zenlytic/metrics_layer/cd16a318a2be64675f515b36c728955c79cd4bdd/tests/config/dbt/analyses/.gitkeep -------------------------------------------------------------------------------- /tests/config/dbt/dashboards/sales_dashboard_dbt.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: dashboard 3 | name: sales_dashboard 4 | label: Sales Dashboard (with campaigns) 5 | description: Sales data broken out by campaign and repurchasing behavior 6 | layout: grid 7 | filters: 8 | - field: orders.new_vs_repeat 9 | value: New 10 | 11 | elements: 12 | - title: First element 13 | type: plot 14 | model: test_model 15 | metric: orders.total_revenue 16 | slice_by: [orders.new_vs_repeat, order_lines.product_name] 17 | 18 | - title: Customer sales stats (by gender) 19 | type: table 20 | model: test_model 21 | metrics: 22 | - orders.total_revenue 23 | - orders.average_order_value 24 | - orders.number_of_orders 25 | slice_by: [customers.gender] 26 | filters: 27 | - field: order_lines.product_name 28 | value: -Handbag -------------------------------------------------------------------------------- /tests/config/dbt/data/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zenlytic/metrics_layer/cd16a318a2be64675f515b36c728955c79cd4bdd/tests/config/dbt/data/.gitkeep -------------------------------------------------------------------------------- /tests/config/dbt/dbt_project.yml: -------------------------------------------------------------------------------- 1 | 2 | # Name your project! Project names should contain only lowercase characters 3 | # and underscores. A good package name should reflect your organization's 4 | # name or the intended use of these models 5 | name: 'test_dbt_project' 6 | version: '1.0.0' 7 | config-version: 2 8 | 9 | # This setting configures which "profile" dbt uses for this project. 10 | profile: 'test_dbt_project' 11 | 12 | # These configurations specify where dbt should look for different types of files. 13 | # The `model-paths` config, for example, states that models in this project can be 14 | # found in the "models/" directory. You probably won't need to change these! 15 | model-paths: ["models"] 16 | analysis-paths: ["analyses"] 17 | test-paths: ["tests"] 18 | seed-paths: ["seeds"] 19 | macro-paths: ["macros"] 20 | snapshot-paths: ["snapshots"] 21 | 22 | target-path: "target" # directory which will store compiled SQL files 23 | clean-targets: # directories to be removed by `dbt clean` 24 | - "target" 25 | - "dbt_packages" 26 | 27 | 28 | # Configuring models 29 | # Full documentation: https://docs.getdbt.com/docs/configuring-models 30 | 31 | # In this example config, we tell dbt to build all models in the example/ directory 32 | # as tables. These settings can be overridden in the individual model files 33 | # using the `{{ config(...) }}` macro. 34 | models: 35 | test_dbt_project: 36 | # Config indicated by + and applies to all files under models/example/ 37 | example: 38 | +materialized: view 39 | -------------------------------------------------------------------------------- /tests/config/dbt/macros/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zenlytic/metrics_layer/cd16a318a2be64675f515b36c728955c79cd4bdd/tests/config/dbt/macros/.gitkeep -------------------------------------------------------------------------------- /tests/config/dbt/models/example/order_LINES.sql: -------------------------------------------------------------------------------- 1 | with source_data as ( 2 | 3 | select 4 | 5 | -- Backbone order lines information 6 | order_lines.order_line_id, 7 | order_lines.order_id, 8 | order_lines.order_date, 9 | order_lines.product_revenue, 10 | order_lines.channel, 11 | order_lines.parent_channel, 12 | order_lines.product_name, 13 | 14 | -- Order information 15 | orders.customer_id, 16 | orders.days_between_orders, 17 | orders.revenue, 18 | orders.sub_channel, 19 | orders.new_vs_repeat, 20 | 21 | -- Customer information 22 | customers.first_order_date, 23 | customers.second_order_date, 24 | customers.region, 25 | customers.gender, 26 | customers.last_product_purchased, 27 | customers.customer_ltv, 28 | customers.total_sessions 29 | 30 | from {{ ref('stg_order_lines')}} order_lines 31 | left join {{ ref('stg_orders')}} orders 32 | on order_lines.order_id=orders.order_id 33 | left join {{ ref('stg_customers')}} customers 34 | on orders.customer_id=customers.customer_id 35 | 36 | ) 37 | 38 | select * from source_data 39 | -------------------------------------------------------------------------------- /tests/config/dbt/models/example/order_lines.yml: -------------------------------------------------------------------------------- 1 | 2 | version: 2 3 | 4 | models: 5 | - name: order_LINES 6 | description: "Shopify Order Lines with order and customer information" 7 | meta: 8 | row_label: "Order line" 9 | default_date: order_date 10 | identifiers: 11 | - name: order_line_id 12 | type: primary 13 | sql: ${order_line_id} 14 | - name: order_id 15 | type: foreign 16 | sql: ${order_id} 17 | - name: customer_id 18 | type: foreign 19 | sql: ${customer_id} 20 | columns: 21 | - name: order_line_id 22 | description: "The primary key for this table" 23 | primary_key: true 24 | tests: 25 | - unique 26 | - not_null 27 | 28 | - name: order_id 29 | description: "The order_id for this table" 30 | is_dimension: true 31 | 32 | - name: order_date 33 | description: "The order_date for this table" 34 | 35 | - name: product_revenue 36 | description: "The product_revenue for this table" 37 | 38 | - name: channel 39 | description: "The channel for this table" 40 | is_dimension: true 41 | 42 | - name: between_first_order_and_now 43 | meta: 44 | field_type: dimension_group 45 | type: duration 46 | sql_start: ${order_date_date} 47 | sql_end: current_date() 48 | intervals: [day, week, month, quarter, year] 49 | 50 | - name: parent_channel 51 | description: "The parent_channel for this table" 52 | is_dimension: true 53 | 54 | - name: product_name 55 | description: "The product_name for this table" 56 | is_dimension: true 57 | 58 | - name: customer_id 59 | description: "The customer_id for this table" 60 | is_dimension: true 61 | meta: {tags: [customers]} 62 | 63 | - name: days_between_orders 64 | description: "The days_between_orders for this table" 65 | 66 | - name: revenue 67 | description: "The revenue for this table" 68 | 69 | - name: sub_channel 70 | description: "The sub_channel for this table" 71 | is_dimension: true 72 | 73 | - name: new_vs_repeat 74 | description: "The new_vs_repeat for this table" 75 | is_dimension: true 76 | 77 | - name: first_order_date 78 | description: "The first_order_date for this table" 79 | 80 | - name: second_order_date 81 | description: "The second_order_date for this table" 82 | 83 | - name: region 84 | description: "The region for this table" 85 | is_dimension: true 86 | 87 | - name: gender 88 | description: "The gender for this table" 89 | is_dimension: true 90 | 91 | - name: last_product_purchased 92 | description: "The last_product_purchased for this table" 93 | is_dimension: true 94 | 95 | - name: customer_ltv 96 | description: "The customer_ltv for this table" 97 | 98 | - name: total_sessions 99 | description: "The total_sessions for this table" 100 | 101 | metrics: 102 | - name: arpu 103 | label: Avg revenue per customer 104 | description: "The avg revenue per customer" 105 | 106 | calculation_method: derived 107 | expression: "{{metric('total_revenue')}} / {{metric('number_of_customers')}}" 108 | 109 | timestamp: order_date 110 | time_grains: [day, week, month, quarter, year] 111 | 112 | - name: test_nested_names 113 | label: Test nested metric 114 | description: "The avg revenue per customer" 115 | 116 | calculation_method: derived 117 | expression: "{{metric('total_revenue')}} / {{metric('total_rev')}}" 118 | 119 | timestamp: order_date 120 | time_grains: [day, week, month, quarter, year] 121 | 122 | - name: total_rev 123 | label: Total Revenue 124 | model: ref('order_LINES') 125 | description: "The total revenue for the period" 126 | 127 | calculation_method: sum 128 | expression: product_revenue 129 | 130 | timestamp: order_date 131 | time_grains: [day, week, month, quarter, year] 132 | 133 | dimensions: 134 | - "*" 135 | 136 | - name: total_revenue 137 | label: Total Revenue 138 | model: ref('order_LINES') 139 | description: "The total revenue for the period" 140 | 141 | calculation_method: sum 142 | expression: product_revenue 143 | 144 | timestamp: order_date 145 | time_grains: [day, week, month, quarter, year] 146 | 147 | dimensions: 148 | - "*" 149 | 150 | - name: number_of_customers 151 | label: Number of customers 152 | model: ref('order_LINES') 153 | description: "The number of customers" 154 | 155 | calculation_method: count_distinct 156 | expression: customer_id 157 | 158 | timestamp: order_date 159 | time_grains: [day, week, month, quarter, year] 160 | 161 | 162 | - name: new_customer_REVENUE 163 | label: New customer revenue 164 | model: ref('order_LINES') 165 | description: "Total revenue from new customers" 166 | 167 | calculation_method: sum 168 | expression: product_revenue 169 | 170 | timestamp: ORDER_DATE 171 | time_grains: [day, week, month, quarter, year] 172 | 173 | filters: 174 | - field: new_vs_repeat 175 | operator: "=" 176 | value: "'New'" 177 | 178 | meta: {team: Finance} 179 | 180 | - name: new_customer_date_filter 181 | label: New customer revenue 182 | model: ref('order_LINES') 183 | description: "Total revenue from new customers" 184 | 185 | calculation_method: sum 186 | expression: product_revenue 187 | 188 | timestamp: ORDER_DATE 189 | time_grains: [day, week, month, quarter, year] 190 | 191 | filters: 192 | - field: ORDER_DATE 193 | operator: ">=" 194 | value: "'2023-08-02'" -------------------------------------------------------------------------------- /tests/config/dbt/models/example/stg_customers.sql: -------------------------------------------------------------------------------- 1 | with source_data as ( 2 | 3 | select 4 | 5 | customer_id, 6 | first_order_date, 7 | second_order_date, 8 | region, 9 | gender, 10 | last_product_purchased, 11 | customer_ltv, 12 | total_sessions 13 | 14 | from raw.shopify.customers 15 | 16 | ) 17 | 18 | select * from source_data -------------------------------------------------------------------------------- /tests/config/dbt/models/example/stg_customers.yml: -------------------------------------------------------------------------------- 1 | 2 | version: 2 3 | 4 | models: 5 | - name: stg_customers 6 | label: Customers 7 | description: "Shopify customers" 8 | meta: 9 | identifiers: 10 | - name: order_id 11 | type: primary 12 | sql: ${order_id} 13 | - name: customer_id 14 | type: foreign 15 | sql: ${customer_id} 16 | columns: 17 | - name: customer_id 18 | description: "The primary key for this table" 19 | tests: 20 | - unique 21 | - not_null 22 | 23 | - name: customer_email 24 | description: "The customer's email address" 25 | is_dimension: true -------------------------------------------------------------------------------- /tests/config/dbt/models/example/stg_discounts.sql: -------------------------------------------------------------------------------- 1 | with source_data as ( 2 | 3 | select 4 | discount_id, 5 | order_id, 6 | country, 7 | order_date, 8 | discount_code, 9 | discount_amt 10 | from raw.shopify.discounts 11 | 12 | ) 13 | 14 | select * from source_data 15 | -------------------------------------------------------------------------------- /tests/config/dbt/models/example/stg_discounts.yml: -------------------------------------------------------------------------------- 1 | 2 | version: 2 3 | 4 | models: 5 | - name: stg_discounts 6 | description: "Shopify discounts" 7 | columns: 8 | - name: discount_id 9 | description: "The primary key for this table" 10 | tests: 11 | - unique 12 | - not_null 13 | -------------------------------------------------------------------------------- /tests/config/dbt/models/example/stg_order_lines.sql: -------------------------------------------------------------------------------- 1 | with source_data as ( 2 | 3 | select 4 | 5 | order_line_id, 6 | order_id, 7 | customer_id, 8 | order_date, 9 | product_revenue, 10 | channel, 11 | parent_channel, 12 | product_name 13 | 14 | from raw.shopify.order_lines 15 | 16 | ) 17 | 18 | select * from source_data -------------------------------------------------------------------------------- /tests/config/dbt/models/example/stg_order_lines.yml: -------------------------------------------------------------------------------- 1 | 2 | version: 2 3 | 4 | models: 5 | - name: stg_order_lines 6 | description: "Shopify order lines" 7 | columns: 8 | - name: order_line_id 9 | description: "The primary key for this table" 10 | tests: 11 | - unique 12 | - not_null 13 | -------------------------------------------------------------------------------- /tests/config/dbt/models/example/stg_orders.sql: -------------------------------------------------------------------------------- 1 | with source_data as ( 2 | 3 | select 4 | 5 | order_id, 6 | customer_id, 7 | order_date, 8 | days_between_orders, 9 | revenue, 10 | sub_channel, 11 | new_vs_repeat 12 | 13 | from raw.shopify.orders 14 | 15 | ) 16 | 17 | select * from source_data 18 | -------------------------------------------------------------------------------- /tests/config/dbt/models/example/stg_orders.yml: -------------------------------------------------------------------------------- 1 | 2 | version: 2 3 | 4 | models: 5 | - name: stg_orders 6 | description: "Shopify orders" 7 | meta: 8 | # default_date: order_date 9 | identifiers: 10 | - name: order_id 11 | type: primary 12 | sql: ${order_id} 13 | - name: customer_id 14 | type: foreign 15 | sql: ${customer_id} 16 | columns: 17 | - name: order_id 18 | description: "The primary key for this table" 19 | tests: 20 | - unique 21 | - not_null 22 | -------------------------------------------------------------------------------- /tests/config/dbt/profiles.yml: -------------------------------------------------------------------------------- 1 | test_dbt_project: 2 | target: temp 3 | outputs: 4 | temp: 5 | type: snowflake 6 | account: fake-url.us-east-1 7 | user: fake 8 | password: fake 9 | warehouse: fake 10 | database: fake 11 | schema: fake 12 | config: 13 | send_anonymous_usage_stats: false 14 | -------------------------------------------------------------------------------- /tests/config/dbt/snapshots/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zenlytic/metrics_layer/cd16a318a2be64675f515b36c728955c79cd4bdd/tests/config/dbt/snapshots/.gitkeep -------------------------------------------------------------------------------- /tests/config/dbt/tests/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zenlytic/metrics_layer/cd16a318a2be64675f515b36c728955c79cd4bdd/tests/config/dbt/tests/.gitkeep -------------------------------------------------------------------------------- /tests/config/dbt/zenlytic_project.yml: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Project name 4 | name: metrics_project 5 | mode: dbt 6 | 7 | dashboard-paths: ['dashboards'] 8 | 9 | view-paths: ['models'] 10 | 11 | # If you don't specify model-paths, the metrics layer will use the profile as the connection_name for the Zenlytic 12 | # In dbt projects it's safe to leave this out because there is only one database connection 13 | # You'll need to include multiple models if you have more than one database or you want to set access grants 14 | # model-paths: [''] -------------------------------------------------------------------------------- /tests/config/metricflow/dbt_project.yml: -------------------------------------------------------------------------------- 1 | 2 | # Name your project! Project names should contain only lowercase characters 3 | # and underscores. A good package name should reflect your organization's 4 | # name or the intended use of these models 5 | name: 'test_dbt_project' 6 | version: '1.0.0' 7 | config-version: 2 8 | 9 | # This setting configures which "profile" dbt uses for this project. 10 | profile: 'test_dbt_project' 11 | 12 | # These configurations specify where dbt should look for different types of files. 13 | # The `model-paths` config, for example, states that models in this project can be 14 | # found in the "models/" directory. You probably won't need to change these! 15 | model-paths: ["models"] 16 | analysis-paths: ["analyses"] 17 | test-paths: ["tests"] 18 | seed-paths: ["seeds"] 19 | macro-paths: ["macros"] 20 | snapshot-paths: ["snapshots"] 21 | 22 | target-path: "target" # directory which will store compiled SQL files 23 | clean-targets: # directories to be removed by `dbt clean` 24 | - "target" 25 | - "dbt_packages" 26 | 27 | 28 | # Configuring models 29 | # Full documentation: https://docs.getdbt.com/docs/configuring-models 30 | 31 | # In this example config, we tell dbt to build all models in the example/ directory 32 | # as tables. These settings can be overridden in the individual model files 33 | # using the `{{ config(...) }}` macro. 34 | models: 35 | test_dbt_project: 36 | # Config indicated by + and applies to all files under models/example/ 37 | example: 38 | +materialized: view 39 | -------------------------------------------------------------------------------- /tests/config/metricflow/models/customers.yml: -------------------------------------------------------------------------------- 1 | models: 2 | - name: customers 3 | columns: 4 | - name: customer_id 5 | description: The unique key of the orders mart. 6 | tests: 7 | - not_null 8 | - unique 9 | - name: customer_name 10 | description: Customers' full name. 11 | - name: count_lifetime_orders 12 | description: Total number of orders a customer has ever placed. 13 | - name: first_ordered_at 14 | description: The timestamp when a customer placed their first order. 15 | - name: last_ordered_at 16 | description: The timestamp of a customer's most recent order. 17 | - name: lifetime_spend_pretax 18 | description: The sum of all the pre-tax subtotals of every order a customer has placed. 19 | - name: lifetime_spend 20 | description: The sum of all the order totals (including tax) that a customer has ever placed. 21 | - name: customer_type 22 | description: Options are 'new' or 'returning', indicating if a customer has ordered more than once or has only placed their first order to date. 23 | tests: 24 | - accepted_values: 25 | values: ['new', 'returning'] 26 | 27 | semantic_models: 28 | - name: customers 29 | defaults: 30 | agg_time_dimension: first_ordered_at 31 | model: ref('customers') 32 | meta: 33 | sql_table_name: 'my-bigquery-project.my_dataset.customers' 34 | zenlytic: 35 | zoe_description: 'Customers table' 36 | entities: 37 | - name: customer 38 | expr: customer_id 39 | type: primary 40 | dimensions: 41 | - name: customer_name 42 | type: categorical 43 | - name: customer_type 44 | type: categorical 45 | - name: first_ordered_at 46 | type: time 47 | type_params: 48 | time_granularity: day 49 | - name: last_ordered_at 50 | type: time 51 | type_params: 52 | time_granularity: day 53 | measures: 54 | - name: count_lifetime_orders 55 | description: Total count of orders per customer. 56 | agg: sum 57 | - name: lifetime_spend_pretax 58 | description: Customer lifetime spend before taxes. 59 | agg: sum 60 | - name: lifetime_spend 61 | agg: sum 62 | description: Gross customer lifetime spend inclusive of taxes. 63 | 64 | metrics: 65 | # Simple metrics 66 | - name: customers_with_orders 67 | description: 'Unique count of customers placing orders' 68 | type: simple 69 | label: Customers w/ Orders 70 | config: 71 | group: 'Customer' 72 | meta: 73 | zenlytic: 74 | zoe_description: 'Distinct count of customers placing orders' 75 | type_params: 76 | measure: 77 | name: customers_with_orders 78 | - name: new_customer 79 | description: Unique count of new customers. 80 | label: New Customers 81 | type: simple 82 | type_params: 83 | measure: 84 | name: customers_with_orders 85 | filter: | 86 | {{ Dimension('customer__customer_type') }} = 'new' 87 | - name: jan_customers 88 | description: Unique count of customers who placed orders in January. 89 | label: January Customers 90 | type: simple 91 | type_params: 92 | measure: 93 | name: customers_with_orders 94 | filter: | 95 | {{ TimeDimension('customer__first_ordered_at') }} = '2024-01-01' 96 | -------------------------------------------------------------------------------- /tests/config/metricflow/models/order_items.yml: -------------------------------------------------------------------------------- 1 | semantic_models: 2 | - name: order_item 3 | defaults: 4 | agg_time_dimension: ordered_at 5 | description: | 6 | Items contatined in each order. The grain of the table is one row per order item. 7 | model: ref('order_items') 8 | entities: 9 | - name: order_item 10 | type: primary 11 | expr: order_item_id 12 | - name: order_id 13 | type: foreign 14 | expr: CAST(order_id AS VARCHAR) 15 | - name: product 16 | type: foreign 17 | expr: product_id 18 | dimensions: 19 | - name: ordered_at 20 | expr: ordered_at 21 | type: time 22 | type_params: 23 | time_granularity: day 24 | - name: is_food_item 25 | type: categorical 26 | - name: is_drink_item 27 | type: categorical 28 | - name: is_ad_item 29 | type: categorical 30 | measures: 31 | - name: revenue 32 | description: The revenue generated for each order item. Revenue is calculated as a sum of revenue associated with each product in an order. 33 | agg: sum 34 | expr: product_price 35 | - name: food_revenue 36 | description: The revenue generated for each order item. Revenue is calculated as a sum of revenue associated with each product in an order. 37 | agg: sum 38 | expr: case when is_food_item = 1 then product_price else 0 end 39 | - name: drink_revenue 40 | description: The revenue generated for each order item. Revenue is calculated as a sum of revenue associated with each product in an order. 41 | agg: sum 42 | expr: case when is_drink_item = 1 then product_price else 0 end 43 | - name: median_revenue 44 | description: The median revenue generated for each order item. 45 | agg: median 46 | expr: product_price 47 | - name: number_of_orders 48 | description: 'The unique number of orders placed.' 49 | agg: count_distinct 50 | expr: order_id 51 | create_metric: true 52 | 53 | metrics: 54 | # Simple metrics 55 | - name: revenue 56 | description: Sum of the product revenue for each order item. Excludes tax. 57 | type: simple 58 | label: Revenue 59 | type_params: 60 | measure: 61 | name: revenue 62 | - name: ad_revenue 63 | description: Sum of the product revenue for each order item. Excludes tax. 64 | type: simple 65 | label: Revenue 66 | type_params: 67 | measure: 68 | name: revenue 69 | filter: | 70 | {{ Dimension('is_ad_item') }} = 1 71 | - name: order_cost 72 | description: Sum of cost for each order item. 73 | label: Order Cost 74 | type: simple 75 | type_params: 76 | measure: 77 | name: order_cost 78 | - name: median_revenue 79 | description: The median revenue for each order item. Excludes tax. 80 | type: simple 81 | label: Median Revenue 82 | type_params: 83 | measure: 84 | name: median_revenue 85 | - name: food_revenue 86 | description: The revenue from food in each order 87 | label: Food Revenue 88 | type: simple 89 | type_params: 90 | measure: 91 | name: food_revenue 92 | - name: food_customers 93 | description: Unique count of customers who placed orders and had food. 94 | label: Food Customers 95 | type: simple 96 | type_params: 97 | measure: 98 | name: customers_with_orders 99 | filter: | 100 | {{ Metric('food_revenue', group_by=['order_id']) }} > 0 101 | - name: food_valid_new_and_jan_or_feb_customers 102 | description: Unique count of customers with many filters 103 | label: Food Customers 104 | type: simple 105 | type_params: 106 | measure: 107 | name: customers_with_orders 108 | filter: | 109 | {{ Dimension('customer__customer_type') }} = 'new' 110 | and ( {{ TimeDimension('customer__first_ordered_at') }} = '2024-01-01' or {{ TimeDimension('customer__first_ordered_at') }} = '2024-02-01' or {{ TimeDimension('customer__first_ordered_at') }} is null) 111 | #Ratio Metrics 112 | - name: food_revenue_pct 113 | description: The % of order revenue from food. 114 | label: Food Revenue % 115 | type: ratio 116 | type_params: 117 | numerator: 118 | name: food_revenue 119 | denominator: revenue 120 | - name: food_revenue_pct_diff_calc # TODO test this 121 | description: The % of order revenue from food. 122 | label: Food Revenue % 123 | type: ratio 124 | type_params: 125 | numerator: 126 | name: revenue 127 | filter: | 128 | {{ Dimension('is_food_item') }} = 1 129 | denominator: revenue 130 | - name: number_of_repeat_orders 131 | type: simple 132 | label: 'Repeat orders' 133 | type_params: 134 | measure: 135 | name: number_of_orders 136 | filter: | 137 | {{ Dimension('NEW_VS_REPEAT') }} != 'Repeat' 138 | - name: repurchase_rate 139 | description: 'Share of orders that are repeat' 140 | type: ratio 141 | label: 'Repurchase Rate' 142 | type_params: 143 | numerator: 144 | name: number_of_repeat_orders 145 | denominator: 146 | name: number_of_orders 147 | 148 | #Derived Metrics 149 | - name: revenue_growth_mom 150 | description: 'Percentage growth of revenue compared to 1 month ago. Excluded tax' 151 | type: derived 152 | label: Revenue Growth % M/M 153 | type_params: 154 | expr: (current_revenue - revenue_prev_month)*100/revenue_prev_month 155 | metrics: 156 | - name: revenue 157 | alias: current_revenue 158 | - name: revenue 159 | offset_window: 1 month 160 | alias: revenue_prev_month 161 | - name: order_gross_profit 162 | description: Gross profit from each order. 163 | type: derived 164 | label: Order Gross Profit 165 | type_params: 166 | expr: revenue - cost 167 | metrics: 168 | - name: revenue 169 | - name: order_cost 170 | alias: cost 171 | - name: pct_rev_from_ads 172 | description: Percentage of revenue from advertising. 173 | type: derived 174 | label: Percentage of Revenue from Advertising 175 | type_params: 176 | expr: ad_revenue / revenue 177 | metrics: 178 | - name: ad_revenue 179 | - name: revenue 180 | 181 | #Cumulative Metrics 182 | - name: cumulative_revenue 183 | description: The cumulative revenue for all orders. 184 | label: Cumulative Revenue (All Time) 185 | type: cumulative 186 | type_params: 187 | measure: 188 | name: revenue 189 | -------------------------------------------------------------------------------- /tests/config/metricflow/models/orders.yml: -------------------------------------------------------------------------------- 1 | models: 2 | - name: orders 3 | description: Order overview data mart, offering key details for each order inlcluding if it's a customer's first order and a food vs. drink item breakdown. One row per order. 4 | columns: 5 | - name: order_id 6 | description: The unique key of the orders mart. 7 | tests: 8 | - not_null 9 | - unique 10 | - name: customer_id 11 | description: The foreign key relating to the customer who placed the order. 12 | tests: 13 | - relationships: 14 | to: ref('stg_customers') 15 | field: customer_id 16 | - name: location_id 17 | description: The foreign key relating to the location the order was placed at. 18 | - name: order_total 19 | description: The total amount of the order in USD including tax. 20 | - name: ordered_at 21 | description: The timestamp the order was placed at. 22 | - name: count_food_items 23 | description: The number of individual food items ordered. 24 | - name: count_drink_items 25 | description: The number of individual drink items ordered. 26 | - name: count_items 27 | description: The total number of both food and drink items ordered. 28 | - name: subtotal_food_items 29 | description: The sum of all the food item prices without tax. 30 | - name: subtotal_drink_items 31 | description: The sum of all the drink item prices without tax. 32 | - name: subtotal 33 | description: The sum total of both food and drink item prices without tax. 34 | - name: order_cost 35 | description: The sum of supply expenses to fulfill the order. 36 | - name: location_name 37 | description: The full location name of where this order was placed. Denormalized from `stg_locations`. 38 | - name: is_first_order 39 | description: A boolean indicating if this order is from a new customer placing their first order. 40 | - name: is_food_order 41 | description: A boolean indicating if this order included any food items. 42 | - name: is_drink_order 43 | description: A boolean indicating if this order included any drink items. 44 | 45 | semantic_models: 46 | - name: orders 47 | defaults: 48 | agg_time_dimension: ordered_at 49 | description: | 50 | Order fact table. This table is at the order grain with one row per order. 51 | model: ref('orders') 52 | entities: 53 | - name: order_id 54 | type: primary 55 | - name: location 56 | type: foreign 57 | expr: location_id 58 | - name: customer 59 | type: foreign 60 | expr: customer_id 61 | dimensions: 62 | - name: ordered_at 63 | expr: ordered_at 64 | type: time 65 | type_params: 66 | time_granularity: day 67 | - name: order_total_dim 68 | type: categorical 69 | expr: order_total 70 | - name: is_food_order 71 | type: categorical 72 | - name: is_drink_order 73 | type: categorical 74 | measures: 75 | - name: order_total 76 | description: The total amount for each order including taxes. 77 | agg: sum 78 | - name: order_count 79 | expr: 1 80 | agg: sum 81 | - name: tax_paid 82 | description: The total tax paid on each order. 83 | agg: sum 84 | - name: customers_with_orders 85 | description: Distinct count of customers placing orders 86 | agg: count_distinct 87 | expr: customer_id 88 | - name: locations_with_orders 89 | description: Distinct count of locations with order 90 | expr: location_id 91 | agg: count_distinct 92 | - name: order_cost 93 | description: The cost for each order item. Cost is calculated as a sum of the supply cost for each order item. 94 | agg: sum 95 | - name: p75_order_total 96 | description: The 75th percentile order total 97 | expr: order_total 98 | agg: percentile 99 | create_metric: true 100 | agg_params: 101 | percentile: .75 102 | use_discrete_percentile: False 103 | - name: p99_order_total 104 | description: The 99th percentile order total 105 | expr: order_total 106 | agg: percentile 107 | create_metric: true 108 | agg_params: 109 | percentile: .99 110 | use_discrete_percentile: True 111 | 112 | metrics: 113 | - name: order_total 114 | description: Sum of total order amonunt. Includes tax + revenue. 115 | type: simple 116 | label: Order Total 117 | type_params: 118 | measure: 119 | name: order_total 120 | - name: large_order 121 | description: 'Count of orders with order total over 20.' 122 | type: simple 123 | label: 'Large Orders' 124 | type_params: 125 | measure: 126 | name: order_count 127 | filter: | 128 | {{ Dimension('orders__order_total_dim') }} >= 20 129 | - name: orders 130 | description: Count of orders. 131 | label: Orders 132 | type: simple 133 | type_params: 134 | measure: 135 | name: order_count 136 | - name: food_orders 137 | description: Count of orders that contain food order items 138 | label: Food Orders 139 | type: simple 140 | type_params: 141 | measure: 142 | name: order_count 143 | filter: | 144 | {{ Dimension('orders__is_food_order') }} = true 145 | -------------------------------------------------------------------------------- /tests/config/metricflow/profiles.yml: -------------------------------------------------------------------------------- 1 | test_dbt_project: 2 | target: temp 3 | outputs: 4 | temp: 5 | type: snowflake 6 | account: fake-url.us-east-1 7 | user: fake 8 | password: fake 9 | warehouse: fake 10 | database: fake 11 | schema: fake 12 | config: 13 | send_anonymous_usage_stats: false 14 | -------------------------------------------------------------------------------- /tests/config/metricflow/zenlytic_project.yml: -------------------------------------------------------------------------------- 1 | # Project name 2 | name: metrics_project 3 | mode: metricflow 4 | 5 | dashboard-paths: ['dashboards'] 6 | 7 | view-paths: ['models'] 8 | # If you don't specify model-paths, the metrics layer will use the profile as the connection_name for the Zenlytic 9 | # In dbt projects it's safe to leave this out because there is only one database connection 10 | # You'll need to include multiple models if you have more than one database or you want to set access grants 11 | # model-paths: [''] 12 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/dashboards/sales_dashboard.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: dashboard 3 | name: sales_dashboard 4 | label: Sales Dashboard (with campaigns) 5 | description: Sales data broken out by campaign and repurchasing behavior 6 | layout: grid 7 | filters: 8 | - field: orders.new_vs_repeat 9 | value: New 10 | 11 | elements: 12 | - title: First element 13 | type: plot 14 | model: test_model 15 | metric: orders.total_revenue 16 | slice_by: [orders.new_vs_repeat, order_lines.product_name] 17 | 18 | - title: Customer sales stats (by gender) 19 | type: table 20 | model: test_model 21 | metrics: 22 | - orders.total_revenue 23 | - orders.average_order_value 24 | - orders.number_of_orders 25 | slice_by: [customers.gender] 26 | filters: 27 | - field: order_lines.product_name 28 | value: -Handbag -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/dashboards/sales_dashboard_v2.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: dashboard 3 | name: sales_dashboard_v2 4 | label: Sales Dashboard (with campaigns) 5 | description: Sales data broken out by campaign and repurchasing behavior 6 | required_access_grants: [test_access_grant_department_dashboard] 7 | layout: grid 8 | 9 | elements: 10 | - title: First element 11 | type: plot 12 | model: test_model 13 | metric: orders.total_revenue 14 | slice_by: [orders.new_vs_repeat, order_lines.product_name] 15 | 16 | - title: Customer sales stats (by gender) 17 | type: table 18 | model: test_model 19 | metrics: 20 | - orders.total_revenue 21 | - orders.average_order_value 22 | - orders.number_of_orders 23 | slice_by: [customers.gender] -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/data_model/model_with_all_fields.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: model 3 | name: model_name 4 | label: "desired label name" 5 | connection: "connection_name" 6 | 7 | week_start_day: monday 8 | 9 | access_grants: 10 | - name: access_grant_name 11 | user_attribute: user_attribute_name 12 | allowed_values: ["value_1", "value_2"] 13 | - name: access_grant_name 14 | user_attribute: user_attribute_name 15 | allowed_values: ["value_1", "value_2"] 16 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/data_model/view_with_all_fields.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: view 3 | name: view_name 4 | 5 | sql_table_name: "{{ ref('customers') }}" 6 | default_date: dimension_group_name 7 | row_label: row_label 8 | row_special_code: customers 9 | extends: [view_name, view_name] 10 | extension: required 11 | required_access_grants: [access_grant_name, access_grant_name] 12 | 13 | sets: 14 | - name: set_name 15 | fields: [field_or_set, field_or_set] 16 | 17 | fields: 18 | - name: field_name 19 | field_type: "dimension" 20 | label: "desired label name" 21 | view_label: "desired label name" 22 | group_label: "label used to group dimensions in the field picker" 23 | group_item_label: "label to use for the field under its group label in the field picker" 24 | description: "description string" 25 | 26 | # This is like dbt meta tag, put whatever you want in here 27 | extra: 28 | zenlytic.exclude: 29 | - field_name 30 | zenlytic.include: 31 | - field_name 32 | parent: parent_field 33 | child: child_field 34 | hidden: no 35 | alias: [old_field_name, old_field_name] 36 | value_format: "excel-style formatting string" 37 | value_format_name: format_name 38 | sql: "SQL expression to generate the field value ;;" 39 | required_fields: [field_name, field_name] 40 | drill_fields: [field_or_set, field_or_set] 41 | can_filter: yes 42 | tags: ["string1", "string2"] 43 | type: field_type 44 | primary_key: no 45 | tiers: [N, N] 46 | sql_latitude: "SQL expression to generate a latitude ;;" 47 | sql_longitude: "SQL expression to generate a longitude ;;" 48 | required_access_grants: [access_grant_name, access_grant_name] 49 | order_by_field: dimension_name 50 | links: 51 | - label: "desired label name;" 52 | url: "desired_url" 53 | icon_url: "url_of_an_ico_file" 54 | - label: "desired label name;" 55 | url: "desired_url" 56 | icon_url: "url_of_an_ico_file" 57 | timeframes: [timeframe, timeframe] 58 | convert_tz: no 59 | datatype: timestamp 60 | intervals: [interval, interval] 61 | sql_start: "SQL expression for start time of duration ;;" 62 | sql_end: "SQL expression for end time of duration ;;" 63 | approximate: no 64 | approximate_threshold: N 65 | sql_distinct_key: "SQL expression to define repeated entities ;;" 66 | percentile: 90 67 | filters: 68 | - field: dimension_name 69 | value: "looker filter expression" 70 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/dbt_models/customers.sql: -------------------------------------------------------------------------------- 1 | with source as ( 2 | select * from shopify.customer 3 | ) 4 | 5 | select * from source 6 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/dbt_project.yml: -------------------------------------------------------------------------------- 1 | 2 | # Name your project! Project names should contain only lowercase characters 3 | # and underscores. A good package name should reflect your organization's 4 | # name or the intended use of these models 5 | name: 'test_dbt_project' 6 | version: '1.0.0' 7 | config-version: 2 8 | 9 | # This setting configures which "profile" dbt uses for this project. 10 | profile: 'sf_creds' 11 | 12 | # These configurations specify where dbt should look for different types of files. 13 | # The `model-paths` config, for example, states that models in this project can be 14 | # found in the "models/" directory. You probably won't need to change these! 15 | model-paths: ["dbt_models"] 16 | analysis-paths: ["analyses"] 17 | test-paths: ["tests"] 18 | seed-paths: ["seeds"] 19 | macro-paths: ["macros"] 20 | snapshot-paths: ["snapshots"] 21 | 22 | target-path: "target" # directory which will store compiled SQL files 23 | clean-targets: # directories to be removed by `dbt clean` 24 | - "target" 25 | - "dbt_packages" 26 | 27 | 28 | # Configuring models 29 | # Full documentation: https://docs.getdbt.com/docs/configuring-models 30 | 31 | # In this example config, we tell dbt to build all models in the example/ directory 32 | # as tables. These settings can be overridden in the individual model files 33 | # using the `{{ config(...) }}` macro. 34 | models: 35 | test_dbt_project: 36 | # Config indicated by + and applies to all files under models/example/ 37 | example: 38 | +materialized: view 39 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/models/commerce_test_model.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: model 3 | name: test_model 4 | label: 'Test commerce data' 5 | connection: testing_snowflake 6 | 7 | mappings: 8 | account_type: 9 | fields: 10 | [ 11 | z_customer_accounts.type_of_account, 12 | aa_acquired_accounts.account_type, 13 | mrr.customer_account_type, 14 | ] 15 | group_label: 'Accounts' 16 | description: 'The account type of the customer' 17 | source: 18 | fields: [sessions.utm_source, orders.sub_channel] 19 | group_label: 'Marketing' 20 | description: 'The source the customer came to our site from' 21 | campaign: 22 | fields: [sessions.utm_campaign, orders.campaign] 23 | gross_revenue: 24 | fields: [order_lines.total_item_revenue, orders.total_revenue] 25 | description: 'Gross revenue (product revenue + shipping - taxes)' 26 | context_os: 27 | fields: [submitted_form.context_os, clicked_on_page.context_os] 28 | description: 'Context OS (from the web tracker)' 29 | device: 30 | fields: [events.device, sessions.session_device, login_events.device] 31 | description: 'Device that made the request' 32 | 33 | access_grants: 34 | - name: test_access_grant_department_view 35 | user_attribute: department 36 | allowed_values: ['finance', 'executive', 'sales'] 37 | - name: test_access_grant_department_topic 38 | user_attribute: department 39 | allowed_values: ['executive'] 40 | - name: test_access_grant_department_customers 41 | user_attribute: department 42 | allowed_values: ['executive', 'marketing', 'sales'] 43 | - name: test_access_grant_department_field 44 | user_attribute: department 45 | allowed_values: ['executive', 'engineering', 'sales'] 46 | - name: test_access_grant_department_dashboard 47 | user_attribute: department 48 | allowed_values: ['executive', 'sales'] 49 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/models/new_model.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: model 3 | name: new_model 4 | connection: testing_snowflake 5 | default_convert_tz: no -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/profiles/bq-test-service-account.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "service_account", 3 | "project_id": "test-data-warehouse", 4 | "client_email": "metrics_layer@testing.iam.gserviceaccount.com" 5 | } -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/profiles/profiles.yml: -------------------------------------------------------------------------------- 1 | sf_creds: 2 | target: dev 3 | outputs: 4 | dev: 5 | type: snowflake 6 | account: 'xyz.us-east-1' 7 | user: test_user 8 | password: test_password 9 | role: test_role 10 | database: transformed 11 | schema: analytics 12 | 13 | bq_creds: 14 | target: prod 15 | outputs: 16 | prod: 17 | type: bigquery 18 | method: service-account 19 | keyfile: ./bq-test-service-account.json -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/topics/order_lines_only_topic.yml: -------------------------------------------------------------------------------- 1 | type: topic 2 | label: Order lines ONLY 3 | base_view: order_lines 4 | model_name: test_model 5 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/topics/order_lines_topic.yml: -------------------------------------------------------------------------------- 1 | type: topic 2 | label: Order lines Topic 3 | base_view: order_lines 4 | model_name: test_model 5 | description: Vanilla order lines topic description 6 | zoe_description: Secret info that is only shown to zoe 7 | hidden: false 8 | 9 | required_access_grants: ['test_access_grant_department_topic'] 10 | always_filter: 11 | - field: orders.revenue_dimension 12 | value: -NULL 13 | 14 | views: 15 | orders: {} 16 | 17 | customers: {} 18 | 19 | discounts: 20 | join: 21 | join_type: left_outer 22 | relationship: many_to_many 23 | sql_on: ${order_lines.order_id} = ${discounts.order_id} and ${discounts.order_date} is not null 24 | 25 | discount_detail: 26 | join: 27 | join_type: left_outer 28 | relationship: one_to_one 29 | sql_on: ${discounts.discount_id} = ${discount_detail.discount_id} and ${orders.order_id} = ${discount_detail.discount_order_id} 30 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/topics/order_lines_topic_no_always_filters.yml: -------------------------------------------------------------------------------- 1 | type: topic 2 | label: Order lines unfiltered 3 | base_view: order_lines 4 | model_name: test_model 5 | description: Vanilla order lines topic description 6 | zoe_description: Secret info that is only shown to zoe 7 | hidden: false 8 | 9 | access_filters: 10 | - field: orders.warehouse_location 11 | user_attribute: warehouse_location 12 | - field: order_lines.order_id 13 | user_attribute: allowed_order_ids 14 | 15 | views: 16 | orders: 17 | override_access_filters: true 18 | 19 | customers: {} 20 | 21 | discounts: 22 | join: 23 | join_type: left_outer 24 | relationship: many_to_many 25 | sql_on: ${order_lines.order_id} = ${discounts.order_id} and ${discounts.order_date} is not null 26 | 27 | discount_detail: 28 | join: 29 | # REMEBER TO TEST AS syntax 30 | join_type: left_outer 31 | relationship: one_to_one 32 | sql_on: ${discounts.discount_id} = ${discount_detail.discount_id} and ${orders.order_id} = ${discount_detail.discount_order_id} 33 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/topics/recurring_revenue_topic.yml: -------------------------------------------------------------------------------- 1 | type: topic 2 | label: Recurring Revenue 3 | base_view: mrr 4 | model_name: test_model 5 | description: Recurring revenue topic description 6 | 7 | views: 8 | accounts: {} 9 | child_account: {} 10 | parent_account: {} 11 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/views/accounts.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: view 3 | name: accounts 4 | 5 | sql_table_name: analytics.accounts 6 | default_date: created 7 | model_name: test_model 8 | 9 | identifiers: 10 | - name: account_id 11 | type: primary 12 | sql: ${account_id} 13 | 14 | - name: child_account_id 15 | type: primary 16 | sql: ${account_id} 17 | join_as: child_account 18 | join_as_label: 'Sub Account' 19 | 20 | - name: parent_account_id 21 | type: primary 22 | sql: ${account_id} 23 | join_as: parent_account 24 | join_as_field_prefix: 'Parent' 25 | include_metrics: yes 26 | 27 | fields: 28 | - name: account_id 29 | field_type: 'dimension' 30 | type: string 31 | primary_key: yes 32 | hidden: yes 33 | sql: '${TABLE}.account_id' 34 | 35 | - name: created 36 | field_type: 'dimension_group' 37 | type: time 38 | timeframes: [raw, time, date, week, month, quarter, year] 39 | sql: '${TABLE}.created_at' 40 | 41 | - name: account_name 42 | field_type: 'dimension' 43 | type: string 44 | sql: '${TABLE}.name' 45 | searchable: true 46 | 47 | - name: n_created_accounts 48 | field_type: 'measure' 49 | type: count 50 | sql: ${account_id} 51 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/views/acquired_accounts.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: view 3 | name: aa_acquired_accounts 4 | 5 | sql_table_name: analytics.accounts 6 | default_date: created 7 | model_name: test_model 8 | 9 | fields: 10 | - name: account_id 11 | field_type: 'dimension' 12 | type: string 13 | primary_key: yes 14 | hidden: yes 15 | sql: '${TABLE}.account_id' 16 | 17 | - name: created 18 | field_type: 'dimension_group' 19 | type: time 20 | timeframes: [raw, time, date, week, month, quarter, year] 21 | sql: '${TABLE}.created_at' 22 | 23 | - name: account_type 24 | field_type: 'dimension' 25 | type: string 26 | sql: '${TABLE}.type' 27 | searchable: true 28 | 29 | - name: number_of_acquired_accounts 30 | field_type: measure 31 | type: count 32 | sql: ${account_id} 33 | 34 | - name: number_of_acquired_accounts_missing 35 | field_type: measure 36 | type: count 37 | sql: ${account_id} 38 | filters: 39 | - field: account_id 40 | value: null 41 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/views/customer_accounts.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: view 3 | name: z_customer_accounts 4 | 5 | sql_table_name: analytics.customer_accounts 6 | model_name: test_model 7 | default_date: created 8 | identifiers: 9 | - name: account_id 10 | type: foreign 11 | sql: ${account_id} 12 | - name: customer_id 13 | type: foreign 14 | sql: ${customer_id} 15 | - name: customer_account 16 | type: primary 17 | identifiers: 18 | - name: account_id 19 | - name: customer_id 20 | 21 | fields: 22 | - name: unique_key 23 | field_type: 'dimension' 24 | type: string 25 | hidden: yes 26 | primary_key: yes 27 | sql: ${TABLE}.account_id || ${TABLE}.customer_id 28 | 29 | - name: account_id 30 | field_type: 'dimension' 31 | type: string 32 | hidden: yes 33 | sql: '${TABLE}.account_id' 34 | 35 | - name: customer_id 36 | field_type: 'dimension' 37 | type: string 38 | hidden: yes 39 | sql: '${TABLE}.customer_id' 40 | 41 | - name: created 42 | field_type: 'dimension_group' 43 | type: time 44 | timeframes: [raw, time, date, week, month, quarter, year] 45 | sql: '${TABLE}.created_at' 46 | 47 | - name: type_of_account 48 | field_type: 'dimension' 49 | type: string 50 | sql: '${TABLE}.account_type' 51 | searchable: true 52 | 53 | - name: number_of_account_customer_connections 54 | field_type: measure 55 | canon_date: ${created} 56 | type: count 57 | sql: ${unique_key} 58 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/views/monthly_aggregates.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: view 3 | name: monthly_aggregates 4 | 5 | sql_table_name: analytics.monthly_rollup 6 | default_date: record 7 | model_name: test_model 8 | 9 | fields: 10 | - name: record 11 | field_type: 'dimension_group' 12 | type: time 13 | timeframes: [raw, time, date, week, month, quarter, year] 14 | sql: '${TABLE}.record_date' 15 | primary_key: yes 16 | 17 | - name: division 18 | field_type: 'dimension' 19 | type: string 20 | sql: '${TABLE}.division' 21 | searchable: true 22 | 23 | - name: count_new_employees 24 | field_type: measure 25 | type: count 26 | sql: ${TABLE}.n_new_employees 27 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/views/mrr.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: view 3 | name: mrr 4 | 5 | sql_table_name: analytics.mrr_by_customer 6 | default_date: record 7 | model_name: test_model 8 | 9 | identifiers: 10 | - name: account_id 11 | type: foreign 12 | sql: ${account_id} 13 | 14 | - name: parent_account_id 15 | type: foreign 16 | sql: ${parent_account_id} 17 | 18 | - name: child_account_id 19 | type: foreign 20 | sql: ${child_account_id} 21 | 22 | fields: 23 | - name: unique_key 24 | field_type: 'dimension' 25 | type: string 26 | primary_key: yes 27 | hidden: yes 28 | sql: ${TABLE}.account_id || record_date 29 | 30 | - name: parent_account_id 31 | field_type: 'dimension' 32 | type: string 33 | hidden: yes 34 | sql: '${TABLE}.parent_account_id' 35 | 36 | - name: child_account_id 37 | field_type: 'dimension' 38 | type: string 39 | hidden: yes 40 | sql: '${TABLE}.child_account_id' 41 | 42 | - name: account_id 43 | field_type: 'dimension' 44 | type: string 45 | hidden: yes 46 | sql: '${TABLE}.account_id' 47 | 48 | - name: record 49 | field_type: 'dimension_group' 50 | type: time 51 | timeframes: [raw, time, date, week, month, quarter, year] 52 | sql: '${TABLE}.record_date' 53 | 54 | - name: mrr_value 55 | field_type: dimension 56 | type: number 57 | sql: ${TABLE}.mrr 58 | hidden: yes 59 | 60 | - name: customer_account_type 61 | field_type: dimension 62 | type: string 63 | sql: ${TABLE}.customer_account_type 64 | searchable: true 65 | 66 | - name: plan_name 67 | field_type: dimension 68 | type: string 69 | sql: ${TABLE}.plan_name 70 | searchable: true 71 | 72 | - name: number_of_billed_accounts 73 | field_type: measure 74 | type: count 75 | sql: ${parent_account_id} 76 | 77 | - name: accounts_beginning_of_month 78 | field_type: measure 79 | type: count 80 | sql: ${parent_account_id} 81 | non_additive_dimension: 82 | name: record_raw 83 | window_choice: min 84 | 85 | - name: accounts_end_of_month 86 | field_type: measure 87 | type: count_distinct 88 | sql: ${parent_account_id} 89 | non_additive_dimension: 90 | name: record_raw 91 | window_choice: max 92 | 93 | - name: mrr_end_of_month 94 | field_type: measure 95 | type: sum 96 | sql: ${mrr_value} 97 | non_additive_dimension: 98 | name: record_raw 99 | window_choice: max 100 | 101 | - name: mrr_beginning_of_month 102 | field_type: measure 103 | type: sum 104 | sql: ${mrr_value} 105 | non_additive_dimension: 106 | name: record_raw 107 | window_choice: min 108 | 109 | - name: mrr_end_of_month_by_account 110 | field_type: measure 111 | type: sum 112 | sql: ${mrr_value} 113 | non_additive_dimension: 114 | name: record_date 115 | window_choice: max 116 | window_groupings: 117 | - account_id 118 | 119 | - name: mrr_end_of_month_by_account_per_customer_connection 120 | field_type: measure 121 | type: number 122 | sql: ${mrr_end_of_month_by_account} / ${z_customer_accounts.number_of_account_customer_connections} 123 | 124 | - name: mrr_beginning_of_month_nulls_equal 125 | field_type: measure 126 | type: sum 127 | sql: ${mrr_value} 128 | non_additive_dimension: 129 | name: record_raw 130 | window_choice: min 131 | nulls_are_equal: true 132 | 133 | - name: mrr_beginning_of_month_no_group_by 134 | field_type: measure 135 | type: sum 136 | sql: ${mrr_value} 137 | non_additive_dimension: 138 | name: record_raw 139 | window_choice: min 140 | window_aware_of_query_dimensions: no 141 | 142 | - name: mrr_change_per_billed_account 143 | field_type: measure 144 | type: number 145 | sql: (${mrr_end_of_month} - ${mrr_beginning_of_month}) / ${number_of_billed_accounts} 146 | 147 | - name: mrr_end_of_month_by_account_no_group_by 148 | field_type: measure 149 | type: sum 150 | sql: ${mrr_value} 151 | non_additive_dimension: 152 | name: record_date 153 | window_choice: max 154 | window_aware_of_query_dimensions: no 155 | window_groupings: 156 | - account_id 157 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/views/other_db_traffic.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: view 3 | name: other_db_traffic 4 | 5 | sql_table_name: "{{ user_attributes['db_name'] }}.analytics.traffic" 6 | model_name: new_model 7 | 8 | fields: 9 | - name: other_traffic_id 10 | field_type: 'dimension' 11 | type: string 12 | primary_key: yes 13 | hidden: yes 14 | sql: '${TABLE}.id' 15 | 16 | - name: other_traffic_source 17 | field_type: 'dimension' 18 | type: string 19 | sql: '${TABLE}.traffic_source' 20 | searchable: false 21 | 22 | - name: original_traffic 23 | field_type: 'dimension_group' 24 | type: time 25 | timeframes: [raw, time, date, week, month, quarter, year] 26 | sql: '${TABLE}.date' 27 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/views/test_clicked_on_page.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: view 3 | name: clicked_on_page 4 | 5 | sql_table_name: analytics.clicked_on_page 6 | model_name: test_model 7 | 8 | default_date: session 9 | row_label: Session 10 | 11 | identifiers: 12 | - name: customer_join 13 | type: join 14 | reference: customers 15 | relationship: many_to_one 16 | sql_on: '${customers.customer_id}=${clicked_on_page.customer_id} AND ${clicked_on_page.session_date} is not null' 17 | 18 | fields: 19 | - name: session_id 20 | field_type: 'dimension' 21 | type: string 22 | primary_key: yes 23 | hidden: yes 24 | sql: '${TABLE}.id' 25 | 26 | - name: customer_id 27 | field_type: dimension 28 | type: string 29 | sql: ${TABLE}.customer_id 30 | searchable: true 31 | 32 | - name: context_os 33 | field_type: dimension 34 | type: string 35 | sql: ${TABLE}.context_os 36 | searchable: true 37 | 38 | - name: session 39 | field_type: 'dimension_group' 40 | type: time 41 | timeframes: [raw, time, date, week, month, quarter, year] 42 | sql: '${TABLE}.session_date' 43 | 44 | - name: number_of_clicks 45 | field_type: measure 46 | type: count 47 | sql: ${session_id} 48 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/views/test_country_detail.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: view 3 | name: country_detail 4 | model_name: test_model 5 | 6 | derived_table: 7 | sql: SELECT * FROM ANALYTICS.COUNTRY_DETAIL WHERE '{{ user_attributes['owned_region'] }}' = COUNTRY_DETAIL.REGION 8 | 9 | identifiers: 10 | - name: country_id 11 | type: primary 12 | sql: ${country_detail.country_id} 13 | - name: order_lines_join 14 | type: join 15 | reference: order_lines 16 | relationship: one_to_many 17 | join_type: left_outer 18 | sql_on: '${discounts.country}=${country_detail.country_id} and ${order_lines.order_date} is not null' 19 | 20 | fields: 21 | - name: country_id 22 | field_type: 'dimension' 23 | type: string 24 | primary_key: yes 25 | hidden: yes 26 | sql: ${TABLE}.country 27 | group_label: "ID's" 28 | 29 | - name: rainfall 30 | field_type: 'dimension' 31 | type: number 32 | sql: ${TABLE}.rain 33 | 34 | - name: rainfall_at 35 | field_type: dimension_group 36 | type: time 37 | timeframes: 38 | - raw 39 | - date 40 | - week 41 | - month 42 | sql: > 43 | case when '{{ query_attributes['dimension_group'] }}' = 'raw' then ${TABLE}.rain_date when '{{ query_attributes['dimension_group'] }}' = 'date' then time_bucket('1 day', ${TABLE}.rain_date) when '{{ query_attributes['dimension_group'] }}' = 'week' then time_bucket('1 week', ${TABLE}.rain_date) when '{{ query_attributes['dimension_group'] }}' = 'month' then time_bucket('1 month', ${TABLE}.rain_date) end 44 | 45 | - name: avg_rainfall 46 | field_type: 'measure' 47 | type: average 48 | canon_date: order_lines.order 49 | sql: ${rainfall} 50 | 51 | - name: avg_rainfall_adj 52 | field_type: 'measure' 53 | type: average 54 | canon_date: order_lines.order 55 | sql: ${rainfall} 56 | filters: 57 | - field: country_id 58 | value: "{{ user_attributes['country_options'] }}" 59 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/views/test_created_workspace.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: view 3 | name: created_workspace 4 | 5 | sql_table_name: analytics.created_workspace 6 | model_name: test_model 7 | 8 | default_date: created 9 | 10 | identifiers: 11 | - name: customer_id 12 | type: foreign 13 | sql: ${TABLE}.customer_id 14 | 15 | always_filter: 16 | - field: customers.is_churned 17 | value: FALSE 18 | - field: context_os 19 | value: -NULL 20 | - field: context_os 21 | value: 1, Google, os:iOS 22 | - field: session_id 23 | value: -1, -44, -087 24 | 25 | fields: 26 | - name: session_id 27 | field_type: 'dimension' 28 | type: number 29 | primary_key: yes 30 | hidden: yes 31 | sql: '${TABLE}.id' 32 | 33 | - name: customer_id 34 | field_type: dimension 35 | type: string 36 | sql: ${TABLE}.customer_id 37 | searchable: false 38 | 39 | - name: context_os 40 | field_type: dimension 41 | type: string 42 | sql: ${TABLE}.context_os 43 | searchable: true 44 | 45 | - name: created 46 | field_type: 'dimension_group' 47 | type: time 48 | timeframes: [raw, time, date, week, month, quarter, year] 49 | sql: '${TABLE}.session_date' 50 | 51 | - name: number_of_workspace_creations 52 | field_type: measure 53 | type: count 54 | sql: ${session_id} 55 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/views/test_customers.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: view 3 | name: customers 4 | 5 | sql_table_name: "{{ ref('customers') }}" 6 | default_date: first_order 7 | row_label: 'Customer' 8 | required_access_grants: [test_access_grant_department_customers] 9 | model_name: test_model 10 | 11 | identifiers: 12 | - name: customer_id 13 | type: primary 14 | sql: ${customer_id} 15 | 16 | fields: 17 | - name: customer_id 18 | field_type: 'dimension' 19 | type: string 20 | primary_key: yes 21 | hidden: yes 22 | sql: '${TABLE}.customer_id' 23 | group_label: "ID's" 24 | tags: ['customer'] 25 | 26 | - name: first_order 27 | field_type: 'dimension_group' 28 | type: time 29 | timeframes: [raw, time, date, week, month, quarter, year] 30 | sql: '${TABLE}.first_order_date' 31 | 32 | - name: cancelled 33 | field_type: 'dimension_group' 34 | type: time 35 | timeframes: [raw, time, date, week, month, quarter, year] 36 | sql: '${TABLE}.cancelled_date' 37 | 38 | - name: number_of_customers 39 | field_type: measure 40 | type: count 41 | sql: '${customer_id}' 42 | 43 | - name: region 44 | field_type: 'dimension' 45 | type: string 46 | sql: '${TABLE}.region' 47 | searchable: true 48 | 49 | - name: gender 50 | field_type: 'dimension' 51 | type: string 52 | sql: '${TABLE}.gender' 53 | searchable: true 54 | 55 | - name: last_product_purchased 56 | field_type: 'dimension' 57 | type: string 58 | sql: '${TABLE}.last_product_purchased' 59 | searchable: true 60 | 61 | - name: is_churned 62 | field_type: dimension 63 | type: yesno 64 | sql: '${TABLE}.is_churned' 65 | 66 | - name: customer_ltv 67 | field_type: 'dimension' 68 | type: number 69 | sql: '${TABLE}.customer_ltv' 70 | 71 | - name: median_customer_ltv 72 | field_type: 'measure' 73 | type: median 74 | sql: '${customer_ltv}' 75 | 76 | - name: average_customer_ltv 77 | field_type: 'measure' 78 | type: average 79 | sql: '${customer_ltv}' 80 | 81 | - name: days_between_first_second_order 82 | field_type: 'measure' 83 | type: average 84 | sql: 'date_diff(cast(${TABLE}.second_order_date as date), cast(${TABLE}.first_order_date as date), day)' 85 | 86 | - name: total_sessions 87 | field_type: 'measure' 88 | type: sum 89 | sql: '${TABLE}.total_sessions' 90 | filters: 91 | - field: is_churned 92 | value: no 93 | 94 | - name: total_sessions_divide 95 | field_type: 'measure' 96 | type: number 97 | sql: '${total_sessions} / (100 * 1.0)' 98 | 99 | - name: unique_user_iphone_sessions 100 | field_type: 'measure' 101 | type: count_distinct 102 | sql: case when ${sessions.session_device} = 'iPhone' then ${customer_id} end 103 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/views/test_discount_detail.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: view 3 | name: discount_detail 4 | model_name: test_model 5 | sql_table_name: analytics.discount_detail 6 | 7 | identifiers: 8 | - name: discount_join 9 | type: join 10 | reference: discounts 11 | relationship: one_to_one 12 | sql_on: '${discounts.discount_id}=${discount_detail.discount_id} AND ${discounts.order_week} is not null' 13 | 14 | fields: 15 | - name: discount_id 16 | field_type: 'dimension' 17 | type: string 18 | primary_key: yes 19 | hidden: yes 20 | sql: ${TABLE}.discount_id 21 | group_label: "ID's" 22 | 23 | - name: discount_order_id 24 | field_type: 'dimension' 25 | type: string 26 | sql: ${TABLE}.order_id 27 | searchable: true 28 | 29 | - name: discount_promo_name 30 | field_type: 'dimension' 31 | type: string 32 | sql: ${TABLE}.promo_name 33 | searchable: true 34 | 35 | - name: discount_usd 36 | field_type: measure 37 | canon_date: discounts.order 38 | type: sum 39 | sql: ${TABLE}.total_usd 40 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/views/test_discounts.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: view 3 | name: discounts 4 | model_name: test_model 5 | 6 | sql_table_name: '-- if dev -- dev_db.discounts 7 | -- if prod -- analytics_live.discounts' 8 | 9 | default_date: order 10 | row_label: 'Discount' 11 | 12 | identifiers: 13 | - name: order_id 14 | type: foreign 15 | allowed_fanouts: [order_lines] 16 | sql: ${order_id} 17 | - name: country_id 18 | type: foreign 19 | sql: ${country} 20 | 21 | fields: 22 | - name: discount_id 23 | field_type: 'dimension' 24 | type: string 25 | primary_key: yes 26 | hidden: yes 27 | sql: '${TABLE}.discount_id' 28 | group_label: "ID's" 29 | 30 | - name: order_id 31 | field_type: 'dimension' 32 | type: string 33 | hidden: yes 34 | sql: '${TABLE}.order_id' 35 | group_label: "ID's" 36 | 37 | - name: country 38 | field_type: 'dimension' 39 | type: string 40 | sql: '${TABLE}.country' 41 | searchable: true 42 | 43 | - name: order 44 | field_type: 'dimension_group' 45 | type: time 46 | timeframes: [date, week, month, year] 47 | sql: '${TABLE}.order_date' 48 | 49 | - name: discount_code 50 | field_type: 'dimension' 51 | type: string 52 | sql: '${TABLE}.code' 53 | searchable: true 54 | 55 | - name: total_discount_amt 56 | field_type: 'measure' 57 | type: sum 58 | sql: '${TABLE}.discount_amt' 59 | canon_date: order 60 | 61 | - name: discount_per_order 62 | field_type: measure 63 | type: number # only type allowed for a field that references an external explore 64 | 65 | # For foreign explores you need to specify the explore, view and measure 66 | # The the one this is defined in, you only need to specify the view and measure (but you can specify both) 67 | sql: ${total_discount_amt} / nullif(${orders.number_of_orders}, 0) 68 | 69 | is_merged_result: yes 70 | 71 | value_format_name: usd 72 | extra: 73 | zenlytic.show: yes 74 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/views/test_events.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: view 3 | name: events 4 | 5 | sql_table_name: analytics.events 6 | model_name: test_model 7 | 8 | default_date: event 9 | row_label: Event 10 | 11 | identifiers: 12 | - name: event_id 13 | type: primary 14 | sql: ${event_id} 15 | 16 | fields: 17 | - name: event_id 18 | field_type: 'dimension' 19 | type: string 20 | primary_key: yes 21 | hidden: yes 22 | sql: '${TABLE}.id' 23 | 24 | - name: event 25 | field_type: 'dimension_group' 26 | type: time 27 | timeframes: [raw, time, date, week, month, quarter, year] 28 | sql: '${TABLE}.event_date' 29 | 30 | - name: device 31 | field_type: 'dimension' 32 | type: string 33 | sql: '${TABLE}.device' 34 | hidden: yes 35 | 36 | - name: event_campaign 37 | field_type: 'dimension' 38 | type: string 39 | sql: '${TABLE}.campaign' 40 | hidden: yes 41 | 42 | - name: number_of_events 43 | field_type: measure 44 | type: count_distinct 45 | sql: ${event_id} 46 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/views/test_login_events.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: view 3 | name: login_events 4 | 5 | sql_table_name: analytics.login_events 6 | model_name: test_model 7 | 8 | default_date: events.event 9 | row_label: Event 10 | 11 | identifiers: 12 | - name: event_id 13 | type: foreign 14 | sql: ${event_id} 15 | 16 | fields: 17 | - name: event_id 18 | field_type: 'dimension' 19 | type: string 20 | primary_key: yes 21 | hidden: yes 22 | sql: '${TABLE}.id' 23 | 24 | - name: device 25 | field_type: 'dimension' 26 | type: string 27 | sql: ${events.device} 28 | searchable: false 29 | 30 | - name: number_of_login_events 31 | field_type: measure 32 | type: count_distinct 33 | sql: ${event_id} 34 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/views/test_order_lines.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: view 3 | name: order_lines 4 | 5 | sql_table_name: analytics.order_line_items 6 | 7 | default_date: order 8 | row_label: 'Order line item' 9 | model_name: test_model 10 | 11 | identifiers: 12 | - name: order_line_id 13 | type: primary 14 | sql: ${order_line_id} 15 | - name: order_id 16 | type: foreign 17 | sql: ${order_id} 18 | - name: customer_id 19 | type: foreign 20 | sql: ${customer_id} 21 | 22 | fields: 23 | - name: order_line_id 24 | field_type: 'dimension' 25 | type: string 26 | label: 'Order line PK' 27 | primary_key: yes 28 | hidden: yes 29 | sql: '${TABLE}.order_line_id' 30 | group_label: "ID's" 31 | 32 | - name: order_id 33 | field_type: 'dimension' 34 | type: string 35 | hidden: yes 36 | sql: '${TABLE}.order_unique_id' 37 | group_label: "ID's" 38 | 39 | - name: customer_id 40 | field_type: 'dimension' 41 | type: string 42 | hidden: yes 43 | sql: '${TABLE}.customer_id' 44 | group_label: "ID's" 45 | 46 | - name: order 47 | field_type: 'dimension_group' 48 | type: time 49 | datatype: date 50 | timeframes: [raw, time, date, week, month, quarter, year] 51 | sql: '${TABLE}.order_date' 52 | 53 | - name: waiting 54 | field_type: dimension_group 55 | type: duration 56 | intervals: [day, week] 57 | sql_start: '${TABLE}.view_date' 58 | sql_end: '${TABLE}.order_date' 59 | 60 | - name: channel 61 | field_type: 'dimension' 62 | type: string 63 | sql: '${TABLE}.sales_channel' 64 | searchable: true 65 | 66 | - name: parent_channel 67 | field_type: 'dimension' 68 | type: string 69 | searchable: true 70 | sql: | 71 | CASE 72 | --- parent channel 73 | WHEN ${channel} ilike '%social%' then 'Social' 74 | ELSE 'Not Social' 75 | END 76 | 77 | - name: product_name 78 | field_type: 'dimension' 79 | type: string 80 | sql: '${TABLE}.product_name' 81 | searchable: true 82 | 83 | - name: product_name_lang 84 | field_type: 'dimension' 85 | type: string 86 | sql: LOOKUP(${TABLE}.product_name, '{{ user_attributes["user_lang"] }}' ) 87 | searchable: true 88 | 89 | - name: inventory_qty 90 | field_type: 'dimension' 91 | hidden: yes 92 | type: number 93 | sql: '${TABLE}.inventory_qty' 94 | 95 | - name: is_on_sale_sql 96 | field_type: dimension 97 | type: yesno 98 | sql: "CASE WHEN ${TABLE}.product_name ilike '%sale%' then TRUE else FALSE end" 99 | 100 | - name: is_on_sale_case 101 | field_type: dimension 102 | type: string 103 | sql: case when ${TABLE}.product_name ilike '%sale%' then 'On sale' else 'Not on sale' end 104 | searchable: true 105 | 106 | - name: order_sequence 107 | field_type: dimension 108 | type: number 109 | sql: dense_rank() over (partition by ${customer_id} order by ${order_date} asc) 110 | window: true 111 | 112 | - name: new_vs_repeat_status 113 | field_type: dimension 114 | type: string 115 | sql: case when ${order_sequence} = 1 then 'New' else 'Repeat' end 116 | searchable: true 117 | 118 | - name: order_tier 119 | field_type: dimension 120 | type: tier 121 | tiers: [0, 20, 50, 100, 300] 122 | sql: '${TABLE}.revenue' 123 | 124 | - name: ending_on_hand_qty 125 | type: number 126 | field_type: measure 127 | value_format_name: decimal_0 128 | description: 'The ending inventory for the time period selected' 129 | sql: split_part(listagg(${inventory_qty}, ',') within group (order by ${order_date} desc), ',', 0)::int 130 | 131 | - name: number_of_new_purchased_items 132 | field_type: measure 133 | type: count 134 | sql: '${TABLE}.order_id' 135 | filters: 136 | - field: new_vs_repeat_status 137 | value: 'New' 138 | 139 | - name: pct_of_total_item_revenue 140 | field_type: measure 141 | type: number 142 | sql: RATIO_TO_REPORT(${total_item_revenue}) OVER () 143 | window: TRUE 144 | value_format_name: percent_1 145 | 146 | - name: number_of_email_purchased_items 147 | field_type: measure 148 | type: count 149 | sql: '${TABLE}.order_id' 150 | filters: 151 | - field: channel 152 | value: 'Email' 153 | 154 | - name: average_order_revenue 155 | field_type: 'measure' 156 | type: average_distinct 157 | sql_distinct_key: ${order_id} 158 | sql: ${TABLE}.order_total 159 | 160 | - name: total_item_revenue 161 | field_type: 'measure' 162 | type: sum 163 | canon_date: order 164 | sql: '${TABLE}.revenue' 165 | 166 | - name: total_item_costs 167 | field_type: 'measure' 168 | type: sum 169 | sql: '${TABLE}.item_costs' 170 | filters: 171 | - field: product_name 172 | value: 'Portable Charger' 173 | - field: product_name 174 | value: 'Portable Charger, Dual Charger' 175 | - field: orders.revenue_in_cents 176 | value: '>100' 177 | 178 | - name: total_item_costs_pct 179 | field_type: measure 180 | type: number 181 | sql: ${total_item_costs} * ${number_of_email_purchased_items} 182 | 183 | - name: line_item_aov 184 | field_type: 'measure' 185 | type: number 186 | sql: '${total_item_revenue} / ${orders.number_of_orders}' 187 | 188 | - name: should_be_number 189 | field_type: measure 190 | type: sum 191 | sql: '${line_item_aov} + ${total_item_costs_pct}' 192 | 193 | - name: costs_per_session 194 | field_type: measure 195 | type: number 196 | sql: ${total_item_costs_pct} / nullif(${sessions.number_of_sessions}, 0) 197 | is_merged_result: yes 198 | value_format_name: usd 199 | 200 | - name: net_per_session 201 | field_type: measure 202 | type: number 203 | sql: ${revenue_per_session} - ${costs_per_session} 204 | is_merged_result: yes 205 | value_format_name: usd 206 | 207 | - name: revenue_per_session 208 | field_type: measure 209 | type: number # only type allowed for a field that references an external explore 210 | 211 | # For foreign explores you need to specify the explore, view and measure 212 | # The the one this is defined in, you only need to specify the view and measure (but you can specify both) 213 | sql: ${total_item_revenue} / nullif(${sessions.number_of_sessions}, 0) 214 | 215 | is_merged_result: yes 216 | 217 | value_format_name: usd 218 | extra: 219 | zenlytic.show: yes 220 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/views/test_orders.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: view 3 | name: orders 4 | 5 | sql_table_name: analytics.orders 6 | required_access_grants: [test_access_grant_department_view] 7 | model_name: test_model 8 | 9 | default_date: order 10 | row_label: 'Order' 11 | 12 | identifiers: 13 | - name: order_id 14 | type: primary 15 | sql: ${order_id} 16 | - name: order_discount_join 17 | type: join 18 | relationship: one_to_many 19 | reference: 'discounts' 20 | sql_on: ${orders.order_id}=${discounts.order_id} 21 | - name: customer_id 22 | type: foreign 23 | sql: ${customer_id} 24 | - name: account_id 25 | type: foreign 26 | sql: ${account_id} 27 | 28 | access_filters: 29 | - field: customers.region 30 | user_attribute: owned_region 31 | - field: orders.warehouse_location 32 | user_attribute: warehouse_location 33 | - field: orders.order_id 34 | user_attribute: allowed_order_ids 35 | 36 | fields: 37 | - name: order_id 38 | field_type: 'dimension' 39 | type: string 40 | primary_key: yes 41 | hidden: yes 42 | sql: '${TABLE}.id' 43 | group_label: "ID's" 44 | 45 | - name: customer_id 46 | field_type: 'dimension' 47 | type: string 48 | hidden: yes 49 | sql: '${TABLE}.customer_id' 50 | group_label: "ID's" 51 | 52 | - name: account_id 53 | field_type: 'dimension' 54 | type: string 55 | hidden: yes 56 | sql: '${TABLE}.account_id' 57 | group_label: "ID's" 58 | 59 | - name: anon_id 60 | field_type: 'dimension' 61 | type: number 62 | sql: '${TABLE}.anon_id' 63 | 64 | - name: do_not_use 65 | field_type: 'dimension' 66 | type: string 67 | sql: '${TABLE}.for_set_testing' 68 | searchable: false 69 | 70 | - name: order 71 | field_type: 'dimension_group' 72 | type: time 73 | timeframes: 74 | [ 75 | raw, 76 | time, 77 | date, 78 | day_of_year, 79 | week, 80 | week_of_year, 81 | month, 82 | month_of_year, 83 | quarter, 84 | year, 85 | day_of_week, 86 | hour_of_day, 87 | ] 88 | sql: '${TABLE}.order_date' 89 | 90 | - name: previous_order 91 | field_type: 'dimension_group' 92 | type: time 93 | timeframes: [raw, time, date, week, month, quarter, year] 94 | sql: '${TABLE}.previous_order_date' 95 | 96 | - name: between_orders 97 | field_type: dimension_group 98 | type: duration 99 | intervals: ['hour', 'day', 'week', 'month', 'quarter', 'year'] 100 | sql_start: ${previous_order_raw} 101 | sql_end: ${order_raw} 102 | label: 'between this and last order' 103 | 104 | - name: revenue_dimension 105 | field_type: 'dimension' 106 | type: number 107 | sql: '${TABLE}.revenue' 108 | 109 | - name: revenue_in_cents 110 | field_type: 'dimension' 111 | type: number 112 | sql: ${revenue_dimension} * 100 113 | 114 | - name: sub_channel 115 | field_type: 'dimension' 116 | type: string 117 | sql: '${TABLE}.sub_channel' 118 | searchable: true 119 | 120 | - name: last_order_channel 121 | field_type: 'dimension' 122 | type: string 123 | sql: lag(${TABLE}.sub_channel) over (partition by ${customers.customer_id} order by ${TABLE}.order_date) 124 | window: true 125 | searchable: true 126 | 127 | - name: last_order_warehouse_location 128 | field_type: 'dimension' 129 | type: string 130 | sql: lag(${TABLE}.warehouselocation) over (partition by ${customers.customer_id} order by ${TABLE}.order_date) 131 | window: true 132 | searchable: true 133 | 134 | - name: warehouse_location 135 | field_type: dimension 136 | type: string 137 | sql: ${TABLE}.warehouselocation 138 | searchable: true 139 | 140 | - name: campaign 141 | field_type: 'dimension' 142 | type: string 143 | sql: '${TABLE}.campaign' 144 | searchable: true 145 | 146 | - name: new_vs_repeat 147 | field_type: 'dimension' 148 | type: string 149 | sql: '${TABLE}.new_vs_repeat' 150 | searchable: true 151 | drill_fields: [test_set*, new_vs_repeat] 152 | 153 | - name: number_of_orders 154 | field_type: measure 155 | type: count 156 | sql: '${order_id}' 157 | canon_date: null 158 | filters: [] 159 | 160 | - name: average_days_between_orders 161 | field_type: measure 162 | type: average 163 | canon_date: previous_order 164 | sql: ${days_between_orders} 165 | 166 | - name: total_revenue 167 | required_access_grants: [test_access_grant_department_field] 168 | field_type: 'measure' 169 | type: sum 170 | sql: ${revenue_dimension} 171 | 172 | - name: total_non_merchant_revenue 173 | field_type: measure 174 | type: sum 175 | sql: ${TABLE}.revenue 176 | filters: 177 | - field: anon_id 178 | value: -9,-3, -22, -9082 179 | 180 | - name: total_lifetime_revenue 181 | field_type: measure 182 | type: cumulative 183 | measure: total_revenue 184 | 185 | - name: cumulative_customers 186 | field_type: measure 187 | type: cumulative 188 | measure: customers.number_of_customers 189 | cumulative_where: ${customers.cancelled_date} < ${cumulative_date} 190 | 191 | - name: cumulative_customers_no_change_grain 192 | field_type: measure 193 | type: cumulative 194 | measure: customers.number_of_customers 195 | cumulative_where: ${customers.cancelled_date} < ${cumulative_date} 196 | update_where_timeframe: no 197 | 198 | - name: cumulative_aov 199 | field_type: measure 200 | type: cumulative 201 | measure: average_order_value_custom 202 | 203 | - name: ltv 204 | field_type: measure 205 | type: number 206 | sql: ${total_lifetime_revenue} / nullif(${cumulative_customers}, 0) 207 | 208 | - name: ltr 209 | field_type: measure 210 | type: number 211 | sql: ${total_lifetime_revenue} / nullif(${customers.number_of_customers}, 0) 212 | 213 | - name: total_modified_revenue 214 | field_type: 'measure' 215 | type: sum 216 | sql: "case when ${TABLE}.order_id not like 'Z%' 217 | and ${TABLE}.order_id not like 'QW%' 218 | and length(${TABLE}.order_id)>=12 219 | then ${TABLE}.revenue else 0 end" 220 | 221 | - name: average_order_value 222 | field_type: 'measure' 223 | type: average 224 | sql: '${TABLE}.revenue' 225 | 226 | - name: average_order_value_custom 227 | field_type: 'measure' 228 | type: number 229 | sql: '${total_revenue} / ${number_of_orders}' 230 | 231 | - name: new_order_count 232 | field_type: 'measure' 233 | type: count 234 | sql: ${orders.order_id} 235 | filters: 236 | - field: new_vs_repeat 237 | value: 'New' 238 | 239 | sets: 240 | - name: test_set 241 | fields: [order_id, customer_id, total_revenue] 242 | 243 | - name: test_set2 244 | fields: [order_id, new_vs_repeat, sub_channel, average_order_value, order_time] 245 | 246 | - name: test_set_composed 247 | fields: [test_set*, test_set2*, -new_vs_repeat, -sub_channel] 248 | 249 | - name: test_set_all_fields 250 | fields: [ALL_FIELDS*, -test_set2*, -new_vs_repeat, -revenue_dimension] 251 | 252 | - name: test_removal 253 | fields: [do_not_use] 254 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/views/test_sessions.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: view 3 | name: sessions 4 | 5 | sql_table_name: analytics.sessions 6 | model_name: test_model 7 | 8 | default_date: session 9 | row_label: Session 10 | 11 | identifiers: 12 | - name: customer_id 13 | type: foreign 14 | sql: ${customer_id} 15 | 16 | fields: 17 | - name: session_id 18 | field_type: 'dimension' 19 | type: string 20 | primary_key: yes 21 | hidden: yes 22 | sql: '${TABLE}.id' 23 | 24 | - name: utm_source 25 | field_type: 'dimension' 26 | type: string 27 | sql: '${TABLE}.utm_source' 28 | searchable: true 29 | 30 | - name: utm_campaign 31 | field_type: 'dimension' 32 | type: string 33 | sql: '${TABLE}.utm_campaign' 34 | searchable: true 35 | 36 | - name: session_device 37 | field_type: 'dimension' 38 | type: string 39 | sql: '${TABLE}.session_device' 40 | searchable: true 41 | 42 | - name: customer_id 43 | field_type: dimension 44 | type: string 45 | sql: ${TABLE}.customer_id 46 | searchable: false 47 | 48 | - name: session_adj_week 49 | field_type: 'dimension' 50 | type: time 51 | datatype: timestamp 52 | sql: date_trunc('week', dateadd(day, 1, ${TABLE}.session_date)) 53 | 54 | - name: session_adj_month 55 | field_type: 'dimension' 56 | type: time 57 | datatype: datetime 58 | sql: date_trunc('month', dateadd(day, 1, ${TABLE}.session_date)) 59 | 60 | - name: session 61 | field_type: 'dimension_group' 62 | type: time 63 | timeframes: [raw, time, date, week, month, quarter, year] 64 | sql: '${TABLE}.session_date' 65 | 66 | - name: number_of_sessions 67 | field_type: measure 68 | type: count 69 | sql: ${session_id} 70 | canon_date: session 71 | 72 | - name: number_of_in_session_clicks 73 | field_type: measure 74 | type: sum_distinct 75 | sql_distinct_key: ${session_id} 76 | sql: ${TABLE}.clicks 77 | 78 | - name: most_recent_session_date 79 | field_type: measure 80 | type: time 81 | sql: max(${session_date}) 82 | 83 | - name: most_recent_session_week 84 | field_type: measure 85 | type: time 86 | datatype: datetime 87 | sql: max(date_trunc('week', ${session_date})) 88 | 89 | - name: most_recent_session_date_is_today 90 | field_type: measure 91 | type: yesno 92 | sql: ${most_recent_session_date} = current_date() 93 | 94 | - name: list_of_devices_used 95 | field_type: measure 96 | type: string 97 | sql: LISTAGG(${session_device}, ', ') 98 | 99 | - name: list_of_sources 100 | field_type: measure 101 | type: string 102 | sql: LISTAGG(${TABLE}.utm_source || ' - ' || ${utm_campaign}, ', ') 103 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/views/test_submitted_form.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: view 3 | name: submitted_form 4 | 5 | sql_table_name: analytics.submitted_form 6 | model_name: test_model 7 | 8 | default_date: session 9 | row_label: Session 10 | 11 | identifiers: 12 | - name: customer_join 13 | type: join 14 | reference: customers 15 | relationship: many_to_one 16 | join_type: full_outer 17 | sql_on: '${customers.customer_id}=${submitted_form.customer_id} AND ${submitted_form.session_date} is not null' 18 | 19 | access_filters: 20 | - field: submitted_form.session_id 21 | user_attribute: products 22 | 23 | fields: 24 | - name: session_id 25 | field_type: 'dimension' 26 | type: string 27 | primary_key: yes 28 | hidden: yes 29 | sql: '${TABLE}.id' 30 | 31 | - name: customer_id 32 | field_type: dimension 33 | type: string 34 | sql: ${TABLE}.customer_id 35 | searchable: false 36 | 37 | - name: context_os 38 | field_type: dimension 39 | type: string 40 | sql: ${TABLE}.context_os 41 | searchable: true 42 | 43 | - name: session 44 | field_type: 'dimension_group' 45 | type: time 46 | timeframes: [raw, time, date, week, month, quarter, year] 47 | sql: '${TABLE}.session_date' 48 | 49 | - name: sent_at 50 | field_type: 'dimension_group' 51 | type: time 52 | timeframes: [raw, time, date, week, month, quarter, year] 53 | sql: '${TABLE}.sent_at' 54 | 55 | - name: number_of_form_submissions 56 | field_type: measure 57 | type: count 58 | sql: ${session_id} 59 | 60 | - name: unique_users_form_submissions 61 | field_type: measure 62 | type: count_distinct 63 | canon_date: sent_at 64 | sql: ${customer_id} 65 | 66 | - name: unique_users_per_form_submission 67 | field_type: measure 68 | type: number 69 | sql: ${unique_users_form_submissions} / ${number_of_form_submissions} 70 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/views/traffic.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | type: view 3 | name: traffic 4 | hidden: true 5 | sql_table_name: analytics.traffic 6 | model_name: test_model 7 | 8 | fields: 9 | - name: traffic_id 10 | field_type: 'dimension' 11 | type: string 12 | primary_key: yes 13 | hidden: yes 14 | sql: '${TABLE}.id' 15 | 16 | - name: traffic_source 17 | field_type: 'dimension' 18 | type: string 19 | sql: '${TABLE}.traffic_source' 20 | searchable: true 21 | -------------------------------------------------------------------------------- /tests/config/metrics_layer_config/zenlytic_project.yml: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Project name 4 | name: metrics_project 5 | 6 | profile: test_profile 7 | 8 | dashboard-paths: ['data_model/dashboards'] 9 | view-paths: ['data_model/views'] 10 | model-paths: ['data_model/models'] 11 | -------------------------------------------------------------------------------- /tests/integration/test_e2e_metricflow_to_zenlytic.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | 5 | from metrics_layer.integrations.metricflow.metricflow_to_zenlytic import ( 6 | convert_mf_project_to_zenlytic_project, 7 | load_mf_project, 8 | ) 9 | 10 | BASE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../config") 11 | 12 | 13 | @pytest.mark.metricflow 14 | def test_e2e_read_project(): 15 | metricflow_folder = os.path.join(BASE_PATH, "metricflow") 16 | 17 | metricflow_project = load_mf_project(metricflow_folder) 18 | print(metricflow_project) 19 | 20 | assert set(metricflow_project.keys()) == {"customers", "orders", "order_item"} 21 | assert metricflow_project["customers"]["metrics"] == [] 22 | assert any("customers_with_orders" in m["name"] for m in metricflow_project["orders"]["metrics"]) 23 | 24 | 25 | @pytest.mark.metricflow 26 | def test_e2e_conversions(): 27 | metricflow_folder = os.path.join(BASE_PATH, "metricflow") 28 | 29 | metricflow_project = load_mf_project(metricflow_folder) 30 | models, views, errors = convert_mf_project_to_zenlytic_project( 31 | metricflow_project, "my_model", "my_company" 32 | ) 33 | 34 | print(views) 35 | assert len(errors) == 3 36 | percentile_error = next(e for e in errors if "p99_order_total" in e["message"]) 37 | assert percentile_error == { 38 | "message": "In view orders discrete percentile is not supported for the measure p99_order_total", 39 | "view_name": "orders", 40 | } 41 | food_customers_error = next(e for e in errors if "food_customers" in e["message"]) 42 | assert food_customers_error == { 43 | "message": ( 44 | "In view orders metric conversion failed for food_customers: Metric type filters are" 45 | " not supported" 46 | ), 47 | "view_name": "orders", 48 | } 49 | cumulative_revenue_error = next(e for e in errors if "cumulative_revenue" in e["message"]) 50 | assert cumulative_revenue_error == { 51 | "message": ( 52 | "In view order_item metric conversion failed for cumulative_revenue: It is a cumulative metric," 53 | " which is not supported." 54 | ), 55 | "view_name": "order_item", 56 | } 57 | 58 | assert len(models) == 1 59 | assert models[0]["name"] == "my_model" 60 | assert models[0]["connection"] == "my_company" 61 | 62 | assert len(views) == 3 63 | 64 | customers_view = next(v for v in views if v["name"] == "customers") 65 | 66 | # All measures are hidden in this view, because there are no metrics 67 | assert all(f["hidden"] for f in customers_view["fields"] if f["field_type"] == "measure") 68 | assert customers_view["model_name"] == "my_model" 69 | assert customers_view["sql_table_name"] == "my-bigquery-project.my_dataset.customers" 70 | assert "description" not in customers_view 71 | assert customers_view["zoe_description"] == "Customers table" 72 | orders_view = next(v for v in views if v["name"] == "orders") 73 | 74 | assert orders_view["model_name"] == "my_model" 75 | assert orders_view["sql_table_name"] == "orders" 76 | order_count = next(m for m in orders_view["fields"] if "_order_count" == m["name"]) 77 | assert order_count["sql"] == "1" 78 | 79 | customers_with_fields_metric = next( 80 | m for m in orders_view["fields"] if "customers_with_orders" == m["name"] 81 | ) 82 | assert customers_with_fields_metric["field_type"] == "measure" 83 | assert not customers_with_fields_metric["hidden"] 84 | assert customers_with_fields_metric["sql"] == "customer_id" 85 | assert customers_with_fields_metric["type"] == "count_distinct" 86 | assert customers_with_fields_metric["label"] == "Customers w/ Orders" 87 | assert customers_with_fields_metric["description"] == "Unique count of customers placing orders" 88 | assert customers_with_fields_metric["zoe_description"] == "Distinct count of customers placing orders" 89 | 90 | order_total_dim = next(m for m in orders_view["fields"] if "order_total_dim" == m["name"]) 91 | assert order_total_dim["field_type"] == "dimension" 92 | assert order_total_dim["sql"] == "${TABLE}.order_total" 93 | assert order_total_dim["type"] == "string" 94 | 95 | large_orders_metric = next(m for m in orders_view["fields"] if "large_order" == m["name"]) 96 | assert large_orders_metric["field_type"] == "measure" 97 | assert not large_orders_metric["hidden"] 98 | assert large_orders_metric["sql"] == "case when ${orders.order_total_dim} >= 20\n then 1 else null end" 99 | 100 | order_item_view = next(v for v in views if v["name"] == "order_item") 101 | 102 | # Validate the outputs are defined in the view 103 | assert next(f for f in order_item_view["fields"] if "ad_revenue" == f["name"]) 104 | assert next(f for f in order_item_view["fields"] if "revenue" == f["name"]) 105 | 106 | pct_rev_from_ads_field = next(f for f in order_item_view["fields"] if "pct_rev_from_ads" == f["name"]) 107 | assert pct_rev_from_ads_field["field_type"] == "measure" 108 | assert pct_rev_from_ads_field["type"] == "number" 109 | assert pct_rev_from_ads_field["sql"] == "${ad_revenue} / ${revenue}" 110 | assert pct_rev_from_ads_field["label"] == "Percentage of Revenue from Advertising" 111 | assert not pct_rev_from_ads_field.get("hidden", False) 112 | 113 | food_revenue_field = next(f for f in order_item_view["fields"] if "food_revenue" == f["name"]) 114 | assert food_revenue_field["field_type"] == "measure" 115 | assert food_revenue_field["type"] == "sum" 116 | assert food_revenue_field["sql"] == "case when is_food_item = 1 then product_price else 0 end" 117 | assert food_revenue_field["label"] == "Food Revenue" 118 | assert not food_revenue_field.get("hidden", False) 119 | 120 | # Validate the inputs are defined in the view 121 | assert next(f for f in order_item_view["fields"] if "number_of_repeat_orders" == f["name"]) 122 | assert next(f for f in order_item_view["fields"] if "number_of_orders" == f["name"]) 123 | 124 | rep_rate_ratio_field = next(f for f in order_item_view["fields"] if "repurchase_rate" == f["name"]) 125 | assert rep_rate_ratio_field["field_type"] == "measure" 126 | assert rep_rate_ratio_field["type"] == "number" 127 | assert rep_rate_ratio_field["sql"] == "${number_of_repeat_orders} / ${number_of_orders}" 128 | assert rep_rate_ratio_field["label"] == "Repurchase Rate" 129 | assert not rep_rate_ratio_field.get("hidden", False) 130 | 131 | revenue_field = next(f for f in order_item_view["fields"] if "revenue" == f["name"]) 132 | assert revenue_field["field_type"] == "measure" 133 | assert revenue_field["type"] == "sum" 134 | assert revenue_field["sql"] == "product_price" 135 | assert revenue_field["label"] == "Revenue" 136 | assert not revenue_field.get("hidden", False) 137 | 138 | # Validate the inputs are defined in the view 139 | assert next(f for f in order_item_view["fields"] if "food_revenue" == f["name"]) 140 | assert next(f for f in order_item_view["fields"] if "revenue" == f["name"]) 141 | 142 | food_ratio_field = next(f for f in order_item_view["fields"] if "food_revenue_pct" == f["name"]) 143 | assert food_ratio_field["field_type"] == "measure" 144 | assert food_ratio_field["type"] == "number" 145 | assert food_ratio_field["sql"] == "${food_revenue} / ${revenue}" 146 | assert food_ratio_field["label"] == "Food Revenue %" 147 | assert not food_ratio_field.get("hidden", False) 148 | 149 | assert order_item_view["identifiers"][0]["name"] == "order_item" 150 | assert order_item_view["identifiers"][0]["type"] == "primary" 151 | assert order_item_view["identifiers"][0]["sql"] == "${TABLE}.order_item_id" 152 | assert order_item_view["identifiers"][1]["name"] == "order_id" 153 | assert order_item_view["identifiers"][1]["type"] == "foreign" 154 | assert order_item_view["identifiers"][1]["sql"] == "CAST(order_id AS VARCHAR)" 155 | -------------------------------------------------------------------------------- /tests/test_access_grants_fields.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from metrics_layer.core.exceptions import AccessDeniedOrDoesNotExistException 4 | 5 | 6 | def test_access_grants_exist(connection): 7 | model = connection.get_model("test_model") 8 | connection.project.set_user({"email": "user@example.com"}) 9 | 10 | assert isinstance(model.access_grants, list) 11 | assert model.access_grants[0]["name"] == "test_access_grant_department_view" 12 | assert model.access_grants[0]["user_attribute"] == "department" 13 | assert model.access_grants[0]["allowed_values"] == ["finance", "executive", "sales"] 14 | 15 | 16 | def test_access_grants_view_visible(connection): 17 | connection.project.set_user(None) 18 | connection.get_view("orders") 19 | 20 | connection.project.set_user({"department": "sales"}) 21 | connection.get_view("orders") 22 | 23 | connection.project.set_user({"department": "marketing"}) 24 | 25 | with pytest.raises(AccessDeniedOrDoesNotExistException) as exc_info: 26 | connection.get_view("orders") 27 | 28 | assert exc_info.value 29 | assert exc_info.value.object_name == "orders" 30 | assert exc_info.value.object_type == "view" 31 | 32 | 33 | def test_access_grants_field_visible(connection): 34 | # None always allows access 35 | connection.project.set_user({"department": None}) 36 | connection.get_field("orders.total_revenue") 37 | 38 | connection.project.set_user({"department": "executive"}) 39 | connection.get_field("orders.total_revenue") 40 | 41 | connection.project.set_user({"department": "sales"}) 42 | connection.get_field("orders.total_revenue") 43 | 44 | # Having permissions on the field isn't enough, you must also have permissions on the view to see field 45 | connection.project.set_user({"department": "engineering"}) 46 | 47 | with pytest.raises(AccessDeniedOrDoesNotExistException) as exc_info: 48 | connection.get_field("orders.total_revenue") 49 | 50 | assert exc_info.value 51 | assert exc_info.value.object_name == "orders" 52 | assert exc_info.value.object_type == "view" 53 | 54 | connection.project.set_user({"department": "operations"}) 55 | 56 | with pytest.raises(AccessDeniedOrDoesNotExistException) as exc_info: 57 | connection.get_field("orders.total_revenue") 58 | 59 | assert exc_info.value 60 | assert exc_info.value.object_name == "orders" 61 | assert exc_info.value.object_type == "view" 62 | 63 | connection.project.set_user({"department": "finance"}) 64 | 65 | with pytest.raises(AccessDeniedOrDoesNotExistException) as exc_info: 66 | connection.get_field("orders.total_revenue") 67 | 68 | assert exc_info.value 69 | assert exc_info.value.object_name == "total_revenue" 70 | assert exc_info.value.object_type == "field" 71 | 72 | 73 | def test_access_grants_field_join_graphs(connection): 74 | connection.project.set_user({"department": "executive"}) 75 | field = connection.get_field("sessions.number_of_sessions") 76 | field.join_graphs() 77 | 78 | # Having permissions on the field isn't enough, you must also have permissions on the view to see field 79 | connection.project.set_user({"department": "engineering"}) 80 | 81 | # This is a regression test for a bug where the call to join_graphs would raise an access denied 82 | # exception because one of the mapped fields was not visible to this user 83 | field = connection.get_field("sessions.number_of_sessions") 84 | field.join_graphs() 85 | 86 | 87 | def test_access_grants_dashboard_visible(connection): 88 | # None always allows access 89 | connection.project.set_user({"department": None}) 90 | connection.get_dashboard("sales_dashboard_v2") 91 | 92 | connection.project.set_user({"department": "sales"}) 93 | connection.get_dashboard("sales_dashboard_v2") 94 | 95 | connection.project.set_user({"department": "operations"}) 96 | with pytest.raises(AccessDeniedOrDoesNotExistException) as exc_info: 97 | connection.get_dashboard("sales_dashboard_v2") 98 | 99 | assert exc_info.value 100 | assert exc_info.value.object_name == "sales_dashboard_v2" 101 | assert exc_info.value.object_type == "dashboard" 102 | -------------------------------------------------------------------------------- /tests/test_access_grants_queries.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from metrics_layer.core.exceptions import AccessDeniedOrDoesNotExistException 4 | 5 | 6 | def test_access_grants_join_permission_block(connection): 7 | connection.project.set_user({"department": "executive"}) 8 | connection.get_sql_query(sql="SELECT * FROM MQL(total_item_revenue BY gender)") 9 | connection.get_sql_query(metrics=["total_item_revenue"], dimensions=["product_name"]) 10 | 11 | connection.project.set_user({"department": "marketing"}) 12 | 13 | connection.get_sql_query(metrics=["total_item_revenue"], dimensions=["product_name"]) 14 | 15 | connection.get_sql_query(metrics=["total_item_revenue"], dimensions=["gender"]) 16 | 17 | with pytest.raises(AccessDeniedOrDoesNotExistException) as exc_info: 18 | connection.get_sql_query(metrics=["total_revenue"], dimensions=["gender"]) 19 | 20 | assert exc_info.value 21 | assert exc_info.value.object_name == "total_revenue" 22 | assert exc_info.value.object_type == "field" 23 | 24 | with pytest.raises(AccessDeniedOrDoesNotExistException) as exc_info: 25 | connection.get_sql_query(sql="SELECT * FROM MQL(total_revenue BY gender)") 26 | 27 | assert exc_info.value 28 | assert exc_info.value.object_name == "total_revenue" 29 | assert exc_info.value.object_type == "field" 30 | 31 | 32 | def test_access_grants_view_permission_block(connection): 33 | connection.project.set_user({"department": "finance"}) 34 | 35 | # Even with explore and view access, they cant run the query due to the 36 | # conditional filter that they don't have access to 37 | with pytest.raises(AccessDeniedOrDoesNotExistException) as exc_info: 38 | connection.get_sql_query(metrics=["total_item_revenue"], dimensions=["new_vs_repeat"]) 39 | 40 | assert exc_info.value 41 | assert exc_info.value.object_name == "customers" 42 | assert exc_info.value.object_type == "view" 43 | 44 | # Even though they have view access, they don't have explore access so still can't run this query 45 | connection.project.set_user({"department": "sales"}) 46 | connection.get_sql_query(metrics=["total_item_revenue"], dimensions=["new_vs_repeat"]) 47 | 48 | connection.project.set_user({"department": "marketing"}) 49 | 50 | connection.get_sql_query(metrics=["total_item_revenue"], dimensions=["product_name"]) 51 | 52 | with pytest.raises(AccessDeniedOrDoesNotExistException) as exc_info: 53 | connection.get_sql_query(metrics=["total_item_revenue"], dimensions=["new_vs_repeat"]) 54 | 55 | assert exc_info.value 56 | assert exc_info.value.object_name == "new_vs_repeat" 57 | assert exc_info.value.object_type == "field" 58 | 59 | 60 | def test_access_grants_field_permission_block(connection): 61 | connection.project.set_user({"department": "executive"}) 62 | 63 | connection.get_sql_query(metrics=["total_revenue"], dimensions=["new_vs_repeat"]) 64 | connection.get_sql_query(metrics=["total_revenue"], dimensions=["product_name"]) 65 | connection.get_sql_query(metrics=["total_revenue"], dimensions=["gender"]) 66 | 67 | connection.project.set_user({"department": "engineering"}) 68 | with pytest.raises(AccessDeniedOrDoesNotExistException) as exc_info: 69 | connection.query(metrics=["total_revenue"], dimensions=["product_name"]) 70 | 71 | assert exc_info.value 72 | assert exc_info.value.object_name == "total_revenue" 73 | assert exc_info.value.object_type == "field" 74 | 75 | 76 | @pytest.mark.query 77 | def test_access_filters_equal_to(connection): 78 | connection.project.set_user({"department": "executive"}) 79 | 80 | query = connection.get_sql_query(metrics=["total_revenue"], dimensions=["new_vs_repeat"]) 81 | 82 | assert "region" not in query 83 | 84 | connection.project.set_user({"department": "executive", "owned_region": "US-West"}) 85 | 86 | query = connection.get_sql_query(metrics=["total_revenue"], dimensions=["new_vs_repeat"]) 87 | 88 | correct = ( 89 | "SELECT orders.new_vs_repeat as orders_new_vs_repeat,SUM(orders.revenue) as orders_total_revenue " 90 | "FROM analytics.orders orders LEFT JOIN analytics.customers customers " 91 | "ON orders.customer_id=customers.customer_id WHERE customers.region='US-West' " 92 | "GROUP BY orders.new_vs_repeat ORDER BY orders_total_revenue DESC NULLS LAST;" 93 | ) 94 | connection.project.set_user({}) 95 | assert correct == query 96 | 97 | 98 | @pytest.mark.query 99 | def test_access_filters_array(connection): 100 | connection.project.set_user({"department": "executive", "owned_region": "US-West, US-East"}) 101 | 102 | query = connection.get_sql_query(metrics=["total_revenue"], dimensions=["new_vs_repeat"]) 103 | 104 | correct = ( 105 | "SELECT orders.new_vs_repeat as orders_new_vs_repeat,SUM(orders.revenue) as orders_total_revenue " 106 | "FROM analytics.orders orders LEFT JOIN analytics.customers customers " 107 | "ON orders.customer_id=customers.customer_id WHERE customers.region IN ('US-West','US-East') " 108 | "GROUP BY orders.new_vs_repeat ORDER BY orders_total_revenue DESC NULLS LAST;" 109 | ) 110 | connection.project.set_user({}) 111 | assert correct == query 112 | 113 | 114 | @pytest.mark.query 115 | def test_access_filters_underscore(connection): 116 | connection.project.set_user({"warehouse_location": "New Jersey"}) 117 | 118 | query = connection.get_sql_query(metrics=["total_revenue"], dimensions=["new_vs_repeat"]) 119 | 120 | correct = ( 121 | "SELECT orders.new_vs_repeat as orders_new_vs_repeat,SUM(orders.revenue) as orders_total_revenue " 122 | "FROM analytics.orders orders WHERE orders.warehouselocation='New Jersey' " 123 | "GROUP BY orders.new_vs_repeat ORDER BY orders_total_revenue DESC NULLS LAST;" 124 | ) 125 | assert correct == query 126 | -------------------------------------------------------------------------------- /tests/test_config_parse.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | 5 | from metrics_layer.core.parse import ( 6 | MetricflowProjectReader, 7 | MetricsLayerProjectReader, 8 | ProjectLoader, 9 | ) 10 | from metrics_layer.core.parse.github_repo import BaseRepo 11 | from metrics_layer.core.query.query import MetricsLayerConnection 12 | 13 | BASE_PATH = os.path.dirname(__file__) 14 | 15 | 16 | class repo_mock(BaseRepo): 17 | def __init__(self, repo_type: str = None): 18 | self.repo_type = repo_type 19 | self.dbt_path = None 20 | 21 | @property 22 | def folder(self): 23 | if self.repo_type == "dbt": 24 | return os.path.join(BASE_PATH, "config/dbt/") 25 | return os.path.join(BASE_PATH, "config/metrics_layer/") 26 | 27 | @property 28 | def warehouse_type(self): 29 | if self.repo_type == "dbt": 30 | return "SNOWFLAKE" 31 | raise NotImplementedError() 32 | 33 | def fetch(self, private_key=None): 34 | return 35 | 36 | def search(self, pattern, folders): 37 | if pattern == "*.yml": 38 | view = os.path.join(BASE_PATH, "config/metrics_layer_config/data_model/view_with_all_fields.yml") 39 | model = os.path.join( 40 | BASE_PATH, "config/metrics_layer_config/data_model/model_with_all_fields.yml" 41 | ) # noqa 42 | return [model, view] 43 | elif pattern == "manifest.json": 44 | return [os.path.join(BASE_PATH, "config/dbt/target/manifest.json")] 45 | return [] 46 | 47 | def delete(self): 48 | return 49 | 50 | 51 | def mock_dbt_search(pattern): 52 | if pattern == "manifest.json": 53 | return [os.path.join(BASE_PATH, "config/dbt/target/manifest.json")] 54 | return [] 55 | 56 | 57 | def test_get_branch_options(): 58 | loader = MetricsLayerConnection(location=os.path.join(BASE_PATH, "config/metrics_layer/")) 59 | loader.load() 60 | assert loader.get_branch_options() == [] 61 | 62 | 63 | def test_config_load_yaml(): 64 | reader = MetricsLayerProjectReader(repo=repo_mock(repo_type="metrics_layer")) 65 | models, views, dashboards, topics, conversion_errors = reader.load() 66 | 67 | model = models[0] 68 | 69 | assert model["type"] == "model" 70 | assert isinstance(model["name"], str) 71 | assert isinstance(model["connection"], str) 72 | 73 | view = views[0] 74 | 75 | assert view["type"] == "view" 76 | assert isinstance(view["name"], str) 77 | assert isinstance(view["sets"], list) 78 | assert isinstance(view["sets"][0], dict) 79 | assert view["sets"][0]["name"] == "set_name" 80 | assert isinstance(view["sql_table_name"], str) 81 | assert isinstance(view["fields"], list) 82 | 83 | field = view["fields"][0] 84 | 85 | assert isinstance(field["name"], str) 86 | assert isinstance(field["field_type"], str) 87 | assert isinstance(field["type"], str) 88 | assert isinstance(field["sql"], str) 89 | 90 | assert len(topics) == 0 91 | 92 | assert len(dashboards) == 0 93 | 94 | assert len(conversion_errors) == 0 95 | 96 | 97 | def test_automatic_choosing(): 98 | assert repo_mock().get_repo_type() == "metrics_layer" 99 | 100 | 101 | def test_bad_repo_type(monkeypatch): 102 | monkeypatch.setattr(ProjectLoader, "_get_repo", lambda *args: repo_mock(repo_type="dne")) 103 | 104 | reader = ProjectLoader(location=None) 105 | with pytest.raises(TypeError) as exc_info: 106 | reader.load() 107 | 108 | assert exc_info.value 109 | 110 | 111 | @pytest.mark.dbt 112 | def test_config_load_dbt(monkeypatch): 113 | mock = repo_mock(repo_type="dbt") 114 | mock.dbt_path = os.path.join(BASE_PATH, "config/dbt/") 115 | monkeypatch.setattr(ProjectLoader, "_get_repo", lambda *args: mock) 116 | 117 | reader = ProjectLoader(location=None) 118 | with pytest.raises(TypeError) as exc_info: 119 | reader.load() 120 | 121 | assert exc_info.value 122 | 123 | 124 | @pytest.mark.dbt 125 | def test_config_load_metricflow(): 126 | mock = repo_mock(repo_type="metricflow") 127 | mock.dbt_path = os.path.join(BASE_PATH, "config/metricflow/") 128 | reader = MetricflowProjectReader(repo=mock) 129 | models, views, dashboards, topics, conversion_errors = reader.load() 130 | 131 | assert len(conversion_errors) == 3 132 | percentile_error = next(e for e in conversion_errors if "p99_order_total" in e["message"]) 133 | assert percentile_error == { 134 | "message": "In view orders discrete percentile is not supported for the measure p99_order_total", 135 | "view_name": "orders", 136 | } 137 | food_customers_error = next(e for e in conversion_errors if "food_customers" in e["message"]) 138 | assert food_customers_error == { 139 | "message": ( 140 | "In view orders metric conversion failed for food_customers: Metric type filters are" 141 | " not supported" 142 | ), 143 | "view_name": "orders", 144 | } 145 | cumulative_revenue_error = next(e for e in conversion_errors if "cumulative_revenue" in e["message"]) 146 | assert cumulative_revenue_error == { 147 | "message": ( 148 | "In view order_item metric conversion failed for cumulative_revenue: It is a cumulative metric," 149 | " which is not supported." 150 | ), 151 | "view_name": "order_item", 152 | } 153 | 154 | model = models[0] 155 | 156 | assert model["type"] == "model" 157 | assert isinstance(model["name"], str) 158 | assert model["connection"] == "test_dbt_project" 159 | 160 | view = next(v for v in views if v["name"] == "order_item") 161 | 162 | assert view["type"] == "view" 163 | assert isinstance(view["name"], str) 164 | assert isinstance(view["identifiers"], list) 165 | # assert view["sql_table_name"] == "order_item" 166 | assert view["default_date"] == "ordered_at" 167 | assert isinstance(view["fields"], list) 168 | 169 | ordered_at = next((f for f in view["fields"] if f["name"] == "ordered_at")) 170 | food_bool = next((f for f in view["fields"] if f["name"] == "is_food_item")) 171 | revenue_measure = next((f for f in view["fields"] if f["name"] == "_revenue")) 172 | median_revenue_metric = next((f for f in view["fields"] if f["name"] == "median_revenue")) 173 | 174 | assert ordered_at["type"] == "time" 175 | assert ordered_at["field_type"] == "dimension_group" 176 | assert "sql" in ordered_at 177 | assert isinstance(ordered_at["timeframes"], list) 178 | assert ordered_at["timeframes"][1] == "date" 179 | 180 | assert food_bool["type"] == "string" 181 | assert food_bool["field_type"] == "dimension" 182 | assert food_bool["sql"] == "${TABLE}.is_food_item" 183 | 184 | assert revenue_measure["type"] == "sum" 185 | assert revenue_measure["field_type"] == "measure" 186 | assert revenue_measure["sql"] == "product_price" 187 | assert revenue_measure["hidden"] 188 | 189 | assert median_revenue_metric["type"] == "median" 190 | assert median_revenue_metric["field_type"] == "measure" 191 | assert median_revenue_metric["sql"] == "product_price" 192 | assert median_revenue_metric["label"] == "Median Revenue" 193 | assert not median_revenue_metric["hidden"] 194 | 195 | assert len(dashboards) == 0 196 | -------------------------------------------------------------------------------- /tests/test_configuration.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | 3 | import pytest 4 | 5 | from metrics_layer import MetricsLayerConnection 6 | from metrics_layer.core.parse import ProjectLoader 7 | from metrics_layer.core.parse import ConfigError 8 | from metrics_layer.core.parse.connections import connection_class_lookup 9 | from metrics_layer.core.parse.connections import ( 10 | BigQueryConnection, 11 | RedshiftConnection, 12 | SnowflakeConnection, 13 | ConnectionType, 14 | ) 15 | 16 | 17 | def test_connection_types_present_in_lookup(): 18 | for connection_type_attr in dir(ConnectionType): 19 | if "__" not in connection_type_attr: 20 | connection_type = getattr(ConnectionType, connection_type_attr) 21 | assert connection_type in connection_class_lookup 22 | 23 | 24 | def test_config_explicit_metrics_layer_single_local(): 25 | loader = ProjectLoader(location="./tests/config/metrics_layer_config/") 26 | 27 | assert loader.repo.repo_path == "./tests/config/metrics_layer_config/" 28 | 29 | 30 | def test_config_explicit_metrics_layer_single_http(): 31 | loader = ProjectLoader(location="https://github.com", branch="dev") 32 | 33 | assert loader.repo.branch == "dev" 34 | assert loader.repo.repo_url == "https://github.com" 35 | 36 | 37 | def test_config_explicit_metrics_layer_single_ssh(): 38 | loader = ProjectLoader(location="git@github.com:", branch="dev") 39 | 40 | assert loader.repo.branch == "dev" 41 | assert loader.repo.repo_url == "git@github.com:" 42 | 43 | 44 | def test_config_explicit_metrics_layer_pickle(project): 45 | loader = ProjectLoader(location="https://github.com", branch="dev") 46 | loader._project = project 47 | 48 | # We should be able to pickle the loader 49 | pickle.dumps(loader) 50 | 51 | 52 | def test_config_explicit_metrics_layer_single_with_connections(): 53 | connections = [ 54 | { 55 | "type": "SNOWFLAKE", 56 | "name": "sf_name", 57 | "account": "sf_account", 58 | "username": "sf_username", 59 | "password": "sf_password", 60 | "role": "my_role", 61 | "warehouse": "compute_wh", 62 | "database": "company", 63 | "schema": "testing", 64 | }, 65 | { 66 | "type": "BIGQUERY", 67 | "name": "bq_name", 68 | "credentials": '{"key": "value", "project_id": "test-1234"}', 69 | }, 70 | { 71 | "type": "REDSHIFT", 72 | "name": "rs_name", 73 | "host": "rs_host", 74 | "username": "rs_username", 75 | "password": "rs_password", 76 | "database": "company", 77 | }, 78 | ] 79 | conn = MetricsLayerConnection(location="https://github.com", branch="dev", connections=connections) 80 | 81 | assert conn.branch == "dev" 82 | assert conn.location == "https://github.com" 83 | 84 | sf_connection = conn.get_connection("sf_name") 85 | assert isinstance(sf_connection, SnowflakeConnection) 86 | assert sf_connection.to_dict() == { 87 | "user": "sf_username", 88 | "password": "sf_password", 89 | "account": "sf_account", 90 | "role": "my_role", 91 | "warehouse": "compute_wh", 92 | "database": "company", 93 | "schema": "testing", 94 | "type": "SNOWFLAKE", 95 | "name": "sf_name", 96 | } 97 | 98 | bq_connection = conn.get_connection("bq_name") 99 | assert isinstance(bq_connection, BigQueryConnection) 100 | assert bq_connection.to_dict() == { 101 | "project_id": "test-1234", 102 | "credentials": {"key": "value", "project_id": "test-1234"}, 103 | "type": "BIGQUERY", 104 | "name": "bq_name", 105 | } 106 | 107 | rs_connection = conn.get_connection("rs_name") 108 | assert isinstance(rs_connection, RedshiftConnection) 109 | assert rs_connection.to_dict() == { 110 | "type": "REDSHIFT", 111 | "name": "rs_name", 112 | "host": "rs_host", 113 | "port": 5439, 114 | "username": "rs_username", 115 | "password": "rs_password", 116 | "database": "company", 117 | } 118 | 119 | result = conn.get_connection("does_not_exist") 120 | assert result is None 121 | 122 | 123 | def test_config_env_metrics_layer(monkeypatch): 124 | monkeypatch.setenv("METRICS_LAYER_LOCATION", "https://github.com") 125 | monkeypatch.setenv("METRICS_LAYER_BRANCH", "dev") 126 | monkeypatch.setenv("METRICS_LAYER_REPO_TYPE", "metrics_layer") 127 | loader = ProjectLoader(None) 128 | 129 | assert loader.repo.repo_type == "metrics_layer" 130 | assert loader.repo.repo_url == "https://github.com" 131 | assert loader.repo.branch == "dev" 132 | 133 | 134 | def test_config_explicit_env_config(monkeypatch): 135 | # Explicit should take priority 136 | monkeypatch.setenv("METRICS_LAYER_LOCATION", "https://github.com") 137 | monkeypatch.setenv("METRICS_LAYER_BRANCH", "dev") 138 | monkeypatch.setenv("METRICS_LAYER_REPO_TYPE", "metrics_layer") 139 | loader = ProjectLoader(location="https://correct.com", branch="master") 140 | 141 | assert loader.repo.branch == "master" 142 | assert loader.repo.repo_url == "https://correct.com" 143 | 144 | 145 | def test_config_does_not_exist(): 146 | # Should raise ConfigError 147 | with pytest.raises(ConfigError) as exc_info: 148 | ProjectLoader(None) 149 | 150 | assert exc_info.value 151 | -------------------------------------------------------------------------------- /tests/test_join_query_raw.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from metrics_layer.core.sql.query_errors import ArgumentError 4 | 5 | 6 | @pytest.mark.query 7 | def test_query_no_join_raw(connection): 8 | query = connection.get_sql_query( 9 | metrics=["total_item_revenue"], 10 | dimensions=["order_lines.order_line_id", "channel"], 11 | ) 12 | 13 | correct = ( 14 | "SELECT order_lines.order_line_id as order_lines_order_line_id," 15 | "order_lines.sales_channel as order_lines_channel" 16 | ",order_lines.revenue as order_lines_total_item_revenue FROM analytics.order_line_items order_lines;" 17 | ) 18 | assert query == correct 19 | 20 | 21 | @pytest.mark.query 22 | def test_query_join_raw_force_group_by_pretty(connection): 23 | query = connection.get_sql_query( 24 | metrics=["total_item_revenue"], 25 | dimensions=["order_lines.order_line_id", "channel"], 26 | force_group_by=True, 27 | pretty=True, 28 | ) 29 | 30 | correct = """select order_lines.order_line_id as order_lines_order_line_id, 31 | order_lines.sales_channel as order_lines_channel, 32 | SUM(order_lines.revenue) as order_lines_total_item_revenue 33 | from analytics.order_line_items order_lines 34 | group by order_lines.order_line_id, 35 | order_lines.sales_channel 36 | order by order_lines_total_item_revenue desc nulls last;""" 37 | 38 | assert query == correct 39 | 40 | 41 | @pytest.mark.query 42 | def test_query_single_join_non_base_primary_key(connection): 43 | query = connection.get_sql_query( 44 | metrics=["total_item_revenue"], 45 | dimensions=["orders.order_id", "channel", "new_vs_repeat"], 46 | ) 47 | 48 | correct = ( 49 | "SELECT orders.id as orders_order_id,order_lines.sales_channel as order_lines_channel," 50 | "orders.new_vs_repeat as orders_new_vs_repeat," 51 | "SUM(order_lines.revenue) as order_lines_total_item_revenue FROM " 52 | "analytics.order_line_items order_lines LEFT JOIN analytics.orders orders ON " 53 | "order_lines.order_unique_id=orders.id GROUP BY orders.id,order_lines.sales_channel," 54 | "orders.new_vs_repeat ORDER BY order_lines_total_item_revenue DESC NULLS LAST;" 55 | ) 56 | assert query == correct 57 | 58 | 59 | @pytest.mark.query 60 | def test_query_single_join_raw(connection): 61 | query = connection.get_sql_query( 62 | metrics=["total_item_revenue"], 63 | dimensions=["order_lines.order_line_id", "channel", "new_vs_repeat"], 64 | ) 65 | 66 | correct = ( 67 | "SELECT order_lines.order_line_id as order_lines_order_line_id," 68 | "order_lines.sales_channel as order_lines_channel," 69 | "orders.new_vs_repeat as orders_new_vs_repeat," 70 | "order_lines.revenue as order_lines_total_item_revenue FROM " 71 | "analytics.order_line_items order_lines LEFT JOIN analytics.orders orders ON " 72 | "order_lines.order_unique_id=orders.id;" 73 | ) 74 | assert query == correct 75 | 76 | 77 | @pytest.mark.query 78 | def test_query_single_join_raw_select_args(connection): 79 | query = connection.get_sql_query( 80 | metrics=["total_item_revenue"], 81 | dimensions=["order_lines.order_line_id", "channel", "new_vs_repeat"], 82 | select_raw_sql=[ 83 | "CAST(new_vs_repeat = 'Repeat' AS INT) as group_1", 84 | "CAST(date_created > '2021-04-02' AS INT) as period", 85 | ], 86 | ) 87 | 88 | correct = ( 89 | "SELECT order_lines.order_line_id as order_lines_order_line_id," 90 | "order_lines.sales_channel as order_lines_channel," 91 | "orders.new_vs_repeat as orders_new_vs_repeat," 92 | "order_lines.revenue as order_lines_total_item_revenue," 93 | "CAST(new_vs_repeat = 'Repeat' AS INT) as group_1," 94 | "CAST(date_created > '2021-04-02' AS INT) as period FROM " 95 | "analytics.order_line_items order_lines LEFT JOIN analytics.orders orders ON " 96 | "order_lines.order_unique_id=orders.id;" 97 | ) 98 | 99 | assert query == correct 100 | 101 | 102 | @pytest.mark.query 103 | def test_query_single_join_having_error(connection): 104 | with pytest.raises(ArgumentError) as exc_info: 105 | connection.get_sql_query( 106 | metrics=["total_item_revenue"], 107 | dimensions=["order_lines.order_line_id", "channel", "new_vs_repeat"], 108 | having=[{"field": "total_item_revenue", "expression": "less_than", "value": 22}], 109 | ) 110 | 111 | assert exc_info.value 112 | 113 | 114 | @pytest.mark.query 115 | def test_query_single_join_order_by_error(connection): 116 | with pytest.raises(ArgumentError) as exc_info: 117 | connection.get_sql_query( 118 | metrics=["total_item_revenue"], 119 | dimensions=["order_lines.order_line_id", "channel", "new_vs_repeat"], 120 | order_by=[{"field": "total_item_revenue"}], 121 | ) 122 | 123 | assert exc_info.value 124 | 125 | 126 | @pytest.mark.query 127 | def test_query_single_join_raw_all(connection): 128 | query = connection.get_sql_query( 129 | metrics=["total_item_revenue"], 130 | dimensions=["order_lines.order_line_id", "channel", "new_vs_repeat"], 131 | where=[{"field": "new_vs_repeat", "expression": "equal_to", "value": "Repeat"}], 132 | ) 133 | 134 | correct = ( 135 | "SELECT order_lines.order_line_id as order_lines_order_line_id," 136 | "order_lines.sales_channel as order_lines_channel," 137 | "orders.new_vs_repeat as orders_new_vs_repeat," 138 | "order_lines.revenue as order_lines_total_item_revenue FROM " 139 | "analytics.order_line_items order_lines LEFT JOIN analytics.orders orders ON " 140 | "order_lines.order_unique_id=orders.id WHERE orders.new_vs_repeat='Repeat';" 141 | ) 142 | assert query == correct 143 | -------------------------------------------------------------------------------- /tests/test_listing_functions.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.mark.project 5 | def test_list_metrics(connection): 6 | metrics = connection.list_metrics() 7 | assert len(metrics) == 70 8 | 9 | metrics = connection.list_metrics(view_name="order_lines", names_only=True) 10 | assert len(metrics) == 13 11 | assert set(metrics) == { 12 | "average_order_revenue", 13 | "costs_per_session", 14 | "ending_on_hand_qty", 15 | "line_item_aov", 16 | "should_be_number", 17 | "net_per_session", 18 | "number_of_email_purchased_items", 19 | "number_of_new_purchased_items", 20 | "pct_of_total_item_revenue", 21 | "revenue_per_session", 22 | "total_item_costs", 23 | "total_item_costs_pct", 24 | "total_item_revenue", 25 | } 26 | 27 | 28 | @pytest.mark.project 29 | def test_list_dimensions(connection): 30 | dimensions = connection.list_dimensions(show_hidden=True) 31 | assert len(dimensions) == 109 32 | 33 | dimensions = connection.list_dimensions() 34 | assert len(dimensions) == 75 35 | 36 | dimensions = connection.list_dimensions(view_name="order_lines", names_only=True, show_hidden=True) 37 | dimensions_present = { 38 | "order_line_id", 39 | "order_id", 40 | "customer_id", 41 | "order", 42 | "waiting", 43 | "channel", 44 | "parent_channel", 45 | "product_name", 46 | "product_name_lang", 47 | "inventory_qty", 48 | "is_on_sale_sql", 49 | "order_sequence", 50 | "new_vs_repeat_status", 51 | "is_on_sale_case", 52 | "order_tier", 53 | } 54 | assert len(dimensions) == 15 55 | assert set(dimensions) == dimensions_present 56 | 57 | 58 | @pytest.mark.project 59 | def test_project_expand_fields(connection): 60 | fields = connection.project.fields(show_hidden=False, expand_dimension_groups=True) 61 | 62 | dim_groups_alias = [f.alias() for f in fields if f.view.name == "orders" and f.name == "order"] 63 | 64 | assert dim_groups_alias == [ 65 | "order_time", 66 | "order_date", 67 | "order_day_of_year", 68 | "order_week", 69 | "order_week_of_year", 70 | "order_month", 71 | "order_month_of_year", 72 | "order_quarter", 73 | "order_year", 74 | "order_day_of_week", 75 | "order_hour_of_day", 76 | ] 77 | -------------------------------------------------------------------------------- /tests/test_query_running.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import pytest 3 | 4 | from metrics_layer.core import MetricsLayerConnection 5 | from metrics_layer.core.model.project import Project 6 | from metrics_layer.core.exceptions import QueryError 7 | from metrics_layer.core.sql import QueryRunner 8 | 9 | 10 | @pytest.mark.running 11 | def test_run_query_snowflake(monkeypatch, models, views): 12 | connections = [ 13 | { 14 | "type": "SNOWFLAKE", 15 | "name": "sf_name", 16 | "account": "sf_account", 17 | "username": "sf_username", 18 | "password": "sf_password", 19 | }, 20 | { 21 | "type": "BIGQUERY", 22 | "name": "bq_name", 23 | "credentials": '{"key": "value", "project_id": "test-1234"}', 24 | }, 25 | ] 26 | # Add reference to snowflake creds 27 | sf_models = [{**m, "connection": "sf_name"} for m in models] 28 | project = Project(models=sf_models, views=views) 29 | 30 | correct_df = pd.DataFrame({"dimension": ["cat1", "cat2", "cat3"], "metric": [12, 21, 34]}) 31 | monkeypatch.setattr(QueryRunner, "_run_snowflake_query", lambda *args, **kwargs: correct_df) 32 | 33 | conn = MetricsLayerConnection(project=project, connections=connections) 34 | df = conn.query(metrics=["total_item_revenue"], dimensions=["channel"]) 35 | 36 | assert df.equals(correct_df) 37 | 38 | 39 | @pytest.mark.running 40 | def test_run_query_bigquery(monkeypatch, models, views): 41 | connections = [ 42 | { 43 | "type": "SNOWFLAKE", 44 | "name": "sf_name", 45 | "account": "sf_account", 46 | "username": "sf_username", 47 | "password": "sf_password", 48 | }, 49 | { 50 | "type": "BIGQUERY", 51 | "name": "bq_name", 52 | "credentials": '{"key": "value", "project_id": "test-1234"}', 53 | }, 54 | ] 55 | # Add reference to BigQuery creds 56 | bq_models = [{**m, "connection": "bq_name"} for m in models] 57 | project = Project(models=bq_models, views=views, looker_env="prod") 58 | 59 | correct_df = pd.DataFrame({"dimension": ["cat7", "cat8", "cat9"], "metric": [98, 86, 65]}) 60 | monkeypatch.setattr(QueryRunner, "_run_bigquery_query", lambda *args, **kwargs: correct_df) 61 | 62 | conn = MetricsLayerConnection(project=project, connections=connections) 63 | df = conn.query(metrics=["total_item_revenue"], dimensions=["channel"]) 64 | 65 | assert df.equals(correct_df) 66 | 67 | 68 | @pytest.mark.running 69 | def test_run_query_no_connection_error(project): 70 | conn = MetricsLayerConnection(project=project, connections=[]) 71 | with pytest.raises(QueryError) as exc_info: 72 | conn.query(metrics=["total_item_revenue"], dimensions=["channel"]) 73 | 74 | assert exc_info.value 75 | -------------------------------------------------------------------------------- /tests/test_seeding.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from metrics_layer.cli.seeding import SeedMetricsLayer 4 | 5 | 6 | @pytest.mark.seeding 7 | @pytest.mark.parametrize( 8 | "db_name,db_conn", 9 | [ 10 | ("test", "testing_snowflake"), 11 | ("test", "testing_bigquery"), 12 | ("test", "testing_databricks"), 13 | (None, "testing_databricks"), 14 | ], 15 | ) 16 | def test_seeding_table_query(connection, db_name, db_conn): 17 | seeder = SeedMetricsLayer(database=db_name, metrics_layer=connection, connection=db_conn) 18 | table_query = seeder.table_query() 19 | 20 | if db_conn == "testing_snowflake": 21 | correct = ( 22 | "SELECT table_catalog as table_database, table_schema as table_schema, " 23 | "table_name as table_name, table_owner as table_owner, table_type as table_type, " 24 | "bytes as table_size, created as table_created, last_altered as table_last_modified, " 25 | "row_count as table_row_count, comment as comment FROM test.INFORMATION_SCHEMA.TABLES;" 26 | ) 27 | elif db_conn == "testing_bigquery": 28 | correct = ( 29 | "SELECT table_catalog as table_database, table_schema as table_schema, " 30 | "table_name as table_name, table_type as table_type, " 31 | "creation_time as table_created FROM `test.test_schema`.INFORMATION_SCHEMA.TABLES;" 32 | ) 33 | elif db_conn == "testing_databricks" and db_name == "test": 34 | correct = ( 35 | "SELECT table_catalog as table_database, table_schema as table_schema, " 36 | "table_name as table_name, table_type as table_type, comment as comment" 37 | " FROM test.INFORMATION_SCHEMA.TABLES;" 38 | ) 39 | elif db_conn == "testing_databricks" and db_name is None: 40 | correct = ( 41 | "SELECT table_catalog as table_database, table_schema as table_schema, " 42 | "table_name as table_name, table_type as table_type, comment as comment" 43 | " FROM INFORMATION_SCHEMA.TABLES;" 44 | ) 45 | else: 46 | raise ValueError(f"Unknown connection: {db_conn}") 47 | 48 | assert table_query == correct 49 | -------------------------------------------------------------------------------- /tests/test_set_functions.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.mark.project 5 | def test_sets(connection): 6 | sets = connection.project.sets() 7 | assert len(sets) == 5 8 | 9 | sets = connection.project.sets(view_name="orders") 10 | assert len(sets) == 5 11 | 12 | sets = connection.project.sets(view_name="order_lines") 13 | assert len(sets) == 0 14 | 15 | _set = connection.project.get_set(set_name="test_set") 16 | assert _set.field_names() == ["orders.order_id", "orders.customer_id", "orders.total_revenue"] 17 | 18 | _set = connection.project.get_set(set_name="test_set2") 19 | assert _set.field_names() == [ 20 | "orders.order_id", 21 | "orders.new_vs_repeat", 22 | "orders.sub_channel", 23 | "orders.average_order_value", 24 | "orders.order_time", 25 | ] 26 | 27 | _set = connection.project.get_set(set_name="test_set_composed") 28 | assert _set.field_names() == [ 29 | "orders.order_id", 30 | "orders.customer_id", 31 | "orders.total_revenue", 32 | "orders.average_order_value", 33 | "orders.order_time", 34 | ] 35 | 36 | _set = connection.project.get_set(set_name="test_set_all_fields") 37 | assert _set.field_names() == [ 38 | "orders.customer_id", 39 | "orders.account_id", 40 | "orders.anon_id", 41 | "orders.do_not_use", 42 | "orders.order_raw", 43 | "orders.order_date", 44 | "orders.order_day_of_year", 45 | "orders.order_week", 46 | "orders.order_week_of_year", 47 | "orders.order_month", 48 | "orders.order_month_of_year", 49 | "orders.order_quarter", 50 | "orders.order_year", 51 | "orders.order_day_of_week", 52 | "orders.order_hour_of_day", 53 | "orders.previous_order_raw", 54 | "orders.previous_order_time", 55 | "orders.previous_order_date", 56 | "orders.previous_order_week", 57 | "orders.previous_order_month", 58 | "orders.previous_order_quarter", 59 | "orders.previous_order_year", 60 | "orders.hours_between_orders", 61 | "orders.days_between_orders", 62 | "orders.weeks_between_orders", 63 | "orders.months_between_orders", 64 | "orders.quarters_between_orders", 65 | "orders.years_between_orders", 66 | "orders.revenue_in_cents", 67 | "orders.last_order_channel", 68 | "orders.last_order_warehouse_location", 69 | "orders.warehouse_location", 70 | "orders.campaign", 71 | "orders.number_of_orders", 72 | "orders.average_days_between_orders", 73 | "orders.total_revenue", 74 | "orders.total_non_merchant_revenue", 75 | "orders.total_lifetime_revenue", 76 | "orders.cumulative_customers", 77 | "orders.cumulative_customers_no_change_grain", 78 | "orders.cumulative_aov", 79 | "orders.ltv", 80 | "orders.ltr", 81 | "orders.total_modified_revenue", 82 | "orders.average_order_value_custom", 83 | "orders.new_order_count", 84 | ] 85 | 86 | 87 | @pytest.mark.project 88 | def test_drill_fields(connection): 89 | field = connection.project.get_field("orders.new_vs_repeat") 90 | 91 | drill_field_names = field.drill_fields 92 | assert field.id() == "orders.new_vs_repeat" 93 | assert drill_field_names == [ 94 | "orders.order_id", 95 | "orders.customer_id", 96 | "orders.total_revenue", 97 | "orders.new_vs_repeat", 98 | ] 99 | 100 | field = connection.project.get_field("orders.total_revenue") 101 | assert field.drill_fields is None 102 | assert field.id() == "orders.total_revenue" 103 | 104 | 105 | @pytest.mark.project 106 | def test_joinable_fields_join(connection): 107 | field = connection.project.get_field("orders.number_of_orders") 108 | 109 | joinable_fields = connection.project.joinable_fields([field], expand_dimension_groups=True) 110 | names = [f.id() for f in joinable_fields] 111 | must_exclude = ["traffic.traffic_source"] 112 | must_include = [ 113 | "orders.order_date", 114 | "order_lines.order_id", 115 | "customers.number_of_customers", 116 | "country_detail.rainfall", 117 | "discount_detail.discount_usd", 118 | "discounts.discount_code", 119 | "order_lines.revenue_per_session", 120 | "sessions.number_of_sessions", 121 | ] 122 | assert all(name in names for name in must_include) 123 | assert all(name not in names for name in must_exclude) 124 | 125 | test_fields = [field] 126 | add_fields = [ 127 | "order_lines.order_month", 128 | "order_lines.product_name", 129 | "customers.gender", 130 | "order_lines.ending_on_hand_qty", 131 | ] 132 | for field_name in add_fields: 133 | test_fields.append(connection.project.get_field(field_name)) 134 | 135 | joinable_fields = connection.project.joinable_fields(test_fields, expand_dimension_groups=True) 136 | names = [f.id() for f in joinable_fields] 137 | assert all(name in names for name in must_include) 138 | 139 | 140 | @pytest.mark.project 141 | def test_joinable_fields_merged(connection): 142 | field = connection.project.get_field("order_lines.revenue_per_session") 143 | 144 | # Add tests for exclusions here 145 | joinable_fields = connection.project.joinable_fields([field], expand_dimension_groups=True) 146 | names = [f.id() for f in joinable_fields] 147 | must_exclude = ["traffic.traffic_source", "discounts.discount_code"] 148 | must_include = [ 149 | "order_lines.order_date", 150 | "customers.gender", 151 | "order_lines.total_item_revenue", 152 | "order_lines.revenue_per_session", 153 | "orders.sub_channel", 154 | "sessions.utm_source", 155 | "sessions.session_year", 156 | "sessions.number_of_sessions", 157 | ] 158 | assert all(name in names for name in must_include) 159 | assert all(name not in names for name in must_exclude) 160 | -------------------------------------------------------------------------------- /tests/test_sql_query_resolver.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from metrics_layer.core.model.project import Project 4 | from metrics_layer.core.sql.resolve import SQLQueryResolver 5 | 6 | 7 | @pytest.mark.query 8 | def test_empty_query(models, views, connections): 9 | SQLQueryResolver( 10 | metrics=[], 11 | dimensions=[], 12 | project=Project(models=models, views=views), 13 | connections=connections, 14 | ) 15 | -------------------------------------------------------------------------------- /tests/test_user_attributes_query.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.mark.query 5 | def test_query_user_attribute_in_sql(connection): 6 | connection.project.set_user({"user_lang": "us-en"}) 7 | 8 | query = connection.get_sql_query( 9 | metrics=["total_item_revenue"], 10 | dimensions=["product_name_lang", "product_name"], 11 | ) 12 | 13 | correct = ( 14 | "SELECT LOOKUP(order_lines.product_name, 'us-en' ) as" 15 | " order_lines_product_name_lang,order_lines.product_name as" 16 | " order_lines_product_name,SUM(order_lines.revenue) as order_lines_total_item_revenue FROM" 17 | " analytics.order_line_items order_lines GROUP BY LOOKUP(order_lines.product_name, 'us-en'" 18 | " ),order_lines.product_name ORDER BY order_lines_total_item_revenue DESC NULLS LAST;" 19 | ) 20 | connection.project.set_user({}) 21 | assert query == correct 22 | 23 | 24 | @pytest.mark.query 25 | def test_query_user_attribute_in_derived_table_sql(connection): 26 | connection.project.set_user({"owned_region": "Europe"}) 27 | query = connection.get_sql_query(metrics=["avg_rainfall"], dimensions=[]) 28 | 29 | correct = ( 30 | "SELECT AVG(country_detail.rain) as country_detail_avg_rainfall FROM (SELECT * FROM" 31 | " ANALYTICS.COUNTRY_DETAIL WHERE 'Europe' = COUNTRY_DETAIL.REGION) as country_detail ORDER BY" 32 | " country_detail_avg_rainfall DESC NULLS LAST;" 33 | ) 34 | connection.project.set_user({}) 35 | assert query == correct 36 | 37 | 38 | @pytest.mark.query 39 | def test_query_user_attribute_in_sql_table_name(connection): 40 | connection.project.set_user({"db_name": "q3lm13dfa"}) 41 | query = connection.get_sql_query(metrics=[], dimensions=["other_traffic_id"]) 42 | 43 | correct = ( 44 | "SELECT other_db_traffic.id as other_db_traffic_other_traffic_id FROM q3lm13dfa.analytics.traffic" 45 | " other_db_traffic GROUP BY other_db_traffic.id ORDER BY other_db_traffic_other_traffic_id ASC NULLS" 46 | " LAST;" 47 | ) 48 | connection.project.set_user({}) 49 | assert query == correct 50 | 51 | 52 | @pytest.mark.query 53 | def test_query_user_attribute_in_filter_sql(connection): 54 | connection.project.set_user({"owned_region": "Europe", "country_options": "Italy, FRANCE"}) 55 | query = connection.get_sql_query(metrics=["avg_rainfall_adj"], dimensions=[]) 56 | 57 | correct = ( 58 | "SELECT AVG(case when country_detail.country IN ('Italy','FRANCE') then country_detail.rain end) as" 59 | " country_detail_avg_rainfall_adj FROM (SELECT * FROM ANALYTICS.COUNTRY_DETAIL WHERE 'Europe' =" 60 | " COUNTRY_DETAIL.REGION) as country_detail ORDER BY country_detail_avg_rainfall_adj DESC NULLS LAST;" 61 | ) 62 | connection.project.set_user({}) 63 | assert query == correct 64 | 65 | 66 | @pytest.mark.query 67 | def test_query_user_attribute_in_filter_sql_error_state(connection): 68 | # When the user attribute is not set, the query will not fill in the user attribute 69 | query = connection.get_sql_query(metrics=["avg_rainfall_adj"], dimensions=[]) 70 | 71 | correct = ( 72 | "SELECT AVG(case when country_detail.country='{{ user_attributes[''country_options''] }}' then" 73 | " country_detail.rain end) as country_detail_avg_rainfall_adj FROM (SELECT * FROM" 74 | " ANALYTICS.COUNTRY_DETAIL WHERE '{{ user_attributes['owned_region'] }}' = COUNTRY_DETAIL.REGION) as" 75 | " country_detail ORDER BY country_detail_avg_rainfall_adj DESC NULLS LAST;" 76 | ) 77 | assert query == correct 78 | 79 | 80 | @pytest.mark.query 81 | def test_query_query_attribute_dimension_group(connection): 82 | connection.project.set_user({"owned_region": "Europe"}) 83 | query = connection.get_sql_query(metrics=["avg_rainfall"], dimensions=["rainfall_at_date"]) 84 | 85 | correct = ( 86 | "SELECT DATE_TRUNC('DAY', case when 'date' = 'raw' then country_detail.rain_date when 'date' = 'date'" 87 | " then time_bucket('1 day', country_detail.rain_date) when 'date' = 'week' then time_bucket('1 week'," 88 | " country_detail.rain_date) when 'date' = 'month' then time_bucket('1 month'," 89 | " country_detail.rain_date) end) as country_detail_rainfall_at_date,AVG(country_detail.rain) as" 90 | " country_detail_avg_rainfall FROM (SELECT * FROM ANALYTICS.COUNTRY_DETAIL WHERE 'Europe' =" 91 | " COUNTRY_DETAIL.REGION) as country_detail GROUP BY DATE_TRUNC('DAY', case when 'date' = 'raw' then" 92 | " country_detail.rain_date when 'date' = 'date' then time_bucket('1 day', country_detail.rain_date)" 93 | " when 'date' = 'week' then time_bucket('1 week', country_detail.rain_date) when 'date' = 'month'" 94 | " then time_bucket('1 month', country_detail.rain_date) end) ORDER BY country_detail_avg_rainfall" 95 | " DESC NULLS LAST;" 96 | ) 97 | connection.project.set_user({}) 98 | assert query == correct 99 | --------------------------------------------------------------------------------