├── .gitignore ├── README.md ├── macros ├── volume_by_demographic.sql ├── volume_ratio.sql └── volume_ratios.sql ├── models ├── example_funnel.sql ├── purchase_sessions_ratio.sql ├── purchase_sessions_ratio_2.sql ├── purchase_volumes.sql └── session_volumes.sql ├── render.py └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # jinja-sql-demo 2 | 3 | A small proof of concept for how to set up a Jinja-powered codebase for an analytics org. 4 | 5 | Designed to be paired with this [thesis article on Medium](https://towardsdatascience.com/jinja-sql-%EF%B8%8F-7e4dff8d8778?sk=2e8e678c133ec42f7ae8427f04a97295). 6 | 7 | **Structure** 8 | 9 | Files in the `macros` directory are templates for SQL queries. 10 | 11 | Files in the `models` directory are callers of those SQL templates, and can be rendered into actual SQL. 12 | 13 | **Dependencies** 14 | 15 | This should work with Python3 on Mac/Linux machines. 16 | 17 | You can install requirements with: 18 | 19 | `pip install -r requirements.txt` 20 | 21 | **Usage** 22 | 23 | To render a model, you can run: 24 | 25 | `./render.py models/{model_name}.sql` 26 | -------------------------------------------------------------------------------- /macros/volume_by_demographic.sql: -------------------------------------------------------------------------------- 1 | {%- macro volume_by_demographic(entity_table, count_col_name) -%} 2 | 3 | select 4 | users.age_group, 5 | users.state, 6 | count(*) as {{ count_col_name }} 7 | from {{ entity_table }} 8 | join users on {{ entity_table }}.user_id=users.user_id 9 | group by 1,2 10 | 11 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/volume_ratio.sql: -------------------------------------------------------------------------------- 1 | {%- from 'volume_by_demographic.sql' import volume_by_demographic -%} 2 | 3 | {% macro volume_ratio( 4 | numerator_vol_ref, 5 | denominator_vol_ref 6 | ) %} 7 | 8 | select 9 | age_group, 10 | state, 11 | N / D as {{ numerator_vol_ref }}_over_{{ denominator_vol_ref }} 12 | from ( 13 | {{ volume_by_demographic(numerator_vol_ref, 'N') }} 14 | ) p 15 | join ( 16 | {{ volume_by_demographic(denominator_vol_ref, 'D') }} 17 | ) s 18 | on p.age_group=s.age_group 19 | and p.state=s.state 20 | 21 | {% endmacro %} 22 | 23 | -------------------------------------------------------------------------------- /macros/volume_ratios.sql: -------------------------------------------------------------------------------- 1 | {% from 'volume_ratio.sql' import volume_ratio %} 2 | 3 | {% macro volume_ratios(numerator_vol_refs, denominator_vol_ref) %} 4 | 5 | select 6 | age_group, 7 | state, 8 | {%- for ref in numerator_vol_refs -%} 9 | {{ ref }}_over_{{ denominator_vol_ref }} 10 | {%- if not loop.last -%},{%- endif -%} 11 | {% endfor %} 12 | from 13 | ({{ volume_ratio(numerator_vol_refs.0, denominator_vol_ref) }}) 14 | as base 15 | {% for ref in numerator_vol_refs %} 16 | {% if not loop.first %} 17 | join 18 | ({{ volume_ratio(ref, denominator_vol_ref) }}) as {{ref}}_r 19 | on {{ref}}_r.age_group=base.age_group 20 | and {{ref}}_r.state=base.state 21 | {%- endif -%} 22 | {%- endfor -%} 23 | 24 | {% endmacro %} -------------------------------------------------------------------------------- /models/example_funnel.sql: -------------------------------------------------------------------------------- 1 | {%- from 'volume_ratios.sql' import volume_ratios -%} 2 | 3 | {{ volume_ratios([ 4 | 'purchases', 5 | 'sessions', 6 | 'supportTickets' 7 | ], 8 | 'marketing_pushes') 9 | }} 10 | -------------------------------------------------------------------------------- /models/purchase_sessions_ratio.sql: -------------------------------------------------------------------------------- 1 | {%- from 'volume_by_demographic.sql' import volume_by_demographic -%} 2 | 3 | -- Get ratio of purchases to sessions for a demographic group 4 | 5 | select 6 | age_group, 7 | state, 8 | purchase_volume / session_volume as ratio 9 | from ( 10 | {{ volume_by_demographic('purchases', 'purchase_volume') }} 11 | ) p 12 | join ( 13 | {{ volume_by_demographic('sessions', 'session_volume') }} 14 | ) s 15 | on p.age_group=s.age_group 16 | and p.state=s.state -------------------------------------------------------------------------------- /models/purchase_sessions_ratio_2.sql: -------------------------------------------------------------------------------- 1 | {%- from 'volume_ratio.sql' import volume_ratio -%} 2 | 3 | -- Get ratio of purchases to sessions for a demographic group 4 | {{ volume_ratio('purchases', 'sessions')}} -------------------------------------------------------------------------------- /models/purchase_volumes.sql: -------------------------------------------------------------------------------- 1 | {%- from 'volume_by_demographic.sql' import volume_by_demographic -%} 2 | 3 | -- Get purchase volume by demographic 4 | {{ volume_by_demographic('purchases', 'purchase_volume') }} -------------------------------------------------------------------------------- /models/session_volumes.sql: -------------------------------------------------------------------------------- 1 | {%- from 'volume_by_demographic.sql' import volume_by_demographic -%} 2 | 3 | -- Get session volume by demographic 4 | {{ volume_by_demographic('sessions', 'session_volume') }} 5 | -------------------------------------------------------------------------------- /render.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | from jinja2 import Environment, FileSystemLoader 5 | 6 | MACRO_DIR = "macros" 7 | 8 | 9 | def get_template_render(path: str) -> str: 10 | """ 11 | Render a Jinja2 template to actual text, using macros in scope. 12 | :param path: Path to the file you'd like to render. 13 | """ 14 | env = Environment(loader=FileSystemLoader(MACRO_DIR)) 15 | render = env.from_string(open(path, 'r').read()).render() 16 | return render 17 | 18 | 19 | if __name__ == '__main__': 20 | assert len(sys.argv) > 1 and sys.argv[1] is not None, """Filepath must be supplied. Example usage: 21 | `./render.py models/purchase_volumes.sql` 22 | """ 23 | filepath = sys.argv[1] 24 | render = get_template_render(filepath) 25 | print(render) 26 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | jinja2 2 | --------------------------------------------------------------------------------