├── .envs └── .local │ ├── .django │ └── .postgres ├── LICENSE ├── README.md ├── compose └── local │ ├── django │ ├── Dockerfile │ ├── celery │ │ ├── beat │ │ │ └── start │ │ └── worker │ │ │ └── start │ ├── entrypoint │ └── start │ ├── docs │ ├── Dockerfile │ └── start │ └── postgres │ ├── Dockerfile │ ├── entrypoint.sh │ └── maintenance │ ├── _sourced │ ├── constants.sh │ ├── countdown.sh │ ├── messages.sh │ └── yes_no.sh │ ├── backup │ ├── backups │ └── restore ├── config ├── __init__.py ├── api_router.py ├── celery_app.py ├── settings │ ├── __init__.py │ ├── base.py │ └── local.py ├── urls.py └── wsgi.py ├── dbt ├── __init__.py ├── analytics │ ├── __init__.py │ ├── admin.py │ ├── celery.py │ ├── migrations │ │ └── __init__.py │ ├── models.py │ ├── serializers.py │ └── views.py ├── contrib │ ├── __init__.py │ └── sites │ │ ├── __init__.py │ │ └── migrations │ │ ├── 0001_initial.py │ │ ├── 0002_alter_domain_unique.py │ │ ├── 0003_set_site_domain_and_name.py │ │ ├── 0004_alter_options_ordering_domain.py │ │ └── __init__.py ├── static │ ├── css │ │ └── project.css │ ├── fonts │ │ └── .gitkeep │ ├── images │ │ └── favicons │ │ │ └── favicon.ico │ └── js │ │ └── project.js ├── templates │ ├── 403.html │ ├── 404.html │ ├── 500.html │ ├── account │ │ ├── account_inactive.html │ │ ├── base.html │ │ ├── email.html │ │ ├── email_confirm.html │ │ ├── login.html │ │ ├── logout.html │ │ ├── password_change.html │ │ ├── password_reset.html │ │ ├── password_reset_done.html │ │ ├── password_reset_from_key.html │ │ ├── password_reset_from_key_done.html │ │ ├── password_set.html │ │ ├── profile.html │ │ ├── signup.html │ │ ├── signup_closed.html │ │ ├── verification_sent.html │ │ └── verified_email_required.html │ ├── admin │ │ ├── base_site.html │ │ └── index.html │ ├── base.html │ └── users │ │ ├── user_detail.html │ │ └── user_form.html ├── users │ ├── __init__.py │ ├── apps.py │ ├── context_processors.py │ ├── forms.py │ ├── management │ │ ├── __init__.py │ │ └── commands │ │ │ ├── __init__.py │ │ │ ├── dbt_command.py │ │ │ ├── dbt_to_db.py │ │ │ └── wait_for_db.py │ ├── migrations │ │ ├── 0001_initial.py │ │ ├── 0002_alter_user_id.py │ │ └── __init__.py │ ├── models.py │ ├── signals.py │ └── views.py └── utils │ ├── __init__.py │ └── common.py ├── docker-compose.yml ├── manage.py ├── print_dbt_current_version.py ├── requirements ├── base.txt └── local.txt └── setup.cfg /.envs/.local/.django: -------------------------------------------------------------------------------- 1 | # General 2 | # ------------------------------------------------------------------------------ 3 | USE_DOCKER=yes 4 | # Redis 5 | 6 | # ------------------------------------------------------------------------------ 7 | REDIS_URL=redis://redis:6379/0 8 | 9 | # Celery 10 | # ------------------------------------------------------------------------------ 11 | CELERY_BROKER_URL=redis://redis:6379/0 12 | -------------------------------------------------------------------------------- /.envs/.local/.postgres: -------------------------------------------------------------------------------- 1 | # PostgreSQL 2 | # ------------------------------------------------------------------------------ 3 | POSTGRES_HOST=postgres 4 | POSTGRES_PORT=5432 5 | POSTGRES_DB=analytics 6 | POSTGRES_USER=dbtuser 7 | 8 | 9 | # Postgrest 10 | # ------------------------------------------------------------------------------ 11 | PGRST_DB_URI=postgres://dbtuser:password@postgres:5432/analytics 12 | PGRST_DB_ANON_ROLE=web_anon 13 | PGRST_DB_SCHEMAS=public 14 | PGRST_OPENAPI_MODE=ignore-privileges 15 | PGRST_OPENAPI_SERVER_PROXY_URI=http://127.0.0.1:3000 16 | #PGRST_JWT_SECRET=pgrst_jwt_secret 17 | 18 | 19 | # ------------------------------------------------------------------------------ 20 | # Swagger 21 | API_URL=http://127.0.0.1:3000/ 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Eric Arsenault 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # django-build-tool 2 | 3 | https://eric-arsenaults.medium.com/django-build-tool-an-open-source-orchestration-tool-for-dbt-a5069dfdadcb 4 | 5 | 6 | -------------------------------------------------------------------------------- /compose/local/django/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG PYTHON_VERSION=3.11-slim-bullseye 2 | 3 | # define an alias for the specfic python version used in this file. 4 | FROM python:${PYTHON_VERSION} as python 5 | 6 | # Python build stage 7 | FROM python as python-build-stage 8 | 9 | ARG BUILD_ENVIRONMENT=local 10 | 11 | # System setup 12 | RUN apt-get update && apt-get install --no-install-recommends -y \ 13 | # dependencies for building Python packages 14 | build-essential \ 15 | # psycopg2 dependencies 16 | libpq-dev 17 | 18 | # Requirements are installed here to ensure they will be cached. 19 | COPY ./requirements . 20 | 21 | # Create Python Dependency and Sub-Dependency Wheels. 22 | RUN pip wheel --wheel-dir /usr/src/app/wheels \ 23 | -r ${BUILD_ENVIRONMENT}.txt 24 | 25 | 26 | # Python 'run' stage 27 | FROM python as python-run-stage 28 | 29 | ARG BUILD_ENVIRONMENT=local 30 | ARG APP_HOME=/app 31 | 32 | ENV PYTHONUNBUFFERED 1 33 | ENV PYTHONDONTWRITEBYTECODE 1 34 | ENV BUILD_ENV ${BUILD_ENVIRONMENT} 35 | 36 | WORKDIR ${APP_HOME} 37 | 38 | # Install required system dependencies 39 | RUN apt-get update && apt-get install --no-install-recommends -y \ 40 | # psycopg2 dependencies 41 | libpq-dev \ 42 | # Translations dependencies 43 | gettext \ 44 | # dbt dependencies 45 | git \ 46 | ssh-client \ 47 | software-properties-common \ 48 | make \ 49 | ca-certificates \ 50 | # cleaning up unused files 51 | && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ 52 | && rm -rf /var/lib/apt/lists/* 53 | 54 | # All absolute dir copies ignore workdir instruction. All relative dir copies are wrt to the workdir instruction 55 | # copy python dependency wheels from python-build-stage 56 | COPY --from=python-build-stage /usr/src/app/wheels /wheels/ 57 | 58 | # use wheels to install python dependencies 59 | RUN pip install --no-cache-dir --no-index --find-links=/wheels/ /wheels/* \ 60 | && rm -rf /wheels/ 61 | 62 | RUN python -m pip install --upgrade pip setuptools wheel --no-cache-dir 63 | RUN python -m pip install dbt-core dbt-postgres dbt-redshift dbt-snowflake dbt-bigquery 64 | 65 | 66 | COPY ./compose/local/django/entrypoint /entrypoint 67 | RUN sed -i 's/\r$//g' /entrypoint 68 | RUN chmod +x /entrypoint 69 | 70 | COPY ./compose/local/django/start /start 71 | RUN sed -i 's/\r$//g' /start 72 | RUN chmod +x /start 73 | 74 | 75 | COPY ./compose/local/django/celery/worker/start /start-celeryworker 76 | RUN sed -i 's/\r$//g' /start-celeryworker 77 | RUN chmod +x /start-celeryworker 78 | 79 | COPY ./compose/local/django/celery/beat/start /start-celerybeat 80 | RUN sed -i 's/\r$//g' /start-celerybeat 81 | RUN chmod +x /start-celerybeat 82 | 83 | RUN mkdir -p /root/.dbt 84 | # copy application code to WORKDIR 85 | COPY . ${APP_HOME} 86 | 87 | ENTRYPOINT ["/entrypoint"] 88 | -------------------------------------------------------------------------------- /compose/local/django/celery/beat/start: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | rm -f './celerybeat.pid' 7 | # wait for migrations applied 8 | sleep 10 9 | 10 | set_database_url() { 11 | postgres_password=$(grep 'postgres_password' /passwords/password.txt | cut -d '=' -f 2) 12 | 13 | # Set DATABASE_URL environment variable 14 | database_url="postgres://dbtuser:$postgres_password@postgres:5432/analytics" 15 | export DATABASE_URL="$database_url" 16 | 17 | } 18 | set_database_url 19 | 20 | celery -A config.celery_app beat -l INFO 21 | -------------------------------------------------------------------------------- /compose/local/django/celery/worker/start: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | # wait for migrations applied 7 | sleep 5 8 | set_database_url() { 9 | postgres_password=$(grep 'postgres_password' /passwords/password.txt | cut -d '=' -f 2) 10 | 11 | # Set DATABASE_URL environment variable 12 | database_url="postgres://dbtuser:$postgres_password@postgres:5432/analytics" 13 | export DATABASE_URL="$database_url" 14 | 15 | } 16 | set_database_url 17 | celery -A config.celery_app worker -l INFO 18 | -------------------------------------------------------------------------------- /compose/local/django/entrypoint: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o pipefail 5 | set -o nounset 6 | 7 | # N.B. If only .env files supported variable expansion... 8 | export CELERY_BROKER_URL="${REDIS_URL}" 9 | 10 | 11 | if [ -z "${POSTGRES_USER}" ]; then 12 | base_postgres_image_default_user='postgres' 13 | export POSTGRES_USER="${base_postgres_image_default_user}" 14 | fi 15 | # Extract POSTGRES_PASSWORD from the passwords file 16 | export POSTGRES_PASSWORD=$(grep postgres_password /passwords/password.txt | cut -d '=' -f2) 17 | 18 | #export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}" 19 | 20 | python << END 21 | import sys 22 | import time 23 | 24 | import psycopg2 25 | 26 | suggest_unrecoverable_after = 30 27 | start = time.time() 28 | 29 | while True: 30 | try: 31 | psycopg2.connect( 32 | dbname="${POSTGRES_DB}", 33 | user="${POSTGRES_USER}", 34 | password="${POSTGRES_PASSWORD}", 35 | host="${POSTGRES_HOST}", 36 | port="${POSTGRES_PORT}", 37 | ) 38 | break 39 | except psycopg2.OperationalError as error: 40 | sys.stderr.write("Waiting for PostgreSQL to become available...\n") 41 | 42 | if time.time() - start > suggest_unrecoverable_after: 43 | sys.stderr.write(" This is taking longer than expected. The following exception may be indicative of an unrecoverable error: '{}'\n".format(error)) 44 | 45 | time.sleep(1) 46 | END 47 | 48 | >&2 echo 'PostgreSQL is available' 49 | 50 | exec "$@" 51 | -------------------------------------------------------------------------------- /compose/local/django/start: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o pipefail 5 | set -o nounset 6 | 7 | set_database_url() { 8 | postgres_password=$(grep 'postgres_password' /passwords/password.txt | cut -d '=' -f 2) 9 | 10 | # Set DATABASE_URL environment variable 11 | database_url="postgres://dbtuser:$postgres_password@postgres:5432/analytics" 12 | export DATABASE_URL="$database_url" 13 | 14 | } 15 | # create superuser 16 | create_superuser() { 17 | # Set predefined values for the superuser credentials 18 | username=$(grep 'dbt_login' /passwords/password.txt | cut -d '=' -f 2) 19 | email="dbtuser@example.com" 20 | # Read password from /passwords/password.txt 21 | password=$(grep 'dbt_password' /passwords/password.txt | cut -d '=' -f 2) 22 | 23 | # Check if the superuser already exists 24 | if python manage.py shell -c "from django.contrib.auth import get_user_model; User = get_user_model(); exists = User.objects.filter(username='$username').exists(); exit(0 if exists else 1)"; then 25 | # Superuser exists, skip creation 26 | echo "Superuser '$username' already exists. Skipping creation." 27 | else 28 | # Run Django management command to create the superuser 29 | python manage.py createsuperuser --noinput --username "$username" --email "$email" 30 | # Set the password for the superuser 31 | echo "from django.contrib.auth import get_user_model; User = get_user_model(); user = User.objects.get(username='$username'); user.set_password('$password'); user.save()" | python manage.py shell 32 | # create token for superuser 33 | python manage.py drf_create_token "$username" 34 | 35 | # Display the created superuser information 36 | echo "Superuser created:" 37 | echo "Username: $username" 38 | echo "Email: $email" 39 | fi 40 | } 41 | 42 | set_database_url 43 | 44 | python manage.py makemigrations 45 | # Run migrations 46 | python manage.py migrate 47 | 48 | create_superuser 49 | # 50 | # Execute DDL statements for the POSTGREST schema 51 | python manage.py shell <.yml (exec |run --rm) postgres backup 8 | 9 | 10 | set -o errexit 11 | set -o pipefail 12 | set -o nounset 13 | 14 | 15 | working_dir="$(dirname ${0})" 16 | source "${working_dir}/_sourced/constants.sh" 17 | source "${working_dir}/_sourced/messages.sh" 18 | 19 | 20 | message_welcome "Backing up the '${POSTGRES_DB}' database..." 21 | 22 | 23 | if [[ "${POSTGRES_USER}" == "postgres" ]]; then 24 | message_error "Backing up as 'postgres' user is not supported. Assign 'POSTGRES_USER' env with another one and try again." 25 | exit 1 26 | fi 27 | 28 | export PGHOST="${POSTGRES_HOST}" 29 | export PGPORT="${POSTGRES_PORT}" 30 | export PGUSER="${POSTGRES_USER}" 31 | export PGPASSWORD="${POSTGRES_PASSWORD}" 32 | export PGDATABASE="${POSTGRES_DB}" 33 | 34 | backup_filename="${BACKUP_FILE_PREFIX}_$(date +'%Y_%m_%dT%H_%M_%S').sql.gz" 35 | pg_dump | gzip > "${BACKUP_DIR_PATH}/${backup_filename}" 36 | 37 | 38 | message_success "'${POSTGRES_DB}' database backup '${backup_filename}' has been created and placed in '${BACKUP_DIR_PATH}'." 39 | -------------------------------------------------------------------------------- /compose/local/postgres/maintenance/backups: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | ### View backups. 5 | ### 6 | ### Usage: 7 | ### $ docker-compose -f .yml (exec |run --rm) postgres backups 8 | 9 | 10 | set -o errexit 11 | set -o pipefail 12 | set -o nounset 13 | 14 | 15 | working_dir="$(dirname ${0})" 16 | source "${working_dir}/_sourced/constants.sh" 17 | source "${working_dir}/_sourced/messages.sh" 18 | 19 | 20 | message_welcome "These are the backups you have got:" 21 | 22 | ls -lht "${BACKUP_DIR_PATH}" 23 | -------------------------------------------------------------------------------- /compose/local/postgres/maintenance/restore: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | ### Restore database from a backup. 5 | ### 6 | ### Parameters: 7 | ### <1> filename of an existing backup. 8 | ### 9 | ### Usage: 10 | ### $ docker-compose -f .yml (exec |run --rm) postgres restore <1> 11 | 12 | 13 | set -o errexit 14 | set -o pipefail 15 | set -o nounset 16 | 17 | 18 | working_dir="$(dirname ${0})" 19 | source "${working_dir}/_sourced/constants.sh" 20 | source "${working_dir}/_sourced/messages.sh" 21 | 22 | 23 | if [[ -z ${1+x} ]]; then 24 | message_error "Backup filename is not specified yet it is a required parameter. Make sure you provide one and try again." 25 | exit 1 26 | fi 27 | backup_filename="${BACKUP_DIR_PATH}/${1}" 28 | if [[ ! -f "${backup_filename}" ]]; then 29 | message_error "No backup with the specified filename found. Check out the 'backups' maintenance script output to see if there is one and try again." 30 | exit 1 31 | fi 32 | 33 | message_welcome "Restoring the '${POSTGRES_DB}' database from the '${backup_filename}' backup..." 34 | 35 | if [[ "${POSTGRES_USER}" == "postgres" ]]; then 36 | message_error "Restoring as 'postgres' user is not supported. Assign 'POSTGRES_USER' env with another one and try again." 37 | exit 1 38 | fi 39 | 40 | export PGHOST="${POSTGRES_HOST}" 41 | export PGPORT="${POSTGRES_PORT}" 42 | export PGUSER="${POSTGRES_USER}" 43 | export PGPASSWORD="${POSTGRES_PASSWORD}" 44 | export PGDATABASE="${POSTGRES_DB}" 45 | 46 | message_info "Dropping the database..." 47 | dropdb "${PGDATABASE}" 48 | 49 | message_info "Creating a new database..." 50 | createdb --owner="${POSTGRES_USER}" 51 | 52 | message_info "Applying the backup to the new database..." 53 | gunzip -c "${backup_filename}" | psql "${POSTGRES_DB}" 54 | 55 | message_success "The '${POSTGRES_DB}' database has been restored from the '${backup_filename}' backup." 56 | -------------------------------------------------------------------------------- /config/__init__.py: -------------------------------------------------------------------------------- 1 | # This will make sure the app is always imported when 2 | # Django starts so that shared_task will use this app. 3 | from .celery_app import app as celery_app 4 | 5 | __all__ = ("celery_app",) 6 | -------------------------------------------------------------------------------- /config/api_router.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from rest_framework.routers import DefaultRouter, SimpleRouter 3 | from dbt.analytics.views import ( 4 | GitRepoAPIViewset, 5 | SSHKeyViewSets, 6 | InterValViewSet, 7 | AddPeriodicTask, 8 | PostYMALDetailsView, 9 | CrontabScheduleViewSet, 10 | DBTCurrentVersionView, 11 | RunDBTTask, 12 | ) 13 | from django.urls import path 14 | from django.conf.urls.static import static 15 | 16 | if settings.DEBUG: 17 | router = DefaultRouter() 18 | else: 19 | router = SimpleRouter() 20 | 21 | 22 | app_name = "api" 23 | 24 | 25 | router.register(r"git-repo", GitRepoAPIViewset, basename="git-repo") 26 | router.register(r"git-ssh-key", SSHKeyViewSets, basename="ssh-key") 27 | router.register(r"interval", InterValViewSet, basename="interval") 28 | router.register(r"crontab", CrontabScheduleViewSet, basename="crontab") 29 | router.register(r"periodic-task", AddPeriodicTask, basename="periodic-task") 30 | router.register(r"profile_yaml", PostYMALDetailsView, basename="profile_yaml") 31 | 32 | urlpatterns = [ 33 | path( 34 | "dbt-current-version", 35 | DBTCurrentVersionView.as_view(), 36 | name="dbt-current-version", 37 | ), 38 | path("run-dbt-task", RunDBTTask.as_view(), name="run-dbt-task"), 39 | ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) 40 | 41 | urlpatterns += router.urls 42 | -------------------------------------------------------------------------------- /config/celery_app.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | from django.core.management import call_command 4 | from celery import Celery, shared_task 5 | 6 | # set the default Django settings module for the 'celery' program. 7 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local") 8 | 9 | app = Celery("dbt") 10 | 11 | # Using a string here means the worker doesn't have to serialize 12 | # the configuration object to child processes. 13 | # - namespace='CELERY' means all celery-related configuration keys 14 | # should have a `CELERY_` prefix. 15 | app.config_from_object("django.conf:settings", namespace="CELERY") 16 | 17 | # Load task modules from all registered Django app configs. 18 | app.autodiscover_tasks() 19 | 20 | 21 | @shared_task 22 | @app.task( 23 | bind=True, name="dbt_runner_task") 24 | def dbt_runner_task(self, *args, **kwargs): 25 | option = "--dbt_command={}".format(self.request.args[0]) 26 | option_two = "--pk={}".format(self.request.kwargs) # pk is git repo object id 27 | call_command("dbt_command", option, option_two) 28 | 29 | 30 | @app.task(bind=True) 31 | def dbt_to_db(self): 32 | call_command("dbt_to_db") 33 | -------------------------------------------------------------------------------- /config/settings/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-arsenault/django-build-tool/db5cf4cab23f38709a1909b868f4d1f59166d2f4/config/settings/__init__.py -------------------------------------------------------------------------------- /config/settings/base.py: -------------------------------------------------------------------------------- 1 | """ 2 | Base settings to build other settings files upon. 3 | """ 4 | import os 5 | from pathlib import Path 6 | 7 | import environ 8 | 9 | ROOT_DIR = Path(__file__).resolve(strict=True).parent.parent.parent 10 | # dbt/ 11 | APPS_DIR = ROOT_DIR / "dbt" 12 | env = environ.Env() 13 | 14 | READ_DOT_ENV_FILE = env.bool("DJANGO_READ_DOT_ENV_FILE", default=False) 15 | if READ_DOT_ENV_FILE: 16 | # OS environment variables take precedence over variables from .env 17 | env.read_env(str(ROOT_DIR / ".env")) 18 | 19 | # GENERAL 20 | # ------------------------------------------------------------------------------ 21 | # https://docs.djangoproject.com/en/dev/ref/settings/#debug 22 | DEBUG = env.bool("DJANGO_DEBUG", False) 23 | # Local time zone. Choices are 24 | # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name 25 | # though not all of them may be available with every OS. 26 | # In Windows, this must be set to your system time zone. 27 | TIME_ZONE = "UTC" 28 | # https://docs.djangoproject.com/en/dev/ref/settings/#language-code 29 | LANGUAGE_CODE = "en-us" 30 | # https://docs.djangoproject.com/en/dev/ref/settings/#site-id 31 | SITE_ID = 1 32 | # https://docs.djangoproject.com/en/dev/ref/settings/#use-i18n 33 | USE_I18N = True 34 | # https://docs.djangoproject.com/en/dev/ref/settings/#use-l10n 35 | USE_L10N = True 36 | # https://docs.djangoproject.com/en/dev/ref/settings/#use-tz 37 | USE_TZ = True 38 | # https://docs.djangoproject.com/en/dev/ref/settings/#locale-paths 39 | LOCALE_PATHS = [str(ROOT_DIR / "locale")] 40 | 41 | # DATABASES 42 | # ------------------------------------------------------------------------------ 43 | # https://docs.djangoproject.com/en/dev/ref/settings/#databases 44 | DATABASES = {"default": env.db("DATABASE_URL")} 45 | DATABASES["default"]["ATOMIC_REQUESTS"] = True 46 | # https://docs.djangoproject.com/en/stable/ref/settings/#std:setting-DEFAULT_AUTO_FIELD 47 | DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" 48 | 49 | # URLS 50 | # ------------------------------------------------------------------------------ 51 | # https://docs.djangoproject.com/en/dev/ref/settings/#root-urlconf 52 | ROOT_URLCONF = "config.urls" 53 | # https://docs.djangoproject.com/en/dev/ref/settings/#wsgi-application 54 | WSGI_APPLICATION = "config.wsgi.application" 55 | 56 | # APPS 57 | # ------------------------------------------------------------------------------ 58 | DJANGO_APPS = [ 59 | "django.contrib.auth", 60 | "django.contrib.contenttypes", 61 | "django.contrib.sessions", 62 | "django.contrib.sites", 63 | "django.contrib.messages", 64 | "django.contrib.staticfiles", 65 | "django.contrib.admin", 66 | "django.forms", 67 | ] 68 | THIRD_PARTY_APPS = [ 69 | "django_celery_beat", 70 | "rest_framework", 71 | "rest_framework.authtoken", 72 | "corsheaders", 73 | "drf_spectacular", 74 | ] 75 | 76 | LOCAL_APPS = [ 77 | "dbt.users", 78 | "dbt.analytics", 79 | ] 80 | # https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps 81 | INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS 82 | 83 | # MIGRATIONS 84 | # ------------------------------------------------------------------------------ 85 | # https://docs.djangoproject.com/en/dev/ref/settings/#migration-modules 86 | MIGRATION_MODULES = {"sites": "dbt.contrib.sites.migrations"} 87 | 88 | # AUTHENTICATION 89 | # ------------------------------------------------------------------------------ 90 | # https://docs.djangoproject.com/en/dev/ref/settings/#authentication-backends 91 | AUTHENTICATION_BACKENDS = [ 92 | "django.contrib.auth.backends.ModelBackend", 93 | ] 94 | # https://docs.djangoproject.com/en/dev/ref/settings/#auth-user-model 95 | AUTH_USER_MODEL = "users.User" 96 | # https://docs.djangoproject.com/en/dev/ref/settings/#login-redirect-url 97 | LOGIN_REDIRECT_URL = "users:redirect" 98 | # https://docs.djangoproject.com/en/dev/ref/settings/#login-url 99 | LOGIN_URL = "account_login" 100 | 101 | # PASSWORDS 102 | # ------------------------------------------------------------------------------ 103 | # https://docs.djangoproject.com/en/dev/ref/settings/#password-hashers 104 | PASSWORD_HASHERS = [ 105 | "django.contrib.auth.hashers.Argon2PasswordHasher", 106 | "django.contrib.auth.hashers.PBKDF2PasswordHasher", 107 | "django.contrib.auth.hashers.PBKDF2SHA1PasswordHasher", 108 | "django.contrib.auth.hashers.BCryptSHA256PasswordHasher", 109 | ] 110 | # https://docs.djangoproject.com/en/dev/ref/settings/#auth-password-validators 111 | AUTH_PASSWORD_VALIDATORS = [ 112 | { 113 | "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator" 114 | }, 115 | {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"}, 116 | {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"}, 117 | {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"}, 118 | ] 119 | 120 | # MIDDLEWARE 121 | # ------------------------------------------------------------------------------ 122 | # https://docs.djangoproject.com/en/dev/ref/settings/#middleware 123 | MIDDLEWARE = [ 124 | "django.middleware.security.SecurityMiddleware", 125 | "corsheaders.middleware.CorsMiddleware", 126 | "whitenoise.middleware.WhiteNoiseMiddleware", 127 | "django.contrib.sessions.middleware.SessionMiddleware", 128 | "django.middleware.locale.LocaleMiddleware", 129 | "django.middleware.common.CommonMiddleware", 130 | "django.contrib.auth.middleware.AuthenticationMiddleware", 131 | "django.contrib.messages.middleware.MessageMiddleware", 132 | "django.middleware.common.BrokenLinkEmailsMiddleware", 133 | "django.middleware.clickjacking.XFrameOptionsMiddleware", 134 | ] 135 | 136 | # STATIC 137 | # ------------------------------------------------------------------------------ 138 | # https://docs.djangoproject.com/en/dev/ref/settings/#static-root 139 | STATIC_ROOT = str(ROOT_DIR / "staticfiles") 140 | # https://docs.djangoproject.com/en/dev/ref/settings/#static-url 141 | STATIC_URL = "/static/" 142 | # https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#std:setting-STATICFILES_DIRS 143 | STATICFILES_DIRS = [str(APPS_DIR / "static")] 144 | # https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#staticfiles-finders 145 | STATICFILES_FINDERS = [ 146 | "django.contrib.staticfiles.finders.FileSystemFinder", 147 | "django.contrib.staticfiles.finders.AppDirectoriesFinder", 148 | ] 149 | 150 | # MEDIA 151 | # ------------------------------------------------------------------------------ 152 | # https://docs.djangoproject.com/en/dev/ref/settings/#media-root 153 | MEDIA_ROOT = str(APPS_DIR / "media") 154 | # https://docs.djangoproject.com/en/dev/ref/settings/#media-url 155 | MEDIA_URL = "/media/" 156 | 157 | # TEMPLATES 158 | # ------------------------------------------------------------------------------ 159 | # https://docs.djangoproject.com/en/dev/ref/settings/#templates 160 | TEMPLATES = [ 161 | { 162 | # https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-TEMPLATES-BACKEND 163 | "BACKEND": "django.template.backends.django.DjangoTemplates", 164 | # https://docs.djangoproject.com/en/dev/ref/settings/#dirs 165 | "DIRS": [str(APPS_DIR / "templates")], 166 | # https://docs.djangoproject.com/en/dev/ref/settings/#app-dirs 167 | "APP_DIRS": True, 168 | "OPTIONS": { 169 | # https://docs.djangoproject.com/en/dev/ref/settings/#template-context-processors 170 | "context_processors": [ 171 | "django.template.context_processors.debug", 172 | "django.template.context_processors.request", 173 | "django.contrib.auth.context_processors.auth", 174 | "django.template.context_processors.i18n", 175 | "django.template.context_processors.media", 176 | "django.template.context_processors.static", 177 | "django.template.context_processors.tz", 178 | "django.contrib.messages.context_processors.messages", 179 | ], 180 | }, 181 | } 182 | ] 183 | 184 | # https://docs.djangoproject.com/en/dev/ref/settings/#form-renderer 185 | FORM_RENDERER = "django.forms.renderers.TemplatesSetting" 186 | 187 | # FIXTURES 188 | # ------------------------------------------------------------------------------ 189 | # https://docs.djangoproject.com/en/dev/ref/settings/#fixture-dirs 190 | FIXTURE_DIRS = (str(APPS_DIR / "fixtures"),) 191 | 192 | # SECURITY 193 | # ------------------------------------------------------------------------------ 194 | # https://docs.djangoproject.com/en/dev/ref/settings/#session-cookie-httponly 195 | SESSION_COOKIE_HTTPONLY = True 196 | # https://docs.djangoproject.com/en/dev/ref/settings/#csrf-cookie-httponly 197 | CSRF_COOKIE_HTTPONLY = True 198 | # https://docs.djangoproject.com/en/dev/ref/settings/#secure-browser-xss-filter 199 | SECURE_BROWSER_XSS_FILTER = True 200 | # https://docs.djangoproject.com/en/dev/ref/settings/#x-frame-options 201 | X_FRAME_OPTIONS = "DENY" 202 | 203 | # EMAIL 204 | # ------------------------------------------------------------------------------ 205 | # https://docs.djangoproject.com/en/dev/ref/settings/#email-backend 206 | EMAIL_BACKEND = env( 207 | "DJANGO_EMAIL_BACKEND", 208 | default="django.core.mail.backends.smtp.EmailBackend", 209 | ) 210 | # https://docs.djangoproject.com/en/dev/ref/settings/#email-timeout 211 | EMAIL_TIMEOUT = 5 212 | 213 | # ADMIN 214 | # ------------------------------------------------------------------------------ 215 | # Django Admin URL. 216 | ADMIN_URL = env("DJANGO_ADMIN_URL", default="") 217 | # https://docs.djangoproject.com/en/dev/ref/settings/#admins 218 | ADMINS = [("""noname""", "noname@example.com")] 219 | # https://docs.djangoproject.com/en/dev/ref/settings/#managers 220 | MANAGERS = ADMINS 221 | 222 | # LOGGING 223 | # ------------------------------------------------------------------------------ 224 | # https://docs.djangoproject.com/en/dev/ref/settings/#logging 225 | # See https://docs.djangoproject.com/en/dev/topics/logging for 226 | # more details on how to customize your logging configuration. 227 | LOGGING = { 228 | "version": 1, 229 | "disable_existing_loggers": False, 230 | "formatters": { 231 | "verbose": { 232 | "format": "%(levelname)s %(asctime)s %(module)s " 233 | "%(process)d %(thread)d %(message)s" 234 | } 235 | }, 236 | "handlers": { 237 | "console": { 238 | "level": "DEBUG", 239 | "class": "logging.StreamHandler", 240 | "formatter": "verbose", 241 | } 242 | }, 243 | "root": {"level": "INFO", "handlers": ["console"]}, 244 | } 245 | 246 | # Celery 247 | # ------------------------------------------------------------------------------ 248 | if USE_TZ: 249 | # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-timezone 250 | CELERY_TIMEZONE = TIME_ZONE 251 | # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-broker_url 252 | CELERY_BROKER_URL = os.getenv("REDIS_URL", "redis://redis:6379/1") # default is running on docker 253 | # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-result_backend 254 | CELERY_RESULT_BACKEND = CELERY_BROKER_URL 255 | # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-accept_content 256 | CELERY_ACCEPT_CONTENT = ["json"] 257 | # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-task_serializer 258 | CELERY_TASK_SERIALIZER = "json" 259 | # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-result_serializer 260 | CELERY_RESULT_SERIALIZER = "json" 261 | # https://docs.celeryq.dev/en/stable/userguide/configuration.html#task-time-limit 262 | # TODO: set to whatever value is adequate in your circumstances 263 | CELERY_TASK_TIME_LIMIT = 43200 264 | # https://docs.celeryq.dev/en/stable/userguide/configuration.html#task-soft-time-limit 265 | # TODO: set to whatever value is adequate in your circumstances 266 | CELERY_TASK_SOFT_TIME_LIMIT = 43200 267 | # https://docs.celeryq.dev/en/stable/userguide/configuration.html#beat-scheduler 268 | CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler" 269 | 270 | # django-rest-framework 271 | # ------------------------------------------------------------------------------- 272 | # django-rest-framework - https://www.django-rest-framework.org/api-guide/settings/ 273 | REST_FRAMEWORK = { 274 | "DEFAULT_AUTHENTICATION_CLASSES": ( 275 | # "rest_framework.authentication.SessionAuthentication", 276 | "rest_framework.authentication.TokenAuthentication", 277 | ), 278 | "DEFAULT_PERMISSION_CLASSES": ("rest_framework.permissions.IsAuthenticated",), 279 | "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema", 280 | 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', 281 | 'PAGE_SIZE': 20, 282 | } 283 | 284 | # django-cors-headers - https://github.com/adamchainz/django-cors-headers#setup 285 | CORS_URLS_REGEX = r"^/api/.*$" 286 | # SECURITY WARNING: don't run with allowing all origin in production! 287 | CORS_ORIGIN_ALLOW_ALL = env.bool("CORS_ORIGIN_ALLOW_ALL", default=True) 288 | 289 | ALLOWED_HOSTS = env.list("ALLOWED_HOSTS", default=["*"]) 290 | 291 | CORS_ALLOWED_ORIGINS = env.list("CORS_ALLOWED_ORIGINS", default=[ 292 | 'http://localhost:8000', 'http://127.0.0.1:8000', 293 | ]) 294 | # # By Default swagger ui is available only to admin user(s). You can change permission classes to change that 295 | # See more configuration options at https://drf-spectacular.readthedocs.io/en/latest/settings.html#settings 296 | SPECTACULAR_SETTINGS = { 297 | "TITLE": "DBT Analytics API", 298 | "DESCRIPTION": "Documentation of API endpoints of DBT Analytics", 299 | "VERSION": "1.0.0", 300 | } 301 | 302 | EXTERNAL_REPO_PREFIX = 'external' 303 | THIS_PROJECT_PATH = '/root/.dbt' 304 | SSH_KEY_PREFIX = "git-django_" 305 | -------------------------------------------------------------------------------- /config/settings/local.py: -------------------------------------------------------------------------------- 1 | from .base import * # noqa 2 | from .base import env 3 | import os 4 | 5 | 6 | # GENERAL 7 | # ------------------------------------------------------------------------------ 8 | # https://docs.djangoproject.com/en/dev/ref/settings/#debug 9 | DEBUG = False 10 | # https://docs.djangoproject.com/en/dev/ref/settings/#secret-key 11 | SECRET_KEY = env( 12 | "DJANGO_SECRET_KEY", 13 | default="PkvhlRoJL7tLsVA9c1xNbS8V3fWcU4G5gcZNGC8VNHvTvC03fHCUYOsRy5GCkWgW", 14 | ) 15 | 16 | # CACHES 17 | # ------------------------------------------------------------------------------ 18 | CACHES = { 19 | "default": { 20 | "BACKEND": "django_redis.cache.RedisCache", 21 | "LOCATION": env("REDIS_URL"), 22 | "OPTIONS": { 23 | "CLIENT_CLASS": "django_redis.client.DefaultClient", 24 | # Mimicing memcache behavior. 25 | # https://github.com/jazzband/django-redis#memcached-exceptions-behavior 26 | "IGNORE_EXCEPTIONS": True, 27 | }, 28 | } 29 | } 30 | 31 | 32 | CORS_ORIGIN_WHITELIST = [ 33 | 'https://localhost:8000','http://localhost:8000', 34 | ] 35 | # EMAIL 36 | # ------------------------------------------------------------------------------ 37 | # https://docs.djangoproject.com/en/dev/ref/settings/#email-backend 38 | EMAIL_BACKEND = env( 39 | "DJANGO_EMAIL_BACKEND", default="django.core.mail.backends.console.EmailBackend" 40 | ) 41 | 42 | 43 | # STATIC 44 | # ------------------------ 45 | STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage" 46 | # https://django-debug-toolbar.readthedocs.io/en/latest/installation.html#internal-ips 47 | 48 | INTERNAL_IPS = ["127.0.0.1", "10.0.2.2"] 49 | if env("USE_DOCKER") == "yes": 50 | import socket 51 | 52 | hostname, _, ips = socket.gethostbyname_ex(socket.gethostname()) 53 | INTERNAL_IPS += [".".join(ip.split(".")[:-1] + ["1"]) for ip in ips] 54 | 55 | # ------------------------------------------------------------------------------ 56 | # https://django-extensions.readthedocs.io/en/latest/installation_instructions.html#configuration 57 | INSTALLED_APPS += ["django_extensions"] # noqa F405 58 | 59 | # Celery 60 | # ------------------------------------------------------------------------------ 61 | # https://docs.celeryq.dev/en/stable/userguide/configuration.html#task-eager-propagates 62 | CELERY_TASK_EAGER_PROPAGATES = True 63 | 64 | LOGGING = { 65 | "version": 1, 66 | "disable_existing_loggers": False, 67 | "filters": {"require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}}, 68 | "formatters": { 69 | "verbose": { 70 | "format": "%(levelname)s %(asctime)s %(module)s " 71 | "%(process)d %(thread)d %(message)s" 72 | } 73 | }, 74 | "handlers": { 75 | "mail_admins": { 76 | "level": "ERROR", 77 | "filters": ["require_debug_false"], 78 | "class": "django.utils.log.AdminEmailHandler", 79 | }, 80 | "console": { 81 | "level": "DEBUG", 82 | "class": "logging.StreamHandler", 83 | "formatter": "verbose", 84 | }, 85 | }, 86 | "root": {"level": "INFO", "handlers": ["console"]}, 87 | "loggers": { 88 | "django.request": { 89 | "handlers": ["mail_admins"], 90 | "level": "ERROR", 91 | "propagate": True, 92 | }, 93 | "django.security.DisallowedHost": { 94 | "level": "ERROR", 95 | "handlers": ["console", "mail_admins"], 96 | "propagate": True, 97 | }, 98 | }, 99 | } -------------------------------------------------------------------------------- /config/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.conf.urls.static import static 3 | from django.urls import include, path 4 | from django.views.generic import TemplateView 5 | from django.views import defaults as default_views 6 | from drf_spectacular.views import ( 7 | SpectacularAPIView, 8 | SpectacularSwaggerView, 9 | SpectacularRedocView, 10 | ) 11 | from django.contrib import admin 12 | 13 | 14 | admin.site.index_title = "Features area" 15 | 16 | urlpatterns = [ 17 | 18 | path("api/", include("config.api_router")), 19 | path("api/schema/", SpectacularAPIView.as_view(), name="api-schema"), 20 | path( 21 | "api/docs/", 22 | SpectacularSwaggerView.as_view(url_name="api-schema"), 23 | name="api-docs", 24 | ), 25 | path( 26 | "api/redoc/", 27 | SpectacularRedocView.as_view(url_name="api-schema"), 28 | name="api-redoc", 29 | ), 30 | path("", admin.site.urls), 31 | ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) 32 | 33 | if settings.DEBUG: 34 | urlpatterns += [ 35 | path( 36 | "400/", 37 | default_views.bad_request, 38 | kwargs={"exception": Exception("Bad Request!")}, 39 | ), 40 | path( 41 | "403/", 42 | default_views.permission_denied, 43 | kwargs={"exception": Exception("Permission Denied")}, 44 | ), 45 | path( 46 | "404/", 47 | default_views.page_not_found, 48 | kwargs={"exception": Exception("Page not Found")}, 49 | ), 50 | path("500/", default_views.server_error), 51 | ] 52 | if "debug_toolbar" in settings.INSTALLED_APPS: 53 | import debug_toolbar 54 | 55 | urlpatterns = [path("__debug__/", include(debug_toolbar.urls))] + urlpatterns 56 | -------------------------------------------------------------------------------- /config/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for DBT Analytics project. 3 | 4 | This module contains the WSGI application used by Django's development server 5 | and any production WSGI deployments. It should expose a module-level variable 6 | named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover 7 | this application via the ``WSGI_APPLICATION`` setting. 8 | 9 | Usually you will have the standard Django WSGI application here, but it also 10 | might make sense to replace the whole Django WSGI application with a custom one 11 | that later delegates to the Django one. For example, you could introduce WSGI 12 | middleware here, or combine a Django application with an application of another 13 | framework. 14 | 15 | """ 16 | import os 17 | import sys 18 | from pathlib import Path 19 | 20 | from django.core.wsgi import get_wsgi_application 21 | 22 | # This allows easy placement of apps within the interior 23 | # dbt directory. 24 | ROOT_DIR = Path(__file__).resolve(strict=True).parent.parent 25 | sys.path.append(str(ROOT_DIR / "dbt")) 26 | # We defer to a DJANGO_SETTINGS_MODULE already in the environment. This breaks 27 | # if running multiple sites in the same mod_wsgi process. To fix this, use 28 | # mod_wsgi daemon mode with each site in its own daemon process, or use 29 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local") 30 | 31 | # This application object is used by any WSGI server configured to use this 32 | # file. This includes Django's development server, if the WSGI_APPLICATION 33 | # setting points here. 34 | application = get_wsgi_application() 35 | # Apply WSGI middleware here. 36 | # from helloworld.wsgi import HelloWorldApplication 37 | # application = HelloWorldApplication(application) 38 | -------------------------------------------------------------------------------- /dbt/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.1.0" 2 | __version_info__ = tuple( 3 | int(num) if num.isdigit() else num 4 | for num in __version__.replace("-", ".", 1).split(".") 5 | ) 6 | -------------------------------------------------------------------------------- /dbt/analytics/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-arsenault/django-build-tool/db5cf4cab23f38709a1909b868f4d1f59166d2f4/dbt/analytics/__init__.py -------------------------------------------------------------------------------- /dbt/analytics/admin.py: -------------------------------------------------------------------------------- 1 | from django import forms 2 | from django.contrib import admin, messages 3 | from django.forms import ModelForm, PasswordInput 4 | from celery import current_app 5 | from django.contrib.auth.models import Group 6 | 7 | from django.contrib.sites.models import Site 8 | from django.forms.widgets import Select 9 | from celery.utils import cached_property 10 | from django_celery_beat.admin import ( 11 | PeriodicTaskAdmin as BasePeriodicTaskAdmin, 12 | PeriodicTaskForm as BasePeriodicTaskForm, 13 | ) 14 | 15 | from dbt.analytics.models import ( 16 | DBTLogs, 17 | GitRepo, 18 | ProfileYAML, 19 | SubProcessLog, 20 | PeriodicTask, 21 | ) 22 | from dbt.utils.common import clone_git_repo 23 | 24 | 25 | class GitRepoForm(ModelForm): 26 | url = forms.CharField(widget=PasswordInput()) 27 | 28 | class Meta: 29 | model = GitRepo 30 | fields = "__all__" 31 | 32 | 33 | @admin.register(DBTLogs) 34 | class DBTLogsLAdmin(admin.ModelAdmin): 35 | list_display = [ 36 | "created_at", 37 | "completed_at", 38 | "success", 39 | "repository_used_name", 40 | "command", 41 | "previous_command", 42 | "periodic_task_name", 43 | "profile_yml_used_name", 44 | ] 45 | readonly_fields = [ 46 | "repository_used_name", 47 | "periodic_task_name", 48 | "profile_yml_used_name", 49 | ] 50 | 51 | 52 | @admin.register(ProfileYAML) 53 | class ProfileYAMLAdmin(admin.ModelAdmin): 54 | list_display = [ 55 | "name", 56 | "profile_yml", 57 | ] 58 | 59 | def has_add_permission(self, request): 60 | count = ProfileYAML.objects.all().count() 61 | if count < 2: 62 | return True 63 | return False 64 | 65 | def has_delete_permission(self, request, obj=None): 66 | return False 67 | 68 | 69 | @admin.register(GitRepo) 70 | class GitRepoAdmin(admin.ModelAdmin): 71 | # form = GitRepoForm 72 | list_display = [ 73 | "id", 74 | "name", 75 | "public_key", 76 | ] 77 | 78 | def save_model(self, request, obj, form, change): 79 | obj.save() 80 | result, msg = clone_git_repo(obj) 81 | if result: 82 | ... 83 | else: 84 | obj.delete() 85 | messages.error(request, f"Something is wrong while git cloning {msg}") 86 | 87 | 88 | @admin.register(SubProcessLog) 89 | class SubprocessAdmin(admin.ModelAdmin): 90 | list_display = [ 91 | "created_at", 92 | "details", 93 | ] 94 | 95 | 96 | class ProfileSelectWidget(Select): 97 | """Widget that lets you choose between task names.""" 98 | 99 | celery_app = current_app 100 | _choices = None 101 | 102 | def profiles_as_choices(self): 103 | _ = self._modules # noqa 104 | tasks = list( 105 | sorted( 106 | name for name in self.celery_app.tasks if not name.startswith("celery.") 107 | ) 108 | ) 109 | return (("", ""),) + tuple(zip(tasks, tasks)) 110 | 111 | @property 112 | def choices(self): 113 | if self._choices is None: 114 | self._choices = self.profiles_as_choices() 115 | return self._choices 116 | 117 | @choices.setter 118 | def choices(self, _): 119 | pass 120 | 121 | @cached_property 122 | def _modules(self): 123 | self.celery_app.loader.import_default_modules() 124 | 125 | 126 | class ProfileChoiceField(forms.ChoiceField): 127 | widget = ProfileSelectWidget 128 | 129 | def valid_value(self, value): 130 | return True 131 | 132 | 133 | class PeriodicTaskForm(BasePeriodicTaskForm): 134 | profile_yml = ProfileChoiceField( 135 | label="Profile YAML", 136 | required=False, 137 | ) 138 | 139 | class Meta: 140 | model = PeriodicTask 141 | exclude = () 142 | 143 | 144 | class PeriodicTaskAdmin(BasePeriodicTaskAdmin): 145 | # form = PeriodicTaskForm 146 | model = PeriodicTask 147 | list_display = ('__str__', 'id', 'enabled', 'interval', 'start_time', 148 | 'last_run_at', 'one_off') 149 | fieldsets = ( 150 | ( 151 | None, 152 | { 153 | "fields": ( 154 | "name", 155 | "git_repo", 156 | "profile_yml", 157 | "regtask", 158 | "task", 159 | "enabled", 160 | "description", 161 | ), 162 | "classes": ("extrapretty", "wide"), 163 | }, 164 | ), 165 | ( 166 | "Schedule", 167 | { 168 | "fields": ( 169 | "interval", 170 | "crontab", 171 | "solar", 172 | "clocked", 173 | "start_time", 174 | "last_run_at", 175 | "one_off", 176 | ), 177 | "classes": ("extrapretty", "wide"), 178 | }, 179 | ), 180 | ( 181 | "Arguments", 182 | { 183 | "fields": ("args",), 184 | "classes": ("extrapretty", "wide", "collapse", "in"), 185 | }, 186 | ), 187 | ( 188 | "Execution Options", 189 | { 190 | "fields": ( 191 | "expires", 192 | "expire_seconds", 193 | "queue", 194 | "exchange", 195 | "routing_key", 196 | "priority", 197 | "headers", 198 | ), 199 | "classes": ("extrapretty", "wide", "collapse", "in"), 200 | }, 201 | ), 202 | ) 203 | 204 | 205 | # 206 | 207 | if PeriodicTask in admin.site._registry: 208 | admin.site.unregister(PeriodicTask) 209 | admin.site.register(PeriodicTask, PeriodicTaskAdmin) 210 | 211 | admin.site.unregister(Group) 212 | admin.site.unregister(Site) 213 | -------------------------------------------------------------------------------- /dbt/analytics/celery.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | import os 4 | 5 | from celery import Celery 6 | from django.core.management import call_command 7 | 8 | # set the default Django settings module for the 'celery' program. 9 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings") 10 | 11 | app = Celery("analytics") 12 | 13 | 14 | app.config_from_object("django.conf:settings", namespace="CELERY") 15 | 16 | 17 | # Load task modules from all registered Django app configs. 18 | app.autodiscover_tasks() 19 | -------------------------------------------------------------------------------- /dbt/analytics/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-arsenault/django-build-tool/db5cf4cab23f38709a1909b868f4d1f59166d2f4/dbt/analytics/migrations/__init__.py -------------------------------------------------------------------------------- /dbt/analytics/models.py: -------------------------------------------------------------------------------- 1 | import os 2 | from django_celery_beat.models import PeriodicTasks 3 | from django.forms import ValidationError 4 | from django.db import models 5 | from django.conf import settings 6 | from django.db.models import ( 7 | Model, 8 | SET_NULL, 9 | CASCADE, 10 | SET_DEFAULT, 11 | CharField, 12 | TextField, 13 | BooleanField, 14 | DateTimeField, 15 | ForeignKey, 16 | JSONField, 17 | OneToOneField, 18 | BigAutoField, 19 | ) 20 | from django_celery_beat.models import PeriodicTask as BasePeriodicTaskModel 21 | from dbt.utils.common import save_profile_yml 22 | from django.db.models.query import QuerySet 23 | 24 | PROFILE_NAME_DEV = "DEV" 25 | PROFILE_NAME_PROD = "PROD" 26 | PROFILE_NAME_DEFAULT = PROFILE_NAME_DEV 27 | PROFILE_NAME_CHOICES = [ 28 | (PROFILE_NAME_DEV, "DEV"), 29 | (PROFILE_NAME_PROD, "PROD"), 30 | ] 31 | SSH_KEY_PREFIX = getattr(settings, "SSH_KEY_PREFIX") 32 | 33 | 34 | class ExtendedQuerySet(QuerySet): 35 | """Base class for query sets.""" 36 | 37 | def update_or_create(self, defaults=None, **kwargs): 38 | obj, created = self.get_or_create(defaults=defaults, **kwargs) 39 | if not created: 40 | self._update_model_with_dict(obj, dict(defaults or {}, **kwargs)) 41 | return obj 42 | 43 | def _update_model_with_dict(self, obj, fields): 44 | [ 45 | setattr(obj, attr_name, attr_value) 46 | for attr_name, attr_value in fields.items() 47 | ] 48 | obj.save() 49 | return obj 50 | 51 | 52 | class ExtendedManager(models.Manager.from_queryset(ExtendedQuerySet)): 53 | """Manager with common utilities.""" 54 | 55 | 56 | class PeriodicTaskManager(ExtendedManager): 57 | """Manager for PeriodicTask model.""" 58 | 59 | def enabled(self): 60 | return self.filter(enabled=True) 61 | 62 | 63 | class PeriodicTask(BasePeriodicTaskModel): 64 | git_repo = ForeignKey( 65 | "GitRepo", 66 | on_delete=SET_NULL, 67 | related_name="periodic_task_git_repo", 68 | null=True, 69 | blank=True, 70 | ) 71 | profile_yml = ForeignKey( 72 | "ProfileYAML", 73 | on_delete=SET_NULL, 74 | related_name="periodic_task_profile_yml", 75 | null=True, 76 | blank=True, 77 | ) 78 | 79 | def save(self, *args, **kwargs): 80 | # # replace ' to " in args 81 | self.args = self.args.replace("'", '"') 82 | self.kwargs = f'{{"task_id":{self.id}}}' 83 | super( 84 | PeriodicTask, 85 | self, 86 | ).save(*args, **kwargs) 87 | PeriodicTasks.update_changed() 88 | 89 | objects = PeriodicTaskManager() 90 | no_changes = False 91 | 92 | 93 | class ProfileYAML(Model): 94 | profile_yml = TextField(max_length=4000) 95 | name = CharField( 96 | max_length=255, 97 | choices=PROFILE_NAME_CHOICES, 98 | default=PROFILE_NAME_DEFAULT, 99 | unique=True, 100 | verbose_name="Profile Name", 101 | null=True, 102 | blank=True, 103 | ) 104 | 105 | def save(self, *args, **kwargs): 106 | save_profile_yml(self.profile_yml, ".dbt/profiles.yml") 107 | super(ProfileYAML, self).save(*args, **kwargs) 108 | 109 | class Meta: 110 | verbose_name = "Profile YAML" 111 | verbose_name_plural = "Profile YAMLs" 112 | 113 | def __str__(self): 114 | return str(self.name) 115 | 116 | 117 | class SSHKey(Model): 118 | name = CharField(max_length=255) 119 | 120 | def __str__(self): 121 | return self.name 122 | 123 | def public_key(self): 124 | pub_key_path = os.path.join( 125 | os.getenv("HOME"), ".ssh/{}{}.pub".format(SSH_KEY_PREFIX, self.id) 126 | ) 127 | 128 | pub_key = "" 129 | with open(pub_key_path, "r") as pub_key_file: 130 | pub_key = pub_key_file.readline() 131 | return pub_key 132 | 133 | class Meta: 134 | verbose_name = "SSH Key" 135 | verbose_name_plural = "SSH Keys" 136 | 137 | 138 | class GitRepo(Model): 139 | name = CharField(max_length=255, blank=True, null=True) 140 | url = CharField(help_text="add with personal token", max_length=600) 141 | ssh_key = OneToOneField(SSHKey, blank=True, null=True, on_delete=CASCADE) 142 | 143 | def public_key(self): 144 | if self.ssh_key: 145 | pub_key_path = os.path.join( 146 | os.getenv("HOME"), 147 | ".ssh/{}{}.pub".format(SSH_KEY_PREFIX, self.ssh_key.id), 148 | ) 149 | 150 | pub_key = "" 151 | with open(pub_key_path, "r") as pub_key_file: 152 | pub_key = pub_key_file.readline() 153 | return pub_key 154 | else: 155 | return "Public key not found" 156 | 157 | def clean(self): 158 | if self.url.startswith("git"): 159 | if not self.ssh_key: 160 | raise ValidationError( 161 | "You must choose a ssh key while you are adding repo of ssh key" 162 | ) 163 | 164 | if self.url.startswith("http"): 165 | if "ghp" not in self.url: 166 | raise ValidationError("Your git repo must be come with personal token") 167 | 168 | def save(self, *args, **kwargs): 169 | self.clean() 170 | super(GitRepo, self).save(*args, **kwargs) 171 | 172 | class Meta: 173 | verbose_name = "Git Repo" 174 | verbose_name_plural = "Git Repos" 175 | 176 | def __str__(self): 177 | return self.name 178 | 179 | 180 | class SubProcessLog(Model): 181 | details = TextField(max_length=1000) 182 | created_at = DateTimeField(auto_now_add=True) 183 | 184 | class Meta: 185 | verbose_name = "Sub Process Log" 186 | verbose_name_plural = "SubProcessLogs" 187 | 188 | 189 | class DBTLogs(Model): 190 | manifest = JSONField(null=True, blank=True) 191 | run_results = JSONField(null=True, blank=True) 192 | sources = JSONField(null=True, blank=True) 193 | catalog = JSONField(null=True, blank=True) 194 | command = CharField(max_length=255, null=True, blank=True) 195 | previous_command = CharField(max_length=255, null=True, blank=True) 196 | success = BooleanField(default=True) 197 | fail_reason = TextField(null=True, blank=True, max_length=10000) 198 | repository_used_name = CharField(max_length=255, null=True, blank=True) 199 | created_at = DateTimeField(auto_now_add=True) 200 | completed_at = DateTimeField(null=True, blank=True) 201 | periodic_task_name = CharField(max_length=255, null=True, blank=True) 202 | profile_yml_used_name = CharField(max_length=255, null=True, blank=True) 203 | dbt_stdout = TextField(null=True, blank=True, ) 204 | 205 | class Meta: 206 | verbose_name = "DBT Log" 207 | verbose_name_plural = "DBT Logs" 208 | 209 | def __str__(self): 210 | return str(self.created_at) 211 | 212 | 213 | class Args(Model): 214 | alias = BigAutoField(primary_key=True, unique=True) 215 | dbt_log = ForeignKey(DBTLogs, on_delete=CASCADE, null=True, blank=True) 216 | quiet = CharField(max_length=255, null=True, blank=True) 217 | which = CharField(max_length=255, null=True, blank=True) 218 | no_print = CharField(max_length=255, null=True, blank=True) 219 | rpc_method = CharField(max_length=255, null=True, blank=True) 220 | use_colors = CharField(max_length=255, null=True, blank=True) 221 | write_json = CharField(max_length=255, null=True, blank=True) 222 | profiles_dir = CharField(max_length=255, null=True, blank=True) 223 | partial_parse = CharField(max_length=255, null=True, blank=True) 224 | printer_width = CharField(max_length=255, null=True, blank=True) 225 | static_parser = CharField(max_length=255, null=True, blank=True) 226 | version_check = CharField(max_length=255, null=True, blank=True) 227 | event_buffer_size = CharField(max_length=255, null=True, blank=True) 228 | indirect_selection = CharField(max_length=255, null=True, blank=True) 229 | send_anonymous = CharField(max_length=255, null=True, blank=True) 230 | usage_stats = CharField(max_length=255, null=True, blank=True) 231 | 232 | class Meta: 233 | verbose_name = "Arg" 234 | verbose_name_plural = "Args" 235 | 236 | def __str__(self): 237 | return str(self.alias) 238 | 239 | -------------------------------------------------------------------------------- /dbt/analytics/serializers.py: -------------------------------------------------------------------------------- 1 | from django_celery_beat.models import IntervalSchedule, CrontabSchedule 2 | from rest_framework import serializers 3 | from timezone_field.rest_framework import TimeZoneSerializerField 4 | from rest_framework.exceptions import ValidationError 5 | from dbt.analytics.models import ( 6 | GitRepo, 7 | ProfileYAML, 8 | SSHKey, 9 | PeriodicTask as PeriodicTaskModel, 10 | ) 11 | from dbt.utils.common import clone_git_repo 12 | 13 | 14 | class GitRepoSerializer(serializers.ModelSerializer): 15 | class Meta: 16 | model = GitRepo 17 | fields = "__all__" 18 | 19 | def create(self, validated_data): 20 | repo = GitRepo.objects.create(**validated_data) 21 | result, msg = clone_git_repo(repo) 22 | if result: 23 | return repo 24 | else: 25 | # delete the repo from db 26 | repo.delete() 27 | raise ValidationError(detail=f"Error creating repo: {msg}") 28 | 29 | def update(self, instance, validated_data): 30 | result, msg = clone_git_repo(validated_data) 31 | if result: 32 | instance = super().update(instance, validated_data) 33 | clone_git_repo(instance) 34 | return instance 35 | else: 36 | raise ValidationError(detail=f"{msg}") 37 | 38 | 39 | class ProfileYAMLSerializer(serializers.ModelSerializer): 40 | class Meta: 41 | model = ProfileYAML 42 | fields = "__all__" 43 | 44 | 45 | class SSHKeySerializer(serializers.ModelSerializer): 46 | class Meta: 47 | model = SSHKey 48 | fields = "__all__" 49 | 50 | 51 | class IntervalScheduleSerializer(serializers.ModelSerializer): 52 | class Meta: 53 | model = IntervalSchedule 54 | fields = "__all__" 55 | 56 | 57 | class PeriodicTaskSerializer(serializers.ModelSerializer): 58 | class Meta: 59 | model = PeriodicTaskModel 60 | fields = "__all__" 61 | 62 | 63 | class CrontabScheduleSerializer(serializers.ModelSerializer): 64 | timezone = TimeZoneSerializerField(use_pytz=False) 65 | 66 | class Meta: 67 | model = CrontabSchedule 68 | fields = "__all__" 69 | 70 | 71 | class WritePeriodicTaskSerializer(serializers.ModelSerializer): 72 | class Meta: 73 | model = PeriodicTaskModel 74 | fields = ( 75 | "name", 76 | "enabled", 77 | "task", 78 | "description", 79 | "start_time", 80 | "crontab", 81 | "one_off", 82 | "args", 83 | "git_repo", 84 | "profile_yml", 85 | ) 86 | 87 | 88 | class DBTCurrentVersionSerializer(serializers.Serializer): 89 | module_name = serializers.CharField() 90 | version = serializers.CharField(allow_null=True) 91 | 92 | 93 | class RunTaskSerializer(serializers.Serializer): 94 | task_id = serializers.IntegerField() 95 | 96 | def validate_task_id(self, value): 97 | if not PeriodicTaskModel.objects.filter(id=value).exists(): 98 | raise serializers.ValidationError('Task with this ID does not exist.') 99 | return value 100 | 101 | def to_representation(self, instance): 102 | ret = super().to_representation(instance) 103 | task = PeriodicTaskModel.objects.get(id=instance['task_id']) 104 | ret['args'] = task.args 105 | return ret 106 | 107 | -------------------------------------------------------------------------------- /dbt/analytics/views.py: -------------------------------------------------------------------------------- 1 | from django_celery_beat.models import IntervalSchedule, CrontabSchedule 2 | from rest_framework.viewsets import ModelViewSet 3 | from rest_framework.views import APIView 4 | from rest_framework.response import Response 5 | from rest_framework import status 6 | from dbt.utils.common import load_dbt_current_version 7 | from config.celery_app import dbt_runner_task 8 | from dbt.analytics.models import ( 9 | GitRepo, 10 | ProfileYAML, 11 | SSHKey, 12 | PeriodicTask as PeriodicTaskModel, 13 | ) 14 | from dbt.analytics.serializers import ( 15 | GitRepoSerializer, 16 | IntervalScheduleSerializer, 17 | PeriodicTaskSerializer, 18 | ProfileYAMLSerializer, 19 | SSHKeySerializer, 20 | WritePeriodicTaskSerializer, 21 | CrontabScheduleSerializer, 22 | DBTCurrentVersionSerializer, 23 | RunTaskSerializer, 24 | ) 25 | 26 | 27 | class GitRepoAPIViewset(ModelViewSet): 28 | http_method_names = ["get", "post", "delete", "head", "options", "trace"] 29 | queryset = GitRepo.objects.all() 30 | serializer_class = GitRepoSerializer 31 | 32 | 33 | class PostYMALDetailsView(ModelViewSet): 34 | serializer_class = ProfileYAMLSerializer 35 | queryset = ProfileYAML.objects.all() 36 | 37 | 38 | class SSHKeyViewSets(ModelViewSet): 39 | http_method_names = ["get", "post", "delete", "head", "options", "trace"] 40 | queryset = SSHKey.objects.all() 41 | serializer_class = SSHKeySerializer 42 | 43 | 44 | class InterValViewSet(ModelViewSet): 45 | queryset = IntervalSchedule.objects.all() 46 | serializer_class = IntervalScheduleSerializer 47 | 48 | 49 | class CrontabScheduleViewSet(ModelViewSet): 50 | queryset = CrontabSchedule.objects.all() 51 | serializer_class = CrontabScheduleSerializer 52 | 53 | 54 | class AddPeriodicTask(ModelViewSet): 55 | queryset = PeriodicTaskModel.objects.all() 56 | serializer_class = WritePeriodicTaskSerializer 57 | 58 | def get_serializer_class(self): 59 | if self.request.method == "POST": 60 | return self.serializer_class 61 | else: 62 | return PeriodicTaskSerializer 63 | 64 | 65 | class DBTCurrentVersionView(APIView): 66 | def get(self, request,): 67 | modules_version_data = load_dbt_current_version() 68 | serializer = DBTCurrentVersionSerializer(data=modules_version_data, many=True) 69 | serializer.is_valid(raise_exception=True) 70 | return Response(serializer.data) 71 | 72 | 73 | class RunDBTTask(APIView): 74 | serializer_class = RunTaskSerializer 75 | 76 | def post(self, request, *args, **kwargs): 77 | serializer = RunTaskSerializer(data=request.data) 78 | if serializer.is_valid(): 79 | task_id = serializer.validated_data["task_id"] 80 | task = PeriodicTaskModel.objects.get(id=task_id) 81 | args = eval(task.args) if task.args else [] 82 | kwargs = eval(task.kwargs) if task.kwargs else {} 83 | dbt_runner_task.delay(*args, **kwargs) 84 | return Response( 85 | {"status": "Task has been initiated"}, status=status.HTTP_200_OK 86 | ) 87 | else: 88 | return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) 89 | 90 | -------------------------------------------------------------------------------- /dbt/contrib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-arsenault/django-build-tool/db5cf4cab23f38709a1909b868f4d1f59166d2f4/dbt/contrib/__init__.py -------------------------------------------------------------------------------- /dbt/contrib/sites/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /dbt/contrib/sites/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | import django.contrib.sites.models 2 | from django.contrib.sites.models import _simple_domain_name_validator 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [] 9 | 10 | operations = [ 11 | migrations.CreateModel( 12 | name="Site", 13 | fields=[ 14 | ( 15 | "id", 16 | models.AutoField( 17 | verbose_name="ID", 18 | serialize=False, 19 | auto_created=True, 20 | primary_key=True, 21 | ), 22 | ), 23 | ( 24 | "domain", 25 | models.CharField( 26 | max_length=100, 27 | verbose_name="domain name", 28 | validators=[_simple_domain_name_validator], 29 | ), 30 | ), 31 | ("name", models.CharField(max_length=50, verbose_name="display name")), 32 | ], 33 | options={ 34 | "ordering": ("domain",), 35 | "db_table": "django_site", 36 | "verbose_name": "site", 37 | "verbose_name_plural": "sites", 38 | }, 39 | bases=(models.Model,), 40 | managers=[("objects", django.contrib.sites.models.SiteManager())], 41 | ) 42 | ] 43 | -------------------------------------------------------------------------------- /dbt/contrib/sites/migrations/0002_alter_domain_unique.py: -------------------------------------------------------------------------------- 1 | import django.contrib.sites.models 2 | from django.db import migrations, models 3 | 4 | 5 | class Migration(migrations.Migration): 6 | 7 | dependencies = [("sites", "0001_initial")] 8 | 9 | operations = [ 10 | migrations.AlterField( 11 | model_name="site", 12 | name="domain", 13 | field=models.CharField( 14 | max_length=100, 15 | unique=True, 16 | validators=[django.contrib.sites.models._simple_domain_name_validator], 17 | verbose_name="domain name", 18 | ), 19 | ) 20 | ] 21 | -------------------------------------------------------------------------------- /dbt/contrib/sites/migrations/0003_set_site_domain_and_name.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.db import migrations 3 | 4 | 5 | def _update_or_create_site_with_sequence(site_model, connection, domain, name): 6 | """Update or create the site with default ID and keep the DB sequence in sync.""" 7 | site, created = site_model.objects.update_or_create( 8 | id=settings.SITE_ID, 9 | defaults={ 10 | "domain": domain, 11 | "name": name, 12 | }, 13 | ) 14 | if created: 15 | # We provided the ID explicitly when creating the Site entry, therefore the DB 16 | # sequence to auto-generate them wasn't used and is now out of sync. If we 17 | # don't do anything, we'll get a unique constraint violation the next time a 18 | # site is created. 19 | # To avoid this, we need to manually update DB sequence and make sure it's 20 | # greater than the maximum value. 21 | max_id = site_model.objects.order_by('-id').first().id 22 | with connection.cursor() as cursor: 23 | cursor.execute("SELECT last_value from django_site_id_seq") 24 | (current_id,) = cursor.fetchone() 25 | if current_id <= max_id: 26 | cursor.execute( 27 | "alter sequence django_site_id_seq restart with %s", 28 | [max_id + 1], 29 | ) 30 | 31 | 32 | def update_site_forward(apps, schema_editor): 33 | """Set site domain and name.""" 34 | Site = apps.get_model("sites", "Site") 35 | _update_or_create_site_with_sequence( 36 | Site, 37 | schema_editor.connection, 38 | "example.com", 39 | "DBT Analytics", 40 | ) 41 | 42 | 43 | def update_site_backward(apps, schema_editor): 44 | """Revert site domain and name to default.""" 45 | Site = apps.get_model("sites", "Site") 46 | _update_or_create_site_with_sequence( 47 | Site, 48 | schema_editor.connection, 49 | "example.com", 50 | "example.com", 51 | "DBT Analytics", 52 | ) 53 | 54 | 55 | class Migration(migrations.Migration): 56 | 57 | dependencies = [("sites", "0002_alter_domain_unique")] 58 | 59 | operations = [migrations.RunPython(update_site_forward, update_site_backward)] 60 | -------------------------------------------------------------------------------- /dbt/contrib/sites/migrations/0004_alter_options_ordering_domain.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.1.7 on 2021-02-04 14:49 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("sites", "0003_set_site_domain_and_name"), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterModelOptions( 14 | name="site", 15 | options={ 16 | "ordering": ["domain"], 17 | "verbose_name": "site", 18 | "verbose_name_plural": "sites", 19 | }, 20 | ), 21 | ] 22 | -------------------------------------------------------------------------------- /dbt/contrib/sites/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /dbt/static/css/project.css: -------------------------------------------------------------------------------- 1 | /* These styles are generated from project.scss. */ 2 | 3 | .alert-debug { 4 | color: black; 5 | background-color: white; 6 | border-color: #d6e9c6; 7 | } 8 | 9 | .alert-error { 10 | color: #b94a48; 11 | background-color: #f2dede; 12 | border-color: #eed3d7; 13 | } 14 | -------------------------------------------------------------------------------- /dbt/static/fonts/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-arsenault/django-build-tool/db5cf4cab23f38709a1909b868f4d1f59166d2f4/dbt/static/fonts/.gitkeep -------------------------------------------------------------------------------- /dbt/static/images/favicons/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-arsenault/django-build-tool/db5cf4cab23f38709a1909b868f4d1f59166d2f4/dbt/static/images/favicons/favicon.ico -------------------------------------------------------------------------------- /dbt/static/js/project.js: -------------------------------------------------------------------------------- 1 | /* Project specific Javascript goes here. */ 2 | -------------------------------------------------------------------------------- /dbt/templates/403.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block title %}Forbidden (403){% endblock %} 4 | 5 | {% block content %} 6 |

Forbidden (403)

7 | 8 |

{% if exception %}{{ exception }}{% else %}You're not allowed to access this page.{% endif %}

9 | {% endblock content %} 10 | -------------------------------------------------------------------------------- /dbt/templates/404.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block title %}Page not found{% endblock %} 4 | 5 | {% block content %} 6 |

Page not found

7 | 8 |

{% if exception %}{{ exception }}{% else %}This is not the page you were looking for.{% endif %}

9 | {% endblock content %} 10 | -------------------------------------------------------------------------------- /dbt/templates/500.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block title %}Server Error{% endblock %} 4 | 5 | {% block content %} 6 |

Ooops!!! 500

7 | 8 |

Looks like something went wrong!

9 | 10 |

We track these errors automatically, but if the problem persists feel free to contact us. In the meantime, try refreshing.

11 | {% endblock content %} 12 | -------------------------------------------------------------------------------- /dbt/templates/account/account_inactive.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n %} 4 | 5 | {% block head_title %}{% translate "Account Inactive" %}{% endblock %} 6 | 7 | {% block inner %} 8 |

{% translate "Account Inactive" %}

9 | 10 |

{% translate "This account is inactive." %}

11 | {% endblock %} 12 | -------------------------------------------------------------------------------- /dbt/templates/account/base.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block title %}{% block head_title %}{% endblock head_title %}{% endblock title %} 3 | 4 | {% block content %} 5 |
6 |
7 | {% block inner %}{% endblock %} 8 |
9 |
10 | {% endblock %} 11 | -------------------------------------------------------------------------------- /dbt/templates/account/email.html: -------------------------------------------------------------------------------- 1 | 2 | {% extends "account/base.html" %} 3 | 4 | {% load i18n %} 5 | {% load crispy_forms_tags %} 6 | 7 | {% block head_title %}{% translate "Account" %}{% endblock %} 8 | 9 | {% block inner %} 10 |

{% translate "E-mail Addresses" %}

11 | 12 | {% if user.emailaddress_set.all %} 13 |

{% translate 'The following e-mail addresses are associated with your account:' %}

14 | 15 | 44 | 45 | {% else %} 46 |

{% translate 'Warning:'%} {% translate "You currently do not have any e-mail address set up. You should really add an e-mail address so you can receive notifications, reset your password, etc." %}

47 | 48 | {% endif %} 49 | 50 | 51 |

{% translate "Add E-mail Address" %}

52 | 53 |
54 | {% csrf_token %} 55 | {{ form|crispy }} 56 | 57 |
58 | 59 | {% endblock %} 60 | 61 | 62 | {% block inline_javascript %} 63 | {{ block.super }} 64 | 78 | {% endblock %} 79 | -------------------------------------------------------------------------------- /dbt/templates/account/email_confirm.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n %} 4 | {% load account %} 5 | 6 | {% block head_title %}{% translate "Confirm E-mail Address" %}{% endblock %} 7 | 8 | 9 | {% block inner %} 10 |

{% translate "Confirm E-mail Address" %}

11 | 12 | {% if confirmation %} 13 | 14 | {% user_display confirmation.email_address.user as user_display %} 15 | 16 |

{% blocktranslate with confirmation.email_address.email as email %}Please confirm that {{ email }} is an e-mail address for user {{ user_display }}.{% endblocktranslate %}

17 | 18 |
19 | {% csrf_token %} 20 | 21 |
22 | 23 | {% else %} 24 | 25 | {% url 'account_email' as email_url %} 26 | 27 |

{% blocktranslate %}This e-mail confirmation link expired or is invalid. Please issue a new e-mail confirmation request.{% endblocktranslate %}

28 | 29 | {% endif %} 30 | 31 | {% endblock %} 32 | -------------------------------------------------------------------------------- /dbt/templates/account/login.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n %} 4 | {% load account socialaccount %} 5 | {% load crispy_forms_tags %} 6 | 7 | {% block head_title %}{% translate "Sign In" %}{% endblock %} 8 | 9 | {% block inner %} 10 | 11 |

{% translate "Sign In" %}

12 | 13 | {% get_providers as socialaccount_providers %} 14 | 15 | {% if socialaccount_providers %} 16 |

17 | {% translate "Please sign in with one of your existing third party accounts:" %} 18 | {% if ACCOUNT_ALLOW_REGISTRATION %} 19 | {% blocktranslate trimmed %} 20 | Or, sign up 21 | for a {{ site_name }} account and sign in below: 22 | {% endblocktranslate %} 23 | {% endif %} 24 |

25 | 26 |
27 | 28 |
    29 | {% include "socialaccount/snippets/provider_list.html" with process="login" %} 30 |
31 | 32 | 33 | 34 |
35 | 36 | {% include "socialaccount/snippets/login_extra.html" %} 37 | 38 | {% else %} 39 | {% if ACCOUNT_ALLOW_REGISTRATION %} 40 |

41 | {% blocktranslate trimmed %} 42 | If you have not created an account yet, then please 43 | sign up first. 44 | {% endblocktranslate %} 45 |

46 | {% endif %} 47 | {% endif %} 48 | 49 | 58 | 59 | {% endblock %} 60 | -------------------------------------------------------------------------------- /dbt/templates/account/logout.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n %} 4 | 5 | {% block head_title %}{% translate "Sign Out" %}{% endblock %} 6 | 7 | {% block inner %} 8 |

{% translate "Sign Out" %}

9 | 10 |

{% translate 'Are you sure you want to sign out?' %}

11 | 12 |
13 | {% csrf_token %} 14 | {% if redirect_field_value %} 15 | 16 | {% endif %} 17 | 18 |
19 | {% endblock %} 20 | -------------------------------------------------------------------------------- /dbt/templates/account/password_change.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n %} 4 | {% load crispy_forms_tags %} 5 | 6 | {% block head_title %}{% translate "Change Password" %}{% endblock %} 7 | 8 | {% block inner %} 9 |

{% translate "Change Password" %}

10 | 11 |
12 | {% csrf_token %} 13 | {{ form|crispy }} 14 | 15 |
16 | {% endblock %} 17 | -------------------------------------------------------------------------------- /dbt/templates/account/password_reset.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n %} 4 | {% load account %} 5 | {% load crispy_forms_tags %} 6 | 7 | {% block head_title %}{% translate "Password Reset" %}{% endblock %} 8 | 9 | {% block inner %} 10 | 11 |

{% translate "Password Reset" %}

12 | {% if user.is_authenticated %} 13 | {% include "account/snippets/already_logged_in.html" %} 14 | {% endif %} 15 | 16 |

{% translate "Forgotten your password? Enter your e-mail address below, and we'll send you an e-mail allowing you to reset it." %}

17 | 18 |
19 | {% csrf_token %} 20 | {{ form|crispy }} 21 | 22 |
23 | 24 |

{% blocktranslate %}Please contact us if you have any trouble resetting your password.{% endblocktranslate %}

25 | {% endblock %} 26 | -------------------------------------------------------------------------------- /dbt/templates/account/password_reset_done.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n %} 4 | {% load account %} 5 | 6 | {% block head_title %}{% translate "Password Reset" %}{% endblock %} 7 | 8 | {% block inner %} 9 |

{% translate "Password Reset" %}

10 | 11 | {% if user.is_authenticated %} 12 | {% include "account/snippets/already_logged_in.html" %} 13 | {% endif %} 14 | 15 |

{% blocktranslate %}We have sent you an e-mail. Please contact us if you do not receive it within a few minutes.{% endblocktranslate %}

16 | {% endblock %} 17 | -------------------------------------------------------------------------------- /dbt/templates/account/password_reset_from_key.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n %} 4 | {% load crispy_forms_tags %} 5 | {% block head_title %}{% translate "Change Password" %}{% endblock %} 6 | 7 | {% block inner %} 8 |

{% if token_fail %}{% translate "Bad Token" %}{% else %}{% translate "Change Password" %}{% endif %}

9 | 10 | {% if token_fail %} 11 | {% url 'account_reset_password' as passwd_reset_url %} 12 |

{% blocktranslate %}The password reset link was invalid, possibly because it has already been used. Please request a new password reset.{% endblocktranslate %}

13 | {% else %} 14 | {% if form %} 15 |
16 | {% csrf_token %} 17 | {{ form|crispy }} 18 | 19 |
20 | {% else %} 21 |

{% translate 'Your password is now changed.' %}

22 | {% endif %} 23 | {% endif %} 24 | {% endblock %} 25 | -------------------------------------------------------------------------------- /dbt/templates/account/password_reset_from_key_done.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n %} 4 | {% block head_title %}{% translate "Change Password" %}{% endblock %} 5 | 6 | {% block inner %} 7 |

{% translate "Change Password" %}

8 |

{% translate 'Your password is now changed.' %}

9 | {% endblock %} 10 | -------------------------------------------------------------------------------- /dbt/templates/account/password_set.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n %} 4 | {% load crispy_forms_tags %} 5 | 6 | {% block head_title %}{% translate "Set Password" %}{% endblock %} 7 | 8 | {% block inner %} 9 |

{% translate "Set Password" %}

10 | 11 |
12 | {% csrf_token %} 13 | {{ form|crispy }} 14 | 15 |
16 | {% endblock %} 17 | -------------------------------------------------------------------------------- /dbt/templates/account/profile.html: -------------------------------------------------------------------------------- 1 | {% extends 'account/base.html' %} 2 | 3 | {% block content %} 4 |

Profile Views

5 |

{{ request.user.username }}

6 | {% endblock %} -------------------------------------------------------------------------------- /dbt/templates/account/signup.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n %} 4 | {% load crispy_forms_tags %} 5 | 6 | {% block head_title %}{% translate "Signup" %}{% endblock %} 7 | 8 | {% block inner %} 9 |

{% translate "Sign Up" %}

10 | 11 |

{% blocktranslate %}Already have an account? Then please sign in.{% endblocktranslate %}

12 | 13 | 21 | 22 | {% endblock %} 23 | -------------------------------------------------------------------------------- /dbt/templates/account/signup_closed.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n %} 4 | 5 | {% block head_title %}{% translate "Sign Up Closed" %}{% endblock %} 6 | 7 | {% block inner %} 8 |

{% translate "Sign Up Closed" %}

9 | 10 |

{% translate "We are sorry, but the sign up is currently closed." %}

11 | {% endblock %} 12 | -------------------------------------------------------------------------------- /dbt/templates/account/verification_sent.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n %} 4 | 5 | {% block head_title %}{% translate "Verify Your E-mail Address" %}{% endblock %} 6 | 7 | {% block inner %} 8 |

{% translate "Verify Your E-mail Address" %}

9 | 10 |

{% blocktranslate %}We have sent an e-mail to you for verification. Follow the link provided to finalize the signup process. Please contact us if you do not receive it within a few minutes.{% endblocktranslate %}

11 | 12 | {% endblock %} 13 | -------------------------------------------------------------------------------- /dbt/templates/account/verified_email_required.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n %} 4 | 5 | {% block head_title %}{% translate "Verify Your E-mail Address" %}{% endblock %} 6 | 7 | {% block inner %} 8 |

{% translate "Verify Your E-mail Address" %}

9 | 10 | {% url 'account_email' as email_url %} 11 | 12 |

{% blocktranslate %}This part of the site requires us to verify that 13 | you are who you claim to be. For this purpose, we require that you 14 | verify ownership of your e-mail address. {% endblocktranslate %}

15 | 16 |

{% blocktranslate %}We have sent an e-mail to you for 17 | verification. Please click on the link inside this e-mail. Please 18 | contact us if you do not receive it within a few minutes.{% endblocktranslate %}

19 | 20 |

{% blocktranslate %}Note: you can still change your e-mail address.{% endblocktranslate %}

21 | {% endblock %} 22 | -------------------------------------------------------------------------------- /dbt/templates/admin/base_site.html: -------------------------------------------------------------------------------- 1 | {% extends "admin/base.html" %} 2 | {% load i18n %} 3 | 4 | {% block title %}{{ title }} {% endblock %} 5 | 6 | {% block branding %} 7 |

DBT Administration

8 | {% endblock %} 9 | 10 | {% block nav-global %}{% endblock %} 11 | -------------------------------------------------------------------------------- /dbt/templates/admin/index.html: -------------------------------------------------------------------------------- 1 | {% extends "admin/index.html" %} 2 | {% load i18n static %} 3 | 4 | {#{% block welcome-msg %}#} 5 | 6 | {#{% endblock %}#} 7 | 8 | {#{% block userlinks %}#} 9 | 10 | {#{% endblock %}#} 11 | -------------------------------------------------------------------------------- /dbt/templates/base.html: -------------------------------------------------------------------------------- 1 | {% load static i18n %} 2 | {% get_current_language as LANGUAGE_CODE %} 3 | 4 | 5 | 6 | 7 | {% block title %}DBT Analytics{% endblock title %} 8 | 9 | 10 | 11 | 12 | 13 | 14 | {% block css %} 15 | 16 | 19 | 20 | 21 | 22 | 23 | {% endblock %} 24 | 26 | {# Placed at the top of the document so pages load faster with defer #} 27 | {% block javascript %} 28 | 29 | 32 | 33 | 34 | 35 | 36 | 37 | {% endblock javascript %} 38 | 39 | 40 | 41 | 42 | 43 |
44 | 68 | 69 |
70 | 71 |
72 | 73 | {% if messages %} 74 | {% for message in messages %} 75 |
76 | {{ message }} 77 | 78 |
79 | {% endfor %} 80 | {% endif %} 81 | 82 | {% block content %} 83 |

Use this document as a way to quick start any new project.

84 | {% endblock content %} 85 | 86 |
87 | 88 | {% block modal %}{% endblock modal %} 89 | 90 | {% block inline_javascript %} 91 | {% comment %} 92 | Script tags with only code, no src (defer by default). To run 93 | with a "defer" so that you run inline code: 94 | 98 | {% endcomment %} 99 | {% endblock inline_javascript %} 100 | 101 | 102 | -------------------------------------------------------------------------------- /dbt/templates/users/user_detail.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% load static %} 3 | 4 | {% block title %}User: {{ object.username }}{% endblock %} 5 | 6 | {% block content %} 7 |
8 | 9 |
10 |
11 | 12 |

{{ object.username }}

13 | {% if object.name %} 14 |

{{ object.name }}

15 | {% endif %} 16 |
17 |
18 | 19 | {% if object == request.user %} 20 | 21 |
22 | 23 |
24 | {# My Info#} 25 | E-Mail 26 | 27 |
28 | 29 |
30 | 31 | {% endif %} 32 | 33 |
34 | {% endblock content %} 35 | -------------------------------------------------------------------------------- /dbt/templates/users/user_form.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% load crispy_forms_tags %} 3 | 4 | {% block title %}{{ user.username }}{% endblock %} 5 | 6 | {% block content %} 7 |

{{ user.username }}

8 |
9 | {% csrf_token %} 10 | {{ form|crispy }} 11 |
12 |
13 | 14 |
15 |
16 |
17 | {% endblock %} 18 | -------------------------------------------------------------------------------- /dbt/users/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-arsenault/django-build-tool/db5cf4cab23f38709a1909b868f4d1f59166d2f4/dbt/users/__init__.py -------------------------------------------------------------------------------- /dbt/users/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | from django.utils.translation import gettext_lazy as _ 3 | 4 | 5 | class UsersConfig(AppConfig): 6 | name = "dbt.users" 7 | verbose_name = _("Users") 8 | 9 | def ready(self): 10 | try: 11 | import dbt.users.signals # noqa F401 12 | except ImportError: 13 | pass 14 | -------------------------------------------------------------------------------- /dbt/users/context_processors.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | -------------------------------------------------------------------------------- /dbt/users/forms.py: -------------------------------------------------------------------------------- 1 | from allauth.account.forms import SignupForm 2 | from allauth.socialaccount.forms import SignupForm as SocialSignupForm 3 | from django.contrib.auth import forms as admin_forms 4 | from django.contrib.auth import get_user_model 5 | from django.utils.translation import gettext_lazy as _ 6 | from allauth.utils import set_form_field_order 7 | from django import forms 8 | 9 | User = get_user_model() 10 | 11 | 12 | class UserAdminChangeForm(admin_forms.UserChangeForm): 13 | class Meta(admin_forms.UserChangeForm.Meta): 14 | model = User 15 | 16 | 17 | class UserAdminCreationForm(admin_forms.UserCreationForm): 18 | """ 19 | Form for User Creation in the Admin Area. 20 | To change user signup, see UserSignupForm and UserSocialSignupForm. 21 | """ 22 | 23 | class Meta(admin_forms.UserCreationForm.Meta): 24 | model = User 25 | 26 | error_messages = { 27 | "username": {"unique": _("This username has already been taken.")} 28 | } 29 | 30 | 31 | class UserSignupForm(SignupForm): 32 | """ 33 | Form that will be rendered on a user sign up section/screen. 34 | Default fields will be added automatically. 35 | Check UserSocialSignupForm for accounts created from social. 36 | """ 37 | 38 | 39 | class UserSocialSignupForm(SocialSignupForm): 40 | """ 41 | Renders the form when user has signed up using social accounts. 42 | Default fields will be added automatically. 43 | See UserSignupForm otherwise. 44 | """ 45 | 46 | 47 | class ExtendedSignupForm(SignupForm): 48 | def __init__(self, *args, **kwargs): 49 | super(ExtendedSignupForm, self).__init__(*args, **kwargs) 50 | self.fields["first_name"] = forms.CharField( 51 | label=_("Fist Name"), 52 | max_length=255, 53 | widget=forms.TextInput( 54 | attrs={"placeholder": _("Fist Name"), "autocomplete": "Fist Name"} 55 | ), 56 | ) 57 | 58 | self.fields["last_name"] = forms.CharField( 59 | label=_("Last Name"), 60 | max_length=255, 61 | widget=forms.TextInput( 62 | attrs={"placeholder": _("Last Name"), "autocomplete": "Fist Name"} 63 | ), 64 | ) 65 | 66 | if hasattr(self, "field_order"): 67 | set_form_field_order(self, self.field_order) 68 | 69 | field_order = [ 70 | "email", 71 | 'first_name', 72 | 'last_name', 73 | "password1", 74 | "password2", # ignored when not present 75 | ] 76 | -------------------------------------------------------------------------------- /dbt/users/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-arsenault/django-build-tool/db5cf4cab23f38709a1909b868f4d1f59166d2f4/dbt/users/management/__init__.py -------------------------------------------------------------------------------- /dbt/users/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-arsenault/django-build-tool/db5cf4cab23f38709a1909b868f4d1f59166d2f4/dbt/users/management/commands/__init__.py -------------------------------------------------------------------------------- /dbt/users/management/commands/dbt_command.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import subprocess 4 | from datetime import datetime 5 | from django.conf import settings 6 | from django.core.management.base import BaseCommand 7 | import paramiko 8 | from dbt.analytics.models import ( 9 | Args, 10 | DBTLogs, 11 | GitRepo, 12 | SubProcessLog, 13 | ProfileYAML, 14 | PeriodicTask, 15 | ) 16 | from dbt.utils.common import save_profile_yml 17 | 18 | SSH_KEY_PREFIX = "git-django_" 19 | 20 | 21 | class Command(BaseCommand): 22 | help = "DBT jobs" 23 | 24 | def add_arguments(self, parser): 25 | print("add_arguments", parser) 26 | parser.add_argument("--dbt_command", action="store", type=str) 27 | parser.add_argument( 28 | "--pk", action="store", type=str 29 | ) # pk is git repo object id 30 | 31 | def read_json(self, filename, pk): 32 | DBT_LOG_TARGET = "{}-{}/target".format( 33 | getattr(settings, "EXTERNAL_REPO_PREFIX"), pk 34 | ) 35 | file_path = os.path.join(DBT_LOG_TARGET, filename) 36 | data = {} 37 | try: 38 | with open(file_path, "r") as state: 39 | data = json.load(state) 40 | except Exception: 41 | print(f"{file_path} not found") 42 | data = {} 43 | return data 44 | 45 | def handle(self, *args, **options): 46 | os.environ["PATH"] += os.pathsep + "/usr/bin" 47 | os.environ["PATH"] += os.pathsep + "/bin" 48 | 49 | stdout_data = "" 50 | try: 51 | dbt_command = options["dbt_command"] 52 | pk = json.loads(options["pk"].replace("'", '"'))["task_id"] 53 | 54 | if dbt_command.startswith("dbt"): 55 | instance = PeriodicTask.objects.get(id=pk) 56 | git_repo = GitRepo.objects.get(id=instance.git_repo_id) 57 | profile_yml = ProfileYAML.objects.get(id=instance.profile_yml_id) 58 | 59 | EXTERNAL_REPO_PREFIX = getattr(settings, "EXTERNAL_REPO_PREFIX") 60 | THIS_PROJECT_PATH = getattr(settings, "THIS_PROJECT_PATH") 61 | EXTERNAL_REPO_NAME = f"{EXTERNAL_REPO_PREFIX}-{instance.git_repo_id}" 62 | EXTERNAL_REPO_PATH = os.path.join(THIS_PROJECT_PATH, EXTERNAL_REPO_NAME) 63 | 64 | os.path.join(THIS_PROJECT_PATH, EXTERNAL_REPO_NAME) 65 | pull_cmd = f"cd {EXTERNAL_REPO_PATH} && git pull origin HEAD" 66 | print(f"Pull cmd: {pull_cmd}") 67 | 68 | profile_yml_content = None 69 | if instance.profile_yml: 70 | profile_yml_content = profile_yml.profile_yml 71 | save_profile_yml(profile_yml_content, ".dbt/profiles.yml") 72 | else: 73 | print("No profile yml found") 74 | exit(-1) 75 | 76 | if git_repo.url.startswith("git"): 77 | pvt_key = os.path.join( 78 | os.getenv("HOME"), 79 | ".ssh/{}{}".format(SSH_KEY_PREFIX, git_repo.ssh_key.id), 80 | ) 81 | cmd = 'eval "$(/usr/bin/ssh-agent -s)" && /usr/bin/ssh-add {} && {}'.format( 82 | pvt_key, pull_cmd 83 | ) 84 | p1 = subprocess.Popen( 85 | cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True 86 | ) 87 | else: 88 | p1 = subprocess.Popen( 89 | pull_cmd, 90 | stdout=subprocess.PIPE, 91 | stderr=subprocess.PIPE, 92 | shell=True, 93 | ) 94 | p1.wait() 95 | SubProcessLog.objects.create(details=str(p1)) 96 | p1.kill() 97 | del p1 98 | 99 | executable_command = "cd {} && {}".format( 100 | EXTERNAL_REPO_PATH, dbt_command 101 | ) 102 | dbt_result = subprocess.Popen( 103 | executable_command, 104 | cwd=EXTERNAL_REPO_PATH, 105 | shell=True, 106 | stdout=subprocess.PIPE, 107 | stderr=subprocess.STDOUT, 108 | ) 109 | # Read the real-time output from stdout and save it to a variable 110 | for line in dbt_result.stdout: 111 | print(line, end="") 112 | stdout_data += line.decode("utf-8") 113 | 114 | dbt_result.wait() 115 | os.system("cd {} && git pull origin HEAD".format(EXTERNAL_REPO_PATH)) 116 | 117 | manifest = self.read_json( 118 | f"{EXTERNAL_REPO_PATH}/target/manifest.json", instance.git_repo_id 119 | ) 120 | run_results = self.read_json( 121 | f"{EXTERNAL_REPO_PATH}/target/run_results.json", 122 | instance.git_repo_id, 123 | ) 124 | sources = self.read_json( 125 | f"{EXTERNAL_REPO_PATH}/target/sources.json", instance.git_repo_id 126 | ) 127 | catalog = self.read_json( 128 | f"{EXTERNAL_REPO_PATH}/target/catalog.json", instance.git_repo_id 129 | ) 130 | 131 | dbt_log = DBTLogs.objects.create( 132 | manifest=manifest, 133 | run_results=run_results, 134 | sources=sources, 135 | catalog=catalog, 136 | command=dbt_command, 137 | repository_used_name=instance.git_repo.name, 138 | profile_yml_used_name=profile_yml.name, 139 | periodic_task_name=instance.name, 140 | completed_at=datetime.now(), 141 | previous_command="this is first commands" 142 | if not DBTLogs.objects.all().exists() 143 | else DBTLogs.objects.last().command, 144 | dbt_stdout=stdout_data, 145 | ) 146 | 147 | args = run_results.get("args", {}) 148 | Args.objects.create( 149 | dbt_log=dbt_log, 150 | quiet=args.get("quiet", ""), 151 | which=args.get("which", ""), 152 | no_print=args.get("no_print", ""), 153 | rpc_method=args.get("rpc_method", ""), 154 | use_colors=args.get("use_colors", ""), 155 | write_json=args.get(" write_json", ""), 156 | profiles_dir=args.get("profiles_dir", ""), 157 | partial_parse=args.get("partial_parse", ""), 158 | printer_width=args.get("printer_width", ""), 159 | static_parser=args.get("static_parser", ""), 160 | version_check=args.get("version_check", ""), 161 | event_buffer_size=args.get("event_buffer_size", ""), 162 | indirect_selection=args.get("indirect_selection", ""), 163 | send_anonymous=args.get("send_anonymous_usage_stats", ""), 164 | usage_stats=args.get("usage_stats", ""), 165 | ) 166 | 167 | dbt_result.kill() 168 | del dbt_result 169 | 170 | except Exception as err: 171 | try: 172 | dbt_result.kill() 173 | del dbt_result 174 | except Exception as err: 175 | pass 176 | instance = PeriodicTask.objects.get(id=pk) 177 | DBTLogs.objects.create( 178 | command=dbt_command, 179 | periodic_task_name=instance.name, 180 | completed_at=datetime.now(), 181 | repository_used_name=instance.git_repo.name, 182 | profile_yml_used_name=profile_yml.name, 183 | previous_command="this is first commands" 184 | if not DBTLogs.objects.all().exists() 185 | else DBTLogs.objects.last().command, 186 | success=False, 187 | fail_reason=str(err), 188 | dbt_stdout=stdout_data 189 | ) 190 | -------------------------------------------------------------------------------- /dbt/users/management/commands/dbt_to_db.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | from django.conf import settings 5 | from django.core.management.base import BaseCommand 6 | 7 | from dbt.analytics.models import DBTLogs 8 | 9 | 10 | class Command(BaseCommand): 11 | help = 'DBT jobs' 12 | 13 | def read_json(self, filename): 14 | 15 | DBT_LOG_TARGET = getattr(settings, 'DBT_LOG_TARGET') 16 | file_path = os.path.join(DBT_LOG_TARGET, filename) 17 | data = {} 18 | with open(file_path, 'r') as state: 19 | data = json.load(state) 20 | return data 21 | 22 | def handle(self, *args, **options): 23 | manifest = self.read_json('manifest.json') 24 | run_results = self.read_json('run_results.json') 25 | 26 | DBTLogs.objects.create( 27 | manifest=manifest, 28 | run_results=run_results, 29 | ) 30 | -------------------------------------------------------------------------------- /dbt/users/management/commands/wait_for_db.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from django.core.management.base import BaseCommand 4 | from django.db import connections 5 | from django.db.utils import OperationalError # db exception 6 | 7 | 8 | class Command(BaseCommand): 9 | """Script to pause execuation until django database is ready""" 10 | 11 | def handle(self, *args, **options): 12 | wait = 1 13 | self.stdout.write("Waiting for database ready...") 14 | db_conn = None 15 | while not db_conn: 16 | try: 17 | db_conn = connections['default'] 18 | except OperationalError: # db exception 19 | self.stdout.write("Database not ready yet, waiting {} second...".format(wait)) 20 | time.sleep(wait) 21 | 22 | self.stdout.write(self.style.SUCCESS('Database connected successfully!')) 23 | -------------------------------------------------------------------------------- /dbt/users/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.9 on 2021-11-20 11:23 2 | import django.contrib.auth.models 3 | import django.contrib.auth.validators 4 | from django.db import migrations, models 5 | import django.utils.timezone 6 | 7 | 8 | import django.contrib.auth.models 9 | import django.contrib.auth.validators 10 | from django.db import migrations, models 11 | import django.utils.timezone 12 | 13 | 14 | class Migration(migrations.Migration): 15 | 16 | initial = True 17 | 18 | dependencies = [ 19 | ("auth", "0012_alter_user_first_name_max_length"), 20 | ] 21 | 22 | operations = [ 23 | migrations.CreateModel( 24 | name="User", 25 | fields=[ 26 | ( 27 | "id", 28 | models.AutoField( 29 | auto_created=True, 30 | primary_key=True, 31 | serialize=False, 32 | verbose_name="ID", 33 | ), 34 | ), 35 | ("password", models.CharField(max_length=128, verbose_name="password")), 36 | ( 37 | "last_login", 38 | models.DateTimeField( 39 | blank=True, null=True, verbose_name="last login" 40 | ), 41 | ), 42 | ( 43 | "is_superuser", 44 | models.BooleanField( 45 | default=False, 46 | help_text="Designates that this user has all permissions without explicitly assigning them.", 47 | verbose_name="superuser status", 48 | ), 49 | ), 50 | ( 51 | "username", 52 | models.CharField( 53 | error_messages={ 54 | "unique": "A user with that username already exists." 55 | }, 56 | help_text="Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.", 57 | max_length=150, 58 | unique=True, 59 | validators=[ 60 | django.contrib.auth.validators.UnicodeUsernameValidator() 61 | ], 62 | verbose_name="username", 63 | ), 64 | ), 65 | ( 66 | "first_name", 67 | models.CharField( 68 | blank=True, max_length=150, verbose_name="first name" 69 | ), 70 | ), 71 | ( 72 | "last_name", 73 | models.CharField( 74 | blank=True, max_length=150, verbose_name="last name" 75 | ), 76 | ), 77 | ( 78 | "email", 79 | models.EmailField( 80 | blank=True, max_length=254, verbose_name="email address" 81 | ), 82 | ), 83 | ( 84 | "is_staff", 85 | models.BooleanField( 86 | default=False, 87 | help_text="Designates whether the user can log into this admin site.", 88 | verbose_name="staff status", 89 | ), 90 | ), 91 | ( 92 | "is_active", 93 | models.BooleanField( 94 | default=True, 95 | help_text="Designates whether this user should be treated as active. Unselect this instead of deleting accounts.", 96 | verbose_name="active", 97 | ), 98 | ), 99 | ( 100 | "date_joined", 101 | models.DateTimeField( 102 | default=django.utils.timezone.now, verbose_name="date joined" 103 | ), 104 | ), 105 | ( 106 | "avatar", 107 | models.ImageField(blank=True, null=True, upload_to="avatars/"), 108 | ), 109 | ( 110 | "groups", 111 | models.ManyToManyField( 112 | blank=True, 113 | help_text="The groups this user belongs to. A user will get all permissions granted to each of their groups.", 114 | related_name="user_set", 115 | related_query_name="user", 116 | to="auth.Group", 117 | verbose_name="groups", 118 | ), 119 | ), 120 | ( 121 | "user_permissions", 122 | models.ManyToManyField( 123 | blank=True, 124 | help_text="Specific permissions for this user.", 125 | related_name="user_set", 126 | related_query_name="user", 127 | to="auth.Permission", 128 | verbose_name="user permissions", 129 | ), 130 | ), 131 | ], 132 | options={ 133 | "verbose_name": "user", 134 | "verbose_name_plural": "users", 135 | "abstract": False, 136 | }, 137 | managers=[ 138 | ("objects", django.contrib.auth.models.UserManager()), 139 | ], 140 | ), 141 | ] 142 | 143 | -------------------------------------------------------------------------------- /dbt/users/migrations/0002_alter_user_id.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.5 on 2022-08-22 21:22 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('users', '0001_initial'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='user', 15 | name='id', 16 | field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /dbt/users/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-arsenault/django-build-tool/db5cf4cab23f38709a1909b868f4d1f59166d2f4/dbt/users/migrations/__init__.py -------------------------------------------------------------------------------- /dbt/users/models.py: -------------------------------------------------------------------------------- 1 | from django.contrib.auth.models import AbstractUser 2 | from django.db.models import CharField, ImageField 3 | from django.urls import reverse 4 | from django.utils.translation import gettext_lazy as _ 5 | 6 | 7 | class User(AbstractUser): 8 | avatar = ImageField(upload_to="avatars/", null=True, blank=True) 9 | first_name = CharField(blank=True, max_length=150, verbose_name="first name") 10 | last_name = CharField(blank=True, max_length=150, verbose_name="last name") 11 | 12 | def get_absolute_url(self): 13 | return reverse("users:detail", kwargs={"username": f"{self.first_name} {self.last_name}"}) 14 | 15 | -------------------------------------------------------------------------------- /dbt/users/signals.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | 4 | from django.conf import settings 5 | from django.contrib.auth import get_user_model 6 | from django.db.models.signals import post_save, pre_delete 7 | from django.dispatch import receiver 8 | 9 | from dbt.analytics.models import GitRepo, SSHKey, PeriodicTask 10 | 11 | SSH_KEY_PREFIX = getattr(settings, "SSH_KEY_PREFIX") 12 | 13 | User = get_user_model() 14 | 15 | 16 | # @receiver(post_save, sender=GitRepo) 17 | def on_git_repo_save(sender, instance, created, **kwargs): 18 | os.environ["PATH"] += os.pathsep + "/usr/bin" 19 | os.environ["PATH"] += os.pathsep + "/bin" 20 | EXTERNAL_REPO_PREFIX = getattr(settings, "EXTERNAL_REPO_PREFIX") 21 | THIS_PROJECT_PATH = getattr(settings, "THIS_PROJECT_PATH") 22 | EXTERNAL_REPO_NAME = "{}-{}".format(EXTERNAL_REPO_PREFIX, instance.id) 23 | destination = os.path.join(THIS_PROJECT_PATH, EXTERNAL_REPO_NAME) 24 | print(EXTERNAL_REPO_PREFIX, THIS_PROJECT_PATH, EXTERNAL_REPO_NAME, destination) 25 | if os.path.isdir(destination): # if exist 26 | subprocess.run(["rm", "-rf", destination], check=True, capture_output=True) 27 | 28 | if instance.url.startswith("git"): 29 | pvt_key = os.path.join( 30 | os.getenv("HOME"), ".ssh/{}{}".format(SSH_KEY_PREFIX, instance.ssh_key.id) 31 | ) 32 | cmd = 'eval "$(/usr/bin/ssh-agent -s)" && /usr/bin/ssh-add {} && /usr/bin/git clone {} {}'.format( 33 | pvt_key, instance.url, destination 34 | ) 35 | else: 36 | cmd = "/usr/bin/git clone {} {}".format(instance.url, destination) 37 | 38 | print(cmd) 39 | p1 = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 40 | 41 | print(p1.stderr, "===") 42 | 43 | 44 | @receiver(pre_delete, sender=GitRepo) 45 | def on_gitrepo_delete(sender, instance, **kwargs): 46 | os.environ["PATH"] += os.pathsep + "/usr/bin" 47 | os.environ["PATH"] += os.pathsep + "/bin" 48 | EXTERNAL_REPO_PREFIX = getattr(settings, "EXTERNAL_REPO_PREFIX") 49 | THIS_PROJECT_PATH = getattr(settings, "THIS_PROJECT_PATH") 50 | EXTERNAL_REPO_NAME = "{}-{}".format(EXTERNAL_REPO_PREFIX, instance.id) 51 | destination = os.path.join(THIS_PROJECT_PATH, EXTERNAL_REPO_NAME) 52 | if os.path.isdir(destination): # if exist 53 | subprocess.run(["rm", "-rf", destination], check=True, capture_output=True) 54 | 55 | 56 | @receiver(post_save, sender=SSHKey) 57 | def on_ssh_key_create(sender, instance, created, **kwargs): 58 | ssh_key_path = os.path.join( 59 | os.getenv("HOME"), ".ssh/{}{}".format(SSH_KEY_PREFIX, instance.id) 60 | ) 61 | 62 | subprocess.run( 63 | ["/usr/bin/ssh-keygen", "-f", ssh_key_path, "-N", ""], 64 | check=False, 65 | capture_output=True, 66 | ) 67 | 68 | 69 | @receiver(pre_delete, sender=SSHKey) 70 | def on_ssh_key_delete(sender, instance, **kwargs): 71 | os.environ["PATH"] += os.pathsep + "/usr/bin" 72 | os.environ["PATH"] += os.pathsep + "/bin" 73 | private_key = os.path.join( 74 | os.getenv("HOME"), ".ssh/{}{}".format(SSH_KEY_PREFIX, instance.id) 75 | ) 76 | public_key = os.path.join( 77 | os.getenv("HOME"), ".ssh/{}{}.pub".format(SSH_KEY_PREFIX, instance.id) 78 | ) 79 | 80 | subprocess.run(["rm", "-rf", private_key], check=True, capture_output=True) 81 | subprocess.run(["rm", "-rf", public_key], check=True, capture_output=True) 82 | 83 | 84 | @receiver(post_save, sender=PeriodicTask) 85 | def on_periodic_task_create(sender, instance, created, **kwargs): 86 | if created: 87 | instance.kwarg = {"task_id": instance.id} 88 | instance.save() 89 | -------------------------------------------------------------------------------- /dbt/users/views.py: -------------------------------------------------------------------------------- 1 | from django.contrib.auth import get_user_model 2 | from django.contrib.auth.mixins import LoginRequiredMixin 3 | from django.contrib.messages.views import SuccessMessageMixin 4 | from django.urls import reverse 5 | from django.utils.translation import gettext_lazy as _ 6 | from django.views.generic import DetailView, RedirectView, UpdateView 7 | 8 | User = get_user_model() 9 | 10 | 11 | class UserDetailView(LoginRequiredMixin, DetailView): 12 | 13 | model = User 14 | slug_field = "username" 15 | slug_url_kwarg = "username" 16 | 17 | 18 | user_detail_view = UserDetailView.as_view() 19 | 20 | 21 | class UserUpdateView(LoginRequiredMixin, SuccessMessageMixin, UpdateView): 22 | 23 | model = User 24 | fields = ["name"] 25 | success_message = _("Information successfully updated") 26 | 27 | def get_success_url(self): 28 | assert ( 29 | self.request.user.is_authenticated 30 | ) # for mypy to know that the user is authenticated 31 | return self.request.user.get_absolute_url() 32 | 33 | def get_object(self): 34 | return self.request.user 35 | 36 | 37 | user_update_view = UserUpdateView.as_view() 38 | 39 | 40 | class UserRedirectView(LoginRequiredMixin, RedirectView): 41 | 42 | permanent = False 43 | 44 | def get_redirect_url(self): 45 | return reverse("users:detail", kwargs={"username": self.request.user.username}) 46 | 47 | 48 | user_redirect_view = UserRedirectView.as_view() 49 | 50 | -------------------------------------------------------------------------------- /dbt/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-arsenault/django-build-tool/db5cf4cab23f38709a1909b868f4d1f59166d2f4/dbt/utils/__init__.py -------------------------------------------------------------------------------- /dbt/utils/common.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | import yaml 4 | import re 5 | from importlib.metadata import version, PackageNotFoundError 6 | from django.conf import settings 7 | 8 | 9 | def save_profile_yml(profile_yml_text_content, profile_dir): 10 | profile_yml_file = os.path.join(os.getenv("HOME"), profile_dir) 11 | dct = yaml.safe_load(profile_yml_text_content) 12 | print(f"Profile yml file: {profile_yml_file}") 13 | print(profile_yml_text_content) 14 | with open(profile_yml_file, "w") as file: 15 | yaml.dump(dct, file) 16 | 17 | 18 | def clone_git_repo(instance) -> (bool, str): 19 | os.environ["PATH"] += os.pathsep + "/usr/bin" 20 | os.environ["PATH"] += os.pathsep + "/bin" 21 | EXTERNAL_REPO_PREFIX = getattr(settings, "EXTERNAL_REPO_PREFIX") 22 | THIS_PROJECT_PATH = getattr(settings, "THIS_PROJECT_PATH") 23 | EXTERNAL_REPO_NAME = "{}-{}".format(EXTERNAL_REPO_PREFIX, instance.id) 24 | destination = os.path.join(THIS_PROJECT_PATH, EXTERNAL_REPO_NAME) 25 | print(EXTERNAL_REPO_PREFIX, THIS_PROJECT_PATH, EXTERNAL_REPO_NAME, destination) 26 | if os.path.isdir(destination): # if exist 27 | subprocess.run(["rm", "-rf", destination], check=True, capture_output=True) 28 | 29 | if instance.url.startswith("git"): 30 | pvt_key = os.path.join( 31 | os.getenv("HOME"), 32 | ".ssh/{}{}".format( 33 | getattr(settings, "SSH_KEY_PREFIX"), instance.ssh_key.id 34 | ), 35 | ) 36 | cmd = 'eval "$(/usr/bin/ssh-agent -s)" && /usr/bin/ssh-add {} && /usr/bin/git clone {} {}'.format( 37 | pvt_key, instance.url, destination 38 | ) 39 | else: 40 | cmd = "/usr/bin/git clone {} {}".format(instance.url, destination) 41 | 42 | print(cmd) 43 | p1 = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 44 | 45 | print(p1.stderr, "===") 46 | result = True 47 | msg = "" 48 | if re.search("fatal:", p1.stderr.decode("UTF-8")): 49 | result = False 50 | msg = p1.stderr.decode("UTF-8") 51 | return result, msg 52 | 53 | 54 | # return current installed dbt version 55 | def load_dbt_current_version() -> list[dict[str, str | None]]: 56 | module_names_list = [ 57 | "dbt-core", 58 | "dbt-postgres", 59 | "dbt-redshift", 60 | "dbt-snowflake", 61 | "dbt-bigquery", 62 | ] 63 | modules_version_data = [] 64 | for module_name in module_names_list: 65 | try: 66 | module_version = version(module_name) 67 | except PackageNotFoundError: 68 | module_version = None 69 | 70 | modules_version_data.append( 71 | {"module_name": module_name, "version": module_version} 72 | ) 73 | return modules_version_data 74 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | volumes: 4 | dbt_scheduler_data: 5 | postgres_data: { } 6 | postgres_data_backups: { } 7 | 8 | services: 9 | password-generator: 10 | image: alpine 11 | volumes: 12 | - ./passwords:/passwords 13 | command: 14 | - /bin/sh 15 | - -c 16 | - > 17 | if [ ! -f /passwords/password.txt ]; then 18 | echo "dbt_login=dbtuser" > /passwords/password.txt; 19 | echo "dbt_password=$$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 16)" >> /passwords/password.txt; 20 | echo "postgres_user=dbtuser" >> /passwords/password.txt; 21 | echo "postgres_password=$$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 24)" >> /passwords/password.txt; 22 | fi; 23 | echo "passwords generated"; 24 | 25 | 26 | django: &django 27 | build: 28 | context: . 29 | dockerfile: ./compose/local/django/Dockerfile 30 | image: analytics_django 31 | container_name: analytics_django 32 | depends_on: 33 | - password-generator 34 | - postgres 35 | - redis 36 | volumes: 37 | - dbt_scheduler_data:/root/.dbt 38 | - ./passwords:/passwords 39 | env_file: 40 | - ./.envs/.local/.django 41 | - ./.envs/.local/.postgres 42 | ports: 43 | - "8000:8000" 44 | restart: "no" 45 | command: /start 46 | 47 | postgres: 48 | build: 49 | context: . 50 | dockerfile: ./compose/local/postgres/Dockerfile 51 | image: analytics_postgres 52 | container_name: analytics_postgres 53 | volumes: 54 | - postgres_data:/var/lib/postgresql/data 55 | - postgres_data_backups:/var/lib/postgresql/backups 56 | - ./passwords:/passwords 57 | depends_on: 58 | - password-generator 59 | env_file: 60 | - ./.envs/.local/.postgres 61 | restart: "no" 62 | ports: 63 | - "5432:5432" 64 | 65 | # pgrst: 66 | # image: postgrest/postgrest 67 | # container_name: analytics_postgrest 68 | # ports: 69 | # - "3000:3000" 70 | # env_file: 71 | # - ./.envs/.local/.postgres 72 | # depends_on: 73 | # - postgres 74 | 75 | swagger: 76 | image: swaggerapi/swagger-ui 77 | container_name: analytics_swagger 78 | restart: "no" 79 | ports: 80 | - "8080:8080" 81 | expose: 82 | - "8080" 83 | env_file: 84 | - ./.envs/.local/.postgres 85 | 86 | redis: 87 | image: redis:6 88 | restart: "no" 89 | container_name: analytics_redis 90 | 91 | celeryworker: 92 | <<: *django 93 | image: analytics_celeryworker 94 | container_name: analytics_celeryworker 95 | depends_on: 96 | - password-generator 97 | - django 98 | - redis 99 | - postgres 100 | restart: "no" 101 | ports: [ ] 102 | command: /start-celeryworker 103 | volumes: 104 | - dbt_scheduler_data:/root/.dbt 105 | - ./passwords:/passwords 106 | 107 | celerybeat: 108 | <<: *django 109 | image: analytics_celerybeat 110 | container_name: analytics_celerybeat 111 | restart: "no" 112 | depends_on: 113 | - password-generator 114 | - django 115 | - redis 116 | - postgres 117 | ports: [ ] 118 | volumes: 119 | - dbt_scheduler_data:/root/.dbt 120 | - ./passwords:/passwords 121 | command: /start-celerybeat 122 | 123 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | from pathlib import Path 5 | 6 | if __name__ == "__main__": 7 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local") 8 | 9 | try: 10 | from django.core.management import execute_from_command_line 11 | except ImportError: 12 | # The above import may fail for some other reason. Ensure that the 13 | # issue is really that Django is missing to avoid masking other 14 | # exceptions on Python 2. 15 | try: 16 | import django # noqa 17 | except ImportError: 18 | raise ImportError( 19 | "Couldn't import Django. Are you sure it's installed and " 20 | "available on your PYTHONPATH environment variable? Did you " 21 | "forget to activate a virtual environment?" 22 | ) 23 | 24 | raise 25 | 26 | # This allows easy placement of apps within the interior 27 | # dbt directory. 28 | current_path = Path(__file__).parent.resolve() 29 | sys.path.append(str(current_path / "dbt")) 30 | 31 | execute_from_command_line(sys.argv) 32 | -------------------------------------------------------------------------------- /print_dbt_current_version.py: -------------------------------------------------------------------------------- 1 | from dbt.utils.common import load_dbt_current_version 2 | 3 | if __name__ == "__main__": 4 | print(load_dbt_current_version()) 5 | -------------------------------------------------------------------------------- /requirements/base.txt: -------------------------------------------------------------------------------- 1 | pytz==2022.2.1 # https://github.com/stub42/pytz 2 | python-slugify==6.1.2 # https://github.com/un33k/python-slugify 3 | Pillow==9.2.0 # https://github.com/python-pillow/Pillow 4 | argon2-cffi==21.3.0 # https://github.com/hynek/argon2_cffi 5 | whitenoise==6.2.0 # https://github.com/evansd/whitenoise 6 | redis==4.3.4 # https://github.com/redis/redis-py 7 | hiredis==2.0.0 # https://github.com/redis/hiredis-py 8 | celery==5.2.7 # pyup: < 6.0 # https://github.com/celery/celery 9 | django-celery-beat==2.3.0 # https://github.com/celery/django-celery-beat 10 | 11 | # Django 12 | # ------------------------------------------------------------------------------ 13 | django==3.2.5 # pyup: < 4.0 # https://www.djangoproject.com/ 14 | django-environ==0.9.0 # https://github.com/joke2k/django-environ 15 | django-model-utils==4.2.0 # https://github.com/jazzband/django-model-utils 16 | django-redis==5.2.0 # https://github.com/jazzband/django-redis 17 | # Django REST Framework 18 | djangorestframework==3.13.1 # https://github.com/encode/django-rest-framework 19 | django-cors-headers==3.13.0 # https://github.com/adamchainz/django-cors-headers 20 | # DRF-spectacular for api documentation 21 | drf-spectacular==0.23.1 # https://github.com/tfranzel/drf-spectacular 22 | django-rest-auth 23 | tqdm 24 | paramiko 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /requirements/local.txt: -------------------------------------------------------------------------------- 1 | -r base.txt 2 | 3 | Werkzeug[watchdog]==2.3.6 # https://github.com/pallets/werkzeug 4 | ipdb==0.13.9 # https://github.com/gotcha/ipdb 5 | psycopg2==2.9.3 # https://github.com/psycopg/psycopg2 6 | watchfiles #==0.16.1 # https://github.com/samuelcolvin/watchfiles 7 | 8 | # Testing 9 | # ------------------------------------------------------------------------------ 10 | mypy==0.971 # https://github.com/python/mypy 11 | django-stubs==1.12.0 # https://github.com/typeddjango/django-stubs 12 | pytest==7.1.2 # https://github.com/pytest-dev/pytest 13 | pytest-sugar==0.9.5 # https://github.com/Frozenball/pytest-sugar 14 | djangorestframework-stubs==1.7.0 # https://github.com/typeddjango/djangorestframework-stubs 15 | 16 | # Documentation 17 | # ------------------------------------------------------------------------------ 18 | sphinx==5.1.1 # https://github.com/sphinx-doc/sphinx 19 | sphinx-autobuild==2021.3.14 # https://github.com/GaretJax/sphinx-autobuild 20 | 21 | # Code quality 22 | # ------------------------------------------------------------------------------ 23 | flake8==5.0.4 # https://github.com/PyCQA/flake8 24 | flake8-isort==4.2.0 # https://github.com/gforcada/flake8-isort 25 | coverage==6.4.3 # https://github.com/nedbat/coveragepy 26 | black==22.6.0 # https://github.com/psf/black 27 | pylint-django==2.5.3 # https://github.com/PyCQA/pylint-django 28 | pylint-celery==0.3 # https://github.com/PyCQA/pylint-celery 29 | pre-commit==2.20.0 # https://github.com/pre-commit/pre-commit 30 | 31 | # Django 32 | # ------------------------------------------------------------------------------ 33 | factory-boy==3.2.1 # https://github.com/FactoryBoy/factory_boy 34 | 35 | django-debug-toolbar==3.5.0 # https://github.com/jazzband/django-debug-toolbar 36 | django-extensions==3.2.0 # https://github.com/django-extensions/django-extensions 37 | django-coverage-plugin==2.0.3 # https://github.com/nedbat/django_coverage_plugin 38 | pytest-django==4.5.2 # https://github.com/pytest-dev/pytest-django 39 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 120 3 | exclude = .tox,.git,*/migrations/*,*/static/CACHE/*,docs,node_modules,venv 4 | 5 | [pycodestyle] 6 | max-line-length = 120 7 | exclude = .tox,.git,*/migrations/*,*/static/CACHE/*,docs,node_modules,venv 8 | 9 | [isort] 10 | line_length = 88 11 | known_first_party = analytics,config 12 | multi_line_output = 3 13 | default_section = THIRDPARTY 14 | skip = venv/ 15 | skip_glob = **/migrations/*.py 16 | include_trailing_comma = true 17 | force_grid_wrap = 0 18 | use_parentheses = true 19 | 20 | [mypy] 21 | python_version = 3.10 22 | check_untyped_defs = True 23 | ignore_missing_imports = True 24 | warn_unused_ignores = True 25 | warn_redundant_casts = True 26 | warn_unused_configs = True 27 | plugins = mypy_django_plugin.main, mypy_drf_plugin.main 28 | 29 | [mypy.plugins.django-stubs] 30 | django_settings_module = config.settings.test 31 | 32 | [mypy-*.migrations.*] 33 | # Django migrations should not produce any errors: 34 | ignore_errors = True 35 | 36 | [coverage:run] 37 | include = analytics/* 38 | omit = *migrations*, *tests* 39 | plugins = 40 | django_coverage_plugin 41 | --------------------------------------------------------------------------------