├── .dockerignore ├── .github ├── FUNDING.yml ├── actions │ ├── ssh-checkout │ │ └── action.yml │ ├── ssh-create-python-unit │ │ └── action.yml │ ├── ssh-install-dbt │ │ └── action.yml │ └── ssh-start-service │ │ └── action.yml └── workflows │ └── ssh-to-server.yml ├── .gitignore ├── Docker.md ├── Dockerfile ├── LICENSE ├── LOCALDEV.md ├── README.md ├── dbt ├── .gitignore ├── README.md ├── analysis │ └── .gitkeep ├── data │ ├── .gitkeep │ └── covid │ │ └── covid_raw.csv ├── dbt_project.yml ├── macros │ ├── .gitkeep │ ├── schema_test │ │ ├── test_relationship_where.sql │ │ └── test_unique_combination_of_columns.sql │ └── sql │ │ └── getutcdate.sql ├── models │ ├── data_mart │ │ └── covid │ │ │ ├── .gitkeep │ │ │ ├── Country.sql │ │ │ ├── Country.yml │ │ │ ├── CovidCase.sql │ │ │ ├── CovidCase.yml │ │ │ ├── Date.sql │ │ │ ├── Date.yml │ │ │ └── _covid_ds.yml │ ├── example │ │ ├── MyFirstModel.sql │ │ └── schema.yml │ ├── ods │ │ ├── .gitkeep │ │ └── covid │ │ │ ├── ODS_Country.sql │ │ │ └── ODS_Covid_Case.sql │ └── staging │ │ ├── .gitkeep │ │ └── covid │ │ └── STAG_Covid_Data.sql ├── packages.yml ├── snapshots │ └── .gitkeep └── tests │ ├── .gitkeep │ └── covid │ ├── assert_number_of_country_differ_zero.sql │ ├── assert_number_of_covidcase_differ_zero.sql │ └── assert_number_of_date_differ_zero.sql ├── dbt_profile_template.yml ├── requirements.txt └── services └── api_service ├── .insomia ├── awesome-dbt-api-2021-10-03.json ├── awesome-dbt-api-docs-2021-10-03.png └── awesome-dbt-api-redoc-2021-10-03.png ├── README.md ├── __init__.py ├── aha_dbt ├── __init__.py └── dbt.py ├── api_v1 ├── __init__.py ├── api.py └── endpoints │ ├── __init__.py │ ├── dbt.py │ └── login.py ├── config.py ├── config.yml ├── deps.py ├── libs ├── __init__.py ├── common.py ├── object_view.py └── yml_helper.py ├── main.py ├── schemas ├── __init__.py ├── dbt.py ├── msg.py └── token.py └── storage ├── __init__.py ├── base.py ├── factory.py ├── model.py ├── pickle.py ├── postgres.py ├── sqlserver.py └── type.py /.dockerignore: -------------------------------------------------------------------------------- 1 | /dbt/target/ 2 | /dbt/dbt_modules/ 3 | /dbt/logs/ -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | custom: ['paypal.me/datnguyenit09', 'https://nhantien.momo.vn/0979414691'] 2 | -------------------------------------------------------------------------------- /.github/actions/ssh-checkout/action.yml: -------------------------------------------------------------------------------- 1 | name: 'checkout repo' 2 | inputs: 3 | project-name: 4 | required: true 5 | ssh-host: 6 | required: true 7 | ssh-username: 8 | required: true 9 | ssh-passphrase: 10 | required: true 11 | ssh-key: 12 | required: true 13 | git-user: 14 | required: true 15 | git-token: 16 | required: true 17 | runs: 18 | using: "composite" 19 | steps: 20 | - name: checkout 21 | uses: appleboy/ssh-action@master 22 | with: 23 | host: ${{ inputs.ssh-host }} 24 | username: ${{ inputs.ssh-username }} 25 | passphrase: ${{ inputs.ssh-passphrase }} 26 | key: ${{ inputs.ssh-key }} 27 | script: | 28 | mkdir /root/archive > /dev/null 29 | mv ${{ inputs.project-name }} /root/archive/${{ inputs.project-name }}-$(date +%s) > /dev/null 30 | git clone https://${{ inputs.git-user }}:${{ inputs.git-token }}@github.com/datnguye/${{ inputs.project-name }}.git -------------------------------------------------------------------------------- /.github/actions/ssh-create-python-unit/action.yml: -------------------------------------------------------------------------------- 1 | name: 'create python unit service file(s)' 2 | inputs: 3 | project-name: 4 | required: true 5 | ssh-host: 6 | required: true 7 | ssh-username: 8 | required: true 9 | ssh-passphrase: 10 | required: true 11 | ssh-key: 12 | required: true 13 | unit-service-path: 14 | required: true 15 | default: /etc/systemd/system/myservice.service 16 | unit-service-exec: 17 | required: true 18 | unit-service-port: 19 | required: true 20 | unit-site-path: 21 | required: true 22 | default: /etc/nginx/sites-enabled/myservice.domain.com 23 | unit-site-domain: 24 | required: true 25 | default: domain.com 26 | ssl-report-to: 27 | required: true 28 | default: datnguyen.it09@gmail.com 29 | runs: 30 | using: "composite" 31 | steps: 32 | - name: checkout 33 | uses: appleboy/ssh-action@master 34 | with: 35 | host: ${{ inputs.ssh-host }} 36 | username: ${{ inputs.ssh-username }} 37 | passphrase: ${{ inputs.ssh-passphrase }} 38 | key: ${{ inputs.ssh-key }} 39 | script: | 40 | echo "[Unit]" > ${{ inputs.unit-service-path }} 41 | echo "Description=${{ inputs.unit-site-domain }} service" >> ${{ inputs.unit-service-path }} 42 | echo "After=multi-user.target" >> ${{ inputs.unit-service-path }} 43 | echo "[Service]" >> ${{ inputs.unit-service-path }} 44 | echo "Type=simple" >> ${{ inputs.unit-service-path }} 45 | echo "Restart=always" >> ${{ inputs.unit-service-path }} 46 | echo "WorkingDirectory=/root/${{ inputs.project-name }}" >> ${{ inputs.unit-service-path }} 47 | echo "VIRTUAL_ENV=/root/${{ inputs.project-name }}/venv" >> ${{ inputs.unit-service-path }} 48 | echo "Enviroment=PATH=\$VIRTUAL_ENV/bin:\$PATH" >> ${{ inputs.unit-service-path }} 49 | echo "ExecStart=/root/${{ inputs.project-name }}/venv/bin/python -m ${{ inputs.unit-service-exec }}" >> ${{ inputs.unit-service-path }} 50 | echo "[Install]" >> ${{ inputs.unit-service-path }} 51 | echo "WantedBy=multi-user.target" >> ${{ inputs.unit-service-path }} 52 | # 53 | echo "server {" > ${{ inputs.unit-site-path }} 54 | echo " server_name ${{ inputs.unit-site-domain }};" >> ${{ inputs.unit-site-path }} 55 | echo " location / {" >> ${{ inputs.unit-site-path }} 56 | echo " proxy_pass http://127.0.0.1:${{inputs.unit-service-port}};" >> ${{ inputs.unit-site-path }} 57 | echo " proxy_set_header Host \$host;" >> ${{ inputs.unit-site-path }} 58 | echo " proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;" >> ${{ inputs.unit-site-path }} 59 | echo " proxy_set_header X-Forwarded-Proto \$scheme;" >> ${{ inputs.unit-site-path }} 60 | echo " }" >> ${{ inputs.unit-site-path }} 61 | echo "}" >> ${{ inputs.unit-site-path }} 62 | # 63 | certbot --nginx -d ${{ inputs.unit-site-domain }} --non-interactive --agree-tos -m ${{ inputs.ssl-report-to }} -------------------------------------------------------------------------------- /.github/actions/ssh-install-dbt/action.yml: -------------------------------------------------------------------------------- 1 | name: 'install service' 2 | inputs: 3 | project-name: 4 | required: true 5 | ssh-host: 6 | required: true 7 | ssh-username: 8 | required: true 9 | ssh-passphrase: 10 | required: true 11 | ssh-key: 12 | required: true 13 | runs: 14 | using: "composite" 15 | steps: 16 | - name: checkout 17 | uses: appleboy/ssh-action@master 18 | with: 19 | host: ${{ inputs.ssh-host }} 20 | username: ${{ inputs.ssh-username }} 21 | passphrase: ${{ inputs.ssh-passphrase }} 22 | key: ${{ inputs.ssh-key }} 23 | script: | 24 | cd /root/${{ inputs.project-name }} 25 | python -m pip install virtualenv 26 | python -m venv venv 27 | . venv/bin/activate 28 | python -m pip install --upgrade pip==21.2.4 29 | python -m pip install -r requirements.txt --ignore-installed PyYAML -------------------------------------------------------------------------------- /.github/actions/ssh-start-service/action.yml: -------------------------------------------------------------------------------- 1 | name: 'start unit service' 2 | inputs: 3 | project-name: 4 | required: true 5 | ssh-host: 6 | required: true 7 | ssh-username: 8 | required: true 9 | ssh-passphrase: 10 | required: true 11 | ssh-key: 12 | required: true 13 | unit-service-name: 14 | required: true 15 | default: myservice 16 | unit-service-env-file: 17 | required: true 18 | dbt-host: 19 | required: true 20 | dbt-user: 21 | required: true 22 | dbt-password: 23 | required: true 24 | runs: 25 | using: "composite" 26 | steps: 27 | - name: checkout 28 | uses: appleboy/ssh-action@master 29 | with: 30 | host: ${{ inputs.ssh-host }} 31 | username: ${{ inputs.ssh-username }} 32 | passphrase: ${{ inputs.ssh-passphrase }} 33 | key: ${{ inputs.ssh-key }} 34 | script: | 35 | echo "env_postgres_user_secret=${{ inputs.dbt-user }}" > ${{ inputs.unit-service-env-file }} 36 | echo "env_postgres_password_secret=${{ inputs.dbt-password }}" >> ${{ inputs.unit-service-env-file }} 37 | echo "env_postgres_host_secret=${{ inputs.dbt-host }}" >> ${{ inputs.unit-service-env-file }} 38 | systemctl enable ${{ inputs.unit-service-name }}.service 39 | systemctl daemon-reload 40 | systemctl restart ${{ inputs.unit-service-name }}.service 41 | # systemctl status ${{ inputs.unit-service-name }}.service -------------------------------------------------------------------------------- /.github/workflows/ssh-to-server.yml: -------------------------------------------------------------------------------- 1 | name: SSH deployment 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | 10 | env: 11 | project_name: dbt-postgres 12 | service_name: awesomedbt 13 | service_port: 6000 14 | nginx_site_path: ${{ secrets.NGINX_SITE_PATH }}/awesomedbt.${{ secrets.DOMAIN }} 15 | nginx_unit_path: ${{ secrets.NGINX_UNIT_PATH }}/awesomedbt.service 16 | 17 | 18 | jobs: 19 | # build_and_dbt_test: 20 | # runs-on: ubuntu-20.04 21 | # steps: 22 | # - 23 | 24 | 25 | # build_and_service_test: 26 | # runs-on: ubuntu-20.04 27 | # steps: 28 | # - 29 | 30 | 31 | ssh_publish: 32 | runs-on: ubuntu-20.04 33 | # conditiom: 34 | steps: 35 | - uses: actions/checkout@v1 36 | 37 | - name: Run ssh checkout 38 | uses: ./.github/actions/ssh-checkout 39 | with: 40 | project-name: ${{ env.project_name }} 41 | ssh-host: ${{ secrets.SSH_HOST }} 42 | ssh-username: ${{ secrets.SSH_USER }} 43 | ssh-passphrase: ${{ secrets.SSH_PASSPHRASE }} 44 | ssh-key: ${{ secrets.SSH_PRIVATE_KEY }} 45 | git-user: ${{ secrets.GIHUB_ACTION_USER }} 46 | git-token: ${{ secrets.GIHUB_ACTION_TOKEN }} 47 | 48 | - name: Run install dbt 49 | uses: ./.github/actions/ssh-install-dbt 50 | with: 51 | project-name: ${{ env.project_name }} 52 | ssh-host: ${{ secrets.SSH_HOST }} 53 | ssh-username: ${{ secrets.SSH_USER }} 54 | ssh-passphrase: ${{ secrets.SSH_PASSPHRASE }} 55 | ssh-key: ${{ secrets.SSH_PRIVATE_KEY }} 56 | 57 | - name: Run create service unit files 58 | uses: ./.github/actions/ssh-create-python-unit 59 | with: 60 | project-name: ${{ env.project_name }} 61 | ssh-host: ${{ secrets.SSH_HOST }} 62 | ssh-username: ${{ secrets.SSH_USER }} 63 | ssh-passphrase: ${{ secrets.SSH_PASSPHRASE }} 64 | ssh-key: ${{ secrets.SSH_PRIVATE_KEY }} 65 | unit-service-path: ${{ env.nginx_unit_path }} 66 | unit-service-exec: uvicorn main:app --host 0.0.0.0 --port ${{ env.service_port }} --app-dir \"/root/${{ env.project_name }}/services/api_service\" 67 | unit-service-port: ${{ env.service_port }} 68 | unit-site-path: ${{ env.nginx_site_path }} 69 | unit-site-domain: ${{ env.service_name }}.${{ secrets.DOMAIN }} 70 | 71 | - name: Run start service 72 | uses: ./.github/actions/ssh-start-service 73 | with: 74 | project-name: ${{ env.project_name }} 75 | ssh-host: ${{ secrets.SSH_HOST }} 76 | ssh-username: ${{ secrets.SSH_USER }} 77 | ssh-passphrase: ${{ secrets.SSH_PASSPHRASE }} 78 | ssh-key: ${{ secrets.SSH_PRIVATE_KEY }} 79 | unit-service-name: ${{ env.service_name }} 80 | unit-service-env-file: ${{ secrets.AWESOME_DBT_ENV_FILE }} 81 | dbt-host: ${{ secrets.DBT_HOST }} 82 | dbt-user: ${{ secrets.DBT_USER }} 83 | dbt-password: ${{ secrets.DBT_PASSWORD }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /Docker.md: -------------------------------------------------------------------------------- 1 | ### Dockerfile 2 | 3 | ### Build image:latest 4 | ``` 5 | docker rm awesome-dbt 6 | docker rmi tuiladat/awesome-dbt 7 | docker build --tag tuiladat/awesome-dbt:latest . -f ./Dockerfile 8 | ``` 9 | 10 | ### Run containter 11 | ``` 12 | docker rm awesome-dbt 13 | docker run --publish 8000:8000 --name "awesome-dbt" ^ 14 | --env SERVER="domain.com" ^ 15 | --env USER="dbt_user" ^ 16 | --env PASSWORD="dbt_user" ^ 17 | tuiladat/awesome-dbt 18 | ``` 19 | 20 | 21 | ### Publish to Hub 22 | ``` 23 | docker push tuiladat/awesome-dbt:latest 24 | ``` 25 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1 2 | FROM python:3.8-slim-buster 3 | RUN mkdir app 4 | WORKDIR /app/ 5 | 6 | # Copy project 7 | RUN mkdir dbt 8 | COPY ./dbt ./dbt 9 | COPY ./services/api_service . 10 | 11 | # Create dbt profiles.yml 12 | # Recommend to use environment variables in: profiles.yml/outputs.prod 13 | # [dbt env_var](https://docs.getdbt.com/reference/dbt-jinja-functions/env_var) 14 | RUN mkdir ~/.dbt 15 | COPY dbt_profile_template.yml dbt_profile_template.yml 16 | RUN cp ./dbt_profile_template.yml ~/.dbt/profiles.yml 17 | 18 | # Install requirements 19 | COPY requirements.txt requirements.txt 20 | RUN pip3 install -r requirements.txt 21 | 22 | # Install dbt packages 23 | RUN dbt deps --project-dir ./dbt 24 | 25 | # Arguments 26 | ENV SERVER="dummy" 27 | # TODO: Use Key Vault here 28 | ENV USER="dummy" 29 | ENV PASSWORD="dummy" 30 | 31 | # Entry point 32 | CMD export env_postgres_host_secret=${SERVER} && \ 33 | export env_postgres_user_secret=${USER} && \ 34 | export env_postgres_password_secret=${PASSWORD} && \ 35 | uvicorn main:app --host 0.0.0.0 --port 8000 --app-dir "/app" -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Dat Nguyen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /LOCALDEV.md: -------------------------------------------------------------------------------- 1 | # Requirements: 2 | - python >=3.7.x 3 | - dbt >=0.20.2 4 | - postgresql 12 5 | 6 | # Getting ready? 7 | 8 | ## Install dbt 9 | Installed version: `dbt 0.20.2` 10 | 11 | - Windows: 12 | ``` 13 | python -m venv env 14 | .\env\Scripts\activate 15 | python -m pip install --upgrade pip==21.2.4 16 | python -m pip install -r requirements.txt 17 | ``` 18 | 19 | - Linux (Recommend to install WSL if you're in Windows): 20 | ``` 21 | python -m venv .venv 22 | source .venv/bin/activate 23 | python -m pip install -r requirements.txt 24 | ``` 25 | 26 | 27 | ## Create dbt profile for SQL Server with 5 threads 28 | - Windows: 29 | ``` 30 | mkdir "%userprofile%/.dbt" 31 | xcopy "dbt_profile_template.yml" "%userprofile%/.dbt/profiles.yml" /Y 32 | ``` 33 | 34 | - Linux: 35 | ``` 36 | mkdir ~/.dbt 37 | cp "dbt_profile_template.yml" ~/.dbt/profiles.yml /Y 38 | ``` 39 | 40 | 41 | ## Install dbt packages 42 | You will find the basic packages in [packages.yml](packages.yml) 43 | ``` 44 | dbt deps --project-dir ./dbt 45 | ``` 46 | 47 | 48 | ## Provision postgresql database (Optional) 49 | ``` 50 | --Connect to server with `sysadmin` 51 | CREATE USER dbt_user WITH ENCRYPTED PASSWORD 'dbt_user'; 52 | ALTER USER dbt_user CREATEDB; 53 | 54 | --Connect to server with `dbt_user` 55 | CREATE DATABASE dbt; 56 | ``` 57 | 58 | 59 | ## Let's run the first model 60 | ``` 61 | dbt run --project-dir ./dbt --target dev --models MyFirstModel 62 | ``` 63 | 64 | Off we go! 65 | ``` 66 | Running with dbt=0.20.2 67 | Found 1 model, 2 tests, 0 snapshots, 0 analyses, 448 macros, 0 operations, 1 seed file, 0 sources, 0 exposures 68 | 69 | 12:37:53 | Concurrency: 10 threads (target='dev') 70 | 12:37:53 | 71 | 12:37:53 | 1 of 1 START table model dbo.MyFirstModel............................ [RUN] 72 | 12:38:00 | 1 of 1 OK created table model dbo.MyFirstModel....................... [OK in 7.34s] 73 | 12:38:00 | 74 | 12:38:00 | Finished running 1 table model in 7.55s. 75 | 76 | Completed successfully 77 | 78 | Done. PASS=1 WARN=0 ERROR=0 SKIP=0 TOTAL=1 79 | ``` 80 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dbt-postgres 2 | This is where to start the data transformation with dbt and PostgreSQL. 3 | > Sample data mart: `covid` 4 | 5 | > Start your local development with `.\env\Scripts\activate`. For more details: [LOCALDEV.md](LOCALDEV.md) 6 | 7 | 8 | ## Common commands: 9 | ### Set enviroment variables before any dbt operations as we're gonna use dbt `env_var` within our `profiles.yml`: 10 | - Windows 11 | ``` 12 | set POSTGRES_HOST=domain.com 13 | set POSTGRES_USER=dbt_user 14 | set POSTGRES_PASS=dbt_user 15 | set POSTGRES_PORT=5432 16 | set POSTGRES_DB=dbt 17 | set POSTGRES_SCHEMA=anly 18 | ``` 19 | 20 | ### Seed data 21 | [Covid](/dbt/data/covid/covid_raw.csv) 22 | ``` 23 | dbt seed --project-dir ./dbt --target dev 24 | ``` 25 | 26 | ### Run all models 27 | ``` 28 | dbt run --project-dir ./dbt --target dev --full-refresh --models +exposure:* 29 | ``` 30 | 31 | ### Run all models - DELTA mode 32 | ``` 33 | dbt run --project-dir ./dbt --target dev --models +exposure:* 34 | ``` 35 | 36 | ### Test models 37 | ``` 38 | dbt test --project-dir ./dbt --target dev --models +exposure:* 39 | ``` 40 | 41 | 42 | 43 | ## `Awesome dbt` service 44 | This is the data service via Restful API ([more details](services/api_service/README.md)) 45 | 46 | [![SSH deployment](https://github.com/datnguye/dbt-postgres/actions/workflows/ssh-to-server.yml/badge.svg?branch=main)](https://github.com/datnguye/dbt-postgres/actions/workflows/ssh-to-server.yml) 47 | 48 | [Sample requests](services/api_service/.insomia/awesome-dbt-api-2021-10-03.json) with using [Insomia](https://insomnia.rest/) 49 | 50 | *Swagger*: 51 | 52 | ![Alt text](/services/api_service/.insomia/awesome-dbt-api-docs-2021-10-03.png?raw=true "api redoc") 53 | 54 | *Redoc*: 55 | 56 | ![Alt text](/services/api_service/.insomia/awesome-dbt-api-redoc-2021-10-03.png?raw=true "api redoc") 57 | 58 | 59 | # WHAT NEXT? 60 | - API Header Key 61 | -------------------------------------------------------------------------------- /dbt/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_modules/ 4 | logs/ 5 | -------------------------------------------------------------------------------- /dbt/README.md: -------------------------------------------------------------------------------- 1 | Welcome to your new dbt project! 2 | 3 | ### Using the starter project 4 | 5 | Try running the following commands: 6 | - dbt run 7 | - dbt test 8 | 9 | 10 | ### Resources: 11 | - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) 12 | - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers 13 | - Join the [chat](http://slack.getdbt.com/) on Slack for live discussions and support 14 | - Find [dbt events](https://events.getdbt.com) near you 15 | - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices 16 | -------------------------------------------------------------------------------- /dbt/analysis/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datnguye/dbt-postgres/1edd458a54d5735dd677298eb6b963f6163aaaa6/dbt/analysis/.gitkeep -------------------------------------------------------------------------------- /dbt/data/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datnguye/dbt-postgres/1edd458a54d5735dd677298eb6b963f6163aaaa6/dbt/data/.gitkeep -------------------------------------------------------------------------------- /dbt/data/covid/covid_raw.csv: -------------------------------------------------------------------------------- 1 | iso_code,continent,location,last_updated_date,total_cases,new_cases,new_cases_smoothed,total_deaths,new_deaths,new_deaths_smoothed,total_cases_per_million,new_cases_per_million,new_cases_smoothed_per_million,total_deaths_per_million,new_deaths_per_million,new_deaths_smoothed_per_million,reproduction_rate,icu_patients,icu_patients_per_million,hosp_patients,hosp_patients_per_million,weekly_icu_admissions,weekly_icu_admissions_per_million,weekly_hosp_admissions,weekly_hosp_admissions_per_million,new_tests,total_tests,total_tests_per_thousand,new_tests_per_thousand,new_tests_smoothed,new_tests_smoothed_per_thousand,positive_rate,tests_per_case,tests_units,total_vaccinations,people_vaccinated,people_fully_vaccinated,total_boosters,new_vaccinations,new_vaccinations_smoothed,total_vaccinations_per_hundred,people_vaccinated_per_hundred,people_fully_vaccinated_per_hundred,total_boosters_per_hundred,new_vaccinations_smoothed_per_million,stringency_index,population,population_density,median_age,aged_65_older,aged_70_older,gdp_per_capita,extreme_poverty,cardiovasc_death_rate,diabetes_prevalence,female_smokers,male_smokers,handwashing_facilities,hospital_beds_per_thousand,life_expectancy,human_development_index,excess_mortality 2 | AFG,Asia,Afghanistan,2021-09-17,154487.0,126.0,75.0,7186.0,3.0,3.143,3878.131,3.163,1.883,180.392,0.075,0.079,0.85,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,39835428.0,54.422,18.6,2.581,1.337,1803.987,,597.029,9.59,,,37.746,0.5,64.83,0.511, 3 | OWID_AFR,,Africa,2021-09-17,8137175.0,16028.0,17313.143,205386.0,444.0,468.857,5924.467,11.67,12.605,149.536,0.323,0.341,,,,,,,,,,,,,,,,,,,129280874.0,81743966.0,51893129.0,,131924.0,1433078.0,9.41,5.95,3.78,,1043.0,,1373486472.0,,,,,,,,,,,,,,, 4 | ALB,Europe,Albania,2021-09-17,161324.0,959.0,861.571,2569.0,6.0,4.857,56153.048,333.805,299.893,894.208,2.088,1.691,1.09,,,,,,,,,,,,,,,,,,1632986.0,911372.0,721614.0,,7350.0,7655.0,56.84,31.72,25.12,,2665.0,,2872934.0,104.871,38.0,13.188,8.643,11803.431,1.1,304.195,10.08,7.1,51.2,,2.89,78.57,0.795, 5 | DZA,Africa,Algeria,2021-09-17,201224.0,235.0,237.714,5670.0,19.0,18.714,4510.068,5.267,5.328,127.083,0.426,0.419,0.7,31.0,0.695,,,,,,,,,,,,,,,,9989662.0,5815039.0,4174623.0,,,248708.0,22.39,13.03,9.36,,5574.0,,44616626.0,17.348,29.1,6.211,3.857,13913.839,0.5,278.364,6.73,0.7,30.4,83.741,1.9,76.88,0.748, 6 | AND,Europe,Andorra,2021-09-17,15124.0,11.0,5.857,130.0,0.0,0.0,195516.715,142.203,75.719,1680.585,0.0,0.0,0.96,,,,,,,,,,210856.0,2725.858,,154.0,1.991,0.034,29.1,people tested,,,,,,,,,,,,51.85,77354.0,163.755,,,,,,109.135,7.97,29.0,37.8,,,83.73,0.868, 7 | AGO,Africa,Angola,2021-09-17,52208.0,381.0,323.571,1378.0,7.0,8.0,1538.534,11.228,9.535,40.609,0.206,0.236,1.15,,,,,,,,,,,,,,,,,,2469333.0,1510880.0,958453.0,,,38301.0,7.28,4.45,2.82,,1129.0,,33933611.0,23.89,16.8,2.405,1.362,5819.495,,276.045,3.94,,,26.664,,61.15,0.581, 8 | AIA,North America,Anguilla,2021-09-17,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,18565.0,9498.0,9067.0,,,12.0,122.74,62.8,59.95,,793.0,,15125.0,,,,,,,,,,,,,81.88,, 9 | ATG,North America,Antigua and Barbuda,2021-09-17,2603.0,299.0,77.714,55.0,7.0,1.143,26365.367,3028.523,787.155,557.086,70.902,11.576,1.6,,,,,,,,,,,,,,,,,,79922.0,43949.0,35973.0,,,335.0,80.95,44.52,36.44,,3393.0,,98728.0,231.845,32.1,6.933,4.631,21490.943,,191.511,13.17,,,,3.8,77.02,0.778, 10 | ARG,South America,Argentina,2021-09-17,5237159.0,2308.0,2192.857,114286.0,185.0,143.429,114835.314,50.608,48.083,2505.952,4.056,3.145,0.69,,,,,,,,,22861.0,13699212.0,300.383,0.501,27663.0,0.607,0.032,31.2,tests performed,48310770.0,28929766.0,19398787.0,,373808.0,287363.0,105.93,63.43,42.54,,6301.0,75.93,45605823.0,16.177,31.9,11.198,7.441,18933.907,0.6,191.032,5.5,16.2,27.7,,5.0,76.67,0.845, 11 | ARM,Asia,Armenia,2021-09-17,252082.0,759.0,630.857,5101.0,26.0,17.429,84929.626,255.717,212.544,1718.592,8.76,5.872,1.12,,,,,,,,,5067.0,1633587.0,550.376,1.707,5541.0,1.867,0.112,9.0,tests performed,335721.0,215278.0,120443.0,,,4628.0,11.31,7.25,4.06,,1559.0,,2968128.0,102.931,35.7,11.232,7.571,8787.58,1.8,341.01,7.11,1.5,52.1,94.043,4.2,75.09,0.776, 12 | ABW,North America,Aruba,2021-09-16,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,154447.0,80606.0,73841.0,,386.0,604.0,144.08,75.2,68.88,,5635.0,15.28,107195.0,584.8,41.2,13.085,7.452,35973.781,,,11.62,,,,,76.29,, 13 | OWID_ASI,,Asia,2021-09-17,73701622.0,154445.0,187167.714,1096584.0,2539.0,3028.0,15749.352,33.003,39.996,234.33,0.543,0.647,,,,,,,,,,,,,,,,,,,3937778340.0,2270834789.0,1589381786.0,14912375.0,9737041.0,21532174.0,84.15,48.53,33.96,0.32,4601.0,,4679660580.0,,,,,,,,,,,,,,, 14 | AUS,Oceania,Australia,2021-09-17,84056.0,1856.0,1724.143,1148.0,7.0,9.143,3259.473,71.971,66.858,44.516,0.271,0.355,1.14,,,,,,,,,223818.0,34911907.0,1353.793,8.679,206264.0,7.998,0.009,117.5,tests performed,24054063.0,14768715.0,9285348.0,,302141.0,269150.0,93.28,57.27,36.01,,10437.0,71.76,25788217.0,3.202,37.9,15.504,10.129,44648.71,0.5,107.791,5.07,13.0,16.5,,3.84,83.44,0.944, 15 | AUT,Europe,Austria,2021-09-17,720455.0,2364.0,2077.429,10882.0,12.0,7.143,79669.276,261.416,229.726,1203.352,1.327,0.79,1.21,182.0,20.126,528.0,58.387,,,,,353847.0,81737713.0,9038.711,39.129,346790.0,38.349,0.006,165.4,tests performed,10685286.0,5640345.0,5330472.0,,14572.0,12428.0,118.16,62.37,58.95,,1374.0,49.07,9043072.0,106.749,44.4,19.202,13.748,45436.686,0.7,145.183,6.35,28.4,30.9,,7.37,81.54,0.922, 16 | AZE,Asia,Azerbaijan,2021-09-17,467173.0,0.0,1732.714,6227.0,0.0,28.0,45696.692,0.0,169.486,609.096,0.0,2.739,1.1,,,,,,,,,12852.0,4702934.0,460.019,1.257,13480.0,1.319,0.206,4.9,tests performed,7806235.0,4570812.0,3235423.0,,60384.0,53684.0,76.36,44.71,31.65,,5251.0,58.33,10223344.0,119.309,32.4,6.018,3.871,15847.419,,559.812,7.11,0.3,42.5,83.241,4.7,73.0,0.756, 17 | BHS,North America,Bahamas,2021-09-17,20030.0,235.0,107.857,504.0,35.0,7.286,50464.332,592.068,271.739,1269.796,88.18,18.356,1.03,,,,,,,,,,,,,,,,,,173130.0,105753.0,65915.0,,,1949.0,43.62,26.64,16.61,,4910.0,54.17,396914.0,39.497,34.3,8.996,5.2,27717.847,,235.954,13.17,3.1,20.4,,2.9,73.92,0.814, 18 | BHR,Asia,Bahrain,2021-09-17,274107.0,66.0,80.714,1388.0,0.0,0.0,156785.325,37.751,46.167,793.916,0.0,0.0,0.97,,,,,,,,,16656.0,6219122.0,3557.25,9.527,17460.0,9.987,0.005,208.2,units unclear,2534296.0,1159989.0,1103361.0,,1417.0,2375.0,144.96,66.35,63.11,,1358.0,39.35,1748295.0,1935.907,32.4,2.372,1.387,43290.705,,151.689,16.52,5.8,37.6,,2.0,77.29,0.852, 19 | BGD,Asia,Bangladesh,2021-09-17,1540110.0,1907.0,1842.143,27147.0,38.0,45.0,9260.84,11.467,11.077,163.238,0.228,0.271,0.7,,,,,,,,,28615.0,9339121.0,56.157,0.172,27976.0,0.168,0.068,14.7,tests performed,35861843.0,21559401.0,14302442.0,,378210.0,444344.0,21.56,12.96,8.6,,2672.0,52.78,166303494.0,1265.036,27.5,5.098,3.262,3523.984,14.8,298.003,8.38,1.0,44.7,34.808,0.8,72.59,0.632, 20 | BRB,North America,Barbados,2021-09-17,6358.0,110.0,85.571,57.0,3.0,0.857,22098.795,382.332,297.425,198.118,10.427,2.979,1.36,,,,,,,,,,,,,,,,,,221000.0,123950.0,97050.0,,1030.0,862.0,76.81,43.08,33.73,,2996.0,62.04,287708.0,664.463,39.8,14.952,9.473,16978.068,,170.05,13.57,1.9,14.5,88.469,5.8,79.19,0.814, 21 | BLR,Europe,Belarus,2021-09-17,512460.0,1979.0,1883.0,3978.0,12.0,12.143,54269.535,209.576,199.41,421.27,1.271,1.286,1.13,,,,,,,,,18329.0,8118970.0,859.799,1.941,21962.0,2.326,0.085,11.8,tests performed,3206505.0,1751254.0,1455251.0,,,17321.0,33.96,18.55,15.41,,1834.0,,9442867.0,46.858,40.3,14.799,9.788,17167.967,,443.129,5.18,10.5,46.1,,11.0,74.79,0.823, 22 | BEL,Europe,Belgium,2021-09-17,1219814.0,2341.0,2042.571,25497.0,3.0,6.143,104864.08,201.249,175.594,2191.907,0.258,0.528,1.05,217.0,18.655,681.0,58.544,,,430.063,36.971,53640.0,19384396.0,1666.424,4.611,46076.0,3.961,0.048,20.8,tests performed,16442235.0,8515499.0,8297278.0,,8716.0,10743.0,141.35,73.21,71.33,,924.0,43.06,11632334.0,375.564,41.8,18.571,12.849,42658.576,0.2,114.898,4.29,25.1,31.4,,5.64,81.63,0.931, 23 | BLZ,North America,Belize,2021-09-17,18532.0,198.0,161.0,389.0,0.0,2.286,45767.63,488.992,397.614,960.695,0.0,5.645,1.29,,,,,,,,,,241828.0,597.232,,1098.0,2.712,0.116,8.6,tests performed,275719.0,180196.0,95523.0,,3712.0,2837.0,68.09,44.5,23.59,,7006.0,,404915.0,16.426,25.0,3.853,2.279,7824.362,,176.957,17.11,,,90.083,1.3,74.62,0.716, 24 | BEN,Africa,Benin,2021-09-17,21450.0,0.0,229.857,146.0,0.0,0.714,1722.749,0.0,18.461,11.726,0.0,0.057,0.64,,,,,,,,,,,,,,,,,,174104.0,152669.0,,,,582.0,1.4,1.23,,,47.0,31.48,12451031.0,99.11,18.8,3.244,1.942,2064.236,49.6,235.848,0.99,0.6,12.3,11.035,0.5,61.77,0.545, 25 | BMU,North America,Bermuda,2021-09-17,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,86194.0,43549.0,42645.0,,,40.0,138.82,70.14,68.68,,644.0,,62092.0,1308.82,,,,50669.315,,139.547,13.0,,,,,82.59,, 26 | BTN,Asia,Bhutan,2021-09-17,2596.0,0.0,0.0,3.0,0.0,0.0,3328.632,0.0,0.0,3.847,0.0,0.0,0.18,,,,,,,,,1874.0,1125970.0,1443.736,2.403,2192.0,2.811,0.0,,samples tested,1046117.0,567220.0,478897.0,,,127.0,134.13,72.73,61.4,,163.0,,779900.0,21.188,28.6,4.885,2.977,8708.597,1.5,217.066,9.75,,,79.807,1.7,71.78,0.654, 27 | BOL,South America,Bolivia,2021-09-17,496700.0,668.0,343.857,18648.0,32.0,12.857,41976.057,56.453,29.059,1575.94,2.704,1.087,0.9,,,,,,,,,6790.0,2371896.0,200.449,0.574,5223.0,0.441,0.069,14.5,tests performed,6347661.0,4161019.0,3017983.0,,24066.0,18726.0,53.64,35.16,25.5,,1583.0,44.91,11832936.0,10.202,25.4,6.704,4.393,6885.829,7.1,204.299,6.89,,,25.383,1.1,71.51,0.718, 28 | BIH,Europe,Bosnia and Herzegovina,2021-09-17,225857.0,995.0,745.857,10203.0,42.0,29.714,69207.856,304.891,228.548,3126.437,12.87,9.105,1.29,,,,,,,,,3222.0,1185467.0,363.255,0.987,2901.0,0.889,0.249,4.0,tests performed,1060145.0,634063.0,426082.0,,,6601.0,32.49,19.43,13.06,,2023.0,,3263459.0,68.496,42.5,16.569,10.711,11713.895,0.2,329.635,10.08,30.2,47.7,97.164,3.5,77.4,0.78, 29 | BWA,Africa,Botswana,2021-09-17,172252.0,0.0,1226.714,2343.0,0.0,2.571,71854.299,0.0,511.719,977.374,0.0,1.073,1.08,,,,,,,,,,,,,,,,,,587595.0,365655.0,221940.0,,,2802.0,24.51,15.25,9.26,,1169.0,,2397240.0,4.044,25.8,3.941,2.242,15807.374,,237.372,4.81,5.7,34.4,,1.8,69.59,0.735, 30 | BRA,South America,Brazil,2021-09-17,21080219.0,11202.0,15052.714,589573.0,327.0,532.429,98508.716,52.347,70.342,2755.098,1.528,2.488,0.81,,,,,,,,,,,,,,,,,,218651740.0,145113195.0,78072189.0,,2040092.0,1826426.0,102.18,67.81,36.48,,8535.0,,213993441.0,25.04,33.5,8.552,5.06,14103.452,3.4,177.961,8.11,10.1,17.9,,2.2,75.88,0.765, 31 | BRN,Asia,Brunei,2021-09-17,4814.0,139.0,131.429,24.0,1.0,0.857,10902.947,314.813,297.665,54.356,2.265,1.941,1.24,,,,,,,,,,,,,,,,,,385602.0,237015.0,148587.0,,,3755.0,87.33,53.68,33.65,,8504.0,79.63,441532.0,81.347,32.4,4.591,2.382,71809.251,,201.285,12.79,2.0,30.9,,2.7,75.86,0.838, 32 | BGR,Europe,Bulgaria,2021-09-17,480777.0,1892.0,1550.143,19955.0,79.0,70.857,69711.621,274.336,224.767,2893.432,11.455,10.274,1.08,380.0,55.099,4572.0,662.93,,,,,,4560069.0,661.2,,20694.0,3.001,0.073,13.7,tests performed,2458928.0,,1272680.0,,10446.0,7996.0,35.65,,18.45,,1159.0,,6896655.0,65.18,44.7,20.801,13.272,18563.307,1.5,424.688,5.81,30.1,44.4,,7.454,75.05,0.816, 33 | BFA,Africa,Burkina Faso,2021-09-17,14003.0,19.0,13.857,172.0,0.0,0.143,651.39,0.884,0.645,8.001,0.0,0.007,1.22,,,,,,,,,,,,,,,,,,166160.0,166160.0,102268.0,,,4780.0,0.77,0.77,0.48,,222.0,16.67,21497097.0,70.151,17.6,2.409,1.358,1703.102,43.7,269.048,2.42,1.6,23.9,11.877,0.4,61.58,0.452, 34 | BDI,Africa,Burundi,2021-09-17,14189.0,0.0,20.286,38.0,0.0,0.0,1157.773,0.0,1.655,3.101,0.0,0.0,0.95,,,,,,,,,,,,,,,,,,,,,,,,,,,,,15.74,12255429.0,423.062,17.5,2.562,1.504,702.225,71.7,293.068,6.05,,,6.144,0.8,61.58,0.433, 35 | KHM,Asia,Cambodia,2021-09-17,102834.0,698.0,664.286,2089.0,11.0,10.0,6068.175,41.189,39.199,123.271,0.649,0.59,1.07,,,,,,,,,,2289480.0,135.101,,,,,,tests performed,21612643.0,11608143.0,10114802.0,915581.0,290798.0,151164.0,127.53,68.5,59.69,5.4,8920.0,,16946446.0,90.672,25.6,4.412,2.385,3645.07,,270.892,4.0,2.0,33.7,66.229,0.8,69.82,0.594, 36 | CMR,Africa,Cameroon,2021-09-17,85414.0,0.0,172.0,1368.0,0.0,1.571,3137.422,0.0,6.318,50.249,0.0,0.058,0.16,,,,,,,,,,,,,,,,,,444521.0,363773.0,80748.0,,,1415.0,1.63,1.34,0.3,,52.0,22.22,27224262.0,50.885,18.8,3.165,1.919,3364.926,23.8,244.661,7.2,,,2.735,1.3,59.29,0.563, 37 | CAN,North America,Canada,2021-09-17,1577335.0,5107.0,4454.429,27421.0,44.0,27.286,41434.764,134.155,117.013,720.318,1.156,0.717,1.2,703.0,18.467,2091.0,54.928,,,,,108965.0,41861477.0,1099.653,2.862,90454.0,2.376,0.048,20.6,tests performed,54934350.0,28629817.0,26304533.0,,162254.0,95642.0,144.31,75.21,69.1,,2512.0,66.2,38067913.0,4.037,41.4,16.984,10.797,44017.591,0.5,105.599,7.37,12.0,16.6,,2.5,82.43,0.929, 38 | CPV,Africa,Cape Verde,2021-09-17,36970.0,79.0,77.571,327.0,0.0,1.0,65794.508,140.594,138.052,581.953,0.0,1.78,1.11,,,,,,,,,1130.0,,,2.011,,,,,tests performed,368789.0,272501.0,94997.0,,,5435.0,65.63,48.5,16.91,,9673.0,59.26,561901.0,135.58,25.7,4.46,3.437,6222.554,,182.219,2.42,2.1,16.5,,2.1,72.98,0.665, 39 | CYM,North America,Cayman Islands,2021-09-16,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,104609.0,54167.0,50442.0,,305.0,299.0,157.31,81.46,75.85,,4496.0,,66498.0,256.496,,,,49903.029,,,13.22,,,,,83.92,, 40 | CAF,Africa,Central African Republic,2021-09-17,11309.0,0.0,0.0,100.0,0.0,0.0,2298.583,0.0,0.0,20.325,0.0,0.0,0.11,,,,,,,,,,,,,,,,,,112492.0,102591.0,9901.0,,,240.0,2.29,2.09,0.2,,49.0,,4919987.0,7.479,18.3,3.655,2.251,661.24,,435.727,6.1,,,16.603,1.0,53.28,0.397, 41 | TCD,Africa,Chad,2021-09-17,5025.0,4.0,1.857,174.0,0.0,0.0,297.074,0.236,0.11,10.287,0.0,0.0,0.93,,,,,,,,,,,,,,,,,,104707.0,86821.0,17886.0,,,4229.0,0.62,0.51,0.11,,250.0,25.0,16914985.0,11.833,16.7,2.486,1.446,1768.153,38.4,280.995,6.1,,,5.818,,54.24,0.398, 42 | CHL,South America,Chile,2021-09-17,1646403.0,583.0,463.857,37318.0,25.0,20.0,85694.981,30.345,24.144,1942.395,1.301,1.041,0.92,,,,,,,,,61697.0,20840102.0,1084.724,3.211,49265.0,2.564,0.009,108.6,tests performed,30772997.0,14671991.0,14007210.0,2654389.0,53648.0,100268.0,160.17,76.37,72.91,13.82,5219.0,63.43,19212362.0,24.282,35.4,11.087,6.938,22767.037,1.3,127.993,8.46,34.2,41.5,,2.11,80.18,0.851, 43 | CHN,Asia,China,2021-09-17,95686.0,46.0,67.143,4636.0,0.0,0.0,66.255,0.032,0.046,3.21,0.0,0.0,1.53,,,,,,,,,,,,,,,,,,2165679000.0,1095000000.0,1011584000.0,,4251000.0,5120857.0,149.96,75.82,70.04,,3546.0,76.39,1444216102.0,147.674,38.7,10.641,5.929,15308.712,0.7,261.899,9.74,1.9,48.4,,4.34,76.91,0.761, 44 | COL,South America,Colombia,2021-09-17,4937596.0,1544.0,1546.286,125826.0,44.0,42.429,96313.567,30.118,30.162,2454.383,0.858,0.828,0.85,,,,,,,,,55581.0,24864554.0,485.012,1.084,45978.0,0.897,0.046,21.7,tests performed,37830713.0,24733487.0,15902438.0,,203207.0,159428.0,73.79,48.25,31.02,,3110.0,51.39,51265841.0,44.223,32.2,7.646,4.312,13254.949,4.5,124.24,7.44,4.7,13.5,65.386,1.71,77.29,0.767, 45 | COM,Africa,Comoros,2021-09-17,4104.0,3.0,1.571,147.0,0.0,0.0,4619.25,3.377,1.769,165.456,0.0,0.0,0.8,,,,,,,,,,,,,,,,,,334369.0,174765.0,159604.0,,,4381.0,37.63,19.67,17.96,,4931.0,,888456.0,437.352,20.4,2.963,1.726,1413.89,18.1,261.516,11.88,4.4,23.6,15.574,2.2,64.32,0.554, 46 | COG,Africa,Congo,2021-09-17,13701.0,0.0,0.0,183.0,0.0,0.0,2421.948,0.0,0.0,32.349,0.0,0.0,0.52,,,,,,,,,,,,,,,,,,319231.0,207265.0,111966.0,,,1563.0,5.64,3.66,1.98,,276.0,35.19,5657017.0,15.405,19.0,3.402,2.063,4881.406,37.0,344.094,7.2,1.7,52.3,47.964,,64.57,0.574, 47 | CRI,North America,Costa Rica,2021-09-17,505163.0,2801.0,2197.0,5949.0,30.0,27.857,98298.85,545.042,427.511,1157.606,5.838,5.421,1.11,,,,,,,,,3678.0,1794154.0,349.122,0.716,6832.0,1.329,0.296,3.4,people tested,4872085.0,3144786.0,1727299.0,,,51230.0,94.81,61.19,33.61,,9969.0,,5139053.0,96.079,33.6,9.468,5.694,15524.995,1.3,137.973,8.78,6.4,17.4,83.841,1.13,80.28,0.81, 48 | CIV,Africa,Cote d'Ivoire,2021-09-17,58743.0,212.0,163.571,553.0,7.0,8.286,2171.354,7.836,6.046,20.441,0.259,0.306,1.02,,,,,,,,,3669.0,964928.0,35.667,0.136,3567.0,0.132,0.044,22.6,samples tested,1698973.0,1193877.0,273400.0,,34019.0,33099.0,6.28,4.41,1.01,,1223.0,22.22,27053629.0,76.399,18.7,2.933,1.582,3601.006,28.2,303.74,2.42,,,19.351,,57.78,0.538, 49 | HRV,Europe,Croatia,2021-09-17,389654.0,1394.0,1101.143,8481.0,9.0,9.714,95464.661,341.528,269.778,2077.833,2.205,2.38,1.27,,,469.0,114.904,,,,,10743.0,2695369.0,660.361,2.632,9608.0,2.354,0.109,9.1,people tested,3356659.0,1775653.0,1663891.0,,7701.0,5056.0,82.24,43.5,40.77,,1239.0,,4081657.0,73.726,44.0,19.724,13.053,22669.797,0.7,253.782,5.59,34.3,39.9,,5.54,78.49,0.851, 50 | CUB,North America,Cuba,2021-09-17,784416.0,8291.0,7897.571,6676.0,75.0,76.571,69310.019,732.582,697.82,589.883,6.627,6.766,1.02,,,,,,,,,,,,,,,,,,17220087.0,7461666.0,4444521.0,11355.0,276791.0,241939.0,152.15,65.93,39.27,0.1,21377.0,,11317498.0,110.408,43.1,14.738,9.719,,,190.968,8.27,17.1,53.3,85.198,5.2,78.8,0.783, 51 | CUW,North America,Curacao,2021-09-16,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,185169.0,97150.0,88019.0,,40.0,157.0,112.36,58.95,53.41,,953.0,,164796.0,362.644,41.7,16.367,10.068,,,,11.62,,,,,78.88,, 52 | CYP,Europe,Cyprus,2021-09-17,116494.0,0.0,74.429,534.0,0.0,1.0,131186.198,0.0,83.815,601.348,0.0,1.126,0.7,20.0,22.522,112.0,126.125,13.0,14.64,65.0,73.198,51255.0,12522182.0,14101.477,57.719,57589.0,64.852,0.002,608.0,tests performed,1129665.0,589098.0,540567.0,,1674.0,1753.0,127.21,66.34,60.87,,1974.0,43.52,888005.0,127.657,37.3,13.416,8.563,32415.132,,141.171,9.24,19.6,52.7,,3.4,80.98,0.887, 53 | CZE,Europe,Czechia,2021-09-17,1685435.0,554.0,422.857,30427.0,5.0,2.0,157156.667,51.657,39.429,2837.135,0.466,0.186,1.27,20.0,1.865,128.0,11.935,41.117,3.834,176.504,16.458,76326.0,,,7.117,65838.0,6.139,0.006,166.7,tests performed,11686508.0,6028517.0,5878531.0,,10764.0,10394.0,108.97,56.21,54.81,,969.0,,10724553.0,137.176,43.3,19.027,11.58,32605.906,,227.485,6.82,30.5,38.3,,6.63,79.38,0.9, 54 | COD,Africa,Democratic Republic of Congo,2021-09-17,56387.0,30.0,41.571,1068.0,0.0,0.0,610.394,0.325,0.45,11.561,0.0,0.0,0.95,,,,,,,,,,,,,,,,,,120062.0,85182.0,25452.0,,,1655.0,0.13,0.09,0.03,,18.0,37.96,92377986.0,35.879,17.0,3.02,1.745,808.133,77.1,318.949,6.1,,,4.472,,60.68,0.48, 55 | DNK,Europe,Denmark,2021-09-17,355575.0,340.0,372.571,2624.0,3.0,2.0,61165.754,58.487,64.089,451.379,0.516,0.344,0.73,32.0,5.505,125.0,21.502,,,147.76,25.417,25726.0,40601051.0,6984.163,4.425,39588.0,6.81,0.011,89.2,tests performed,8716356.0,4440213.0,4322533.0,,6014.0,7313.0,149.94,76.38,74.36,,1258.0,24.07,5813302.0,136.52,42.3,19.677,12.325,46682.515,0.2,114.767,6.41,19.3,18.8,,2.5,80.9,0.94, 56 | DJI,Africa,Djibouti,2021-09-17,11960.0,0.0,17.286,157.0,0.0,0.0,11933.781,0.0,17.248,156.656,0.0,0.0,1.58,,,,,,,,,,,,,,,,,,66010.0,39923.0,26087.0,,,266.0,6.59,3.98,2.6,,265.0,,1002197.0,41.285,25.4,4.213,2.38,2705.406,22.5,258.037,6.05,1.7,24.5,,1.4,67.11,0.524, 57 | DMA,North America,Dominica,2021-09-17,2758.0,0.0,68.286,8.0,0.0,0.286,38214.266,0.0,946.152,110.846,0.0,3.959,1.41,,,,,,,,,,,,,,,,,,43971.0,23300.0,20671.0,,,82.0,60.93,32.28,28.64,,1136.0,72.22,72172.0,98.567,,,,9673.367,,227.376,11.62,,,,3.8,75.0,0.742, 58 | DOM,North America,Dominican Republic,2021-09-17,354716.0,273.0,279.143,4027.0,2.0,1.857,32383.172,24.923,25.484,367.638,0.183,0.17,1.08,,,,,,,,,,2010899.0,183.581,,3873.0,0.354,0.073,13.8,samples tested,11585713.0,5957432.0,4802290.0,825991.0,32793.0,24480.0,105.77,54.39,43.84,7.54,2235.0,53.24,10953714.0,222.873,27.6,6.981,4.419,14600.861,1.6,266.653,8.2,8.5,19.1,55.182,1.6,74.08,0.756, 59 | ECU,South America,Ecuador,2021-09-17,505860.0,0.0,112.143,32559.0,0.0,19.0,28278.544,0.0,6.269,1820.111,0.0,1.062,0.71,,,,,,,,,2884.0,1710032.0,95.594,0.161,1812.0,0.101,0.108,9.3,people tested,20426849.0,10699102.0,9727747.0,,47669.0,28753.0,114.19,59.81,54.38,,1607.0,65.74,17888474.0,66.939,28.1,7.104,4.458,10581.936,3.6,140.448,5.55,2.0,12.3,80.635,1.5,77.01,0.759, 60 | EGY,Africa,Egypt,2021-09-17,295639.0,588.0,517.286,16935.0,14.0,12.571,2835.639,5.64,4.962,162.433,0.134,0.121,1.32,,,,,,,,,,,,,,,,,,11989196.0,7743420.0,4245776.0,,,186725.0,11.5,7.43,4.07,,1791.0,,104258327.0,97.999,25.3,5.159,2.891,10550.206,1.3,525.432,17.31,0.2,50.1,89.827,1.6,71.99,0.707, 61 | SLV,North America,El Salvador,2021-09-17,99701.0,0.0,225.571,3078.0,12.0,10.429,15295.083,0.0,34.605,472.195,1.841,1.6,0.87,,,,,,,,,,,,,,,,,,7065918.0,3875388.0,3190530.0,,56061.0,34130.0,108.4,59.45,48.95,,5236.0,,6518500.0,307.811,27.6,8.273,5.417,7292.458,2.2,167.295,8.87,2.5,18.8,90.65,1.3,73.32,0.673, 62 | GNQ,Africa,Equatorial Guinea,2021-09-17,11063.0,0.0,111.286,137.0,0.0,1.143,7630.229,0.0,76.755,94.49,0.0,0.788,1.27,,,,,,,,,2175.0,210877.0,145.443,1.5,1012.0,0.698,0.134,7.4,tests performed,370163.0,210358.0,159805.0,,,1131.0,25.53,14.51,11.02,,780.0,,1449891.0,45.194,22.4,2.846,1.752,22604.873,,202.812,7.78,,,24.64,2.1,58.74,0.592, 63 | ERI,Africa,Eritrea,2021-09-17,6669.0,2.0,2.0,40.0,0.0,0.0,1851.748,0.555,0.555,11.107,0.0,0.0,0.85,,,,,,,,,,,,,,,,,,,,,,,,,,,,,50.93,3601462.0,44.304,19.3,3.607,2.171,1510.459,,311.11,6.05,0.2,11.4,,0.7,66.32,0.459, 64 | EST,Europe,Estonia,2021-09-17,148923.0,532.0,434.0,1322.0,2.0,1.857,112378.772,401.452,327.501,997.594,1.509,1.401,1.11,15.0,11.319,180.0,135.83,9.971,7.525,128.632,97.067,6453.0,1848009.0,1394.526,4.869,5831.0,4.4,0.075,13.4,tests performed,1326099.0,745168.0,580931.0,,3783.0,3427.0,100.07,56.23,43.84,,2586.0,,1325188.0,31.033,42.7,19.452,13.491,29481.252,0.5,255.569,4.02,24.5,39.3,,4.69,78.74,0.892, 65 | SWZ,Africa,Eswatini,2021-09-17,45317.0,43.0,77.143,1194.0,0.0,3.857,38654.212,36.678,65.801,1018.451,0.0,3.29,0.59,,,,,,,,,,,,,,,,,,210277.0,180709.0,172985.0,,,897.0,17.94,15.41,14.76,,765.0,59.26,1172369.0,79.492,21.5,3.163,1.845,7738.975,,333.436,3.94,1.7,16.5,24.097,2.1,60.19,0.611, 66 | ETH,Africa,Ethiopia,2021-09-17,330494.0,1759.0,1243.857,5090.0,31.0,29.429,2803.738,14.922,10.552,43.181,0.263,0.25,0.99,,,,,,,,,9725.0,3358285.0,28.49,0.083,6961.0,0.059,0.17,5.9,tests performed,2960109.0,2414898.0,545211.0,,42676.0,29712.0,2.51,2.05,0.46,,252.0,43.52,117876226.0,104.957,19.8,3.526,2.063,1729.927,26.7,182.634,7.47,0.4,8.5,7.96,0.3,66.6,0.485, 67 | OWID_EUR,,Europe,2021-09-17,57839559.0,128770.0,116373.286,1204783.0,1783.0,1671.714,77351.736,172.211,155.632,1611.217,2.384,2.236,,,,,,,,,,,,,,,,,,,777904615.0,415705743.0,378818966.0,1904616.0,825061.0,1550309.0,104.03,55.59,50.66,0.25,2073.0,,747747396.0,,,,,,,,,,,,,,, 68 | OWID_EUN,,European Union,2021-09-17,37314706.0,52745.0,48703.571,763775.0,544.0,531.429,83442.638,117.948,108.91,1707.943,1.216,1.188,,,,,,,,,,,,,,,,,,,551016108.0,295637602.0,272668276.0,1463931.0,345069.0,900472.0,123.22,66.11,60.97,0.33,2014.0,,447189915.0,,,,,,,,,,,,,,, 69 | FRO,Europe,Faeroe Islands,2021-09-16,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,73208.0,37568.0,35640.0,,157.0,113.0,149.24,76.59,72.66,,2304.0,13.89,49053.0,35.308,,,,,,,,,,,,80.67,, 70 | FJI,Oceania,Fiji,2021-09-17,49719.0,132.0,143.429,566.0,22.0,4.571,55065.96,146.196,158.853,626.87,24.366,5.063,0.67,,,,,,,,,,,,,,,,,,912149.0,569958.0,342191.0,,,6571.0,101.02,63.13,37.9,,7278.0,86.11,902899.0,49.562,28.6,6.224,3.284,8702.975,1.4,412.82,14.49,10.2,34.8,,2.3,67.44,0.743, 71 | FIN,Europe,Finland,2021-09-17,135815.0,501.0,450.429,1052.0,0.0,1.286,24478.4,90.297,81.182,189.606,0.0,0.232,0.94,22.0,3.965,95.0,17.122,,,,,16118.0,6933023.0,1249.562,2.905,15396.0,2.775,0.029,34.3,tests performed,7332815.0,4094729.0,3238086.0,,58850.0,25636.0,132.16,73.8,58.36,,4620.0,36.57,5548361.0,18.136,42.8,21.228,13.264,40585.721,,153.507,5.76,18.3,22.6,,3.28,81.91,0.938, 72 | FRA,Europe,France,2021-09-17,7029959.0,7756.0,8146.429,116618.0,107.0,81.286,104048.5,114.794,120.573,1726.031,1.584,1.203,0.84,2129.0,31.511,10012.0,148.185,939.393,13.904,3377.198,49.985,663977.0,,,9.827,565113.0,8.364,0.018,55.6,people tested,92262410.0,49707676.0,43042655.0,540489.0,286019.0,232974.0,136.56,73.57,63.71,0.8,3448.0,66.67,67564251.0,122.578,42.0,19.718,13.079,38605.671,,86.06,4.77,30.1,35.6,,5.98,82.66,0.901, 73 | PYF,Oceania,French Polynesia,2021-09-14,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,271181.0,146890.0,124291.0,,,1988.0,95.98,51.99,43.99,,7036.0,,282534.0,77.324,32.7,7.775,4.593,,,,22.63,,,,,77.66,, 74 | GAB,Africa,Gabon,2021-09-17,27643.0,352.0,143.571,175.0,1.0,0.714,12130.353,154.465,63.002,76.794,0.439,0.313,1.43,,,,,,,,,,1127524.0,494.782,,2701.0,1.185,0.038,26.0,samples tested,,,,,,,,,,,,58.33,2278829.0,7.859,23.1,4.45,2.976,16562.413,3.4,259.967,7.2,,,,6.3,66.47,0.703, 75 | GMB,Africa,Gambia,2021-09-17,9867.0,0.0,6.714,330.0,0.0,0.286,3967.531,0.0,2.7,132.693,0.0,0.115,0.74,,,,,,,,,,106259.0,42.727,,370.0,0.149,0.043,23.1,samples tested,201835.0,179910.0,165785.0,,,651.0,8.12,7.23,6.67,,262.0,,2486937.0,207.566,17.5,2.339,1.417,1561.767,10.1,331.43,1.91,0.7,31.2,7.876,1.1,62.05,0.496, 76 | GEO,Asia,Georgia,2021-09-17,591766.0,2039.0,2365.143,8446.0,56.0,51.0,148693.405,512.341,594.291,2122.232,14.071,12.815,0.88,,,,,,,,,50000.0,8500000.0,2135.8,12.564,28143.0,7.072,0.087,11.5,tests performed,1610309.0,935094.0,675215.0,,26616.0,28460.0,40.46,23.5,16.97,,7151.0,,3979773.0,65.032,38.7,14.864,10.244,9745.079,4.2,496.218,7.11,5.3,55.5,,2.6,73.77,0.812, 77 | DEU,Europe,Germany,2021-09-17,4137062.0,9904.0,9345.571,92928.0,22.0,46.286,49309.163,118.045,111.389,1107.598,0.262,0.552,0.99,1441.0,17.175,,,,,1286.249,15.331,,72301440.0,861.752,,141624.0,1.688,0.08,12.5,tests performed,105229636.0,55703730.0,52245087.0,328459.0,263701.0,190702.0,125.42,66.39,62.27,0.39,2273.0,62.04,83900471.0,237.016,46.6,21.453,15.957,45229.245,,156.139,8.31,28.2,33.1,,8.0,81.33,0.947, 78 | GHA,Africa,Ghana,2021-09-17,124740.0,866.0,280.143,1116.0,18.0,4.143,3931.032,27.291,8.828,35.169,0.567,0.131,1.09,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,31732128.0,126.719,21.1,3.385,1.948,4227.63,12.0,298.245,4.97,0.3,7.7,41.047,0.9,64.07,0.611, 79 | GIB,Europe,Gibraltar,2021-09-17,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,79269.0,39837.0,39432.0,,36.0,25.0,235.28,118.24,117.04,,742.0,,33691.0,3457.1,,,,,,,,,,,,79.93,, 80 | GRC,Europe,Greece,2021-09-17,627314.0,2231.0,2238.0,14394.0,40.0,41.714,60488.796,215.124,215.799,1387.942,3.857,4.022,0.91,,,,,130.619,12.595,,,155295.0,17406729.0,1678.445,14.974,153156.0,14.768,0.015,68.9,samples tested,11848069.0,6349134.0,5920231.0,,30957.0,26032.0,114.25,61.22,57.09,,2510.0,64.35,10370747.0,83.479,45.3,20.396,14.524,24574.382,1.5,175.695,4.55,35.3,52.0,,4.21,82.24,0.888, 81 | GRL,North America,Greenland,2021-09-17,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,76772.0,40922.0,35850.0,,512.0,324.0,135.0,71.96,63.04,,5697.0,30.09,56868.0,0.137,,,,,,199.941,2.16,,,,,71.7,, 82 | GRD,North America,Grenada,2021-09-17,3490.0,228.0,220.429,48.0,2.0,4.143,30880.857,2017.431,1950.436,424.722,17.697,36.658,1.55,,,,,,,,,,,,,,,,,,48140.0,28511.0,19629.0,,,383.0,42.6,25.23,17.37,,3389.0,,113015.0,317.132,29.4,7.304,5.021,13593.877,,243.964,10.71,,,,3.7,72.4,0.779, 83 | GTM,North America,Guatemala,2021-09-17,525161.0,4068.0,2788.714,12947.0,40.0,48.714,28776.153,222.906,152.807,709.43,2.192,2.669,1.03,,,,,,,,,6773.0,2145502.0,117.563,0.371,10442.0,0.572,0.312,3.2,people tested,5964194.0,4017554.0,1946640.0,,101416.0,90002.0,32.68,22.01,10.67,,4932.0,,18249868.0,157.834,22.9,4.694,3.016,7423.808,8.7,155.898,10.18,,,76.665,0.6,74.3,0.663, 84 | GGY,Europe,Guernsey,2021-09-17,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,101487.0,,,,,76.0,151.36,,,,1133.0,,67052.0,,,,,,,,,,,,,,, 85 | GIN,Africa,Guinea,2021-09-17,30136.0,0.0,21.286,370.0,0.0,1.0,2232.753,0.0,1.577,27.413,0.0,0.074,0.91,,,,,,,,,,,,,,,,,,1387699.0,953942.0,433757.0,,20296.0,12561.0,10.28,7.07,3.21,,931.0,59.26,13497237.0,51.755,19.0,3.135,1.733,1998.926,35.3,336.717,2.42,,,17.45,0.3,61.6,0.477, 86 | GNB,Africa,Guinea-Bissau,2021-09-17,6069.0,16.0,14.429,130.0,0.0,0.714,3011.178,7.939,7.159,64.5,0.0,0.354,0.87,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2015490.0,66.191,19.4,3.002,1.565,1548.675,67.1,382.474,2.42,,,6.403,,58.32,0.48, 87 | GUY,South America,Guyana,2021-09-17,29072.0,241.0,253.0,706.0,3.0,6.714,36784.681,304.936,320.12,893.299,3.796,8.496,1.24,,,,,,,,,,,,,,,,,,518295.0,341600.0,176695.0,,6844.0,2319.0,65.58,43.22,22.36,,2934.0,62.96,790329.0,3.952,26.3,5.305,2.837,7435.047,,373.159,11.62,,,77.159,1.6,69.91,0.682, 88 | HTI,North America,Haiti,2021-09-17,21330.0,12.0,24.0,596.0,0.0,1.143,1848.084,1.04,2.079,51.639,0.0,0.099,1.14,,,,,,,,,,,,,,,,,,57341.0,40198.0,17484.0,,1767.0,707.0,0.5,0.35,0.15,,61.0,53.7,11541683.0,398.448,24.3,4.8,2.954,1653.173,23.5,430.548,6.65,2.9,23.1,22.863,0.7,64.0,0.51, 89 | HND,North America,Honduras,2021-09-17,356707.0,1737.0,832.571,9452.0,52.0,19.0,35447.403,172.613,82.736,939.283,5.167,1.888,1.04,,,,,,,,,,,,,,,,,,4637151.0,2947650.0,1689501.0,,,47940.0,46.08,29.29,16.79,,4764.0,75.0,10062994.0,82.805,24.9,4.652,2.883,4541.795,16.0,240.208,7.21,2.0,,84.169,0.7,75.27,0.634, 90 | HKG,Asia,Hong Kong,2021-09-17,12152.0,3.0,2.0,213.0,0.0,0.143,1608.94,0.397,0.265,28.201,0.0,0.019,,,,,,,,,,,,,,,,,,,8315971.0,4387270.0,3928701.0,,37524.0,36273.0,110.1,58.09,52.02,,4803.0,48.15,7552800.0,7039.714,44.8,16.303,10.158,56054.92,,,8.33,,,,,84.86,0.949, 91 | HUN,Europe,Hungary,2021-09-17,817159.0,479.0,346.714,30123.0,5.0,5.286,84818.898,49.719,35.988,3126.686,0.519,0.549,1.49,,,220.0,22.835,,,,,15682.0,6302559.0,654.189,1.628,12642.0,1.312,0.026,39.0,tests performed,,5855039.0,5564586.0,590000.0,,,,60.77,57.76,6.12,,30.56,9634162.0,108.043,43.4,18.577,11.976,26777.561,0.5,278.296,7.55,26.8,34.8,,7.02,76.88,0.854, 92 | ISL,Europe,Iceland,2021-09-17,11404.0,27.0,29.286,33.0,0.0,0.0,33212.954,78.635,85.292,96.109,0.0,0.0,0.71,,,6.0,17.474,0.0,0.0,4.715,13.731,1762.0,615816.0,1793.5,5.132,1569.0,4.57,,,tests performed,547788.0,281141.0,274175.0,46674.0,1925.0,1769.0,159.54,81.88,79.85,13.59,5152.0,41.67,343360.0,3.404,37.3,14.431,9.207,46482.958,0.2,117.992,5.31,14.3,15.2,,2.91,82.99,0.949, 93 | IND,Asia,India,2021-09-17,33381728.0,0.0,24771.143,444248.0,0.0,275.857,23956.876,0.0,17.777,318.821,0.0,0.198,0.89,,,,,,,,,1430891.0,544444967.0,390.729,1.027,1607946.0,1.154,0.02,51.2,samples tested,777387850.0,586105233.0,191282617.0,,6046209.0,7610446.0,55.79,42.06,13.73,,5462.0,70.83,1393409033.0,450.419,28.2,5.989,3.414,6426.674,21.2,282.28,10.39,1.9,20.6,59.55,0.53,69.66,0.645, 94 | IDN,Asia,Indonesia,2021-09-17,4185144.0,3835.0,3773.286,140138.0,219.0,243.857,15143.714,13.877,13.653,507.082,0.792,0.882,0.59,,,,,,,,,143135.0,23199268.0,83.945,0.518,141506.0,0.512,0.037,27.0,people tested,121542203.0,77417488.0,44124715.0,,1214070.0,1356993.0,43.98,28.01,15.97,,4910.0,68.98,276361788.0,145.725,29.3,5.319,3.053,11188.744,5.7,342.864,6.32,2.8,76.1,64.204,1.04,71.72,0.718, 95 | OWID_INT,,International,2021-09-17,721.0,0.0,0.0,15.0,0.0,0.0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, 96 | IRN,Asia,Iran,2021-09-17,5396013.0,17605.0,19585.714,116436.0,364.0,436.571,63461.034,207.048,230.342,1369.372,4.281,5.134,0.79,,,,,,,,,115179.0,30672916.0,360.736,1.355,109829.0,1.292,0.183,5.5,tests performed,36753231.0,22943141.0,12549387.0,,1706088.0,965517.0,43.22,26.98,14.76,,11355.0,69.91,85028760.0,49.831,32.4,5.44,3.182,19082.62,0.2,270.308,9.59,0.8,21.1,,1.5,76.68,0.783, 97 | IRQ,Asia,Iraq,2021-09-17,1970746.0,3559.0,3803.0,21739.0,56.0,49.286,47857.627,86.427,92.352,527.91,1.36,1.197,0.78,,,,,,,,,32036.0,14895398.0,361.72,0.778,31284.0,0.76,0.127,7.9,samples tested,6672182.0,4203578.0,2468604.0,,,431125.0,16.2,10.21,5.99,,10469.0,,41179351.0,88.125,20.0,3.186,1.957,15663.986,2.5,218.612,8.83,,,94.576,1.4,70.6,0.674, 98 | IRL,Europe,Ireland,2021-09-17,372687.0,1386.0,1262.857,5179.0,0.0,3.429,74793.133,278.151,253.438,1039.354,0.0,0.688,0.95,59.0,11.84,315.0,63.216,27.1,5.439,207.77,41.697,28135.0,7079448.0,1420.747,5.646,24731.0,4.963,0.052,19.1,tests performed,7088705.0,3745660.0,3577773.0,,12816.0,14328.0,142.26,75.17,71.8,,2875.0,48.15,4982904.0,69.874,38.7,13.928,8.678,67335.293,0.2,126.459,3.28,23.0,25.7,,2.96,82.3,0.955, 99 | IMN,Europe,Isle of Man,2021-09-16,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,128915.0,65430.0,63485.0,,81.0,56.0,150.94,76.61,74.33,,656.0,,85410.0,147.872,,,,,,,,,,,,81.4,, 100 | ISR,Asia,Israel,2021-09-17,1211443.0,3040.0,8165.286,7494.0,29.0,24.714,137824.104,345.856,928.953,852.581,3.299,2.812,1.26,233.0,26.508,1097.0,124.804,533.0,60.639,1204.0,136.977,54984.0,25581567.0,2910.378,6.255,155106.0,17.646,0.058,17.4,tests performed,14652030.0,6059158.0,5572156.0,3020716.0,44253.0,39615.0,166.69,68.93,63.39,34.37,4507.0,56.48,8789776.0,402.606,30.6,11.733,7.359,33132.32,0.5,93.32,6.74,15.4,35.4,,2.99,82.97,0.919, 101 | ITA,Europe,Italy,2021-09-17,4627699.0,4544.0,4448.714,130233.0,66.0,57.857,76658.818,75.272,73.694,2157.337,1.093,0.958,0.88,559.0,9.26,4672.0,77.393,,,1224.729,20.288,306267.0,88516407.0,1466.293,5.073,278528.0,4.614,0.017,60.5,tests performed,82019420.0,44110138.0,39371032.0,,158635.0,202981.0,135.87,73.07,65.22,,3362.0,68.98,60367471.0,205.859,47.9,23.021,16.24,35220.084,2.0,113.151,4.78,19.8,27.8,,3.18,83.51,0.892, 102 | JAM,North America,Jamaica,2021-09-17,78590.0,601.0,563.571,1772.0,4.0,11.286,26430.471,202.121,189.534,595.938,1.345,3.795,1.01,,,,,,,,,1564.0,578267.0,194.476,0.526,1714.0,0.576,0.333,3.0,samples tested,702693.0,506797.0,216103.0,,8866.0,13772.0,23.63,17.04,7.27,,4632.0,74.07,2973462.0,266.879,31.4,9.684,6.39,8193.571,,206.537,11.28,5.3,28.6,66.425,1.7,74.47,0.734, 103 | JPN,Asia,Japan,2021-09-17,1669982.0,6164.0,6296.143,17120.0,69.0,57.571,13248.484,48.901,49.949,135.818,0.547,0.457,0.62,,,,,,,,,107105.0,22590267.0,179.216,0.85,101948.0,0.809,0.074,13.6,people tested,149882538.0,82677879.0,67204659.0,,1107527.0,1207116.0,118.91,65.59,53.32,,9576.0,52.31,126050796.0,347.778,48.2,27.049,18.493,39002.223,,79.37,5.72,11.2,33.7,,13.05,84.63,0.919, 104 | JEY,Europe,Jersey,2021-09-12,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,150379.0,76818.0,73561.0,,,90.0,148.78,76.0,72.78,,890.0,,101073.0,,,,,,,,,,,,,,, 105 | JOR,Asia,Jordan,2021-09-17,812253.0,790.0,891.571,10594.0,8.0,9.0,79097.406,76.93,86.821,1031.646,0.779,0.876,1.12,,,,,,,,,28071.0,9706117.0,945.184,2.734,33055.0,3.219,0.031,32.4,tests performed,6770298.0,3615365.0,3154933.0,,3594.0,21260.0,65.93,35.21,30.72,,2070.0,,10269022.0,109.285,23.2,3.81,2.361,8337.49,0.1,208.257,11.75,,,,1.4,74.53,0.729, 106 | KAZ,Asia,Kazakhstan,2021-09-17,931458.0,3117.0,3294.571,15031.0,0.0,86.857,49037.118,164.096,173.445,791.315,0.0,4.573,0.83,,,,,,,,,,,,,,,,,,13287776.0,7228047.0,6059729.0,,73889.0,63260.0,69.95,38.05,31.9,,3330.0,,18994958.0,6.681,30.6,6.991,4.625,24055.588,0.1,466.792,7.11,7.0,43.1,98.999,6.7,73.6,0.825, 107 | KEN,Africa,Kenya,2021-09-17,245781.0,444.0,405.143,4965.0,4.0,9.857,4469.907,8.075,7.368,90.296,0.073,0.179,0.75,,,,,,,,,,,,,,,,,,3290450.0,2439528.0,850922.0,,48699.0,36960.0,5.98,4.44,1.55,,672.0,56.94,54985702.0,87.324,20.0,2.686,1.528,2993.028,36.8,218.637,2.92,1.2,20.4,24.651,1.4,66.7,0.601, 108 | KIR,Oceania,Kiribati,2021-09-17,2.0,0.0,0.0,,,0.0,16.476,0.0,0.0,,,0.0,,,,,,,,,,,,,,,,,,,31332.0,24429.0,6903.0,,,1494.0,25.81,20.12,5.69,,12308.0,37.5,121388.0,143.701,23.2,3.895,2.21,1981.132,,434.657,22.66,35.9,58.9,,1.9,68.37,0.63, 109 | OWID_KOS,Europe,Kosovo,2021-09-17,158675.0,279.0,382.857,2878.0,19.0,17.143,82097.027,144.352,198.087,1489.051,9.83,8.87,0.81,,,,,,,,,6882.0,1138715.0,589.161,3.561,7932.0,4.104,0.056,18.0,tests performed,1110286.0,710945.0,399341.0,,20671.0,19287.0,57.45,36.78,20.66,,9979.0,,1932774.0,168.155,,,,9795.834,0.6,,,,,,,,, 110 | KWT,Asia,Kuwait,2021-09-17,411081.0,63.0,55.429,2437.0,1.0,1.143,94969.612,14.555,12.805,563.006,0.231,0.264,0.58,,,,,,,,,14694.0,4003280.0,924.854,3.395,14390.0,3.324,0.004,260.3,tests performed,,,,,,,,,,,,49.07,4328553.0,232.128,33.7,2.345,1.114,65530.537,,132.235,15.84,2.7,37.0,,2.0,75.49,0.806, 111 | KGZ,Asia,Kyrgyzstan,2021-09-17,177567.0,83.0,83.429,2583.0,2.0,2.429,26789.032,12.522,12.587,389.69,0.302,0.366,0.65,,,,,,,,,,,,,,,,,,1321880.0,760466.0,557094.0,,6355.0,9426.0,19.94,11.47,8.4,,1422.0,,6628347.0,32.333,26.3,4.489,2.882,3393.474,1.4,436.362,7.11,3.6,50.5,89.22,4.5,71.45,0.697, 112 | LAO,Asia,Laos,2021-09-17,18347.0,288.0,201.571,16.0,0.0,0.0,2486.26,39.028,27.316,2.168,0.0,0.0,0.9,,,,,,,,,3430.0,487290.0,66.034,0.465,3650.0,0.495,0.052,19.4,people tested,4402770.0,2650948.0,1860415.0,,13946.0,23461.0,59.66,35.92,25.21,,3179.0,50.0,7379358.0,29.715,24.4,4.029,2.322,6397.36,22.7,368.111,4.0,7.3,51.2,49.839,1.5,67.92,0.613, 113 | LVA,Europe,Latvia,2021-09-17,149614.0,533.0,461.857,2635.0,3.0,4.714,80138.88,285.495,247.388,1411.405,1.607,2.525,1.32,,,234.0,125.339,41.103,22.016,214.323,114.799,,,,,,,,,,1580201.0,888910.0,802871.0,,5700.0,4357.0,84.64,47.61,43.0,,2334.0,37.96,1866934.0,31.212,43.9,19.754,14.136,25063.846,0.7,350.06,4.91,25.6,51.0,,5.57,75.29,0.866, 114 | LBN,Asia,Lebanon,2021-09-17,617036.0,857.0,732.0,8228.0,4.0,8.429,91154.12,126.604,108.138,1215.514,0.591,1.245,0.85,,,,,,,,,,,,,,,,,,2704158.0,1499772.0,1204386.0,,13812.0,15072.0,39.95,22.16,17.79,,2227.0,65.74,6769151.0,594.561,31.1,8.514,5.43,13367.565,,266.591,12.71,26.9,40.7,,2.9,78.93,0.744, 115 | LSO,Africa,Lesotho,2021-09-17,14395.0,0.0,0.0,403.0,0.0,0.0,6667.232,0.0,0.0,186.655,0.0,0.0,0.26,,,,,,,,,,,,,,,,,,,,,,,,,,,,,60.19,2159067.0,73.562,22.2,4.506,2.647,2851.153,59.6,405.126,3.94,0.4,53.9,2.117,,54.33,0.527, 116 | LBR,Africa,Liberia,2021-09-17,5904.0,177.0,25.286,283.0,38.0,5.429,1139.723,34.169,4.881,54.631,7.336,1.048,0.17,,,,,,,,,,,,,,,,,,,,,,,,,,,,,38.89,5180208.0,49.127,19.2,3.057,1.756,752.788,38.6,272.509,2.42,1.5,18.1,1.188,0.8,64.1,0.48, 117 | LBY,Africa,Libya,2021-09-17,329824.0,968.0,1048.143,4490.0,10.0,11.429,47398.462,139.11,150.627,645.25,1.437,1.642,0.87,,,,,,,,,5039.0,,,0.724,,,,,samples tested,1381309.0,1246217.0,135092.0,,48231.0,17883.0,19.85,17.91,1.94,,2570.0,,6958538.0,3.623,29.0,4.424,2.816,17881.509,,341.862,10.43,,,,3.7,72.91,0.724, 118 | LIE,Europe,Liechtenstein,2021-09-17,3402.0,8.0,4.571,60.0,0.0,0.0,88931.876,209.128,119.502,1568.463,0.0,0.0,0.92,,,,,,,0.987,25.808,164.0,64615.0,1689.104,4.287,189.0,4.941,0.026,38.5,tests performed,44789.0,23699.0,21383.0,,10.0,213.0,117.08,61.95,55.9,,5568.0,,38254.0,237.012,,,,,,,7.77,,,,2.397,82.49,0.919, 119 | LTU,Europe,Lithuania,2021-09-17,313692.0,1211.0,1019.143,4763.0,12.0,15.429,116620.109,450.209,378.883,1770.723,4.461,5.736,1.19,,,761.0,282.914,53.911,20.042,308.063,114.527,22720.0,4812417.0,1789.094,8.447,16061.0,5.971,0.061,16.4,tests performed,3306744.0,1716628.0,1590116.0,,11039.0,8341.0,122.93,63.82,59.12,,3101.0,30.56,2689862.0,45.135,43.5,19.002,13.778,29524.265,0.7,342.989,3.67,21.3,38.0,,6.56,75.93,0.882, 120 | LUX,Europe,Luxembourg,2021-09-17,77189.0,91.0,79.571,834.0,0.0,0.0,121593.097,143.349,125.346,1313.771,0.0,0.0,1.07,9.0,14.177,37.0,58.285,,,13.181,20.763,6400.0,3493106.0,5502.566,10.082,5294.0,8.339,0.015,68.5,tests performed,779620.0,416918.0,,506.0,37.0,949.0,122.81,65.68,,0.08,1495.0,,634814.0,231.447,39.7,14.312,9.842,94277.965,0.2,128.275,4.42,20.9,26.0,,4.51,82.25,0.916, 121 | MAC,Asia,Macao,2021-09-16,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,633071.0,338450.0,296643.0,,,2782.0,96.15,51.41,45.06,,4225.0,31.48,658391.0,20546.766,39.2,9.798,4.991,104861.851,,,,,,,,84.24,, 122 | MDG,Africa,Madagascar,2021-09-17,42898.0,0.0,1.0,958.0,0.0,0.0,1509.041,0.0,0.035,33.7,0.0,0.0,0.8,,,,,,,,,,,,,,,,,,283091.0,210666.0,86090.0,,,1230.0,1.0,0.74,0.3,,43.0,,28427333.0,43.951,19.6,2.929,1.686,1416.44,77.6,405.994,3.94,,,50.54,0.2,67.04,0.528, 123 | MWI,Africa,Malawi,2021-09-17,61287.0,37.0,33.571,2252.0,1.0,2.857,3119.299,1.883,1.709,114.619,0.051,0.145,0.57,,,,,,,,,,,,,,,,,,974830.0,742169.0,469604.0,,6530.0,5898.0,4.96,3.78,2.39,,300.0,39.81,19647681.0,197.519,18.1,2.979,1.783,1095.042,71.4,227.349,3.94,4.4,24.7,8.704,1.3,64.26,0.483, 124 | MYS,Asia,Malaysia,2021-09-17,2067327.0,17577.0,18053.857,22743.0,388.0,416.571,63074.039,536.273,550.822,693.888,11.838,12.71,0.95,,,,,,,,,,,,,,,,,,39923416.0,21863951.0,18135561.0,,238002.0,235409.0,121.81,66.71,55.33,,7182.0,,32776195.0,96.254,29.9,6.293,3.407,26808.164,0.1,260.942,16.74,1.0,42.4,,1.9,76.16,0.81, 125 | MDV,Asia,Maldives,2021-09-17,83332.0,84.0,107.714,229.0,2.0,0.286,153290.902,154.52,198.143,421.25,3.679,0.526,0.94,,,,,,,,,5127.0,1460154.0,2685.983,9.431,5101.0,9.383,0.022,46.2,samples tested,720666.0,390644.0,330022.0,,1751.0,2658.0,132.57,71.86,60.71,,4889.0,,543620.0,1454.433,30.6,4.12,2.875,15183.616,,164.905,9.19,2.1,55.0,95.803,,78.92,0.74, 126 | MLI,Africa,Mali,2021-09-17,15029.0,9.0,7.429,545.0,0.0,0.286,720.618,0.432,0.356,26.132,0.0,0.014,0.85,,,,,,,,,,,,,,,,,,370529.0,284052.0,86477.0,,,3768.0,1.78,1.36,0.41,,181.0,50.93,20855724.0,15.196,16.4,2.519,1.486,2014.306,,268.024,2.42,1.6,23.0,52.232,0.1,59.31,0.434, 127 | MLT,Europe,Malta,2021-09-17,36927.0,27.0,36.143,453.0,1.0,1.0,71763.668,52.472,70.24,880.357,1.943,1.943,1.0,,,,,,,6.0,11.66,2601.0,1211456.0,2354.335,5.055,3465.0,6.734,0.016,62.4,tests performed,810125.0,418251.0,417636.0,4477.0,1157.0,879.0,157.44,81.28,81.16,0.87,1708.0,43.52,514564.0,1454.037,42.4,19.426,11.324,36513.323,0.2,168.711,8.83,20.9,30.2,,4.485,82.53,0.895, 128 | MHL,Oceania,Marshall Islands,2021-09-17,4.0,0.0,0.0,,,0.0,67.094,0.0,0.0,,,0.0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,59618.0,295.15,,,,3819.202,,557.793,30.53,,,82.502,2.7,73.7,0.704, 129 | MRT,Africa,Mauritania,2021-09-17,35328.0,116.0,90.857,763.0,2.0,2.286,7398.364,24.293,19.027,159.787,0.419,0.479,0.74,,,,,,,,,,,,,,,,,,333089.0,311544.0,21545.0,,,4503.0,6.98,6.52,0.45,,943.0,44.44,4775110.0,4.289,20.3,3.138,1.792,3597.633,6.0,232.347,2.42,,,15.95,,64.92,0.546, 130 | MUS,Africa,Mauritius,2021-09-17,14243.0,170.0,208.571,50.0,2.0,1.857,11184.771,133.498,163.787,39.264,1.571,1.458,1.03,,,,,,,,,,,,,,,,,,1625865.0,842642.0,783223.0,,,5394.0,127.68,66.17,61.51,,4236.0,,1273428.0,622.962,37.4,10.945,5.884,20292.745,0.5,224.644,22.02,3.2,40.7,,3.4,74.99,0.804, 131 | MEX,North America,Mexico,2021-09-17,3552983.0,3754.0,10426.286,270538.0,190.0,626.857,27275.621,28.819,80.041,2076.872,1.459,4.812,0.85,,,,,,,,,7450.0,9795982.0,75.202,0.057,18285.0,0.14,0.358,2.8,people tested,94144251.0,61573170.0,40997724.0,,555532.0,666333.0,72.27,47.27,31.47,,5115.0,32.41,130262220.0,66.444,29.3,6.857,4.321,17336.469,2.5,152.783,13.06,6.9,21.4,87.847,1.38,75.05,0.779, 132 | FSM,Oceania,Micronesia (country),2021-09-17,1.0,0.0,0.0,,,0.0,8.602,0.0,0.0,,,0.0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,116255.0,150.777,23.0,4.81,2.392,3299.464,16.0,454.343,12.02,,,,,67.88,0.62, 133 | MDA,Europe,Moldova,2021-09-17,280175.0,1032.0,893.857,6578.0,9.0,12.286,69625.561,256.46,222.13,1634.682,2.237,3.053,1.42,,,,,,,,,,1673626.0,415.908,,7725.0,1.92,0.113,8.9,tests performed,1357495.0,,750553.0,,4741.0,4267.0,33.73,,18.65,,1060.0,29.63,4024025.0,123.655,37.6,10.864,6.955,5189.972,0.2,408.502,5.72,5.9,44.6,86.979,5.8,71.9,0.75, 134 | MCO,Europe,Monaco,2021-09-17,3290.0,2.0,2.429,33.0,0.0,0.0,83248.988,50.607,61.452,835.02,0.0,0.0,0.7,,,,,,,,,,,,,,,,,,49980.0,26672.0,23308.0,,,103.0,126.47,67.49,58.98,,2606.0,,39520.0,19347.5,,,,,,,5.46,,,,13.8,86.75,, 135 | MNG,Asia,Mongolia,2021-09-17,269476.0,2796.0,2866.857,1083.0,9.0,10.857,80941.176,839.821,861.104,325.295,2.703,3.261,1.16,,,,,,,,,1106416.0,4973007.0,1493.718,332.329,166705.0,50.072,0.277,3.6,samples tested,4367512.0,2248058.0,2119454.0,,1343.0,1617.0,131.18,67.52,63.66,,486.0,53.7,3329282.0,1.98,28.6,4.031,2.421,11840.846,0.5,460.043,4.82,5.5,46.5,71.18,7.0,69.87,0.737, 136 | MNE,Europe,Montenegro,2021-09-17,125128.0,643.0,629.286,1832.0,10.0,5.714,199232.228,1023.802,1001.966,2916.961,15.922,9.098,1.11,,,,,,,,,,,,,,,,,,432167.0,229399.0,202768.0,,5505.0,2765.0,68.81,36.53,32.29,,4403.0,,628051.0,46.28,39.1,14.762,9.395,16409.288,1.0,387.305,10.08,44.0,47.9,,3.861,76.88,0.829, 137 | MSR,North America,Montserrat,2021-09-10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2840.0,1466.0,1374.0,,,0.0,57.02,29.43,27.58,,0.0,,4981.0,,,,,,,,,,,,,74.16,, 138 | MAR,Africa,Morocco,2021-09-17,915835.0,2412.0,2322.0,13828.0,53.0,56.0,24523.771,64.587,62.177,370.279,1.419,1.5,0.71,,,,,,,,,23052.0,8533903.0,228.517,0.617,22181.0,0.594,0.106,9.4,people tested,37238088.0,20362745.0,16875343.0,,267550.0,230801.0,99.71,54.53,45.19,,6180.0,,37344787.0,80.08,29.6,6.769,4.209,7485.013,1.0,419.146,7.14,0.8,47.1,,1.1,76.68,0.686, 139 | MOZ,Africa,Mozambique,2021-09-17,149906.0,102.0,143.571,1903.0,1.0,1.571,4660.815,3.171,4.464,59.167,0.031,0.049,0.64,,,,,,,,,705.0,878110.0,27.302,0.022,1844.0,0.057,0.093,10.7,units unclear,3427486.0,1864229.0,1563257.0,,,145942.0,10.66,5.8,4.86,,4538.0,,32163045.0,37.728,17.7,3.158,1.87,1136.103,62.9,329.942,3.3,5.1,29.1,12.227,0.7,60.85,0.456, 140 | MMR,Asia,Myanmar,2021-09-17,442928.0,2187.0,2201.714,16944.0,75.0,84.429,8081.741,39.904,40.173,309.163,1.368,1.54,0.9,,,,,,,,,24822.0,4002846.0,73.037,0.453,24501.0,0.447,0.094,10.6,samples tested,8184418.0,4944654.0,3239764.0,,,91940.0,14.93,9.02,5.91,,1678.0,,54806014.0,81.721,29.1,5.732,3.12,5591.597,6.4,202.104,4.61,6.3,35.2,79.287,0.9,67.13,0.583, 141 | NAM,Africa,Namibia,2021-09-17,126628.0,91.0,86.143,3461.0,5.0,5.0,48941.308,35.171,33.294,1337.665,1.932,1.932,0.79,,,,,,,,,790.0,681868.0,263.54,0.305,1299.0,0.502,0.059,17.1,samples tested,374178.0,232051.0,142127.0,,,3458.0,14.46,8.97,5.49,,1337.0,,2587344.0,3.078,22.0,3.552,2.085,9541.808,13.4,243.811,3.94,9.7,34.2,44.6,,63.71,0.646, 142 | NPL,Asia,Nepal,2021-09-17,783075.0,1086.0,1075.286,11012.0,10.0,12.571,26388.445,36.597,36.236,371.088,0.337,0.424,0.81,,,,,,,,,11208.0,4068591.0,137.105,0.378,9871.0,0.333,0.107,9.3,tests performed,11590504.0,6083124.0,5507380.0,,60159.0,104183.0,39.06,20.5,18.56,,3511.0,52.31,29674920.0,204.43,25.0,5.809,3.212,2442.804,15.0,260.797,7.26,9.5,37.8,47.782,0.3,70.78,0.602, 143 | NLD,Europe,Netherlands,2021-09-17,2018461.0,2236.0,2186.857,18495.0,11.0,7.429,117536.246,130.204,127.342,1076.975,0.641,0.433,1.02,205.0,11.937,435.0,25.33,73.99,4.308,350.218,20.393,,12823298.0,746.709,,20975.0,1.221,0.085,11.8,people tested,22196529.0,12068854.0,10924705.0,,,9605.0,129.25,70.28,63.62,,559.0,41.67,17173094.0,508.544,43.2,18.779,11.881,48472.545,,109.361,5.29,24.4,27.3,,3.32,82.28,0.944, 144 | NCL,Oceania,New Caledonia,2021-09-14,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,176813.0,101168.0,75645.0,,,2649.0,61.35,35.1,26.25,,9191.0,,288217.0,15.342,33.4,9.954,6.489,,,,23.36,,,,,77.55,, 145 | NZL,Oceania,New Zealand,2021-09-17,4039.0,24.0,21.143,27.0,0.0,0.0,830.96,4.938,4.35,5.555,0.0,0.0,0.9,,,,,,,,,15418.0,3206325.0,659.651,3.172,13177.0,2.711,0.002,612.4,tests performed,4569255.0,3015345.0,1553910.0,,62155.0,57654.0,94.01,62.04,31.97,,11861.0,81.02,4860642.0,18.206,37.9,15.322,9.72,36085.843,,128.797,8.08,14.8,17.2,,2.61,82.29,0.931, 146 | NIC,North America,Nicaragua,2021-09-17,13025.0,0.0,96.429,202.0,0.0,0.143,1943.34,0.0,14.387,30.139,0.0,0.021,0.01,,,,,,,,,,,,,,,,,,769105.0,490472.0,278633.0,,,4836.0,11.48,7.32,4.16,,722.0,2.78,6702379.0,51.667,27.3,5.445,3.519,5321.444,3.2,137.016,11.47,,,,0.9,74.48,0.66, 147 | NER,Africa,Niger,2021-09-17,5947.0,0.0,5.0,201.0,0.0,0.286,236.642,0.0,0.199,7.998,0.0,0.011,1.03,,,,,,,,,,,,,,,,,,,,,,,,,,,,,19.44,25130810.0,16.955,15.1,2.553,1.378,926.0,44.5,238.339,2.42,0.1,15.4,8.978,0.3,62.42,0.394, 148 | NGA,Africa,Nigeria,2021-09-17,201294.0,938.0,436.429,2649.0,9.0,8.714,952.192,4.437,2.064,12.531,0.043,0.041,0.96,,,,,,,,,,2942578.0,13.919,,10226.0,0.048,0.051,19.7,samples tested,5858737.0,4146028.0,1712709.0,,175033.0,71926.0,2.77,1.96,0.81,,340.0,54.63,211400704.0,209.588,18.1,2.751,1.447,5338.454,,181.013,2.42,0.6,10.8,41.949,,54.69,0.539, 149 | OWID_NAM,,North America,2021-09-17,50443159.0,236269.0,179239.429,1025098.0,3151.0,2833.0,84553.707,396.038,300.444,1718.287,5.282,4.749,,,,,,,,,,,,,,,,,,,593795090.0,334200308.0,269575541.0,837346.0,1210667.0,1661972.0,99.53,56.02,45.19,0.14,2786.0,,596581283.0,,,,,,,,,,,,,,, 150 | MKD,Europe,North Macedonia,2021-09-17,186010.0,560.0,508.571,6396.0,29.0,26.0,89313.623,268.887,244.193,3071.071,13.924,12.484,0.88,,,,,,,,,,1222443.0,586.962,,6101.0,2.929,0.081,12.3,tests performed,1435392.0,767653.0,667739.0,,23834.0,11046.0,68.92,36.86,32.06,,5304.0,,2082661.0,82.6,39.1,13.26,8.16,13111.214,5.0,322.688,10.08,,,,4.28,75.8,0.774, 151 | NOR,Europe,Norway,2021-09-17,181195.0,763.0,935.143,841.0,0.0,2.0,33151.72,139.6,171.095,153.871,0.0,0.366,0.95,,,116.0,21.224,22.402,4.099,82.48,15.091,24735.0,7537286.0,1379.034,4.526,22658.0,4.146,0.052,19.1,people tested,7586705.0,4046955.0,3539750.0,,45344.0,25749.0,138.81,74.04,64.76,,4711.0,38.89,5465629.0,14.462,39.7,16.821,10.813,64800.057,0.2,114.316,5.31,19.6,20.7,,3.6,82.4,0.957, 152 | OWID_OCE,,Oceania,2021-09-17,156395.0,2012.0,1918.143,1946.0,29.0,15.143,3618.583,46.553,44.381,45.025,0.671,0.35,,,,,,,,,,,,,,,,,,,30590164.0,19025370.0,11564794.0,,302141.0,338514.0,70.78,44.02,26.76,,7832.0,,43219954.0,,,,,,,,,,,,,,, 153 | OMN,Asia,Oman,2021-09-17,303309.0,0.0,55.0,4092.0,0.0,1.143,58067.618,0.0,10.53,783.401,0.0,0.219,0.78,,,,,,,,,,,,,,,,,,4336872.0,2710021.0,1626851.0,,,37565.0,83.03,51.88,31.15,,7192.0,64.81,5223376.0,14.98,30.7,2.355,1.53,37960.709,,266.342,12.61,0.5,15.6,97.4,1.6,77.86,0.813, 154 | PAK,Asia,Pakistan,2021-09-17,1221261.0,2512.0,2842.0,27135.0,63.0,67.571,5423.008,11.155,12.62,120.493,0.28,0.3,0.9,,,,,,,,,57626.0,18740356.0,83.217,0.256,56108.0,0.249,0.053,19.0,tests performed,71311567.0,53396378.0,23241257.0,,886043.0,693617.0,31.67,23.71,10.32,,3080.0,62.96,225199929.0,255.573,23.5,4.495,2.78,5034.708,4.0,423.031,8.35,2.8,36.7,59.607,0.6,67.27,0.557, 155 | PLW,Oceania,Palau,2021-09-17,5.0,0.0,0.429,,,0.0,275.118,0.0,23.582,,,0.0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,18174.0,47.237,,,,13240.405,,,15.89,7.7,22.7,,4.8,73.7,0.826, 156 | PSE,Asia,Palestine,2021-09-17,381854.0,2219.0,2168.429,3890.0,19.0,15.0,73113.506,424.871,415.189,744.817,3.638,2.872,1.15,,,,,,,,,11228.0,,,2.15,10163.0,1.946,0.215,4.7,tests performed,1810211.0,1271801.0,538410.0,,28827.0,19486.0,34.66,24.35,10.31,,3731.0,,5222756.0,778.202,20.4,3.043,1.726,4449.898,1.0,265.91,10.59,,,,,74.05,0.708, 157 | PAN,North America,Panama,2021-09-17,463783.0,324.0,313.286,7169.0,3.0,6.714,105848.274,73.946,71.501,1636.167,0.685,1.532,0.71,,,,,,,,,8191.0,3772110.0,860.901,1.869,6761.0,1.543,0.048,21.0,tests performed,5042581.0,2891906.0,2150675.0,,46443.0,31921.0,115.09,66.0,49.08,,7285.0,77.31,4381583.0,55.133,29.7,7.918,5.03,22267.037,2.2,128.346,8.33,2.4,9.9,,2.3,78.51,0.815, 158 | PNG,Oceania,Papua New Guinea,2021-09-17,18542.0,0.0,29.0,204.0,0.0,1.429,2033.336,0.0,3.18,22.371,0.0,0.157,0.52,,,,,,,,,,,,,,,,,,154710.0,114419.0,40291.0,,,2895.0,1.7,1.25,0.44,,317.0,49.07,9119005.0,18.22,22.6,3.808,2.142,3823.194,,561.494,17.65,23.5,48.8,,,64.5,0.555, 159 | PRY,South America,Paraguay,2021-09-17,459524.0,0.0,55.857,16120.0,0.0,13.143,63649.148,0.0,7.737,2232.798,0.0,1.82,0.87,,,,,,,,,2232.0,1809486.0,250.634,0.309,2220.0,0.307,0.025,39.7,tests performed,4401222.0,2550991.0,1850231.0,,1.0,18596.0,60.96,35.33,25.63,,2576.0,49.07,7219641.0,17.144,26.5,6.378,3.833,8827.01,1.7,199.128,8.27,5.0,21.6,79.602,1.3,74.25,0.728, 160 | PER,South America,Peru,2021-09-17,2164380.0,0.0,724.857,198891.0,0.0,31.143,64880.634,0.0,21.729,5962.065,0.0,0.934,0.93,,,,,,,,,,,,,,,,,,21271177.0,12354190.0,8916987.0,,22384.0,110510.0,63.76,37.03,26.73,,3313.0,64.81,33359415.0,25.129,29.1,7.151,4.455,12236.706,3.5,85.755,5.95,4.8,,,1.6,76.74,0.777, 161 | PHL,Asia,Philippines,2021-09-17,2324475.0,20283.0,20672.143,36328.0,310.0,204.143,20932.37,182.653,186.157,327.141,2.792,1.838,1.09,,,,,,,,,69289.0,18837782.0,169.638,0.624,71702.0,0.646,0.297,3.4,people tested,40518845.0,,17977462.0,,488457.0,398676.0,36.49,,16.19,,3590.0,74.54,111046910.0,351.873,25.2,4.803,2.661,7599.188,,370.437,7.07,7.8,40.8,78.463,1.0,71.23,0.718, 162 | PCN,Oceania,Pitcairn,2021-09-07,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,94.0,47.0,47.0,,,0.0,200.0,100.0,100.0,,0.0,,47.0,,,,,,,,,,,,,,, 163 | POL,Europe,Poland,2021-09-17,2896599.0,652.0,565.286,75473.0,9.0,8.0,76635.685,17.25,14.956,1996.799,0.238,0.212,1.32,,,675.0,17.859,,,10.953,0.29,40596.0,19961353.0,528.12,1.074,37441.0,0.991,0.015,68.4,people tested,36878694.0,19601546.0,19191249.0,,37503.0,31613.0,97.57,51.86,50.77,,836.0,38.89,37797000.0,124.027,41.8,16.763,10.202,27216.445,,227.331,5.91,23.3,33.1,,6.62,78.73,0.88, 164 | PRT,Europe,Portugal,2021-09-17,1060432.0,1023.0,997.429,17895.0,7.0,7.429,104291.899,100.611,98.096,1759.946,0.688,0.731,0.76,120.0,11.802,569.0,55.96,,,73.08,7.187,61620.0,17806549.0,1751.247,6.06,54623.0,5.372,0.02,50.1,tests performed,15550992.0,8846756.0,8319395.0,,55441.0,54982.0,152.94,87.01,81.82,,5407.0,52.78,10167923.0,112.371,46.2,21.502,14.924,27936.896,0.5,127.842,9.85,16.3,30.0,,3.39,82.05,0.864, 165 | QAT,Asia,Qatar,2021-09-17,235304.0,117.0,134.571,604.0,0.0,0.0,80294.173,39.925,45.921,206.106,0.0,0.0,0.79,,,,,,,,,5671.0,2584515.0,881.929,1.935,5246.0,1.79,0.026,38.2,people tested,4633897.0,2360308.0,2218292.0,,9718.0,10341.0,158.13,80.54,75.7,,3529.0,50.0,2930524.0,227.322,31.9,1.307,0.617,116935.6,,176.69,16.52,0.8,26.9,,1.2,80.23,0.848, 166 | ROU,Europe,Romania,2021-09-17,1139505.0,4478.0,3372.0,35359.0,73.0,63.571,59573.326,234.11,176.288,1848.569,3.816,3.324,1.57,,,,,,,14691.566,768.075,,9335522.0,488.061,,13557.0,0.709,0.225,4.4,tests performed,9912393.0,5347116.0,5237170.0,,11173.0,8870.0,51.82,27.95,27.38,,464.0,,19127772.0,85.129,43.0,17.85,11.69,23313.199,5.7,370.946,9.74,22.9,37.1,,6.892,76.05,0.828, 167 | RUS,Europe,Russia,2021-09-17,7130245.0,19589.0,18515.571,193111.0,771.0,763.286,48866.741,134.252,126.895,1323.476,5.284,5.231,0.99,,,,,,,,,,185649676.0,1272.34,,420297.0,2.88,0.044,23.0,tests performed,87267626.0,46745074.0,40522552.0,,311372.0,335976.0,59.81,32.04,27.77,,2303.0,52.31,145912022.0,8.823,39.6,14.178,9.393,24765.954,0.1,431.297,6.18,23.4,58.3,,8.05,72.58,0.824, 168 | RWA,Africa,Rwanda,2021-09-17,94893.0,415.0,409.714,1199.0,6.0,5.857,7147.432,31.258,30.86,90.31,0.452,0.441,0.99,,,,,,,,,14495.0,2644414.0,199.18,1.092,12229.0,0.921,0.035,28.5,samples tested,2882134.0,1827371.0,1054763.0,,,39280.0,21.71,13.76,7.94,,2959.0,,13276517.0,494.869,20.3,2.974,1.642,1854.211,56.0,191.375,4.28,4.7,21.0,4.617,,69.02,0.543, 169 | KNA,North America,Saint Kitts and Nevis,2021-09-17,1589.0,0.0,33.857,9.0,0.0,0.571,29675.419,0.0,632.3,168.08,0.0,10.672,1.17,,,,,,,,,257.0,35529.0,663.523,4.8,279.0,5.21,0.109,9.2,people tested,46884.0,24968.0,21916.0,,,42.0,87.56,46.63,40.93,,784.0,,53546.0,212.865,,,,24654.385,,,12.84,,,,2.3,76.23,0.779, 170 | LCA,North America,Saint Lucia,2021-09-17,10344.0,127.0,134.429,148.0,9.0,2.857,56095.14,688.716,729.001,802.599,48.807,15.494,1.25,,,,,,,,,,,,,,,,,,71517.0,42594.0,28923.0,,588.0,427.0,38.78,23.1,15.68,,2316.0,,184401.0,293.187,34.9,9.721,6.405,12951.839,,204.62,11.62,,,87.202,1.3,76.2,0.759, 171 | VCT,North America,Saint Vincent and the Grenadines,2021-09-17,2613.0,0.0,27.429,13.0,0.0,0.143,23483.63,0.0,246.507,116.834,0.0,1.284,1.56,,,,,,,,,,,,,,,,,,32126.0,19512.0,12644.0,,842.0,47.0,28.87,17.54,11.36,,422.0,,111269.0,281.787,31.8,7.724,4.832,10727.146,,252.675,11.62,,,,2.6,72.53,0.738, 172 | WSM,Oceania,Samoa,2021-09-17,3.0,0.0,0.0,,,0.0,14.989,0.0,0.0,,,0.0,,,,,,,,,,,,,,,,,,,142566.0,97032.0,45534.0,,,658.0,71.23,48.48,22.75,,3288.0,,200144.0,69.413,22.0,5.606,3.564,6021.557,,348.977,9.21,16.7,38.1,,,73.32,0.715, 173 | SMR,Europe,San Marino,2021-09-17,5412.0,4.0,5.571,90.0,0.0,0.0,159129.668,117.612,163.817,2646.281,0.0,0.0,1.23,,,,,,,,,,,,,,,,,,46917.0,24329.0,24329.0,,9.0,51.0,137.95,71.53,71.53,,1500.0,50.0,34010.0,556.667,,,,56861.47,,,5.64,,,,3.8,84.97,, 174 | STP,Africa,Sao Tome and Principe,2021-09-17,3025.0,41.0,34.0,43.0,1.0,0.571,13542.916,183.557,152.218,192.511,4.477,2.558,1.47,,,,,,,,,,,,,,,,,,57154.0,45088.0,12066.0,,,1284.0,25.59,20.19,5.4,,5748.0,,223364.0,212.841,18.7,2.886,2.162,3052.714,32.3,270.113,2.42,,,41.34,2.9,70.39,0.625, 175 | SAU,Asia,Saudi Arabia,2021-09-17,546411.0,75.0,83.143,8651.0,6.0,5.857,15461.247,2.122,2.353,244.789,0.17,0.166,0.26,,,,,,,,,49548.0,28248324.0,799.315,1.402,47578.0,1.346,0.001,1027.9,tests performed,40573432.0,23022785.0,17550647.0,,92500.0,201440.0,114.81,65.15,49.66,,5700.0,55.56,35340680.0,15.322,31.9,3.295,1.845,49045.411,,259.538,17.72,1.8,25.4,,2.7,75.13,0.854, 176 | SEN,Africa,Senegal,2021-09-17,73588.0,26.0,28.143,1842.0,1.0,2.286,4279.291,1.512,1.637,107.116,0.058,0.133,0.52,,,,,,,,,2564.0,787408.0,45.789,0.149,2150.0,0.125,0.014,72.0,tests performed,1768064.0,1203066.0,564998.0,,,3633.0,10.28,7.0,3.29,,211.0,,17196308.0,82.328,18.7,3.008,1.796,2470.58,38.0,241.219,2.42,0.4,16.6,20.859,,67.94,0.512, 177 | SRB,Europe,Serbia,2021-09-17,853686.0,7572.0,6748.286,7698.0,34.0,29.286,125457.264,1112.777,991.725,1131.294,4.997,4.304,1.47,,,,,,,,,26183.0,5342565.0,785.141,3.848,22864.0,3.36,0.286,3.5,people tested,6238655.0,2990502.0,2854142.0,394011.0,20350.0,21255.0,91.68,43.95,41.94,5.79,3124.0,36.11,6804596.0,80.291,41.2,17.366,,14048.881,,439.415,10.08,37.7,40.2,97.719,5.609,76.0,0.806, 178 | SYC,Africa,Seychelles,2021-09-17,20593.0,0.0,0.0,106.0,0.0,0.0,208199.373,0.0,0.0,1071.681,0.0,0.0,1.02,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,98910.0,208.354,36.2,8.606,5.586,26382.287,1.1,242.648,10.55,7.1,35.7,,3.6,73.4,0.796, 179 | SLE,Africa,Sierra Leone,2021-09-17,6391.0,3.0,1.714,121.0,0.0,0.0,785.006,0.368,0.211,14.862,0.0,0.0,0.68,,,,,,,,,,,,,,,,,,221110.0,181065.0,40045.0,,,1621.0,2.72,2.22,0.49,,199.0,29.63,8141343.0,104.7,19.1,2.538,1.285,1390.3,52.2,325.721,2.42,8.8,41.3,19.275,,54.7,0.452, 180 | SGP,Asia,Singapore,2021-09-17,75783.0,935.0,738.714,59.0,0.0,0.143,12851.799,158.564,125.276,10.006,0.0,0.024,1.81,266.0,45.11,,,,,,,,18638817.0,3160.898,,63021.0,10.688,0.008,130.0,samples tested,9084721.0,4654992.0,4544159.0,,11919.0,10350.0,154.06,78.94,77.06,,1755.0,47.22,5896684.0,7915.731,42.4,12.922,7.049,85535.383,,92.243,10.99,5.2,28.3,,2.4,83.62,0.938, 181 | SXM,North America,Sint Maarten (Dutch part),2021-09-10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,48882.0,25516.0,23366.0,,,53.0,112.58,58.76,53.81,,1221.0,,43421.0,1209.088,,,,36327.232,,,,,,,,78.95,, 182 | SVK,Europe,Slovakia,2021-09-17,401250.0,902.0,498.714,12569.0,3.0,1.571,73479.241,165.18,91.327,2301.709,0.549,0.288,1.53,,,143.0,26.187,4.002,0.733,29.015,5.313,27933.0,41455749.0,7591.619,5.115,26970.0,4.939,0.023,43.5,tests performed,4636179.0,2415058.0,2221121.0,,39.0,2899.0,84.9,44.23,40.67,,531.0,34.72,5460726.0,113.128,41.2,15.07,9.167,30155.152,0.7,287.959,7.29,23.1,37.7,,5.82,77.54,0.86, 183 | SVN,Europe,Slovenia,2021-09-17,281687.0,1143.0,1022.714,4492.0,6.0,3.714,135509.637,549.857,491.992,2160.942,2.886,1.787,1.42,70.0,33.675,292.0,140.471,53.558,25.765,222.168,106.877,6528.0,1519459.0,730.958,3.14,5074.0,2.441,0.201,5.0,tests performed,2034157.0,1071680.0,962477.0,,7485.0,11329.0,97.86,51.55,46.3,,5450.0,35.19,2078723.0,102.619,44.5,19.062,12.93,31400.84,,153.493,7.25,20.1,25.0,,4.5,81.32,0.917, 184 | SLB,Oceania,Solomon Islands,2021-09-17,20.0,0.0,0.0,,,0.0,28.409,0.0,0.0,,,0.0,,,,,,,,,,,,,,,,,,,97744.0,74573.0,23171.0,,,1867.0,13.88,10.59,3.29,,2652.0,38.89,703995.0,21.841,20.8,3.507,2.043,2205.923,25.1,459.78,18.68,,,35.89,1.4,73.0,0.567, 185 | SOM,Africa,Somalia,2021-09-17,18820.0,0.0,78.143,1041.0,0.0,3.429,1150.402,0.0,4.777,63.633,0.0,0.21,1.23,,,,,,,,,,,,,,,,,,351305.0,231089.0,120216.0,,,4209.0,2.15,1.41,0.73,,257.0,,16359500.0,23.5,16.8,2.731,1.496,,,365.769,6.05,,,9.831,0.9,57.4,, 186 | ZAF,Africa,South Africa,2021-09-17,2877063.0,3648.0,4019.714,85952.0,173.0,192.0,47917.511,60.757,66.948,1431.531,2.881,3.198,0.71,,,,,,,,,45342.0,17236691.0,287.077,0.755,43746.0,0.729,0.099,10.1,people tested,15683797.0,11367049.0,7734069.0,,236763.0,222541.0,26.12,18.93,12.88,,3706.0,51.85,60041996.0,46.754,27.3,5.344,3.053,12294.876,18.9,200.38,5.52,8.1,33.2,43.993,2.32,64.13,0.709, 187 | OWID_SAM,,South America,2021-09-17,37334191.0,19913.0,22537.429,1145044.0,654.0,841.857,85971.95,45.855,51.898,2636.77,1.506,1.939,,,,,,,,,,,,,,,,,,,406021857.0,253314189.0,158061704.0,3394537.0,536485.0,2042057.0,93.5,58.33,36.4,0.78,4702.0,,434260138.0,,,,,,,,,,,,,,, 188 | KOR,Asia,South Korea,2021-09-17,284022.0,2084.0,1827.857,2394.0,5.0,5.143,5535.932,40.62,35.627,46.662,0.097,0.1,1.02,,,,,,,,,47324.0,13017726.0,253.731,0.922,41620.0,0.811,0.042,23.8,people tested,55514851.0,35414516.0,21489009.0,,731206.0,747375.0,108.21,69.03,41.88,,14567.0,47.22,51305184.0,527.967,43.4,13.914,8.622,35938.374,0.2,85.998,6.8,6.2,40.9,,12.27,83.03,0.916, 189 | SSD,Africa,South Sudan,2021-09-17,11800.0,10.0,27.714,121.0,0.0,0.143,1036.781,0.879,2.435,10.631,0.0,0.013,1.4,,,,,,,,,,,,,,,,,,67109.0,55182.0,11927.0,,,1312.0,0.59,0.48,0.1,,115.0,,11381377.0,,19.2,3.441,2.032,1569.888,,280.775,10.43,,,,,57.85,0.433, 190 | ESP,Europe,Spain,2021-09-17,4929546.0,3222.0,3155.0,85783.0,44.0,70.429,105455.637,68.927,67.494,1835.118,0.941,1.507,0.68,1227.0,26.249,4975.0,106.428,20.739,0.444,549.1,11.747,,56889626.0,1217.015,,90685.0,1.94,0.05,20.1,tests performed,69094664.0,37385758.0,35784743.0,,139473.0,126996.0,147.81,79.98,76.55,,2717.0,47.69,46745211.0,93.105,45.5,19.436,13.799,34272.36,1.0,99.403,7.17,27.4,31.4,,2.97,83.56,0.904, 191 | LKA,Asia,Sri Lanka,2021-09-17,500772.0,2078.0,2899.143,11938.0,121.0,134.714,23294.64,96.663,134.861,555.325,5.629,6.267,0.78,,,,,,,,,9672.0,5163974.0,240.215,0.45,10899.0,0.507,0.276,3.6,tests performed,24558102.0,13690202.0,10867900.0,,94649.0,154537.0,114.24,63.68,50.55,,7189.0,,21497306.0,341.955,34.1,10.069,5.331,11669.077,0.7,197.093,10.68,0.3,27.0,,3.6,76.98,0.782, 192 | SDN,Africa,Sudan,2021-09-17,37995.0,0.0,9.143,2875.0,0.0,5.429,846.038,0.0,0.204,64.018,0.0,0.121,0.51,,,,,,,,,,,,,,,,,,1423327.0,648840.0,505413.0,,,25076.0,3.17,1.44,1.13,,558.0,13.89,44909351.0,23.258,19.7,3.548,2.034,4466.507,,431.388,15.67,,,23.437,0.8,65.31,0.51, 193 | SUR,South America,Suriname,2021-09-17,36428.0,611.0,523.429,796.0,8.0,5.714,61554.787,1032.447,884.472,1345.054,13.518,9.656,1.44,,,,,,,,,,,,,,,,,,376042.0,216536.0,159506.0,,3613.0,2032.0,63.54,36.59,26.95,,3434.0,58.33,591798.0,3.612,29.6,6.933,4.229,13767.119,,258.314,12.54,7.4,42.9,67.779,3.1,71.68,0.738, 194 | SWE,Europe,Sweden,2021-09-17,1144982.0,1009.0,995.0,14775.0,22.0,10.286,112693.315,99.309,97.932,1454.21,2.165,1.012,1.24,55.0,5.413,340.0,33.464,21.643,2.13,,,37493.0,,,3.69,37493.0,3.69,0.029,35.0,tests performed,13403076.0,7070462.0,6332614.0,,49609.0,37840.0,131.92,69.59,62.33,,3724.0,37.04,10160159.0,24.718,41.0,19.985,13.433,46949.283,0.5,133.982,4.79,18.8,18.9,,2.22,82.8,0.945, 195 | CHE,Europe,Switzerland,2021-09-17,823078.0,2096.0,2145.714,10999.0,8.0,6.714,94438.479,240.491,246.195,1262.005,0.918,0.77,0.99,262.0,30.061,770.0,88.348,,,318.0,36.487,32826.0,9882439.0,1133.893,3.766,34348.0,3.941,0.073,13.7,tests performed,10055192.0,5270807.0,4587999.0,,33212.0,30592.0,115.37,60.48,52.64,,3510.0,44.44,8715494.0,214.243,43.1,18.436,12.644,57410.166,,99.739,5.59,22.6,28.9,,4.53,83.78,0.955, 196 | SYR,Asia,Syria,2021-09-17,30337.0,184.0,159.429,2112.0,8.0,6.571,1659.963,10.068,8.724,115.563,0.438,0.36,1.11,,,,,,,,,,,,,,,,,,,,,,,,,,,,,28.7,18275704.0,,21.7,,2.577,,,376.264,,,,70.598,1.5,72.7,0.567, 197 | TWN,Asia,Taiwan,2021-09-17,16123.0,8.0,7.714,839.0,0.0,0.0,675.875,0.335,0.323,35.171,0.0,0.0,0.93,,,,,,,,,24536.0,3092342.0,129.631,1.029,23858.0,1.0,0.0,3553.5,people tested,12938098.0,11536036.0,1402062.0,,229670.0,139489.0,54.24,48.36,5.88,,5847.0,,23855008.0,,42.2,,8.353,,,103.957,,,,,,80.46,, 198 | TJK,Asia,Tajikistan,2021-09-17,17484.0,0.0,6.0,125.0,0.0,0.0,1793.3,0.0,0.615,12.821,0.0,0.0,0.6,,,,,,,,,,,,,,,,,,3446777.0,2168537.0,1278240.0,,113265.0,38892.0,35.35,22.24,13.11,,3989.0,,9749625.0,64.281,23.3,3.466,2.155,2896.913,4.8,427.698,7.11,,,72.704,4.8,71.1,0.668, 199 | TZA,Africa,Tanzania,2021-09-17,1367.0,0.0,0.0,50.0,0.0,0.0,22.228,0.0,0.0,0.813,0.0,0.0,-0.0,,,,,,,,,,,,,,,,,,350000.0,350000.0,350000.0,,,3243.0,0.57,0.57,0.57,,53.0,,61498438.0,64.699,17.7,3.108,1.874,2683.304,49.1,217.288,5.75,3.3,26.7,47.953,0.7,65.46,0.529, 200 | THA,Asia,Thailand,2021-09-17,1448792.0,14555.0,13691.286,15124.0,171.0,172.0,20711.573,208.075,195.727,216.209,2.445,2.459,0.9,,,,,,,,,41354.0,12894770.0,184.34,0.591,50234.0,0.718,0.293,3.4,tests performed,43342103.0,28436015.0,14285995.0,620093.0,864589.0,638392.0,61.96,40.65,20.42,0.89,9126.0,55.09,69950844.0,135.132,40.1,11.373,6.89,16277.671,0.1,109.861,7.04,1.9,38.8,90.67,2.1,77.15,0.777, 201 | TLS,Asia,Timor,2021-09-17,18943.0,87.0,90.714,103.0,1.0,2.143,14095.805,64.738,67.502,76.644,0.744,1.595,0.79,,,,,,,,,,194367.0,144.632,,809.0,0.602,0.164,6.1,tests performed,643324.0,408918.0,234406.0,,,3980.0,47.87,30.43,17.44,,2962.0,54.63,1343875.0,87.176,18.0,3.556,1.897,6570.102,30.3,335.346,6.86,6.3,78.1,28.178,5.9,69.5,0.606, 202 | TGO,Africa,Togo,2021-09-17,24369.0,276.0,131.286,211.0,2.0,1.0,2874.299,32.554,15.485,24.887,0.236,0.118,1.08,,,,,,,,,1836.0,491349.0,57.954,0.217,1759.0,0.207,0.087,11.5,tests performed,651941.0,414249.0,237692.0,,,7276.0,7.69,4.89,2.8,,858.0,50.93,8478242.0,143.366,19.4,2.839,1.525,1429.813,49.2,280.033,6.15,0.9,14.2,10.475,0.7,61.04,0.515, 203 | TTO,North America,Trinidad and Tobago,2021-09-17,48143.0,218.0,201.143,1405.0,8.0,6.571,34305.182,155.34,143.328,1001.159,5.701,4.683,0.96,,,,,,,,,5319.0,331651.0,236.324,3.79,1364.0,0.972,0.143,7.0,people tested,1014945.0,555534.0,459411.0,,5072.0,6301.0,72.32,39.59,32.74,,4490.0,83.33,1403374.0,266.886,36.2,10.014,5.819,28763.071,,228.467,10.97,,,89.443,3.0,73.51,0.796, 204 | TUN,Africa,Tunisia,2021-09-17,698427.0,1006.0,2394.714,24415.0,32.0,47.0,58515.483,84.285,200.634,2045.533,2.681,3.938,1.32,,,,,,,,,9547.0,2830291.0,237.127,0.8,17362.0,1.455,0.143,7.0,people tested,7017171.0,4610453.0,3128815.0,,56532.0,98621.0,58.79,38.63,26.21,,8263.0,41.67,11935764.0,74.228,32.7,8.001,5.075,10849.297,2.0,318.991,8.52,1.1,65.8,78.687,2.3,76.7,0.74, 205 | TUR,Asia,Turkey,2021-09-17,6794670.0,27692.0,25817.714,61140.0,237.0,250.857,79897.124,325.625,303.585,718.933,2.787,2.95,1.1,,,,,,,,,343142.0,81382545.0,956.961,4.035,325888.0,3.832,0.077,12.9,tests performed,104593216.0,52497989.0,41739242.0,10355985.0,572278.0,506364.0,122.99,61.73,49.08,12.18,5954.0,,85042736.0,104.914,31.6,8.153,5.061,25129.341,0.2,171.285,12.13,14.1,41.1,,2.81,77.69,0.82, 206 | TCA,North America,Turks and Caicos Islands,2021-09-10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,51838.0,27220.0,24618.0,,,138.0,132.15,69.39,62.76,,3518.0,,39226.0,37.312,,,,,,,,,,,,80.22,, 207 | UGA,Africa,Uganda,2021-09-17,121984.0,200.0,116.857,3119.0,4.0,4.857,2588.6,4.244,2.48,66.188,0.085,0.103,0.33,,,,,,,,,3374.0,1635630.0,34.709,0.072,3643.0,0.077,0.035,28.2,samples tested,,,,,,,,,,,,81.48,47123533.0,213.759,16.4,2.168,1.308,1697.707,41.6,213.333,2.5,3.4,16.7,21.222,0.5,63.37,0.544, 208 | UKR,Europe,Ukraine,2021-09-17,2442344.0,6940.0,4257.286,58265.0,119.0,88.143,56188.695,159.662,97.943,1340.448,2.738,2.028,1.26,,,,,,,,,35171.0,12412498.0,285.563,0.809,27924.0,0.642,0.142,7.1,tests performed,11362214.0,6248153.0,5114061.0,,132559.0,110123.0,26.14,14.37,11.77,,2533.0,50.93,43466822.0,77.39,41.4,16.462,11.133,7894.393,0.1,539.849,7.11,13.5,47.4,,8.8,72.06,0.779, 209 | ARE,Asia,United Arab Emirates,2021-09-17,731828.0,521.0,612.429,2071.0,2.0,1.571,73248.115,52.146,61.298,207.285,0.2,0.157,0.8,,,,,,,,,275823.0,79589959.0,7966.099,27.607,313989.0,31.427,0.002,487.3,tests performed,19330107.0,9059751.0,7971790.0,,82943.0,75566.0,193.47,90.68,79.79,,7563.0,,9991083.0,112.442,34.0,1.144,0.526,67293.483,,317.84,17.26,1.2,37.4,,1.2,77.97,0.89, 210 | GBR,Europe,United Kingdom,2021-09-17,7406017.0,32566.0,29115.0,135314.0,180.0,143.0,108581.298,477.458,426.862,1983.869,2.639,2.097,0.89,1081.0,15.849,8339.0,122.26,,,6782.0,99.432,1063453.0,259880899.0,3810.173,15.592,1012527.0,14.845,0.031,32.3,tests performed,92826977.0,48528901.0,44298076.0,,87832.0,93004.0,136.1,71.15,64.95,,1364.0,41.2,68207114.0,272.898,40.8,18.517,12.527,39753.244,0.2,122.137,4.28,20.0,24.7,,2.54,81.32,0.932, 211 | USA,North America,United States,2021-09-17,41993789.0,207886.0,148023.143,672635.0,2635.0,1949.0,126139.644,624.442,444.627,2020.44,7.915,5.854,1.03,24893.0,74.773,89181.0,267.879,,,81187.0,243.867,382262.0,551079079.0,1655.314,1.148,1210978.0,3.637,0.142,7.0,tests performed,383994877.0,211097597.0,180572171.0,,956474.0,775023.0,114.17,62.77,53.69,,2304.0,61.57,332915074.0,35.608,38.3,15.413,9.732,54225.446,1.2,151.089,10.79,19.1,24.6,,2.77,78.86,0.926, 212 | URY,South America,Uruguay,2021-09-17,387449.0,150.0,148.429,6046.0,0.0,1.286,111171.335,43.04,42.589,1734.788,0.0,0.369,1.14,,,,,,,,,10352.0,3472637.0,996.409,2.97,8302.0,2.382,0.018,55.8,tests performed,6015778.0,2716871.0,2558759.0,740148.0,32506.0,14047.0,172.61,77.96,73.42,21.24,4031.0,52.78,3485152.0,19.751,35.6,14.655,10.361,20551.409,0.1,160.708,6.93,14.0,19.9,,2.8,77.91,0.817, 213 | UZB,Asia,Uzbekistan,2021-09-17,167268.0,624.0,594.286,1179.0,4.0,5.286,4928.959,18.388,17.512,34.742,0.118,0.156,0.89,,,,,,,,,,,,,,,,,,17435318.0,9787973.0,3345225.0,,281054.0,208168.0,51.38,28.84,9.86,,6134.0,59.26,33935765.0,76.134,28.2,4.469,2.873,6253.104,,724.417,7.57,1.3,24.7,,4.0,71.72,0.72, 214 | VUT,Oceania,Vanuatu,2021-09-17,4.0,0.0,0.0,1.0,0.0,0.0,12.72,0.0,0.0,3.18,0.0,0.0,,,,,,,,,,,,,,,,,,,50620.0,39347.0,11273.0,,,994.0,16.1,12.51,3.58,,3161.0,22.22,314464.0,22.662,23.1,4.394,2.62,2921.909,13.2,546.3,12.02,2.8,34.5,25.209,,70.47,0.609, 215 | VAT,Europe,Vatican,2021-09-17,27.0,0.0,0.0,,,0.0,33251.232,0.0,0.0,,,0.0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,812.0,,,,,,,,,,,,,75.12,, 216 | VEN,South America,Venezuela,2021-09-17,353401.0,2606.0,1120.143,4275.0,30.0,13.714,12311.502,90.786,39.023,148.929,1.045,0.478,1.03,,,,,,,,,,,,,,,,,,11094206.0,6822809.0,4271397.0,,,125099.0,38.65,23.77,14.88,,4358.0,,28704947.0,36.253,29.0,6.614,3.915,16745.022,,204.85,6.47,,,,0.8,72.06,0.711, 217 | VNM,Asia,Vietnam,2021-09-17,667650.0,11521.0,11176.143,16637.0,212.0,270.286,6801.039,117.359,113.846,169.473,2.16,2.753,1.0,,,,,,,,,209988.0,18407857.0,187.512,2.139,259000.0,2.638,0.046,21.6,samples tested,32296517.0,26307653.0,5988864.0,,1041661.0,1245514.0,32.9,26.8,6.1,,12687.0,72.69,98168829.0,308.127,32.6,7.15,4.718,6171.884,2.0,245.465,6.0,1.0,45.9,85.847,2.6,75.4,0.704, 218 | WLF,Oceania,Wallis and Futuna,2021-09-14,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,9902.0,4952.0,4950.0,,,7.0,89.26,44.64,44.62,,631.0,,11094.0,,,,,,,,,,,,,79.94,, 219 | OWID_WRL,,World,2021-09-17,227613687.0,557437.0,524549.143,4678856.0,8600.0,8858.571,28903.451,70.786,66.61,594.143,1.092,1.125,0.94,,,,,,,,,,,,,,,,,,5875370940.0,3374824365.0,2459295920.0,21048874.0,12743319.0,28558103.0,74.61,42.86,31.23,0.27,3626.0,,7874965732.0,58.045,30.9,8.696,5.355,15469.207,10.0,233.07,8.51,6.434,34.635,60.13,2.705,72.58,0.737, 220 | YEM,Asia,Yemen,2021-09-17,8593.0,36.0,41.0,1628.0,9.0,9.571,281.824,1.181,1.345,53.393,0.295,0.314,1.17,,,,,,,,,,,,,,,,,,322934.0,308025.0,14909.0,,,244.0,1.06,1.01,0.05,,8.0,,30490639.0,53.508,20.3,2.922,1.583,1479.147,18.8,495.003,5.35,7.6,29.2,49.542,0.7,66.12,0.47, 221 | ZMB,Africa,Zambia,2021-09-17,208353.0,86.0,93.857,3637.0,1.0,1.571,11011.933,4.545,4.961,192.224,0.053,0.083,0.7,,,,,,,,,6888.0,2375797.0,125.566,0.364,6122.0,0.324,0.016,60.6,tests performed,636600.0,311049.0,291947.0,,,3734.0,3.36,1.64,1.54,,197.0,43.52,18920657.0,22.995,17.7,2.48,1.542,3689.251,57.5,234.499,3.94,3.1,24.7,13.938,2.0,63.89,0.584, 222 | ZWE,Africa,Zimbabwe,2021-09-17,127632.0,264.0,209.857,4562.0,2.0,4.286,8456.835,17.493,13.905,302.276,0.133,0.284,0.91,,,,,,,,,6482.0,1200458.0,79.542,0.429,3899.0,0.258,0.042,23.7,tests performed,4964302.0,2930550.0,2033752.0,,108486.0,51755.0,32.89,19.42,13.48,,3429.0,65.74,15092171.0,42.729,19.6,2.822,1.882,1899.775,21.4,307.846,1.82,1.6,30.7,36.791,1.7,61.49,0.571, 223 | -------------------------------------------------------------------------------- /dbt/dbt_project.yml: -------------------------------------------------------------------------------- 1 | 2 | # Name your project! Project names should contain only lowercase characters 3 | # and underscores. A good package name should reflect your organization's 4 | # name or the intended use of these models 5 | name: 'dbt_postgresql' 6 | version: '1.0.0' 7 | config-version: 2 8 | 9 | # This setting configures which "profile" dbt uses for this project. 10 | profile: 'dbt_postgresql' 11 | 12 | # These configurations specify where dbt should look for different types of files. 13 | # The `source-paths` config, for example, states that models in this project can be 14 | # found in the "models/" directory. You probably won't need to change these! 15 | source-paths: ["models"] 16 | analysis-paths: ["analysis"] 17 | test-paths: ["tests"] 18 | data-paths: ["data"] 19 | macro-paths: ["macros"] 20 | snapshot-paths: ["snapshots"] 21 | 22 | target-path: "target" # directory which will store compiled SQL files 23 | clean-targets: # directories to be removed by `dbt clean` 24 | - "target" 25 | - "dbt_modules" 26 | 27 | 28 | # Configuring models 29 | # Full documentation: https://docs.getdbt.com/docs/configuring-models 30 | 31 | # In this example config, we tell dbt to build all models in the example/ directory 32 | # as tables. These settings can be overridden in the individual model files 33 | # using the `{{ config(...) }}` macro. 34 | models: 35 | dbt_postgresql: 36 | +materialized: incremental 37 | +incremental_strategy: merge 38 | 39 | vars: 40 | dbt_utils_dispatch_list: ['tsql_utils'] 41 | dbt_date_dispatch_list: ['tsql_utils'] 42 | audit_helper_dispatch_list: ['tsql_utils'] 43 | dbt_expectations_dispatch_list: ['tsql_utils'] 44 | 45 | seeds: 46 | dbt_postgresql: 47 | covid: 48 | covid_raw: 49 | +enabled: true 50 | +schema: staging 51 | +alias: Covid_Raw 52 | +full_refresh: true 53 | +quote_columns: false 54 | +column_types: 55 | iso_code: varchar(255) 56 | continent: varchar(255) 57 | location: varchar(255) 58 | last_updated_date: date 59 | total_cases: varchar(255) 60 | new_cases: varchar(255) 61 | new_cases_smoothed: varchar(255) 62 | total_deaths: varchar(255) 63 | new_deaths: varchar(255) 64 | new_deaths_smoothed: varchar(255) 65 | total_cases_per_million: varchar(255) 66 | new_cases_per_million: varchar(255) 67 | new_cases_smoothed_per_million: varchar(255) 68 | total_deaths_per_million: varchar(255) 69 | new_deaths_per_million: varchar(255) 70 | new_deaths_smoothed_per_million: varchar(255) 71 | reproduction_rate: varchar(255) 72 | icu_patients: varchar(255) 73 | icu_patients_per_million: varchar(255) 74 | hosp_patients: varchar(255) 75 | hosp_patients_per_million: varchar(255) 76 | weekly_icu_admissions: varchar(255) 77 | weekly_icu_admissions_per_million: varchar(255) 78 | weekly_hosp_admissions: varchar(255) 79 | weekly_hosp_admissions_per_million: varchar(255) 80 | new_tests: varchar(255) 81 | total_tests: varchar(255) 82 | total_tests_per_thousand: varchar(255) 83 | new_tests_per_thousand: varchar(255) 84 | new_tests_smoothed: varchar(255) 85 | new_tests_smoothed_per_thousand: varchar(255) 86 | positive_rate: varchar(255) 87 | tests_per_case: varchar(255) 88 | tests_units: varchar(255) 89 | total_vaccinations: varchar(255) 90 | people_vaccinated: varchar(255) 91 | people_fully_vaccinated: varchar(255) 92 | total_boosters: varchar(255) 93 | new_vaccinations: varchar(255) 94 | new_vaccinations_smoothed: varchar(255) 95 | total_vaccinations_per_hundred: varchar(255) 96 | people_vaccinated_per_hundred: varchar(255) 97 | people_fully_vaccinated_per_hundred: varchar(255) 98 | total_boosters_per_hundred: varchar(255) 99 | new_vaccinations_smoothed_per_million: varchar(255) 100 | stringency_index: varchar(255) 101 | population: varchar(255) 102 | population_density: varchar(255) 103 | median_age: varchar(255) 104 | aged_65_older: varchar(255) 105 | aged_70_older: varchar(255) 106 | gdp_per_capita: varchar(255) 107 | extreme_poverty: varchar(255) 108 | cardiovasc_death_rate: varchar(255) 109 | diabetes_prevalence: varchar(255) 110 | female_smokers: varchar(255) 111 | male_smokers: varchar(255) 112 | handwashing_facilities: varchar(255) 113 | hospital_beds_per_thousand: varchar(255) 114 | life_expectancy: varchar(255) 115 | human_development_index: varchar(255) 116 | excess_mortality: varchar(255) -------------------------------------------------------------------------------- /dbt/macros/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datnguye/dbt-postgres/1edd458a54d5735dd677298eb6b963f6163aaaa6/dbt/macros/.gitkeep -------------------------------------------------------------------------------- /dbt/macros/schema_test/test_relationship_where.sql: -------------------------------------------------------------------------------- 1 | {#-- Cloned from https://github.com/dbt-labs/dbt-utils/blob/a7290442b401cdfb930d1f527cb9782022d867d9/macros/schema_tests/relationships_where.sql#} 2 | 3 | {% macro test_relationship_where(model, column_name, to, field, from_condition="1=1", to_condition="1=1") %} 4 | 5 | select left_table.id, 6 | right_table.id as right_id 7 | from ( 8 | select {{column_name}} as id 9 | from {{model}} 10 | where {{column_name}} is not null 11 | and {{from_condition}} 12 | ) as left_table 13 | left join ( 14 | select {{field}} as id 15 | from {{to}} 16 | where {{field}} is not null 17 | and {{to_condition}} 18 | ) as right_table 19 | on left_table.id = right_table.id 20 | where right_table.id is null 21 | 22 | {% endmacro %} -------------------------------------------------------------------------------- /dbt/macros/schema_test/test_unique_combination_of_columns.sql: -------------------------------------------------------------------------------- 1 | {#-- Cloned from https://github.com/dbt-labs/dbt-utils/blob/a7290442b401cdfb930d1f527cb9782022d867d9/macros/schema_tests/unique_combination_of_columns.sql #} 2 | {% macro test_unique_combination_of_columns(model, combination_of_columns, quote_columns=false) %} 3 | 4 | {% if not quote_columns %} 5 | {%- set column_list=combination_of_columns %} 6 | {% elif quote_columns %} 7 | {%- set column_list=[] %} 8 | {% for column in combination_of_columns -%} 9 | {% set column_list = column_list.append( adapter.quote(column) ) %} 10 | {%- endfor %} 11 | {% else %} 12 | {{ exceptions.raise_compiler_error( 13 | "`quote_columns` argument for unique_combination_of_columns test must be one of [True, False] Got: '" ~ quote ~"'.'" 14 | ) }} 15 | {% endif %} 16 | 17 | {%- set columns_csv=column_list | join(', ') %} 18 | 19 | select 20 | {{ columns_csv }} 21 | from {{ model }} 22 | group by {{ columns_csv }} 23 | having count(*) > 1 24 | 25 | {% endmacro %} -------------------------------------------------------------------------------- /dbt/macros/sql/getutcdate.sql: -------------------------------------------------------------------------------- 1 | {% macro getutcdate(db='postgresql') %} 2 | {%- if db == "postgresql" -%} 3 | now() at time zone 'utc' 4 | {%- elif db == "mssql" -%} 5 | getutcdate() 6 | {%- endif -%} 7 | {% endmacro %} -------------------------------------------------------------------------------- /dbt/models/data_mart/covid/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datnguye/dbt-postgres/1edd458a54d5735dd677298eb6b963f6163aaaa6/dbt/models/data_mart/covid/.gitkeep -------------------------------------------------------------------------------- /dbt/models/data_mart/covid/Country.sql: -------------------------------------------------------------------------------- 1 | {{ config(alias='Country', unique_key='"CountryKey"') }} 2 | 3 | {% if is_incremental() %} 4 | {%- call statement('get_lastest_time', fetch_result=True) -%} 5 | SELECT MAX("LastUpdatedTimestamp") FROM {{ this }} 6 | {%- endcall -%} 7 | {%- set lastest_time = load_result('get_lastest_time')['data'][0][0] -%} 8 | {% endif %} 9 | 10 | SELECT "CountryKey", 11 | {{getutcdate()}} as "LastUpdatedTimestamp", 12 | 13 | "CountryCode" as "Country Code", 14 | "Continent" as "Continent", 15 | "Country" as "Country" 16 | 17 | FROM {{ref('ODS_Country')}} 18 | 19 | WHERE 1=1 20 | {% if is_incremental() %} 21 | AND ( 22 | --update once a day 23 | CAST({{getutcdate()}} as date) > CAST('{{lastest_time}}' as date) 24 | ) 25 | {% endif %} -------------------------------------------------------------------------------- /dbt/models/data_mart/covid/Country.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: Country 5 | description: "covid - Country model" 6 | tests: 7 | - unique_combination_of_columns: 8 | combination_of_columns: 9 | - '"Continent"' 10 | - '"Country"' 11 | columns: 12 | - name: '"CountryKey"' 13 | description: "The primary key for this table" 14 | tests: 15 | - unique 16 | - not_null 17 | - name: '"Continent"' 18 | tests: 19 | - not_null 20 | - name: '"Country"' 21 | tests: 22 | - not_null -------------------------------------------------------------------------------- /dbt/models/data_mart/covid/CovidCase.sql: -------------------------------------------------------------------------------- 1 | {{ config(alias='CovidCase', unique_key='"CovidCaseKey"') }} 2 | 3 | {% if is_incremental() %} 4 | {%- call statement('get_lastest_time', fetch_result=True) -%} 5 | SELECT MAX("LastUpdatedTimestamp") FROM {{ this }} 6 | {%- endcall -%} 7 | {%- set lastest_time = load_result('get_lastest_time')['data'][0][0] -%} 8 | {% endif %} 9 | 10 | SELECT CC."CovidCaseKey", 11 | C."CountryKey", 12 | D."DateKey", 13 | {{getutcdate()}} as "LastUpdatedTimestamp", 14 | 15 | CC."TotalCase" as "Total Case", 16 | CC."NewCase" as "New Case", 17 | CC."TotalDeath" as "Total Death", 18 | CC."NewDeath" as "New Death" 19 | 20 | FROM {{ref('ODS_Covid_Case')}} as CC 21 | JOIN {{ref('Country')}} as C 22 | ON C."Country" = CC."Country" 23 | AND C."Continent" = CC."Continent" 24 | JOIN {{ref('Date')}} as D 25 | ON D."Date Value" = CC."UpdatedDate" 26 | 27 | WHERE 1=1 28 | {% if is_incremental() %} 29 | AND ( 30 | --update once a day 31 | CAST({{getutcdate()}} as date) > CAST('{{lastest_time}}' as date) 32 | ) 33 | {% endif %} -------------------------------------------------------------------------------- /dbt/models/data_mart/covid/CovidCase.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: CovidCase 5 | description: "covid - CovidCase model" 6 | columns: 7 | - name: '"CovidCaseKey"' 8 | description: "The primary key for this table" 9 | tests: 10 | - unique 11 | - not_null 12 | - name: '"CountryKey"' 13 | tests: 14 | - relationship_where: 15 | to: ref('Country') 16 | field: '"CountryKey"' 17 | from_condition: 1=1 18 | - name: '"DateKey"' 19 | tests: 20 | - relationship_where: 21 | to: ref('Date') 22 | field: '"DateKey"' 23 | from_condition: 1=1 -------------------------------------------------------------------------------- /dbt/models/data_mart/covid/Date.sql: -------------------------------------------------------------------------------- 1 | {{ config(alias='Date', unique_key='"DateKey"') }} 2 | 3 | {% if is_incremental() %} 4 | {%- call statement('get_lastest_time', fetch_result=True) -%} 5 | SELECT MAX("LastUpdatedTimestamp") FROM {{ this }} 6 | {%- endcall -%} 7 | {%- set lastest_time = load_result('get_lastest_time')['data'][0][0] -%} 8 | {% endif %} 9 | 10 | SELECT DISTINCT 11 | {{ dbt_utils.surrogate_key(['"UpdatedDate"']) }} as "DateKey", 12 | {{getutcdate()}} as "LastUpdatedTimestamp", 13 | 14 | "UpdatedDate" as "Date Value", 15 | DATE_PART('day', "UpdatedDate") as "Day", 16 | DATE_PART('month', "UpdatedDate") as "Month", 17 | TO_CHAR("UpdatedDate", 'month') as "Month Name", 18 | 'Q' || CAST(DATE_PART('quarter', "UpdatedDate") as varchar) as "Quarter", 19 | DATE_PART('year', "UpdatedDate") as "Year" 20 | 21 | FROM {{ref('ODS_Covid_Case')}} 22 | 23 | WHERE 1=1 24 | {% if is_incremental() %} 25 | AND ( 26 | --update once a day 27 | CAST({{getutcdate()}} as date) > CAST('{{lastest_time}}' as date) 28 | ) 29 | {% endif %} -------------------------------------------------------------------------------- /dbt/models/data_mart/covid/Date.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: Date 5 | description: "covid - Date model" 6 | columns: 7 | - name: '"DateKey"' 8 | description: "The primary key for this table" 9 | tests: 10 | - unique 11 | - not_null 12 | - name: '"Date Value"' 13 | tests: 14 | - unique 15 | - not_null -------------------------------------------------------------------------------- /dbt/models/data_mart/covid/_covid_ds.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | exposures: 4 | - name: _covid_ds 5 | type: dashboard 6 | description: > 7 | All `Covid` dataset's tables 8 | 9 | depends_on: 10 | - ref('Country') 11 | - ref('Date') 12 | - ref('CovidCase') 13 | 14 | owner: 15 | name: covid 16 | email: datnguyen.it09@gmail.com -------------------------------------------------------------------------------- /dbt/models/example/MyFirstModel.sql: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | Welcome to your first dbt model! 4 | Did you know that you can also configure models directly within SQL files? 5 | This will override configurations stated in dbt_project.yml 6 | 7 | Try changing "table" to "view" below 8 | */ 9 | 10 | {{ config(materialized='table', schema='firstrun') }} 11 | 12 | with source_data as ( 13 | 14 | select 1 as id 15 | --union all 16 | --select null as id 17 | 18 | ) 19 | 20 | select * 21 | from source_data 22 | 23 | /* 24 | Uncomment the line below to remove records with null `id` values 25 | */ 26 | 27 | -- where id is not null 28 | -------------------------------------------------------------------------------- /dbt/models/example/schema.yml: -------------------------------------------------------------------------------- 1 | 2 | version: 2 3 | 4 | models: 5 | - name: MyFirstModel 6 | description: "A starter dbt model" 7 | columns: 8 | - name: id 9 | description: "The primary key for this table" 10 | tests: 11 | - unique 12 | - not_null -------------------------------------------------------------------------------- /dbt/models/ods/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datnguye/dbt-postgres/1edd458a54d5735dd677298eb6b963f6163aaaa6/dbt/models/ods/.gitkeep -------------------------------------------------------------------------------- /dbt/models/ods/covid/ODS_Country.sql: -------------------------------------------------------------------------------- 1 | {{ config(schema='ods', alias='ODS_Country', unique_key='"CountryKey"') }} 2 | 3 | {% if is_incremental() %} 4 | {%- call statement('get_lastest_time', fetch_result=True) -%} 5 | SELECT MAX("LastUpdatedTimestamp") FROM {{ this }} 6 | {%- endcall -%} 7 | {%- set lastest_time = load_result('get_lastest_time')['data'][0][0] -%} 8 | {% endif %} 9 | 10 | SELECT {{ dbt_utils.surrogate_key(['"Continent"', '"Country"']) }} as "CountryKey", 11 | "CountryCode" , 12 | "Continent" , 13 | "Country" , 14 | 15 | {{getutcdate()}} as "LastUpdatedTimestamp" 16 | 17 | FROM {{ref('STAG_Covid_Data')}} 18 | 19 | WHERE 1=1 20 | {% if is_incremental() %} 21 | AND ( 22 | --update once a day 23 | CAST({{getutcdate()}} as date) > CAST('{{lastest_time}}' as date) 24 | ) 25 | {% endif %} -------------------------------------------------------------------------------- /dbt/models/ods/covid/ODS_Covid_Case.sql: -------------------------------------------------------------------------------- 1 | {{ config(schema='ods', alias='ODS_Covid_Case', unique_key='"CovidCaseKey"') }} 2 | 3 | {% if is_incremental() %} 4 | {%- call statement('get_lastest_time', fetch_result=True) -%} 5 | SELECT MAX("UpdatedDate") FROM {{ this }} 6 | {%- endcall -%} 7 | {%- set lastest_time = load_result('get_lastest_time')['data'][0][0] -%} 8 | {% endif %} 9 | 10 | SELECT {{ dbt_utils.surrogate_key(['"Continent"', '"Country"', '"UpdatedDate"']) }} as "CovidCaseKey", 11 | 12 | "Continent", 13 | "Country", 14 | "UpdatedDate", 15 | "TotalCase", 16 | "NewCase", 17 | "TotalDeath", 18 | "NewDeath", 19 | 20 | {{getutcdate()}} as "LastUpdatedTimestamp" 21 | 22 | FROM {{ref('STAG_Covid_Data')}} 23 | 24 | WHERE 1=1 25 | {% if is_incremental() %} 26 | AND ( 27 | "UpdatedDate" >= '{{lastest_time}}' 28 | ) 29 | {% endif %} -------------------------------------------------------------------------------- /dbt/models/staging/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datnguye/dbt-postgres/1edd458a54d5735dd677298eb6b963f6163aaaa6/dbt/models/staging/.gitkeep -------------------------------------------------------------------------------- /dbt/models/staging/covid/STAG_Covid_Data.sql: -------------------------------------------------------------------------------- 1 | {{ config(schema='staging', alias='STAG_Covid_Data', unique_key='"CovidDataKey"') }} 2 | 3 | {% if is_incremental() %} 4 | {%- call statement('get_lastest_time', fetch_result=True) -%} 5 | SELECT MAX("UpdatedDate") FROM {{ this }} 6 | {%- endcall -%} 7 | {%- set lastest_time = load_result('get_lastest_time')['data'][0][0] -%} 8 | {% endif %} 9 | 10 | SELECT DISTINCT 11 | {{ dbt_utils.surrogate_key(['continent', 'location', 'last_updated_date']) }} as "CovidDataKey", 12 | 13 | iso_code::varchar(3) as "CountryCode", 14 | continent::varchar(255) as "Continent", 15 | location::varchar(255) as "Country", 16 | last_updated_date::date as "UpdatedDate", 17 | COALESCE(total_cases::decimal(17,0),0) as "TotalCase", 18 | COALESCE(new_cases::decimal(17,0),0) as "NewCase", 19 | COALESCE(total_deaths::decimal(17,0),0) as "TotalDeath", 20 | COALESCE(new_deaths::decimal(17,0),0) as "NewDeath", 21 | 22 | {{getutcdate()}} as "LastUpdatedTimestamp" 23 | 24 | FROM {{ref('covid_raw')}} 25 | WHERE continent IS NOT NULL 26 | 27 | {% if is_incremental() %} 28 | AND ( 29 | last_updated_date::date >= '{{lastest_time}}' 30 | ) 31 | {% endif %} -------------------------------------------------------------------------------- /dbt/packages.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - package: dbt-labs/dbt_utils 3 | version: 0.7.3 -------------------------------------------------------------------------------- /dbt/snapshots/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datnguye/dbt-postgres/1edd458a54d5735dd677298eb6b963f6163aaaa6/dbt/snapshots/.gitkeep -------------------------------------------------------------------------------- /dbt/tests/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datnguye/dbt-postgres/1edd458a54d5735dd677298eb6b963f6163aaaa6/dbt/tests/.gitkeep -------------------------------------------------------------------------------- /dbt/tests/covid/assert_number_of_country_differ_zero.sql: -------------------------------------------------------------------------------- 1 | SELECT COUNT("CountryKey") as count_value 2 | FROM {{ ref('Country') }} 3 | HAVING NOT(COUNT("CountryKey") > 0) -------------------------------------------------------------------------------- /dbt/tests/covid/assert_number_of_covidcase_differ_zero.sql: -------------------------------------------------------------------------------- 1 | SELECT COUNT("CovidCaseKey") as count_value 2 | FROM {{ ref('CovidCase') }} 3 | HAVING NOT(COUNT("CovidCaseKey") > 0) -------------------------------------------------------------------------------- /dbt/tests/covid/assert_number_of_date_differ_zero.sql: -------------------------------------------------------------------------------- 1 | SELECT COUNT("DateKey") as count_value 2 | FROM {{ ref('Date') }} 3 | HAVING NOT(COUNT("DateKey") > 0) -------------------------------------------------------------------------------- /dbt_profile_template.yml: -------------------------------------------------------------------------------- 1 | dbt_postgresql: 2 | target: dev 3 | outputs: 4 | dev: 5 | type: postgres 6 | host: "{{ env_var('POSTGRES_HOST') }}" 7 | user: "{{ env_var('POSTGRES_USER') }}" 8 | pass: "{{ env_var('POSTGRES_PASS') }}" 9 | port: "{{ env_var('POSTGRES_PORT') | as_number }}" 10 | dbname: "{{ env_var('POSTGRES_DB') }}" 11 | schema: "{{ env_var('POSTGRES_SCHEMA') }}" 12 | threads: 5 13 | prod: 14 | type: postgres 15 | host: "{{ env_var('POSTGRES_HOST') }}" 16 | user: "{{ env_var('POSTGRES_USER') }}" 17 | pass: "{{ env_var('POSTGRES_PASS') }}" 18 | port: "{{ env_var('POSTGRES_PORT') | as_number }}" 19 | dbname: "{{ env_var('POSTGRES_DB') }}" 20 | schema: "{{ env_var('POSTGRES_SCHEMA') }}" 21 | threads: 5 -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | #[dbt] 2 | dbt==0.20.2 3 | 4 | #[workflow] 5 | prefect==0.15.6 6 | 7 | #[api] 8 | uvicorn==0.15.0 9 | fastapi==0.54.1 10 | sqlmodel==0.0.4 11 | python-jose==3.3.0 12 | pydantic==1.8.2 13 | pyyaml==5.4.1 -------------------------------------------------------------------------------- /services/api_service/.insomia/awesome-dbt-api-2021-10-03.json: -------------------------------------------------------------------------------- 1 | {"_type":"export","__export_format":4,"__export_date":"2021-10-03T10:21:31.635Z","__export_source":"insomnia.desktop.app:v2021.5.3","resources":[{"_id":"req_14297e175cb64556a8392fb5cff07cbb","parentId":"wrk_46ed8e0af6aa43ccbd8c045226355a26","modified":1633251813237,"created":1632633862864,"url":"http://127.0.0.1:8000/api/v1/dbt/processing","name":"api/v1/dbt/processing","description":"","method":"POST","body":{},"parameters":[],"headers":[],"authentication":{},"metaSortKey":-1632633862864,"isPrivate":false,"settingStoreCookies":true,"settingSendCookies":true,"settingDisableRenderRequestBody":false,"settingEncodeUrl":true,"settingRebuildPath":true,"settingFollowRedirects":"global","_type":"request"},{"_id":"wrk_46ed8e0af6aa43ccbd8c045226355a26","parentId":null,"modified":1632633572573,"created":1632633572573,"name":"dbt-mssql","description":"","scope":"collection","_type":"workspace"},{"_id":"req_dce6343d443a440fa6f30cdb55ca27d8","parentId":"wrk_46ed8e0af6aa43ccbd8c045226355a26","modified":1632633598155,"created":1632633595875,"url":"http://127.0.0.1:8000/api/v1/dbt/provision","name":"api/v1/dbt/provision","description":"","method":"POST","body":{},"parameters":[],"headers":[],"authentication":{},"metaSortKey":-1632633595875,"isPrivate":false,"settingStoreCookies":true,"settingSendCookies":true,"settingDisableRenderRequestBody":false,"settingEncodeUrl":true,"settingRebuildPath":true,"settingFollowRedirects":"global","_type":"request"},{"_id":"req_7e5ed9a88a0c428f9988203b6badad4c","parentId":"wrk_46ed8e0af6aa43ccbd8c045226355a26","modified":1633254914272,"created":1633231845106,"url":"http://127.0.0.1:8000/api/v1/dbt/custom-run","name":"api/v1/dbt/custom-run (full refresh a dataset)","description":"","method":"POST","body":{"mimeType":"application/json","text":"{\n\t\"action\": \"run\",\n\t\"args\": [\n\t\t{\"value\":\"--full-refresh\"}\n\t],\n\t\"kwargs\": [\n\t\t{\"key\": \"--target\", \"value\":\"dev\"},\n\t\t{\"key\": \"--models\", \"value\":\"+exposure:_covid_ds\"}\n\t]\n}"},"parameters":[],"headers":[{"name":"Content-Type","value":"application/json","id":"pair_c179be5cc45f484db3037b939eac8180"}],"authentication":{},"metaSortKey":-1630920373694.125,"isPrivate":false,"settingStoreCookies":true,"settingSendCookies":true,"settingDisableRenderRequestBody":false,"settingEncodeUrl":true,"settingRebuildPath":true,"settingFollowRedirects":"global","_type":"request"},{"_id":"req_079efcf7d79542fb81047d361b7b6d16","parentId":"wrk_46ed8e0af6aa43ccbd8c045226355a26","modified":1633256329276,"created":1633252811984,"url":"http://127.0.0.1:8000/api/v1/dbt/task/e6741ec5-144b-4758-81f8-58b8c12be0f3","name":"api/v1/dbt/provision/{taskid}","description":"","method":"GET","body":{},"parameters":[],"headers":[],"authentication":{},"metaSortKey":-1629207151513.25,"isPrivate":false,"settingStoreCookies":true,"settingSendCookies":true,"settingDisableRenderRequestBody":false,"settingEncodeUrl":true,"settingRebuildPath":true,"settingFollowRedirects":"global","_type":"request"},{"_id":"env_83c0b7d83703363968313035e2a2232afde16814","parentId":"wrk_46ed8e0af6aa43ccbd8c045226355a26","modified":1632633572737,"created":1632633572737,"name":"Base Environment","data":{},"dataPropertyOrder":null,"color":null,"isPrivate":false,"metaSortKey":1632633572737,"_type":"environment"},{"_id":"jar_83c0b7d83703363968313035e2a2232afde16814","parentId":"wrk_46ed8e0af6aa43ccbd8c045226355a26","modified":1632633572741,"created":1632633572741,"name":"Default Jar","cookies":[],"_type":"cookie_jar"},{"_id":"spc_dd1f4d6acf4c4959a7231fbdd8de132a","parentId":"wrk_46ed8e0af6aa43ccbd8c045226355a26","modified":1632633572580,"created":1632633572580,"fileName":"dbt-mssql","contents":"","contentType":"yaml","_type":"api_spec"}]} -------------------------------------------------------------------------------- /services/api_service/.insomia/awesome-dbt-api-docs-2021-10-03.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datnguye/dbt-postgres/1edd458a54d5735dd677298eb6b963f6163aaaa6/services/api_service/.insomia/awesome-dbt-api-docs-2021-10-03.png -------------------------------------------------------------------------------- /services/api_service/.insomia/awesome-dbt-api-redoc-2021-10-03.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datnguye/dbt-postgres/1edd458a54d5735dd677298eb6b963f6163aaaa6/services/api_service/.insomia/awesome-dbt-api-redoc-2021-10-03.png -------------------------------------------------------------------------------- /services/api_service/README.md: -------------------------------------------------------------------------------- 1 | # Awesome dbt 2 | This is the data services via Restful API 3 | 4 | [![SSH deployment](https://github.com/datnguye/dbt-postgres/actions/workflows/ssh-to-server.yml/badge.svg?branch=main)](https://github.com/datnguye/dbt-postgres/actions/workflows/ssh-to-server.yml) 5 | 6 | [Sample requests](.insomia/awesome-dbt-api-2021-10-03.json) with using [Insomia](https://insomnia.rest/) 7 | 8 | *Swagger*: 9 | 10 | ![Alt text](.insomia/awesome-dbt-api-docs-2021-10-03.png?raw=true "api redoc") 11 | 12 | *Redoc*: 13 | 14 | ![Alt text](.insomia/awesome-dbt-api-redoc-2021-10-03.png?raw=true "api redoc") 15 | 16 | 17 | 18 | ### DEV 19 | - [Optional] Set the `python` alias (if Linux) 20 | ``` 21 | update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1 22 | python --version 23 | ``` 24 | - [Install dbt & activate env](../../LOCALDEV.md) 25 | 26 | 27 | #### Start service locally 28 | ``` 29 | set env_postgres_user_secret=dbt_user 30 | set env_postgres_password_secret=dbt_user 31 | set env_postgres_host_secret=domain.com 32 | uvicorn main:app --host 0.0.0.0 --port 8000 --app-dir "./services/api_service" --reload 33 | ``` 34 | 35 | - To see the API document: http://127.0.0.1:8000/docs 36 | - To see the Open API document: http://127.0.0.1:8000/redoc 37 | 38 | 39 | -------------------------------------------------------------------------------- /services/api_service/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datnguye/dbt-postgres/1edd458a54d5735dd677298eb6b963f6163aaaa6/services/api_service/__init__.py -------------------------------------------------------------------------------- /services/api_service/aha_dbt/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datnguye/dbt-postgres/1edd458a54d5735dd677298eb6b963f6163aaaa6/services/api_service/aha_dbt/__init__.py -------------------------------------------------------------------------------- /services/api_service/aha_dbt/dbt.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | import json 3 | import os 4 | import subprocess 5 | import threading 6 | import prefect 7 | from enum import Enum 8 | from queue import Queue 9 | from prefect import Flow, Task 10 | from prefect.engine.state import Pending, Running 11 | from prefect.schedules.schedules import IntervalSchedule 12 | from config import settings 13 | from storage.factory import StorageFactory 14 | from storage.base import BaseStorage 15 | from storage.type import StorageType 16 | 17 | class DbtAction(Enum): 18 | DEPS = 'deps' 19 | RUN = 'run' 20 | SEED = 'seed' 21 | TEST = 'test' 22 | RUN_OPERATION = 'run-operation' 23 | 24 | 25 | class DBT(): 26 | def __init__(self, 27 | taskid: str = "UNKOWN_TASKID", 28 | action: str = DbtAction.RUN.value, 29 | macro: str = None, 30 | macro_args: dict = None, 31 | profiles_dir: str = None, 32 | target: str = None, 33 | project_dir: str = '.', 34 | vars: dict = None, 35 | models: str = None, 36 | full_refresh: bool = False, 37 | args: list = [], 38 | kwargs: list = [] 39 | ) -> None: 40 | self.taskid = taskid 41 | self.action = action 42 | self.macro = macro 43 | self.macro_args = macro_args 44 | self.profiles_dir = profiles_dir 45 | self.target = target 46 | self.project_dir = project_dir 47 | self.vars = vars 48 | self.models = models 49 | self.full_refresh = full_refresh 50 | self.args = args 51 | self.kwargs = kwargs 52 | 53 | 54 | def build(self): 55 | """ 56 | Build args 57 | """ 58 | arguments = [self.action] 59 | 60 | # Standard arguments 61 | if self.macro is not None: 62 | arguments.extend([self.macro]) 63 | 64 | if self.macro_args is not None: 65 | arguments.extend(["--args", json.dumps(self.macro_args)]) 66 | 67 | if self.target is not None: 68 | arguments.extend(["--target", self.target]) 69 | 70 | if self.profiles_dir is not None: 71 | arguments.extend(["--profiles-dir", self.profiles_dir]) 72 | 73 | if self.project_dir is not None: 74 | arguments.extend(["--project-dir", self.project_dir]) 75 | 76 | if self.vars is not None: 77 | arguments.extend(["--vars", json.dumps(self.vars)]) 78 | 79 | if self.models is not None: 80 | arguments.extend(["--models", self.models]) 81 | 82 | if self.full_refresh: 83 | arguments.extend(["--full-refresh"]) 84 | 85 | # Additional non-keyword arguments 86 | if self.args: 87 | arguments.extend(self.args) 88 | 89 | # Additional keyword arguments 90 | if self.kwargs: 91 | for key, value in self.kwargs: 92 | arguments.extend([key, value]) 93 | 94 | return arguments 95 | 96 | 97 | class DbtTask(Task): 98 | def __set_var__(self): 99 | """ 100 | Initiate enviroment variables 101 | """ 102 | if settings.DBT_STORAGE["type"] == StorageType.SQLSERVER.value: 103 | os.environ["ENV_DBT_SERVER"] = settings.DBT_STORAGE["server"] 104 | os.environ["ENV_DBT_PORT"] = str(settings.DBT_STORAGE["port"]) 105 | os.environ["ENV_DBT_DATABASE"] = settings.DBT_STORAGE["database"] 106 | os.environ["ENV_DBT_SCHEMA"] = settings.DBT_STORAGE.get("schema", None) or "dbo" 107 | os.environ["ENV_DBT_USER"] = settings.DBT_STORAGE["user"] 108 | os.environ["ENV_DBT_PASSWORD"] = settings.DBT_STORAGE["password"] 109 | 110 | if settings.DBT_STORAGE["type"] == StorageType.POSTGRES.value: 111 | os.environ["POSTGRES_HOST"] = settings.DBT_STORAGE["server"] 112 | os.environ["POSTGRES_PORT"] = str(settings.DBT_STORAGE["port"]) 113 | os.environ["POSTGRES_DB"] = settings.DBT_STORAGE["database"] 114 | os.environ["POSTGRES_SCHEMA"] = settings.DBT_STORAGE.get("schema", None) or "dbo" 115 | os.environ["POSTGRES_USER"] = settings.DBT_STORAGE["user"] 116 | os.environ["POSTGRES_PASS"] = settings.DBT_STORAGE["password"] 117 | 118 | 119 | 120 | def __del_var__(self): 121 | """ 122 | Clean up the enviroment variables 123 | """ 124 | if settings.DBT_STORAGE["type"] == StorageType.SQLSERVER.value: 125 | del os.environ["ENV_DBT_SERVER"] 126 | del os.environ["ENV_DBT_PORT"] 127 | del os.environ["ENV_DBT_DATABASE"] 128 | del os.environ["ENV_DBT_SCHEMA"] 129 | del os.environ["ENV_DBT_USER"] 130 | del os.environ["ENV_DBT_PASSWORD"] 131 | 132 | if settings.DBT_STORAGE["type"] == StorageType.POSTGRES.value: 133 | del os.environ["POSTGRES_HOST"] 134 | del os.environ["POSTGRES_PORT"] 135 | del os.environ["POSTGRES_DB"] 136 | del os.environ["POSTGRES_SCHEMA"] 137 | del os.environ["POSTGRES_USER"] 138 | del os.environ["POSTGRES_PASS"] 139 | 140 | 141 | def run(self, instance: DBT): 142 | """ 143 | Run dbt in subprocess 144 | """ 145 | logger = prefect.context.get("logger") 146 | self.__set_var__() 147 | 148 | dbt_cmd = ["dbt"] 149 | dbt_cmd.extend(instance.build()) 150 | logger.info(dbt_cmd) 151 | sp = subprocess.Popen( 152 | dbt_cmd, 153 | stdout=subprocess.PIPE, 154 | stderr=subprocess.STDOUT, 155 | cwd=os.getcwd(), 156 | close_fds=True 157 | ) 158 | logs = '' 159 | for line in iter(sp.stdout.readline, b''): 160 | line = line.decode('utf-8').rstrip() 161 | logger.info(line) 162 | logs = f"{logs}\n{line}" 163 | sp.wait() 164 | if sp.returncode != 0: 165 | raise prefect.engine.signals.FAIL() 166 | 167 | self.__del_var__() 168 | 169 | return ( 170 | f"Command exited with return code {sp.returncode}", 171 | logs, 172 | sp.returncode == 0 173 | ) 174 | 175 | 176 | class DbtExec(): 177 | def __init__(self, singleton: bool = True) -> None: 178 | self.singleton = singleton 179 | self.queue = Queue(maxsize=100) # Config max size 180 | if self.singleton: 181 | threading.Thread(target=self.__run_flow_worker__, daemon=True).start() 182 | self.log_storage = StorageFactory(base=BaseStorage(settings.LOG_STORAGE)).get_storage_instance() 183 | if self.log_storage: 184 | threading.Thread(target=self.__maintenance_worker__, daemon=True).start() 185 | 186 | 187 | def __maintenance_worker__(self): 188 | """" 189 | Maintenance worker 190 | """ 191 | schedule = IntervalSchedule( 192 | start_date=datetime.utcnow() + timedelta(seconds=1), 193 | interval=timedelta(days=1) 194 | ) 195 | with Flow("Dbt Maintenance", schedule=schedule) as flow: 196 | log_task = self.log_storage.maintenance() 197 | 198 | flow.run() 199 | 200 | 201 | def __run_flow_worker__(self): 202 | """" 203 | Queue worker - Running flow in queue if configured 204 | """ 205 | while True: 206 | flow_id, flow = self.queue.get() 207 | print(f'Working on {flow}') 208 | self.__run_flow__(flow_id=flow_id, flow=flow) 209 | print(f'Finished {flow}') 210 | self.queue.task_done() 211 | 212 | 213 | def __run_flow__(self, flow_id: str, flow: Flow): 214 | """ 215 | Run a flow 216 | """ 217 | self.log_storage.save(id=flow_id, data=Running(message=f"Task:{flow_id} is running")) 218 | state = flow.run() 219 | self.log_storage.save(id=flow_id, data=state) 220 | 221 | 222 | def get_execution_state(self, taskid: str = None): 223 | """ 224 | Get state of an execution 225 | """ 226 | return self.log_storage.get(id=taskid) 227 | 228 | 229 | def execute(self, 230 | flow_name: str = "Execution of dbt series | execute", 231 | dbts: list = [], 232 | taskid: str = None 233 | ): 234 | """ 235 | General dbt execution 236 | """ 237 | self.log_storage.save(id=taskid, data=Pending(message=f"Task:{taskid} is in queue")) 238 | 239 | dbt_tasks = [DbtTask(name=f"Flow:{taskid} - Task:{idx}") for idx, x in enumerate(dbts)] 240 | with Flow(name=flow_name) as f: 241 | prev_task = None 242 | for idx, dbt in enumerate(dbts): 243 | task = f.add_task(dbt_tasks[idx](instance=dbt)) 244 | if prev_task is not None: 245 | task.set_dependencies(upstream_tasks=[prev_task]) 246 | prev_task = task 247 | 248 | if self.singleton: 249 | self.queue.put((taskid, f)) 250 | return "Task queued" 251 | 252 | return self.__run_flow__(flow_id=taskid, flow=f) 253 | 254 | 255 | instance = DbtExec(singleton=settings.DBT_SINGLETON) -------------------------------------------------------------------------------- /services/api_service/api_v1/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datnguye/dbt-postgres/1edd458a54d5735dd677298eb6b963f6163aaaa6/services/api_service/api_v1/__init__.py -------------------------------------------------------------------------------- /services/api_service/api_v1/api.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter 2 | 3 | from api_v1.endpoints import login, dbt 4 | 5 | api_router = APIRouter() 6 | api_router.include_router(dbt.router, prefix="/dbt", tags=["dbt"]) 7 | api_router.include_router(login.router, tags=["login"]) -------------------------------------------------------------------------------- /services/api_service/api_v1/endpoints/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datnguye/dbt-postgres/1edd458a54d5735dd677298eb6b963f6163aaaa6/services/api_service/api_v1/endpoints/__init__.py -------------------------------------------------------------------------------- /services/api_service/api_v1/endpoints/dbt.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | from fastapi import APIRouter, BackgroundTasks 3 | from prefect.engine.serializers import Serializer 4 | from prefect.engine.state import Pending, Running 5 | import schemas 6 | from aha_dbt.dbt import instance, DBT, DbtAction 7 | from config import settings 8 | import libs 9 | 10 | router = APIRouter() 11 | PROJECT_DIR_ARGUMENT = "--project-dir" 12 | 13 | def dbt_custom_run(arguments: list = []): 14 | """ 15 | dbt custom run 16 | """ 17 | taskid = arguments[0] 18 | 19 | dbt = DBT( 20 | action=arguments[1], 21 | args=arguments[2:] 22 | ) 23 | if PROJECT_DIR_ARGUMENT not in arguments: 24 | dbt.project_dir=settings.DBT_PROJECT_DIR 25 | result = instance.execute(f"Custom dbt flow | Task: {taskid}", [dbt], taskid=taskid) 26 | 27 | return result #TODO: Parse result to readable message 28 | 29 | 30 | def dbt_run(taskid: str, full: bool = None): 31 | """ 32 | dbt run 33 | """ 34 | result = None 35 | if full: # Provision job 36 | result = instance.execute(f"Provisioning flow | Task: {taskid}", 37 | [ 38 | DBT( 39 | action=DbtAction.SEED.value, 40 | target=settings.DBT_TARGET, 41 | project_dir=settings.DBT_PROJECT_DIR, 42 | full_refresh=True 43 | ), 44 | DBT( 45 | action=DbtAction.RUN.value, 46 | target=settings.DBT_TARGET, 47 | project_dir=settings.DBT_PROJECT_DIR, 48 | models='+exposure:*', 49 | full_refresh=True 50 | ) 51 | ], 52 | taskid=taskid 53 | ) 54 | else: # Processing job 55 | result = instance.execute(f"Processing flow | Task: {taskid}", 56 | [ 57 | DBT( 58 | action=DbtAction.RUN.value, 59 | target=settings.DBT_TARGET, 60 | project_dir=settings.DBT_PROJECT_DIR, 61 | models='+exposure:*' 62 | ) 63 | ], 64 | taskid=taskid 65 | ) 66 | 67 | return result #TODO: Parse result to readable message 68 | 69 | 70 | @router.post("/provision", response_model=schemas.TaskMsg) 71 | async def provision( 72 | background_tasks: BackgroundTasks 73 | ) -> Any: 74 | """ 75 | Run dbt FULL for all models 76 | """ 77 | taskid = libs.new_taskid() 78 | background_tasks.add_task(dbt_run, taskid=taskid, full=True) 79 | return dict( 80 | taskid=taskid, 81 | msg="Provision job has been sent in the background" 82 | ) 83 | 84 | 85 | @router.post("/processing", response_model=schemas.TaskMsg) 86 | async def processing( 87 | background_tasks: BackgroundTasks 88 | ) -> Any: 89 | """ 90 | Run dbt DELTA for all models 91 | """ 92 | taskid = libs.new_taskid() 93 | background_tasks.add_task(dbt_run, taskid=taskid, full=False) 94 | return dict( 95 | taskid=taskid, 96 | msg="Processing job has been sent in the background" 97 | ) 98 | 99 | 100 | @router.post("/custom-run", response_model=schemas.TaskMsg) 101 | async def custom_run( 102 | dbt_args: schemas.DbtArgument, 103 | background_tasks: BackgroundTasks 104 | ) -> Any: 105 | """ 106 | Run dbt in whatever its native arguments 107 | """ 108 | taskid = libs.new_taskid() 109 | args = [ 110 | taskid, 111 | dbt_args.action 112 | ] 113 | 114 | args.extend( 115 | [x.value for x in dbt_args.args] \ 116 | if dbt_args.args and len(dbt_args.args) > 0 117 | else [] 118 | ) 119 | 120 | if dbt_args.kwargs and len(dbt_args.kwargs) > 0: 121 | for kw in dbt_args.kwargs: 122 | args.extend([kw.key, kw.value]) 123 | 124 | background_tasks.add_task(dbt_custom_run, arguments=args) 125 | return dict( 126 | taskid=taskid, 127 | msg="A dbt job has been sent in the background" 128 | ) 129 | 130 | 131 | @router.get("/task/{taskid}", response_model=schemas.TaskState) 132 | async def get_task_status( 133 | taskid: str 134 | ) -> Any: 135 | """ 136 | Get custom run state 137 | Returned code: 138 | - (99): Pending or Running 139 | - (-1): Failed 140 | - ( 0): Success 141 | """ 142 | data = instance.get_execution_state(taskid=taskid) 143 | 144 | if isinstance(data, Pending) or isinstance(data, Running): 145 | return dict( 146 | code = 99, 147 | msg = data.message 148 | ) 149 | 150 | return dict( 151 | code = -1 if data.is_failed() else 0, 152 | msg = data.message, 153 | result = str(data.result) if data.is_failed() else None 154 | #TODO: Serialize result 155 | ) -------------------------------------------------------------------------------- /services/api_service/api_v1/endpoints/login.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | from fastapi import APIRouter 3 | from schemas import token 4 | 5 | router = APIRouter() 6 | 7 | @router.post("/login/access-token", response_model=token.Token) 8 | def login_access_token() -> Any: 9 | """ 10 | [Not Implemented] OAuth2 compatible token login, get an access token for future requests 11 | """ 12 | pass -------------------------------------------------------------------------------- /services/api_service/config.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseSettings 2 | import os 3 | from libs.yml_helper import load_yaml_text 4 | from libs.object_view import ObjectView 5 | from storage.type import StorageType 6 | 7 | 8 | def load___config___config(): 9 | """ 10 | Load __config__ial config 11 | """ 12 | with open( 13 | f"{os.path.dirname(os.path.realpath(__file__))}/config.yml" 14 | ) as __config___file: 15 | return load_yaml_text(__config___file) 16 | __config__ = load___config___config() 17 | 18 | class Settings(BaseSettings): 19 | """ 20 | All API service settings 21 | """ 22 | API_V1_STR: str = "/api/v1" 23 | PROJECT_NAME: str = "Awesome dbt" 24 | 25 | DEBUG = bool(__config__.debug) 26 | DBT_SINGLETON = __config__.dbt.singleton 27 | DBT_TARGET = __config__.dbt.target 28 | DBT_PROJECT_DIR = __config__.dbt.project_dir 29 | 30 | def __get_log_storage__(type: str): 31 | """ 32 | Get storage configuration dict 33 | """ 34 | if type == StorageType.PICKLE.value: 35 | return dict( 36 | type=type, 37 | path=__config__.log_storage[0].path 38 | ) 39 | 40 | if type == StorageType.SQLSERVER.value: 41 | return dict( 42 | type=type, 43 | server=os.environ[__config__.log_storage[0].server], 44 | port=__config__.log_storage[0].port or 1433, 45 | database=__config__.log_storage[0].database, 46 | user=os.environ[__config__.log_storage[0].user], 47 | password=os.environ[__config__.log_storage[0].password] 48 | ) 49 | 50 | if type == StorageType.POSTGRES.value: 51 | return dict( 52 | type=type, 53 | server=os.environ[__config__.log_storage[0].server], 54 | port=__config__.log_storage[0].port or 1433, 55 | database=__config__.log_storage[0].database, 56 | user=os.environ[__config__.log_storage[0].user], 57 | password=os.environ[__config__.log_storage[0].password] 58 | ) 59 | 60 | return None 61 | LOG_STORAGE = None 62 | if __config__.log_storage and len(__config__.log_storage) > 0: 63 | type = __config__.log_storage[0].type 64 | LOG_STORAGE = __get_log_storage__(type=type) 65 | 66 | 67 | def __get_dbt_storage__(type: str): 68 | """ 69 | Get dbt storage configuration dict 70 | """ 71 | if type == StorageType.SQLSERVER.value: 72 | return dict( 73 | type=type, 74 | server=os.environ[__config__.dbt.storage[0].server], 75 | port=__config__.dbt.storage[0].port or 1433, 76 | database=__config__.dbt.storage[0].database, 77 | schema=__config__.dbt.storage[0].schema, 78 | user=os.environ[__config__.dbt.storage[0].user], 79 | password=os.environ[__config__.dbt.storage[0].password] 80 | ) 81 | 82 | if type == StorageType.POSTGRES.value: 83 | return dict( 84 | type=type, 85 | server=os.environ[__config__.dbt.storage[0].server], 86 | port=__config__.dbt.storage[0].port or 5432, 87 | database=__config__.dbt.storage[0].database, 88 | schema=__config__.dbt.storage[0].schema, 89 | user=os.environ[__config__.dbt.storage[0].user], 90 | password=os.environ[__config__.dbt.storage[0].password] 91 | ) 92 | 93 | raise "Not support dbt storage" 94 | DBT_STORAGE = None 95 | if __config__.dbt.storage and len(__config__.dbt.storage) > 0: 96 | type = __config__.dbt.storage[0].type 97 | DBT_STORAGE = __get_dbt_storage__(type=type) 98 | 99 | 100 | settings = Settings() 101 | -------------------------------------------------------------------------------- /services/api_service/config.yml: -------------------------------------------------------------------------------- 1 | debug: False 2 | 3 | dbt: 4 | singleton: True 5 | target: dev 6 | project_dir: "./dbt" 7 | storage: # Take first entry ONLY 8 | - entry: 9 | type: postgres 10 | server: env_postgres_host_secret 11 | user: env_postgres_user_secret 12 | password: env_postgres_password_secret 13 | port: 5432 14 | database: dbt 15 | schema: anly 16 | - entry: 17 | type: sqlserver 18 | server: env_sqlserver_host_secret 19 | port: 1433 20 | database: dbt 21 | schema: analytic 22 | user: env_sqlserver_user_secret 23 | password: env_sqlserver_password_secret 24 | 25 | log_storage: # Take first entry ONLY 26 | - entry: 27 | type: postgres 28 | server: env_postgres_host_secret 29 | port: 5432 30 | database: dbt 31 | user: env_postgres_user_secret 32 | password: env_postgres_password_secret 33 | - entry: 34 | type: sqlserver 35 | server: env_sqlserver_host_secret 36 | port: 1433 37 | database: dbt 38 | user: env_sqlserver_user_secret 39 | password: env_sqlserver_password_secret 40 | - entry: 41 | type: pickle 42 | path: C:/Temp/ -------------------------------------------------------------------------------- /services/api_service/deps.py: -------------------------------------------------------------------------------- 1 | # API Depedencies here -------------------------------------------------------------------------------- /services/api_service/libs/__init__.py: -------------------------------------------------------------------------------- 1 | from .common import new_taskid -------------------------------------------------------------------------------- /services/api_service/libs/common.py: -------------------------------------------------------------------------------- 1 | from uuid import uuid4 2 | 3 | def new_taskid(): 4 | """ 5 | Return a UUID string 6 | """ 7 | return str(uuid4()) -------------------------------------------------------------------------------- /services/api_service/libs/object_view.py: -------------------------------------------------------------------------------- 1 | class ObjectView(object): 2 | ''' 3 | Convert nested dict to dynamic object 4 | ''' 5 | def __init__(self, d, nested=True): 6 | self.origin = d 7 | for a, b in d.items(): 8 | if nested: 9 | if isinstance(b, (list, tuple)): 10 | setattr(self, a, [ObjectView(x) if isinstance(x, dict) else x for x in b]) 11 | else: 12 | setattr(self, a, ObjectView(b) if isinstance(b, dict) else b) 13 | else: 14 | setattr(self, a, b) 15 | 16 | def has_field(self, field): 17 | fields = field.split('.') 18 | if len(fields) == 0: 19 | return False 20 | 21 | obj = self.origin 22 | for f in fields: 23 | if f not in obj: 24 | return False 25 | obj = obj[f] 26 | 27 | return True -------------------------------------------------------------------------------- /services/api_service/libs/yml_helper.py: -------------------------------------------------------------------------------- 1 | from libs.object_view import ObjectView 2 | import yaml 3 | import yaml.scanner 4 | 5 | # the C version is faster, but it doesn't always exist 6 | try: 7 | from yaml import CSafeLoader as SafeLoader 8 | except ImportError: 9 | from yaml import SafeLoader # type: ignore # noqa: F401 10 | 11 | 12 | def load_yaml_text(contents): 13 | result = None 14 | try: 15 | result = yaml.load(contents, Loader=SafeLoader) 16 | except (yaml.scanner.ScannerError, yaml.YAMLError) as e: 17 | raise str(e) 18 | 19 | return ObjectView(result) 20 | 21 | def save_yaml_file(file_path, json): 22 | result = None 23 | try: 24 | yaml.add_representer(str, quoted_presenter) 25 | with open(file_path, 'w') as file: 26 | result = yaml.dump( 27 | data=json, 28 | stream=file, 29 | indent=2, 30 | default_flow_style=False 31 | ) 32 | except (yaml.scanner.ScannerError, yaml.YAMLError) as e: 33 | raise str(e) 34 | return result 35 | 36 | 37 | def quoted_presenter(dumper, data): 38 | """ 39 | Dump style 40 | """ 41 | # if "env_var" in data: 42 | # return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='"') 43 | return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='') -------------------------------------------------------------------------------- /services/api_service/main.py: -------------------------------------------------------------------------------- 1 | from fastapi import FastAPI 2 | from api_v1.api import api_router 3 | from config import settings 4 | 5 | app = FastAPI( 6 | title=settings.PROJECT_NAME, 7 | openapi_url=f"{settings.API_V1_STR}/openapi.json" 8 | ) 9 | 10 | app.include_router(api_router, prefix=settings.API_V1_STR) -------------------------------------------------------------------------------- /services/api_service/schemas/__init__.py: -------------------------------------------------------------------------------- 1 | from .msg import Msg, TaskMsg, TaskState 2 | from .token import Token 3 | from .dbt import DbtArgument -------------------------------------------------------------------------------- /services/api_service/schemas/dbt.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | from pydantic import BaseModel 3 | 4 | 5 | 6 | class DbtArgs(BaseModel): 7 | value: str 8 | 9 | 10 | class DbtKWArgs(BaseModel): 11 | key: str 12 | value: str 13 | 14 | 15 | class DbtArgument(BaseModel): 16 | action: str 17 | args: List[DbtArgs] = [] 18 | kwargs: List[DbtKWArgs] = [] -------------------------------------------------------------------------------- /services/api_service/schemas/msg.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | from pydantic import BaseModel 3 | 4 | class Msg(BaseModel): 5 | msg: str 6 | 7 | class TaskMsg(BaseModel): 8 | taskid: str 9 | msg: str 10 | 11 | class TaskState(BaseModel): 12 | code: int 13 | msg: Optional[str] 14 | result: Optional[str] -------------------------------------------------------------------------------- /services/api_service/schemas/token.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | 4 | class Token(BaseModel): 5 | access_token: str 6 | token_type: str -------------------------------------------------------------------------------- /services/api_service/storage/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datnguye/dbt-postgres/1edd458a54d5735dd677298eb6b963f6163aaaa6/services/api_service/storage/__init__.py -------------------------------------------------------------------------------- /services/api_service/storage/base.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | 4 | class BaseStorage(): 5 | def __init__(self, storage_config: dict = None) -> None: 6 | self.storage_config = storage_config 7 | self.type = self.storage_config["type"] \ 8 | if self.storage_config else None 9 | 10 | 11 | def save(self, id: str, data: Any) -> bool: 12 | raise Exception("Not yet implemented") 13 | 14 | 15 | def get(self, id) -> Any: 16 | raise Exception("Not yet implemented") 17 | 18 | 19 | def maintenance(self, log_retention_day: int = 30) -> bool: 20 | return True -------------------------------------------------------------------------------- /services/api_service/storage/factory.py: -------------------------------------------------------------------------------- 1 | from storage.postgres import PostgresStorage 2 | from storage.sqlserver import SqlServerStorage 3 | from storage.base import BaseStorage 4 | from storage.pickle import PickleStorage 5 | from storage.type import StorageType 6 | 7 | 8 | class StorageFactory(object): 9 | def __init__(self, base: BaseStorage) -> None: 10 | super().__init__() 11 | self.base = base 12 | 13 | 14 | def get_storage_instance(self): 15 | if self.base.type is None: 16 | return self.base 17 | 18 | if self.base.type == StorageType.PICKLE.value: 19 | return PickleStorage(self.base.storage_config) 20 | 21 | if self.base.type == StorageType.SQLSERVER.value: 22 | return SqlServerStorage(self.base.storage_config) 23 | 24 | if self.base.type == StorageType.POSTGRES.value: 25 | return PostgresStorage(self.base.storage_config) -------------------------------------------------------------------------------- /services/api_service/storage/model.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy.sql.schema import MetaData 2 | from sqlmodel import Field, SQLModel 3 | from datetime import datetime 4 | from uuid import UUID, uuid4 5 | 6 | 7 | class DbtLog(SQLModel, table=True): 8 | """ 9 | Table: __Dbt_Log 10 | """ 11 | __tablename__ = "__Dbt_Log" 12 | 13 | Id: UUID = Field(default_factory=uuid4, primary_key=True) 14 | TaskId: str = Field(max_length=128) 15 | Data: str = Field(index=False) 16 | Timestamp: datetime = Field(index=False, default_factory=datetime.utcnow) -------------------------------------------------------------------------------- /services/api_service/storage/pickle.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | from prefect.engine.state import State, TriggerFailed 3 | from storage.base import BaseStorage 4 | import pickle 5 | 6 | 7 | class PickleStorage(BaseStorage): 8 | def __init__(self, storage_config: dict) -> None: 9 | super().__init__(storage_config=storage_config) 10 | self.path = storage_config["path"] 11 | self.file_path = f'{self.path}{self.type}_{{id}}.pickle' 12 | 13 | 14 | def save(self, id: str, data: State) -> bool: 15 | """ 16 | Save data to a pickle file 17 | """ 18 | try: 19 | with open(self.file_path.format(id=id), 'wb') as handle: 20 | pickle.dump(data, handle, protocol=pickle.HIGHEST_PROTOCOL) 21 | except Exception as e: 22 | return False 23 | return True 24 | 25 | 26 | 27 | def get(self, id) -> State: 28 | """ 29 | Get data from a pickle file 30 | """ 31 | try: 32 | with open(self.file_path.format(id=id), 'rb') as handle: 33 | return pickle.load(handle) 34 | except Exception as e: 35 | return TriggerFailed(message=f"ID: {id} Not found") -------------------------------------------------------------------------------- /services/api_service/storage/postgres.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | from sqlalchemy.sql.expression import desc 3 | from storage.model import DbtLog 4 | from storage.base import BaseStorage 5 | from sqlmodel import Session, SQLModel, create_engine, select 6 | from prefect.engine.state import Pending, Running, State, Success, TriggerFailed 7 | from config import settings 8 | 9 | class PostgresStorage(BaseStorage): 10 | def __init__(self, storage_config: dict = None) -> None: 11 | super().__init__(storage_config=storage_config) 12 | self.engine = create_engine( 13 | url=( 14 | "postgresql://" 15 | f"{self.storage_config['user']}:{self.storage_config['password']}" 16 | f"@{self.storage_config['server']}:{self.storage_config['port']}" 17 | f"/{self.storage_config['database']}" 18 | ), 19 | echo=settings.DEBUG # shouldn't be enabled in prod 20 | ) 21 | SQLModel.metadata.create_all( 22 | bind=self.engine 23 | ) 24 | 25 | 26 | def __migration__(self): 27 | """ 28 | Schema mirgration 29 | """ 30 | raise "Not yet impletmented" 31 | 32 | 33 | def save(self, id: str, data: State) -> bool: 34 | """ 35 | Save task status 36 | """ 37 | with Session(bind=self.engine) as session: 38 | session.add(DbtLog(TaskId=id, Data=str(data))) 39 | session.commit() 40 | 41 | return True 42 | 43 | 44 | def get(self, id) -> State: 45 | """ 46 | Get task status 47 | """ 48 | with Session(bind=self.engine) as session: 49 | statement = select(DbtLog)\ 50 | .where(DbtLog.TaskId == id)\ 51 | .order_by(desc(DbtLog.Timestamp))\ 52 | .limit(1) 53 | result = session.exec(statement).first() 54 | 55 | if not result: 56 | return TriggerFailed(message=f"ID: {id} Not found") 57 | if "success" in result.Data.lower(): 58 | return Success(message=result.Data) 59 | if "running" in result.Data.lower(): 60 | return Running(message=result.Data) 61 | 62 | return Pending(message=result.Data) 63 | 64 | 65 | def maintenance(self, log_retention_day: int = 30): 66 | """ 67 | Run maintenance: 68 | - Log retention 69 | """ 70 | with Session(bind=self.engine) as session: 71 | penalty_date = datetime.utcnow() - timedelta(days=log_retention_day) 72 | statement = select(DbtLog).where(DbtLog.Timestamp < penalty_date) 73 | deletes = session.exec(statement) 74 | 75 | if deletes: 76 | for delete in deletes: 77 | session.delete(delete) 78 | session.commit() -------------------------------------------------------------------------------- /services/api_service/storage/sqlserver.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | from typing import Any 3 | from sqlalchemy.sql.expression import desc 4 | from storage.model import DbtLog 5 | from storage.base import BaseStorage 6 | from sqlmodel import Session, SQLModel, create_engine, select 7 | from prefect.engine.state import Pending, Running, State, Success, TriggerFailed 8 | from config import settings 9 | 10 | class SqlServerStorage(BaseStorage): 11 | def __init__(self, storage_config: dict = None) -> None: 12 | super().__init__(storage_config=storage_config) 13 | self.engine = create_engine( 14 | url="mssql+pyodbc://{user}:{password}@{server}:{port}/{database}?driver=ODBC+Driver+17+for+SQL+Server"\ 15 | .format( 16 | user=self.storage_config['user'], 17 | password=self.storage_config['password'], 18 | server=self.storage_config['server'], 19 | port=self.storage_config['port'], 20 | database=self.storage_config['database'] 21 | ), 22 | echo=settings.DEBUG # shouldn't be enabled in prod 23 | ) 24 | SQLModel.metadata.create_all( 25 | bind=self.engine 26 | ) 27 | 28 | 29 | def __migration__(self): 30 | """ 31 | Schema mirgration 32 | """ 33 | raise "Not yet impletmented" 34 | 35 | 36 | def save(self, id: str, data: State) -> bool: 37 | """ 38 | Save task status 39 | """ 40 | with Session(bind=self.engine) as session: 41 | session.add(DbtLog(TaskId=id, Data=str(data))) 42 | session.commit() 43 | 44 | return True 45 | 46 | 47 | def get(self, id) -> State: 48 | """ 49 | Get task status 50 | """ 51 | with Session(bind=self.engine) as session: 52 | statement = select(DbtLog)\ 53 | .where(DbtLog.TaskId == id)\ 54 | .order_by(desc(DbtLog.Timestamp))\ 55 | .limit(1) 56 | result = session.exec(statement).first() 57 | 58 | if not result: 59 | return TriggerFailed(message=f"ID: {id} Not found") 60 | if "success" in result.Data.lower(): 61 | return Success(message=result.Data) 62 | if "running" in result.Data.lower(): 63 | return Running(message=result.Data) 64 | 65 | return Pending(message=result.Data) 66 | 67 | 68 | def maintenance(self, log_retention_day: int = 30): 69 | """ 70 | Run maintenance: 71 | - Log retention 72 | """ 73 | with Session(bind=self.engine) as session: 74 | penalty_date = datetime.utcnow() - timedelta(days=log_retention_day) 75 | statement = select(DbtLog).where(DbtLog.Timestamp < penalty_date) 76 | deletes = session.exec(statement) 77 | 78 | if deletes: 79 | for delete in deletes: 80 | session.delete(delete) 81 | session.commit() 82 | -------------------------------------------------------------------------------- /services/api_service/storage/type.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class StorageType(Enum): 5 | """ 6 | Supported storage type 7 | """ 8 | PICKLE = 'pickle' 9 | SQLSERVER = 'sqlserver' 10 | POSTGRES = 'postgres' --------------------------------------------------------------------------------