├── tests ├── __init__.py ├── unit │ ├── __init__.py │ ├── test_model_utils.py │ └── test_limiting.py ├── README.md ├── url_list.txt ├── fastapi-test.py ├── test_logging.py ├── test_sensors.py ├── locustfile.py ├── test_sensors_latest.py ├── test_main.py └── locust-test-V3.py ├── openaq_api ├── __init__.py ├── models │ ├── __init__ .py │ ├── auth.py │ ├── responses.py │ └── logging.py ├── v3 │ ├── models │ │ ├── __init__.py │ │ └── utils.py │ └── routers │ │ ├── __init__.py │ │ ├── owners.py │ │ ├── countries.py │ │ ├── licenses.py │ │ ├── manufacturers.py │ │ ├── providers.py │ │ ├── locations.py │ │ ├── sensors.py │ │ ├── flags.py │ │ ├── trends.py │ │ ├── instruments.py │ │ ├── parameters.py │ │ └── latest.py ├── static │ ├── favicon.png │ ├── index.html │ └── openaq-logo.svg ├── exceptions.py ├── settings.py ├── requirements.txt ├── middleware.py ├── dependencies.py └── main.py ├── cloudfront_logs ├── __init__.py ├── cloudfront_logs │ ├── __init__.py │ ├── settings.py │ └── models.py ├── requirements.txt └── pyproject.toml ├── cdk ├── requirements.devcontainer.txt ├── cdk.json ├── Dockerfile ├── settings.py ├── README.md ├── setup.py ├── app.py ├── stacks │ ├── utils.py │ └── waf_rules.py └── cdk.context.json ├── pages ├── index.js ├── login │ ├── index.js │ ├── style.scss │ └── index.html ├── verify │ ├── index.js │ ├── style.scss │ └── index.html ├── check_email │ ├── index.js │ ├── style.scss │ └── index.html ├── revalidate │ ├── index.js │ ├── style.scss │ └── index.html ├── .gitignore ├── email_key │ ├── index.js │ ├── style.scss │ └── index.html ├── vite.config.js ├── style.scss ├── package.json ├── index.html ├── register │ ├── style.scss │ ├── index.html │ └── index.js └── logo.svg ├── .gitignore ├── .github └── workflows │ ├── test.yml │ ├── deploy-staging.yml │ └── deploy-aeolus.yml ├── pyproject.toml ├── CONTRIBUTING.md ├── MAINTENANCE.md └── README.md /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /openaq_api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cloudfront_logs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /openaq_api/models/__init__ .py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /openaq_api/v3/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /openaq_api/v3/routers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cdk/requirements.devcontainer.txt: -------------------------------------------------------------------------------- 1 | -e . 2 | -------------------------------------------------------------------------------- /cloudfront_logs/cloudfront_logs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pages/index.js: -------------------------------------------------------------------------------- 1 | import './style.scss'; 2 | -------------------------------------------------------------------------------- /pages/login/index.js: -------------------------------------------------------------------------------- 1 | import './style.scss'; -------------------------------------------------------------------------------- /pages/verify/index.js: -------------------------------------------------------------------------------- 1 | import './style.scss'; 2 | -------------------------------------------------------------------------------- /pages/check_email/index.js: -------------------------------------------------------------------------------- 1 | import './style.scss'; 2 | -------------------------------------------------------------------------------- /pages/revalidate/index.js: -------------------------------------------------------------------------------- 1 | import './style.scss'; 2 | -------------------------------------------------------------------------------- /openaq_api/static/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openaq/openaq-api-v2/HEAD/openaq_api/static/favicon.png -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | How to tell pytest which env settings to use 2 | ``` 3 | DOTENV=.env pytest 4 | DOTENV=.env.staging pytest 5 | ``` 6 | -------------------------------------------------------------------------------- /openaq_api/models/auth.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | 4 | class User(BaseModel): 5 | full_name: str 6 | email_address: str 7 | password_hash: str 8 | entity_type: str 9 | ip_address: str 10 | -------------------------------------------------------------------------------- /cdk/cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "python3 app.py", 3 | "context": { 4 | "aws-cdk:enableDiffNoFail": "true", 5 | "@aws-cdk/core:stackRelativeExports": "true", 6 | "@aws-cdk/aws-ecr-assets:dockerIgnoreSupport": true 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.ipynb 2 | .env* 3 | __pycache__/ 4 | *.py[cod] 5 | cdk.out/ 6 | .DS_Store 7 | *.egg-info 8 | .build 9 | openaq_api/openaq_api/templates/* 10 | openaq_api/openaq_api/static/assets/* 11 | .python-version 12 | Pipfile 13 | /openaq_api/venv/ 14 | /.scratch/ 15 | -------------------------------------------------------------------------------- /tests/unit/test_model_utils.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from datetime import date 3 | 4 | from openaq_api.v3.models.utils import fix_date 5 | 6 | 7 | def test_infinity_date(): 8 | d = fix_date("infinity") 9 | assert d == None 10 | 11 | 12 | def test_string_date(): 13 | d = fix_date("2024-01-01") 14 | assert isinstance(d, date) 15 | -------------------------------------------------------------------------------- /pages/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | 15 | # Editor directories and files 16 | .vscode/* 17 | !.vscode/extensions.json 18 | .idea 19 | .DS_Store 20 | *.suo 21 | *.ntvs* 22 | *.njsproj 23 | *.sln 24 | *.sw? 25 | -------------------------------------------------------------------------------- /openaq_api/exceptions.py: -------------------------------------------------------------------------------- 1 | from fastapi import ( 2 | HTTPException, 3 | status, 4 | ) 5 | 6 | NOT_AUTHENTICATED_EXCEPTION = HTTPException( 7 | status_code=status.HTTP_401_UNAUTHORIZED, 8 | detail="Invalid credentials", 9 | ) 10 | 11 | def TOO_MANY_REQUESTS(headers=None): 12 | return HTTPException( 13 | status_code=status.HTTP_429_TOO_MANY_REQUESTS, 14 | detail="Too many requests", 15 | headers=headers, 16 | ) 17 | -------------------------------------------------------------------------------- /openaq_api/models/responses.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Any 3 | 4 | from pydantic import BaseModel 5 | 6 | logger = logging.getLogger("responses") 7 | 8 | 9 | class Meta(BaseModel): 10 | name: str = "openaq-api" 11 | license: str = "" 12 | website: str = "/" 13 | page: int = 1 14 | limit: int = 100 15 | found: int | str | None = None 16 | 17 | 18 | # Abstract class for all responses 19 | class OpenAQResult(BaseModel): 20 | meta: Meta = Meta() 21 | results: list[Any] = [] 22 | -------------------------------------------------------------------------------- /openaq_api/v3/models/utils.py: -------------------------------------------------------------------------------- 1 | from dateutil.parser import parse 2 | from dateutil.tz import UTC 3 | from datetime import date, datetime 4 | 5 | 6 | def fix_date( 7 | d: date | str | int | None, 8 | ): 9 | if isinstance(d, date): 10 | pass 11 | elif isinstance(d, str): 12 | if d == "infinity": 13 | d = None 14 | elif d == "-infinity": 15 | d = None 16 | else: 17 | d = parse(d).date() 18 | elif isinstance(d, datetime): 19 | d = d.date() 20 | 21 | return d 22 | -------------------------------------------------------------------------------- /cloudfront_logs/cloudfront_logs/settings.py: -------------------------------------------------------------------------------- 1 | from os import environ 2 | from pathlib import Path 3 | 4 | from pydantic_settings import BaseSettings, SettingsConfigDict 5 | 6 | 7 | def get_env(): 8 | parent = Path(__file__).resolve().parent.parent 9 | env_file = Path.joinpath(parent, environ.get("DOTENV", ".env")) 10 | return env_file 11 | 12 | 13 | class Settings(BaseSettings): 14 | ENV: str = "staging" 15 | CF_LOGS_LOG_LEVEL: str = "INFO" 16 | 17 | model_config = SettingsConfigDict( 18 | extra="ignore", 19 | env_file=get_env(), 20 | env_file_encoding="utf-8", 21 | ) 22 | 23 | 24 | settings = Settings() 25 | -------------------------------------------------------------------------------- /cloudfront_logs/requirements.txt: -------------------------------------------------------------------------------- 1 | annotated-types==0.7.0 ; python_version >= "3.11" 2 | botocore==1.37.22 ; python_version >= "3.11" 3 | jmespath==1.0.1 ; python_version >= "3.11" 4 | pydantic-core==2.33.0 ; python_version >= "3.11" 5 | pydantic-settings==2.8.1 ; python_version >= "3.11" 6 | pydantic==2.11.0 ; python_version >= "3.11" 7 | pyhumps==3.8.0 ; python_version >= "3.11" 8 | python-dateutil==2.9.0.post0 ; python_version >= "3.11" 9 | python-dotenv==1.1.0 ; python_version >= "3.11" 10 | s3transfer==0.11.4 ; python_version >= "3.11" 11 | six==1.17.0 ; python_version >= "3.11" 12 | typing-extensions==4.13.0 ; python_version >= "3.11" 13 | typing-inspection==0.4.0 ; python_version >= "3.11" 14 | urllib3==2.3.0 ; python_version >= "3.11" 15 | -------------------------------------------------------------------------------- /pages/email_key/index.js: -------------------------------------------------------------------------------- 1 | import './style.scss'; 2 | 3 | 4 | const emailInput = document.querySelector('.js-email-input'); 5 | const passwordInput = document.querySelector('.js-password-input'); 6 | 7 | const submitBtn = document.querySelector('.js-submit-btn'); 8 | 9 | emailInput.addEventListener('input', () => { 10 | checkFormFieldsComplete() 11 | }) 12 | 13 | passwordInput.addEventListener('input', (e) => { 14 | checkFormFieldsComplete(); 15 | }); 16 | 17 | let formfieldsCheckTimeout; 18 | 19 | function checkFormFieldsComplete() { 20 | clearTimeout(formfieldsCheckTimeout) 21 | formfieldsCheckTimeout = setTimeout(() => { 22 | if (emailInput.value != '' && passwordInput.value != '') { 23 | submitBtn.disabled = false; 24 | } else { 25 | submitBtn.disabled = true; 26 | } 27 | }, 300) 28 | } 29 | -------------------------------------------------------------------------------- /cdk/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM public.ecr.aws/sam/build-python3.11:1.97.0-20230905191328-x86_64 2 | 3 | WORKDIR /var/task 4 | 5 | COPY requirements.docker.txt /var/task/requirements.txt 6 | 7 | RUN pip install --upgrade pip 8 | 9 | RUN mkdir -p /var/task/python 10 | 11 | RUN pip install -t /var/task/python -r /var/task/requirements.txt 12 | 13 | # Reduce package size and remove useless files 14 | RUN \ 15 | cd python \ 16 | find . -type f -name '*.pyc' | \ 17 | while read f; do n=$(echo $f | \ 18 | sed 's/__pycache__\///' | \ 19 | sed 's/.cpython-[2-3] [0-9]//'); \ 20 | cp $f $n; \ 21 | done \ 22 | && find . -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf \ 23 | && find . -type d -a -name 'tests' -print0 | xargs -0 rm -rf \ 24 | && find . -type d -a -name '*.dist-info' -print0 | xargs -0 rm -rf \ 25 | && find . -type f -a -name '*.so' -exec strip "{}" \; 26 | 27 | CMD cp -r /var/task/python /tmp/ 28 | 29 | -------------------------------------------------------------------------------- /pages/vite.config.js: -------------------------------------------------------------------------------- 1 | import { resolve } from 'path'; 2 | import { defineConfig } from 'vite'; 3 | import { splitVendorChunkPlugin } from 'vite'; 4 | import lightningcss from 'vite-plugin-lightningcss'; 5 | import htmlPurge from 'vite-plugin-purgecss' 6 | 7 | 8 | export default defineConfig({ 9 | plugins: [ 10 | splitVendorChunkPlugin(), 11 | htmlPurge({ 12 | safelist: ['strength-meter__bar--ok','strength-meter__bar--alert','strength-meter__bar--warning'] 13 | }), 14 | lightningcss({ 15 | browserslist: '>= 0.25%', 16 | }), 17 | ], 18 | 19 | build: { 20 | minify: true, 21 | rollupOptions: { 22 | input: { 23 | main: resolve(__dirname, 'index.html'), 24 | register: resolve(__dirname, 'register/index.html'), 25 | login: resolve(__dirname, 'login/index.html'), 26 | check_email: resolve(__dirname, 'check_email/index.html'), 27 | verify: resolve(__dirname, 'verify/index.html'), 28 | email_key: resolve(__dirname, 'email_key/index.html'), 29 | }, 30 | }, 31 | }, 32 | }); 33 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | pull_request: 5 | types: [opened, reopened] 6 | branches: 7 | - main 8 | - staging 9 | workflow_dispatch: 10 | 11 | 12 | env: 13 | DATABASE_READ_USER: "placeholder" 14 | DATABASE_READ_PASSWORD: "placeholder" 15 | DATABASE_WRITE_USER: "placeholder" 16 | DATABASE_WRITE_PASSWORD: "placeholder" 17 | DATABASE_DB: "placeholder" 18 | DATABASE_HOST: "placeholder" 19 | DATABASE_PORT: 42 20 | EXPLORER_API_KEY: "placeholder" 21 | 22 | jobs: 23 | test: 24 | runs-on: ubuntu-latest 25 | steps: 26 | - name: Checkout repo 27 | uses: actions/checkout@v3 28 | 29 | - uses: actions/setup-python@v3 30 | with: 31 | python-version: '3.11' 32 | 33 | - name: Install dependencies 34 | working-directory: ./openaq_api 35 | run: | 36 | pip install -r requirements_dev.txt 37 | 38 | - name: Run tests 39 | working-directory: ./openaq_api 40 | run: | 41 | pytest tests/unit/test_v3_queries.py -vv -s 42 | 43 | -------------------------------------------------------------------------------- /cloudfront_logs/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "cloudfront_logs" 3 | version = "0.1.0" 4 | description = "OpenAQ REST API log transfer" 5 | authors = [ 6 | {name = "Russ Biggs",email = "russ@openaq.org"}, 7 | {name = "Christian Parker", email = "chris@talloaks.io"} 8 | ] 9 | license = {text = "MIT"} 10 | readme = "README.md" 11 | requires-python = ">=3.11" 12 | dependencies = [ 13 | "pyhumps (>=3.8.0,<4.0.0)", 14 | "python-dotenv (>=1.1.0,<2.0.0)", 15 | "s3transfer (>=0.11.4,<0.12.0)", 16 | "six (>=1.17.0,<2.0.0)", 17 | "urllib3 (>=2.3.0,<3.0.0)", 18 | "python-dateutil (>=2.9.0.post0,<3.0.0)", 19 | "pydantic (>=2.11.0,<3.0.0)", 20 | "pydantic-settings (>=2.8.1,<3.0.0)", 21 | "jmespath (>=1.0.1,<2.0.0)", 22 | "annotated-types (>=0.7.0,<0.8.0)" 23 | ] 24 | 25 | [tool.poetry.group.test.dependencies] 26 | pytest = "^8.3.5" 27 | httpx = "^0.28.1" 28 | 29 | [tool.poetry.group.lint.dependencies] 30 | ruff = "^0.9.10" 31 | black = "^25.1.0" 32 | 33 | [build-system] 34 | requires = ["poetry-core>=2.0.0,<3.0.0"] 35 | build-backend = "poetry.core.masonry.api" 36 | -------------------------------------------------------------------------------- /pages/style.scss: -------------------------------------------------------------------------------- 1 | @use 'openaq-design-system/scss/utilities'; 2 | @use 'openaq-design-system/scss/header'; 3 | @use 'openaq-design-system/scss/variables' as variables; 4 | 5 | body, 6 | html { 7 | margin: 0; 8 | padding: 0; 9 | } 10 | 11 | * { 12 | box-sizing: border-box; 13 | } 14 | 15 | .container { 16 | margin-top: 50px; 17 | display: flex; 18 | align-items: center; 19 | flex-direction: column; 20 | } 21 | 22 | .openaq-logo { 23 | padding: 15px; 24 | width: 20%; 25 | height: auto; 26 | } 27 | 28 | .links-list { 29 | margin: 0; 30 | padding: 0; 31 | display: flex; 32 | flex-direction: column; 33 | justify-content: center; 34 | list-style: none; 35 | } 36 | 37 | .links-list > li { 38 | padding: 5px 0 5px 0; 39 | } 40 | 41 | .gradient-title { 42 | background-color: variables.$lavender100; 43 | background-image: linear-gradient( 44 | 92deg, 45 | variables.$sky120 0%, 46 | variables.$lavender100 10% 47 | ); 48 | background-size: 100%; 49 | background-clip: text; 50 | -webkit-background-clip: text; 51 | -moz-background-clip: text; 52 | -webkit-text-fill-color: transparent; 53 | -moz-text-fill-color: transparent; 54 | } 55 | -------------------------------------------------------------------------------- /pages/verify/style.scss: -------------------------------------------------------------------------------- 1 | @use 'openaq-design-system/scss/variables' as variables; 2 | @use 'openaq-design-system/scss/resets'; 3 | @use 'openaq-design-system/scss/bubbles'; 4 | @use 'openaq-design-system/scss/utilities'; 5 | @use 'openaq-design-system/scss/header'; 6 | 7 | html body { 8 | margin: 0; 9 | padding: 0; 10 | } 11 | 12 | * { 13 | font-family: 'Space Grotesk', sans-serif; 14 | } 15 | 16 | .gradient-title { 17 | background-color: variables.$lavender100; 18 | background-image: linear-gradient( 19 | 92deg, 20 | variables.$sky120 0%, 21 | variables.$lavender100 10% 22 | ); 23 | background-size: 100%; 24 | background-clip: text; 25 | -webkit-background-clip: text; 26 | -moz-background-clip: text; 27 | -webkit-text-fill-color: transparent; 28 | -moz-text-fill-color: transparent; 29 | } 30 | 31 | .main { 32 | background-color: white; 33 | display: flex; 34 | min-height: calc(100vh - 80px); 35 | justify-content: center; 36 | } 37 | 38 | .content { 39 | width: 800px; 40 | display: flex; 41 | flex-direction: column; 42 | align-items: center; 43 | 44 | @media (max-width: 450px) { 45 | & { 46 | margin: 30px; 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /pages/check_email/style.scss: -------------------------------------------------------------------------------- 1 | @use 'openaq-design-system/scss/variables' as variables; 2 | @use 'openaq-design-system/scss/resets'; 3 | @use 'openaq-design-system/scss/bubbles'; 4 | @use 'openaq-design-system/scss/utilities'; 5 | @use 'openaq-design-system/scss/header'; 6 | 7 | html body { 8 | margin: 0; 9 | padding: 0; 10 | } 11 | 12 | * { 13 | font-family: 'Space Grotesk', sans-serif; 14 | } 15 | 16 | .gradient-title { 17 | background-color: variables.$lavender100; 18 | background-image: linear-gradient( 19 | 92deg, 20 | variables.$sky120 0%, 21 | variables.$lavender100 10% 22 | ); 23 | background-size: 100%; 24 | background-clip: text; 25 | -webkit-background-clip: text; 26 | -moz-background-clip: text; 27 | -webkit-text-fill-color: transparent; 28 | -moz-text-fill-color: transparent; 29 | } 30 | 31 | .main { 32 | background-color: white; 33 | display: flex; 34 | min-height: calc(100vh - 80px); 35 | justify-content: center; 36 | } 37 | 38 | .content { 39 | width: 800px; 40 | display: flex; 41 | flex-direction: column; 42 | align-items: center; 43 | 44 | @media (max-width: 450px) { 45 | & { 46 | margin: 30px; 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /tests/url_list.txt: -------------------------------------------------------------------------------- 1 | /v1/measurements?location=%E4%BA%91%E6%A0%96&page=1&limit=100&date_from=2020-11-19T18%3A00%3A00.000Z&date_to=2020-11-27T18%3A00%3A00.000Z 2 | /v1/measurements?location=%E9%9D%92%E6%B3%A5%E6%B4%BC%E6%A1%A5&page=1&limit=100&date_from=2020-11-19T18%3A00%3A00.000Z&date_to=2020-11-27T18%3A00%3A00.000Z 3 | /v1/measurements 4 | /v1/countries 5 | /v2/averages?temporal=dow¶meter=pm10&location=2&spatial=location 6 | /v2/measurements?location=%E4%BA%91%E6%A0%96&page=1&limit=10000&date_from=2020-11-19T18%3A00%3A00.000Z&date_to=2020-11-27T18%3A00%3A00.000Z 7 | /v2/measurements?location=%E9%9D%92%E6%B3%A5%E6%B4%BC%E6%A1%A5&page=1&limit=10000&date_from=2020-11-19T18%3A00%3A00.000Z&date_to=2020-11-27T18%3A00%3A00.000Z 8 | /v2/averages?temporal=moy¶meter=pm10&spatial=location&location=9 9 | /v2/averages?temporal=dow¶meter=pm10&spatial=location&location=9 10 | /v2/measurements 11 | /v2/countries 12 | /v2/locations 13 | /v2/locations/6 14 | /v2/locations?page=3237¶meter=pm25&limit=1 15 | /v1/locations 16 | /v1/latest 17 | /v2/latest 18 | /v2/locations?limit=100&page=1&offset=0&sort=desc¶meter=pm10¶meter=pm25&radius=1000&order_by=lastUpdated 19 | /v2/locations?page=3662¶meter=pm25&limit=1 20 | -------------------------------------------------------------------------------- /tests/fastapi-test.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import requests 3 | 4 | base_url = "http://localhost:8000" 5 | 6 | # list of endpoint paths 7 | endpoints = [ 8 | "/v2/averages", 9 | "/v2/cities", 10 | "/v1/cities", 11 | "/v1/countries", 12 | "/v1/countries/13", 13 | "/v2/countries", 14 | "/v2/countries/13", 15 | "/v1/locations", 16 | "/v1/locations/2178", 17 | "/v1/latest", 18 | "/v1/latest/2178", 19 | "/v2/locations", 20 | "/v2/locations/2178", 21 | "/v2/latest", 22 | "/v2/latest/2178", 23 | "/v2/manufacturers", 24 | "/v2/models", 25 | "/v1/measurements", 26 | "/v2/measurements", 27 | "/v2/locations/tiles/2/2/1.pbf", 28 | "/v1/parameters", 29 | "/v2/parameters", 30 | "/v2/projects", 31 | "/v2/projects/22", 32 | "/v1/sources", 33 | "/v2/sources", 34 | "/v2/summary", 35 | "/v3/countries", 36 | "/v3/countries/13", 37 | "/v3/locations", 38 | "/v3/locations/2178", 39 | "/v3/locations/2178/measurements", 40 | "/v3/parameters", 41 | "/v3/parameters/2", 42 | "/v3/providers", 43 | "/v3/providers/62", 44 | "/v3/sensors/662", 45 | "/v3/locations/tiles/2/2/1.pbf", 46 | ] 47 | 48 | 49 | @pytest.mark.parametrize("endpoint", endpoints) 50 | def test_endpoints(endpoint): 51 | response = requests.get(base_url + endpoint) 52 | assert response.status_code == 200 53 | -------------------------------------------------------------------------------- /cdk/settings.py: -------------------------------------------------------------------------------- 1 | from os import environ 2 | from pathlib import Path 3 | from typing import List 4 | 5 | 6 | from pydantic_settings import BaseSettings, SettingsConfigDict 7 | 8 | 9 | def get_env(): 10 | parent = Path(__file__).resolve().parent.parent 11 | env_file = Path.joinpath(parent, environ.get("DOTENV", ".env")) 12 | return env_file 13 | 14 | 15 | class Settings(BaseSettings): 16 | CDK_ACCOUNT: str 17 | CDK_REGION: str 18 | VPC_ID: str | None = None 19 | ENV: str = "staging" 20 | PROJECT: str = "openaq" 21 | API_CACHE_TIMEOUT: int = 900 22 | LOG_LEVEL: str = "INFO" 23 | REDIS_PORT: int | None = 6379 24 | REDIS_SECURITY_GROUP_ID: str | None = None 25 | API_LAMBDA_MEMORY_SIZE: int = 1536 26 | API_LAMBDA_TIMEOUT: int = 15 # lambda timeout in seconds 27 | HOSTED_ZONE_ID: str = None 28 | HOSTED_ZONE_NAME: str = None 29 | DOMAIN_NAME: str = None 30 | CERTIFICATE_ARN: str = None 31 | CF_LOGS_LAMBDA_MEMORY_SIZE: int = 1792 32 | CF_LOG_LAMBDA_TIMEOUT: int = 180 # lambda timeout in seconds 33 | WAF_RATE_LIMIT_EVALUATION_WINDOW: int = 60 # in seconds 34 | WAF_RATE_LIMIT: int 35 | WAF_BLOCK_IPS: List[str] | None = None 36 | 37 | model_config = SettingsConfigDict( 38 | extra="ignore", 39 | env_file=get_env(), 40 | env_file_encoding="utf-8", 41 | ) 42 | 43 | 44 | settings = Settings() 45 | -------------------------------------------------------------------------------- /cdk/README.md: -------------------------------------------------------------------------------- 1 | 2 | # OpenAQ Version 2 API CDK Deployment 3 | 4 | This is an [AWS CDK](https://aws.amazon.com/cdk/) project that can be used to deploy Lambda Functions for data ingest and the FastAPI based OpenAQ Version 2 API. 5 | 6 | This code will package the API code and all dependencies into a package.zip file using Docker. CDK can then be used to deploy the code as an AWS CloudFormation Stack. 7 | 8 | It is recommended to install this code in a virtual environment. 9 | 10 | Before install the code, you must have Docker and CDK installed on the system. 11 | 12 | CDK can be installed using the node package manager: 13 | 14 | ` 15 | npm install -g aws-cdk 16 | ` 17 | 18 | Install the cdk project: 19 | 20 | ` 21 | pip install -e . 22 | ` 23 | 24 | You must have your environment set up ([Setting up your Environment](../README.md)) prior to deploying. 25 | 26 | There are three targets for building: 27 | - openaq-lcs-apistaging (Staging API) 28 | - openaq-lcs-api (Production API) 29 | - openaq-lcs-ingeststaging (Ingest Lambda) 30 | 31 | To build the project: 32 | 33 | ` 34 | cdk synth [target] 35 | ` 36 | 37 | To see what changes will be made in a deploy: 38 | 39 | ` 40 | cdk diff [target] 41 | ` 42 | 43 | To deploy: 44 | 45 | ` 46 | cdk deploy [target] 47 | ` 48 | 49 | **Note that you do not need to run the synth and diff separately to deploy.** 50 | 51 | `cdk deploy` will return the URL that each resource is available through. -------------------------------------------------------------------------------- /pages/check_email/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 10 | 11 | 16 | 20 | 24 | OpenAQ API Registration 25 | 26 | 27 |
28 |
29 | 32 |
33 |
34 |
35 |
36 |

API Key Registration

37 | 38 |

Check your email for a verification link

39 |
40 |
41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /cdk/setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | aws_cdk_version = "1.79.0" 4 | aws_cdk_reqs = [ 5 | "core", 6 | "aws-lambda", 7 | "aws-apigatewayv2", 8 | "aws-apigatewayv2-integrations", 9 | "aws-lambda-python", 10 | "aws-lambda-nodejs", 11 | "aws-s3", 12 | "aws-lambda-event-sources", 13 | "aws_s3_notifications", 14 | ] 15 | 16 | install_requires = ["docker"] 17 | install_requires.append([f"aws_cdk.{x}=={aws_cdk_version}" for x in aws_cdk_reqs]) 18 | 19 | setuptools.setup( 20 | name="openaq-fastapi", 21 | version="0.0.1", 22 | description="An empty CDK Python app", 23 | long_description="hi", 24 | long_description_content_type="text/markdown", 25 | author="author", 26 | # package_dir={"": "../openaq_api"}, 27 | # packages=setuptools.find_packages(where="openaq_api"), 28 | install_requires=install_requires, 29 | python_requires=">=3.6", 30 | classifiers=[ 31 | "Development Status :: 4 - Beta", 32 | "Intended Audience :: Developers", 33 | "License :: OSI Approved :: Apache Software License", 34 | "Programming Language :: JavaScript", 35 | "Programming Language :: Python :: 3 :: Only", 36 | "Programming Language :: Python :: 3.6", 37 | "Programming Language :: Python :: 3.7", 38 | "Programming Language :: Python :: 3.8", 39 | "Topic :: Software Development :: Code Generators", 40 | "Topic :: Utilities", 41 | "Typing :: Typed", 42 | ], 43 | ) 44 | -------------------------------------------------------------------------------- /pages/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pages", 3 | "private": true, 4 | "version": "0.0.0", 5 | "type": "module", 6 | "scripts": { 7 | "dev": "vite", 8 | "build": "NODE_ENV=production vite build", 9 | "clean": "", 10 | "mkdir": "mkdir -p ../openaq_api/openaq_api/templates/register && mkdir -p ../openaq_api/openaq_api/templates/verify && mkdir -p ../openaq_api/openaq_api/templates/check_email && mkdir -p ../openaq_api/openaq_api/templates/email_key", 11 | "move": "cp ./dist/index.html ../openaq_api/openaq_api/templates/index.html && cp ./dist/register/index.html ../openaq_api/openaq_api/templates/register/index.html && cp ./dist/verify/index.html ../openaq_api/openaq_api/templates/verify/index.html && cp ./dist/check_email/index.html ../openaq_api/openaq_api/templates/check_email/index.html && cp ./dist/email_key/index.html ../openaq_api/openaq_api/templates/email_key/index.html && cp -r ./dist/assets ../openaq_api/openaq_api/static", 12 | "deploy": "yarn run build && yarn run mkdir && yarn run move", 13 | "preview": "vite preview" 14 | }, 15 | "devDependencies": { 16 | "lightningcss": "^1.18.0", 17 | "sass": "^1.59.3", 18 | "vite": "^4.2.0", 19 | "vite-plugin-lightningcss": "^0.0.3", 20 | "vite-plugin-purgecss": "^0.2.12" 21 | }, 22 | "dependencies": { 23 | "@zxcvbn-ts/core": "^2.2.1", 24 | "@zxcvbn-ts/language-common": "^2.0.1", 25 | "@zxcvbn-ts/language-en": "^2.1.0", 26 | "openaq-design-system": "github:openaq/openaq-design-system#v-4.0.0" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /tests/test_logging.py: -------------------------------------------------------------------------------- 1 | from fastapi.testclient import TestClient 2 | import pytest 3 | from starlette.middleware.base import BaseHTTPMiddleware 4 | 5 | from openaq_api.main import app 6 | from openaq_api.models.logging import HTTPLog, LogType 7 | 8 | captured_requests = [] 9 | 10 | 11 | class CaptureMiddleware(BaseHTTPMiddleware): 12 | async def dispatch(self, request, call_next): 13 | captured_requests.append(request) 14 | return await call_next(request) 15 | 16 | 17 | app.add_middleware(CaptureMiddleware) 18 | 19 | 20 | @pytest.fixture 21 | def client(): 22 | with TestClient(app) as c: 23 | c.captured_requests = captured_requests 24 | yield c 25 | 26 | 27 | paths = [ 28 | ("/v3/instruments/3", {"instruments_id": "3"}), 29 | ("/v3/locations/42", {"locations_id": "42"}), 30 | ("/v3/locations/42?locations_id=24", {"locations_id": "42"}), 31 | ("/v3/sensors/42/measurements", {"sensors_id": "42"}), 32 | ( 33 | "/v3/sensors/42/measurements?limit=100&page=2", 34 | {"sensors_id": "42", "limit": "100", "page": "2"}, 35 | ), 36 | ("/v3/sensors/54", {"sensors_id": "54"}), 37 | ] 38 | 39 | 40 | @pytest.mark.parametrize("path,params_obj", paths) 41 | class TestHTTPLog: 42 | def test_params_obj_property(self, client, path, params_obj): 43 | client.captured_requests.clear() 44 | response = client.get( 45 | path, 46 | ) 47 | request = client.captured_requests[0] 48 | log = HTTPLog( 49 | type=LogType.SUCCESS, request=request, http_code=response.status_code 50 | ) 51 | assert log.params_obj == params_obj 52 | -------------------------------------------------------------------------------- /pages/verify/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 10 | 11 | 16 | 20 | 24 | OpenAQ API Registration 25 | 26 | 27 |
28 |
29 | 32 |
33 |
34 |
35 |
36 |

API Key Registration

37 | {% if not error %} 38 |

{% if verify %} Email address verified. {% endif %}You will recieve a email containing your OpenAQ API key shortly.

39 | {% else %} 40 |

{{ error_message }}

41 | {% endif %} 42 |
43 |
44 | 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /cdk/app.py: -------------------------------------------------------------------------------- 1 | import aws_cdk 2 | from aws_cdk import ( 3 | Environment, 4 | Tags, 5 | ) 6 | 7 | from stacks.lambda_api_stack import LambdaApiStack 8 | 9 | from settings import settings 10 | 11 | # this is the only way that I can see to allow us to have 12 | # one settings file and import it from there. I would recommend 13 | # a better package structure in the future. 14 | import os 15 | import sys 16 | 17 | p = os.path.abspath("../api") 18 | sys.path.insert(1, p) 19 | from openaq_api.settings import settings as lambda_env 20 | 21 | p = os.path.abspath("../cloudfront_logs") 22 | sys.path.insert(1, p) 23 | from cloudfront_logs.settings import settings as cloudfront_logs_lambda_env 24 | 25 | app = aws_cdk.App() 26 | 27 | env = Environment(account=settings.CDK_ACCOUNT, region=settings.CDK_REGION) 28 | 29 | api = LambdaApiStack( 30 | app, 31 | f"openaq-api-{settings.ENV}", 32 | env_name=settings.ENV, 33 | lambda_env=lambda_env, 34 | vpc_id=settings.VPC_ID, 35 | redis_security_group_id=settings.REDIS_SECURITY_GROUP_ID, 36 | redis_port=settings.REDIS_PORT, 37 | api_lambda_timeout=settings.API_LAMBDA_TIMEOUT, 38 | api_lambda_memory_size=settings.API_LAMBDA_MEMORY_SIZE, 39 | hosted_zone_name=settings.HOSTED_ZONE_NAME, 40 | hosted_zone_id=settings.HOSTED_ZONE_ID, 41 | domain_name=settings.DOMAIN_NAME, 42 | cert_arn=settings.CERTIFICATE_ARN, 43 | cloudfront_logs_lambda_env=cloudfront_logs_lambda_env, 44 | cf_logs_lambda_memory_size=settings.CF_LOGS_LAMBDA_MEMORY_SIZE, 45 | cf_logs_lambda_timeout=settings.CF_LOG_LAMBDA_TIMEOUT, 46 | waf_evaluation_window_sec=settings.WAF_RATE_LIMIT_EVALUATION_WINDOW, 47 | waf_rate_limit=settings.WAF_RATE_LIMIT, 48 | waf_block_ips=settings.WAF_BLOCK_IPS, 49 | env=env, 50 | ) 51 | 52 | Tags.of(api).add("project", settings.PROJECT) 53 | Tags.of(api).add("product", "api") 54 | Tags.of(api).add("env", settings.ENV) 55 | 56 | app.synth() 57 | -------------------------------------------------------------------------------- /tests/test_sensors.py: -------------------------------------------------------------------------------- 1 | from fastapi.testclient import TestClient 2 | import json 3 | import time 4 | import os 5 | import pytest 6 | from main import app 7 | from db import db_pool 8 | 9 | 10 | @pytest.fixture 11 | def client(): 12 | with TestClient(app) as c: 13 | yield c 14 | 15 | 16 | # mock sensor and node 17 | sensor = 1 18 | node = 1 19 | 20 | urls = [ 21 | ## v2 22 | {"path": "/v2/averages?locations_id=:node", "status": 200}, 23 | {"path": "/v2/locations/:node", "status": 200}, 24 | {"path": "/v2/latest/:node", "status": 200}, 25 | {"path": "/v2/measurements?location_id=:node", "status": 200}, 26 | # all of the following have an added where clause 27 | # and we just want to make sure the sql works 28 | {"path": "/v2/cities?limit=1", "status": 200}, 29 | {"path": "/v2/countries?limit=1", "status": 200}, 30 | {"path": "/v2/sources?limit=1", "status": 200}, 31 | {"path": "/v3/manufacturers?limit=1", "status": 200}, 32 | {"path": "/v3/locations?limit=1", "status": 200}, 33 | {"path": "/v3/licenses", "status": 200}, 34 | {"path": "/v3/licenses/:node", "status": 200}, 35 | ## v3 36 | {"path": "/v3/instruments/3", "status": 200}, 37 | {"path": "/v3/locations/:node", "status": 200}, # after 38 | {"path": "/v3/sensors/:sensor/measurements", "status": 200}, # after 39 | {"path": "/v3/sensors/:sensor", "status": 200}, # after 40 | ] 41 | 42 | 43 | @pytest.mark.parametrize("url", urls) 44 | class TestUrls: 45 | def test_urls(self, client, url): 46 | path = url.get("path") 47 | path = path.replace(":sensor", str(sensor)) 48 | path = path.replace(":node", str(node)) 49 | response = client.get(path) 50 | code = url.get("status") 51 | if code == 404: 52 | data = json.loads(response.content) 53 | assert len(data["results"]) == 0 54 | else: 55 | assert response.status_code == url.get("status") 56 | -------------------------------------------------------------------------------- /pages/revalidate/style.scss: -------------------------------------------------------------------------------- 1 | @use 'openaq-design-system/scss/variables' as variables; 2 | @use 'openaq-design-system/scss/badges'; 3 | @use 'openaq-design-system/scss/buttons'; 4 | @use 'openaq-design-system/scss/inputs'; 5 | @use 'openaq-design-system/scss/resets'; 6 | @use 'openaq-design-system/scss/bubbles'; 7 | @use 'openaq-design-system/scss/utilities'; 8 | @use 'openaq-design-system/scss/header'; 9 | 10 | html body { 11 | margin: 0; 12 | padding: 0; 13 | } 14 | 15 | * { 16 | font-family: 'Space Grotesk', sans-serif; 17 | } 18 | 19 | .gradient-title { 20 | background-color: variables.$lavender100; 21 | background-image: linear-gradient( 22 | 92deg, 23 | variables.$sky120 0%, 24 | variables.$lavender100 10% 25 | ); 26 | background-size: 100%; 27 | background-clip: text; 28 | -webkit-background-clip: text; 29 | -moz-background-clip: text; 30 | -webkit-text-fill-color: transparent; 31 | -moz-text-fill-color: transparent; 32 | } 33 | 34 | .main { 35 | background-color: white; 36 | display: flex; 37 | min-height: calc(100vh - 80px); 38 | justify-content: center; 39 | } 40 | 41 | .content { 42 | width: 800px; 43 | display: flex; 44 | flex-direction: column; 45 | align-items: center; 46 | 47 | @media (max-width: 450px) { 48 | & { 49 | margin: 30px; 50 | } 51 | } 52 | } 53 | 54 | .request-validation-form { 55 | width: 360px; 56 | margin: 0 0 20px 0; 57 | 58 | @media (max-width: 450px) { 59 | & { 60 | width: 100%; 61 | } 62 | } 63 | } 64 | 65 | .form-element { 66 | display: flex; 67 | flex-direction: column; 68 | margin: 9px 0; 69 | gap: 8px; 70 | } 71 | 72 | .form-header { 73 | display: flex; 74 | flex-direction: column; 75 | gap: 15px; 76 | width: 450px; 77 | margin: 20px 0; 78 | 79 | @media (max-width: 450px) { 80 | & { 81 | width: 300px; 82 | } 83 | } 84 | } 85 | 86 | .form-submit { 87 | grid-column: 1/-1; 88 | display: flex; 89 | justify-content: center; 90 | } 91 | 92 | .submit-btn { 93 | flex: 1; 94 | } 95 | -------------------------------------------------------------------------------- /pages/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 14 | 18 | OpenAQ REST API 19 | 20 | 21 |
22 |
23 | 26 |
27 |
28 |
29 | 30 |

Welcome to the OpenAQ REST API

31 |

Links

32 | 39 |
40 | 41 | 42 | -------------------------------------------------------------------------------- /pages/revalidate/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 13 | 17 | 21 | OpenAQ Request Email Validation 22 | 23 | 24 |
25 |
26 | 29 |
30 |
31 |
32 |
33 |
34 |

Request new email validation

35 |
36 |
37 |
38 | 39 | 40 |
41 |
42 | 43 |
44 |
45 |
46 |
47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "cdk" 3 | version = "0.1.0" 4 | description = "OpenAQ REST API deployment" 5 | authors = [ 6 | {name = "Russ Biggs",email = "russ@openaq.org"}, 7 | {name = "Christian Parker", email = "chris@talloaks.io"} 8 | ] 9 | license = {text = "MIT"} 10 | readme = "README.md" 11 | requires-python = ">=3.11,<4.0" 12 | dependencies = [ 13 | ## openaq_api 14 | "fastapi (>=0.115.11,<0.116.0)", 15 | "python-dotenv (>=1.1.0,<2.0.0)", 16 | "pydantic (>=2.11.0,<3.0.0)", 17 | "pydantic-settings (>=2.8.1,<3.0.0)", 18 | "asyncpg (>=0.30.0,<0.31.0)", 19 | "orjson (>=3.10.15,<4.0.0)", 20 | "redis (>=5.2.1,<6.0.0)", 21 | "mangum (>=0.19.0,<0.20.0)", 22 | "uvicorn (>=0.34.0,<0.35.0)", 23 | "starlette (>=0.46.1,<0.47.0)", 24 | "buildpg (>=0.4,<0.5)", 25 | "aiocache (>=0.12.3,<0.13.0)", 26 | "pyhumps (>=3.8.0,<4.0.0)", 27 | "annotated-types (>=0.7.0,<0.8.0)", 28 | "jinja2 (>=3.1.6,<4.0.0)", 29 | ## cloudfront_logs 30 | "s3transfer (>=0.11.4,<0.12.0)", 31 | "six (>=1.17.0,<2.0.0)", 32 | "urllib3 (>=2.3.0,<3.0.0)", 33 | "python-dateutil (>=2.9.0.post0,<3.0.0)", 34 | "jmespath (>=1.0.1,<2.0.0)", 35 | # "pyhumps (>=3.8.0,<4.0.0)", 36 | # "python-dotenv (>=1.1.0,<2.0.0)", 37 | # "pydantic (>=2.11.0,<3.0.0)", 38 | # "pydantic-settings (>=2.8.1,<3.0.0)", 39 | # "annotated-types (>=0.7.0,<0.8.0)" 40 | "boto3 (>1.34.0)" 41 | ] 42 | 43 | [tool.poetry.group.deploy.dependencies] 44 | aws-cdk-lib = "^2.186.0" 45 | docker = "^7.1.0" 46 | 47 | [tool.poetry.group.test.dependencies] 48 | pytest = "^8.3.5" 49 | httpx = "^0.28.1" 50 | 51 | [tool.poetry.group.lint.dependencies] 52 | ruff = "^0.9.10" 53 | black = "^25.1.0" 54 | 55 | [build-system] 56 | requires = ["poetry-core>=2.0.0,<3.0.0"] 57 | build-backend = "poetry.core.masonry.api" 58 | 59 | [tool.pytest.ini_options] 60 | pythonpath = "openaq_api" 61 | log_format = "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s" 62 | log_date_format = "%H:%M:%S" 63 | log_level = "DEBUG" 64 | filterwarnings = [ 65 | "ignore::DeprecationWarning" 66 | ] 67 | -------------------------------------------------------------------------------- /pages/login/style.scss: -------------------------------------------------------------------------------- 1 | @use 'openaq-design-system/scss/variables' as variables; 2 | @use 'openaq-design-system/scss/badges'; 3 | @use 'openaq-design-system/scss/buttons'; 4 | @use 'openaq-design-system/scss/inputs'; 5 | @use 'openaq-design-system/scss/resets'; 6 | @use 'openaq-design-system/scss/bubbles'; 7 | @use 'openaq-design-system/scss/utilities'; 8 | @use 'openaq-design-system/scss/header'; 9 | 10 | html body { 11 | margin: 0; 12 | padding: 0; 13 | } 14 | 15 | * { 16 | font-family: 'Space Grotesk', sans-serif; 17 | } 18 | 19 | .gradient-title { 20 | background-color: variables.$lavender100; 21 | background-image: linear-gradient( 22 | 92deg, 23 | variables.$sky120 0%, 24 | variables.$lavender100 10% 25 | ); 26 | background-size: 100%; 27 | background-clip: text; 28 | -webkit-background-clip: text; 29 | -moz-background-clip: text; 30 | -webkit-text-fill-color: transparent; 31 | -moz-text-fill-color: transparent; 32 | } 33 | 34 | .main { 35 | background-color: white; 36 | display: flex; 37 | min-height: calc(100vh - 80px); 38 | justify-content: center; 39 | } 40 | 41 | .content { 42 | width: 800px; 43 | display: flex; 44 | flex-direction: column; 45 | align-items: center; 46 | 47 | @media (max-width: 450px) { 48 | & { 49 | margin: 30px; 50 | } 51 | } 52 | } 53 | 54 | .login-form { 55 | width: 360px; 56 | margin: 0 0 20px 0; 57 | 58 | @media (max-width: 450px) { 59 | & { 60 | width: 100%; 61 | } 62 | } 63 | } 64 | 65 | .privacy-section { 66 | margin: 20px 0; 67 | } 68 | 69 | .form-element { 70 | display: flex; 71 | flex-direction: column; 72 | margin: 9px 0; 73 | gap: 8px; 74 | } 75 | 76 | .form-header { 77 | display: flex; 78 | flex-direction: column; 79 | gap: 15px; 80 | width: 450px; 81 | margin: 20px 0; 82 | 83 | @media (max-width: 450px) { 84 | & { 85 | width: 300px; 86 | } 87 | } 88 | } 89 | 90 | .form-submit { 91 | grid-column: 1/-1; 92 | display: flex; 93 | justify-content: center; 94 | } 95 | 96 | .submit-btn { 97 | flex: 1; 98 | } 99 | -------------------------------------------------------------------------------- /cloudfront_logs/cloudfront_logs/models.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, date 2 | from humps import camelize 3 | from pydantic import BaseModel, validator 4 | 5 | 6 | class CloudwatchLog(BaseModel): 7 | timestamp: int 8 | message: str 9 | 10 | 11 | class HTTPStatusLog(BaseModel): 12 | status_code: int 13 | count: int = 1 14 | 15 | class Config: 16 | alias_generator = camelize 17 | allow_population_by_field_name = True 18 | 19 | def increment_count(self): 20 | self.count = self.count + 1 21 | 22 | 23 | class CloudfrontLog(BaseModel): 24 | date: date | None 25 | time: str | None 26 | location: str | None 27 | bytes: int | None 28 | request_ip: str | None 29 | method: str | None 30 | host: str | None 31 | uri: str | None 32 | status: int | None 33 | referrer: str | None 34 | user_agent: str | None 35 | query_string: str | None 36 | cookie: str | None 37 | result_type: str | None 38 | request_id: str | None 39 | host_header: str | None 40 | request_protocol: str | None 41 | request_bytes: int | None 42 | time_taken: float | None 43 | xforwarded_for: str | None 44 | ssl_protocol: str | None 45 | ssl_cipher: str | None 46 | response_result_type: str | None 47 | http_version: str | None 48 | fle_status: str | None 49 | fle_encrypted_fields: int | None 50 | c_port: int | None 51 | time_to_first_byte: float | None 52 | x_edge_detailed_result_type: str | None 53 | sc_content_type: str | None 54 | sc_content_len: int | None 55 | sc_range_start: int | None 56 | sc_range_end: int | None 57 | 58 | @validator("date", pre=True) 59 | def parse_date(cls, v): 60 | return datetime.strptime(v, "%Y-%m-%d").date() 61 | 62 | @validator("*", pre=True) 63 | def check_null(cls, v): 64 | if v == "-": 65 | return None 66 | return v 67 | 68 | class Config: 69 | alias_generator = camelize 70 | arbitrary_types_allowed = True 71 | allow_population_by_field_name = True 72 | -------------------------------------------------------------------------------- /pages/email_key/style.scss: -------------------------------------------------------------------------------- 1 | @use 'openaq-design-system/scss/variables' as variables; 2 | @use 'openaq-design-system/scss/badges'; 3 | @use 'openaq-design-system/scss/buttons'; 4 | @use 'openaq-design-system/scss/inputs'; 5 | @use 'openaq-design-system/scss/resets'; 6 | @use 'openaq-design-system/scss/bubbles'; 7 | @use 'openaq-design-system/scss/utilities'; 8 | @use 'openaq-design-system/scss/header'; 9 | 10 | html body { 11 | margin: 0; 12 | padding: 0; 13 | } 14 | 15 | * { 16 | font-family: 'Space Grotesk', sans-serif; 17 | } 18 | 19 | .gradient-title { 20 | background-color: variables.$lavender100; 21 | background-image: linear-gradient( 22 | 92deg, 23 | variables.$sky120 0%, 24 | variables.$lavender100 10% 25 | ); 26 | background-size: 100%; 27 | background-clip: text; 28 | -webkit-background-clip: text; 29 | -moz-background-clip: text; 30 | -webkit-text-fill-color: transparent; 31 | -moz-text-fill-color: transparent; 32 | } 33 | 34 | .main { 35 | background-color: white; 36 | display: flex; 37 | min-height: calc(100vh - 80px); 38 | justify-content: center; 39 | } 40 | 41 | .content { 42 | width: 800px; 43 | display: flex; 44 | flex-direction: column; 45 | align-items: center; 46 | 47 | @media (max-width: 450px) { 48 | & { 49 | margin: 30px; 50 | } 51 | } 52 | } 53 | 54 | .error-message { 55 | display: flex; 56 | flex-direction: column; 57 | align-items: center; 58 | } 59 | 60 | .email-key-form { 61 | width: 360px; 62 | margin: 0 0 20px 0; 63 | 64 | @media (max-width: 450px) { 65 | & { 66 | width: 100%; 67 | } 68 | } 69 | } 70 | 71 | .privacy-section { 72 | margin: 20px 0; 73 | } 74 | 75 | .form-element { 76 | display: flex; 77 | flex-direction: column; 78 | margin: 9px 0; 79 | gap: 8px; 80 | } 81 | 82 | .form-header { 83 | display: flex; 84 | flex-direction: column; 85 | gap: 15px; 86 | width: 450px; 87 | margin: 20px 0; 88 | 89 | @media (max-width: 450px) { 90 | & { 91 | width: 300px; 92 | } 93 | } 94 | } 95 | 96 | .form-submit { 97 | grid-column: 1/-1; 98 | display: flex; 99 | justify-content: center; 100 | } 101 | 102 | .submit-btn { 103 | flex: 1; 104 | } 105 | 106 | input[type='radio'] { 107 | accent-color: variables.$ocean120; 108 | } 109 | -------------------------------------------------------------------------------- /tests/locustfile.py: -------------------------------------------------------------------------------- 1 | from locust import HttpUser, task 2 | # import time 3 | from random import randrange 4 | 5 | 6 | class HelloWorldUser(HttpUser): 7 | 8 | @task(1) 9 | def LocationsV2(self): 10 | id = randrange(30000) 11 | self.client.get(f"/v2/locations/{id}", name='v2/locations/:id') 12 | 13 | @task(0) 14 | def LocationsRadiusSourceV2(self): 15 | lon = randrange(35000, 45000)/1000 16 | lat = randrange(-100000, -80000)/1000 17 | self.client.get( 18 | f"/v2/locations?coordinates={lon},{lat}&sourceName=us-epa-airnow&radius=10000", 19 | name='v2/locations?radius' 20 | ) 21 | 22 | @task(40) 23 | def LatestV2(self): 24 | lon = randrange(40000, 60000)/1000 25 | lat = randrange(50000, 60000)/1000 26 | rad = 80000 27 | self.client.get( 28 | f"/v2/latest?limit=10", 29 | name='v2/latest/empty' 30 | ) 31 | 32 | 33 | @task(40) 34 | def LatestRadiusV2b(self): 35 | lon = randrange(40000, 60000)/1000 36 | lat = randrange(50000, 60000)/1000 37 | rad = 8000 38 | self.client.get( 39 | f"/v2/latest?coordinates={lon},{lat}&radius={rad}", 40 | name='v2/latest/8K' 41 | ) 42 | 43 | @task(2) 44 | def LatestLocationV1(self): 45 | id = randrange(30000) 46 | self.client.get(f"/v1/latest?location={id}", name='v1/latest?location') 47 | 48 | @task(0) 49 | def LatestCoordinatesV1(self): 50 | coords = randrange(50000, 60000)/1000 51 | self.client.get( 52 | f"/v1/latest?coordinates={coords}", 53 | name='v1/latest?coordinates' 54 | ) 55 | 56 | @task(2) 57 | def LatestRadiusV1(self): 58 | lon = randrange(40000, 60000)/1000 59 | lat = randrange(50000, 60000)/1000 60 | rad = 8000 61 | self.client.get( 62 | f"/v1/latest?coordinates={lon},{lat}&radius={rad}", 63 | name='v1/latest?radius' 64 | ) 65 | 66 | @task(1) 67 | def LocationsV1(self): 68 | self.client.get("/v1/locations", name='v1/locations') 69 | 70 | @task(0) 71 | def GetMeasurementsV3(self): 72 | id = randrange(30000) 73 | self.client.get( 74 | f"/v3/locations/{id}/measurements", 75 | name='v3/locations/:id/meausurements' 76 | ) 77 | -------------------------------------------------------------------------------- /openaq_api/settings.py: -------------------------------------------------------------------------------- 1 | from os import environ, getcwd, path 2 | 3 | from pydantic import computed_field 4 | from pydantic_settings import BaseSettings, SettingsConfigDict 5 | 6 | 7 | def get_env(): 8 | env_name = environ.get('DOTENV', '.env') 9 | dir_name = path.basename(getcwd()) 10 | if not env_name.startswith(".env"): 11 | env_name = f".env.{env_name}" 12 | if dir_name == 'openaq_api': 13 | env_name = f"../{env_name}" 14 | elif dir_name == 'cdk': 15 | env_name = f"../{env_name}" 16 | return env_name 17 | 18 | 19 | 20 | class Settings(BaseSettings): 21 | DATABASE_READ_USER: str 22 | DATABASE_WRITE_USER: str 23 | DATABASE_READ_PASSWORD: str 24 | DATABASE_WRITE_PASSWORD: str 25 | DATABASE_DB: str 26 | DATABASE_HOST: str 27 | DATABASE_PORT: int 28 | API_CACHE_TIMEOUT: int = 900 29 | USE_SHARED_POOL: bool = False 30 | LOG_LEVEL: str = "INFO" 31 | LOG_BUCKET: str | None = None 32 | DOMAIN_NAME: str | None = None 33 | 34 | REDIS_HOST: str | None = None 35 | REDIS_PORT: int | None = 6379 36 | 37 | RATE_LIMITING: bool = False 38 | RATE_AMOUNT_KEY: int | None = None 39 | USER_AGENT: str | None = None 40 | ORIGIN: str | None = None 41 | 42 | EMAIL_SENDER: str | None = None 43 | SMTP_EMAIL_HOST: str | None = None 44 | SMTP_EMAIL_USER: str | None = None 45 | SMTP_EMAIL_PASSWORD: str | None = None 46 | 47 | EXPLORER_API_KEY: str 48 | 49 | @computed_field(return_type=str, alias="DATABASE_READ_URL") 50 | @property 51 | def DATABASE_READ_URL(self): 52 | return f"postgresql://{self.DATABASE_READ_USER}:{self.DATABASE_READ_PASSWORD}@{self.DATABASE_HOST}:{self.DATABASE_PORT}/{self.DATABASE_DB}" 53 | 54 | @computed_field(return_type=str, alias="DATABASE_WRITE_URL") 55 | @property 56 | def DATABASE_WRITE_URL(self): 57 | return f"postgresql://{self.DATABASE_WRITE_USER}:{self.DATABASE_WRITE_PASSWORD}@{self.DATABASE_HOST}:{self.DATABASE_PORT}/{self.DATABASE_DB}" 58 | 59 | @computed_field(return_type=str, alias="DATABASE_WRITE_URL") 60 | @property 61 | def USE_SMTP_EMAIL(self): 62 | return None not in [ 63 | self.SMTP_EMAIL_HOST, 64 | self.SMTP_EMAIL_USER, 65 | self.SMTP_EMAIL_PASSWORD, 66 | ] 67 | 68 | model_config = SettingsConfigDict(extra="ignore", env_file=get_env()) 69 | 70 | 71 | settings = Settings() 72 | -------------------------------------------------------------------------------- /openaq_api/requirements.txt: -------------------------------------------------------------------------------- 1 | aiocache==0.12.3 ; python_version >= "3.11" and python_version < "4.0" 2 | annotated-types==0.7.0 ; python_version >= "3.11" and python_version < "4.0" 3 | anyio==4.9.0 ; python_version >= "3.11" and python_version < "4.0" 4 | async-timeout==5.0.1 ; python_version >= "3.11" and python_full_version < "3.11.3" 5 | asyncpg==0.30.0 ; python_version >= "3.11" and python_version < "4.0" 6 | boto3==1.37.24 ; python_version >= "3.11" and python_version < "4.0" 7 | botocore==1.37.24 ; python_version >= "3.11" and python_version < "4.0" 8 | buildpg==0.4 ; python_version >= "3.11" and python_version < "4.0" 9 | click==8.1.8 ; python_version >= "3.11" and python_version < "4.0" 10 | colorama==0.4.6 ; python_version >= "3.11" and python_version < "4.0" and platform_system == "Windows" 11 | fastapi==0.115.12 ; python_version >= "3.11" and python_version < "4.0" 12 | h11==0.14.0 ; python_version >= "3.11" and python_version < "4.0" 13 | idna==3.10 ; python_version >= "3.11" and python_version < "4.0" 14 | jinja2==3.1.6 ; python_version >= "3.11" and python_version < "4.0" 15 | jmespath==1.0.1 ; python_version >= "3.11" and python_version < "4.0" 16 | mangum==0.19.0 ; python_version >= "3.11" and python_version < "4.0" 17 | markupsafe==3.0.2 ; python_version >= "3.11" and python_version < "4.0" 18 | orjson==3.10.16 ; python_version >= "3.11" and python_version < "4.0" 19 | pydantic-core==2.33.0 ; python_version >= "3.11" and python_version < "4.0" 20 | pydantic-settings==2.8.1 ; python_version >= "3.11" and python_version < "4.0" 21 | pydantic==2.11.1 ; python_version >= "3.11" and python_version < "4.0" 22 | pyhumps==3.8.0 ; python_version >= "3.11" and python_version < "4.0" 23 | python-dateutil==2.9.0.post0 ; python_version >= "3.11" and python_version < "4.0" 24 | python-dotenv==1.1.0 ; python_version >= "3.11" and python_version < "4.0" 25 | redis==5.2.1 ; python_version >= "3.11" and python_version < "4.0" 26 | s3transfer==0.11.4 ; python_version >= "3.11" and python_version < "4.0" 27 | six==1.17.0 ; python_version >= "3.11" and python_version < "4.0" 28 | sniffio==1.3.1 ; python_version >= "3.11" and python_version < "4.0" 29 | starlette==0.46.1 ; python_version >= "3.11" and python_version < "4.0" 30 | typing-extensions==4.13.0 ; python_version >= "3.11" and python_version < "4.0" 31 | typing-inspection==0.4.0 ; python_version >= "3.11" and python_version < "4.0" 32 | urllib3==2.3.0 ; python_version >= "3.11" and python_version < "4.0" 33 | uvicorn==0.34.0 ; python_version >= "3.11" and python_version < "4.0" 34 | -------------------------------------------------------------------------------- /pages/login/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 13 | 17 | 21 | OpenAQ Login - API Key 22 | 23 | 24 |
25 |
26 | 29 |
30 |
31 |
32 |
33 |
34 |

API Key Login

35 |
36 | 49 |
50 |

Don't have an account? Register

51 |
52 |
53 |

See our privacy policy for more information on the personal data we store.

54 |
55 |
56 |
57 | 58 | 59 | -------------------------------------------------------------------------------- /tests/test_sensors_latest.py: -------------------------------------------------------------------------------- 1 | from fastapi.testclient import TestClient 2 | import json 3 | import time 4 | import pytest 5 | from main import app 6 | 7 | 8 | @pytest.fixture 9 | def client(): 10 | with TestClient(app) as c: 11 | yield c 12 | 13 | 14 | measurands_id = 2 15 | # location 1 is at -10 hrs 16 | # last value is on 2024-08-27 19:30 17 | locations_id = 1 18 | 19 | 20 | class TestLocations: 21 | def test_default_good(self, client): 22 | response = client.get(f"/v3/locations/{locations_id}/latest") 23 | assert response.status_code == 200 24 | data = json.loads(response.content).get("results", []) 25 | assert len(data) == 1 26 | 27 | def test_date_filter(self, client): 28 | response = client.get( 29 | f"/v3/locations/{locations_id}/latest?datetime_min=2024-08-27" 30 | ) 31 | assert response.status_code == 200 32 | data = json.loads(response.content).get("results", []) 33 | assert len(data) == 1 34 | 35 | def test_timestamp_filter(self, client): 36 | response = client.get( 37 | f"/v3/locations/{locations_id}/latest?datetime_min=2024-08-27 19:00:00" 38 | ) 39 | assert response.status_code == 200 40 | data = json.loads(response.content).get("results", []) 41 | assert len(data) == 1 42 | 43 | def test_timestamptz_filter(self, client): 44 | response = client.get( 45 | f"/v3/locations/{locations_id}/latest?datetime_min=2024-08-27 19:00:00-10:00" 46 | ) 47 | assert response.status_code == 200 48 | data = json.loads(response.content).get("results", []) 49 | assert len(data) == 1 50 | 51 | 52 | class TestMeasurands: 53 | def test_default_good(self, client): 54 | response = client.get(f"/v3/parameters/{measurands_id}/latest") 55 | assert response.status_code == 200 56 | data = json.loads(response.content).get("results", []) 57 | assert len(data) == 6 58 | 59 | def test_date_filter(self, client): 60 | response = client.get( 61 | f"/v3/parameters/{measurands_id}/latest?datetime_min=2024-08-27" 62 | ) 63 | assert response.status_code == 200 64 | data = json.loads(response.content).get("results", []) 65 | assert len(data) == 1 66 | 67 | def test_timestamp_filter(self, client): 68 | response = client.get( 69 | f"/v3/parameters/{measurands_id}/latest?datetime_min=2024-08-27 19:00:00" 70 | ) 71 | assert response.status_code == 200 72 | data = json.loads(response.content).get("results", []) 73 | assert len(data) == 1 74 | 75 | def test_timestamptz_filter(self, client): 76 | response = client.get( 77 | f"/v3/parameters/{measurands_id}/latest?datetime_min=2024-08-27 19:00:00-10:00" 78 | ) 79 | assert response.status_code == 200 80 | data = json.loads(response.content).get("results", []) 81 | assert len(data) == 1 82 | -------------------------------------------------------------------------------- /pages/email_key/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 13 | 17 | 21 | OpenAQ API 22 | 23 | 24 |
25 |
26 | 29 |
30 |
31 |
32 |
33 |
34 |

API Key Form

35 |

Request your OpenAQ API Key to be sent to your registered email. Don't have an account? Register at https://api.openaq.org/register

36 |
37 | 55 | 56 |
57 |

See our privacy policy for more information on the personal data we store.

58 |
59 |
60 |
61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /openaq_api/v3/routers/owners.py: -------------------------------------------------------------------------------- 1 | from enum import StrEnum, auto 2 | import logging 3 | from typing import Annotated 4 | 5 | from fastapi import APIRouter, Depends, HTTPException, Path, Query 6 | 7 | from openaq_api.db import DB 8 | from openaq_api.v3.models.queries import ( 9 | Paging, 10 | QueryBaseModel, 11 | QueryBuilder, 12 | SortingBase, 13 | ) 14 | from openaq_api.v3.models.responses import OwnersResponse 15 | 16 | logger = logging.getLogger("owners") 17 | 18 | router = APIRouter( 19 | prefix="/v3", 20 | tags=["v3"], 21 | include_in_schema=True, 22 | ) 23 | 24 | 25 | class OwnerPathQuery(QueryBaseModel): 26 | """Path query to filter results by Owners ID. 27 | 28 | Inherits from QueryBaseModel. 29 | 30 | Attributes: 31 | owners_id: owners ID value. 32 | """ 33 | 34 | owners_id: int = Path( 35 | description="Limit the results to a specific owner by id", 36 | ge=1, 37 | ) 38 | 39 | def where(self) -> str: 40 | """Generates SQL condition for filtering to a single owners_id 41 | 42 | Overrides the base QueryBaseModel `where` method 43 | 44 | Returns: 45 | string of WHERE clause 46 | """ 47 | return "entities_id = :owners_id" 48 | 49 | 50 | class OwnersSortFields(StrEnum): 51 | ID = auto() 52 | 53 | 54 | class OwnersSorting(SortingBase): 55 | order_by: OwnersSortFields | None = Query( 56 | "id", 57 | description="The field by which to order results", 58 | examples=["order_by=id"], 59 | ) 60 | 61 | 62 | class OwnersQueries(Paging, OwnersSorting): ... 63 | 64 | 65 | @router.get( 66 | "/owners/{owners_id}", 67 | response_model=OwnersResponse, 68 | summary="Get a owner by ID", 69 | description="Provides a owner by owner ID", 70 | ) 71 | async def owner_get( 72 | owners: Annotated[OwnerPathQuery, Depends(OwnerPathQuery.depends())], 73 | db: DB = Depends(), 74 | ): 75 | response = await fetch_owners(owners, db) 76 | if len(response.results) == 0: 77 | raise HTTPException(status_code=404, detail="Owner not found") 78 | return response 79 | 80 | 81 | @router.get( 82 | "/owners", 83 | response_model=OwnersResponse, 84 | summary="Get owners", 85 | description="Provides a list of owners", 86 | ) 87 | async def owners_get( 88 | owner: Annotated[OwnersQueries, Depends(OwnersQueries.depends())], 89 | db: DB = Depends(), 90 | ): 91 | response = await fetch_owners(owner, db) 92 | return response 93 | 94 | 95 | async def fetch_owners(query, db): 96 | query_builder = QueryBuilder(query) 97 | sql = f""" 98 | SELECT e.entities_id AS id 99 | , e.full_name AS name 100 | FROM entities e 101 | JOIN sensor_nodes sn ON e.entities_id = sn.owner_entities_id 102 | {query_builder.where()} 103 | AND sn.is_public 104 | GROUP BY e.entities_id, name 105 | ORDER BY e.entities_id 106 | {query_builder.pagination()}; 107 | """ 108 | response = await db.fetchPage(sql, query_builder.params()) 109 | return response 110 | -------------------------------------------------------------------------------- /tests/test_main.py: -------------------------------------------------------------------------------- 1 | from fastapi.testclient import TestClient 2 | import json 3 | import time 4 | import os 5 | import pytest 6 | from main import app 7 | from db import db_pool 8 | 9 | 10 | @pytest.fixture 11 | def client(): 12 | with TestClient(app) as c: 13 | yield c 14 | 15 | 16 | def test_ping(client): 17 | response = client.get("/ping") 18 | assert response.status_code == 200 19 | assert response.json() == {"ping": "pong!"} 20 | 21 | 22 | endpoints = [ 23 | "countries", 24 | "owners", 25 | "manufacturers", 26 | "providers", 27 | "sensors", 28 | "locations", 29 | "parameters", 30 | ] 31 | 32 | 33 | # @pytest.mark.parametrize("endpoint", endpoints) 34 | # class TestEndpointsHealth: 35 | # def test_endpoint_list_good(self, client, endpoint): 36 | # response = client.get(f"/v3/{endpoint}") 37 | # assert response.status_code == 200 38 | 39 | # def test_endpoint_path_good(self, client, endpoint): 40 | # response = client.get(f"/v3/{endpoint}/1") 41 | # assert response.status_code == 200 42 | 43 | # def test_endpoint_path_bad(self, client, endpoint): 44 | # response = client.get(f"/v3/{endpoint}/0") 45 | # assert response.status_code == 422 46 | 47 | 48 | dir_path = os.path.dirname(os.path.realpath(__file__)) 49 | with open(os.path.join(dir_path, "url_list.txt")) as file: 50 | urls = [line.rstrip() for line in file] 51 | 52 | 53 | @pytest.mark.parametrize("url", urls) 54 | class TestUrls: 55 | def test_urls(self, client, url): 56 | response = client.get(url) 57 | assert response.status_code == 200 58 | 59 | 60 | class TestLocations: 61 | def test_locations_radius_good(self, client): 62 | response = client.get("/v3/locations?coordinates=38.907,-77.037&radius=1000") 63 | assert response.status_code == 200 64 | 65 | def test_locations_bbox_good(self, client): 66 | response = client.get("/v3/locations?bbox=-77.037,38.907,-77.0,39.910") 67 | assert response.status_code == 200 68 | 69 | def test_locations_query_bad(self, client): 70 | response = client.get( 71 | "/v3/locations?coordinates=42,42&radius=1000&bbox=42,42,42,42" 72 | ) 73 | assert response.status_code == 422 74 | 75 | def test_locations_providers_id_param(self, client): 76 | response = client.get("/v3/locations?providers_id=1") 77 | res = json.loads(response.content) 78 | assert all(result["provider"]["id"] == 1 for result in res["results"]) 79 | 80 | def test_locations_is_monitor_param(self, client): 81 | response = client.get("/v3/locations?monitor=true") 82 | res = json.loads(response.content) 83 | assert all(result["isMonitor"] for result in res["results"]) 84 | response = client.get("/v3/locations?monitor=false") 85 | res = json.loads(response.content) 86 | assert all(result["isMonitor"] == False for result in res["results"]) 87 | 88 | def test_locations_countries_id_param(self, client): 89 | response = client.get("/v3/locations?countries_id=1") 90 | res = json.loads(response.content) 91 | assert all(result["country"]["id"] == 1 for result in res["results"]) 92 | -------------------------------------------------------------------------------- /openaq_api/v3/routers/countries.py: -------------------------------------------------------------------------------- 1 | from enum import StrEnum, auto 2 | import logging 3 | from typing import Annotated 4 | 5 | from fastapi import APIRouter, Depends, HTTPException, Path, Query 6 | 7 | from openaq_api.db import DB 8 | from openaq_api.v3.models.queries import ( 9 | Paging, 10 | ParametersQuery, 11 | ProviderQuery, 12 | QueryBaseModel, 13 | QueryBuilder, 14 | SortingBase, 15 | ) 16 | from openaq_api.v3.models.responses import CountriesResponse 17 | 18 | logger = logging.getLogger("countries") 19 | 20 | router = APIRouter( 21 | prefix="/v3", 22 | tags=["v3"], 23 | include_in_schema=True, 24 | ) 25 | 26 | 27 | class CountryPathQuery(QueryBaseModel): 28 | """Path query to filter results by countries ID 29 | 30 | Inherits from QueryBaseModel 31 | 32 | Attributes: 33 | countries_id: countries ID value 34 | """ 35 | 36 | countries_id: int = Path( 37 | description="Limit the results to a specific country by id", 38 | ge=1, 39 | ) 40 | 41 | def where(self) -> str: 42 | """Generates SQL condition for filtering to a single countries_id 43 | 44 | Overrides the base QueryBaseModel `where` method 45 | 46 | Returns: 47 | string of WHERE clause 48 | """ 49 | return "id = :countries_id" 50 | 51 | 52 | class CountriesSortFields(StrEnum): 53 | ID = auto() 54 | 55 | 56 | class CountriesSorting(SortingBase): 57 | order_by: CountriesSortFields | None = Query( 58 | "id", 59 | description="The field by which to order results", 60 | examples=["order_by=id"], 61 | ) 62 | 63 | 64 | class CountriesQueries(Paging, ParametersQuery, ProviderQuery, CountriesSorting): ... 65 | 66 | 67 | @router.get( 68 | "/countries/{countries_id}", 69 | response_model=CountriesResponse, 70 | summary="Get a country by ID", 71 | description="Provides a country by country ID", 72 | ) 73 | async def country_get( 74 | countries: Annotated[CountryPathQuery, Depends(CountryPathQuery)], 75 | db: DB = Depends(), 76 | ): 77 | response = await fetch_countries(countries, db) 78 | if len(response.results) == 0: 79 | raise HTTPException(status_code=404, detail="Country not found") 80 | return response 81 | 82 | 83 | @router.get( 84 | "/countries", 85 | response_model=CountriesResponse, 86 | summary="Get countries", 87 | description="Provides a list of countries", 88 | ) 89 | async def countries_get( 90 | countries: Annotated[CountriesQueries, Depends(CountriesQueries.depends())], 91 | db: DB = Depends(), 92 | ): 93 | response = await fetch_countries(countries, db) 94 | return response 95 | 96 | 97 | async def fetch_countries(query, db): 98 | query_builder = QueryBuilder(query) 99 | sql = f""" 100 | SELECT id 101 | , code 102 | , name 103 | , datetime_first 104 | , datetime_last 105 | , parameters 106 | {query_builder.total()} 107 | FROM countries_view_cached 108 | {query_builder.where()} 109 | {query_builder.pagination()} 110 | """ 111 | response = await db.fetchPage(sql, query_builder.params()) 112 | return response 113 | -------------------------------------------------------------------------------- /cdk/stacks/utils.py: -------------------------------------------------------------------------------- 1 | from os import environ 2 | from pathlib import Path 3 | import subprocess 4 | import platform 5 | import docker 6 | import shutil 7 | import os 8 | import pathlib 9 | from aws_cdk import aws_lambda 10 | 11 | 12 | def dictstr(item): 13 | return item[0], str(item[1]) 14 | 15 | 16 | def stringify_settings(data: dict): 17 | return dict(map(dictstr, data.model_dump(exclude_unset=True).items())) 18 | 19 | 20 | def create_dependencies_layer( 21 | self, 22 | env_name: str, 23 | function_name: str, 24 | python_version, 25 | ) -> aws_lambda.LayerVersion: 26 | output_dir = f"../.build/{function_name}" 27 | layer_id = f"openaq-{function_name}-{env_name}-dependencies" 28 | 29 | if not environ.get("SKIP_BUILD"): 30 | print(f'Building {layer_id} into {output_dir}') 31 | if 'arm' in platform.uname().version.lower(): 32 | shutil.copy(requirements_file, f"./requirements.docker.txt") 33 | client = docker.from_env() 34 | print("starting docker image build...") 35 | client.images.build( 36 | path=str("."), 37 | dockerfile="Dockerfile", 38 | platform="linux/amd64", 39 | tag="openaqapidependencies", 40 | nocache=False, 41 | ) 42 | print("docker image built.") 43 | print("running docker container.") 44 | client.containers.run( 45 | image="openaqapidependencies", 46 | remove=True, 47 | volumes=[f"{str(Path(__file__).resolve().parent.parent)}:/tmp/"], 48 | user=0, 49 | ) 50 | p = pathlib.Path(f"{output_dir}").resolve().absolute() 51 | if not os.path.exists(p): 52 | os.mkdir(p) 53 | print("cleaning up") 54 | shutil.move("./python", str(p)) 55 | os.remove(f"./requirements.docker.txt") 56 | else: 57 | ## migrate to the package/function directory to export and install 58 | subprocess.run( 59 | f""" 60 | cd ../{function_name} && poetry export --only main -o requirements.txt --without-hashes && \ 61 | poetry run python -m pip install -qq -r requirements.txt \ 62 | -t {output_dir}/python && \ 63 | cd {output_dir} && \ 64 | find . -type f -name '*.pyc' | \ 65 | while read f; do n=$(echo $f | \ 66 | sed 's/__pycache__\///' | \ 67 | sed 's/.cpython-[2-3][0-9]+//'); \ 68 | cp $f $n; \ 69 | done \ 70 | && find . -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf \ 71 | && find . -type d -a -name 'tests' -print0 | xargs -0 rm -rf \ 72 | && find . -type d -a -name '*.dist-info' -print0 | xargs -0 rm -rf \ 73 | && find . -type f -a -name '*.so' -print0 | xargs -0 strip --strip-unneeded 74 | """, 75 | shell=True, 76 | ) 77 | 78 | layer_code = aws_lambda.Code.from_asset(output_dir) 79 | 80 | return aws_lambda.LayerVersion( 81 | self, 82 | layer_id, 83 | code=layer_code, 84 | compatible_architectures=[aws_lambda.Architecture.X86_64], 85 | compatible_runtimes=[python_version], 86 | ) 87 | -------------------------------------------------------------------------------- /pages/register/style.scss: -------------------------------------------------------------------------------- 1 | @use 'openaq-design-system/scss/variables' as variables; 2 | @use 'openaq-design-system/scss/badges'; 3 | @use 'openaq-design-system/scss/buttons'; 4 | @use 'openaq-design-system/scss/inputs'; 5 | @use 'openaq-design-system/scss/resets'; 6 | @use 'openaq-design-system/scss/bubbles'; 7 | @use 'openaq-design-system/scss/utilities'; 8 | @use 'openaq-design-system/scss/header'; 9 | 10 | html body { 11 | margin: 0; 12 | padding: 0; 13 | } 14 | 15 | * { 16 | font-family: 'Space Grotesk', sans-serif; 17 | } 18 | 19 | .gradient-title { 20 | background-color: variables.$lavender100; 21 | background-image: linear-gradient( 22 | 92deg, 23 | variables.$sky120 0%, 24 | variables.$lavender100 10% 25 | ); 26 | background-size: 100%; 27 | background-clip: text; 28 | -webkit-background-clip: text; 29 | -moz-background-clip: text; 30 | -webkit-text-fill-color: transparent; 31 | -moz-text-fill-color: transparent; 32 | } 33 | 34 | .main { 35 | background-color: white; 36 | display: flex; 37 | min-height: calc(100vh - 80px); 38 | justify-content: center; 39 | } 40 | 41 | .content { 42 | width: 800px; 43 | display: flex; 44 | flex-direction: column; 45 | align-items: center; 46 | 47 | @media (max-width: 450px) { 48 | & { 49 | margin: 30px; 50 | } 51 | } 52 | } 53 | 54 | .registration-form { 55 | width: 360px; 56 | margin: 0 0 20px 0; 57 | 58 | @media (max-width: 450px) { 59 | & { 60 | width: 100%; 61 | } 62 | } 63 | } 64 | 65 | .privacy-section { 66 | margin: 20px 0; 67 | } 68 | 69 | .form-element { 70 | display: flex; 71 | flex-direction: column; 72 | margin: 9px 0; 73 | gap: 8px; 74 | } 75 | 76 | .entity-select { 77 | display: grid; 78 | grid-template-columns: 1fr 1fr; 79 | gap: 12px; 80 | font-size: 24px; 81 | margin: 0 auto; 82 | width: 240px; 83 | } 84 | 85 | .form-header { 86 | display: flex; 87 | flex-direction: column; 88 | gap: 15px; 89 | width: 450px; 90 | margin: 20px 0; 91 | 92 | @media (max-width: 450px) { 93 | & { 94 | width: 300px; 95 | } 96 | } 97 | } 98 | 99 | .form-submit { 100 | grid-column: 1/-1; 101 | display: flex; 102 | justify-content: center; 103 | } 104 | 105 | .submit-btn { 106 | flex: 1; 107 | } 108 | 109 | input[type='radio'] { 110 | accent-color: variables.$ocean120; 111 | } 112 | 113 | .password-strength { 114 | display: flex; 115 | flex-direction: column; 116 | gap: 8px; 117 | justify-content: center; 118 | align-items: center; 119 | margin: 20px auto; 120 | } 121 | 122 | .strenth-message { 123 | display: flex; 124 | align-items: center; 125 | justify-content: center; 126 | } 127 | 128 | .strength-meter { 129 | display: flex; 130 | } 131 | 132 | .strength-meter-bars { 133 | width: 150px; 134 | height: 15px; 135 | display: grid; 136 | grid-template-columns: 1fr 1fr 1fr 1fr 1fr; 137 | border-radius: 6px; 138 | overflow: hidden; 139 | } 140 | 141 | .strength-meter__bar { 142 | border: 0.5px solid white; 143 | background-color: #eee; 144 | 145 | &--ok { 146 | background-color: variables.$mantis100; 147 | } 148 | 149 | &--alert { 150 | background-color: variables.$corn100; 151 | } 152 | 153 | &--warning { 154 | background-color: variables.$fire100; 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /openaq_api/v3/routers/licenses.py: -------------------------------------------------------------------------------- 1 | from enum import StrEnum, auto 2 | import logging 3 | from typing import Annotated 4 | 5 | from fastapi import APIRouter, Depends, HTTPException, Path, Query 6 | 7 | from openaq_api.db import DB 8 | from openaq_api.v3.models.queries import ( 9 | Paging, 10 | QueryBaseModel, 11 | QueryBuilder, 12 | SortingBase, 13 | ) 14 | from openaq_api.v3.models.responses import LicensesResponse 15 | 16 | logger = logging.getLogger("licenses") 17 | 18 | router = APIRouter( 19 | prefix="/v3", 20 | tags=["v3"], 21 | include_in_schema=True, 22 | ) 23 | 24 | 25 | class LicensesPathQuery(QueryBaseModel): 26 | """Path query to filter results by license ID 27 | 28 | Inherits from QueryBaseModel 29 | 30 | Attributes: 31 | licenses_id: license ID value 32 | """ 33 | 34 | licenses_id: int = Path( 35 | ..., description="Limit the results to a specific licenses id", ge=1 36 | ) 37 | 38 | def where(self) -> str: 39 | """Generates SQL condition for filtering to a single licenses_id 40 | 41 | Overrides the base QueryBaseModel `where` method 42 | 43 | Returns: 44 | string of WHERE clause 45 | """ 46 | return "l.licenses_id = :licenses_id" 47 | 48 | 49 | class LicensesSortFields(StrEnum): 50 | ID = auto() 51 | 52 | 53 | class LicensesSorting(SortingBase): 54 | order_by: LicensesSortFields | None = Query( 55 | "id", 56 | description="The field by which to order results", 57 | examples=["order_by=id"], 58 | ) 59 | 60 | 61 | class LicensesQueries(Paging, LicensesSorting): ... 62 | 63 | 64 | @router.get( 65 | "/licenses/{licenses_id}", 66 | response_model=LicensesResponse, 67 | summary="Get an instrument by ID", 68 | description="Provides a instrument by instrument ID", 69 | ) 70 | async def license_get( 71 | licenses: Annotated[LicensesPathQuery, Depends(LicensesPathQuery.depends())], 72 | db: DB = Depends(), 73 | ): 74 | response = await fetch_licenses(licenses, db) 75 | if len(response.results) == 0: 76 | raise HTTPException(status_code=404, detail="License not found") 77 | return response 78 | 79 | 80 | @router.get( 81 | "/licenses", 82 | response_model=LicensesResponse, 83 | summary="Get licenses", 84 | description="Provides a list of licenses", 85 | ) 86 | async def instruments_get( 87 | licenses: Annotated[LicensesQueries, Depends(LicensesQueries.depends())], 88 | db: DB = Depends(), 89 | ): 90 | response = await fetch_licenses(licenses, db) 91 | return response 92 | 93 | 94 | async def fetch_licenses(query, db): 95 | query_builder = QueryBuilder(query) 96 | sql = f""" 97 | SELECT 98 | licenses_id AS id 99 | , name 100 | , attribution_required 101 | , share_alike_required 102 | , commercial_use_allowed 103 | , redistribution_allowed 104 | , modification_allowed 105 | , url AS source_url 106 | FROM licenses l 107 | {query_builder.where()} 108 | ORDER BY 109 | licenses_id 110 | {query_builder.pagination()}; 111 | """ 112 | 113 | response = await db.fetchPage(sql, query_builder.params()) 114 | return response 115 | -------------------------------------------------------------------------------- /openaq_api/middleware.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import time 4 | from starlette.middleware.base import BaseHTTPMiddleware 5 | from starlette.requests import Request 6 | from starlette.types import ASGIApp 7 | from starlette.background import BackgroundTask 8 | 9 | from openaq_api.db import DB 10 | from fastapi import Depends 11 | 12 | from openaq_api.models.logging import ( 13 | HTTPLog, 14 | LogType, 15 | ) 16 | 17 | logger = logging.getLogger("middleware") 18 | 19 | 20 | class CacheControlMiddleware(BaseHTTPMiddleware): 21 | """MiddleWare to add CacheControl in response headers.""" 22 | 23 | def __init__(self, app: ASGIApp, cachecontrol: str | None = None) -> None: 24 | """Init Middleware.""" 25 | super().__init__(app) 26 | self.cachecontrol = cachecontrol 27 | 28 | async def dispatch(self, request: Request, call_next): 29 | """Add cache-control.""" 30 | response = await call_next(request) 31 | 32 | if ( 33 | not response.headers.get("Cache-Control") 34 | and self.cachecontrol 35 | and request.method in ["HEAD", "GET"] 36 | and response.status_code < 500 37 | ): 38 | response.headers["Cache-Control"] = self.cachecontrol 39 | return response 40 | 41 | 42 | class Timer: 43 | def __init__(self): 44 | self.start_time = time.time() 45 | self.last_mark = self.start_time 46 | self.marks = [] 47 | 48 | def mark(self, key: str, return_time: str = "total") -> float: 49 | now = time.time() 50 | mrk = { 51 | "key": key, 52 | "since": round((now - self.last_mark) * 1000, 1), 53 | "total": round((now - self.start_time) * 1000, 1), 54 | } 55 | self.last_make = now 56 | self.marks.append(mrk) 57 | logger.debug(f"TIMER ({key}): {mrk['since']}") 58 | return mrk.get(return_time) 59 | 60 | 61 | async def logEntry(entry: HTTPLog, db: DB): 62 | await db.post_log(entry) 63 | ## delete me later 64 | logger.info(entry.model_dump_json()) 65 | 66 | 67 | class LoggingMiddleware(BaseHTTPMiddleware): 68 | """MiddleWare to set servers url on App with current url.""" 69 | 70 | async def dispatch( 71 | self, 72 | request: Request, 73 | call_next, 74 | ): 75 | request.state.timer = Timer() 76 | response = await call_next(request) 77 | timing = request.state.timer.mark("process") 78 | if hasattr(request.state, "rate_limiter"): 79 | rate_limiter = request.state.rate_limiter 80 | else: 81 | rate_limiter = None 82 | if hasattr(request.app.state, "counter"): 83 | counter = request.app.state.counter 84 | else: 85 | counter = None 86 | api_key = request.headers.get("x-api-key", None) 87 | 88 | entry = HTTPLog( 89 | request=request, 90 | type=LogType.SUCCESS if response.status_code == 200 else LogType.WARNING, 91 | http_code=response.status_code, 92 | timing=timing, 93 | rate_limiter=rate_limiter, 94 | counter=counter, 95 | api_key=api_key, 96 | ) 97 | 98 | if os.environ.get("LOGGING_DB"): 99 | response.background = BackgroundTask(logEntry, entry, DB(request)) 100 | 101 | return response 102 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing guidelines 2 | 3 | ## Before contributing 4 | 5 | Welcome to [Openaq](https://github.com/openaq/openaq-api-v2)! Before sending your pull requests, make sure that you **read the whole guidelines**. If you have any doubt on the contributing guide, please feel free to [state it clearly in an issue](https://github.com/openaq/openaq-api-v2/issues/new) or ask the community in [Slack](https://slack.com). 6 | 7 | ## Contributing 8 | 9 | There are many ways to contribute to a project, below are some examples: 10 | 11 | - Report bugs, ideas, requests for features by creating “Issues” in the project repository. 12 | - Fork the code and play with it, whether you later choose to make a pull request or not. 13 | - Create pull requests of changes that you think are laudatory. From typos to major design flaws, you will find a target-rich environment for improvements. 14 | 15 | ## Issues 16 | 17 | When creating a task through the issue tracker, please include the following where applicable: 18 | 19 | * A summary of identified tasks related to the issue; and 20 | * Any dependencies related to completion of the task (include links to tickets with the dependency). 21 | 22 | ### Design and feature request issues should include: 23 | * What the goal of the task being accomplished is; and 24 | * The user need being addressed. 25 | 26 | ### Development issues should include: 27 | * Unknowns tasks or dependencies that need investigation. 28 | 29 | Use checklists (via `- [ ]`) to keep track of sub-items wherever possible. 30 | 31 | ## Coding style 32 | 33 | When writing code it is generally a good idea to try and match your 34 | formatting to that of any existing code in the same file, or to other 35 | similar files if you are writing new code. Consistency of layout is 36 | far more important that the layout itself as it makes reading code 37 | much easier. 38 | 39 | One golden rule of formatting -- please don't use tabs in your code 40 | as they will cause the file to be formatted differently for different 41 | people depending on how they have their editor configured. 42 | 43 | ## Comments 44 | 45 | Sometimes it's not apparent from the code itself what it does, or, 46 | more importantly, **why** it does that. Good comments help your fellow 47 | developers to read the code and satisfy themselves that it's doing the 48 | right thing. 49 | 50 | When developing, you should: 51 | 52 | * Comment your code - don't go overboard, but explain the bits which 53 | might be difficult to understand what the code does, why it does it 54 | and why it should be the way it is. 55 | * Check existing comments to ensure that they are not misleading. 56 | 57 | ## Committing 58 | 59 | When you submit patches, the project maintainer has to read them and 60 | understand them. This is difficult enough at the best of times, and 61 | misunderstanding patches can lead to them being more difficult to 62 | merge. To help with this, when submitting you should: 63 | 64 | * Split up large patches into smaller units of functionality. 65 | * Keep your commit messages relevant to the changes in each individual 66 | unit. 67 | 68 | When writing commit messages please try and stick to the same style as 69 | other commits, namely: 70 | 71 | * A one line summary, starting with a capital and with no full stop. 72 | * A blank line. 73 | * Full description, as proper sentences with capitals and full stops. 74 | 75 | For simple commits the one line summary is often enough and the body 76 | of the commit message can be left out. 77 | 78 | If you have forked on GitHub then the best way to submit your patches is to 79 | push your changes back to GitHub and then send a "pull request" on GitHub. 80 | -------------------------------------------------------------------------------- /.github/workflows/deploy-staging.yml: -------------------------------------------------------------------------------- 1 | name: Deploy staging 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'feature/**' 7 | 8 | jobs: 9 | deploy: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout repo 13 | uses: actions/checkout@v4 14 | 15 | - name: Configure aws credentials 16 | uses: aws-actions/configure-aws-credentials@master 17 | with: 18 | aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PROD }} 19 | aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY_PROD }} 20 | aws-region: ${{ secrets.AWS_REGION }} 21 | 22 | - name: Get envionmental values 23 | uses: aws-actions/aws-secretsmanager-get-secrets@v2 24 | with: 25 | secret-ids: | 26 | STAGING, openaq-env/staging 27 | name-transformation: uppercase 28 | parse-json-secrets: true 29 | 30 | - uses: actions/setup-node@v3 31 | with: 32 | node-version: "20" 33 | 34 | - name: build pages 35 | working-directory: ./pages 36 | run: | 37 | yarn install 38 | yarn run deploy 39 | 40 | - name: Install CDK 41 | run: | 42 | npm install -g aws-cdk 43 | 44 | - uses: actions/setup-python@v3 45 | with: 46 | python-version: '3.11' 47 | 48 | - name: Install Poetry 49 | uses: snok/install-poetry@v1 50 | 51 | - name: Deploy stack 52 | env: 53 | ENV: "staging" 54 | PROJECT: "openaq" 55 | DATABASE_READ_USER: ${{ env.STAGING_DATABASE_READ_USER }} 56 | DATABASE_READ_PASSWORD: ${{ env.STAGING_DATABASE_READ_PASSWORD }} 57 | DATABASE_WRITE_USER: ${{ env.STAGING_DATABASE_WRITE_USER }} 58 | DATABASE_WRITE_PASSWORD: ${{ env.STAGING_DATABASE_WRITE_PASSWORD }} 59 | DATABASE_DB: ${{ env.STAGING_DATABASE_DB }} 60 | DATABASE_HOST: ${{ env.STAGING_DATABASE_HOST }} 61 | DATABASE_PORT: ${{ env.STAGING_DATABASE_PORT }} 62 | API_LAMBDA_MEMORY_SIZE: ${{ env.STAGING_API_LAMBDA_MEMORY_SIZE }} 63 | 64 | CDK_ACCOUNT: ${{ secrets.CDK_ACCOUNT }} 65 | CDK_REGION: ${{ secrets.CDK_REGION }} 66 | 67 | VPC_ID: ${{ env.STAGING_VPC_ID }} 68 | 69 | HOSTED_ZONE_ID: ${{ env.STAGING_HOSTED_ZONE_ID }} 70 | HOSTED_ZONE_NAME: ${{ env.STAGING_HOSTED_ZONE_NAME }} 71 | DOMAIN_NAME: ${{ env.STAGING_DOMAIN_NAME }} 72 | CERTIFICATE_ARN: ${{ env.STAGING_CERTIFICATE_ARN }} 73 | 74 | RATE_LIMITING: True 75 | RATE_AMOUNT: 10 76 | RATE_AMOUNT_KEY: 60 77 | RATE_TIME: 1 78 | USER_AGENT: ${{ env.STAGING_USER_AGENT }} 79 | ORIGIN: ${{ env.STAGING_ORIGIN }} 80 | 81 | EMAIL_SENDER: ${{ env.STAGING_EMAIL_SENDER }} 82 | SMTP_EMAIL_HOST: ${{ env.STAGING_SMTP_EMAIL_HOST }} 83 | SMTP_EMAIL_USER: ${{ env.STAGING_SMTP_EMAIL_USER }} 84 | SMTP_EMAIL_PASSWORD: ${{ env.STAGING_SMTP_EMAIL_PASSWORD }} 85 | 86 | EXPLORER_API_KEY: ${{ env.STAGING_EXPLORER_API_KEY }} 87 | 88 | REDIS_HOST: ${{ env.STAGING_REDIS_HOST }} 89 | REDIS_PORT: ${{ env.STAGING_REDIS_PORT }} 90 | REDIS_SECURITY_GROUP_ID: ${{ env.STAGING_REDIS_SECURITY_GROUP_ID }} 91 | WAF_RATE_LIMIT_EVALUATION_WINDOW: ${{ secrets.WAF_RATE_LIMIT_EVALUATION_WINDOW }} 92 | WAF_RATE_LIMIT: ${{ secrets.WAF_RATE_LIMIT }} 93 | 94 | working-directory: ./cdk 95 | run: | 96 | poetry self add poetry-plugin-export 97 | poetry lock 98 | poetry install 99 | poetry run cdk deploy openaq-api-staging --require-approval never 100 | -------------------------------------------------------------------------------- /.github/workflows/deploy-aeolus.yml: -------------------------------------------------------------------------------- 1 | name: Deploy aeolus 2 | 3 | on: 4 | release: 5 | types: [published] 6 | workflow_dispatch: 7 | 8 | jobs: 9 | deploy: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout repo 13 | uses: actions/checkout@v4 14 | 15 | - name: Configure aws credentials 16 | uses: aws-actions/configure-aws-credentials@master 17 | with: 18 | aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PROD }} 19 | aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY_PROD }} 20 | aws-region: ${{ secrets.AWS_REGION }} 21 | 22 | - name: Get envionmental values 23 | uses: aws-actions/aws-secretsmanager-get-secrets@v2 24 | with: 25 | secret-ids: | 26 | AEOLUS, openaq-env/aeolus 27 | name-transformation: uppercase 28 | parse-json-secrets: true 29 | 30 | - uses: actions/setup-node@v4 31 | with: 32 | node-version: "20" 33 | 34 | - name: build pages 35 | working-directory: ./pages 36 | run: | 37 | yarn install 38 | yarn run deploy 39 | 40 | - name: Install CDK 41 | run: | 42 | npm install -g aws-cdk 43 | 44 | - uses: actions/setup-python@v3 45 | with: 46 | python-version: '3.11' 47 | 48 | - name: Install Poetry 49 | uses: snok/install-poetry@v1 50 | 51 | - name: Deploy stack 52 | env: 53 | ENV: "aeolus" 54 | PROJECT: "openaq" 55 | DATABASE_READ_USER: ${{ env.AEOLUS_DATABASE_READ_USER }} 56 | DATABASE_READ_PASSWORD: ${{ env.AEOLUS_DATABASE_READ_PASSWORD }} 57 | DATABASE_WRITE_USER: ${{ env.AEOLUS_DATABASE_WRITE_USER }} 58 | DATABASE_WRITE_PASSWORD: ${{ env.AEOLUS_DATABASE_WRITE_PASSWORD }} 59 | DATABASE_DB: ${{ env.AEOLUS_DATABASE_DB }} 60 | DATABASE_HOST: ${{ env.AEOLUS_DATABASE_HOST }} 61 | DATABASE_PORT: ${{ env.AEOLUS_DATABASE_PORT }} 62 | API_LAMBDA_MEMORY_SIZE: ${{ env.AEOLUS_API_LAMBDA_MEMORY_SIZE }} 63 | 64 | HOSTED_ZONE_ID: ${{ secrets.HOSTED_ZONE_ID }} 65 | HOSTED_ZONE_NAME: ${{ secrets.HOSTED_ZONE_NAME }} 66 | DOMAIN_NAME: "api.openaq.org" 67 | CERTIFICATE_ARN: ${{ secrets.CERTIFICATE_ARN }} 68 | 69 | CDK_ACCOUNT: ${{ secrets.CDK_ACCOUNT }} 70 | CDK_REGION: ${{ secrets.CDK_REGION }} 71 | 72 | VPC_ID: ${{ env.AEOLUS_VPC_ID }} 73 | 74 | RATE_LIMITING: True 75 | LOGGING_DB: True 76 | RATE_AMOUNT_KEY: 60 77 | RATE_TIME: 1 78 | USER_AGENT: ${{ env.AEOLUS_USER_AGENT }} 79 | ORIGIN: ${{ env.AEOLUS_ORIGIN }} 80 | REDIS_HOST: ${{ env.AEOLUS_REDIS_HOST }} 81 | REDIS_PORT: ${{ env.AEOLUS_REDIS_PORT }} 82 | REDIS_SECURITY_GROUP_ID: ${{ env.AEOLUS_REDIS_SECURITY_GROUP_ID }} 83 | 84 | EMAIL_SENDER: ${{ env.AEOLUS_EMAIL_SENDER }} 85 | SMTP_EMAIL_HOST: ${{ env.AEOLUS_SMTP_EMAIL_HOST }} 86 | SMTP_EMAIL_USER: ${{ env.AEOLUS_SMTP_EMAIL_USER }} 87 | SMTP_EMAIL_PASSWORD: ${{ env.AEOLUS_SMTP_EMAIL_PASSWORD }} 88 | 89 | EXPLORER_API_KEY: ${{ env.AEOLUS_EXPLORER_API_KEY }} 90 | WAF_RATE_LIMIT_EVALUATION_WINDOW: ${{ secrets.WAF_RATE_LIMIT_EVALUATION_WINDOW }} 91 | WAF_RATE_LIMIT: ${{ secrets.WAF_RATE_LIMIT }} 92 | WAF_BLOCK_IPS: ${{ secrets.WAF_BLOCK_IPS }} 93 | 94 | working-directory: ./cdk 95 | run: | 96 | poetry self add poetry-plugin-export 97 | poetry lock 98 | poetry install 99 | poetry run cdk deploy openaq-api-aeolus --require-approval never 100 | -------------------------------------------------------------------------------- /openaq_api/v3/routers/manufacturers.py: -------------------------------------------------------------------------------- 1 | from enum import StrEnum, auto 2 | import logging 3 | from typing import Annotated 4 | 5 | from fastapi import APIRouter, Depends, HTTPException, Path, Query 6 | 7 | from openaq_api.db import DB 8 | from openaq_api.v3.models.queries import ( 9 | Paging, 10 | QueryBaseModel, 11 | QueryBuilder, 12 | SortingBase, 13 | ) 14 | from openaq_api.v3.models.responses import ManufacturersResponse 15 | 16 | logger = logging.getLogger("manufacturers") 17 | 18 | router = APIRouter( 19 | prefix="/v3", 20 | tags=["v3"], 21 | include_in_schema=True, 22 | ) 23 | 24 | 25 | class ManufacturerPathQuery(QueryBaseModel): 26 | """Path query to filter results by manufacturers ID 27 | 28 | Inherits from QueryBaseModel 29 | 30 | Attributes: 31 | manufacturers_id: manufacturers ID value 32 | """ 33 | 34 | manufacturers_id: int = Path( 35 | ..., description="Limit the results to a specific manufacturers id", ge=1 36 | ) 37 | 38 | def where(self) -> str: 39 | """Generates SQL condition for filtering to a single manufacturers_id 40 | 41 | Overrides the base QueryBaseModel `where` method 42 | 43 | Returns: 44 | string of WHERE clause 45 | """ 46 | return "e.entities_id = :manufacturers_id" 47 | 48 | 49 | class ManufacturersSortFields(StrEnum): 50 | ID = auto() 51 | 52 | 53 | class InstrumentsSorting(SortingBase): 54 | order_by: ManufacturersSortFields | None = Query( 55 | "id", 56 | description="The field by which to order results", 57 | examples=["order_by=id"], 58 | ) 59 | 60 | 61 | class ManufacturersQueries(Paging, InstrumentsSorting): ... 62 | 63 | 64 | @router.get( 65 | "/manufacturers/{manufacturers_id}", 66 | response_model=ManufacturersResponse, 67 | summary="Get a manufacturer by ID", 68 | description="Provides a manufacturer by manufacturer ID", 69 | ) 70 | async def manufacturer_get( 71 | manufacturers: Annotated[ 72 | ManufacturerPathQuery, Depends(ManufacturerPathQuery.depends()) 73 | ], 74 | db: DB = Depends(), 75 | ): 76 | response = await fetch_manufacturers(manufacturers, db) 77 | if len(response.results) == 0: 78 | raise HTTPException(status_code=404, detail="Manufacturer not found") 79 | return response 80 | 81 | 82 | @router.get( 83 | "/manufacturers", 84 | response_model=ManufacturersResponse, 85 | summary="Get manufacturers", 86 | description="Provides a list of manufacturers", 87 | ) 88 | async def manufacturers_get( 89 | manufacturer: Annotated[ 90 | ManufacturersQueries, Depends(ManufacturersQueries.depends()) 91 | ], 92 | db: DB = Depends(), 93 | ): 94 | response = await fetch_manufacturers(manufacturer, db) 95 | return response 96 | 97 | 98 | async def fetch_manufacturers(query, db): 99 | query_builder = QueryBuilder(query) 100 | sql = f""" 101 | SELECT 102 | e.entities_id AS id 103 | , e.full_name AS name 104 | , ARRAY_AGG(DISTINCT (jsonb_build_object('id', i.instruments_id, 'name', i.label))) AS instruments 105 | , COUNT(1) OVER() AS found 106 | FROM 107 | sensor_nodes sn 108 | JOIN 109 | sensor_systems ss ON sn.sensor_nodes_id = ss.sensor_nodes_id 110 | JOIN 111 | instruments i ON i.instruments_id = ss.instruments_id 112 | JOIN 113 | entities e ON e.entities_id = i.manufacturer_entities_id 114 | {query_builder.where()} 115 | GROUP BY id, name 116 | {query_builder.pagination()}; 117 | 118 | """ 119 | 120 | response = await db.fetchPage(sql, query_builder.params()) 121 | return response 122 | -------------------------------------------------------------------------------- /MAINTENANCE.md: -------------------------------------------------------------------------------- 1 | # Error Checking 2 | 3 | There are two places to check for errors, the rejects table and the fetchlogs. You can refer to the openaq-db schema for details on both tables and ways to query them but generally you can do the following. 4 | 5 | ## Tools 6 | Tools to make error checking and fixing easier 7 | 8 | ### check.py 9 | List, summarize, evaluate and fix errors related to file ingestion. The tool has the following options: 10 | * --env: (string) provide the name of the .env file to load. If no value is provided we default to `.env` 11 | * --profile: (string) the name of the AWS profile to use. If no value is provided the default profile is used, if set. 12 | * --summary: Provide a summary of errors instead of a list 13 | * --fix: Attempt to fix the errors after listing 14 | * --dryrun: Echo out the updated file instead of saving it back to the s3 bucket 15 | * --debug: Debug level logging 16 | * --id: (int) check a specific log record based on the fetchlogs_id value 17 | * --n: (int) limit the list to n records or the summary to the past n days 18 | * --rejects: Echo out the list of rejects. If used with --summary it will show a summary of the rejects table 19 | * --resubmit: Update the fetchlogs table to force a reload of that id 20 | 21 | 22 | 23 | ## Ingestion errors 24 | 25 | Whenever we catch an error during ingestion we register that error in the fetchlogs in the `last_message` field. The error should always take the format of `ERROR: message` and should be as specific as possible. Right now the predominant error is a writing error that results in a truncated JSON object that leads to a parsing error during ingestion. The current approach to fixing such an error is to remove the truncated lines and resubmit the file. 26 | 27 | To see a summary of the last 30 days use the following 28 | ```shell 29 | python3 check.py --summary --n 30 30 | ``` 31 | 32 | Or see a more detailed list of the last 10 errors. The list method will also download the file and check it for errors. 33 | 34 | ```shell 35 | python3 check.py --n 10 36 | ``` 37 | 38 | Or you can check on a specific file by using the `--id` argument. This will also download the file and check it. 39 | ```shell 40 | python3 check.py --id 5555555 41 | ``` 42 | 43 | And then if you want to try and fix the file you can use 44 | ```shell 45 | python3 check.py --id 5555555 --fix 46 | ``` 47 | 48 | Or you can batch fix files by skipping the `--id` argument. The following will check the last 10 errors and fix them if possible. 49 | ```shell 50 | python3 check.py --n 10 --fix 51 | ``` 52 | 53 | ## Ingestion rejects 54 | For the LCS pipeline we can have files that contain rejected values but not errors. In this case we add the rejected records into the `rejects` table for later review. The following line will display a rejects summary based on the ingest id and the file the data comes from. The ingest id is broken up into the first part, the provider, and the second part, the source id for reference. 55 | 56 | ```shell 57 | python3 check.py --rejects --n 10 58 | ``` 59 | 60 | We also attempt to match that data to the sensor nodes table to determine if a node already exists. If the node id is returned with the rest of the data than the likely reason for the rejection is that the node did not exist at the time the measurements were being ingested but it does now. For this type of error the likely fix is to just resubmit the file. 61 | 62 | ```shell 63 | python3 check.py --id 555555 --resubmit 64 | ``` 65 | 66 | If the node id is not returned than it is likely that it does not exist for some reason. In that scenario we need to search for the station file for that node and see if that exists. If the node does not exist the `--rejects` method will automatically check for files matching the `provider/source` pattern and return those files. If one of them looks like a good candidate you can resubmit that file. 67 | 68 | ```shell 69 | python3 check.py --id 555554 --resubmit 70 | ``` 71 | -------------------------------------------------------------------------------- /openaq_api/v3/routers/providers.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Annotated 3 | from enum import StrEnum, auto 4 | 5 | from fastapi import APIRouter, Depends, HTTPException, Path, Query 6 | 7 | from openaq_api.db import DB 8 | from openaq_api.v3.models.queries import ( 9 | BboxQuery, 10 | CountryIdQuery, 11 | CountryIsoQuery, 12 | MonitorQuery, 13 | Paging, 14 | ParametersQuery, 15 | QueryBaseModel, 16 | QueryBuilder, 17 | RadiusQuery, 18 | SortingBase, 19 | ) 20 | from openaq_api.v3.models.responses import ProvidersResponse 21 | 22 | logger = logging.getLogger("providers") 23 | 24 | router = APIRouter( 25 | prefix="/v3", 26 | tags=["v3"], 27 | include_in_schema=True, 28 | ) 29 | 30 | 31 | class ProvidersSortFields(StrEnum): 32 | ID = auto() 33 | 34 | 35 | class ProvidersSorting(SortingBase): 36 | order_by: ProvidersSortFields | None = Query( 37 | "id", 38 | description="""Order results by ID""", 39 | examples=["order_by=id"], 40 | ) 41 | 42 | 43 | class ProviderPathQuery(QueryBaseModel): 44 | """Path query to filter results by providers ID 45 | 46 | Inherits from QueryBaseModel 47 | 48 | Attributes: 49 | providers_id: providers ID value 50 | """ 51 | 52 | providers_id: int = Path( 53 | description="Limit the results to a specific provider by id", 54 | ge=1, 55 | ) 56 | 57 | def where(self): 58 | """Generates SQL condition for filtering to a single providers_id 59 | 60 | Overrides the base QueryBaseModel `where` method 61 | 62 | Returns: 63 | string of WHERE clause 64 | """ 65 | return "id = :providers_id" 66 | 67 | 68 | class ProvidersSortFields(StrEnum): 69 | ID = auto() 70 | 71 | 72 | class ParametersSorting(SortingBase): 73 | order_by: ProvidersSortFields | None = Query( 74 | "id", 75 | description="The field by which to order results", 76 | examples=["order_by=id"], 77 | ) 78 | 79 | 80 | ## TODO 81 | class ProvidersQueries( 82 | Paging, 83 | RadiusQuery, 84 | BboxQuery, 85 | CountryIdQuery, 86 | CountryIsoQuery, 87 | MonitorQuery, 88 | ParametersQuery, 89 | ParametersSorting, 90 | ): ... 91 | 92 | 93 | @router.get( 94 | "/providers/{providers_id}", 95 | response_model=ProvidersResponse, 96 | summary="Get a provider by ID", 97 | description="Provides a provider by provider ID", 98 | ) 99 | async def provider_get( 100 | providers: Annotated[ProviderPathQuery, Depends(ProviderPathQuery.depends())], 101 | db: DB = Depends(), 102 | ): 103 | response = await fetch_providers(providers, db) 104 | if len(response.results) == 0: 105 | raise HTTPException(status_code=404, detail="Provider not found") 106 | return response 107 | 108 | 109 | @router.get( 110 | "/providers", 111 | response_model=ProvidersResponse, 112 | summary="Get providers", 113 | description="Provides a list of providers", 114 | ) 115 | async def providers_get( 116 | provider: Annotated[ProvidersQueries, Depends(ProvidersQueries.depends())], 117 | db: DB = Depends(), 118 | ): 119 | response = await fetch_providers(provider, db) 120 | return response 121 | 122 | 123 | async def fetch_providers(query, db): 124 | query_builder = QueryBuilder(query) 125 | sql = f""" 126 | SELECT id 127 | , name 128 | , source_name 129 | , export_prefix 130 | , datetime_first 131 | , datetime_last 132 | , datetime_added 133 | , owner_entity->>'id' AS entities_id 134 | , parameters 135 | , st_asgeojson(extent)::json as bbox 136 | {query_builder.total()} 137 | FROM providers_view_cached 138 | {query_builder.where()} 139 | {query_builder.pagination()} 140 | """ 141 | response = await db.fetchPage(sql, query_builder.params()) 142 | return response 143 | -------------------------------------------------------------------------------- /tests/locust-test-V3.py: -------------------------------------------------------------------------------- 1 | from locust import HttpUser, task 2 | from random import randrange, choice 3 | from datetime import datetime, timedelta 4 | 5 | 6 | class HelloWorldUser(HttpUser): 7 | @task(10) 8 | def LocationsV3(self): 9 | id = randrange(30000) 10 | self.client.get(f"/v3/locations/{id}", name="v3/locations/:id") 11 | 12 | @task(2) 13 | def ProvidersV3(self): 14 | limit = randrange(1000) 15 | self.client.get(f"/v3/providers?{limit}", name="v3/providers") 16 | 17 | @task(2) 18 | def ParametersV3(self): 19 | self.client.get("/v3/parameters?", name="v3/parameters") 20 | 21 | @task(1) 22 | def load_tiles(self): 23 | z = choice([1, 2, 3]) 24 | x = choice([0, 1, 2]) 25 | y = choice([0, 1, 2]) 26 | parameter_ids = [ 27 | 1, 28 | 2, 29 | 3, 30 | 4, 31 | 5, 32 | 6, 33 | 7, 34 | 8, 35 | 9, 36 | 10, 37 | # 11, 38 | # 19, 39 | # 27, 40 | # 35, 41 | # 95, 42 | # 98, 43 | # 100, 44 | # 125, 45 | # 126, 46 | # 128, 47 | # 129, 48 | # 130, 49 | # 132, 50 | # 133, 51 | # 134, 52 | # 135, 53 | # 150, 54 | # 676, 55 | # 19840, 56 | # 19843, 57 | # 19844, 58 | # 19860, 59 | ] 60 | parameters_id = choice(parameter_ids) 61 | self.client.get( 62 | f"/v3/locations/tiles/{z}/{x}/{y}.pbf?parameters_id={parameters_id}&active=true", 63 | name="/v3/locations/tiles", 64 | ) 65 | 66 | @task(20) 67 | def LocationsV3PeriodAndParameter(self): 68 | id = randrange(3000) 69 | limit = randrange(1000) 70 | base_date = datetime(2022, 6, 1) 71 | random_days = randrange( 72 | 30 73 | ) # Changed from 300 to 30 to avoid exceeding the maximum of 30 days in the month of June 74 | random_hours = randrange(24) 75 | date_from = base_date + timedelta(days=random_days, hours=random_hours) 76 | 77 | # Ensure there are remaining days in the month for the date_to 78 | remaining_days_in_month = 30 - random_days 79 | if remaining_days_in_month > 1: 80 | random_days_to = randrange(1, remaining_days_in_month) 81 | else: 82 | random_days_to = 1 83 | random_hours_to = randrange(24) 84 | date_to = date_from + timedelta(days=random_days_to, hours=random_hours_to) 85 | date_from_str = date_from.isoformat(timespec="seconds") 86 | date_to_str = date_to.isoformat(timespec="seconds") 87 | parameter_ids = [ 88 | 1, 89 | 2, 90 | 3, 91 | 4, 92 | 5, 93 | 6, 94 | 7, 95 | 8, 96 | 9, 97 | 10, 98 | # 11, 99 | # 19, 100 | # 27, 101 | # 35, 102 | # 95, 103 | # 98, 104 | # 100, 105 | # 125, 106 | # 126, 107 | # 128, 108 | # 129, 109 | # 130, 110 | # 132, 111 | # 133, 112 | # 134, 113 | # 135, 114 | # 150, 115 | # 676, 116 | # 19840, 117 | # 19843, 118 | # 19844, 119 | # 19860, 120 | ] 121 | parameter_id = choice(parameter_ids) 122 | self.client.get( 123 | f"/v3/locations/{id}/measurements?period_name=hour&limit={limit}¶meters_id={parameter_id}&date_from={date_from_str}&date_to={date_to_str}", 124 | name="v3/locations/:id/measurements/", 125 | ) 126 | -------------------------------------------------------------------------------- /openaq_api/v3/routers/locations.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Annotated 3 | from enum import StrEnum, auto 4 | from fastapi import APIRouter, Depends, HTTPException, Path, Query, Request 5 | 6 | from openaq_api.db import DB 7 | from openaq_api.v3.models.queries import ( 8 | BboxQuery, 9 | CountryIdQuery, 10 | CountryIsoQuery, 11 | ManufacturersQuery, 12 | InstrumentsQuery, 13 | LicenseQuery, 14 | MobileQuery, 15 | MonitorQuery, 16 | OwnerQuery, 17 | Paging, 18 | ParametersQuery, 19 | ProviderQuery, 20 | QueryBaseModel, 21 | QueryBuilder, 22 | RadiusQuery, 23 | SortingBase, 24 | ) 25 | from openaq_api.v3.models.responses import LocationsResponse 26 | 27 | logger = logging.getLogger("locations") 28 | 29 | router = APIRouter( 30 | prefix="/v3", 31 | tags=["v3"], 32 | include_in_schema=True, 33 | ) 34 | 35 | 36 | class LocationsSortFields(StrEnum): 37 | ID = auto() 38 | 39 | 40 | class LocationsSorting(SortingBase): 41 | order_by: LocationsSortFields | None = Query( 42 | "id", 43 | description="The field by which to order results", 44 | examples=["order_by=id"], 45 | ) 46 | 47 | 48 | class LocationPathQuery(QueryBaseModel): 49 | """Path query to filter results by locations ID. 50 | 51 | Inherits from QueryBaseModel. 52 | 53 | Attributes: 54 | locations_id: locations ID value. 55 | """ 56 | 57 | locations_id: int = Path( 58 | description="Limit the results to a specific location by id", ge=1 59 | ) 60 | 61 | def where(self) -> str: 62 | """Generates SQL condition for filtering to a single locations_id 63 | 64 | Overrides the base QueryBaseModel `where` method 65 | 66 | Returns: 67 | string of WHERE clause 68 | """ 69 | return "id = :locations_id" 70 | 71 | 72 | class LocationsQueries( 73 | BboxQuery, 74 | CountryIdQuery, 75 | CountryIsoQuery, 76 | InstrumentsQuery, 77 | MobileQuery, 78 | MonitorQuery, 79 | LicenseQuery, 80 | LocationsSorting, 81 | ManufacturersQuery, 82 | OwnerQuery, 83 | Paging, 84 | ParametersQuery, 85 | ProviderQuery, 86 | RadiusQuery, 87 | ): ... 88 | 89 | 90 | @router.get( 91 | "/locations/{locations_id}", 92 | response_model=LocationsResponse, 93 | summary="Get a location by ID", 94 | description="Provides a location by location ID", 95 | ) 96 | async def location_get( 97 | locations: Annotated[LocationPathQuery, Depends(LocationPathQuery.depends())], 98 | request: Request, 99 | db: DB = Depends(), 100 | ): 101 | response = await fetch_locations(locations, db) 102 | if len(response.results) == 0: 103 | raise HTTPException(status_code=404, detail="Location not found") 104 | return response 105 | 106 | 107 | @router.get( 108 | "/locations", 109 | response_model=LocationsResponse, 110 | summary="Get locations", 111 | description="Provides a list of locations", 112 | ) 113 | async def locations_get( 114 | locations: Annotated[LocationsQueries, Depends(LocationsQueries.depends())], 115 | db: DB = Depends(), 116 | ): 117 | response = await fetch_locations(locations, db) 118 | return response 119 | 120 | 121 | async def fetch_locations(query, db): 122 | query_builder = QueryBuilder(query) 123 | sql = f""" 124 | SELECT id 125 | , name 126 | , ismobile as is_mobile 127 | , ismonitor as is_monitor 128 | , city as locality 129 | , country 130 | , owner 131 | , provider 132 | , coordinates 133 | , instruments 134 | , sensors 135 | , timezone 136 | , bbox(geom) as bounds 137 | , datetime_first 138 | , datetime_last 139 | , licenses 140 | {query_builder.fields() or ''} 141 | FROM locations_view_cached 142 | {query_builder.where()} 143 | {query_builder.order_by()} 144 | {query_builder.pagination()} 145 | """ 146 | print("SQL", sql) 147 | response = await db.fetchPage(sql, query_builder.params()) 148 | return response 149 | -------------------------------------------------------------------------------- /openaq_api/v3/routers/sensors.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Annotated 3 | 4 | from fastapi import APIRouter, Depends, HTTPException, Path 5 | 6 | from openaq_api.db import DB 7 | from openaq_api.v3.models.queries import ( 8 | QueryBaseModel, 9 | QueryBuilder, 10 | ) 11 | 12 | from openaq_api.v3.models.responses import ( 13 | SensorsResponse, 14 | ) 15 | 16 | logger = logging.getLogger("sensors") 17 | 18 | router = APIRouter( 19 | prefix="/v3", 20 | tags=["v3"], 21 | include_in_schema=True, 22 | ) 23 | 24 | 25 | class SensorPathQuery(QueryBaseModel): 26 | sensors_id: int = Path( 27 | ..., description="Limit the results to a specific sensors id", ge=1 28 | ) 29 | 30 | def where(self): 31 | return "s.sensors_id = :sensors_id" 32 | 33 | 34 | class LocationSensorQuery(QueryBaseModel): 35 | locations_id: int = Path( 36 | ..., description="Limit the results to a specific sensors id", ge=1 37 | ) 38 | 39 | def where(self): 40 | return "n.sensor_nodes_id = :locations_id" 41 | 42 | 43 | @router.get( 44 | "/locations/{locations_id}/sensors", 45 | response_model=SensorsResponse, 46 | summary="Get sensors by location ID", 47 | description="Provides a list of sensors by location ID", 48 | ) 49 | async def sensors_get( 50 | location_sensors: Annotated[ 51 | LocationSensorQuery, Depends(LocationSensorQuery.depends()) 52 | ], 53 | db: DB = Depends(), 54 | ): 55 | return await fetch_sensors(location_sensors, db) 56 | 57 | 58 | @router.get( 59 | "/sensors/{sensors_id}", 60 | response_model=SensorsResponse, 61 | summary="Get a sensor by ID", 62 | description="Provides a sensor by sensor ID", 63 | ) 64 | async def sensor_get( 65 | sensors: Annotated[SensorPathQuery, Depends(SensorPathQuery.depends())], 66 | db: DB = Depends(), 67 | ): 68 | response = await fetch_sensors(sensors, db) 69 | if len(response.results) == 0: 70 | raise HTTPException(status_code=404, detail="Sensor not found") 71 | return response 72 | 73 | 74 | async def fetch_sensors(q, db): 75 | query = QueryBuilder(q) 76 | 77 | logger.debug(query.params()) 78 | sql = f""" 79 | SELECT s.sensors_id as id 80 | , m.measurand||' '||m.units as name 81 | , json_build_object( 82 | 'id', m.measurands_id 83 | , 'name', m.measurand 84 | , 'units', m.units 85 | , 'display_name', m.display 86 | ) as parameter 87 | , s.sensors_id 88 | , CASE 89 | WHEN r.value_latest IS NOT NULL THEN 90 | json_build_object( 91 | 'min', r.value_min 92 | , 'max', r.value_max 93 | , 'avg', r.value_avg 94 | , 'sd', r.value_sd 95 | ) 96 | ELSE NULL 97 | END as summary 98 | , CASE 99 | WHEN r.value_latest IS NOT NULL THEN 100 | jsonb_build_object( 101 | 'datetime_from', get_datetime_object(r.datetime_first, t.tzid), 102 | 'datetime_to', get_datetime_object(r.datetime_last, t.tzid) 103 | ) || calculate_coverage( 104 | r.value_count, 105 | s.data_averaging_period_seconds, 106 | s.data_logging_period_seconds 107 | )::jsonb 108 | ELSE NULL 109 | END as coverage 110 | , get_datetime_object(r.datetime_first, t.tzid) as datetime_first 111 | , get_datetime_object(r.datetime_last, t.tzid) as datetime_last 112 | ,CASE 113 | WHEN r.value_latest IS NOT NULL THEN 114 | json_build_object( 115 | 'datetime', get_datetime_object(r.datetime_last, t.tzid) 116 | , 'value', r.value_latest 117 | , 'coordinates', json_build_object( 118 | 'latitude', st_y(COALESCE(r.geom_latest, n.geom)) 119 | ,'longitude', st_x(COALESCE(r.geom_latest, n.geom)) 120 | )) 121 | ELSE NULL 122 | END as latest 123 | FROM sensors s 124 | JOIN sensor_systems sy ON (s.sensor_systems_id = sy.sensor_systems_id) 125 | JOIN sensor_nodes n ON (sy.sensor_nodes_id = n.sensor_nodes_id) 126 | JOIN timezones t ON (n.timezones_id = t.timezones_id) 127 | JOIN measurands m ON (s.measurands_id = m.measurands_id) 128 | LEFT JOIN sensors_rollup r ON (s.sensors_id = r.sensors_id) 129 | {query.where()} AND n.is_public AND s.is_public 130 | {query.pagination()} 131 | """ 132 | return await db.fetchPage(sql, query.params()) 133 | -------------------------------------------------------------------------------- /pages/logo.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /openaq_api/v3/routers/flags.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Annotated, Any 3 | from datetime import datetime, date 4 | 5 | from fastapi import APIRouter, Depends, Path, Query 6 | from fastapi.exceptions import RequestValidationError 7 | 8 | from pydantic import model_validator 9 | 10 | from openaq_api.db import DB 11 | from openaq_api.v3.models.queries import ( 12 | Paging, 13 | QueryBaseModel, 14 | QueryBuilder, 15 | ) 16 | 17 | from openaq_api.v3.models.responses import ( 18 | LocationFlagsResponse, 19 | ) 20 | 21 | logger = logging.getLogger("flags") 22 | 23 | router = APIRouter( 24 | prefix="/v3", 25 | tags=["v3"], 26 | include_in_schema=True, 27 | ) 28 | 29 | 30 | class DatetimePeriodQuery(QueryBaseModel): 31 | datetime_from: datetime | date | None = Query( 32 | None, 33 | description="To when?", 34 | examples=["2022-10-01T11:19:38-06:00", "2022-10-01"], 35 | ) 36 | datetime_to: datetime | date | None = Query( 37 | None, 38 | description="To when?", 39 | examples=["2022-10-01T11:19:38-06:00", "2022-10-01"], 40 | ) 41 | 42 | @model_validator(mode="after") 43 | @classmethod 44 | def check_dates_are_in_order(cls, data: Any) -> Any: 45 | dt = getattr(data, "datetime_to") 46 | df = getattr(data, "datetime_from") 47 | if dt and df and dt <= df: 48 | raise RequestValidationError( 49 | f"Date/time from must be older than the date/time to. User passed {df} - {dt}" 50 | ) 51 | 52 | def where(self) -> str: 53 | pd = self.map("period", "period") 54 | if self.datetime_to is None and self.datetime_from is None: 55 | return None 56 | if self.datetime_to is not None and self.datetime_from is not None: 57 | return f"{pd} && tstzrange(:datetime_from, :datetime_to, '[]')" 58 | elif self.datetime_to is not None: 59 | return f"{pd} && tstzrange('-infinity'::timestamptz, :datetime_to, '[]')" 60 | elif self.datetime_from is not None: 61 | return f"{pd} && tstzrange(:datetime_from, 'infinity'::timestamptz, '[]')" 62 | 63 | 64 | class LocationFlagQuery(QueryBaseModel): 65 | locations_id: int = Path( 66 | ..., description="Limit the results to a specific locations", ge=1 67 | ) 68 | 69 | def where(self): 70 | return "f.sensor_nodes_id = :locations_id" 71 | 72 | 73 | class SensorFlagQuery(QueryBaseModel): 74 | sensor_id: int = Path( 75 | ..., description="Limit the results to a specific sensor", ge=1 76 | ) 77 | 78 | def where(self): 79 | return "ARRAY[:sensor_id::int] @> f.sensors_ids" 80 | 81 | 82 | class LocationFlagQueries(LocationFlagQuery, DatetimePeriodQuery, Paging): ... 83 | 84 | 85 | class SensorFlagQueries(SensorFlagQuery, DatetimePeriodQuery, Paging): ... 86 | 87 | 88 | @router.get( 89 | "/locations/{locations_id}/flags", 90 | response_model=LocationFlagsResponse, 91 | summary="Get flags by location ID", 92 | description="Provides a list of flags by location ID", 93 | ) 94 | async def location_flags_get( 95 | location_flags: Annotated[ 96 | LocationFlagQueries, Depends(LocationFlagQueries.depends()) 97 | ], 98 | db: DB = Depends(), 99 | ): 100 | return await fetch_flags(location_flags, db) 101 | 102 | 103 | @router.get( 104 | "/sensors/{sensor_id}/flags", 105 | response_model=LocationFlagsResponse, 106 | summary="Get flags by sensor ID", 107 | description="Provides a list of flags by sensor ID", 108 | ) 109 | async def sensor_flags_get( 110 | sensor_flags: Annotated[SensorFlagQueries, Depends(SensorFlagQueries.depends())], 111 | db: DB = Depends(), 112 | ): 113 | return await fetch_flags(sensor_flags, db) 114 | 115 | 116 | async def fetch_flags(q, db): 117 | query = QueryBuilder(q) 118 | query.set_column_map({"timezone": "tz.tzid", "datetime": "lower(period)"}) 119 | 120 | sql = f""" 121 | SELECT f.sensor_nodes_id as location_id 122 | , json_build_object('id', ft.flag_types_id, 'label', ft.label, 'level', ft.flag_level) as flag_type 123 | , sensors_ids 124 | , get_datetime_object(lower(f.period), t.tzid) as datetime_from 125 | , get_datetime_object(upper(f.period), t.tzid) as datetime_to 126 | , note 127 | FROM flags f 128 | JOIN flag_types ft ON (f.flag_types_id = ft.flag_types_id) 129 | JOIN sensor_nodes n ON (f.sensor_nodes_id = n.sensor_nodes_id) 130 | JOIN timezones t ON (n.timezones_id = t.timezones_id) 131 | {query.where()} 132 | """ 133 | return await db.fetchPage(sql, query.params()) 134 | -------------------------------------------------------------------------------- /cdk/cdk.context.json: -------------------------------------------------------------------------------- 1 | { 2 | "vpc-provider:account=470049585876:filter.vpc-id=vpc-03806358296cb96fd:region=us-east-1:returnAsymmetricSubnets=true": { 3 | "vpcId": "vpc-03806358296cb96fd", 4 | "vpcCidrBlock": "10.0.0.0/16", 5 | "availabilityZones": [], 6 | "subnetGroups": [ 7 | { 8 | "name": "Public", 9 | "type": "Public", 10 | "subnets": [ 11 | { 12 | "subnetId": "subnet-0b61da99aa57365ce", 13 | "cidr": "10.0.0.0/20", 14 | "availabilityZone": "us-east-1a", 15 | "routeTableId": "rtb-03bd0bea591a6edb4" 16 | } 17 | ] 18 | }, 19 | { 20 | "name": "Isolated", 21 | "type": "Isolated", 22 | "subnets": [ 23 | { 24 | "subnetId": "subnet-0cbb8600d4e95bb61", 25 | "cidr": "10.0.128.0/20", 26 | "availabilityZone": "us-east-1a", 27 | "routeTableId": "rtb-0c5e253afcdad70d7" 28 | } 29 | ] 30 | } 31 | ] 32 | }, 33 | "vpc-provider:account=470049585876:filter.vpc-id=vpc-5e5ed73a:region=us-east-1:returnAsymmetricSubnets=true": { 34 | "vpcId": "vpc-5e5ed73a", 35 | "vpcCidrBlock": "172.31.0.0/16", 36 | "availabilityZones": [], 37 | "subnetGroups": [ 38 | { 39 | "name": "Public", 40 | "type": "Public", 41 | "subnets": [ 42 | { 43 | "subnetId": "subnet-809cfde5", 44 | "cidr": "172.31.64.0/20", 45 | "availabilityZone": "us-east-1a", 46 | "routeTableId": "rtb-74c68310" 47 | }, 48 | { 49 | "subnetId": "subnet-fd9b92d6", 50 | "cidr": "172.31.48.0/20", 51 | "availabilityZone": "us-east-1b", 52 | "routeTableId": "rtb-74c68310" 53 | }, 54 | { 55 | "subnetId": "subnet-f024e086", 56 | "cidr": "172.31.0.0/20", 57 | "availabilityZone": "us-east-1c", 58 | "routeTableId": "rtb-74c68310" 59 | }, 60 | { 61 | "subnetId": "subnet-a5736dfc", 62 | "cidr": "172.31.16.0/20", 63 | "availabilityZone": "us-east-1d", 64 | "routeTableId": "rtb-74c68310" 65 | }, 66 | { 67 | "subnetId": "subnet-8a4efcb7", 68 | "cidr": "172.31.32.0/20", 69 | "availabilityZone": "us-east-1e", 70 | "routeTableId": "rtb-74c68310" 71 | }, 72 | { 73 | "subnetId": "subnet-d33fbcdf", 74 | "cidr": "172.31.80.0/20", 75 | "availabilityZone": "us-east-1f", 76 | "routeTableId": "rtb-74c68310" 77 | } 78 | ] 79 | } 80 | ] 81 | }, 82 | "vpc-provider:account=470049585876:filter.vpc-id=vpc-01de015177eedd05e:region=us-east-1:returnAsymmetricSubnets=true": { 83 | "vpcId": "vpc-01de015177eedd05e", 84 | "vpcCidrBlock": "10.0.0.0/16", 85 | "availabilityZones": [], 86 | "subnetGroups": [ 87 | { 88 | "name": "Public", 89 | "type": "Public", 90 | "subnets": [ 91 | { 92 | "subnetId": "subnet-0baeac8d7cea3fece", 93 | "cidr": "10.0.0.0/19", 94 | "availabilityZone": "us-east-1a", 95 | "routeTableId": "rtb-000e19ce83d0905d6" 96 | }, 97 | { 98 | "subnetId": "subnet-07a17a8257f4250c5", 99 | "cidr": "10.0.32.0/19", 100 | "availabilityZone": "us-east-1b", 101 | "routeTableId": "rtb-088a4b51a5453d51c" 102 | }, 103 | { 104 | "subnetId": "subnet-0524632c1b5d3e5ea", 105 | "cidr": "10.0.64.0/19", 106 | "availabilityZone": "us-east-1c", 107 | "routeTableId": "rtb-097d6b80c46bd7fd8" 108 | } 109 | ] 110 | }, 111 | { 112 | "name": "Private", 113 | "type": "Private", 114 | "subnets": [ 115 | { 116 | "subnetId": "subnet-09f93828e47072297", 117 | "cidr": "10.0.96.0/19", 118 | "availabilityZone": "us-east-1a", 119 | "routeTableId": "rtb-03e599fde8ca3336a" 120 | }, 121 | { 122 | "subnetId": "subnet-01f1f2600e62bd260", 123 | "cidr": "10.0.128.0/19", 124 | "availabilityZone": "us-east-1b", 125 | "routeTableId": "rtb-01c38feaf28ba3134" 126 | }, 127 | { 128 | "subnetId": "subnet-08c5b31b1d655912b", 129 | "cidr": "10.0.160.0/19", 130 | "availabilityZone": "us-east-1c", 131 | "routeTableId": "rtb-0cf296aaf6574cb3f" 132 | } 133 | ] 134 | } 135 | ] 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /openaq_api/v3/routers/trends.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Annotated 3 | 4 | from fastapi import APIRouter, Depends, Path 5 | 6 | from db import DB 7 | from v3.models.responses import TrendsResponse 8 | 9 | logger = logging.getLogger("trends") 10 | 11 | from v3.models.queries import ( 12 | DatetimeFromQuery, 13 | DatetimeToQuery, 14 | Paging, 15 | PeriodNameQuery, 16 | QueryBaseModel, 17 | QueryBuilder, 18 | ) 19 | 20 | router = APIRouter( 21 | prefix="/v3", 22 | tags=["v3"], 23 | include_in_schema=True, 24 | ) 25 | 26 | 27 | class ParameterPathQuery(QueryBaseModel): 28 | measurands_id: int = Path(description="The parameter to query") 29 | 30 | def where(self) -> str: 31 | return "s.measurands_id = :measurands_id" 32 | 33 | 34 | class LocationPathQuery(QueryBaseModel): 35 | locations_id: int = Path( 36 | description="Limit the results to a specific location by id", ge=1 37 | ) 38 | 39 | def where(self) -> str: 40 | return "sy.sensor_nodes_id = :locations_id" 41 | 42 | 43 | class LocationTrendsQueries( 44 | Paging, 45 | LocationPathQuery, 46 | ParameterPathQuery, 47 | DatetimeFromQuery, 48 | DatetimeToQuery, 49 | PeriodNameQuery, 50 | ): ... 51 | 52 | 53 | @router.get( 54 | "/locations/{locations_id}/trends/{measurands_id}", 55 | response_model=TrendsResponse, 56 | summary="Get trends by location", 57 | description="Provides a list of aggregated measurements by location ID and factor", 58 | ) 59 | async def trends_get( 60 | trends: Annotated[LocationTrendsQueries, Depends(LocationTrendsQueries)], 61 | db: DB = Depends(), 62 | ): 63 | response = await fetch_trends(trends, db) 64 | return response 65 | 66 | 67 | async def fetch_trends(q, db): 68 | fmt = "" 69 | if q.period_name == "hour": 70 | fmt = "HH24" 71 | dur = "01:00:00" 72 | elif q.period_name == "day": 73 | fmt = "ID" 74 | dur = "24:00:00" 75 | elif q.period_name == "month": 76 | fmt = "MM" 77 | dur = "1 month" 78 | 79 | query = QueryBuilder(q) 80 | sql = f""" 81 | WITH trends AS ( 82 | SELECT 83 | sn.id 84 | , s.measurands_id 85 | , sn.timezone 86 | , to_char(timezone(sn.timezone, datetime - '1sec'::interval), '{fmt}') as factor 87 | , AVG(s.data_averaging_period_seconds) as avg_seconds 88 | , AVG(s.data_logging_period_seconds) as log_seconds 89 | , MIN(datetime - '1sec'::interval) as datetime_from 90 | , MAX(datetime - '1sec'::interval) as datetime_to 91 | , COUNT(1) as value_count 92 | , AVG(value_avg) as value_avg 93 | , STDDEV(value_avg) as value_sd 94 | , MIN(value_avg) as value_min 95 | , MAX(value_avg) as value_max 96 | , PERCENTILE_CONT(0.02) WITHIN GROUP(ORDER BY value_avg) as value_p02 97 | , PERCENTILE_CONT(0.25) WITHIN GROUP(ORDER BY value_avg) as value_p25 98 | , PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY value_avg) as value_p50 99 | , PERCENTILE_CONT(0.75) WITHIN GROUP(ORDER BY value_avg) as value_p75 100 | , PERCENTILE_CONT(0.98) WITHIN GROUP(ORDER BY value_avg) as value_p98 101 | , current_timestamp as calculated_on 102 | FROM hourly_data m 103 | JOIN sensors s ON (m.sensors_id = s.sensors_id) 104 | JOIN sensor_systems sy ON (s.sensor_systems_id = sy.sensor_systems_id) 105 | JOIN locations_view_cached sn ON (sy.sensor_nodes_id = sn.id) 106 | {query.where()} 107 | GROUP BY 1, 2, 3, 4) 108 | SELECT t.id 109 | , jsonb_build_object( 110 | 'label', factor 111 | , 'interval', '{dur}' 112 | , 'order', factor::int 113 | ) as factor 114 | , value_avg as value 115 | , json_build_object( 116 | 'id', t.measurands_id 117 | , 'units', m.units 118 | , 'name', m.measurand 119 | ) as parameter 120 | , json_build_object( 121 | 'sd', t.value_sd 122 | , 'min', t.value_min 123 | , 'q02', t.value_p02 124 | , 'q25', t.value_p25 125 | , 'median', t.value_p50 126 | , 'q75', t.value_p75 127 | , 'q98', t.value_p98 128 | , 'max', t.value_max 129 | ) as summary 130 | , calculate_coverage( 131 | t.value_count::int 132 | , t.avg_seconds 133 | , t.log_seconds 134 | , expected_hours(datetime_from, datetime_to, '{q.period_name}', factor) * 3600.0 135 | )||jsonb_build_object( 136 | 'datetime_from', get_datetime_object(datetime_from, t.timezone) 137 | , 'datetime_to', get_datetime_object(datetime_to, t.timezone) 138 | ) as coverage 139 | FROM trends t 140 | JOIN measurands m ON (t.measurands_id = m.measurands_id) 141 | {query.pagination()} 142 | """ 143 | 144 | logger.debug( 145 | f"expected_hours(datetime_from, datetime_to, '{q.period_name}', factor) * 3600.0" 146 | ) 147 | 148 | response = await db.fetchPage(sql, query.params()) 149 | return response 150 | -------------------------------------------------------------------------------- /openaq_api/v3/routers/instruments.py: -------------------------------------------------------------------------------- 1 | from enum import StrEnum, auto 2 | import logging 3 | from typing import Annotated 4 | 5 | from fastapi import APIRouter, Depends, HTTPException, Path, Query 6 | 7 | from openaq_api.db import DB 8 | from openaq_api.v3.models.queries import ( 9 | Paging, 10 | QueryBaseModel, 11 | QueryBuilder, 12 | SortingBase, 13 | ) 14 | from openaq_api.v3.models.responses import InstrumentsResponse 15 | 16 | logger = logging.getLogger("instruments") 17 | 18 | router = APIRouter( 19 | prefix="/v3", 20 | tags=["v3"], 21 | include_in_schema=True, 22 | ) 23 | 24 | 25 | class ManufacturerInstrumentsQuery(QueryBaseModel): 26 | """ 27 | Path query to filter results by manufacturers ID 28 | 29 | Inherits from QueryBaseModel 30 | 31 | Attributes: 32 | manufacturers_id: manufacturers ID value 33 | """ 34 | 35 | manufacturers_id: int = Path( 36 | ..., description="Limit results to a specific manufacturer id", ge=1 37 | ) 38 | 39 | def where(self) -> str: 40 | return "i.manufacturer_entities_id = :manufacturers_id" 41 | 42 | 43 | class InstrumentPathQuery(QueryBaseModel): 44 | """Path query to filter results by instruments ID 45 | 46 | Inherits from QueryBaseModel 47 | 48 | Attributes: 49 | instruments_id: instruments ID value 50 | """ 51 | 52 | instruments_id: int = Path( 53 | ..., description="Limit the results to a specific instruments id", ge=1 54 | ) 55 | 56 | def where(self) -> str: 57 | """Generates SQL condition for filtering to a single instruments_id 58 | 59 | Overrides the base QueryBaseModel `where` method 60 | 61 | Returns: 62 | string of WHERE clause 63 | """ 64 | return "i.instruments_id = :instruments_id" 65 | 66 | 67 | class InstrumentsSortFields(StrEnum): 68 | ID = auto() 69 | 70 | 71 | class InstrumentsSorting(SortingBase): 72 | order_by: InstrumentsSortFields | None = Query( 73 | "id", 74 | description="The field by which to order results", 75 | examples=["order_by=id"], 76 | ) 77 | 78 | 79 | class InstrumentsQueries(Paging, InstrumentsSorting): ... 80 | 81 | 82 | @router.get( 83 | "/instruments/{instruments_id}", 84 | response_model=InstrumentsResponse, 85 | summary="Get an instrument by ID", 86 | description="Provides a instrument by instrument ID", 87 | ) 88 | async def instrument_get( 89 | instruments: Annotated[InstrumentPathQuery, Depends(InstrumentPathQuery.depends())], 90 | db: DB = Depends(), 91 | ): 92 | response = await fetch_instruments(instruments, db) 93 | if len(response.results) == 0: 94 | raise HTTPException(status_code=404, detail="Instrument not found") 95 | return response 96 | 97 | 98 | @router.get( 99 | "/instruments", 100 | response_model=InstrumentsResponse, 101 | summary="Get instruments", 102 | description="Provides a list of instruments", 103 | ) 104 | async def instruments_get( 105 | instruments: Annotated[InstrumentsQueries, Depends(InstrumentsQueries.depends())], 106 | db: DB = Depends(), 107 | ): 108 | response = await fetch_instruments(instruments, db) 109 | return response 110 | 111 | 112 | @router.get( 113 | "/manufacturers/{manufacturers_id}/instruments", 114 | response_model=InstrumentsResponse, 115 | summary="Get instruments by manufacturer ID", 116 | description="Provides a list of instruments for a specific manufacturer", 117 | ) 118 | async def get_instruments_by_manufacturer( 119 | manufacturer: Annotated[ 120 | ManufacturerInstrumentsQuery, Depends(ManufacturerInstrumentsQuery.depends()) 121 | ], 122 | db: DB = Depends(), 123 | ): 124 | response = await fetch_instruments(manufacturer, db) 125 | return response 126 | 127 | 128 | async def fetch_instruments(query, db): 129 | query_builder = QueryBuilder(query) 130 | sql = f""" 131 | WITH locations_summary AS ( 132 | SELECT 133 | i.instruments_id 134 | FROM 135 | sensor_nodes sn 136 | JOIN 137 | sensor_systems ss ON sn.sensor_nodes_id = ss.sensor_nodes_id 138 | JOIN 139 | instruments i ON i.instruments_id = ss.instruments_id 140 | 141 | GROUP BY i.instruments_id 142 | ) 143 | SELECT 144 | instruments_id AS id 145 | , label AS name 146 | , is_monitor 147 | , json_build_object('id', e.entities_id, 'name', e.full_name) AS manufacturer 148 | FROM 149 | instruments i 150 | JOIN 151 | locations_summary USING (instruments_id) 152 | JOIN 153 | entities e 154 | ON 155 | i.manufacturer_entities_id = e.entities_id 156 | {query_builder.where()} 157 | ORDER BY 158 | instruments_id 159 | {query_builder.pagination()}; 160 | 161 | """ 162 | 163 | response = await db.fetchPage(sql, query_builder.params()) 164 | return response 165 | -------------------------------------------------------------------------------- /openaq_api/static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 51 | OpenAQ REST API 52 | 53 | 54 |
55 | 56 |

Welcome to the OpenAQ REST API

57 |

Links

58 | 65 |
66 | 67 | -------------------------------------------------------------------------------- /tests/unit/test_limiting.py: -------------------------------------------------------------------------------- 1 | from fastapi.testclient import TestClient 2 | import json 3 | import time 4 | import os 5 | import pytest 6 | from openaq_api.main import app 7 | from openaq_api.db import db_pool 8 | import re 9 | 10 | 11 | class FakePipeline: 12 | 13 | async def __aenter__(self): 14 | return self 15 | 16 | async def __aexit__(self, exc_type, exc_value, exc_tb): 17 | ... 18 | 19 | def incr(self, key): 20 | key = re.sub(r"[\d+:]", "", key) 21 | self.key = self.api_key_data.get(key, {}) 22 | return self 23 | 24 | def expire(self, key, sec): 25 | return self 26 | 27 | async def execute(self): 28 | value = self.key.get('get'); 29 | if value: 30 | # just incrd 31 | return [int(value) + 1] 32 | else: 33 | # incr and then expire 34 | return [1, None] 35 | 36 | 37 | class FakeRedisClient: 38 | def __init__(self): 39 | self.api_keys = [ 40 | "limited-api-key", 41 | "not-limited-api-key", 42 | "new-api-key", 43 | ] 44 | self.api_key_data = { 45 | "limited-api-key": {"get": "60", "ttl": 5, "rate": 60}, 46 | "not-limited-api-key": {"get": "58", "ttl": 2, "rate": 60}, 47 | "new-api-key": {"ttl": -2, "rate": None}, ## this key does not exist 48 | } 49 | 50 | # is this key in the set 51 | async def sismember(self, scope, key): 52 | value = 1 if key in self.api_keys else 0 53 | print(f"redis sismember: {key} = {value}") 54 | return value 55 | 56 | # number of requests made on this key 57 | async def get(self, key): 58 | key = re.sub(r"[\d+:]", "", key) 59 | value = self.api_key_data.get(key, {}).get("get") 60 | print(f"redis get: {key} = {value}") 61 | return value 62 | 63 | async def hget(self, key, field): 64 | key = re.sub(r"[\d+:]", "", key) 65 | value = self.api_key_data.get(key, {}).get(field) 66 | print(f"redis get: {key} = {value}") 67 | return value 68 | 69 | # time to live 70 | # how many seconds are left for this key 71 | async def ttl(self, key): 72 | key = re.sub(r"[\d+:]", "", key) 73 | value = self.api_key_data.get(key, {}).get("ttl") 74 | print(f"redis ttl: {key} = {value}") 75 | return value 76 | 77 | # just a way to increment the number of requests 78 | def pipeline(self): 79 | pipe = FakePipeline() 80 | pipe.api_key_data = self.api_key_data; 81 | return pipe 82 | 83 | 84 | @pytest.fixture 85 | def client(): 86 | app.redis = FakeRedisClient() 87 | with TestClient(app) as c: 88 | yield c 89 | 90 | 91 | def test_whitelisted_path_returns_200(client): 92 | response = client.get("/openapi.json") 93 | assert response.status_code == 200 94 | 95 | 96 | def test_no_key_returns_401(client): 97 | response = client.get("/ping") 98 | assert response.status_code == 401 99 | 100 | 101 | def test_empty_key_returns_401(client): 102 | response = client.get("/ping", headers={"X-API-Key": ""}) 103 | assert response.status_code == 401 104 | 105 | 106 | def test_invalid_key_returns_401(client): 107 | response = client.get("/ping", headers={"X-API-Key": "invalid-key"}) 108 | assert response.status_code == 401 109 | 110 | 111 | def test_limited_key_returns_429(client): 112 | response = client.get("/ping", headers={"X-API-Key": "limited-api-key"}) 113 | assert response.status_code == 429 114 | 115 | 116 | def test_not_limited_key_returns_valid_rate_headers(client): 117 | response = client.get("/ping", headers={"X-API-Key": "not-limited-api-key"}) 118 | assert response.headers.get('x-ratelimit-limit') == '60' 119 | assert response.headers.get('x-ratelimit-used') == '59' 120 | assert response.headers.get('x-ratelimit-remaining') == '1' 121 | assert response.headers.get('x-ratelimit-reset') == '2' 122 | 123 | 124 | def test_limited_key_returns_valid_rate_headers(client): 125 | response = client.get("/ping", headers={"X-API-Key": "limited-api-key"}) 126 | assert response.headers.get('x-ratelimit-limit') == '60' 127 | assert response.headers.get('x-ratelimit-used') == '60' 128 | assert response.headers.get('x-ratelimit-remaining') == '0' 129 | assert response.headers.get('x-ratelimit-reset') == '5' 130 | 131 | def test_new_key_returns_valid_rate_headers(client): 132 | response = client.get("/ping", headers={"X-API-Key": "new-api-key"}) 133 | print(response.headers) 134 | assert response.headers.get('x-ratelimit-limit') == '60' 135 | assert response.headers.get('x-ratelimit-used') == '1' 136 | assert response.headers.get('x-ratelimit-remaining') == '59' 137 | assert response.headers.get('x-ratelimit-reset') == '-2' 138 | 139 | 140 | def test_not_limited_key_returns_200(client): 141 | response = client.get("/ping", headers={"X-API-Key": "not-limited-api-key"}) 142 | assert response.status_code == 200 143 | 144 | 145 | def test_new_api_key_returns_200(client): 146 | response = client.get("/ping", headers={"X-API-Key": "new-api-key"}) 147 | assert response.status_code == 200 148 | -------------------------------------------------------------------------------- /pages/register/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 13 | 17 | 21 | OpenAQ API Registration 22 | 23 | 24 |
25 |
26 | 29 |
30 |
31 |
32 |
33 |
34 |

API Key Registration

35 | 36 |

Registering for an API key provided authenticated access to the OpenAQ API. Authenticated users are allowed an increased rate limit when accessing the API. To learn more visit docs.openaq.org

37 |
38 |
39 | 40 |
41 |
42 | 43 | 44 |
45 |
46 | 47 | 48 |
49 |
50 |
51 | 52 | 53 |
54 |
55 | 56 | 57 |
58 |
59 | 60 | 61 |
62 |
63 | 64 | 65 |
66 |
67 | 68 |
69 |
70 |
71 | Password strength 72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 | 82 | 83 |
84 |
85 |
86 | 87 |
88 |
89 | 90 |
91 |

See our privacy policy for more information on the personal data we store.

92 |
93 |
94 |
95 | 96 | 97 | 98 | -------------------------------------------------------------------------------- /openaq_api/v3/routers/parameters.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from enum import StrEnum, auto 3 | from typing import Annotated 4 | 5 | from fastapi import APIRouter, Depends, HTTPException, Path, Query 6 | 7 | from openaq_api.db import DB 8 | from openaq_api.v3.models.queries import ( 9 | BboxQuery, 10 | CountryIdQuery, 11 | CountryIsoQuery, 12 | Paging, 13 | QueryBaseModel, 14 | QueryBuilder, 15 | RadiusQuery, 16 | SortingBase, 17 | ) 18 | from openaq_api.v3.models.responses import ParametersResponse 19 | 20 | logger = logging.getLogger("parameters") 21 | 22 | router = APIRouter( 23 | prefix="/v3", 24 | tags=["v3"], 25 | include_in_schema=True, 26 | ) 27 | 28 | 29 | class ProvidersSortFields(StrEnum): 30 | ID = auto() 31 | 32 | 33 | class ProvidersSorting(SortingBase): 34 | order_by: ProvidersSortFields | None = Query( 35 | "id", 36 | description="""Order results by ID""", 37 | examples=["order_by=id"], 38 | ) 39 | 40 | 41 | class ParameterPathQuery(QueryBaseModel): 42 | """Path query to filter results by parameters ID 43 | 44 | Inherits from QueryBaseModel 45 | 46 | Attributes: 47 | parameters_id: countries ID value 48 | """ 49 | 50 | parameters_id: int = Path( 51 | ..., description="Limit the results to a specific parameters id", ge=1 52 | ) 53 | 54 | def where(self) -> str: 55 | """Generates SQL condition for filtering to a single parameters_id 56 | 57 | Overrides the base QueryBaseModel `where` method 58 | 59 | Returns: 60 | string of WHERE clause 61 | """ 62 | return "id = :parameters_id" 63 | 64 | 65 | class ParameterType(StrEnum): 66 | pollutant = "pollutant" 67 | meteorological = "meteorological" 68 | 69 | 70 | class ParameterTypeQuery(QueryBaseModel): 71 | """Query to filter results by parameter_type 72 | 73 | Inherits from QueryBaseModel 74 | 75 | Attributes: 76 | parameter_type: a string representing the parameter type to filter 77 | """ 78 | 79 | parameter_type: ParameterType | None = Query( 80 | None, 81 | description="Limit the results to a specific parameters type", 82 | examples=["pollutant", "meteorological"], 83 | ) 84 | 85 | def where(self) -> str | None: 86 | """Generates SQL condition for filtering to a single parameters_id 87 | 88 | Overrides the base QueryBaseModel `where` method 89 | 90 | Returns: 91 | string of WHERE clause if `parameter_type` is set 92 | """ 93 | if self.parameter_type == None: 94 | return None 95 | return "m.parameter_type = :parameter_type" 96 | 97 | 98 | class ParametersCountryIsoQuery(CountryIsoQuery): 99 | """Pydantic query model for the `iso` query parameter. 100 | 101 | Specialty query object for parameters_view_cached to handle ISO code IN ARRAY 102 | 103 | Inherits from CountryIsoQuery 104 | """ 105 | 106 | def where(self) -> str | None: 107 | """Generates SQL condition for filtering to country ISO code 108 | 109 | Overrides the base QueryBaseModel `where` method 110 | 111 | Returns: 112 | string of WHERE clause 113 | """ 114 | if self.iso is not None: 115 | return "country->>'code' IN :iso" 116 | 117 | 118 | class ParametersSortFields(StrEnum): 119 | ID = auto() 120 | 121 | 122 | class ParametersSorting(SortingBase): 123 | order_by: ParametersSortFields | None = Query( 124 | "id", 125 | description="The field by which to order results", 126 | examples=["order_by=id"], 127 | ) 128 | 129 | 130 | class ParametersQueries( 131 | Paging, 132 | CountryIdQuery, 133 | CountryIsoQuery, # TODO replace with ParametersCountryIsoQuery when parameters_view_cached is updated with ISO array field 134 | BboxQuery, 135 | RadiusQuery, 136 | ParameterTypeQuery, 137 | ParametersSorting, 138 | ): ... 139 | 140 | 141 | @router.get( 142 | "/parameters/{parameters_id}", 143 | response_model=ParametersResponse, 144 | summary="Get a parameter by ID", 145 | description="Provides a parameter by parameter ID", 146 | ) 147 | async def parameter_get( 148 | parameter: Annotated[ParameterPathQuery, Depends(ParameterPathQuery.depends())], 149 | db: DB = Depends(), 150 | ) -> ParametersResponse: 151 | response = await fetch_parameters(parameter, db) 152 | if len(response.results) == 0: 153 | raise HTTPException(status_code=404, detail="Parameter not found") 154 | return response 155 | 156 | 157 | @router.get( 158 | "/parameters", 159 | response_model=ParametersResponse, 160 | summary="Get a parameters", 161 | description="Provides a list of parameters", 162 | ) 163 | async def parameters_get( 164 | parameter: Annotated[ParametersQueries, Depends(ParametersQueries.depends())], 165 | db: DB = Depends(), 166 | ): 167 | response = await fetch_parameters(parameter, db) 168 | return response 169 | 170 | 171 | async def fetch_parameters(query, db) -> ParametersResponse: 172 | query_builder = QueryBuilder(query) 173 | ## TODO 174 | sql = f""" 175 | SELECT id 176 | , p.name 177 | , p.display_name 178 | , p.units 179 | , p.description 180 | {query_builder.total()} 181 | FROM 182 | parameters_view_cached p 183 | JOIN 184 | measurands m ON p.id = m.measurands_id 185 | {query_builder.where()} 186 | {query_builder.pagination()} 187 | """ 188 | response = await db.fetchPage(sql, query_builder.params()) 189 | return response 190 | -------------------------------------------------------------------------------- /openaq_api/dependencies.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from datetime import datetime 3 | from openaq_api.settings import settings 4 | from fastapi import Security, Response 5 | from starlette.requests import Request 6 | 7 | from fastapi.security import ( 8 | APIKeyHeader, 9 | ) 10 | 11 | from openaq_api.models.logging import ( 12 | TooManyRequestsLog, 13 | UnauthorizedLog, 14 | ) 15 | 16 | from openaq_api.exceptions import ( 17 | NOT_AUTHENTICATED_EXCEPTION, 18 | TOO_MANY_REQUESTS, 19 | ) 20 | 21 | logger = logging.getLogger("dependencies") 22 | 23 | 24 | def in_allowed_list(route: str) -> bool: 25 | logger.debug(f"Checking if '{route}' is allowed") 26 | allow_list = ["/", "/openapi.json", "/docs", "/register"] 27 | if route in allow_list: 28 | return True 29 | if "/v3/locations/tiles" in route: 30 | return True 31 | if "/assets" in route: 32 | return True 33 | if ".css" in route: 34 | return True 35 | if ".js" in route: 36 | return True 37 | return False 38 | 39 | 40 | async def check_api_key( 41 | request: Request, 42 | response: Response, 43 | api_key=Security(APIKeyHeader(name="X-API-Key", auto_error=False)), 44 | ): 45 | """ 46 | Check for an api key and then to see if they are rate limited. Throws a 47 | `not authenticated` or `too many reqests` error if appropriate. 48 | Meant to be used as a dependency either at the app, router or function level 49 | """ 50 | route = request.url.path 51 | # no checking or limiting for whitelistted routes 52 | logger.debug(f'Explorer api key: {settings.EXPLORER_API_KEY}') 53 | if in_allowed_list(route): 54 | return api_key 55 | elif api_key == settings.EXPLORER_API_KEY: 56 | return api_key 57 | else: 58 | # check to see if we are limiting 59 | redis = request.app.redis 60 | 61 | if redis is None: 62 | logger.warning("No redis client found") 63 | return api_key 64 | elif api_key is None: 65 | logging.info( 66 | UnauthorizedLog( 67 | request=request, detail="api key not provided" 68 | ).model_dump_json() 69 | ) 70 | raise NOT_AUTHENTICATED_EXCEPTION 71 | else: 72 | 73 | # check valid key 74 | if await redis.sismember("keys", api_key) == 0: 75 | logging.info( 76 | UnauthorizedLog( 77 | request=request, detail="api key not found" 78 | ).model_dump_json() 79 | ) 80 | raise NOT_AUTHENTICATED_EXCEPTION 81 | # check api key 82 | limit = await redis.hget(api_key, "rate") 83 | try: 84 | limit = int(limit) 85 | except TypeError: 86 | limit = 60 87 | limited = False 88 | # check if its limited 89 | now = datetime.now() 90 | # Using a sliding window rate limiting algorithm 91 | # we add the current time to the minute to the api key and use that as our check 92 | key = f"{api_key}:{now.year}{now.month}{now.day}{now.hour}{now.minute}" 93 | # if the that key is in our redis db it will return the number of requests 94 | # that key has made during the current minute 95 | requests_used = await redis.get(key) 96 | 97 | if requests_used is None: 98 | # if the value is none than we need to add that key to the redis db 99 | # and set it, increment it and set it to timeout/delete is 60 seconds 100 | logger.debug("redis no key for current minute so not limited") 101 | async with redis.pipeline() as pipe: 102 | [requests_used, _] = await pipe.incr(key).expire(key, 60).execute() 103 | elif int(requests_used) < limit: 104 | # if that key does exist and the value is below the allowed number of requests 105 | # wea re going to increment it and move on 106 | logger.debug( 107 | f"redis - has key for current minute ({requests_used}) < limit ({limit})" 108 | ) 109 | async with redis.pipeline() as pipe: 110 | [requests_used] = await pipe.incr(key).execute() 111 | else: 112 | # otherwise the user is over their limit and so we are going to throw a 429 113 | # after we set the headers 114 | logger.debug( 115 | f"redis - has key for current minute ({requests_used}) >= limit ({limit})" 116 | ) 117 | limited = True 118 | 119 | ttl = await redis.ttl(key) 120 | request.state.rate_limiter = ( 121 | f"{key}/{limit}/{requests_used}/{limit - int(requests_used)}/{ttl}" 122 | ) 123 | rate_limit_headers = { 124 | "x-ratelimit-limit": str(limit), 125 | "x-ratelimit-used": str(requests_used), 126 | "x-ratelimit-remaining": str(limit - int(requests_used)), 127 | "x-ratelimit-reset": str(ttl), 128 | } 129 | response.headers.update(rate_limit_headers) 130 | 131 | if limited: 132 | logging.info( 133 | TooManyRequestsLog( 134 | request=request, 135 | rate_limiter=f"{key}/{limit}/{requests_used}", 136 | ).model_dump_json() 137 | ) 138 | raise TOO_MANY_REQUESTS(rate_limit_headers) 139 | 140 | # it would be ideal if we were returing the user information right here 141 | # even it was just an email address it might be useful 142 | return api_key 143 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OpenAQ API 2 | [![Slack Chat](https://img.shields.io/badge/Chat-Slack-ff69b4.svg "Join us. Anyone is welcome!")](https://join.slack.com/t/openaq/shared_invite/zt-yzqlgsva-v6McumTjy2BZnegIK9XCVw) 3 | 4 | ## Overview 5 | This repository contains the source code for the [OpenAQ API](https://api.openaq.org), a publicly-accessible API that provides endpoints to query the real-time and historical air quality measurements on the OpenAQ platform. 6 | 7 | > [!NOTE] 8 | > This repository is for setting up and deploying the OpenAQ API. If you just wish to access the public API to query data from the OpenAQ platform, visit https://api.openaq.org or https://docs.openaq.org to learn more. 9 | 10 | ## Package management 11 | We are currently using [Poetry](https://python-poetry.org/) to manage our dependencies and run locally. 12 | 13 | ## Local development 14 | In production, the OpenAQ API runs on AWS Lambda with the help of the [mangum](https://mangum.io/) library. This allows the application to run in a serverless environment and take advantage of async Python and FastAPI routing. Despite the serverless deployment, running the API locally as a standard FastAPI application is largely unchanged, making local development much easier. 15 | 16 | ### Settings 17 | Settings can be loaded using `.env` files, and multiple files can be kept and used. The easiest way to manage multiple environment files is to add an extension describing your environment. For example, if I wanted to keep a production, staging and local environment, I would save them as `.env.production`, `.env.staging` and `.env.local` each with their own settings. 18 | 19 | ``` 20 | DATABASE_READ_USER=database-read-user 21 | DATABASE_WRITE_USER=database-write-user 22 | DATABASE_READ_PASSWORD=database-read-password 23 | DATABASE_WRITE_PASSWORD=database-write-password 24 | DATABASE_DB=database-name 25 | DATABASE_HOST=localhost 26 | DATABASE_PORT=5432 27 | LOG_LEVEL=info 28 | ``` 29 | 30 | ### Running locally 31 | The easiest way to run the API locally is to use uvicorn. Make sure that you have your settings (`.env`) file setup. Once that is done, you can run the following from the `openaq_api` directory. Variables from the `.env` files can be overrode by setting them inline. 32 | 33 | ```bash 34 | # Run using the default .env file 35 | poetry run uvicorn openaq_api.main:app --reload --lifespan on 36 | ``` 37 | You can also specify which `.env` file to load by passing the `ENV` variable. This should not include the `.env.` prefix 38 | 39 | ```bash 40 | DOTENV=local poetry run uvicorn openaq_api.main:app --reload --lifespan on 41 | ``` 42 | If you are connecting to our production environment you will AWS credentials therefor you may need to provdide the profile name to access the right credentials. 43 | 44 | ``` 45 | AWS_PROFILE=optional-profile-name \ 46 | DOTENV=production \ 47 | poetry run uvicorn openaq_api.main:app --reload --lifespan on 48 | ``` 49 | And you can always override variables by setting them inline. This is handy for when you want to change something for the purpose of debugging. 50 | ``` 51 | # Run the staging environment and add verbose logging 52 | ENV=staging LOG_LEVEL=debug uvicorn main:app --reload 53 | DOTENV=staging \ 54 | LOG_LEVEL=debug \ 55 | poetry run uvicorn openaq_api.main:app --reload --lifespan on 56 | ``` 57 | 58 | ## Testing 59 | From the root directory 60 | ```bash 61 | DOTENV=local poetry run pytest tests/ 62 | ``` 63 | 64 | ## Rate limiting 65 | 66 | In the production environment, rate limiting is handled in two places, AWS WAF and at the application level with [Starlette Middleware](https://www.starlette.io/middleware/). The application rate limiting is configurable via environment variables. The rate limiting middleware requires access to an instance of a [redis](https://redis.io/) cluster. For local development, [docker](https://www.docker.com/) can be a convenient method to set up a local redis cluster. With docker, use the following command: 67 | 68 | ```sh 69 | docker run -e "IP=0.0.0.0" -p 7000-7005:7000-7005 grokzen/redis-cluster:7.0.7 70 | ``` 71 | 72 | Now a redis instance will be available at ``` http://localhost:7000 ```. Configure the REDIS_HOST to `localhost` and REDIS_PORT to `7000`. 73 | 74 | > [!TIP] 75 | > On some macOS systems port 7000 is used by Airplay which can complicate the mapping of ports from the Docker container. The easiest option is to disable the Airplay reciever in system settings. `System settings -> General -> Airplay receiver (toggle off)` 76 | 77 | ### Rate limiting values 78 | 79 | Rate limiting can be toggled off for local develop via the `RATE_LIMITING` environment variable. Other rate limiting values are: 80 | * `RATE_AMOUNT_KEY` - The number of requests allowed with a valid API key 81 | * `RATE_TIME` - The number of minutes for the rate 82 | 83 | e.g. `RATE_AMOUNT_KEY=5` and `RATE_TIME=1` would allow 5 requests per 1 minute. 84 | 85 | > [!NOTE] 86 | > With AWS WAF, rate limiting also occurs at the cloudfront stage. The application level rate limiting should be less than or equal to the value set at AWS WAF. 87 | 88 | 89 | ### Deployment 90 | 91 | Deployment is managed with Amazon Web Services (AWS) Cloud Development Kit (CDK). Additional environment variables are required for a full deployment to the AWS Cloud. 92 | # Deploying 93 | ```python 94 | AWS_PROFILE=optional-profile-name DOTENV=production cdk deploy openaq-api-production 95 | ``` 96 | 97 | ## Platform Overview 98 | 99 | [openaq-fetch](https://github.com/openaq/openaq-fetch) and [openaq-fetch-lcs](https://github.com/openaq/openaq-fetch-lcs) take care of fetching new data and writing to [S3](https://openaq-fetches.s3.amazonaws.com/index.html). Lambda functions defined in [openaq-ingestor](https://github.com/openaq/openaq-ingestor), then load data into the database, defined in [openaq-db](https://github.com/openaq/openaq-db). 100 | 101 | 102 | ## Contributing 103 | There are many ways to contribute to this project; more details can be found in the [contributing guide](CONTRIBUTING.md). 104 | -------------------------------------------------------------------------------- /openaq_api/models/logging.py: -------------------------------------------------------------------------------- 1 | from enum import StrEnum 2 | from typing import Any 3 | 4 | from fastapi import Request, status 5 | from humps import camelize 6 | from pydantic import BaseModel, ConfigDict, Field, computed_field 7 | import re 8 | from dateutil.parser import parse 9 | 10 | 11 | class LogType(StrEnum): 12 | SUCCESS = "SUCCESS" 13 | VALIDATION_ERROR = "VALIDATION_ERROR" 14 | INFRASTRUCTURE_ERROR = "INFRASTRUCTURE_ERROR" 15 | UNPROCESSABLE_ENTITY = "UNPROCESSABLE_ENTITY" 16 | UNAUTHORIZED = "UNAUTHORIZED" 17 | TOO_MANY_REQUESTS = "TOO_MANY_REQUESTS" 18 | WARNING = "WARNING" 19 | INFO = "INFO" 20 | ERROR = "ERROR" 21 | 22 | 23 | class BaseLog(BaseModel): 24 | """Abstract base class for logging. 25 | 26 | Inherits from Pydantic BaseModel 27 | """ 28 | 29 | type: LogType 30 | detail: str | None = None 31 | 32 | def model_dump_json(self, **kwargs): 33 | kwargs["by_alias"] = True 34 | return super().model_dump_json(**kwargs) 35 | 36 | model_config = ConfigDict( 37 | alias_generator=camelize, arbitrary_types_allowed=True, populate_by_name=True 38 | ) 39 | 40 | 41 | class InfoLog(BaseLog): 42 | type: LogType = LogType.INFO 43 | 44 | 45 | class WarnLog(BaseLog): 46 | type: LogType = LogType.WARNING 47 | 48 | 49 | class ErrorLog(BaseLog): 50 | type: LogType = LogType.ERROR 51 | 52 | 53 | class InfrastructureErrorLog(BaseLog): 54 | type: LogType = LogType.INFRASTRUCTURE_ERROR 55 | 56 | 57 | class AuthLog(BaseLog): 58 | type: LogType = LogType.INFO 59 | 60 | 61 | class SESEmailLog(BaseLog): 62 | type: LogType = LogType.INFO 63 | 64 | 65 | class HTTPLog(BaseLog): 66 | """A base class for logging HTTP requests 67 | 68 | inherits from BaseLog 69 | 70 | Attributes: 71 | request: 72 | http_code: 73 | timing: 74 | rate_limiter: 75 | counter: 76 | ip: 77 | api_key: 78 | user_agent: 79 | path: 80 | params: 81 | params_obj: 82 | params_keys: 83 | 84 | """ 85 | 86 | request: Request = Field(..., exclude=True) 87 | http_code: int 88 | timing: float | None = None 89 | rate_limiter: str | None = None 90 | counter: int | None = None 91 | 92 | @computed_field(return_type=str) 93 | @property 94 | def ip(self) -> str: 95 | """str: returns IP address from request client""" 96 | return self.request.client.host 97 | 98 | @computed_field(return_type=str) 99 | @property 100 | def api_key(self) -> str: 101 | """str: returns API Key from request headers""" 102 | return self.request.headers.get("X-API-Key", None) 103 | 104 | @computed_field(return_type=str) 105 | @property 106 | def user_agent(self) -> str: 107 | """str: returns User-Agent from request headers""" 108 | return self.request.headers.get("User-Agent", None) 109 | 110 | @computed_field(return_type=str) 111 | @property 112 | def path(self) -> str: 113 | """str: returns URL path from request but replaces numbers in the path with :id""" 114 | return re.sub(r"/[0-9]+", "/:id", self.request.url.path) 115 | 116 | @computed_field(return_type=dict) 117 | @property 118 | def path_params(self) -> dict[str, Any] | None: 119 | """str: returns URL path from request but replaces numbers in the path with :id""" 120 | return self.request.path_params 121 | 122 | @computed_field(return_type=str) 123 | @property 124 | def params(self) -> str: 125 | """str: returns URL query params from request""" 126 | return self.request.url.query 127 | 128 | @computed_field(return_type=dict) 129 | @property 130 | def params_obj(self) -> dict: 131 | """returns URL query params and path params as key values from request""" 132 | params = dict(x.split("=", 1) for x in self.params.split("&") if "=" in x) 133 | if self.path_params: 134 | params = params | self.path_params 135 | try: 136 | # if bad strings make it past our validation than this will protect the log 137 | if "date_from" in params.keys(): 138 | params["date_from_epoch"] = parse(params["date_from"]).timestamp() 139 | if "date_to" in params.keys(): 140 | params["date_to_epoch"] = parse(params["date_to"]).timestamp() 141 | except Exception: 142 | pass 143 | 144 | return params 145 | 146 | @computed_field(return_type=list) 147 | @property 148 | def params_keys(self) -> list: 149 | """list: returns URL query params keys as list/array from request""" 150 | return [] if self.params_obj is None else list(self.params_obj.keys()) 151 | 152 | 153 | class HTTPErrorLog(HTTPLog): 154 | """Log for HTTP 500. 155 | 156 | Inherits from HTTPLog 157 | """ 158 | 159 | http_code: int = status.HTTP_500_INTERNAL_SERVER_ERROR 160 | 161 | 162 | class UnprocessableEntityLog(HTTPLog): 163 | """Log for HTTP 422. 164 | 165 | Inherits from HTTPLog 166 | """ 167 | 168 | http_code: int = status.HTTP_422_UNPROCESSABLE_ENTITY 169 | type: LogType = LogType.UNPROCESSABLE_ENTITY 170 | 171 | 172 | class TooManyRequestsLog(HTTPLog): 173 | """Log for HTTP 429. 174 | 175 | Inherits from HTTPLog 176 | """ 177 | 178 | http_code: int = status.HTTP_429_TOO_MANY_REQUESTS 179 | type: LogType = LogType.TOO_MANY_REQUESTS 180 | 181 | 182 | class UnauthorizedLog(HTTPLog): 183 | """Log for HTTP 401. 184 | 185 | Inherits from HTTPLog 186 | """ 187 | 188 | http_code: int = status.HTTP_401_UNAUTHORIZED 189 | type: LogType = LogType.UNAUTHORIZED 190 | 191 | 192 | class ModelValidationError(HTTPErrorLog): 193 | """Log for model validations 194 | 195 | Inherits from ErrorLog 196 | """ 197 | 198 | type: LogType = LogType.VALIDATION_ERROR 199 | 200 | 201 | class RedisErrorLog(ErrorLog): 202 | detail: str 203 | -------------------------------------------------------------------------------- /pages/register/index.js: -------------------------------------------------------------------------------- 1 | import './style.scss'; 2 | 3 | import { zxcvbn, zxcvbnOptions } from '@zxcvbn-ts/core'; 4 | import zxcvbnCommonPackage from '@zxcvbn-ts/language-common'; 5 | import zxcvbnEnPackage from '@zxcvbn-ts/language-en'; 6 | 7 | const options = { 8 | translations: zxcvbnEnPackage.translations, 9 | graphs: zxcvbnCommonPackage.adjacencyGraphs, 10 | dictionary: { 11 | ...zxcvbnCommonPackage.dictionary, 12 | ...zxcvbnEnPackage.dictionary, 13 | }, 14 | }; 15 | 16 | let passwordInputTimeout; 17 | 18 | const errorSvg = 19 | ''; 20 | const checkSvg = 21 | ''; 22 | 23 | zxcvbnOptions.setOptions(options); 24 | 25 | const nameInput = document.querySelector('.js-name-input'); 26 | const emailInput = document.querySelector('.js-email-input'); 27 | const passwordInput = document.querySelector('.js-password-input'); 28 | const passwordConfirmInput = document.querySelector( 29 | '.js-password-confirm-input' 30 | ); 31 | const passwordMatchWarning = document.querySelector( 32 | '.js-password-match-warning' 33 | ); 34 | const submitBtn = document.querySelector('.js-submit-btn'); 35 | 36 | let score; 37 | 38 | nameInput.addEventListener('input', () => { 39 | checkFormFieldsComplete(); 40 | }); 41 | 42 | emailInput.addEventListener('input', () => { 43 | checkFormFieldsComplete(); 44 | }); 45 | 46 | passwordInput.addEventListener('input', (e) => { 47 | clearTimeout(passwordInputTimeout); 48 | const result = zxcvbn(e.target.value); 49 | score = result.score; 50 | const strengthMessage = document.querySelector( 51 | '.js-strength-message' 52 | ); 53 | strengthMessage.innerText = ''; 54 | resetBars(); 55 | // only show bars if password field has value 56 | if (e.target.value.length > 0) { 57 | setPasswordStrength(result); 58 | } 59 | if (passwordConfirmInput.value != '') { 60 | passwordMatchOnInput(); 61 | } 62 | if (result.score < 3) { 63 | e.target.setCustomValidity( 64 | 'password not strong enough, please choose a stronger password.' 65 | ); 66 | } else { 67 | e.target.setCustomValidity(''); 68 | } 69 | if (e.target.value.length === 0) { 70 | e.target.setCustomValidity(''); 71 | } 72 | checkFormFieldsComplete(); 73 | passwordInputTimeout = setTimeout( 74 | () => e.target.reportValidity(), 75 | 1000 76 | ); 77 | }); 78 | 79 | let passwordMatchTimeout; 80 | 81 | passwordConfirmInput.addEventListener('input', () => { 82 | passwordMatchOnInput(); 83 | checkFormFieldsComplete(); 84 | }); 85 | 86 | function resetBars() { 87 | const bars = document.querySelectorAll('.strength-meter__bar'); 88 | for (const bar of bars) { 89 | bar.classList.remove(...bar.classList); 90 | bar.classList.add(`strength-meter__bar`); 91 | } 92 | } 93 | 94 | function passwordMatchOnInput() { 95 | clearTimeout(passwordMatchTimeout); 96 | passwordMatchTimeout = setTimeout( 97 | () => 98 | checkPasswordsMatch( 99 | passwordInput.value, 100 | passwordConfirmInput.value 101 | ), 102 | 300 103 | ); 104 | } 105 | 106 | function checkPasswordsMatch(password, confirmPassword) { 107 | while (passwordMatchWarning.firstChild) { 108 | passwordMatchWarning.removeChild(passwordMatchWarning.firstChild); 109 | } 110 | if (password != confirmPassword) { 111 | passwordMatchWarning.insertAdjacentHTML('afterbegin', errorSvg); 112 | passwordMatchWarning.innerText = 'Passwords do not match.'; 113 | } else { 114 | passwordMatchWarning.innerText = ''; 115 | } 116 | } 117 | 118 | let formfieldsCheckTimeout; 119 | 120 | function checkFormFieldsComplete() { 121 | clearTimeout(formfieldsCheckTimeout); 122 | formfieldsCheckTimeout = setTimeout(() => { 123 | if ( 124 | nameInput.value != '' && 125 | emailInput.value != '' && 126 | passwordInput.value != '' && 127 | passwordConfirmInput.value != '' && 128 | passwordInput.value == passwordConfirmInput.value && 129 | score >= 3 130 | ) { 131 | submitBtn.disabled = false; 132 | } else { 133 | submitBtn.disabled = true; 134 | } 135 | }, 300); 136 | } 137 | 138 | function setPasswordStrength(result) { 139 | let color; 140 | let message; 141 | let symbol; 142 | 143 | switch (result.score) { 144 | case 0: 145 | color = 'warning'; 146 | symbol = errorSvg; 147 | message = `Very weak password`; 148 | break; 149 | case 1: 150 | color = 'warning'; 151 | symbol = errorSvg; 152 | message = `Weak password`; 153 | break; 154 | case 2: 155 | color = 'alert'; 156 | symbol = errorSvg; 157 | message = `Somewhat secure password`; 158 | break; 159 | case 3: 160 | color = 'ok'; 161 | message = 'Strong password'; 162 | symbol = checkSvg; 163 | break; 164 | case 4: 165 | color = 'ok'; 166 | message = 'Very strong password'; 167 | symbol = checkSvg; 168 | break; 169 | default: 170 | color = 'red'; 171 | symbol = errorSvg; 172 | message = 'Weak password'; 173 | } 174 | const strengthMessage = document.querySelector( 175 | '.js-strength-message' 176 | ); 177 | const strengthWarning = document.querySelector( 178 | '.js-strength-warning' 179 | ); 180 | while (strengthMessage.firstChild) { 181 | strengthMessage.removeChild(strengthMessage.firstChild); 182 | } 183 | strengthMessage.innerText = message; 184 | strengthMessage.insertAdjacentHTML('afterbegin', symbol); 185 | strengthWarning.innerText = result.feedback.warning; 186 | const bars = document.querySelectorAll('.strength-meter__bar'); 187 | let barsArr = Array.prototype.slice.call(bars); 188 | barsArr = barsArr.splice(0, result.score + 1); 189 | for (const bar of barsArr) { 190 | bar.classList.add(`strength-meter__bar--${color}`); 191 | } 192 | } 193 | -------------------------------------------------------------------------------- /openaq_api/v3/routers/latest.py: -------------------------------------------------------------------------------- 1 | from datetime import date, datetime 2 | import logging 3 | from typing import Annotated 4 | 5 | from fastapi import APIRouter, Depends, HTTPException, Path, Query 6 | 7 | from openaq_api.db import DB 8 | from openaq_api.v3.routers.locations import LocationPathQuery, fetch_locations 9 | from openaq_api.v3.routers.parameters import fetch_parameters 10 | from openaq_api.v3.models.queries import QueryBaseModel, QueryBuilder, Paging 11 | from openaq_api.v3.models.responses import LatestResponse 12 | 13 | logger = logging.getLogger("latest") 14 | 15 | router = APIRouter( 16 | prefix="/v3", 17 | tags=["v3"], 18 | include_in_schema=True, 19 | ) 20 | 21 | 22 | class DatetimeMinQuery(QueryBaseModel): 23 | """Pydantic query model for the `datetime_min` query parameter 24 | 25 | Inherits from QueryBaseModel 26 | 27 | Attributes: 28 | datetime_min: date or datetime in ISO-8601 format to filter results to a mininum data 29 | """ 30 | 31 | datetime_min: datetime | date | None = Query( 32 | None, 33 | description="Minimum datetime", 34 | examples=["2022-10-01T11:19:38-06:00", "2022-10-01"], 35 | ) 36 | 37 | def where(self) -> str: 38 | """Generates SQL condition for filtering to datetime. 39 | 40 | Overrides the base QueryBaseModel `where` method 41 | 42 | If `datetime_min` is a `date` or `datetime` without a timezone a timezone 43 | is added as local timezone. 44 | 45 | Returns: 46 | string of WHERE clause if `datetime_min` is set 47 | """ 48 | tz = self.map("timezone", "tzid") 49 | dt = self.map("datetime", "datetime_last") 50 | 51 | if self.datetime_min is None: 52 | return None 53 | elif isinstance(self.datetime_min, datetime): 54 | if self.datetime_min.tzinfo is None: 55 | return f"{dt} > (:datetime_min::timestamp AT TIME ZONE {tz})" 56 | else: 57 | return f"{dt} > :datetime_min" 58 | elif isinstance(self.datetime_min, date): 59 | return f"{dt} > (:datetime_min::timestamp AT TIME ZONE {tz})" 60 | 61 | 62 | class ParameterLatestPathQuery(QueryBaseModel): 63 | """Path query to filter results by parameters ID 64 | 65 | Inherits from QueryBaseModel 66 | 67 | Attributes: 68 | parameters_id: countries ID value 69 | """ 70 | 71 | parameters_id: int = Path( 72 | ..., description="Limit the results to a specific parameters id", ge=1 73 | ) 74 | 75 | def where(self) -> str: 76 | """Generates SQL condition for filtering to a single parameters_id 77 | 78 | Overrides the base QueryBaseModel `where` method 79 | 80 | Returns: 81 | string of WHERE clause 82 | """ 83 | return "m.measurands_id = :parameters_id" 84 | 85 | 86 | class ParametersLatestQueries(ParameterLatestPathQuery, DatetimeMinQuery, Paging): ... 87 | 88 | 89 | @router.get( 90 | "/parameters/{parameters_id}/latest", 91 | response_model=LatestResponse, 92 | summary="", 93 | description="", 94 | ) 95 | async def parameters_latest_get( 96 | parameters_latest: Annotated[ 97 | ParametersLatestQueries, Depends(ParametersLatestQueries.depends()) 98 | ], 99 | db: DB = Depends(), 100 | ): 101 | response = await fetch_latest(parameters_latest, db) 102 | if len(response.results) == 0: 103 | parameters_response = await fetch_parameters(parameters_latest, db) 104 | if len(parameters_response.results) == 0: 105 | raise HTTPException(status_code=404, detail="Parameter not found") 106 | return response 107 | 108 | 109 | class LocationLatestPathQuery(QueryBaseModel): 110 | """Path query to filter results by locations ID. 111 | 112 | Inherits from QueryBaseModel. 113 | 114 | Attributes: 115 | locations_id: locations ID value. 116 | """ 117 | 118 | locations_id: int = Path( 119 | description="Limit the results to a specific location by id", ge=1 120 | ) 121 | 122 | def where(self) -> str: 123 | """Generates SQL condition for filtering to a single locations_id 124 | 125 | Overrides the base QueryBaseModel `where` method 126 | 127 | Returns: 128 | string of WHERE clause 129 | """ 130 | return "n.sensor_nodes_id = :locations_id" 131 | 132 | 133 | class LocationsLatestQueries(LocationLatestPathQuery, DatetimeMinQuery, Paging): ... 134 | 135 | 136 | @router.get( 137 | "/locations/{locations_id}/latest", 138 | response_model=LatestResponse, 139 | summary="Get a location's latest measurements", 140 | description="Providers a location's latest measurement values", 141 | ) 142 | async def location_latest_get( 143 | locations_latest: Annotated[ 144 | LocationsLatestQueries, Depends(LocationsLatestQueries.depends()) 145 | ], 146 | db: DB = Depends(), 147 | ): 148 | response = await fetch_latest(locations_latest, db) 149 | if len(response.results) == 0: 150 | locations_response = await fetch_locations( 151 | LocationPathQuery(locations_id=locations_latest.locations_id), db 152 | ) 153 | if len(locations_response.results) == 0: 154 | raise HTTPException(status_code=404, detail="Location not found") 155 | return response 156 | 157 | 158 | async def fetch_latest(query, db): 159 | query_builder = QueryBuilder(query) 160 | sql = f""" 161 | SELECT 162 | n.sensor_nodes_id AS locations_id 163 | ,s.sensors_id AS sensors_id 164 | ,get_datetime_object(r.datetime_last, t.tzid) as datetime 165 | ,r.value_latest AS value 166 | ,json_build_object( 167 | 'latitude', st_y(COALESCE(r.geom_latest, n.geom)) 168 | ,'longitude', st_x(COALESCE(r.geom_latest, n.geom)) 169 | ) AS coordinates 170 | {query_builder.total()} 171 | FROM 172 | sensors s 173 | JOIN 174 | sensor_systems sy ON (s.sensor_systems_id = sy.sensor_systems_id) 175 | JOIN 176 | sensor_nodes n ON (sy.sensor_nodes_id = n.sensor_nodes_id) 177 | JOIN 178 | timezones t ON (n.timezones_id = t.timezones_id) 179 | JOIN 180 | measurands m ON (s.measurands_id = m.measurands_id) 181 | INNER JOIN 182 | sensors_rollup r ON (s.sensors_id = r.sensors_id) 183 | {query_builder.where()} 184 | {query_builder.pagination()} 185 | """ 186 | response = await db.fetchPage(sql, query_builder.params()) 187 | return response 188 | -------------------------------------------------------------------------------- /openaq_api/main.py: -------------------------------------------------------------------------------- 1 | from contextlib import asynccontextmanager 2 | import datetime 3 | import logging 4 | import time 5 | from os import environ 6 | from pathlib import Path 7 | from typing import Any 8 | 9 | import orjson 10 | from fastapi import FastAPI, Request, Depends 11 | from fastapi.encoders import jsonable_encoder 12 | from fastapi.exceptions import RequestValidationError 13 | from fastapi.middleware.cors import CORSMiddleware 14 | from fastapi.middleware.gzip import GZipMiddleware 15 | from fastapi.staticfiles import StaticFiles 16 | from mangum import Mangum 17 | from pydantic import BaseModel, ValidationError 18 | from starlette.responses import JSONResponse, RedirectResponse 19 | 20 | from openaq_api.db import db_pool 21 | from openaq_api.dependencies import check_api_key 22 | from openaq_api.middleware import ( 23 | CacheControlMiddleware, 24 | LoggingMiddleware, 25 | ) 26 | from openaq_api.models.logging import InfrastructureErrorLog 27 | 28 | from openaq_api.settings import settings 29 | 30 | # V3 routers 31 | from openaq_api.v3.routers import ( 32 | auth, 33 | countries, 34 | instruments, 35 | locations, 36 | manufacturers, 37 | measurements, 38 | owners, 39 | parameters, 40 | providers, 41 | sensors, 42 | tiles, 43 | licenses, 44 | latest, 45 | flags, 46 | ) 47 | 48 | logging.basicConfig( 49 | format="[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s", 50 | level=settings.LOG_LEVEL.upper(), 51 | force=True, 52 | ) 53 | # When debuging we dont want to debug these libraries 54 | logging.getLogger("boto3").setLevel(logging.WARNING) 55 | logging.getLogger("botocore").setLevel(logging.WARNING) 56 | logging.getLogger("urllib3").setLevel(logging.WARNING) 57 | logging.getLogger("aiocache").setLevel(logging.WARNING) 58 | logging.getLogger("uvicorn").setLevel(logging.WARNING) 59 | logging.getLogger("mangum").setLevel(logging.WARNING) 60 | 61 | logger = logging.getLogger("main") 62 | 63 | # Make sure that we are using UTC timezone 64 | # this is required because the datetime class will automatically 65 | # add the env timezone when passing the value to a sql query parameter 66 | environ["TZ"] = "UTC" 67 | 68 | 69 | # this is instead of importing settings elsewhere 70 | if settings.DOMAIN_NAME is not None: 71 | environ["DOMAIN_NAME"] = settings.DOMAIN_NAME 72 | 73 | 74 | def default(obj): 75 | if isinstance(obj, float): 76 | return round(obj, 5) 77 | if isinstance(obj, datetime.datetime): 78 | return obj.strptime("%Y-%m-%dT%H:%M:%SZ") 79 | if isinstance(obj, datetime.date): 80 | return obj.strptime("%Y-%m-%d") 81 | 82 | 83 | class ORJSONResponse(JSONResponse): 84 | def render(self, content: Any) -> bytes: 85 | # logger.debug(f'rendering content {content}') 86 | return orjson.dumps(content, default=default) 87 | 88 | 89 | @asynccontextmanager 90 | async def lifespan(app: FastAPI): 91 | if not hasattr(app.state, "pool"): 92 | logger.debug("initializing connection pool") 93 | app.state.pool = await db_pool(None) 94 | logger.debug("Connection pool established") 95 | 96 | if hasattr(app.state, "counter"): 97 | app.state.counter += 1 98 | else: 99 | app.state.counter = 0 100 | 101 | yield 102 | if hasattr(app.state, "pool") and not settings.USE_SHARED_POOL: 103 | logger.debug("Closing connection") 104 | await app.state.pool.close() 105 | delattr(app.state, "pool") 106 | logger.debug("Connection closed") 107 | 108 | 109 | app = FastAPI( 110 | title="OpenAQ", 111 | description="OpenAQ API", 112 | version="3.0.0", 113 | default_response_class=ORJSONResponse, 114 | dependencies=[Depends(check_api_key)], 115 | docs_url="/docs", 116 | lifespan=lifespan, 117 | ) 118 | 119 | 120 | app.redis = None 121 | if settings.RATE_LIMITING is True: 122 | if settings.RATE_LIMITING: 123 | logger.debug("Connecting to redis") 124 | from redis.asyncio.cluster import RedisCluster 125 | 126 | try: 127 | redis_client = RedisCluster( 128 | host=settings.REDIS_HOST, 129 | port=settings.REDIS_PORT, 130 | decode_responses=True, 131 | socket_timeout=5, 132 | ) 133 | # attach to the app so it can be retrieved via the request 134 | app.redis = redis_client 135 | logger.debug("Redis connected") 136 | 137 | except Exception as e: 138 | logging.error( 139 | InfrastructureErrorLog(detail=f"failed to connect to redis: {e}") 140 | ) 141 | 142 | 143 | app.add_middleware( 144 | CORSMiddleware, 145 | allow_origins=["*"], 146 | allow_credentials=True, 147 | allow_methods=["*"], 148 | allow_headers=["*"], 149 | ) 150 | app.add_middleware(CacheControlMiddleware, cachecontrol="public, max-age=900") 151 | app.add_middleware(LoggingMiddleware) 152 | app.add_middleware(GZipMiddleware, minimum_size=1000) 153 | 154 | 155 | class OpenAQValidationResponseDetail(BaseModel): 156 | loc: list[str] | None = None 157 | msg: str | None = None 158 | type: str | None = None 159 | 160 | 161 | class OpenAQValidationResponse(BaseModel): 162 | detail: list[OpenAQValidationResponseDetail] | None = None 163 | 164 | 165 | @app.exception_handler(RequestValidationError) 166 | async def openaq_request_validation_exception_handler( 167 | request: Request, exc: RequestValidationError 168 | ): 169 | return ORJSONResponse(status_code=422, content=jsonable_encoder(str(exc))) 170 | 171 | 172 | @app.exception_handler(ValidationError) 173 | async def openaq_exception_handler(request: Request, exc: ValidationError): 174 | return ORJSONResponse(status_code=422, content=jsonable_encoder(str(exc))) 175 | 176 | 177 | @app.get("/ping", include_in_schema=False) 178 | def pong(): 179 | """ 180 | health check. 181 | This will let the user know that the service is operational. 182 | And this path operation will: 183 | * show a lifesign 184 | """ 185 | return {"ping": "pong!"} 186 | 187 | 188 | @app.get("/favicon.ico", include_in_schema=False) 189 | def favico(): 190 | return RedirectResponse("https://openaq.org/assets/graphics/meta/favicon.png") 191 | 192 | 193 | # v3 194 | app.include_router(auth.router) 195 | app.include_router(instruments.router) 196 | app.include_router(locations.router) 197 | app.include_router(licenses.router) 198 | app.include_router(parameters.router) 199 | app.include_router(tiles.router) 200 | app.include_router(countries.router) 201 | app.include_router(manufacturers.router) 202 | app.include_router(measurements.router) 203 | app.include_router(owners.router) 204 | app.include_router(providers.router) 205 | app.include_router(sensors.router) 206 | app.include_router(latest.router) 207 | app.include_router(flags.router) 208 | 209 | 210 | static_dir = Path.joinpath(Path(__file__).resolve().parent, "static") 211 | 212 | 213 | app.mount("/", StaticFiles(directory=str(static_dir), html=True)) 214 | 215 | 216 | def handler(event, context): 217 | asgi_handler = Mangum(app) 218 | return asgi_handler(event, context) 219 | 220 | 221 | def run(): 222 | attempts = 0 223 | while attempts < 10: 224 | try: 225 | import uvicorn 226 | 227 | uvicorn.run( 228 | "main:app", 229 | host="0.0.0.0", 230 | port=8888, 231 | reload=True, 232 | ) 233 | except Exception: 234 | attempts += 1 235 | logger.debug("waiting for database to start") 236 | time.sleep(3) 237 | pass 238 | 239 | 240 | if __name__ == "__main__": 241 | run() 242 | -------------------------------------------------------------------------------- /cdk/stacks/waf_rules.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | from aws_cdk.aws_wafv2 import CfnWebACL, CfnIPSet 3 | from constructs import Construct 4 | 5 | 6 | custom_response_bodies = { 7 | "ForbiddenMessage": CfnWebACL.CustomResponseBodyProperty( 8 | content='{"message": "Forbidden. violation of rate limit policy. Contact dev@openaq.org"}', 9 | content_type="APPLICATION_JSON", 10 | ), 11 | "UnauthorizedMessage": CfnWebACL.CustomResponseBodyProperty( 12 | content='{"message": "Unauthorized. A valid API key must be provided in the X-API-Key header."}', 13 | content_type="APPLICATION_JSON", 14 | ), 15 | "GoneMessage": CfnWebACL.CustomResponseBodyProperty( 16 | content='{"message": "Gone. Version 1 and Version 2 API endpoints are retired and no longer available. Please migrate to Version 3 endpoints."}', 17 | content_type="APPLICATION_JSON", 18 | ), 19 | } 20 | 21 | amazon_ip_reputation_list = CfnWebACL.RuleProperty( 22 | name="AWS-AWSManagedRulesAmazonIpReputationList", 23 | priority=0, 24 | statement=CfnWebACL.StatementProperty( 25 | managed_rule_group_statement=CfnWebACL.ManagedRuleGroupStatementProperty( 26 | vendor_name="AWS", name="AWSManagedRulesAmazonIpReputationList" 27 | ) 28 | ), 29 | override_action=CfnWebACL.OverrideActionProperty(none={}), 30 | visibility_config=CfnWebACL.VisibilityConfigProperty( 31 | sampled_requests_enabled=True, 32 | cloud_watch_metrics_enabled=True, 33 | metric_name="AWS-AWSManagedRulesAmazonIpReputationList", 34 | ), 35 | ) 36 | 37 | known_bad_inputs_rule_set = CfnWebACL.RuleProperty( 38 | name="AWS-AWSManagedRulesKnownBadInputsRuleSet", 39 | priority=1, 40 | statement=CfnWebACL.StatementProperty( 41 | managed_rule_group_statement=CfnWebACL.ManagedRuleGroupStatementProperty( 42 | vendor_name="AWS", name="AWSManagedRulesKnownBadInputsRuleSet" 43 | ) 44 | ), 45 | override_action=CfnWebACL.OverrideActionProperty(none={}), 46 | visibility_config=CfnWebACL.VisibilityConfigProperty( 47 | sampled_requests_enabled=True, 48 | cloud_watch_metrics_enabled=True, 49 | metric_name="AWS-AWSManagedRulesKnownBadInputsRuleSet", 50 | ), 51 | ) 52 | 53 | api_key_header_rule = CfnWebACL.RuleProperty( 54 | name="CheckXApiKeyHeader", 55 | priority=2, 56 | action=CfnWebACL.RuleActionProperty( 57 | block=CfnWebACL.BlockActionProperty( 58 | custom_response=CfnWebACL.CustomResponseProperty( 59 | response_code=401, custom_response_body_key="UnauthorizedMessage" 60 | ) 61 | ) 62 | ), 63 | statement=CfnWebACL.StatementProperty( 64 | and_statement=CfnWebACL.AndStatementProperty( 65 | statements=[ 66 | CfnWebACL.StatementProperty( 67 | not_statement=CfnWebACL.NotStatementProperty( 68 | statement=CfnWebACL.StatementProperty( 69 | size_constraint_statement=CfnWebACL.SizeConstraintStatementProperty( 70 | field_to_match=CfnWebACL.FieldToMatchProperty( 71 | single_header={"Name": "x-api-key"} 72 | ), 73 | comparison_operator="GT", 74 | size=0, 75 | text_transformations=[ 76 | CfnWebACL.TextTransformationProperty( 77 | priority=0, type="NONE" 78 | ) 79 | ], 80 | ) 81 | ) 82 | ) 83 | ), 84 | CfnWebACL.StatementProperty( 85 | byte_match_statement=CfnWebACL.ByteMatchStatementProperty( 86 | search_string="/v3/", 87 | field_to_match=CfnWebACL.FieldToMatchProperty(uri_path={}), 88 | text_transformations=[ 89 | CfnWebACL.TextTransformationProperty( 90 | priority=0, type="NONE" 91 | ) 92 | ], 93 | positional_constraint="CONTAINS", 94 | ) 95 | ), 96 | ] 97 | ) 98 | ), 99 | visibility_config=CfnWebACL.VisibilityConfigProperty( 100 | sampled_requests_enabled=True, 101 | cloud_watch_metrics_enabled=True, 102 | metric_name="CheckXApiKeyHeader", 103 | ), 104 | ) 105 | 106 | retired_endpoints_rule = CfnWebACL.RuleProperty( 107 | name="retiredVersionsEndpoints", 108 | priority=4, 109 | action=CfnWebACL.RuleActionProperty( 110 | block=CfnWebACL.BlockActionProperty( 111 | custom_response=CfnWebACL.CustomResponseProperty( 112 | response_code=410, custom_response_body_key="GoneMessage" 113 | ) 114 | ) 115 | ), 116 | statement=CfnWebACL.StatementProperty( 117 | or_statement=CfnWebACL.OrStatementProperty( 118 | statements=[ 119 | CfnWebACL.StatementProperty( 120 | byte_match_statement=CfnWebACL.ByteMatchStatementProperty( 121 | search_string="/v1/", 122 | field_to_match=CfnWebACL.FieldToMatchProperty(uri_path={}), 123 | text_transformations=[ 124 | CfnWebACL.TextTransformationProperty( 125 | priority=0, type="NONE" 126 | ) 127 | ], 128 | positional_constraint="CONTAINS", 129 | ) 130 | ), 131 | CfnWebACL.StatementProperty( 132 | byte_match_statement=CfnWebACL.ByteMatchStatementProperty( 133 | search_string="/v2/", 134 | field_to_match=CfnWebACL.FieldToMatchProperty(uri_path={}), 135 | text_transformations=[ 136 | CfnWebACL.TextTransformationProperty( 137 | priority=0, type="NONE" 138 | ) 139 | ], 140 | positional_constraint="CONTAINS", 141 | ) 142 | ), 143 | ] 144 | ) 145 | ), 146 | visibility_config=CfnWebACL.VisibilityConfigProperty( 147 | sampled_requests_enabled=True, 148 | cloud_watch_metrics_enabled=True, 149 | metric_name="retiredVersionsEndpoints", 150 | ), 151 | ) 152 | 153 | 154 | def ip_rate_limiter( 155 | limit: int, evaluation_window_sec: int = 60 156 | ) -> CfnWebACL.RuleProperty: 157 | return CfnWebACL.RuleProperty( 158 | name="IPRateLimiter", 159 | priority=5, 160 | statement=CfnWebACL.StatementProperty( 161 | rate_based_statement=CfnWebACL.RateBasedStatementProperty( 162 | aggregate_key_type="IP", 163 | evaluation_window_sec=evaluation_window_sec, 164 | limit=limit, 165 | ) 166 | ), 167 | action=CfnWebACL.RuleActionProperty(block={}), 168 | visibility_config=CfnWebACL.VisibilityConfigProperty( 169 | sampled_requests_enabled=True, 170 | cloud_watch_metrics_enabled=True, 171 | metric_name="IpRateLimiter", 172 | ), 173 | ) 174 | 175 | 176 | def ip_block_rule(stack: Construct, ips: List[str]) -> CfnWebACL.RuleProperty: 177 | ip_set = CfnIPSet( 178 | stack, 179 | "waf_ip_block_set", 180 | addresses=ips, 181 | ip_address_version="IPV4", 182 | scope="CLOUDFRONT", 183 | description="Set of IPs to specifically block to prevent abuse", 184 | name="OpenAQAPIWAFIPBlockList", 185 | ) 186 | 187 | return CfnWebACL.RuleProperty( 188 | name="IpBlockRule", 189 | priority=3, 190 | statement=CfnWebACL.StatementProperty( 191 | ip_set_reference_statement=CfnWebACL.IPSetReferenceStatementProperty( 192 | arn=ip_set.attr_arn 193 | ) 194 | ), 195 | action=CfnWebACL.RuleActionProperty( 196 | block=CfnWebACL.BlockActionProperty( 197 | custom_response=CfnWebACL.CustomResponseProperty( 198 | response_code=403, custom_response_body_key="ForbiddenMessage" 199 | ) 200 | ) 201 | ), 202 | visibility_config=CfnWebACL.VisibilityConfigProperty( 203 | sampled_requests_enabled=True, 204 | cloud_watch_metrics_enabled=True, 205 | metric_name="IpBlockRule", 206 | ), 207 | ) 208 | -------------------------------------------------------------------------------- /openaq_api/static/openaq-logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 14 | 16 | 40 | 45 | 50 | 55 | 56 | --------------------------------------------------------------------------------