├── .github └── workflows │ ├── main.yml │ └── test.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .python-version ├── Dockerfile ├── LICENSE ├── README.md ├── api ├── __init__.py ├── auth.py ├── bills.py ├── committees.py ├── db │ ├── __init__.py │ └── models │ │ ├── __init__.py │ │ ├── auth.py │ │ ├── bills.py │ │ ├── common.py │ │ ├── events.py │ │ ├── jurisdiction.py │ │ ├── people_orgs.py │ │ └── votes.py ├── events.py ├── jurisdictions.py ├── main.py ├── pagination.py ├── people.py ├── schemas.py ├── tests │ ├── __init__.py │ ├── conftest.py │ ├── fixtures.py │ ├── test_bills.py │ ├── test_committees.py │ ├── test_events.py │ ├── test_jurisdictions.py │ ├── test_pagination.py │ └── test_people.py └── utils.py ├── docker-compose.yml ├── poetry.lock ├── pyproject.toml └── setup.cfg /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Build and push Docker images 2 | on: 3 | push: 4 | branches: 5 | - main 6 | tags: 7 | - '*' 8 | jobs: 9 | publish: 10 | steps: 11 | - uses: actions/checkout@v3 12 | - name: Set up QEMU 13 | uses: docker/setup-qemu-action@v2 14 | - name: Set up Docker Buildx 15 | uses: docker/setup-buildx-action@v2 16 | - name: Docker Login 17 | uses: docker/login-action@v2 18 | with: 19 | username: ${{ secrets.DOCKER_USERNAME }} 20 | password: ${{ secrets.DOCKER_PASSWORD }} 21 | - name: build docker image 22 | uses: docker/build-push-action@v3 23 | with: 24 | tags: "openstates/api-v3:latest,openstates/api-v3:${{ github.ref_name }}" 25 | platforms: amd64,arm64 26 | push: true 27 | runs-on: ubuntu-latest 28 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test Python 2 | 3 | on: 4 | push: 5 | branches: 6 | - "**" 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | jobs: 12 | test: 13 | runs-on: ubuntu-latest 14 | 15 | services: 16 | postgres: 17 | image: postgres 18 | env: 19 | POSTGRES_USER: 'v3test' 20 | POSTGRES_PASSWORD: 'v3test' 21 | POSTGRES_DB: 'v3test' 22 | options: >- 23 | --health-cmd pg_isready 24 | --health-interval 10s 25 | --health-timeout 5s 26 | --health-retries 5 27 | ports: 28 | - 5432:5432 29 | 30 | steps: 31 | # Python & dependency installation 32 | - uses: actions/checkout@v3 33 | - name: setup Python 3.9 34 | uses: actions/setup-python@v2 35 | with: 36 | python-version: 3.9 37 | - name: install Poetry 38 | uses: snok/install-poetry@v1.3.3 39 | - name: cache Poetry virtualenv 40 | uses: actions/cache@v4 41 | id: cache 42 | with: 43 | path: ~/.virtualenvs/ 44 | key: poetry-${{ hashFiles('**/poetry.lock') }} 45 | restore-keys: | 46 | poetry-${{ hashFiles('**/poetry.lock') }} 47 | - name: set poetry config path 48 | run: poetry config virtualenvs.path ~/.virtualenvs 49 | - name: install dependencies 50 | run: poetry install --no-root 51 | # if: steps.cache.outputs.cache-hit != 'true' 52 | - name: lint with black 53 | run: poetry run black --diff --check . 54 | - name: lint with flake8 55 | run: poetry run flake8 api --show-source --statistics 56 | - name: pytest 57 | run: poetry run pytest 58 | env: 59 | PYTHONPATH: scripts 60 | DATABASE_URL: postgresql://v3test:v3test@localhost/v3test 61 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .idea/ 3 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | default_language_version: 2 | python: python3.8 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v2.2.3 # Use the ref you want to point at 6 | hooks: 7 | - id: check-merge-conflict 8 | - id: debug-statements 9 | - id: flake8 10 | - repo: https://github.com/ambv/black 11 | rev: 20.8b1 12 | hooks: 13 | - id: black 14 | -------------------------------------------------------------------------------- /.python-version: -------------------------------------------------------------------------------- 1 | 3.9.15 2 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM bmltenabled/uvicorn-gunicorn-fastapi:python3.9-slim 2 | 3 | # improve logging performance (don't buffer messages to output) 4 | ENV PYTHONUNBUFFERED=1 5 | # reduce images size 6 | ENV PYTHONDONTWRITEBYTECODE=1 7 | # next two environment files ensure UTF-8 processing works consistently 8 | ENV PYTHONIOENCODING='utf-8' 9 | ENV LANG='C.UTF-8' 10 | 11 | # install Poetry 12 | # also disable virtual environment creation so global installs (like gunicorn) 13 | # can actually see packages 14 | RUN pip3 install --disable-pip-version-check --no-cache-dir wheel \ 15 | && pip3 install --disable-pip-version-check --no-cache-dir poetry crcmod \ 16 | && poetry config virtualenvs.create false 17 | 18 | ENV MODULE_NAME=api.main 19 | WORKDIR /app 20 | 21 | COPY pyproject.toml /app/ 22 | COPY poetry.lock /app/ 23 | 24 | # the extra delete steps here probably aren't needed, but good to have available 25 | RUN poetry install --no-root --only main \ 26 | && rm -r /root/.cache/pypoetry/cache /root/.cache/pypoetry/artifacts/ \ 27 | && apt-get autoremove -yqq \ 28 | && apt-get clean \ 29 | && rm -rf /var/lib/apt/lists/* 30 | 31 | COPY . /app 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 James Turk, Open States 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Open States API v3 2 | 3 | This repository contains the code responsible for v3 of the Open States API. 4 | 5 | Report API Issues at https://github.com/openstates/issues/ 6 | 7 | ## Links 8 | 9 | * [Contributor's Guide](https://docs.openstates.org/en/latest/contributing/getting-started.html) 10 | * [Documentation](https://docs.openstates.org/en/latest/api/v3/) 11 | * [Code of Conduct](https://docs.openstates.org/en/latest/contributing/code-of-conduct.html) 12 | 13 | ## Multi-arch builds 14 | 15 | The selected base image supports `amd64` and `arm64` build targets (and this is shown in the CI workflow). 16 | Use `docker buildx` for local multi-arch builds, e.g. 17 | 18 | ```bash 19 | docker buildx create --use 20 | docker buildx build --platform amd64,arm64 . 21 | ``` 22 | 23 | ## Deploy 24 | 25 | See [infrastructure repo](https://github.com/openstates/infrastructure#api-restarts). Plural employees also see the 26 | [Open States Maintenance ops guide](https://civic-eagle.atlassian.net/wiki/spaces/ENG/pages/1393459207/Open+States+Maintenance#%E2%80%9CRestarting%E2%80%9D-the-API) 27 | 28 | ## Running locally 29 | 30 | To run locally, you first need to have a running local 31 | database [following these instructions](https://docs.openstates.org/contributing/local-database/) 32 | 33 | You also need to have a redis instance running. That can be run via the docker-compose config included in this package 34 | by running `docker compose up redis` 35 | 36 | You can then run the app directly with the command: 37 | 38 | ```bash 39 | DATABASE_URL=postgresql://openstates:openstates@localhost:5405/openstatesorg poetry run uvicorn api.main:app 40 | ``` 41 | 42 | * Check that the port is correct for your locally running database 43 | * Username/password/dbname in example above are from openstates-scrapers docker-compose.yml, they also need to match the 44 | local DB. 45 | 46 | To test out hitting the API, there will need to be a user account + profile entry with an API key in the local DB. The 47 | scripts involved in the above-mentioned instructions (openstates.org repo init DB) should result in the creation of an 48 | API key called `testkey` that can be used for local testing. 49 | 50 | ### Run tests 51 | 52 | * In addition to having the normal DB container running (as described above), you need to also start the `db-test` 53 | service available in this project's `docker-compose.yml`: in this repo directory, run `docker compose up -d db-test` 54 | * Run all tests: `poetry run pytest` in the CLI; or in PyCharm configure a python test run targeting the 55 | `pytest` module 56 | * Run an individual test: add an argument to either of the above specifying the test file and function name, eg 57 | `api/tests/test_bills.py::test_bills_filter_by_jurisdiction_abbr` 58 | 59 | ## Architecture 60 | 61 | Components of this FastAPI application: 62 | 63 | * Routes, found in the `api/` folder, such as `api/bills.py`. These expose HTTP API routes and contain the business 64 | logic executed when a request hits that route. 65 | * SQL Alchemy models, found in the `api/db/models` folder, such as `api/db/models/bills.py` that define the data models 66 | used by business logic to query data. 67 | * Pydantic schemas, found in the `api/schemas.py` folder, which define how data from the database is transformed into 68 | output that is returned by business logic to the client. 69 | * Pagination logic, found in `api/pagination.py` and in route files, provides a semi-magic way for business logic to 70 | use SQL Alchemy models to manage things like includes, pagination of results, etc.. 71 | 72 | If you want to make a change, such as adding a related entity to output on an endpoint, you likely need to make changes 73 | to: 74 | 75 | * The Model file: add the new entity as a model, and a relationship property on the entity that is related to it. This 76 | informs which columns are selected via which join logic. 77 | * The Pydantic schema: add the entity and fields to the output schema. Even if the Model changes are correct, the data 78 | will not show up in API output unless it is in the schema. 79 | * The relevant Pagination object in the route file: you may need to add to `include_map_overrides` to tell the 80 | pagination system that sub-entities should be fetched when an include is requested. If you add a sub-sub entity here, 81 | such as "actions.related_entities" to the `BillPagination`, make sure to explicitly add the sub-entity as well: 82 | "actions". Otherwise, additional queries will be generated to lazy-load the sub-entity. -------------------------------------------------------------------------------- /api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstates/api-v3/af82501a464ce91988bbe3d95b154dc3f701c925/api/__init__.py -------------------------------------------------------------------------------- /api/auth.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | from fastapi import Header, HTTPException, Depends 3 | from sqlalchemy.orm.exc import NoResultFound 4 | from rrl import RateLimiter, Tier, RateLimitExceeded 5 | from .db import SessionLocal, get_db, models 6 | 7 | limiter = RateLimiter( 8 | prefix="v3", 9 | tiers=[ 10 | Tier("default", 10, 0, 250), 11 | Tier("bronze", 40, 0, 1000), 12 | Tier("silver", 80, 0, 50000), 13 | Tier("unlimited", 360, 0, 1_000_000_000), 14 | ], 15 | use_redis_time=False, 16 | track_daily_usage=True, 17 | ) 18 | 19 | 20 | def apikey_auth( 21 | apikey: Optional[str] = None, 22 | x_api_key: Optional[str] = Header(None), 23 | db: SessionLocal = Depends(get_db), 24 | ): 25 | provided_apikey = x_api_key or apikey 26 | if not provided_apikey: 27 | raise HTTPException( 28 | 403, 29 | detail="Must provide API Key as ?apikey or X-API-KEY. " 30 | "Login and visit https://openstates.org/account/profile/ for your API key.", 31 | ) 32 | 33 | try: 34 | key = ( 35 | db.query(models.Profile) 36 | .filter(models.Profile.api_key == provided_apikey) 37 | .one() 38 | ) 39 | try: 40 | limiter.check_limit(provided_apikey, key.api_tier) 41 | except RateLimitExceeded as e: 42 | raise HTTPException(429, detail=str(e)) 43 | except ValueError: 44 | raise HTTPException( 45 | 401, 46 | detail="Inactive API Key. " 47 | "Login and visit https://openstates.org/account/profile/ for details.", 48 | ) 49 | except NoResultFound: 50 | raise HTTPException( 51 | 401, 52 | detail="Invalid API Key. " 53 | "Login and visit https://openstates.org/account/profile/ for your API key.", 54 | ) 55 | -------------------------------------------------------------------------------- /api/bills.py: -------------------------------------------------------------------------------- 1 | import re 2 | import datetime 3 | from typing import Optional, List 4 | from enum import Enum 5 | from fastapi import APIRouter, Depends, Query, HTTPException 6 | from sqlalchemy import func, desc, nullslast 7 | from sqlalchemy.orm import contains_eager 8 | from openstates.utils.transformers import fix_bill_id 9 | from .db import SessionLocal, get_db, models 10 | from .schemas import Bill 11 | from .pagination import Pagination 12 | from .auth import apikey_auth 13 | from .utils import jurisdiction_filter 14 | 15 | 16 | class BillInclude(str, Enum): 17 | sponsorships = "sponsorships" 18 | abstracts = "abstracts" 19 | other_titles = "other_titles" 20 | other_identifiers = "other_identifiers" 21 | actions = "actions" 22 | sources = "sources" 23 | documents = "documents" 24 | versions = "versions" 25 | votes = "votes" 26 | related_bills = "related_bills" 27 | 28 | 29 | class BillSortOption(str, Enum): 30 | updated_asc = "updated_asc" 31 | updated_desc = "updated_desc" 32 | first_action_asc = "first_action_asc" 33 | first_action_desc = "first_action_desc" 34 | latest_action_asc = "latest_action_asc" 35 | latest_action_desc = "latest_action_desc" 36 | 37 | 38 | class BillPagination(Pagination): 39 | ObjCls = Bill 40 | IncludeEnum = BillInclude 41 | include_map_overrides = { 42 | BillInclude.sponsorships: ["sponsorships", "sponsorships.person"], 43 | BillInclude.versions: ["versions", "versions.links"], 44 | BillInclude.documents: ["documents", "documents.links"], 45 | BillInclude.votes: [ 46 | "votes", 47 | "votes.votes", 48 | "votes.counts", 49 | "votes.sources", 50 | "votes.votes.voter", 51 | ], 52 | BillInclude.actions: ["actions", "actions.related_entities"], 53 | } 54 | max_per_page = 20 55 | 56 | 57 | router = APIRouter() 58 | 59 | 60 | _likely_bill_id = re.compile(r"\w{1,3}\s*\d{1,5}") 61 | 62 | 63 | def base_query(db): 64 | return ( 65 | db.query(models.Bill) 66 | .join(models.Bill.legislative_session) 67 | .join(models.LegislativeSession.jurisdiction) 68 | .join(models.Bill.from_organization) 69 | .options( 70 | contains_eager( 71 | models.Bill.legislative_session, models.LegislativeSession.jurisdiction 72 | ) 73 | ) 74 | .options(contains_eager(models.Bill.from_organization)) 75 | ) 76 | 77 | 78 | @router.get( 79 | "/bills", 80 | response_model=BillPagination.response_model(), 81 | response_model_exclude_none=True, 82 | tags=["bills"], 83 | ) 84 | async def bills_search( 85 | jurisdiction: Optional[str] = Query( 86 | None, description="Filter by jurisdiction name or ID." 87 | ), 88 | session: Optional[str] = Query(None, description="Filter by session identifier."), 89 | chamber: Optional[str] = Query( 90 | None, description="Filter by chamber of origination." 91 | ), 92 | identifier: Optional[List[str]] = Query( 93 | [], 94 | description="Filter to only include bills with this identifier.", 95 | ), 96 | classification: Optional[str] = Query( 97 | None, description="Filter by classification, e.g. bill or resolution" 98 | ), 99 | subject: Optional[List[str]] = Query( 100 | [], description="Filter by one or more subjects." 101 | ), 102 | updated_since: Optional[str] = Query( 103 | None, 104 | description="Filter to only include bills with updates since a given date.", 105 | ), 106 | created_since: Optional[str] = Query( 107 | None, description="Filter to only include bills created since a given date." 108 | ), 109 | action_since: Optional[str] = Query( 110 | None, 111 | description="Filter to only include bills with an action since a given date.", 112 | ), 113 | sort: Optional[BillSortOption] = Query( 114 | BillSortOption.updated_desc, description="Desired sort order for bill results." 115 | ), 116 | sponsor: Optional[str] = Query( 117 | None, 118 | description="Filter to only include bills sponsored by a given name or person ID.", 119 | ), 120 | sponsor_classification: Optional[str] = Query( 121 | None, 122 | description="Filter matched sponsors to only include particular types of sponsorships.", 123 | ), 124 | q: Optional[str] = Query(None, description="Filter by full text search term."), 125 | include: List[BillInclude] = Query( 126 | [], description="Additional information to include in response." 127 | ), 128 | db: SessionLocal = Depends(get_db), 129 | pagination: BillPagination = Depends(), 130 | auth: str = Depends(apikey_auth), 131 | ): 132 | """ 133 | Search for bills matching given criteria. 134 | 135 | Must either specify a jurisdiction or a full text query (q). Additional parameters will 136 | futher restrict bills returned. 137 | """ 138 | query = base_query(db) 139 | 140 | if sort == BillSortOption.updated_asc: 141 | query = query.order_by(models.Bill.updated_at) 142 | elif sort == BillSortOption.updated_desc: 143 | query = query.order_by(desc(models.Bill.updated_at)) 144 | elif sort == BillSortOption.first_action_asc: 145 | query = query.order_by(nullslast(models.Bill.first_action_date)) 146 | elif sort == BillSortOption.first_action_desc: 147 | query = query.order_by(nullslast(desc(models.Bill.first_action_date))) 148 | elif sort == BillSortOption.latest_action_asc: 149 | query = query.order_by(nullslast(models.Bill.latest_action_date)) 150 | elif sort == BillSortOption.latest_action_desc: 151 | query = query.order_by(nullslast(desc(models.Bill.latest_action_date))) 152 | else: 153 | raise HTTPException(500, "Unknown sort option, this shouldn't happen!") 154 | 155 | if jurisdiction: 156 | query = query.filter( 157 | jurisdiction_filter( 158 | jurisdiction, jid_field=models.LegislativeSession.jurisdiction_id 159 | ) 160 | ) 161 | if session: 162 | if not jurisdiction: 163 | raise HTTPException( 164 | 400, "filtering by session requires a jurisdiction parameter as well" 165 | ) 166 | query = query.filter(models.LegislativeSession.identifier == session) 167 | if chamber: 168 | query = query.filter(models.Organization.classification == chamber) 169 | if identifier: 170 | if len(identifier) > 20: 171 | raise HTTPException( 172 | 400, 173 | "can only provide up to 20 identifiers in one request", 174 | ) 175 | identifiers = [fix_bill_id(bill_id).upper() for bill_id in identifier] 176 | query = query.filter(models.Bill.identifier.in_(identifiers)) 177 | if classification: 178 | query = query.filter(models.Bill.classification.any(classification)) 179 | if subject: 180 | query = query.filter(models.Bill.subject.contains(subject)) 181 | if sponsor: 182 | # need to join this way, or sqlalchemy will try to join via from_organization 183 | query = query.join(models.Bill.sponsorships) 184 | if sponsor.startswith("ocd-person/"): 185 | query = query.filter(models.BillSponsorship.person_id == sponsor) 186 | else: 187 | query = query.filter(models.BillSponsorship.name == sponsor) 188 | if sponsor_classification: 189 | if not sponsor: 190 | raise HTTPException( 191 | 400, 192 | "filtering by sponsor_classification requires sponsor parameter as well", 193 | ) 194 | query = query.filter( 195 | models.BillSponsorship.classification == sponsor_classification 196 | ) 197 | try: 198 | if updated_since: 199 | query = query.filter( 200 | models.Bill.updated_at >= datetime.datetime.fromisoformat(updated_since) 201 | ) 202 | if created_since: 203 | query = query.filter( 204 | models.Bill.created_at >= datetime.datetime.fromisoformat(created_since) 205 | ) 206 | except ValueError: 207 | raise HTTPException( 208 | 400, 209 | "datetime must be in ISO-8601 format, try YYYY-MM-DD or YYYY-MM-DDTHH:MM:SS", 210 | ) 211 | 212 | if action_since: 213 | query = query.filter(models.Bill.latest_action_date >= action_since) 214 | if q: 215 | if _likely_bill_id.match(q): 216 | query = query.filter( 217 | func.upper(models.Bill.identifier) == fix_bill_id(q).upper() 218 | ) 219 | else: 220 | query = query.join(models.SearchableBill).filter( 221 | models.SearchableBill.search_vector.op("@@")( 222 | func.websearch_to_tsquery("english", q) 223 | ) 224 | ) 225 | 226 | if not q and not jurisdiction: 227 | raise HTTPException(400, "either 'jurisdiction' or 'q' required") 228 | 229 | # handle includes 230 | 231 | resp = pagination.paginate(query, includes=include) 232 | 233 | return resp 234 | 235 | 236 | @router.get( 237 | # we have to use the Starlette path type to allow slashes here 238 | "/bills/ocd-bill/{openstates_bill_id}", 239 | response_model=Bill, 240 | response_model_exclude_none=True, 241 | tags=["bills"], 242 | ) 243 | async def bill_detail_by_id( 244 | openstates_bill_id: str, 245 | include: List[BillInclude] = Query([]), 246 | db: SessionLocal = Depends(get_db), 247 | auth: str = Depends(apikey_auth), 248 | ): 249 | """Obtain bill information by internal ID in the format ocd-bill/*uuid*.""" 250 | query = base_query(db).filter(models.Bill.id == "ocd-bill/" + openstates_bill_id) 251 | return BillPagination.detail(query, includes=include) 252 | 253 | 254 | @router.get( 255 | # we have to use the Starlette path type to allow slashes here 256 | "/bills/{jurisdiction}/{session}/{bill_id}", 257 | response_model=Bill, 258 | response_model_exclude_none=True, 259 | tags=["bills"], 260 | ) 261 | async def bill_detail( 262 | jurisdiction: str, 263 | session: str, 264 | bill_id: str, 265 | include: List[BillInclude] = Query([]), 266 | db: SessionLocal = Depends(get_db), 267 | auth: str = Depends(apikey_auth), 268 | ): 269 | """Obtain bill information based on (state, session, bill_id).""" 270 | query = base_query(db).filter( 271 | models.Bill.identifier == fix_bill_id(bill_id).upper(), 272 | models.LegislativeSession.identifier == session, 273 | jurisdiction_filter( 274 | jurisdiction, jid_field=models.LegislativeSession.jurisdiction_id 275 | ), 276 | ) 277 | return BillPagination.detail(query, includes=include) 278 | -------------------------------------------------------------------------------- /api/committees.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | from typing import List, Optional 3 | from fastapi import APIRouter, Depends, Query 4 | from .db import SessionLocal, get_db, models 5 | from .schemas import Committee, OrgClassification, CommitteeClassification 6 | from .pagination import Pagination 7 | from .auth import apikey_auth 8 | from .utils import jurisdiction_filter 9 | 10 | 11 | router = APIRouter() 12 | 13 | 14 | class CommitteeInclude(str, Enum): 15 | memberships = "memberships" 16 | links = "links" 17 | sources = "sources" 18 | 19 | 20 | class CommitteePagination(Pagination): 21 | ObjCls = Committee 22 | IncludeEnum = CommitteeInclude 23 | include_map_overrides = { 24 | CommitteeInclude.memberships: ["memberships", "memberships.person"], 25 | CommitteeInclude.links: [], 26 | CommitteeInclude.sources: [], 27 | } 28 | max_per_page = 20 29 | 30 | def __init__(self, page: int = 1, per_page: int = 20): 31 | self.page = page 32 | self.per_page = per_page 33 | 34 | 35 | @router.get( 36 | "/committees", 37 | response_model=CommitteePagination.response_model(), 38 | response_model_exclude_none=True, 39 | tags=["committees"], 40 | ) 41 | async def committee_list( 42 | jurisdiction: str = Query(None, description="Filter by jurisdiction name or ID."), 43 | classification: Optional[CommitteeClassification] = None, 44 | parent: Optional[str] = Query( 45 | None, description="ocd-organization ID of parent committee." 46 | ), 47 | chamber: Optional[OrgClassification] = Query( 48 | None, description="Chamber of committee, generally upper or lower." 49 | ), 50 | include: List[CommitteeInclude] = Query( 51 | [], description="Additional includes for the Committee response." 52 | ), 53 | db: SessionLocal = Depends(get_db), 54 | auth: str = Depends(apikey_auth), 55 | pagination: CommitteePagination = Depends(), 56 | ): 57 | query = ( 58 | db.query(models.Organization) 59 | .filter( 60 | models.Organization.classification.in_(("committee", "subcommittee")), 61 | jurisdiction_filter( 62 | jurisdiction, jid_field=models.Organization.jurisdiction_id 63 | ), 64 | ) 65 | .order_by(models.Organization.name) 66 | ) 67 | 68 | # handle parameters 69 | if classification: 70 | query = query.filter(models.Organization.classification == classification) 71 | if parent: 72 | query = query.filter(models.Organization.parent_id == parent) 73 | if chamber: 74 | subquery = ( 75 | db.query(models.Organization.id) 76 | .filter( 77 | models.Organization.classification == chamber, 78 | jurisdiction_filter( 79 | jurisdiction, jid_field=models.Organization.jurisdiction_id 80 | ), 81 | ) 82 | .scalar_subquery() 83 | ) 84 | query = query.filter(models.Organization.parent_id == subquery) 85 | 86 | return pagination.paginate(query, includes=include) 87 | 88 | 89 | @router.get( 90 | # we have to use the Starlette path type to allow slashes here 91 | "/committees/{committee_id:path}", 92 | response_model=Committee, 93 | response_model_exclude_none=True, 94 | tags=["committees"], 95 | ) 96 | async def committee_detail( 97 | committee_id: str, 98 | include: List[CommitteeInclude] = Query( 99 | [], description="Additional includes for the Committee response." 100 | ), 101 | db: SessionLocal = Depends(get_db), 102 | auth: str = Depends(apikey_auth), 103 | ): 104 | """Get details on a single committee by ID.""" 105 | query = db.query(models.Organization).filter( 106 | models.Organization.id == committee_id, 107 | models.Organization.classification.in_(("committee", "subcommittee")), 108 | ) 109 | return CommitteePagination.detail(query, includes=include) 110 | -------------------------------------------------------------------------------- /api/db/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | from sqlalchemy import create_engine 3 | from sqlalchemy.ext.declarative import declarative_base 4 | from sqlalchemy.orm import sessionmaker 5 | 6 | DATABASE_URL = os.environ["DATABASE_URL"].replace("postgres://", "postgresql://") 7 | """ 8 | From https://docs.sqlalchemy.org/en/14/core/pooling.html 9 | Default pool/overflow size is 5/10, timeout 30 seconds 10 | 11 | max_overflow=15 - the number of connections to allow in connection pool “overflow”, 12 | that is connections that can be opened above and beyond the pool_size setting, 13 | which defaults to five. this is only used with QueuePool. 14 | 15 | pool_size=10 - the number of connections to keep open inside the connection pool. 16 | This used with QueuePool as well as SingletonThreadPool. 17 | With QueuePool, a pool_size setting of 0 indicates no limit; to disable pooling, 18 | set poolclass to NullPool instead. 19 | 20 | pool_timeout=30 - number of seconds to wait before giving up on getting a connection from the pool. 21 | This is only used with QueuePool. This can be a float but is subject to the limitations of 22 | Python time functions which may not be reliable in the tens of milliseconds. 23 | 24 | pool_recycle=28800 - this setting causes the pool to recycle connections after the given number 25 | of seconds has passed. It defaults to -1, or no timeout. For example, setting to 3600 means 26 | connections will be recycled after one hour. Note that MySQL in particular will disconnect 27 | automatically if no activity is detected on a connection for eight hours 28 | (although this is configurable with the MySQLDB connection itself and the server configuration as well). 29 | """ 30 | engine = create_engine( 31 | DATABASE_URL, 32 | pool_size=10, 33 | max_overflow=7, 34 | pool_timeout=45, 35 | pool_recycle=7200, 36 | connect_args={"application_name": "os_api_v3"}, 37 | ) 38 | SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) 39 | 40 | Base = declarative_base() 41 | 42 | 43 | def get_db(): 44 | db = SessionLocal() 45 | try: 46 | yield db 47 | finally: 48 | db.close() 49 | -------------------------------------------------------------------------------- /api/db/models/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .jurisdiction import Jurisdiction, LegislativeSession, RunPlan, DataExport 3 | from .people_orgs import ( 4 | Organization, 5 | Post, 6 | Person, 7 | PersonName, 8 | PersonLink, 9 | PersonSource, 10 | PersonOffice, 11 | Membership, 12 | ) 13 | from .bills import ( 14 | Bill, 15 | BillAbstract, 16 | BillTitle, 17 | BillIdentifier, 18 | BillAction, 19 | BillActionRelatedEntity, 20 | RelatedBill, 21 | BillSponsorship, 22 | BillSource, 23 | BillDocument, 24 | BillDocumentLink, 25 | BillVersion, 26 | BillVersionLink, 27 | SearchableBill, 28 | ) 29 | from .votes import VoteEvent, PersonVote, VoteCount, VoteSource 30 | from .auth import Profile 31 | from .events import ( 32 | Event, 33 | EventLocation, 34 | EventMedia, 35 | EventDocument, 36 | EventParticipant, 37 | EventAgendaItem, 38 | EventRelatedEntity, 39 | EventAgendaMedia, 40 | ) 41 | -------------------------------------------------------------------------------- /api/db/models/auth.py: -------------------------------------------------------------------------------- 1 | from .. import Base 2 | from sqlalchemy import Column, String 3 | 4 | 5 | class Profile(Base): 6 | __tablename__ = "profiles_profile" 7 | 8 | id = Column(String, primary_key=True, index=True) 9 | api_key = Column(String, index=True) 10 | api_tier = Column(String) 11 | 12 | # lots of other fields but we just need these for API 13 | -------------------------------------------------------------------------------- /api/db/models/bills.py: -------------------------------------------------------------------------------- 1 | from functools import lru_cache 2 | from sqlalchemy import Column, String, ForeignKey, DateTime, Integer, Boolean, Text 3 | from sqlalchemy.ext.declarative import declared_attr 4 | from sqlalchemy.dialects.postgresql import UUID, JSONB, ARRAY, TSVECTOR 5 | from sqlalchemy.orm import relationship 6 | from .. import Base 7 | from .common import PrimaryUUID, LinkBase, RelatedEntityBase 8 | from .jurisdiction import LegislativeSession 9 | from .people_orgs import Organization, Person 10 | 11 | 12 | @lru_cache(100) 13 | def _jid_to_abbr(jid): 14 | return jid.split(":")[-1].split("/")[0] 15 | 16 | 17 | class Bill(Base): 18 | __tablename__ = "opencivicdata_bill" 19 | 20 | id = Column(String, primary_key=True, index=True) 21 | identifier = Column(String) 22 | title = Column(String) 23 | # important for ARRAY types to use Text to avoid casting to String 24 | classification = Column(ARRAY(Text)) 25 | subject = Column(ARRAY(Text)) 26 | extras = Column(JSONB) 27 | created_at = Column(DateTime(timezone=True)) 28 | updated_at = Column(DateTime(timezone=True)) 29 | 30 | from_organization_id = Column(String, ForeignKey(Organization.id)) 31 | from_organization = relationship("Organization") 32 | legislative_session_id = Column( 33 | UUID(as_uuid=True), ForeignKey(LegislativeSession.id) 34 | ) 35 | legislative_session = relationship("LegislativeSession") 36 | 37 | abstracts = relationship("BillAbstract", back_populates="bill") 38 | other_titles = relationship("BillTitle", back_populates="bill") 39 | other_identifiers = relationship("BillIdentifier", back_populates="bill") 40 | sponsorships = relationship("BillSponsorship", back_populates="bill") 41 | actions = relationship("BillAction", back_populates="bill") 42 | sources = relationship("BillSource", back_populates="bill") 43 | documents = relationship("BillDocument", back_populates="bill") 44 | versions = relationship("BillVersion", back_populates="bill") 45 | votes = relationship("VoteEvent", back_populates="bill") 46 | related_bills = relationship( 47 | "RelatedBill", back_populates="bill", foreign_keys="RelatedBill.bill_id" 48 | ) 49 | 50 | @property 51 | def jurisdiction(self): 52 | return self.legislative_session.jurisdiction 53 | 54 | @property 55 | def session(self): 56 | return self.legislative_session.identifier 57 | 58 | @property 59 | def openstates_url(self): 60 | abbr = _jid_to_abbr(self.legislative_session.jurisdiction_id) 61 | identifier = self.identifier.replace(" ", "") 62 | return f"https://openstates.org/{abbr}/bills/{self.session}/{identifier}/" 63 | 64 | # computed fields 65 | first_action_date = Column(String) 66 | latest_action_date = Column(String) 67 | latest_action_description = Column(String) 68 | latest_passage_date = Column(String) 69 | 70 | 71 | class BillRelatedBase(PrimaryUUID): 72 | @declared_attr 73 | def bill_id(cls): 74 | return Column("bill_id", ForeignKey(Bill.id)) 75 | 76 | @declared_attr 77 | def bill(cls): 78 | return relationship(Bill) 79 | 80 | 81 | class BillAbstract(BillRelatedBase, Base): 82 | __tablename__ = "opencivicdata_billabstract" 83 | 84 | abstract = Column(String) 85 | note = Column(String) 86 | 87 | 88 | class BillTitle(BillRelatedBase, Base): 89 | __tablename__ = "opencivicdata_billtitle" 90 | 91 | title = Column(String) 92 | note = Column(String) 93 | 94 | 95 | class BillIdentifier(BillRelatedBase, Base): 96 | __tablename__ = "opencivicdata_billidentifier" 97 | 98 | identifier = Column(String) 99 | 100 | 101 | class BillAction(BillRelatedBase, Base): 102 | __tablename__ = "opencivicdata_billaction" 103 | 104 | organization_id = Column(String, ForeignKey(Organization.id)) 105 | organization = relationship(Organization) 106 | description = Column(String) 107 | date = Column(String) 108 | classification = Column(ARRAY(Text), default=list) 109 | order = Column(Integer) 110 | related_entities = relationship("BillActionRelatedEntity", back_populates="action") 111 | 112 | 113 | class BillActionRelatedEntity(RelatedEntityBase, Base): 114 | __tablename__ = "opencivicdata_billactionrelatedentity" 115 | 116 | action_id = Column(UUID(as_uuid=True), ForeignKey(BillAction.id)) 117 | action = relationship(BillAction, back_populates="related_entities") 118 | name = Column(String) 119 | entity_type = Column(String) 120 | organization_id = Column(String, ForeignKey(Organization.id)) 121 | person_id = Column(String, ForeignKey(Person.id)) 122 | 123 | 124 | class RelatedBill(PrimaryUUID, Base): 125 | __tablename__ = "opencivicdata_relatedbill" 126 | 127 | bill_id = Column(String, ForeignKey(Bill.id)) 128 | bill = relationship(Bill, foreign_keys=[bill_id]) 129 | related_bill_id = Column(String, ForeignKey(Bill.id)) 130 | related_bill = relationship(Bill, foreign_keys=[related_bill_id]) 131 | 132 | identifier = Column(String) 133 | legislative_session = Column(String) # not a FK, it can be unknown 134 | relation_type = Column(String) 135 | 136 | 137 | class BillSponsorship(RelatedEntityBase, BillRelatedBase, Base): 138 | __tablename__ = "opencivicdata_billsponsorship" 139 | 140 | primary = Column(Boolean) 141 | classification = Column(String) 142 | 143 | 144 | class BillSource(LinkBase, BillRelatedBase, Base): 145 | __tablename__ = "opencivicdata_billsource" 146 | 147 | 148 | class DocVerBase(BillRelatedBase): 149 | """base class for document & version""" 150 | 151 | note = Column(String) 152 | date = Column(String) 153 | extras = Column(JSONB) 154 | 155 | 156 | class DocumentLinkBase(PrimaryUUID): 157 | media_type = Column(String) 158 | url = Column(String) 159 | 160 | 161 | class BillDocument(DocVerBase, Base): 162 | __tablename__ = "opencivicdata_billdocument" 163 | 164 | classification = Column(String) 165 | links = relationship("BillDocumentLink", back_populates="document") 166 | 167 | 168 | class BillVersion(DocVerBase, Base): 169 | __tablename__ = "opencivicdata_billversion" 170 | 171 | classification = Column(String) 172 | links = relationship("BillVersionLink", back_populates="version") 173 | 174 | 175 | class BillDocumentLink(DocumentLinkBase, Base): 176 | __tablename__ = "opencivicdata_billdocumentlink" 177 | 178 | document_id = Column(UUID(as_uuid=True), ForeignKey(BillDocument.id)) 179 | document = relationship(BillDocument) 180 | 181 | 182 | class BillVersionLink(DocumentLinkBase, Base): 183 | __tablename__ = "opencivicdata_billversionlink" 184 | 185 | version_id = Column(UUID(as_uuid=True), ForeignKey(BillVersion.id)) 186 | version = relationship(BillVersion) 187 | 188 | 189 | class SearchableBill(Base): 190 | __tablename__ = "opencivicdata_searchablebill" 191 | 192 | id = Column(Integer, primary_key=True, index=True) 193 | search_vector = Column(TSVECTOR) 194 | bill_id = Column(String, ForeignKey(Bill.id)) 195 | bill = relationship(Bill) 196 | -------------------------------------------------------------------------------- /api/db/models/common.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | from sqlalchemy import Column, String, ForeignKey 3 | from sqlalchemy.ext.declarative import declared_attr 4 | from sqlalchemy.orm import relationship 5 | from sqlalchemy.dialects.postgresql import UUID 6 | 7 | 8 | class PrimaryUUID: 9 | id = Column(UUID(as_uuid=True), primary_key=True, index=True, default=uuid.uuid4) 10 | 11 | 12 | class LinkBase(PrimaryUUID): 13 | url = Column(String) 14 | note = Column(String) 15 | 16 | 17 | class RelatedEntityBase(PrimaryUUID): 18 | name = Column(String) 19 | entity_type = Column(String) 20 | 21 | @declared_attr 22 | def organization_id(cls): 23 | return Column("organization_id", ForeignKey("opencivicdata_organization.id")) 24 | 25 | @declared_attr 26 | def organization(cls): 27 | return relationship("Organization") 28 | 29 | @declared_attr 30 | def person_id(cls): 31 | return Column("person_id", ForeignKey("opencivicdata_person.id")) 32 | 33 | @declared_attr 34 | def person(cls): 35 | return relationship("Person") 36 | -------------------------------------------------------------------------------- /api/db/models/events.py: -------------------------------------------------------------------------------- 1 | from .. import Base 2 | from sqlalchemy import ( 3 | Column, 4 | String, 5 | ForeignKey, 6 | Integer, 7 | Boolean, 8 | Text, 9 | ARRAY, 10 | ) 11 | from sqlalchemy.orm import relationship 12 | from sqlalchemy.dialects.postgresql import JSONB, UUID 13 | from .common import PrimaryUUID, RelatedEntityBase 14 | from .jurisdiction import Jurisdiction 15 | from .bills import Bill 16 | from .votes import VoteEvent 17 | 18 | 19 | class EventLocation(PrimaryUUID, Base): 20 | __tablename__ = "opencivicdata_eventlocation" 21 | 22 | name = Column(String) 23 | url = Column(String) 24 | 25 | jurisdiction_id = Column(String, ForeignKey(Jurisdiction.id)) 26 | jurisdiction = relationship("Jurisdiction") 27 | 28 | 29 | class Event(Base): 30 | __tablename__ = "opencivicdata_event" 31 | 32 | id = Column(String, primary_key=True, index=True) 33 | name = Column(String) 34 | description = Column(Text) 35 | classification = Column(String) 36 | start_date = Column(String) 37 | end_date = Column(String) 38 | all_day = Column(Boolean) 39 | status = Column(String) 40 | upstream_id = Column(String) 41 | deleted = Column(Boolean) 42 | 43 | sources = Column(JSONB) 44 | links = Column(JSONB) 45 | 46 | jurisdiction_id = Column(String, ForeignKey(Jurisdiction.id)) 47 | jurisdiction = relationship(Jurisdiction) 48 | location_id = Column(UUID(as_uuid=True), ForeignKey(EventLocation.id)) 49 | location = relationship(EventLocation) 50 | 51 | media = relationship("EventMedia", back_populates="event") 52 | documents = relationship("EventDocument", back_populates="event") 53 | participants = relationship("EventParticipant", back_populates="event") 54 | agenda = relationship("EventAgendaItem", back_populates="event") 55 | 56 | 57 | class EventMediaBase(PrimaryUUID): 58 | note = Column(String) 59 | date = Column(String) 60 | offset = Column(Integer) 61 | classification = Column(String) 62 | links = Column(JSONB) 63 | 64 | 65 | class EventMedia(EventMediaBase, Base): 66 | __tablename__ = "opencivicdata_eventmedia" 67 | 68 | event_id = Column(String, ForeignKey(Event.id)) 69 | event = relationship(Event, back_populates="media") 70 | 71 | 72 | class EventDocument(PrimaryUUID, Base): 73 | __tablename__ = "opencivicdata_eventdocument" 74 | 75 | event_id = Column(String, ForeignKey(Event.id)) 76 | event = relationship("Event", back_populates="documents") 77 | 78 | note = Column(String) 79 | date = Column(String) 80 | classification = Column(String) 81 | links = Column(JSONB) 82 | 83 | 84 | class EventParticipant(RelatedEntityBase, Base): 85 | __tablename__ = "opencivicdata_eventparticipant" 86 | 87 | event_id = Column(String, ForeignKey(Event.id)) 88 | event = relationship("Event", back_populates="participants") 89 | 90 | note = Column(Text) 91 | 92 | 93 | class EventAgendaItem(PrimaryUUID, Base): 94 | __tablename__ = "opencivicdata_eventagendaitem" 95 | 96 | event_id = Column(String, ForeignKey(Event.id)) 97 | event = relationship("Event", back_populates="agenda") 98 | 99 | description = Column(Text) 100 | classification = Column(ARRAY(Text)) 101 | order = Column(Integer) 102 | subjects = Column(ARRAY(Text)) 103 | notes = Column(ARRAY(Text)) 104 | extras = Column(JSONB) 105 | 106 | related_entities = relationship("EventRelatedEntity", back_populates="agenda_item") 107 | media = relationship("EventAgendaMedia", back_populates="agenda_item") 108 | 109 | 110 | class EventRelatedEntity(RelatedEntityBase, Base): 111 | __tablename__ = "opencivicdata_eventrelatedentity" 112 | 113 | agenda_item_id = Column(UUID(as_uuid=True), ForeignKey(EventAgendaItem.id)) 114 | agenda_item = relationship(EventAgendaItem) 115 | 116 | bill_id = Column(String, ForeignKey(Bill.id)) 117 | bill = relationship(Bill) 118 | vote_event_id = Column(String, ForeignKey(VoteEvent.id)) 119 | vote_event = relationship(VoteEvent) 120 | note = Column(Text) 121 | 122 | 123 | class EventAgendaMedia(EventMediaBase, Base): 124 | __tablename__ = "opencivicdata_eventagendamedia" 125 | 126 | agenda_item_id = Column(UUID(as_uuid=True), ForeignKey(EventAgendaItem.id)) 127 | agenda_item = relationship(EventAgendaItem) 128 | -------------------------------------------------------------------------------- /api/db/models/jurisdiction.py: -------------------------------------------------------------------------------- 1 | from .. import Base 2 | from sqlalchemy import Column, String, ForeignKey, Integer, Boolean, DateTime 3 | from sqlalchemy.dialects.postgresql import UUID 4 | from sqlalchemy.orm import relationship, object_session 5 | from .common import PrimaryUUID 6 | 7 | 8 | class Jurisdiction(Base): 9 | __tablename__ = "opencivicdata_jurisdiction" 10 | 11 | id = Column(String, primary_key=True, index=True) 12 | name = Column(String) 13 | url = Column(String) 14 | classification = Column(String, index=True) 15 | division_id = Column(String) 16 | latest_bill_update = Column(DateTime) 17 | latest_people_update = Column(DateTime) 18 | 19 | organizations = relationship( 20 | "Organization", 21 | primaryjoin="""and_( 22 | Jurisdiction.id == Organization.jurisdiction_id, 23 | Organization.classification.in_(('upper', 'lower', 'legislature', 'executive')) 24 | )""", 25 | ) 26 | legislative_sessions = relationship( 27 | "LegislativeSession", 28 | order_by="LegislativeSession.start_date", 29 | back_populates="jurisdiction", 30 | ) 31 | run_plans = relationship( 32 | "RunPlan", order_by="desc(RunPlan.end_time)", back_populates="jurisdiction" 33 | ) 34 | 35 | def get_latest_runs(self): 36 | """ 37 | We only want the last <=20 most recent runs, 38 | so ask for the last 20 objects in the returned 39 | list. 40 | """ 41 | return object_session(self).query(RunPlan).with_parent(self)[-20:] 42 | 43 | 44 | class LegislativeSession(PrimaryUUID, Base): 45 | __tablename__ = "opencivicdata_legislativesession" 46 | 47 | identifier = Column(String) 48 | name = Column(String) 49 | classification = Column(String, default="") 50 | start_date = Column(String, default="") 51 | end_date = Column(String, default="") 52 | 53 | jurisdiction_id = Column(String, ForeignKey(Jurisdiction.id)) 54 | jurisdiction = relationship("Jurisdiction") 55 | 56 | downloads = relationship( 57 | "DataExport", 58 | back_populates="session", 59 | primaryjoin="""and_(DataExport.session_id == LegislativeSession.id, DataExport.data_type == 'csv')""", 60 | ) 61 | 62 | 63 | class DataExport(Base): 64 | __tablename__ = "bulk_dataexport" 65 | 66 | id = Column(Integer, primary_key=True, index=True) 67 | created_at = Column(DateTime) 68 | updated_at = Column(DateTime) 69 | session_id = Column(UUID(as_uuid=True), ForeignKey(LegislativeSession.id)) 70 | data_type = Column(String) 71 | url = Column(String) 72 | 73 | session = relationship(LegislativeSession) 74 | 75 | 76 | class RunPlan(Base): 77 | __tablename__ = "pupa_runplan" 78 | 79 | id = Column(Integer, primary_key=True, index=True) 80 | success = Column(Boolean) 81 | start_time = Column(DateTime) 82 | end_time = Column(DateTime) 83 | 84 | jurisdiction_id = Column(String, ForeignKey(Jurisdiction.id)) 85 | jurisdiction = relationship("Jurisdiction") 86 | -------------------------------------------------------------------------------- /api/db/models/people_orgs.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | import base62 3 | from slugify import slugify 4 | from sqlalchemy import Column, String, ForeignKey, DateTime, Integer 5 | from sqlalchemy.dialects.postgresql import JSONB 6 | from sqlalchemy.orm import relationship 7 | from .. import Base 8 | from .common import PrimaryUUID, LinkBase 9 | from .jurisdiction import Jurisdiction 10 | 11 | 12 | def encode_uuid(id): 13 | uuid_portion = str(id).split("/")[1] 14 | as_int = uuid.UUID(uuid_portion).int 15 | return base62.encode(as_int) 16 | 17 | 18 | class Organization(Base): 19 | __tablename__ = "opencivicdata_organization" 20 | 21 | id = Column(String, primary_key=True, index=True) 22 | name = Column(String) 23 | classification = Column(String) 24 | parent_id = Column(String, index=True) 25 | 26 | links = Column(JSONB) 27 | sources = Column(JSONB) 28 | extras = Column(JSONB, default=dict) 29 | 30 | jurisdiction_id = Column(String, ForeignKey(Jurisdiction.id)) 31 | jurisdiction = relationship("Jurisdiction", back_populates="organizations") 32 | 33 | posts = relationship("Post", order_by="Post.label", back_populates="organization") 34 | memberships = relationship("Membership", back_populates="organization") 35 | 36 | @property 37 | def districts(self): 38 | return self.posts 39 | 40 | 41 | class Post(Base): 42 | __tablename__ = "opencivicdata_post" 43 | 44 | id = Column(String, primary_key=True, index=True) 45 | label = Column(String) 46 | role = Column(String) 47 | division_id = Column(String) 48 | maximum_memberships = Column(Integer) 49 | 50 | organization_id = Column(String, ForeignKey(Organization.id)) 51 | organization = relationship("Organization") 52 | 53 | 54 | class Person(Base): 55 | __tablename__ = "opencivicdata_person" 56 | 57 | id = Column(String, primary_key=True, index=True) 58 | name = Column(String, index=True) 59 | family_name = Column(String, default="") 60 | given_name = Column(String, default="") 61 | image = Column(String, default="") 62 | gender = Column(String, default="") 63 | email = Column(String, default="") 64 | biography = Column(String, default="") 65 | birth_date = Column(String, default="") 66 | death_date = Column(String, default="") 67 | party = Column("primary_party", String) 68 | current_role = Column(JSONB) 69 | created_at = Column(DateTime(timezone=True)) 70 | updated_at = Column(DateTime(timezone=True)) 71 | extras = Column(JSONB, default=dict) 72 | jurisdiction_id = Column( 73 | "current_jurisdiction_id", String, ForeignKey(Jurisdiction.id) 74 | ) 75 | jurisdiction = relationship("Jurisdiction") 76 | 77 | other_identifiers = relationship("PersonIdentifier", back_populates="person") 78 | other_names = relationship("PersonName", back_populates="person") 79 | links = relationship("PersonLink", back_populates="person") 80 | sources = relationship("PersonSource", back_populates="person") 81 | offices = relationship("PersonOffice", back_populates="person") 82 | 83 | @property 84 | def openstates_url(self): 85 | """get canonical URL for openstates.org""" 86 | return f"https://openstates.org/person/{slugify(self.name)}-{encode_uuid(self.id)}/" 87 | 88 | 89 | class PersonIdentifier(PrimaryUUID, Base): 90 | __tablename__ = "opencivicdata_personidentifier" 91 | 92 | person_id = Column(String, ForeignKey(Person.id)) 93 | person = relationship(Person) 94 | identifier = Column(String) 95 | scheme = Column(String) 96 | 97 | 98 | class PersonName(PrimaryUUID, Base): 99 | __tablename__ = "opencivicdata_personname" 100 | 101 | person_id = Column(String, ForeignKey(Person.id)) 102 | person = relationship(Person) 103 | name = Column(String) 104 | note = Column(String) 105 | 106 | 107 | class PersonLink(LinkBase, Base): 108 | __tablename__ = "opencivicdata_personlink" 109 | 110 | person_id = Column(String, ForeignKey(Person.id)) 111 | person = relationship(Person) 112 | 113 | 114 | class PersonSource(LinkBase, Base): 115 | __tablename__ = "opencivicdata_personsource" 116 | 117 | person_id = Column(String, ForeignKey(Person.id)) 118 | person = relationship(Person) 119 | 120 | 121 | class PersonOffice(PrimaryUUID, Base): 122 | __tablename__ = "openstates_personoffice" 123 | 124 | person_id = Column(String, ForeignKey(Person.id)) 125 | person = relationship(Person) 126 | classification = Column(String) 127 | address = Column(String) 128 | voice = Column(String) 129 | fax = Column(String) 130 | name_ = Column("name", String) 131 | 132 | @property 133 | def name(self): 134 | if self.name_: 135 | return self.name_ 136 | else: 137 | return self.classification.title() + " Office" 138 | 139 | 140 | class Membership(PrimaryUUID, Base): 141 | __tablename__ = "opencivicdata_membership" 142 | 143 | organization_id = Column(String, ForeignKey(Organization.id)) 144 | organization = relationship("Organization") 145 | 146 | person_name = Column(String) 147 | person_id = Column(String, ForeignKey(Person.id)) 148 | person = relationship("Person") 149 | 150 | role = Column(String) 151 | -------------------------------------------------------------------------------- /api/db/models/votes.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, String, ForeignKey, Integer, Text 2 | from sqlalchemy.dialects.postgresql import UUID, JSONB, ARRAY 3 | from sqlalchemy.orm import relationship 4 | from .. import Base 5 | from .common import PrimaryUUID, LinkBase 6 | from .jurisdiction import LegislativeSession 7 | from .people_orgs import Organization, Person 8 | from .bills import Bill, BillAction 9 | 10 | 11 | class VoteEvent(Base): 12 | __tablename__ = "opencivicdata_voteevent" 13 | 14 | id = Column(String, primary_key=True, index=True) 15 | identifier = Column(String) 16 | motion_text = Column(String) 17 | motion_classification = Column(ARRAY(Text), default=list) 18 | start_date = Column(String) 19 | result = Column(String) 20 | extras = Column(JSONB, default=dict) 21 | 22 | organization_id = Column(String, ForeignKey(Organization.id)) 23 | organization = relationship(Organization) 24 | legislative_session_id = Column(UUID, ForeignKey(LegislativeSession.id)) 25 | legislative_session = relationship(LegislativeSession) 26 | bill_id = Column(String, ForeignKey(Bill.id)) 27 | bill = relationship(Bill) 28 | bill_action_id = Column(UUID, ForeignKey(BillAction.id)) 29 | bill_action = relationship(BillAction) 30 | 31 | votes = relationship("PersonVote", back_populates="vote_event") 32 | counts = relationship("VoteCount", back_populates="vote_event") 33 | sources = relationship("VoteSource", back_populates="vote_event") 34 | 35 | 36 | class VoteCount(PrimaryUUID, Base): 37 | __tablename__ = "opencivicdata_votecount" 38 | 39 | vote_event_id = Column(String, ForeignKey(VoteEvent.id)) 40 | vote_event = relationship(VoteEvent) 41 | 42 | option = Column(String) 43 | value = Column(Integer) 44 | 45 | 46 | class VoteSource(LinkBase, Base): 47 | __tablename__ = "opencivicdata_votesource" 48 | 49 | vote_event_id = Column(String, ForeignKey(VoteEvent.id)) 50 | vote_event = relationship(VoteEvent) 51 | 52 | 53 | class PersonVote(PrimaryUUID, Base): 54 | __tablename__ = "opencivicdata_personvote" 55 | 56 | vote_event_id = Column(String, ForeignKey(VoteEvent.id)) 57 | vote_event = relationship(VoteEvent) 58 | 59 | option = Column(String) 60 | voter_name = Column(String) 61 | voter_id = Column(String, ForeignKey(Person.id)) 62 | voter = relationship(Person) 63 | note = Column(String) 64 | -------------------------------------------------------------------------------- /api/events.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | from typing import List # , Optional 3 | from fastapi import APIRouter, Depends, Query, HTTPException 4 | from sqlalchemy import func 5 | from sqlalchemy.orm import contains_eager 6 | from .db import SessionLocal, get_db, models 7 | from .schemas import Event 8 | from .pagination import Pagination 9 | from .auth import apikey_auth 10 | from .utils import jurisdiction_filter 11 | 12 | router = APIRouter() 13 | 14 | 15 | class EventInclude(str, Enum): 16 | links = "links" 17 | sources = "sources" 18 | media = "media" 19 | documents = "documents" 20 | participants = "participants" 21 | agenda = "agenda" 22 | 23 | 24 | class EventPagination(Pagination): 25 | ObjCls = Event 26 | IncludeEnum = EventInclude 27 | include_map_overrides = { 28 | EventInclude.links: [], 29 | EventInclude.sources: [], 30 | EventInclude.agenda: ["agenda", "agenda.related_entities", "agenda.media"], 31 | } 32 | max_per_page = 20 33 | 34 | def __init__(self, page: int = 1, per_page: int = 20): 35 | self.page = page 36 | self.per_page = per_page 37 | 38 | 39 | @router.get( 40 | "/events", 41 | response_model=EventPagination.response_model(), 42 | response_model_exclude_none=True, 43 | tags=["events"], 44 | ) 45 | async def event_list( 46 | jurisdiction: str = Query(None, description="Filter by jurisdiction name or ID."), 47 | deleted: bool = Query(False, description="Return events marked as deleted?"), 48 | before: str = Query( 49 | None, description="Limit results to those starting before a given datetime." 50 | ), 51 | after: str = Query( 52 | None, description="Limit results to those starting before a given datetime." 53 | ), 54 | require_bills: bool = Query( 55 | False, 56 | description="Limit results to events with associated bills.", 57 | ), 58 | include: List[EventInclude] = Query( 59 | [], description="Additional includes for the Event response." 60 | ), 61 | db: SessionLocal = Depends(get_db), 62 | auth: str = Depends(apikey_auth), 63 | pagination: EventPagination = Depends(), 64 | ): 65 | if not jurisdiction: 66 | raise HTTPException( 67 | 400, 68 | "must provide 'jurisdiction' parameter", 69 | ) 70 | query = ( 71 | db.query(models.Event) 72 | .join(models.Event.jurisdiction) 73 | .outerjoin(models.Event.location) 74 | .filter( 75 | jurisdiction_filter(jurisdiction, jid_field=models.Event.jurisdiction_id), 76 | ) 77 | .order_by(models.Event.start_date, models.Event.id) 78 | ).options( 79 | contains_eager( 80 | models.Event.jurisdiction, 81 | ), 82 | contains_eager( 83 | models.Event.location, 84 | ), 85 | ) 86 | 87 | # handle parameters 88 | query = query.filter(models.Event.deleted.is_(deleted)) 89 | 90 | if before: 91 | query = query.filter(models.Event.start_date < before) 92 | if after: 93 | query = query.filter(models.Event.start_date > after) 94 | if require_bills: 95 | query = ( 96 | query.outerjoin(models.Event.agenda) 97 | .outerjoin(models.EventAgendaItem.related_entities) 98 | .group_by(models.Event, models.Jurisdiction, models.EventLocation) 99 | .filter(models.EventRelatedEntity.entity_type == "bill") 100 | .having(func.count_(models.EventRelatedEntity.id) > 0) 101 | ) 102 | 103 | return pagination.paginate(query, includes=include) 104 | 105 | 106 | @router.get( 107 | # we have to use the Starlette path type to allow slashes here 108 | "/events/{event_id:path}", 109 | response_model=Event, 110 | response_model_exclude_none=True, 111 | tags=["events"], 112 | ) 113 | async def event_detail( 114 | event_id: str, 115 | include: List[EventInclude] = Query( 116 | [], description="Additional includes for the Event response." 117 | ), 118 | db: SessionLocal = Depends(get_db), 119 | auth: str = Depends(apikey_auth), 120 | ): 121 | """Get details on a single event by ID.""" 122 | query = ( 123 | db.query(models.Event) 124 | .filter(models.Event.id == event_id) 125 | .join(models.Event.jurisdiction) 126 | .outerjoin(models.Event.location) 127 | .options( 128 | contains_eager( 129 | models.Event.jurisdiction, 130 | ), 131 | contains_eager( 132 | models.Event.location, 133 | ), 134 | ) 135 | ) 136 | return EventPagination.detail(query, includes=include) 137 | -------------------------------------------------------------------------------- /api/jurisdictions.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | from typing import Optional, List 3 | from fastapi import APIRouter, Depends, Query 4 | from .db import SessionLocal, get_db, models 5 | from .schemas import Jurisdiction, JurisdictionClassification 6 | from .pagination import Pagination 7 | from .auth import apikey_auth 8 | from .utils import jurisdiction_filter 9 | 10 | 11 | class JurisdictionInclude(str, Enum): 12 | organizations = "organizations" 13 | legislative_sessions = "legislative_sessions" 14 | latest_runs = "latest_runs" 15 | 16 | 17 | class JurisdictionPagination(Pagination): 18 | ObjCls = Jurisdiction 19 | IncludeEnum = JurisdictionInclude 20 | include_map_overrides = { 21 | JurisdictionInclude.organizations: ["organizations", "organizations.posts"], 22 | JurisdictionInclude.latest_runs: [], # no preloading for this one, needs to be dynamic 23 | JurisdictionInclude.legislative_sessions: [ 24 | "legislative_sessions", 25 | "legislative_sessions.downloads", 26 | ], 27 | } 28 | max_per_page = 52 29 | 30 | @classmethod 31 | def postprocess_includes(cls, obj, data, includes): 32 | # latest runs needs to be set on each object individually, the 20-item 33 | # limit makes a subquery approach not work 34 | if JurisdictionInclude.latest_runs in includes: 35 | obj.latest_runs = data.get_latest_runs() 36 | 37 | def __init__(self, page: int = 1, per_page: int = 52): 38 | self.page = page 39 | self.per_page = per_page 40 | 41 | 42 | router = APIRouter() 43 | 44 | 45 | @router.get( 46 | "/jurisdictions", 47 | response_model=JurisdictionPagination.response_model(), 48 | response_model_exclude_none=True, 49 | tags=["jurisdictions"], 50 | ) 51 | async def jurisdiction_list( 52 | classification: Optional[JurisdictionClassification] = Query( 53 | None, description="Filter returned jurisdictions by type." 54 | ), 55 | include: List[JurisdictionInclude] = Query( 56 | [], description="Additional information to include in response." 57 | ), 58 | db: SessionLocal = Depends(get_db), 59 | pagination: JurisdictionPagination = Depends(), 60 | auth: str = Depends(apikey_auth), 61 | ): 62 | """ 63 | Get list of supported Jurisdictions, a Jurisdiction is a state or municipality. 64 | """ 65 | query = db.query(models.Jurisdiction).order_by(models.Jurisdiction.name) 66 | 67 | # handle parameters 68 | if classification: 69 | query = query.filter(models.Jurisdiction.classification == classification) 70 | 71 | return pagination.paginate(query, includes=include) 72 | 73 | 74 | @router.get( 75 | # we have to use the Starlette path type to allow slashes here 76 | "/jurisdictions/{jurisdiction_id:path}", 77 | response_model=Jurisdiction, 78 | response_model_exclude_none=True, 79 | tags=["jurisdictions"], 80 | ) 81 | async def jurisdiction_detail( 82 | jurisdiction_id: str, 83 | include: List[JurisdictionInclude] = Query( 84 | [], description="Additional includes for the Jurisdiction response." 85 | ), 86 | db: SessionLocal = Depends(get_db), 87 | auth: str = Depends(apikey_auth), 88 | ): 89 | """Get details on a single Jurisdiction (e.g. state or municipality).""" 90 | query = db.query(models.Jurisdiction).filter( 91 | jurisdiction_filter(jurisdiction_id, jid_field=models.Jurisdiction.id) 92 | ) 93 | return JurisdictionPagination.detail(query, includes=include) 94 | -------------------------------------------------------------------------------- /api/main.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sentry_sdk 3 | from fastapi import FastAPI 4 | from fastapi.openapi.utils import get_openapi 5 | from fastapi.middleware.cors import CORSMiddleware 6 | from fastapi.responses import RedirectResponse 7 | from prometheus_fastapi_instrumentator import Instrumentator 8 | from uvicorn.workers import UvicornWorker 9 | from . import jurisdictions, people, bills, committees, events 10 | 11 | if "SENTRY_URL" in os.environ: 12 | sentry_sdk.init(os.environ["SENTRY_URL"], traces_sample_rate=0) 13 | 14 | app = FastAPI() 15 | app.include_router(jurisdictions.router) 16 | app.include_router(people.router) 17 | app.include_router(bills.router) 18 | app.include_router(committees.router) 19 | app.include_router(events.router) 20 | app.add_middleware( 21 | CORSMiddleware, 22 | allow_origins=["*"], 23 | allow_headers=["*"], 24 | ) 25 | 26 | instrumentator = Instrumentator( 27 | should_group_status_codes=False, 28 | should_ignore_untemplated=True, 29 | should_instrument_requests_inprogress=True, 30 | excluded_handlers=[ 31 | "/metrics", 32 | ], 33 | inprogress_name="inprogress", 34 | inprogress_labels=True, 35 | ) 36 | instrumentator.instrument(app) 37 | instrumentator.expose(app, include_in_schema=True, should_gzip=True) 38 | 39 | 40 | @app.get("/healthz", include_in_schema=False) 41 | async def health(): 42 | return "OK" 43 | 44 | 45 | @app.get("/", include_in_schema=False) 46 | async def read_typer(): 47 | return RedirectResponse("/docs") 48 | 49 | 50 | # @app.get("/debug") 51 | # def debug(request: Request): 52 | # print(request.scope) 53 | # return { 54 | # "headers": request.headers, 55 | # "url": request.url, 56 | # "scope.type": request.scope["type"], 57 | # "scope.http_version": request.scope["http_version"], 58 | # "scope.server": request.scope["server"], 59 | # "scope.client": request.scope["client"], 60 | # } 61 | 62 | 63 | def custom_openapi(): 64 | if app.openapi_schema: 65 | return app.openapi_schema 66 | openapi_schema = get_openapi( 67 | title="Open States API v3", 68 | version="2021.11.12", 69 | description=""" 70 | * [More documentation](https://docs.openstates.org/en/latest/api/v3/index.html) 71 | * [Register for an account](https://openstates.org/accounts/signup/) 72 | 73 | 74 | **We are currently working to restore experimental support for committees & events.** 75 | 76 | During this period please note that data is not yet available for all states 77 | and the exact format of the new endpoints may change slightly depending on user feedback. 78 | 79 | If you have any issues or questions use our 80 | [GitHub Issues](https://github.com/openstates/issues/issues) to give feedback. 81 | """, 82 | routes=app.routes, 83 | ) 84 | 85 | # if we want to publish divisions.geo we can like this 86 | # openapi_schema["paths"]["/divisions.geo"] = { 87 | # "get": { 88 | # "summary": "Divisions Geo", 89 | # "operationId": "divisions_geo_get", 90 | # "tags": ["divisions"], 91 | # "parameters": [ 92 | # { 93 | # "required": True, 94 | # "schema": {"title": "Latitude", "type": "number"}, 95 | # "name": "lat", 96 | # "in": "query", 97 | # }, 98 | # { 99 | # "required": True, 100 | # "schema": {"title": "Longitude", "type": "number"}, 101 | # "name": "lng", 102 | # "in": "query", 103 | # }, 104 | # ], 105 | # "responses": { 106 | # "200": {"description": "Successful Response"}, 107 | # "content": { 108 | # "application/json": { 109 | # "schema": "#/components/schemas/JurisdictionList" 110 | # } 111 | # }, 112 | # }, 113 | # } 114 | # } 115 | app.openapi_schema = openapi_schema 116 | return app.openapi_schema 117 | 118 | 119 | app.openapi = custom_openapi 120 | 121 | 122 | # based on suggestion in https://github.com/encode/uvicorn/issues/343 to add proxy_headers 123 | # also need to set environment variable FORWARDED_ALLOW_IPS (can be * in ECS) 124 | class CustomUvicornWorker(UvicornWorker): 125 | CONFIG_KWARGS = {"loop": "asyncio", "http": "h11", "proxy_headers": True} 126 | -------------------------------------------------------------------------------- /api/pagination.py: -------------------------------------------------------------------------------- 1 | import math 2 | from typing import List 3 | from pydantic import create_model, BaseModel, Field 4 | from fastapi import HTTPException 5 | from sqlalchemy.orm import noload, selectinload 6 | from sqlalchemy.orm.exc import NoResultFound 7 | 8 | 9 | class PaginationMeta(BaseModel): 10 | per_page: int = Field(..., example=20) 11 | page: int = Field(..., example=1) 12 | max_page: int = Field(..., example=3) 13 | total_items: int = Field(..., example=52) 14 | 15 | 16 | class Pagination: 17 | """ 18 | Base class that handles pagination and includes= behavior together. 19 | 20 | Must set the following properties on subclasses: 21 | - ObjCls - the Pydantic model to use for the objects 22 | - IncludeEnum - the valid include= parameters enumeration 23 | - include_map_overrides - mapping of what fields to select-in if included 24 | (default to same name as IncludeEnum properties) 25 | - postprocess_includes - function to call on each object to set includes 26 | 27 | Once those are set all of the basic methods work as classmethods so they can be called by 28 | PaginationSubclass.detail. 29 | 30 | The only time the class is insantiated is when it is used as a dependency for actual 31 | pagination. 32 | """ 33 | 34 | def __init__(self, page: int = 1, per_page: int = 10): 35 | self.page = page 36 | self.per_page = per_page 37 | 38 | @classmethod 39 | def include_map(cls): 40 | if not hasattr(cls, "_include_map"): 41 | cls._include_map = {key: [key] for key in cls.IncludeEnum} 42 | cls._include_map.update(cls.include_map_overrides) 43 | return cls._include_map 44 | 45 | @classmethod 46 | def response_model(cls): 47 | return create_model( 48 | f"{cls.ObjCls.__name__}List", 49 | results=(List[cls.ObjCls], ...), 50 | pagination=(PaginationMeta, ...), 51 | ) 52 | 53 | def paginate( 54 | self, 55 | results, 56 | *, 57 | includes=None, 58 | skip_count=False, 59 | ): 60 | # shouldn't happen, but help log if it does 61 | if not results._order_by_clauses: 62 | raise HTTPException( 63 | status_code=500, detail="ordering is required for pagination" 64 | ) 65 | 66 | if self.per_page < 1 or self.per_page > self.max_per_page: 67 | raise HTTPException( 68 | status_code=400, 69 | detail=f"invalid per_page, must be in [1, {self.max_per_page}]", 70 | ) 71 | 72 | if not skip_count: 73 | total_items = results.count() 74 | num_pages = math.ceil(total_items / self.per_page) or 1 75 | else: 76 | # used for people.geo, always fits on one page 77 | total_items = 0 78 | num_pages = 1 79 | 80 | if self.page < 1 or self.page > num_pages: 81 | raise HTTPException( 82 | status_code=404, detail=f"invalid page, must be in [1, {num_pages}]" 83 | ) 84 | 85 | # before the query, do the appropriate joins and noload operations 86 | results = self.select_or_noload(results, includes) 87 | results = ( 88 | results.limit(self.per_page).offset((self.page - 1) * self.per_page).all() 89 | ) 90 | results = [self.to_obj_with_includes(data, includes) for data in results] 91 | 92 | # make the data correct without the extra query 93 | if skip_count: 94 | total_items = len(results) 95 | 96 | meta = PaginationMeta( 97 | total_items=total_items, 98 | per_page=self.per_page, 99 | page=self.page, 100 | max_page=num_pages, 101 | ) 102 | 103 | return {"pagination": meta, "results": results} 104 | 105 | @classmethod 106 | def detail(cls, query, *, includes): 107 | """convert a single instance query to a model with the appropriate includes""" 108 | query = cls.select_or_noload(query, includes) 109 | try: 110 | obj = query.one() 111 | except NoResultFound: 112 | raise HTTPException( 113 | status_code=404, detail=f"No such {cls.ObjCls.__name__}." 114 | ) 115 | return cls.to_obj_with_includes(obj, includes) 116 | 117 | @classmethod 118 | def postprocess_includes(cls, obj, data, includes): 119 | pass 120 | 121 | @classmethod 122 | def to_obj_with_includes(cls, data, includes): 123 | """ 124 | remove the non-included data from the response by setting the fields to 125 | None instead of [], and returning the Pydantic objects directly 126 | """ 127 | newobj = cls.ObjCls.from_orm(data) 128 | for include in cls.IncludeEnum: 129 | if include not in includes: 130 | setattr(newobj, include, None) 131 | cls.postprocess_includes(newobj, data, includes) 132 | return newobj 133 | 134 | @classmethod 135 | def select_or_noload(cls, query, includes): 136 | """either pre-join or no-load data based on whether it has been requested""" 137 | for fieldname in cls.IncludeEnum: 138 | if fieldname in includes: 139 | # selectinload seems like a strong default choice, but it is possible that 140 | # some configurability might be desirable eventually for some types of data 141 | loader = selectinload 142 | else: 143 | loader = noload 144 | 145 | # update the query with appropriate loader 146 | for dbname in cls.include_map()[fieldname]: 147 | query = query.options(loader(dbname)) 148 | return query 149 | -------------------------------------------------------------------------------- /api/people.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, List 2 | from enum import Enum 3 | import requests 4 | from fastapi import APIRouter, Depends, Query, HTTPException 5 | from sqlalchemy import func, or_ 6 | from sqlalchemy.orm import contains_eager 7 | from .db import SessionLocal, get_db, models 8 | from .schemas import Person, OrgClassification 9 | from .pagination import Pagination, PaginationMeta 10 | from .auth import apikey_auth 11 | from .utils import jurisdiction_filter, add_state_divisions 12 | 13 | 14 | class PersonInclude(str, Enum): 15 | other_names = "other_names" 16 | other_identifiers = "other_identifiers" 17 | links = "links" 18 | sources = "sources" 19 | offices = "offices" 20 | 21 | 22 | class PeoplePagination(Pagination): 23 | ObjCls = Person 24 | IncludeEnum = PersonInclude 25 | include_map_overrides = {} 26 | max_per_page = 50 27 | 28 | 29 | # have to save this here since generating it twice caused a FastAPI error 30 | PersonList = PeoplePagination.response_model() 31 | 32 | 33 | def people_query(db): 34 | return ( 35 | db.query(models.Person) 36 | .join(models.Person.jurisdiction) 37 | .order_by(models.Person.name) 38 | .options(contains_eager(models.Person.jurisdiction)) 39 | ) 40 | 41 | 42 | router = APIRouter() 43 | session = requests.Session() 44 | 45 | 46 | @router.get( 47 | "/people", 48 | response_model=PersonList, 49 | response_model_exclude_none=True, 50 | tags=["people"], 51 | ) 52 | async def people_search( 53 | jurisdiction: Optional[str] = Query( 54 | None, description="Filter by jurisdiction name or id." 55 | ), 56 | name: Optional[str] = Query( 57 | None, description="Filter by name, case-insensitive match." 58 | ), 59 | id: Optional[List[str]] = Query( 60 | [], 61 | description="Filter by id, can be specified multiple times for multiple people.", 62 | ), 63 | org_classification: Optional[OrgClassification] = Query( 64 | None, description="Filter by current role." 65 | ), 66 | district: Optional[str] = Query( 67 | None, 68 | description="Filter by district name.", 69 | ), 70 | include: List[PersonInclude] = Query( 71 | [], description="Additional information to include in response." 72 | ), 73 | db: SessionLocal = Depends(get_db), 74 | pagination: PeoplePagination = Depends(), 75 | auth: str = Depends(apikey_auth), 76 | ): 77 | """ 78 | Get list of people matching selected criteria. 79 | 80 | Must provide either **jurisdiction**, **name**, or one or more **id** parameters. 81 | """ 82 | 83 | query = people_query(db) 84 | filtered = False 85 | 86 | if jurisdiction: 87 | query = query.filter( 88 | jurisdiction_filter(jurisdiction, jid_field=models.Person.jurisdiction_id), 89 | models.Person.current_role.isnot(None), # current members only for now 90 | ) 91 | filtered = True 92 | if name: 93 | lname = f"%{name.lower()}%" 94 | query = query.filter( 95 | or_( 96 | func.lower(models.Person.name).like(lname), 97 | func.lower(models.PersonName.name).like(lname), 98 | ) 99 | ).outerjoin(models.PersonName) 100 | filtered = True 101 | if id: 102 | query = query.filter(models.Person.id.in_(id)) 103 | filtered = True 104 | if org_classification: 105 | query = query.filter( 106 | models.Person.current_role["org_classification"].astext 107 | == org_classification 108 | ) 109 | if district: 110 | if not jurisdiction: 111 | raise HTTPException(400, "cannot specify 'district' without 'jurisdiction'") 112 | query = query.filter(models.Person.current_role["district"].astext == district) 113 | 114 | if not filtered: 115 | raise HTTPException(400, "either 'jurisdiction', 'name', or 'id' is required") 116 | 117 | return pagination.paginate(query, includes=include) 118 | 119 | 120 | @router.get( 121 | "/people.geo", 122 | response_model=PersonList, 123 | response_model_exclude_none=True, 124 | tags=["people"], 125 | ) 126 | async def people_geo( 127 | lat: float = Query(..., description="Latitude of point."), 128 | lng: float = Query(..., description="Longitude of point."), 129 | include: List[PersonInclude] = Query( 130 | [], description="Additional information to include in the response." 131 | ), 132 | db: SessionLocal = Depends(get_db), 133 | auth: str = Depends(apikey_auth), 134 | ): 135 | """ 136 | Get list of people currently representing a given location. 137 | 138 | **Note:** Currently limited to state legislators and US Congress. Governors & mayors are not included. 139 | """ 140 | url = f"https://v3.openstates.org/divisions.geo?lat={lat}&lng={lng}" 141 | try: 142 | data = session.get(url).json() 143 | except Exception as e: 144 | raise HTTPException(500, f"Failed to retrieve data from Geo endpoint :: {e}") 145 | try: 146 | divisions = [d["id"] for d in data["divisions"]] 147 | if len(data["divisions"]) > 0: 148 | divisions.append(add_state_divisions(data["divisions"][0]["state"])) 149 | except KeyError: 150 | raise HTTPException( 151 | 500, "unexpected upstream response, try again in 60 seconds" 152 | ) 153 | 154 | # skip the rest of the logic if there are no divisions 155 | if not divisions: 156 | return { 157 | "pagination": PaginationMeta( 158 | total_items=0, per_page=100, page=1, max_page=1 159 | ), 160 | "results": [], 161 | } 162 | 163 | query = people_query(db).filter( 164 | models.Person.current_role["division_id"].astext.in_(divisions) 165 | ) 166 | # paginate without looking for page= params 167 | pagination = PeoplePagination() 168 | return pagination.paginate(query, includes=include, skip_count=True) 169 | -------------------------------------------------------------------------------- /api/schemas.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from typing import Optional, List, Union 3 | from enum import Enum 4 | from uuid import UUID 5 | from pydantic import BaseModel, Field 6 | 7 | 8 | class JurisdictionClassification(str, Enum): 9 | state = "state" 10 | municipality = "municipality" 11 | country = "country" 12 | 13 | 14 | class OrgClassification(str, Enum): 15 | legislature = "legislature" 16 | executive = "executive" 17 | lower = "lower" 18 | upper = "upper" 19 | government = "government" 20 | 21 | 22 | # class LegislativeSessionClassification(str, Enum): 23 | # primary = "primary" 24 | # special = "special" 25 | # unset = "" 26 | 27 | 28 | class Post(BaseModel): 29 | label: str = Field(..., example="2") 30 | role: str = Field(..., example="Senator") 31 | division_id: str = Field(..., example="ocd-division/country:us/state:mn/sldu:4") 32 | maximum_memberships: int = Field(..., example=1) 33 | 34 | class Config: 35 | orm_mode = True 36 | 37 | 38 | class Organization(BaseModel): 39 | id: str = Field( 40 | ..., example="ocd-organization/32aab083-d7a0-44e0-9b95-a7790c542605" 41 | ) 42 | name: str = Field(..., example="North Carolina General Assembly") 43 | classification: str = Field(..., example="legislature") 44 | 45 | class Config: 46 | orm_mode = True 47 | 48 | 49 | class Chamber(Organization): 50 | districts: Optional[List[Post]] = None 51 | 52 | 53 | class DataExport(BaseModel): 54 | created_at: datetime.datetime 55 | updated_at: datetime.datetime 56 | data_type: str 57 | url: str 58 | 59 | class Config: 60 | orm_mode = True 61 | 62 | 63 | class LegislativeSession(BaseModel): 64 | identifier: str 65 | name: str 66 | classification: str 67 | start_date: str 68 | end_date: str 69 | downloads: Optional[List[DataExport]] 70 | 71 | class Config: 72 | orm_mode = True 73 | 74 | 75 | class RunPlan(BaseModel): 76 | success: bool 77 | start_time: datetime.datetime 78 | end_time: datetime.datetime 79 | 80 | class Config: 81 | orm_mode = True 82 | 83 | 84 | class Jurisdiction(BaseModel): 85 | id: str = Field(..., example="ocd-jurisdiction/country:us/state:nc/government") 86 | name: str = Field(..., example="North Carolina") 87 | classification: JurisdictionClassification = Field(..., example="state") 88 | division_id: Optional[str] = Field( 89 | "", example="ocd-division/country:us/state:nc" 90 | ) # never exclude 91 | url: str = Field(..., example="https://nc.gov") 92 | latest_bill_update: datetime.datetime 93 | latest_people_update: datetime.datetime 94 | organizations: Optional[List[Chamber]] = None 95 | legislative_sessions: Optional[List[LegislativeSession]] = None 96 | latest_runs: Optional[List[RunPlan]] = None 97 | 98 | class Config: 99 | orm_mode = True 100 | 101 | 102 | class CurrentRole(BaseModel): 103 | title: str = Field(..., example="Senator") 104 | org_classification: OrgClassification = Field(..., example="upper") 105 | district: Union[str, int, None] = Field("", example=3) 106 | division_id: Optional[str] = Field( 107 | "", example="ocd-division/country:us/state:nc/sldu:3" 108 | ) 109 | 110 | class Config: 111 | orm_mode = True 112 | 113 | 114 | class AltIdentifier(BaseModel): 115 | identifier: str = Field(..., example="NCL000123") 116 | scheme: str = Field(..., example="legacy_openstates") 117 | 118 | class Config: 119 | orm_mode = True 120 | 121 | 122 | class AltName(BaseModel): 123 | name: str = Field(..., example="Auggie") 124 | note: str = Field(..., example="nickname") 125 | 126 | class Config: 127 | orm_mode = True 128 | 129 | 130 | class Link(BaseModel): 131 | url: str = Field(..., example="https://example.com/") 132 | note: Union[str, None] = Field(None, example="homepage") 133 | 134 | class Config: 135 | orm_mode = True 136 | 137 | 138 | class Office(BaseModel): 139 | name: str = Field(..., example="District Office") 140 | fax: Optional[str] = Field(None, example="919-555-1234") 141 | voice: Optional[str] = Field(None, example="919-555-0064") 142 | address: Optional[str] = Field(None, example="212 Maple Lane; Raleigh NC; 27526") 143 | classification: Optional[str] = Field(None, example="capitol") 144 | 145 | class Config: 146 | orm_mode = True 147 | 148 | 149 | class CompactJurisdiction(BaseModel): 150 | id: str = Field(..., example="ocd-jurisdiction/country:us/state:nc/government") 151 | name: str = Field(..., example="North Carolina") 152 | classification: JurisdictionClassification = Field(..., example="state") 153 | 154 | class Config: 155 | orm_mode = True 156 | 157 | 158 | class CompactPerson(BaseModel): 159 | id: str = Field(..., example="ocd-person/adb58f21-f2fd-4830-85b6-f490b0867d14") 160 | name: str = Field(..., example="Angela Augusta") 161 | party: str = Field(..., example="Democratic") 162 | current_role: Optional[CurrentRole] 163 | 164 | class Config: 165 | orm_mode = True 166 | 167 | 168 | class CompactBill(BaseModel): 169 | id: str = Field(..., example="ocd-bill/12345678-0000-1111-2222-333344445555") 170 | session: str 171 | identifier: str 172 | title: str 173 | 174 | class Config: 175 | orm_mode = True 176 | 177 | 178 | class CompactVoteEvent(BaseModel): 179 | id: str = Field(..., example="ocd-vote/12345678-0000-1111-2222-333344445555") 180 | motion_text: str = Field(..., example="Shall the bill be passed?") 181 | 182 | class Config: 183 | orm_mode = True 184 | 185 | 186 | class Person(CompactPerson): 187 | jurisdiction: CompactJurisdiction 188 | 189 | # extra detail 190 | given_name: str = Field(..., example="Angela") 191 | family_name: str = Field(..., example="Augusta") 192 | image: str = Field(..., example="https://example.com/ncimg/3.png") 193 | email: str = Field(..., example="aperson@example.com") 194 | gender: str = Field(..., example="female") 195 | birth_date: str = Field(..., example="1960-05-04") 196 | death_date: str = Field(..., example="2019-04-10") 197 | extras: dict = Field(..., example={"profession": "Doctor"}) 198 | created_at: datetime.datetime 199 | updated_at: datetime.datetime 200 | openstates_url: str = Field( 201 | ..., 202 | example="https://openstates.org/person/amos-l-quick-iii-28NRPPfJA6FGVl9RrjpKjl/", 203 | ) 204 | 205 | # joins 206 | other_identifiers: Optional[List[AltIdentifier]] 207 | other_names: Optional[List[AltName]] 208 | links: Optional[List[Link]] 209 | sources: Optional[List[Link]] 210 | offices: Optional[List[Office]] 211 | 212 | class Config: 213 | orm_mode = True 214 | 215 | 216 | class RelatedBill(BaseModel): 217 | identifier: str = Field(..., example="HB 123") 218 | legislative_session: str = Field(..., example="2022S1") 219 | relation_type: str = Field(..., example="companion") 220 | 221 | class Config: 222 | orm_mode = True 223 | 224 | 225 | class BillAbstract(BaseModel): 226 | abstract: str = Field(..., example="This bill designates a new state arachnid.") 227 | note: str = Field(..., example="house abstract") 228 | 229 | class Config: 230 | orm_mode = True 231 | 232 | 233 | class BillTitle(BaseModel): 234 | title: str = Field(..., example="Designating the scorpion as the state arachnid.") 235 | note: str = Field(..., example="short title") 236 | 237 | class Config: 238 | orm_mode = True 239 | 240 | 241 | class BillIdentifier(BaseModel): 242 | identifier: str = Field(..., example="HB 74") 243 | 244 | class Config: 245 | orm_mode = True 246 | 247 | 248 | class BillSponsorship(BaseModel): 249 | id: UUID = Field(..., example="f0049138-1ad8-4506-a2a4-f4dd1251bbba") 250 | name: str = Field(..., example="JONES") 251 | entity_type: str = Field(..., example="person") 252 | organization: Optional[Organization] = Field(None, example=None) 253 | person: Optional[CompactPerson] 254 | primary: bool 255 | classification: str = Field(..., example="primary") 256 | 257 | class Config: 258 | orm_mode = True 259 | 260 | 261 | class BillActionRelatedEntity(BaseModel): 262 | name: str = Field(..., example="Senate Committee of the Whole") 263 | entity_type: str = Field(..., example="organization") 264 | organization: Optional[Organization] = Field(None, example=None) 265 | person: Optional[CompactPerson] 266 | 267 | class Config: 268 | orm_mode = True 269 | 270 | 271 | class BillAction(BaseModel): 272 | id: UUID = Field(..., example="f0049138-1ad8-4506-a2a4-f4dd1251bbba") 273 | organization: Organization 274 | description: str = Field(..., example="Passed 1st Reading") 275 | date: str = Field(..., example="2020-03-14") 276 | # TODO: enumerate billaction classifiers 277 | classification: List[str] = Field(..., example=["passed"]) 278 | order: int 279 | related_entities: Optional[List[BillActionRelatedEntity]] 280 | 281 | class Config: 282 | orm_mode = True 283 | 284 | 285 | class BillDocumentLink(BaseModel): 286 | url: str = Field(..., example="https://example.com/doc.pdf") 287 | media_type: str = Field(..., example="application/pdf") 288 | 289 | class Config: 290 | orm_mode = True 291 | 292 | 293 | class BillDocumentOrVersion(BaseModel): 294 | id: UUID = Field(..., example="f0049138-1ad8-4506-a2a4-f4dd1251bbba") 295 | note: str = Field(..., example="Latest Version") 296 | date: str = Field(..., example="2020-10-01") 297 | classification: str = Field(..., example="amendment") 298 | links: List[BillDocumentLink] 299 | 300 | class Config: 301 | orm_mode = True 302 | 303 | 304 | class VoteCount(BaseModel): 305 | option: str = Field(..., example="yes") 306 | value: int = Field(..., example=48) 307 | 308 | class Config: 309 | orm_mode = True 310 | 311 | 312 | class PersonVote(BaseModel): 313 | id: UUID = Field(..., example="f0049138-1ad8-4506-a2a4-f4dd1251bbba") 314 | option: str = Field(..., example="no") 315 | voter_name: str = Field(..., example="Wu") 316 | voter: Optional[CompactPerson] 317 | 318 | class Config: 319 | orm_mode = True 320 | 321 | 322 | class VoteEvent(BaseModel): 323 | id: str 324 | motion_text: str = Field(..., example="Shall the bill be passed?") 325 | motion_classification: List[str] = Field([], example=["passage"]) 326 | start_date: str = Field(..., example="2020-09-18") 327 | result: str = Field(..., example="pass") 328 | identifier: str = Field(..., example="HV #3312") 329 | extras: dict 330 | 331 | organization: Organization 332 | votes: List[PersonVote] 333 | counts: List[VoteCount] 334 | sources: List[Link] 335 | 336 | class Config: 337 | orm_mode = True 338 | 339 | 340 | class Bill(BaseModel): 341 | id: str = Field(..., example="ocd-bill/f0049138-1ad8-4506-a2a4-f4dd1251bbba") 342 | session: str = Field(..., example="2020") 343 | jurisdiction: CompactJurisdiction 344 | from_organization: Organization 345 | identifier: str = Field(..., example="SB 113") 346 | title: str = Field(..., example="Adopting a State Scorpion") 347 | classification: List[str] = Field([], example=["resolution"]) 348 | subject: List[str] = Field([], example=["SCORPIONS", "SYMBOLS"]) 349 | extras: dict = Field({}, example={}) 350 | created_at: datetime.datetime 351 | updated_at: datetime.datetime 352 | 353 | # computed fields 354 | openstates_url: str = Field( 355 | ..., example="https://openstates.org/nc/bills/2019/HB1105/" 356 | ) 357 | first_action_date: Optional[str] = Field("", example="2020-01-01") 358 | latest_action_date: Optional[str] = Field("", example="2020-02-01") 359 | latest_action_description: Optional[str] = Field("", example="Introduced in House") 360 | latest_passage_date: Optional[str] = Field("", example="2020-03-01") 361 | 362 | related_bills: Optional[List[RelatedBill]] 363 | abstracts: Optional[List[BillAbstract]] 364 | other_titles: Optional[List[BillTitle]] 365 | other_identifiers: Optional[List[BillIdentifier]] 366 | sponsorships: Optional[List[BillSponsorship]] 367 | actions: Optional[List[BillAction]] 368 | sources: Optional[List[Link]] 369 | versions: Optional[List[BillDocumentOrVersion]] 370 | documents: Optional[List[BillDocumentOrVersion]] 371 | votes: Optional[List[VoteEvent]] 372 | 373 | class Config: 374 | orm_mode = True 375 | 376 | 377 | class CommitteeClassification(str, Enum): 378 | committee = "committee" 379 | subcommittee = "subcommittee" 380 | 381 | 382 | class CommitteeMembership(BaseModel): 383 | person_name: str 384 | role: str 385 | person: Optional[CompactPerson] 386 | 387 | class Config: 388 | orm_mode = True 389 | 390 | 391 | class Committee(BaseModel): 392 | id: str = Field( 393 | ..., example="ocd-organization/aabbbbcc-dddd-eeee-ffff-0123456789ab" 394 | ) 395 | name: str = Field(..., example="Health & Public Services") 396 | classification: CommitteeClassification 397 | parent_id: str = Field( 398 | ..., example="ocd-organization/aabbbbcc-dddd-eeee-ffff-999988887777" 399 | ) 400 | extras: dict = Field(..., example={"room": "Room 4B"}) 401 | 402 | # joins 403 | memberships: Optional[List[CommitteeMembership]] 404 | other_names: Optional[List[AltName]] 405 | links: Optional[List[Link]] 406 | sources: Optional[List[Link]] 407 | 408 | class Config: 409 | orm_mode = True 410 | 411 | 412 | class EventLocation(BaseModel): 413 | name: str 414 | url: str 415 | 416 | class Config: 417 | orm_mode = True 418 | 419 | 420 | class EventMedia(BaseModel): 421 | note: str 422 | date: str 423 | offset: Union[int, None] 424 | classification: str 425 | links: list[Link] 426 | 427 | class Config: 428 | orm_mode = True 429 | 430 | 431 | class EventDocument(BaseModel): 432 | note: str 433 | date: str 434 | classification: str 435 | links: list[Link] 436 | 437 | class Config: 438 | orm_mode = True 439 | 440 | 441 | class EventParticipant(BaseModel): 442 | note: str 443 | name: str = Field(..., example="JONES") 444 | entity_type: str = Field(..., example="person") 445 | organization: Optional[Organization] = Field(None, example=None) 446 | person: Optional[CompactPerson] 447 | 448 | class Config: 449 | orm_mode = True 450 | 451 | 452 | class EventRelatedEntity(BaseModel): 453 | note: str 454 | name: str = Field(..., example="JONES") 455 | entity_type: str = Field(..., example="person") 456 | organization: Optional[Organization] = Field(None, example=None) 457 | person: Optional[CompactPerson] 458 | bill: Optional[CompactBill] 459 | vote: Optional[CompactVoteEvent] 460 | 461 | class Config: 462 | orm_mode = True 463 | 464 | 465 | class EventAgendaItem(BaseModel): 466 | description: str 467 | classification: List[str] 468 | order: int 469 | subjects: List[str] 470 | notes: List[str] 471 | extras: dict 472 | 473 | related_entities: List[EventRelatedEntity] 474 | media: List[EventMedia] 475 | 476 | class Config: 477 | orm_mode = True 478 | 479 | 480 | class Event(BaseModel): 481 | id: str 482 | name: str 483 | jurisdiction: CompactJurisdiction 484 | description: str 485 | classification: str 486 | start_date: str 487 | end_date: str 488 | all_day: bool 489 | status: str 490 | upstream_id: str 491 | deleted: bool 492 | location: Optional[EventLocation] 493 | 494 | # related fields 495 | links: Optional[List[Link]] 496 | sources: Optional[List[Link]] 497 | media: Optional[List[EventMedia]] 498 | documents: Optional[List[EventDocument]] 499 | participants: Optional[List[EventParticipant]] 500 | agenda: Optional[List[EventAgendaItem]] 501 | 502 | class Config: 503 | orm_mode = True 504 | -------------------------------------------------------------------------------- /api/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstates/api-v3/af82501a464ce91988bbe3d95b154dc3f701c925/api/tests/__init__.py -------------------------------------------------------------------------------- /api/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pytest 3 | from sqlalchemy import create_engine, event 4 | from sqlalchemy.orm import sessionmaker 5 | from sqlalchemy_utils.functions import database_exists, drop_database, create_database 6 | from sqlalchemy.exc import OperationalError 7 | from fastapi.testclient import TestClient 8 | from api.main import app 9 | from api.auth import apikey_auth 10 | from api.db import Base, get_db 11 | from . import fixtures 12 | 13 | TEST_DATABASE_URL = os.environ.get( 14 | "TEST_DATABASE_URL", "postgresql://v3test:v3test@localhost/v3test" 15 | ) 16 | engine = create_engine(TEST_DATABASE_URL) 17 | TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) 18 | 19 | 20 | class QueryLogger: 21 | def __init__(self): 22 | self.count = 0 23 | 24 | def reset(self): 25 | self.count = 0 26 | 27 | def callback(self, conn, cursor, statement, parameters, context, executemany): 28 | self.count += 1 29 | print(f"==== QUERY #{self.count} ====\n", statement, parameters) 30 | 31 | 32 | query_logger = QueryLogger() 33 | 34 | 35 | def get_test_db(): 36 | try: 37 | db = TestingSessionLocal() 38 | query_logger.reset() 39 | event.listen(engine, "after_cursor_execute", query_logger.callback) 40 | yield db 41 | finally: 42 | db.close() 43 | 44 | 45 | app.dependency_overrides[get_db] = get_test_db 46 | app.dependency_overrides[apikey_auth] = lambda: "disable api key" 47 | 48 | 49 | @pytest.fixture(scope="session", autouse=True) 50 | def create_test_database(): 51 | url = TEST_DATABASE_URL 52 | try: 53 | if database_exists(url): 54 | drop_database(url) 55 | except OperationalError: 56 | pass 57 | create_database(url) 58 | Base.metadata.create_all(bind=engine) 59 | yield # run the tests 60 | drop_database(url) 61 | 62 | 63 | @pytest.fixture(scope="session", autouse=True) 64 | def common_data(create_test_database): 65 | db = TestingSessionLocal() 66 | for obj in fixtures.nebraska(): 67 | db.add(obj) 68 | for obj in fixtures.ohio(): 69 | db.add(obj) 70 | for obj in fixtures.mentor(): 71 | db.add(obj) 72 | db.commit() 73 | 74 | 75 | @pytest.fixture 76 | def client(): 77 | client = TestClient(app) 78 | return client 79 | -------------------------------------------------------------------------------- /api/tests/fixtures.py: -------------------------------------------------------------------------------- 1 | import random 2 | import uuid 3 | import datetime 4 | from sqlalchemy import func 5 | from api.db.models import ( 6 | Bill, 7 | BillAction, 8 | BillDocument, 9 | BillDocumentLink, 10 | BillSource, 11 | BillSponsorship, 12 | BillVersion, 13 | BillVersionLink, 14 | RelatedBill, 15 | Event, 16 | EventAgendaItem, 17 | EventAgendaMedia, 18 | EventDocument, 19 | EventLocation, 20 | EventMedia, 21 | EventParticipant, 22 | EventRelatedEntity, 23 | Jurisdiction, 24 | LegislativeSession, 25 | DataExport, 26 | Membership, 27 | Organization, 28 | Person, 29 | PersonOffice, 30 | PersonLink, 31 | PersonName, 32 | PersonSource, 33 | PersonVote, 34 | Post, 35 | RunPlan, 36 | SearchableBill, 37 | VoteCount, 38 | VoteEvent, 39 | ) 40 | 41 | 42 | def dummy_person_id(n): 43 | return f"ocd-person/{n*8}-{n*4}-{n*4}-{n*4}-{n*12}" 44 | 45 | 46 | def create_test_bill( 47 | session, 48 | chamber, 49 | *, 50 | sponsors=0, 51 | actions=0, 52 | votes=0, 53 | versions=0, 54 | documents=0, 55 | sources=0, 56 | subjects=None, 57 | identifier=None, 58 | classification=None, 59 | ): 60 | b = Bill( 61 | id="ocd-bill/" + str(uuid.uuid4()), 62 | identifier=identifier or ("Bill " + str(random.randint(1000, 9000))), 63 | title="Random Bill", 64 | legislative_session=session, 65 | from_organization=chamber, 66 | subject=subjects or [], 67 | classification=classification or ["bill"], 68 | extras={}, 69 | created_at=datetime.datetime.utcnow(), 70 | updated_at=datetime.datetime.utcnow(), 71 | latest_action_date=f"{session.identifier}-02-{random.randint(10,30)}", 72 | first_action_date=f"{session.identifier}-01-{random.randint(10,30)}", 73 | ) 74 | yield b 75 | for n in range(sponsors): 76 | yield BillSponsorship( 77 | bill=b, 78 | primary=True, 79 | classification="sponsor", 80 | name="Someone", 81 | entity_type="person", 82 | ) 83 | for n in range(actions): 84 | yield BillAction( 85 | bill=b, 86 | description="an action took place", 87 | date=session.identifier, 88 | organization=chamber, 89 | order=n, 90 | ) 91 | for n in range(sources): 92 | yield BillSource(bill=b, url="https://example.com/source", note="") 93 | for n in range(versions): 94 | bv = BillVersion(bill=b, note=f"Version {n}", date="2020", classification="") 95 | yield bv 96 | yield BillVersionLink( 97 | version=bv, url=f"https://example.com/{n}", media_type="text/html" 98 | ) 99 | for n in range(documents): 100 | bd = BillDocument(bill=b, note=f"Version {n}", date="2020", classification="") 101 | yield bd 102 | yield BillDocumentLink( 103 | document=bd, url=f"https://example.com/{n}", media_type="text/html" 104 | ) 105 | 106 | 107 | def create_test_event( 108 | jid, n, *, start_date, deleted=False, related_bill=False, related_committee=False 109 | ): 110 | loc = EventLocation( 111 | id=str(uuid.uuid4()), name=f"Location #{n}", jurisdiction_id=jid, url="" 112 | ) 113 | e = Event( 114 | jurisdiction_id=jid, 115 | id=f"ocd-event/00000000-0000-0000-0000-{n:012d}", 116 | name=f"Event #{n}", 117 | description="", 118 | classification="", 119 | start_date=start_date, 120 | end_date="", 121 | all_day=False, 122 | status="normal", 123 | upstream_id="", 124 | deleted=deleted, 125 | location=loc, 126 | links=[{"note": "source", "url": f"https://example.com/{n}"}], 127 | sources=[{"note": "source", "url": f"https://example.com/{n}"}], 128 | ) 129 | yield loc 130 | yield e 131 | yield EventMedia( 132 | event=e, 133 | note="", 134 | date=start_date, 135 | offset=0, 136 | classification="", 137 | links=[], 138 | ) 139 | yield EventDocument( 140 | event=e, 141 | date=start_date, 142 | note="document 1", 143 | classification="", 144 | links=[], 145 | ) 146 | yield EventDocument( 147 | event=e, 148 | date=start_date, 149 | note="document 2", 150 | classification="", 151 | links=[], 152 | ) 153 | yield EventParticipant( 154 | event=e, 155 | note="", 156 | name="John", 157 | entity_type="person", 158 | ) 159 | yield EventParticipant( 160 | event=e, 161 | note="", 162 | name="Jane", 163 | entity_type="person", 164 | ) 165 | yield EventParticipant( 166 | event=e, 167 | note="", 168 | name="Javier", 169 | entity_type="person", 170 | ) 171 | a1 = EventAgendaItem( 172 | id=str(uuid.uuid4()), 173 | event=e, 174 | description="Agenda Item 1", 175 | classification="", 176 | subjects=[], 177 | notes=[], 178 | extras={}, 179 | order=1, 180 | ) 181 | yield a1 182 | yield EventAgendaMedia( 183 | agenda_item=a1, 184 | note="", 185 | date=start_date, 186 | offset=0, 187 | classification="", 188 | links=[], 189 | ) 190 | yield EventAgendaItem( 191 | id=str(uuid.uuid4()), 192 | event=e, 193 | description="Agenda Item 2", 194 | classification="", 195 | subjects=[], 196 | notes=[], 197 | extras={}, 198 | order=2, 199 | ) 200 | if related_bill: 201 | yield EventRelatedEntity( 202 | agenda_item=a1, 203 | note="", 204 | name="SB 1", 205 | entity_type="bill", 206 | ) 207 | if related_committee: 208 | yield EventRelatedEntity( 209 | agenda_item=a1, note="", name="Finance", entity_type="organization" 210 | ) 211 | 212 | 213 | def nebraska(): 214 | j = Jurisdiction( 215 | id="ocd-jurisdiction/country:us/state:ne/government", 216 | name="Nebraska", 217 | url="https://nebraska.gov", 218 | classification="state", 219 | division_id="ocd-division/country:us/state:ne", 220 | latest_bill_update=datetime.datetime(2021, 8, 1), 221 | latest_people_update=datetime.datetime(2021, 8, 2), 222 | ) 223 | runs = [] 224 | runs_from = datetime.datetime(2020, 1, 1) 225 | for n in range(100): 226 | runs.append( 227 | RunPlan( 228 | jurisdiction=j, 229 | start_time=runs_from + datetime.timedelta(days=n), 230 | end_time=runs_from + datetime.timedelta(days=n, hours=3), 231 | success=n % 2 == 0, 232 | ) 233 | ) 234 | ls2020 = LegislativeSession( 235 | jurisdiction=j, 236 | identifier="2020", 237 | name="2020", 238 | start_date="2020-01-01", 239 | end_date="2020-12-31", 240 | ) 241 | data_export = DataExport( 242 | session=ls2020, 243 | data_type="csv", 244 | created_at="2021-01-01", 245 | updated_at="2021-01-01", 246 | url="https://example.com", 247 | ) 248 | ls2021 = LegislativeSession( 249 | jurisdiction=j, identifier="2021", name="2020", start_date="2021-01-01" 250 | ) 251 | leg = Organization( 252 | id="nel", 253 | name="Nebraska Legislature", 254 | classification="legislature", 255 | jurisdiction=j, 256 | ) 257 | bills = [] 258 | for n in range(5): 259 | bills.extend( 260 | create_test_bill( 261 | ls2020, 262 | leg, 263 | sponsors=2, 264 | actions=5, 265 | versions=2, 266 | documents=3, 267 | sources=1, 268 | subjects=["sample"], 269 | ) 270 | ) 271 | for n in range(2): 272 | bills.extend( 273 | create_test_bill( 274 | ls2021, 275 | leg, 276 | subjects=["futurism"], 277 | classification=["resolution"], 278 | identifier=f"SB {n+1}", 279 | ) 280 | ) 281 | events = [] 282 | for n in range(3): 283 | events.extend( 284 | create_test_event( 285 | j.id, 286 | n, 287 | start_date=f"2021-01-0{n+1}", 288 | related_bill=(n == 0), 289 | related_committee=(n > 1), 290 | ) 291 | ) 292 | events.extend(create_test_event(j.id, 4, start_date="2021-01-04", deleted=True)) 293 | 294 | return [ 295 | j, 296 | ls2020, 297 | data_export, 298 | ls2021, 299 | leg, 300 | *runs, 301 | *bills, 302 | *events, 303 | Organization( 304 | id="nee", 305 | name="Nebraska Executive", 306 | classification="executive", 307 | jurisdiction=j, 308 | ), 309 | Post( 310 | id="a", 311 | organization=leg, 312 | label="1", 313 | role="Senator", 314 | maximum_memberships=1, 315 | division_id="ocd-division/country:us/state:ne/sldu:1", 316 | ), 317 | Person( 318 | id=dummy_person_id("1"), 319 | name="Amy Adams", 320 | family_name="Amy", 321 | given_name="Adams", 322 | gender="female", 323 | email="aa@example.com", 324 | birth_date="2000-01-01", 325 | party="Democratic", 326 | current_role={ 327 | "org_classification": "legislature", 328 | "district": 1, 329 | "title": "Senator", 330 | "division_id": "ocd-division/country:us/state:ne/sldu:1", 331 | }, 332 | jurisdiction_id=j.id, 333 | created_at=datetime.datetime.utcnow(), 334 | updated_at=datetime.datetime.utcnow(), 335 | ), 336 | PersonName( 337 | person_id=dummy_person_id("1"), name="Amy 'Aardvark' Adams", note="nickname" 338 | ), 339 | PersonLink( 340 | person_id=dummy_person_id("1"), url="https://example.com/amy", note="" 341 | ), 342 | PersonSource( 343 | person_id=dummy_person_id("1"), url="https://example.com/amy", note="" 344 | ), 345 | PersonOffice( 346 | person_id=dummy_person_id("1"), 347 | classification="capitol", 348 | voice="555-555-5555", 349 | fax="", 350 | address="123 Main St", 351 | name_="", 352 | ), 353 | Person( 354 | id=dummy_person_id("2"), 355 | name="Boo Berri", 356 | birth_date="1973-12-25", 357 | party="Libertarian", 358 | current_role={"org_classification": "executive", "title": "Governor"}, 359 | jurisdiction_id=j.id, 360 | created_at=datetime.datetime.utcnow(), 361 | updated_at=datetime.datetime.utcnow(), 362 | ), 363 | Person( 364 | id=dummy_person_id("3"), 365 | name="Rita Red", # retired 366 | birth_date="1973-12-25", 367 | party="Republican", 368 | jurisdiction_id=j.id, 369 | created_at=datetime.datetime.utcnow(), 370 | updated_at=datetime.datetime.utcnow(), 371 | ), 372 | ] 373 | 374 | 375 | def ohio(): 376 | j = Jurisdiction( 377 | id="ocd-jurisdiction/country:us/state:oh/government", 378 | name="Ohio", 379 | url="https://ohio.gov", 380 | classification="state", 381 | division_id="ocd-division/country:us/state:oh", 382 | latest_bill_update=datetime.datetime(2021, 8, 4), 383 | latest_people_update=datetime.datetime(2021, 8, 5), 384 | ) 385 | ls2021 = LegislativeSession(jurisdiction=j, identifier="2021", name="2021") 386 | leg = Organization( 387 | id="ohl", 388 | name="Ohio Legislature", 389 | classification="legislature", 390 | jurisdiction=j, 391 | ) 392 | upper = Organization( 393 | id="ohs", 394 | name="Ohio Senate", 395 | classification="upper", 396 | jurisdiction=j, 397 | ) 398 | lower = Organization( 399 | id="ohh", 400 | name="Ohio House", 401 | classification="lower", 402 | jurisdiction=j, 403 | ) 404 | house_education = Organization( 405 | id="ocd-organization/11112222-3333-4444-5555-666677778888", 406 | jurisdiction=j, 407 | parent_id="ohh", 408 | name="House Committee on Education", 409 | classification="committee", 410 | links=[{"url": "https://example.com/education-link", "note": ""}], 411 | sources=[{"url": "https://example.com/education-source", "note": ""}], 412 | extras={"example-room": "Room 84"}, 413 | ) 414 | senate_education = Organization( 415 | id="ocd-organization/11112222-3333-4444-5555-666677779999", 416 | jurisdiction=j, 417 | parent_id="ohs", 418 | name="Senate Committee on Education", 419 | classification="committee", 420 | ) 421 | k5_sub = Organization( 422 | id="ocd-organization/11112222-3333-4444-5555-000000000000", 423 | jurisdiction=j, 424 | parent_id=house_education.id, 425 | name="K-5 Education Subcommittee", 426 | classification="subcommittee", 427 | ) 428 | hb1 = Bill( 429 | id="ocd-bill/1234", 430 | identifier="HB 1", 431 | title="Alphabetization of OHIO Act", 432 | legislative_session=ls2021, 433 | from_organization=upper, 434 | subject=[], 435 | classification=["bill"], 436 | extras={}, 437 | created_at=datetime.datetime.utcnow(), 438 | updated_at=datetime.datetime.utcnow(), 439 | latest_action_date="2021-01-01", 440 | ) 441 | # sb1 = Bill( 442 | # id="ocd-bill/9999", 443 | # identifier="SB 1", 444 | related_bill = RelatedBill( 445 | bill=hb1, 446 | identifier="SB 1", 447 | legislative_session="2021", 448 | relation_type="companion", 449 | ) 450 | ruth = Person( 451 | id=dummy_person_id("9"), 452 | name="Ruth", 453 | party="Democratic", 454 | current_role={ 455 | "org_classification": "upper", 456 | "district": 9, 457 | "title": "Senator", 458 | "division_id": "ocd-division/country:us/state:oh/sldu:9", 459 | }, 460 | ) 461 | marge = Person( 462 | id=dummy_person_id("7"), 463 | name="Marge", 464 | party="Democratic", 465 | current_role={ 466 | "org_classification": "upper", 467 | "district": 7, 468 | "title": "Senator", 469 | "division_id": "ocd-division/country:us/state:oh/sldu:7", 470 | }, 471 | ) 472 | sp1 = BillSponsorship( 473 | bill=hb1, 474 | primary=True, 475 | classification="sponsor", 476 | name="Ruth", 477 | entity_type="person", 478 | person=ruth, 479 | ) 480 | sp2 = BillSponsorship( 481 | bill=hb1, 482 | primary=True, 483 | classification="cosponsor", 484 | name="Marge", 485 | entity_type="person", 486 | person=marge, 487 | ) 488 | btext = SearchableBill( 489 | bill=hb1, 490 | search_vector=func.to_tsvector( 491 | "This bill renames Ohio to HIOO, it is a good idea.", config="english" 492 | ), 493 | ) 494 | v1 = VoteEvent( 495 | id="ocd-vote/1", 496 | bill=hb1, 497 | identifier="Vote on HB1", 498 | motion_text="Floor Vote", 499 | start_date="2021-01-01", 500 | result="passed", 501 | organization=lower, 502 | ) 503 | v2 = VoteEvent( 504 | id="ocd-vote/2", 505 | bill=hb1, 506 | identifier="Vote on HB1", 507 | motion_text="Floor Vote", 508 | start_date="2021-02-01", 509 | result="passed", 510 | organization=upper, 511 | ) 512 | com_mem1 = Membership( 513 | organization_id=senate_education.id, 514 | person_id=ruth.id, 515 | role="Chair", 516 | person_name="Ruth", 517 | ) 518 | com_mem2 = Membership( 519 | organization_id=senate_education.id, 520 | person_id=marge.id, 521 | role="Member", 522 | person_name="Marge", 523 | ) 524 | return [ 525 | j, 526 | leg, 527 | upper, 528 | lower, 529 | ls2021, 530 | hb1, 531 | related_bill, 532 | ruth, 533 | marge, 534 | sp1, 535 | sp2, 536 | v1, 537 | v2, 538 | VoteCount(vote_event=v1, option="yes", value=2), 539 | VoteCount(vote_event=v1, option="no", value=1), 540 | PersonVote(vote_event=v1, option="yes", voter_name="Bart"), 541 | PersonVote(vote_event=v1, option="yes", voter_name="Harley"), 542 | PersonVote(vote_event=v1, option="no", voter_name="Jarvis"), 543 | VoteCount(vote_event=v2, option="yes", value=42), 544 | VoteCount(vote_event=v2, option="no", value=0), 545 | btext, 546 | Organization( 547 | id="ohe", name="Ohio Executive", classification="executive", jurisdiction=j 548 | ), 549 | house_education, 550 | senate_education, 551 | k5_sub, 552 | com_mem1, 553 | com_mem2, 554 | ] 555 | 556 | 557 | def mentor(): 558 | j = Jurisdiction( 559 | id="ocd-jurisdiction/country:us/state:oh/place:mentor", 560 | name="Mentor", 561 | url="https://mentoroh.gov", 562 | classification="municipality", 563 | division_id="ocd-division/country:us/state:oh/place:mentor", 564 | latest_bill_update=datetime.datetime(2021, 8, 1), 565 | latest_people_update=datetime.datetime(2021, 8, 2), 566 | ) 567 | return [j] 568 | -------------------------------------------------------------------------------- /api/tests/test_bills.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | from .conftest import query_logger 3 | from api.bills import BillSortOption 4 | 5 | 6 | def test_bills_filter_by_jurisdiction_abbr(client): 7 | # state short ID lower case 8 | response = client.get("/bills?jurisdiction=ne") 9 | assert query_logger.count == 2 10 | response = response.json() 11 | assert len(response["results"]) == 7 12 | 13 | # state short ID upper case 14 | response = client.get("/bills?jurisdiction=NE") 15 | assert query_logger.count == 2 16 | response = response.json() 17 | assert len(response["results"]) == 7 18 | 19 | 20 | def test_bills_filter_by_jurisdiction_name(client): 21 | # by full name 22 | response = client.get("/bills?jurisdiction=Nebraska") 23 | assert query_logger.count == 2 24 | response = response.json() 25 | assert len(response["results"]) == 7 26 | 27 | 28 | def test_bills_filter_by_jurisdiction_jid(client): 29 | # by jid 30 | response = client.get( 31 | "/bills?jurisdiction=ocd-jurisdiction/country:us/state:ne/government" 32 | ) 33 | assert query_logger.count == 2 34 | response = response.json() 35 | assert len(response["results"]) == 7 36 | 37 | 38 | def test_bills_filter_by_session_no_jurisdiction(client): 39 | # 5 bills are in 2020 40 | response = client.get("/bills?session=2020") 41 | assert query_logger.count == 0 42 | assert response.status_code == 400 43 | assert "jurisdiction" in response.json()["detail"] 44 | 45 | 46 | def test_bills_filter_by_session(client): 47 | # 5 bills are in 2020 48 | response = client.get("/bills?jurisdiction=ne&session=2020") 49 | assert query_logger.count == 2 50 | assert len(response.json()["results"]) == 5 51 | 52 | 53 | def test_bills_filter_by_identifier(client): 54 | # spaces corrected 55 | response = client.get("/bills?jurisdiction=oh&identifier=HB1") 56 | assert query_logger.count == 2 57 | assert len(response.json()["results"]) == 1 58 | # case insensitive 59 | response = client.get("/bills?jurisdiction=oh&identifier=hb 1") 60 | assert query_logger.count == 2 61 | assert len(response.json()["results"]) == 1 62 | 63 | 64 | def test_bills_filter_by_identifier_multi(client): 65 | response = client.get("/bills?jurisdiction=ne&identifier=sb1&identifier=SB 2") 66 | assert query_logger.count == 2 67 | assert len(response.json()["results"]) == 2 68 | 69 | 70 | def test_bills_filter_by_chamber(client): 71 | response = client.get("/bills?jurisdiction=ne&chamber=legislature") 72 | assert len(response.json()["results"]) == 7 73 | response = client.get("/bills?jurisdiction=oh&chamber=upper") 74 | assert len(response.json()["results"]) == 1 75 | response = client.get("/bills?jurisdiction=oh&chamber=lower") 76 | assert len(response.json()["results"]) == 0 77 | 78 | 79 | def test_bills_filter_by_classification(client): 80 | response = client.get("/bills?jurisdiction=ne&classification=bill") 81 | assert len(response.json()["results"]) == 5 82 | response = client.get("/bills?jurisdiction=ne&classification=resolution") 83 | assert len(response.json()["results"]) == 2 84 | 85 | 86 | def test_bills_filter_by_subject(client): 87 | response = client.get("/bills?jurisdiction=ne&subject=futurism") 88 | response = response.json() 89 | assert len(response["results"]) == 2 90 | 91 | 92 | def test_bills_filter_by_updated_since(client): 93 | response = client.get("/bills?jurisdiction=ne&updated_since=2020-01-01") 94 | assert len(response.json()["results"]) == 7 95 | response = client.get("/bills?jurisdiction=ne&updated_since=2032-01-01T00:00:00") 96 | assert len(response.json()["results"]) == 0 97 | 98 | 99 | def test_bills_filter_by_created_since(client): 100 | response = client.get("/bills?jurisdiction=ne&created_since=2020-01-01T00:00") 101 | assert len(response.json()["results"]) == 7 102 | response = client.get("/bills?jurisdiction=ne&created_since=2032-01-01") 103 | assert len(response.json()["results"]) == 0 104 | 105 | 106 | def test_bills_filter_by_bad_datetime(client): 107 | response = client.get("/bills?jurisdiction=ne&created_since=2022") 108 | assert response.status_code == 400 109 | assert "ISO-8601" in response.json()["detail"] 110 | 111 | 112 | def test_bills_filter_by_action_since(client): 113 | response = client.get("/bills?jurisdiction=ne&action_since=2020") 114 | assert len(response.json()["results"]) == 7 115 | response = client.get("/bills?jurisdiction=ne&action_since=2021") 116 | assert len(response.json()["results"]) == 2 117 | 118 | 119 | def test_bills_filter_by_query(client): 120 | response = client.get("/bills?q=HIOO") 121 | assert len(response.json()["results"]) == 1 122 | response = client.get("/bills?q=Ohio") 123 | assert len(response.json()["results"]) == 1 124 | response = client.get("/bills?q=Cleveland") 125 | assert len(response.json()["results"]) == 0 126 | 127 | 128 | def test_bills_filter_by_query_bill_id(client): 129 | response = client.get("/bills?q=HB 1") 130 | assert len(response.json()["results"]) == 1 131 | response = client.get("/bills?q=hb1") 132 | assert len(response.json()["results"]) == 1 133 | response = client.get("/bills?q=hb6") 134 | assert len(response.json()["results"]) == 0 135 | 136 | 137 | def test_bills_filter_by_sponsor_name(client): 138 | response = client.get("/bills?jurisdiction=oh&sponsor=Ruth") 139 | assert len(response.json()["results"]) == 1 140 | # non-used name 141 | response = client.get("/bills?jurisdiction=oh&sponsor=Willis") 142 | assert len(response.json()["results"]) == 0 143 | 144 | 145 | def test_bills_filter_by_sponsor_id(client): 146 | response = client.get( 147 | "/bills?jurisdiction=oh&sponsor=ocd-person/99999999-9999-9999-9999-999999999999" 148 | ) 149 | assert len(response.json()["results"]) == 1 150 | # bad ID 151 | response = client.get( 152 | "/bills?jurisdiction=oh&sponsor=ocd-person/99999999-9999-9999-9999-000000000000" 153 | ) 154 | assert len(response.json()["results"]) == 0 155 | 156 | 157 | def test_bills_filter_by_sponsor_classification(client): 158 | response = client.get( 159 | "/bills?jurisdiction=oh&sponsor=Ruth&sponsor_classification=sponsor" 160 | ) 161 | assert len(response.json()["results"]) == 1 162 | response = client.get( 163 | "/bills?jurisdiction=oh&sponsor=Ruth&sponsor_classification=cosponsor" 164 | ) 165 | assert len(response.json()["results"]) == 0 166 | response = client.get( 167 | "/bills?jurisdiction=oh&sponsor=Marge&sponsor_classification=cosponsor" 168 | ) 169 | assert len(response.json()["results"]) == 1 170 | 171 | 172 | def test_bills_no_filter(client): 173 | response = client.get("/bills") 174 | assert query_logger.count == 0 175 | assert response.status_code == 400 176 | assert "required" in response.json()["detail"] 177 | 178 | 179 | def test_bills_include_basics(client): 180 | response = client.get( 181 | "/bills?jurisdiction=ne&session=2020&include=sponsorships&include=abstracts" 182 | "&include=other_titles&include=other_identifiers&include=actions&include=sources" 183 | ) 184 | assert query_logger.count == 9 185 | assert response.status_code == 200 186 | for b in response.json()["results"]: 187 | assert len(b["sponsorships"]) == 2 188 | assert len(b["abstracts"]) == 0 189 | assert len(b["other_titles"]) == 0 190 | assert len(b["other_identifiers"]) == 0 191 | assert len(b["actions"]) == 5 192 | assert len(b["sources"]) == 1 193 | assert "documents" not in b 194 | 195 | 196 | def test_bills_include_documents_versions(client): 197 | response = client.get( 198 | "/bills?jurisdiction=ne&session=2020&include=documents&include=versions" 199 | ) 200 | assert query_logger.count == 6 201 | assert response.status_code == 200 202 | for b in response.json()["results"]: 203 | assert len(b["documents"]) == 3 204 | assert len(b["versions"]) == 2 205 | version = b["versions"][0] 206 | version_id = version.pop("id") # they are dynamic uuids so... 207 | assert uuid.UUID(version_id) 208 | assert isinstance(version_id, str) 209 | assert version == { 210 | "note": "Version 0", 211 | "date": "2020", 212 | "links": [{"media_type": "text/html", "url": "https://example.com/0"}], 213 | "classification": "", 214 | } 215 | 216 | for doc in b["documents"]: 217 | doc_id = doc.pop("id") 218 | assert uuid.UUID(doc_id) 219 | assert "note" in doc 220 | assert "date" in doc 221 | assert "links" in doc 222 | assert "other_titles" not in b 223 | 224 | 225 | def test_bills_include_votes(client): 226 | response = client.get("/bills?q=HB1&include=votes") 227 | assert query_logger.count == 7 228 | assert response.status_code == 200 229 | b = response.json()["results"][0] 230 | votes = b["votes"] 231 | for vote in votes: 232 | for person_vote in vote["votes"]: 233 | vote_people_id = person_vote.pop("id") 234 | assert uuid.UUID(vote_people_id) 235 | assert isinstance(vote_people_id, str) 236 | 237 | assert b["votes"] == [ 238 | { 239 | "id": "ocd-vote/1", 240 | "identifier": "Vote on HB1", 241 | "motion_text": "Floor Vote", 242 | "motion_classification": [], 243 | "start_date": "2021-01-01", 244 | "result": "passed", 245 | "organization": { 246 | "id": "ohh", 247 | "name": "Ohio House", 248 | "classification": "lower", 249 | }, 250 | "votes": [ 251 | {"option": "yes", "voter_name": "Bart"}, 252 | {"option": "yes", "voter_name": "Harley"}, 253 | {"option": "no", "voter_name": "Jarvis"}, 254 | ], 255 | "counts": [{"option": "yes", "value": 2}, {"option": "no", "value": 1}], 256 | "sources": [], 257 | "extras": {}, 258 | }, 259 | { 260 | "id": "ocd-vote/2", 261 | "identifier": "Vote on HB1", 262 | "motion_text": "Floor Vote", 263 | "motion_classification": [], 264 | "start_date": "2021-02-01", 265 | "result": "passed", 266 | "organization": { 267 | "id": "ohs", 268 | "name": "Ohio Senate", 269 | "classification": "upper", 270 | }, 271 | "votes": [], 272 | "counts": [{"option": "yes", "value": 42}, {"option": "no", "value": 0}], 273 | "sources": [], 274 | "extras": {}, 275 | }, 276 | ] 277 | 278 | 279 | def test_bills_include_related_bills(client): 280 | response = client.get("/bills?q=HB1&include=related_bills") 281 | assert query_logger.count == 3 282 | assert response.status_code == 200 283 | b = response.json()["results"][0] 284 | assert b["related_bills"] == [ 285 | { 286 | "identifier": "SB 1", 287 | "legislative_session": "2021", 288 | "relation_type": "companion", 289 | } 290 | ] 291 | 292 | 293 | def test_bills_all_sort_options_valid(client): 294 | for sort_param in BillSortOption: 295 | response = client.get(f"/bills/?jurisdiction=ne&sort={sort_param}") 296 | print(response.json()) 297 | assert response.status_code == 200 298 | 299 | 300 | def test_bills_sort_ordering_correct(client): 301 | bills = client.get("/bills/?jurisdiction=ne&sort=updated_desc").json()["results"] 302 | assert bills[0]["updated_at"] > bills[1]["updated_at"] > bills[2]["updated_at"] 303 | 304 | bills = client.get("/bills/?jurisdiction=ne&sort=updated_asc").json()["results"] 305 | assert bills[0]["updated_at"] < bills[1]["updated_at"] < bills[2]["updated_at"] 306 | 307 | bills = client.get("/bills/?jurisdiction=ne&sort=first_action_asc").json()[ 308 | "results" 309 | ] 310 | assert ( 311 | bills[0]["first_action_date"] 312 | <= bills[1]["first_action_date"] 313 | <= bills[2]["first_action_date"] 314 | ) 315 | 316 | bills = client.get("/bills/?jurisdiction=ne&sort=first_action_desc").json()[ 317 | "results" 318 | ] 319 | assert ( 320 | bills[0]["first_action_date"] 321 | >= bills[1]["first_action_date"] 322 | >= bills[2]["first_action_date"] 323 | ) 324 | 325 | bills = client.get("/bills/?jurisdiction=ne&sort=latest_action_asc").json()[ 326 | "results" 327 | ] 328 | assert ( 329 | bills[0]["latest_action_date"] 330 | <= bills[1]["latest_action_date"] 331 | <= bills[2]["latest_action_date"] 332 | ) 333 | 334 | bills = client.get("/bills/?jurisdiction=ne&sort=latest_action_desc").json()[ 335 | "results" 336 | ] 337 | assert ( 338 | bills[0]["latest_action_date"] 339 | >= bills[1]["latest_action_date"] 340 | >= bills[2]["latest_action_date"] 341 | ) 342 | 343 | 344 | def test_bill_detail_basic(client): 345 | response = client.get("/bills/oh/2021/HB 1").json() 346 | assert response["id"] == "ocd-bill/1234" 347 | assert query_logger.count == 1 348 | 349 | 350 | def test_bill_detail_404(client): 351 | response = client.get("/bills/oh/2021/HB 66") 352 | assert response.status_code == 404 353 | assert response.json() == {"detail": "No such Bill."} 354 | 355 | 356 | def test_bill_detail_id_normalization(client): 357 | response = client.get("/bills/oh/2021/HB1").json() 358 | assert response["id"] == "ocd-bill/1234" 359 | assert query_logger.count == 1 360 | 361 | response = client.get("/bills/oh/2021/hb1").json() 362 | assert response["id"] == "ocd-bill/1234" 363 | assert query_logger.count == 1 364 | 365 | 366 | def test_bill_openstates_url(client): 367 | response = client.get("/bills/oh/2021/HB1").json() 368 | assert response["openstates_url"] == "https://openstates.org/oh/bills/2021/HB1/" 369 | assert query_logger.count == 1 370 | 371 | 372 | def test_bill_detail_includes(client): 373 | response = client.get("/bills/oh/2021/HB 1?include=votes").json() 374 | assert response["id"] == "ocd-bill/1234" 375 | assert len(response["votes"]) == 2 376 | assert query_logger.count == 6 377 | 378 | 379 | def test_bill_detail_by_internal_id(client): 380 | response = client.get("/bills/ocd-bill/1234").json() 381 | assert response["id"] == "ocd-bill/1234" 382 | assert response["identifier"] == "HB 1" 383 | assert query_logger.count == 1 384 | 385 | 386 | def test_bill_detail_sponsorship_resolution(client): 387 | response = client.get("/bills/oh/2021/HB 1?include=sponsorships").json() 388 | assert response["id"] == "ocd-bill/1234" 389 | assert len(response["sponsorships"]) == 2 390 | # uses the compact person representation, no more joins 391 | sponsorship = response["sponsorships"][0] 392 | sponsor_id = sponsorship.pop("id") 393 | assert uuid.UUID(sponsor_id) 394 | assert isinstance(sponsor_id, str) 395 | assert sponsorship == { 396 | "name": "Ruth", 397 | "entity_type": "person", 398 | "classification": "sponsor", 399 | "primary": True, 400 | "person": { 401 | "id": "ocd-person/99999999-9999-9999-9999-999999999999", 402 | "name": "Ruth", 403 | "party": "Democratic", 404 | "current_role": { 405 | "title": "Senator", 406 | "org_classification": "upper", 407 | "district": "9", 408 | "division_id": "ocd-division/country:us/state:oh/sldu:9", 409 | }, 410 | }, 411 | } 412 | assert query_logger.count == 3 413 | 414 | 415 | def test_bills_include_actions(client): 416 | response = client.get("/bills?jurisdiction=ne&session=2020&include=actions") 417 | 418 | for bill in response.json()["results"]: 419 | assert "actions" in bill 420 | assert len(bill["actions"]) > 0 421 | 422 | for action in bill["actions"]: 423 | action_id = action.pop("id") 424 | assert uuid.UUID(action_id) 425 | 426 | assert action["description"] 427 | assert action["date"] 428 | assert isinstance(action["classification"], list) 429 | assert isinstance(action["organization"], dict) 430 | assert isinstance(action["related_entities"], list) 431 | 432 | assert "id" in action["organization"] 433 | org_id = action["organization"]["id"] 434 | assert isinstance(org_id, str) 435 | assert "name" in action["organization"] 436 | assert "classification" in action["organization"] 437 | 438 | for entity in action["related_entities"]: 439 | assert "id" in entity 440 | assert "name" in entity 441 | assert "type" in entity 442 | -------------------------------------------------------------------------------- /api/tests/test_committees.py: -------------------------------------------------------------------------------- 1 | from .conftest import query_logger 2 | 3 | HOUSE_COM_ID = "ocd-organization/11112222-3333-4444-5555-666677778888" 4 | SENATE_COM_ID = "ocd-organization/11112222-3333-4444-5555-666677779999" 5 | SENATE_COM_RESPONSE = { 6 | "id": SENATE_COM_ID, 7 | "name": "Senate Committee on Education", 8 | "classification": "committee", 9 | "parent_id": "ohs", 10 | "extras": {}, 11 | } 12 | SENATE_MEMBERSHIPS = [ 13 | { 14 | "person_name": "Ruth", 15 | "role": "Chair", 16 | "person": { 17 | "id": "ocd-person/99999999-9999-9999-9999-999999999999", 18 | "name": "Ruth", 19 | "party": "Democratic", 20 | "current_role": { 21 | "district": "9", 22 | "division_id": "ocd-division/country:us/state:oh/sldu:9", 23 | "org_classification": "upper", 24 | "title": "Senator", 25 | }, 26 | }, 27 | }, 28 | { 29 | "person_name": "Marge", 30 | "role": "Member", 31 | "person": { 32 | "id": "ocd-person/77777777-7777-7777-7777-777777777777", 33 | "name": "Marge", 34 | "party": "Democratic", 35 | "current_role": { 36 | "district": "7", 37 | "division_id": "ocd-division/country:us/state:oh/sldu:7", 38 | "org_classification": "upper", 39 | "title": "Senator", 40 | }, 41 | }, 42 | }, 43 | ] 44 | 45 | 46 | def test_committee_detail(client): 47 | response = client.get("/committees/" + SENATE_COM_ID) 48 | assert query_logger.count == 1 49 | response = response.json() 50 | assert response == SENATE_COM_RESPONSE 51 | 52 | 53 | def test_committee_detail_include_memberships(client): 54 | response = client.get("/committees/" + SENATE_COM_ID + "?include=memberships") 55 | assert query_logger.count == 3 56 | response = response.json() 57 | assert response == dict(memberships=SENATE_MEMBERSHIPS, **SENATE_COM_RESPONSE) 58 | 59 | 60 | def test_committee_list(client): 61 | response = client.get("/committees?jurisdiction=oh") 62 | assert query_logger.count == 2 63 | response = response.json() 64 | assert len(response["results"]) == 3 65 | assert "House Committee on Education" == response["results"][0]["name"] 66 | assert "K-5 Education Subcommittee" == response["results"][1]["name"] 67 | assert "Senate Committee on Education" == response["results"][2]["name"] 68 | 69 | 70 | def test_committee_list_empty(client): 71 | response = client.get("/committees?jurisdiction=nh") 72 | assert query_logger.count == 2 73 | response = response.json() 74 | assert len(response["results"]) == 0 75 | 76 | 77 | def test_committee_list_with_members(client): 78 | response = client.get("/committees?jurisdiction=oh&include=memberships") 79 | assert query_logger.count == 4 80 | response = response.json() 81 | assert len(response["results"]) == 3 82 | assert response["results"][0]["memberships"] == [] 83 | assert response["results"][1]["memberships"] == [] 84 | assert "Senate Committee on Education" == response["results"][2]["name"] 85 | assert response["results"][2]["memberships"] == SENATE_MEMBERSHIPS 86 | 87 | 88 | def test_committee_list_with_links_sources_extras(client): 89 | response = client.get("/committees?jurisdiction=oh&include=links&include=sources") 90 | assert query_logger.count == 2 91 | response = response.json() 92 | assert response["results"][0]["links"] == [ 93 | {"url": "https://example.com/education-link", "note": ""} 94 | ] 95 | assert response["results"][0]["sources"] == [ 96 | {"url": "https://example.com/education-source", "note": ""} 97 | ] 98 | assert response["results"][0]["extras"] == {"example-room": "Room 84"} 99 | 100 | 101 | def test_committee_list_by_chamber(client): 102 | response = client.get("/committees?jurisdiction=oh&chamber=upper") 103 | assert query_logger.count == 2 104 | response = response.json() 105 | assert len(response["results"]) == 1 106 | assert "Senate Committee on Education" == response["results"][0]["name"] 107 | 108 | 109 | def test_committee_list_by_parent(client): 110 | response = client.get("/committees?jurisdiction=oh&parent=ohs") 111 | assert query_logger.count == 2 112 | response = response.json() 113 | assert len(response["results"]) == 1 114 | assert "Senate Committee on Education" == response["results"][0]["name"] 115 | 116 | 117 | def test_committee_list_by_classification(client): 118 | response = client.get("/committees?jurisdiction=oh&classification=subcommittee") 119 | assert query_logger.count == 2 120 | response = response.json() 121 | assert len(response["results"]) == 1 122 | assert "K-5 Education Subcommittee" == response["results"][0]["name"] 123 | -------------------------------------------------------------------------------- /api/tests/test_events.py: -------------------------------------------------------------------------------- 1 | from .conftest import query_logger 2 | 3 | 4 | def test_events_list(client): 5 | response = client.get("/events?jurisdiction=ne").json() 6 | assert query_logger.count == 2 7 | assert len(response["results"]) == 3 8 | assert response["results"][0]["name"] == "Event #0" 9 | assert "links" not in response["results"][0] 10 | assert "sources" not in response["results"][0] 11 | 12 | 13 | def test_events_list_simple_includes(client): 14 | response = client.get( 15 | "/events?jurisdiction=ne&include=sources&include=links" 16 | ).json() 17 | # no extra queries for these 18 | assert query_logger.count == 2 19 | assert len(response["results"]) == 3 20 | assert response["results"][0]["name"] == "Event #0" 21 | assert response["results"][0]["links"][0]["url"] == "https://example.com/0" 22 | assert response["results"][0]["sources"][0]["url"] == "https://example.com/0" 23 | 24 | 25 | def test_events_list_join_includes(client): 26 | response = client.get( 27 | "/events?jurisdiction=ne&include=media&include=documents&include=participants" 28 | ).json() 29 | # one extra query each 30 | assert query_logger.count == 5 31 | assert len(response["results"]) == 3 32 | assert response["results"][0]["name"] == "Event #0" 33 | assert len(response["results"][0]["media"]) == 1 34 | assert len(response["results"][0]["documents"]) == 2 35 | assert len(response["results"][0]["participants"]) == 3 36 | 37 | 38 | def test_events_list_join_agenda(client): 39 | response = client.get("/events?jurisdiction=ne&include=agenda").json() 40 | # agenda is 3 extra queries together 41 | assert query_logger.count == 5 42 | assert len(response["results"]) == 3 43 | assert response["results"][0]["name"] == "Event #0" 44 | assert len(response["results"][0]["agenda"]) == 2 45 | assert response["results"][0]["agenda"][0]["description"] == "Agenda Item 1" 46 | assert response["results"][0]["agenda"][0]["order"] == 1 47 | assert response["results"][0]["agenda"][0]["media"][0]["date"] == "2021-01-01" 48 | assert response["results"][0]["agenda"][0]["related_entities"] == [ 49 | {"note": "", "name": "SB 1", "entity_type": "bill"} 50 | ] 51 | assert response["results"][0]["agenda"][1]["description"] == "Agenda Item 2" 52 | 53 | 54 | def test_events_list_deleted(client): 55 | response = client.get("/events?jurisdiction=ne&deleted=true").json() 56 | assert query_logger.count == 2 57 | assert len(response["results"]) == 1 58 | assert response["results"][0]["name"] == "Event #4" 59 | assert response["results"][0]["deleted"] is True 60 | 61 | 62 | def test_events_list_before(client): 63 | response = client.get("/events?jurisdiction=ne&before=2021-01-02").json() 64 | assert query_logger.count == 2 65 | assert len(response["results"]) == 1 66 | assert response["results"][0]["start_date"] == "2021-01-01" 67 | 68 | 69 | def test_events_list_after(client): 70 | response = client.get("/events?jurisdiction=ne&after=2021-01-02").json() 71 | assert query_logger.count == 2 72 | assert len(response["results"]) == 1 73 | assert response["results"][0]["start_date"] == "2021-01-03" 74 | 75 | 76 | def test_events_list_before_and_after(client): 77 | response = client.get( 78 | "/events?jurisdiction=ne&after=2021-01-01&before=2021-01-03" 79 | ).json() 80 | assert query_logger.count == 2 81 | assert len(response["results"]) == 1 82 | assert response["results"][0]["start_date"] == "2021-01-02" 83 | 84 | 85 | def test_events_list_require_bills(client): 86 | response = client.get("/events?jurisdiction=ne&require_bills=true").json() 87 | assert query_logger.count == 2 88 | assert len(response["results"]) == 1 89 | assert response["results"][0]["name"] == "Event #0" 90 | 91 | 92 | def test_events_list_require_bills_include_agenda(client): 93 | response = client.get( 94 | "/events?jurisdiction=ne&require_bills=true&include=agenda" 95 | ).json() 96 | # join count should still be 5, checking for weirdness w/ group by/agenda join 97 | assert query_logger.count == 5 98 | assert len(response["results"]) == 1 99 | assert response["results"][0]["name"] == "Event #0" 100 | 101 | 102 | BASIC_EVENT = { 103 | "all_day": False, 104 | "classification": "", 105 | "deleted": False, 106 | "description": "", 107 | "end_date": "", 108 | "id": "ocd-event/00000000-0000-0000-0000-000000000000", 109 | "jurisdiction": { 110 | "classification": "state", 111 | "id": "ocd-jurisdiction/country:us/state:ne/government", 112 | "name": "Nebraska", 113 | }, 114 | "location": {"name": "Location #0", "url": ""}, 115 | "name": "Event #0", 116 | "start_date": "2021-01-01", 117 | "status": "normal", 118 | "upstream_id": "", 119 | } 120 | 121 | FULL_EVENT = BASIC_EVENT.copy() 122 | FULL_EVENT["sources"] = FULL_EVENT["links"] = [ 123 | {"note": "source", "url": "https://example.com/0"} 124 | ] 125 | FULL_EVENT["media"] = [ 126 | {"classification": "", "date": "2021-01-01", "links": [], "note": "", "offset": 0} 127 | ] 128 | FULL_EVENT["participants"] = [ 129 | {"entity_type": "person", "note": "", "name": "John"}, 130 | {"entity_type": "person", "note": "", "name": "Jane"}, 131 | {"entity_type": "person", "note": "", "name": "Javier"}, 132 | ] 133 | FULL_EVENT["documents"] = [ 134 | {"classification": "", "date": "2021-01-01", "note": "document 1", "links": []}, 135 | {"classification": "", "date": "2021-01-01", "note": "document 2", "links": []}, 136 | ] 137 | FULL_EVENT["agenda"] = [ 138 | { 139 | "classification": [], 140 | "description": "Agenda Item 1", 141 | "media": [ 142 | { 143 | "classification": "", 144 | "date": "2021-01-01", 145 | "links": [], 146 | "note": "", 147 | "offset": 0, 148 | } 149 | ], 150 | "notes": [], 151 | "subjects": [], 152 | "order": 1, 153 | "extras": {}, 154 | "related_entities": [{"entity_type": "bill", "name": "SB 1", "note": ""}], 155 | }, 156 | { 157 | "classification": [], 158 | "description": "Agenda Item 2", 159 | "media": [], 160 | "notes": [], 161 | "subjects": [], 162 | "order": 2, 163 | "extras": {}, 164 | "related_entities": [], 165 | }, 166 | ] 167 | 168 | 169 | def test_event_by_id(client): 170 | response = client.get( 171 | "/events/ocd-event/00000000-0000-0000-0000-000000000000" 172 | ).json() 173 | assert query_logger.count == 1 174 | assert response == BASIC_EVENT 175 | 176 | 177 | def test_event_by_id_all_includes(client): 178 | response = client.get( 179 | "/events/ocd-event/00000000-0000-0000-0000-000000000000?" 180 | "include=links&include=sources&include=media&include=documents&include=participants&include=agenda" 181 | ).json() 182 | # 3 joins for media, documents, participants, and 3 more for agenda 183 | assert query_logger.count == 7 184 | assert response == FULL_EVENT 185 | -------------------------------------------------------------------------------- /api/tests/test_jurisdictions.py: -------------------------------------------------------------------------------- 1 | from .conftest import query_logger 2 | 3 | 4 | def test_jurisdictions_simplest(client): 5 | response = client.get("/jurisdictions") 6 | assert query_logger.count == 2 7 | response = response.json() 8 | assert len(response["results"]) == 3 9 | assert response["results"][0]["name"] == "Mentor" 10 | assert response["results"][1]["name"] == "Nebraska" 11 | assert response["results"][2] == { 12 | "id": "ocd-jurisdiction/country:us/state:oh/government", 13 | "name": "Ohio", 14 | "url": "https://ohio.gov", 15 | "division_id": "ocd-division/country:us/state:oh", 16 | "classification": "state", 17 | "latest_bill_update": "2021-08-04T00:00:00", 18 | "latest_people_update": "2021-08-05T00:00:00", 19 | # note that organizations are not included here as a key 20 | } 21 | 22 | 23 | def test_jurisdictions_filter(client): 24 | response = client.get("/jurisdictions?classification=state") 25 | response = response.json() 26 | assert len(response["results"]) == 2 27 | assert query_logger.count == 2 28 | response = client.get("/jurisdictions?classification=municipality") 29 | response = response.json() 30 | assert len(response["results"]) == 1 31 | assert query_logger.count == 2 32 | 33 | 34 | def test_jurisdiction_include_organizations(client): 35 | response = client.get( 36 | "/jurisdictions?classification=state&per_page=1&include=organizations" 37 | ) 38 | response = response.json() 39 | # is included, organizations are inline 40 | assert query_logger.count == 4 41 | assert len(response["results"][0]["organizations"]) == 2 42 | assert response["results"][0]["organizations"][0] == { 43 | "id": "nel", 44 | "classification": "legislature", 45 | "name": "Nebraska Legislature", 46 | "districts": [ 47 | { 48 | "label": "1", 49 | "division_id": "ocd-division/country:us/state:ne/sldu:1", 50 | "role": "Senator", 51 | "maximum_memberships": 1, 52 | }, 53 | ], 54 | } 55 | 56 | 57 | def test_jurisdiction_include_organizations_empty(client): 58 | response = client.get( 59 | "/jurisdictions?classification=municipality&per_page=1&include=organizations" 60 | ) 61 | response = response.json() 62 | # is included, but the field is empty 63 | assert len(response["results"][0]["organizations"]) == 0 64 | assert query_logger.count == 3 65 | 66 | 67 | def test_jurisdictions_include_runs(client): 68 | response = client.get( 69 | "/jurisdictions?classification=state&per_page=5&include=latest_runs" 70 | ) 71 | response = response.json() 72 | # is included, but the field is empty 73 | assert len(response["results"][0]["latest_runs"]) == 20 74 | # this necessarily does N+1 queries, might need to restrict 75 | assert query_logger.count == 4 76 | 77 | 78 | def test_jurisdictions_include_runs_empty(client): 79 | response = client.get( 80 | "/jurisdictions?classification=municipality&per_page=1&include=latest_runs" 81 | ) 82 | response = response.json() 83 | # is included, but the field is empty 84 | assert len(response["results"][0]["latest_runs"]) == 0 85 | assert query_logger.count == 3 86 | 87 | 88 | def test_jurisdiction_include_sessions(client): 89 | response = client.get( 90 | "/jurisdictions?classification=state&per_page=1&include=legislative_sessions" 91 | ) 92 | response = response.json() 93 | # is included, legislative sessions are inline 94 | assert query_logger.count == 4 95 | assert len(response["results"][0]["legislative_sessions"]) == 2 96 | assert response["results"][0]["legislative_sessions"][0] == { 97 | "identifier": "2020", 98 | "name": "2020", 99 | "classification": "", 100 | "start_date": "2020-01-01", 101 | "end_date": "2020-12-31", 102 | "downloads": [ 103 | { 104 | "created_at": "2021-01-01T00:00:00", 105 | "updated_at": "2021-01-01T00:00:00", 106 | "data_type": "csv", 107 | "url": "https://example.com", 108 | } 109 | ], 110 | } 111 | 112 | 113 | NEBRASKA_RESPONSE = { 114 | "id": "ocd-jurisdiction/country:us/state:ne/government", 115 | "name": "Nebraska", 116 | "classification": "state", 117 | "division_id": "ocd-division/country:us/state:ne", 118 | "url": "https://nebraska.gov", 119 | "latest_bill_update": "2021-08-01T00:00:00", 120 | "latest_people_update": "2021-08-02T00:00:00", 121 | } 122 | 123 | 124 | def test_jurisdiction_detail_by_abbr(client): 125 | response = client.get("/jurisdictions/ne").json() 126 | assert response == NEBRASKA_RESPONSE 127 | assert query_logger.count == 1 128 | 129 | 130 | def test_jurisdiction_detail_by_name(client): 131 | response = client.get("/jurisdictions/Nebraska").json() 132 | assert response == NEBRASKA_RESPONSE 133 | assert query_logger.count == 1 134 | 135 | 136 | def test_jurisdiction_detail_by_jid(client): 137 | response = client.get( 138 | "/jurisdictions/ocd-jurisdiction/country:us/state:ne/government" 139 | ).json() 140 | assert response == NEBRASKA_RESPONSE 141 | assert query_logger.count == 1 142 | 143 | 144 | def test_jurisdiction_detail_404(client): 145 | response = client.get("/jurisdictions/xy") 146 | assert response.status_code == 404 147 | assert response.json() == {"detail": "No such Jurisdiction."} 148 | 149 | 150 | def test_jurisdiction_include_orgs(client): 151 | response = client.get("/jurisdictions/ne?include=organizations").json() 152 | assert len(response["organizations"]) == 2 153 | assert query_logger.count == 3 154 | 155 | 156 | def test_jurisdiction_include_latest_runs(client): 157 | response = client.get("/jurisdictions/ne?include=latest_runs").json() 158 | assert len(response["latest_runs"]) == 20 159 | assert query_logger.count == 2 160 | -------------------------------------------------------------------------------- /api/tests/test_pagination.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from api.db import get_db, models 3 | from api.pagination import Pagination 4 | 5 | 6 | def test_pagination_basic(client): 7 | response = client.get("/jurisdictions") 8 | response = response.json() 9 | assert response["pagination"] == { 10 | "page": 1, 11 | "max_page": 1, 12 | "per_page": 52, 13 | "total_items": 3, 14 | } 15 | 16 | 17 | def test_pagination_empty(client): 18 | response = client.get("/people?jurisdiction=wi") 19 | response = response.json() 20 | assert response["pagination"] == { 21 | "page": 1, 22 | "max_page": 1, 23 | "per_page": 10, 24 | "total_items": 0, 25 | } 26 | 27 | 28 | def test_pagination_per_page(client): 29 | response = client.get("/jurisdictions?per_page=2") 30 | response = response.json() 31 | assert response["pagination"] == { 32 | "page": 1, 33 | "max_page": 2, 34 | "per_page": 2, 35 | "total_items": 3, 36 | } 37 | 38 | 39 | def test_pagination_page2(client): 40 | response = client.get("/jurisdictions?per_page=2&page=2") 41 | response = response.json() 42 | assert response["results"][0]["name"] == "Ohio" 43 | assert response["pagination"] == { 44 | "page": 2, 45 | "max_page": 2, 46 | "per_page": 2, 47 | "total_items": 3, 48 | } 49 | 50 | 51 | def test_pagination_invalid_per_page(client): 52 | response = client.get("/jurisdictions?per_page=0") 53 | assert response.status_code == 400 54 | response = response.json() 55 | assert "invalid per_page" in response["detail"] 56 | 57 | response = client.get("/jurisdictions?per_page=999") 58 | assert response.status_code == 400 59 | response = response.json() 60 | assert "invalid per_page" in response["detail"] 61 | 62 | 63 | def test_pagination_invalid_page(client): 64 | response = client.get("/jurisdictions?per_page=2&page=0") 65 | assert response.status_code == 404 66 | response = response.json() 67 | assert "invalid page" in response["detail"] 68 | 69 | response = client.get("/jurisdictions?per_page=2&page=5") 70 | assert response.status_code == 404 71 | response = response.json() 72 | assert "invalid page" in response["detail"] 73 | 74 | 75 | def test_pagination_no_order_by(): 76 | db = list(get_db())[0] 77 | query = db.query(models.Jurisdiction) 78 | p = Pagination() 79 | with pytest.raises(Exception) as e: 80 | p.paginate(query) 81 | assert "ordering is required for pagination" in str(e) 82 | -------------------------------------------------------------------------------- /api/tests/test_people.py: -------------------------------------------------------------------------------- 1 | from .conftest import query_logger 2 | from unittest import mock 3 | 4 | 5 | def test_by_jurisdiction_abbr(client): 6 | # by abbr 7 | response = client.get("/people?jurisdiction=ne").json() 8 | assert query_logger.count == 2 9 | assert len(response["results"]) == 2 10 | assert response["results"][0]["name"] == "Amy Adams" 11 | assert response["results"][1]["name"] == "Boo Berri" 12 | 13 | 14 | def test_by_jurisdiction_name(client): 15 | # by name 16 | response = client.get("/people?jurisdiction=Nebraska").json() 17 | assert query_logger.count == 2 18 | assert len(response["results"]) == 2 19 | assert response["results"][0]["name"] == "Amy Adams" 20 | assert response["results"][1]["name"] == "Boo Berri" 21 | 22 | 23 | def test_by_jurisdiction_jid(client): 24 | # by full ID 25 | response = client.get( 26 | "/people?jurisdiction=ocd-jurisdiction/country:us/state:ne/government" 27 | ).json() 28 | assert query_logger.count == 2 29 | assert len(response["results"]) == 2 30 | assert response["results"][0]["name"] == "Amy Adams" 31 | assert response["results"][1]["name"] == "Boo Berri" 32 | 33 | 34 | def test_by_district(client): 35 | response = client.get( 36 | "/people?jurisdiction=ocd-jurisdiction/country:us/state:ne/government&district=1" 37 | ).json() 38 | assert query_logger.count == 2 39 | assert len(response["results"]) == 1 40 | assert response["results"][0]["name"] == "Amy Adams" 41 | 42 | 43 | def test_by_district_text(client): 44 | response = client.get( 45 | "/people?jurisdiction=ocd-jurisdiction/country:us/state:ne/government&district=1A" 46 | ).json() 47 | assert query_logger.count == 2 48 | assert len(response["results"]) == 0 49 | 50 | 51 | def test_err_district_without_jursidiction(client): 52 | response = client.get("/people?district=1") 53 | assert response.status_code == 400 54 | assert query_logger.count == 0 55 | assert "without 'jurisdiction'" in response.json()["detail"] 56 | 57 | 58 | def test_by_jurisdiction_and_org_classification(client): 59 | response = client.get( 60 | "/people?jurisdiction=ne&org_classification=legislature" 61 | ).json() 62 | assert query_logger.count == 2 63 | assert len(response["results"]) == 1 64 | assert response["results"][0]["name"] == "Amy Adams" 65 | 66 | response = client.get("/people?jurisdiction=ne&org_classification=executive").json() 67 | assert query_logger.count == 2 68 | assert len(response["results"]) == 1 69 | assert response["results"][0]["name"] == "Boo Berri" 70 | response = client.get("/people?jurisdiction=ne&org_classification=lower").json() 71 | assert len(response["results"]) == 0 72 | 73 | 74 | def test_by_name(client): 75 | response = client.get("/people?name=Amy Adams").json() 76 | assert query_logger.count == 2 77 | assert len(response["results"]) == 1 78 | assert response["results"][0]["name"] == "Amy Adams" 79 | assert response["results"][0]["gender"] == "female" 80 | assert response["results"][0]["email"] == "aa@example.com" 81 | 82 | # lower case (also retired) 83 | response = client.get("/people?name=rita red").json() 84 | assert query_logger.count == 2 85 | assert len(response["results"]) == 1 86 | assert response["results"][0]["name"] == "Rita Red" 87 | 88 | 89 | def test_by_name_fuzzy(client): 90 | response = client.get("/people?name=amy").json() 91 | assert query_logger.count == 2 92 | assert len(response["results"]) == 1 93 | assert response["results"][0]["name"] == "Amy Adams" 94 | 95 | 96 | def test_by_name_other_name(client): 97 | response = client.get("/people?name=Amy 'Aardvark' Adams").json() 98 | assert query_logger.count == 2 99 | assert len(response["results"]) == 1 100 | assert response["results"][0]["name"] == "Amy Adams" 101 | 102 | 103 | def test_by_id(client): 104 | response = client.get( 105 | "/people?id=ocd-person/11111111-1111-1111-1111-111111111111&id=ocd-person/33333333-3333-3333-3333-333333333333" 106 | ).json() 107 | assert query_logger.count == 2 108 | assert len(response["results"]) == 2 109 | assert response["results"][0]["name"] == "Amy Adams" 110 | assert response["results"][1]["name"] == "Rita Red" 111 | 112 | 113 | def test_openstates_url(client): 114 | response = client.get( 115 | "/people?id=ocd-person/11111111-1111-1111-1111-111111111111" 116 | ).json() 117 | assert query_logger.count == 2 118 | assert len(response["results"]) == 1 119 | assert response["results"][0]["name"] == "Amy Adams" 120 | assert ( 121 | response["results"][0]["openstates_url"] 122 | == "https://openstates.org/person/amy-adams-WCfTognxqNqfz8qrH12uH/" 123 | ) 124 | 125 | 126 | def test_no_filter(client): 127 | response = client.get("/people") 128 | assert query_logger.count == 0 129 | assert response.status_code == 400 130 | assert "is required" in response.json()["detail"] 131 | 132 | response = client.get("/people?org_classification=upper") 133 | assert query_logger.count == 0 134 | assert response.status_code == 400 135 | assert "is required" in response.json()["detail"] 136 | 137 | 138 | def test_people_includes_normal(client): 139 | response = client.get( 140 | "/people?id=ocd-person/11111111-1111-1111-1111-111111111111" 141 | "&include=other_names&include=other_identifiers&include=links&include=sources" 142 | ).json() 143 | assert query_logger.count == 6 # 4 extra queries 144 | assert response["results"][0]["other_names"] == [ 145 | {"name": "Amy 'Aardvark' Adams", "note": "nickname"} 146 | ] 147 | assert response["results"][0]["other_identifiers"] == [] # empty is ok 148 | assert response["results"][0]["links"] == [ 149 | {"url": "https://example.com/amy", "note": ""} 150 | ] 151 | assert response["results"][0]["sources"] == [ 152 | {"url": "https://example.com/amy", "note": ""} 153 | ] 154 | 155 | 156 | def test_people_include_office(client): 157 | response = client.get( 158 | "/people?id=ocd-person/11111111-1111-1111-1111-111111111111" "&include=offices" 159 | ).json() 160 | assert query_logger.count == 3 # 1 extra query 161 | assert response["results"][0]["offices"] == [ 162 | { 163 | "name": "Capitol Office", 164 | "address": "123 Main St", 165 | "voice": "555-555-5555", 166 | "fax": "", 167 | "classification": "capitol", 168 | } 169 | ] 170 | 171 | 172 | def test_people_geo_basic(client): 173 | with mock.patch("api.people.session.get") as mock_get: 174 | mock_get.return_value.json.return_value = { 175 | "divisions": [ 176 | { 177 | "id": "ocd-division/country:us/state:ne/sldu:1", 178 | "state": "ne", 179 | "name": "1", 180 | "division_set": "sldu", 181 | }, 182 | ] 183 | } 184 | response = client.get("/people.geo?lat=41.5&lng=-100").json() 185 | # 1 query b/c we bypass count() since pagination isn't really needed 186 | assert query_logger.count == 1 187 | assert len(response["results"]) == 1 188 | assert response["results"][0]["name"] == "Amy Adams" 189 | # check this since skip_count=True was used 190 | assert response["pagination"]["total_items"] == 1 191 | 192 | 193 | def test_people_geo_bad_param(client): 194 | # missing parameter 195 | with mock.patch("api.people.session.get") as mock_get: 196 | response = client.get("/people.geo?lat=38") 197 | assert response.status_code == 422 198 | assert response.json() 199 | assert query_logger.count == 0 200 | assert mock_get.called is False 201 | 202 | # non-float param 203 | with mock.patch("api.people.session.get") as mock_get: 204 | response = client.get("/people.geo?lat=38&lng=abc") 205 | assert response.status_code == 422 206 | assert response.json() 207 | assert query_logger.count == 0 208 | assert mock_get.called is False 209 | 210 | 211 | def test_people_geo_bad_upstream(client): 212 | # unexpected response from upstream 213 | with mock.patch("api.people.session.get") as mock_get: 214 | mock_get.return_value.json.return_value = {"endpoint disabled": True} 215 | response = client.get("/people.geo?lat=50&lng=50") 216 | assert response.status_code == 500 217 | assert response.json() 218 | assert query_logger.count == 0 219 | assert mock_get.called is True 220 | 221 | 222 | def test_people_geo_empty(client): 223 | with mock.patch("api.people.session.get") as mock_get: 224 | mock_get.return_value.json.return_value = {"divisions": []} 225 | response = client.get("/people.geo?lat=0&lng=0") 226 | assert response.json() == { 227 | "results": [], 228 | "pagination": {"max_page": 1, "per_page": 100, "page": 1, "total_items": 0}, 229 | } 230 | assert query_logger.count == 0 231 | assert mock_get.called is True 232 | -------------------------------------------------------------------------------- /api/utils.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import and_ 2 | from .db import models 3 | from openstates.metadata import lookup 4 | 5 | 6 | def jurisdiction_filter(j: str, *, jid_field): 7 | if not j: 8 | # an empty object can't equal anything 9 | return False 10 | # check either by Jurisdiction.name or a specified field's jurisdiction_id 11 | if len(j) == 2: 12 | try: 13 | return jid_field == lookup(abbr=j).jurisdiction_id 14 | except KeyError: 15 | return and_( 16 | models.Jurisdiction.name == j, 17 | models.Jurisdiction.classification == "state", 18 | ) 19 | elif j.startswith("ocd-jurisdiction"): 20 | return jid_field == j 21 | else: 22 | return and_( 23 | models.Jurisdiction.name == j, models.Jurisdiction.classification == "state" 24 | ) 25 | 26 | 27 | def add_state_divisions(d: str): 28 | return lookup(abbr=d).division_id 29 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.5" 2 | 3 | networks: 4 | openstates-network: 5 | name: openstates-network 6 | external: true 7 | 8 | services: 9 | api: 10 | build: 11 | context: . 12 | volumes: 13 | - .:/app 14 | environment: 15 | - DATABASE_URL=postgresql://openstates:openstates@db/openstatesorg 16 | - RRL_REDIS_HOST=redis 17 | ports: 18 | - "7000:80" 19 | networks: 20 | - openstates-network 21 | redis: 22 | image: redis 23 | hostname: redis 24 | ports: 25 | - "6379:6379" 26 | networks: 27 | - openstates-network 28 | db-test: 29 | image: postgres 30 | environment: 31 | POSTGRES_USER: 'v3test' 32 | POSTGRES_PASSWORD: 'v3test' 33 | POSTGRES_DB: 'v3test' 34 | healthcheck: 35 | test: ["CMD-SHELL", "pg_isready -q -d v3test -U v3test"] 36 | interval: 10s 37 | timeout: 5s 38 | retries: 5 39 | start_period: 60s 40 | ports: 41 | - 5432:5432 42 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "openstates-api" 3 | version = "3.0.0" 4 | description = "Open States API v3" 5 | authors = ["James Turk "] 6 | license = "MIT" 7 | 8 | [tool.poetry.dependencies] 9 | python = "^3.9" 10 | openstates = "^6.7.0" 11 | sqlalchemy = "^1.4" 12 | psycopg2-binary = "^2.8.5" 13 | sqlalchemy_utils = "^0.37.0" 14 | gunicorn = "^20.0.4" 15 | sentry-sdk = "^1.0.0" 16 | pybase62 = "^0.4.3" 17 | python-slugify = "^4.0.1" 18 | rrl = "^0.3.1" 19 | prometheus-fastapi-instrumentator = "^5.8.2" 20 | fastapi = {extras = ["all"], version = "^0.87.0"} 21 | 22 | [tool.poetry.dev-dependencies] 23 | black = "^22.10.0" 24 | flake8 = "^5.0.4" 25 | pytest = "^6.0.1" 26 | 27 | [build-system] 28 | requires = ["poetry>=0.12"] 29 | build-backend = "poetry.masonry.api" 30 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max_line_length=120 3 | 4 | 5 | [coverage:report] 6 | exclude_lines = 7 | pragma: no cover 8 | if __name__ == .__main__.: 9 | --------------------------------------------------------------------------------