├── enip_backend ├── __init__.py ├── export │ ├── __init__.py │ ├── schemas │ │ ├── __init__.py │ │ ├── state.schema.json │ │ ├── ga.example.json │ │ ├── national.example.json │ │ └── national.schema.json │ ├── bulk.py │ ├── state.py │ ├── structs.py │ ├── run.py │ ├── helpers.py │ ├── state_test.py │ ├── national.py │ └── national_test.py ├── ingest │ ├── __init__.py │ ├── ingest_run.py │ ├── apapi.py │ └── run.py ├── enip_common │ ├── __init__.py │ ├── pg.py │ ├── config.py │ ├── s3.py │ ├── states.py │ └── gsheets.py ├── calls_gsheet_sync │ ├── __init__.py │ └── run.py ├── comments_gsheet_sync │ ├── __init__.py │ └── run.py └── lambda_handlers.py ├── .tool-versions ├── scripts ├── psql.sh └── migrate-db.sh ├── docker-compose.yml ├── .env.example ├── mypy.ini ├── .travis.yml ├── package.json ├── LICENSE ├── Pipfile ├── README.md ├── .gitignore ├── serverless.yml └── db └── init.sql /enip_backend/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.tool-versions: -------------------------------------------------------------------------------- 1 | python 3.7.9 2 | -------------------------------------------------------------------------------- /enip_backend/export/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /enip_backend/ingest/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /enip_backend/enip_common/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /enip_backend/calls_gsheet_sync/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /enip_backend/comments_gsheet_sync/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scripts/psql.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Connects to the development database 4 | PGPASSWORD=postgres psql -h localhost -U postgres 5 | -------------------------------------------------------------------------------- /scripts/migrate-db.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Migrates the development database 4 | 5 | cd "$(dirname "$0")" 6 | 7 | PGPASSWORD=postgres psql -h localhost -U postgres -f ../db/init.sql 8 | -------------------------------------------------------------------------------- /enip_backend/export/schemas/__init__.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os.path 3 | 4 | with open(os.path.join(os.path.dirname(__file__), "national.schema.json")) as f: 5 | national_schema = json.load(f) 6 | 7 | with open(os.path.join(os.path.dirname(__file__), "state.schema.json")) as f: 8 | state_schema = json.load(f) 9 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | # This docker-compose file can be used during local development to spin up 2 | # the Postgres server you need. 3 | # 4 | # It does NOT run the app server for you. 5 | version: '3' 6 | services: 7 | postgres: 8 | image: postgres:alpine 9 | restart: always 10 | environment: 11 | POSTGRES_DB: postgres 12 | POSTGRES_PASSWORD: postgres 13 | POSTGRES_USER: postgres 14 | volumes: 15 | - postgres:/var/lib/postgresql/data 16 | ports: 17 | - 5432:5432 18 | volumes: 19 | postgres: 20 | external: false 21 | -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | # This points to a local postgres server, which you can run with docker-compose 2 | POSTGRES_URL=postgres://postgres:postgres@localhost/postgres 3 | 4 | # Request test data from the AP 5 | INGEST_TEST_DATA=true 6 | 7 | # Request data for the 2020 general 8 | ELECTION_DATE=2020-11-03 9 | 10 | # Fill this in 11 | AP_API_KEY=xxx 12 | 13 | # Leave these as-is for local development 14 | # (you will need AWS creds in your environment) 15 | S3_BUCKET=voteamerica-enip-data 16 | S3_PREFIX=local 17 | CALLS_GSHEET_ID=xxx 18 | COMMENTS_GSHEET_ID=xxx 19 | GSHEET_API_CREDENTIALS_SSM_PATH=/shared/enip/gsheets_service_account 20 | -------------------------------------------------------------------------------- /enip_backend/enip_common/pg.py: -------------------------------------------------------------------------------- 1 | from contextlib import contextmanager 2 | 3 | import psycopg2 4 | import psycopg2.extras 5 | import psycopg2.pool 6 | 7 | from ..enip_common.config import POSTGRES_RO_URL, POSTGRES_URL 8 | 9 | 10 | @contextmanager 11 | def get_cursor(): 12 | with psycopg2.connect(POSTGRES_URL) as conn: 13 | with conn.cursor(cursor_factory=psycopg2.extras.NamedTupleCursor) as cursor: 14 | yield cursor 15 | 16 | 17 | @contextmanager 18 | def get_ro_cursor(): 19 | with psycopg2.connect(POSTGRES_RO_URL) as conn: 20 | with conn.cursor(cursor_factory=psycopg2.extras.NamedTupleCursor) as cursor: 21 | yield cursor 22 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | python_version=3.7 3 | 4 | [mypy-environs.*] 5 | ignore_missing_imports = True 6 | 7 | [mypy-psycopg2.*] 8 | ignore_missing_imports = True 9 | 10 | [mypy-elex.*] 11 | ignore_missing_imports = True 12 | 13 | [mypy-pydantic.*] 14 | ignore_missing_imports = True 15 | 16 | [mypy-humps.*] 17 | ignore_missing_imports = True 18 | 19 | [mypy-boto3.*] 20 | ignore_missing_imports = True 21 | 22 | [mypy-botocore.*] 23 | ignore_missing_imports = True 24 | 25 | [mypy-jsonschema.*] 26 | ignore_missing_imports = True 27 | 28 | [mypy-pygsheets.*] 29 | ignore_missing_imports = True 30 | 31 | [mypy-ddtrace.*] 32 | ignore_missing_imports = True 33 | 34 | [mypy-pytest.*] 35 | ignore_missing_imports = True 36 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | distro: bionic 2 | language: node_js 3 | node_js: 4 | - 14 5 | install: 6 | - sudo apt-get update 7 | - sudo apt-get install python3-pip python3-setuptools python3-wheel 8 | - python3 -m pip install --user pipenv 9 | - pipenv install --dev 10 | script: 11 | - pipenv run pytest 12 | 13 | env: 14 | global: 15 | - "POSTGRES_URL=postgres://postgres:postgres@localhost/postgres" 16 | - "INGEST_TEST_DATA=true" 17 | - "ELECTION_DATE=2020-11-03" 18 | - "AP_API_KEY=xxx" 19 | - "S3_BUCKET=voteamerica-enip-data" 20 | - "S3_PREFIX=local" 21 | - "CALLS_GSHEET_ID=xxx" 22 | - "COMMENTS_GSHEET_ID=xxx" 23 | - "GSHEET_API_CREDENTIALS_SSM_PATH=/shared/enip/gsheets_service_account" 24 | -------------------------------------------------------------------------------- /enip_backend/enip_common/config.py: -------------------------------------------------------------------------------- 1 | from environs import Env 2 | 3 | env = Env() 4 | env.read_env() 5 | 6 | POSTGRES_URL = env("POSTGRES_URL") 7 | POSTGRES_RO_URL = env("POSTGRES_RO_URL", POSTGRES_URL) 8 | AP_API_KEY = env("AP_API_KEY") 9 | INGEST_TEST_DATA = env.bool("INGEST_TEST_DATA") 10 | ELECTION_DATE = env("ELECTION_DATE") 11 | SENTRY_DSN = env("SENTRY_DSN", None) 12 | SENTRY_ENVIRONMENT = env("SENTRY_ENVIRONMENT", "unknown") 13 | S3_BUCKET = env("S3_BUCKET") 14 | S3_PREFIX = env("S3_PREFIX") 15 | HISTORICAL_START = env.datetime("HISTORICAL_START", "2020-10-01T00:00:00Z") 16 | CDN_URL = f"https://enip-data.voteamerica.com/{S3_PREFIX}/" 17 | 18 | GSHEET_API_CREDENTIALS_SSM_PATH = env("GSHEET_API_CREDENTIALS_SSM_PATH") 19 | CALLS_GSHEET_ID = env("CALLS_GSHEET_ID") 20 | COMMENTS_GSHEET_ID = env("COMMENTS_GSHEET_ID") 21 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "enip-backend", 3 | "version": "1.0.0", 4 | "license": "MIT", 5 | "description": "This is the backend for the [Election Night Integrity Project](https://2020.dataforprogress.org/), built by [Data for Progress](https://www.dataforprogress.org/) and [VoteAmerica](https://www.voteamerica.com/).", 6 | "dependencies": {}, 7 | "devDependencies": { 8 | "husky": "^4.3.0", 9 | "serverless": "^2.7.0", 10 | "serverless-plugin-datadog": "^2.5.0", 11 | "serverless-prune-plugin": "^1.4.3", 12 | "serverless-python-requirements": "^5.1.0" 13 | }, 14 | "scripts": { 15 | "sls": "AWS_SDK_LOAD_CONFIG=1 serverless" 16 | }, 17 | "repository": { 18 | "type": "git", 19 | "url": "git+https://github.com/vote/enip-backend.git" 20 | }, 21 | "husky": { 22 | "hooks": { 23 | "pre-commit": "pipenv run format" 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 VoteAmerica 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | name = "pypi" 3 | url = "https://pypi.org/simple" 4 | verify_ssl = true 5 | 6 | [dev-packages] 7 | ipython = "*" 8 | mypy = "*" 9 | black = "*" 10 | autoflake = "*" 11 | isort = "*" 12 | pytest = "*" 13 | pytest-mock = "*" 14 | pytest-cov = "*" 15 | 16 | [packages] 17 | elex = "*" 18 | psycopg2-binary = "*" 19 | environs = "*" 20 | sentry-sdk = "*" 21 | pydantic = "*" 22 | pyhumps = "*" 23 | boto3 = "==1.14.63" 24 | botocore = "==1.17.63" 25 | jsonschema = "*" 26 | ddtrace = "*" 27 | pygsheets = "*" 28 | pytz = "*" 29 | 30 | [requires] 31 | python_version = "3.7" 32 | 33 | [scripts] 34 | autoflake = "autoflake --remove-unused-variables --remove-all-unused-imports --ignore-init-module-imports -i --recursive enip_backend" 35 | isort = "isort --recursive enip_backend" 36 | black = "black enip_backend" 37 | mypy = "mypy enip_backend --strict-optional" 38 | format = "bash -c 'pipenv run autoflake && pipenv run isort && pipenv run black'" 39 | pytest = "pytest ./enip_backend/export" 40 | pytest_cov = "pytest ./enip_backend/export --cov enip_backend --cov-report xml:cov.xml" 41 | ci = "bash -c 'pipenv run mypy && pipenv run pytest'" 42 | 43 | [pipenv] 44 | allow_prereleases = true 45 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Election Night Integrity Project Backend 2 | 3 | This is the backend for the [Election Night Integrity Project](https://2020.dataforprogress.org/), built by [Data for Progress](https://www.dataforprogress.org/) and [VoteAmerica](https://www.voteamerica.com/). 4 | 5 | The job of the backend is to: 6 | - Ingest data from the AP Elections API, historical data sources, and live commentary spreadsheets and write it to a Postgres database 7 | - Export data from the Postgres database to populate static JSON files in S3 that are consumed by the frontend. 8 | 9 | ## Dev environment 10 | 11 | 1. To spin up a Postgres database for development, run `docker-compose up`. 12 | 2. Then, migrate your database with `scripts/migrate-db.sh`. You can connect with 13 | `scripts/psql.sh`. 14 | 3. Copy `.env.example` to `.env` and fill in your AP API key. 15 | 16 | ## Ingest 17 | 18 | The ingester is responsible for importing all of our data sources into Postgres. 19 | 20 | You can run the ingester with: `pipenv run python -m enip_backend.ingest.run`. 21 | 22 | ## Export 23 | 24 | The exporter is responsible for updates all the exports in S3 based on the 25 | latest data in Postgres. 26 | 27 | You can run the exporter with: `pipenv run python -m enip_backend.export.run` 28 | -------------------------------------------------------------------------------- /enip_backend/enip_common/s3.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os.path 4 | 5 | import boto3 6 | import botocore.config 7 | from botocore.exceptions import ClientError 8 | 9 | from .config import S3_BUCKET, S3_PREFIX 10 | 11 | client_config = botocore.config.Config(max_pool_connections=50,) 12 | 13 | s3 = boto3.client("s3", config=client_config) 14 | 15 | 16 | def read_json(path): 17 | try: 18 | response = s3.get_object(Bucket=S3_BUCKET, Key=os.path.join(S3_PREFIX, path)) 19 | except ClientError as ex: 20 | if ex.response["Error"]["Code"] == "NoSuchKey": 21 | logging.warning("No latest file") 22 | return None 23 | else: 24 | raise 25 | 26 | return json.loads(response["Body"].read()) 27 | 28 | 29 | def write_string(path, content, content_type, acl, cache_control): 30 | s3.put_object( 31 | Bucket=S3_BUCKET, 32 | Key=os.path.join(S3_PREFIX, path), 33 | Body=content.encode(), 34 | ContentType=content_type, 35 | ACL=acl, 36 | CacheControl=cache_control, 37 | ) 38 | 39 | 40 | def write_cacheable_json(path, content): 41 | write_string( 42 | path, 43 | content, 44 | content_type="application/json", 45 | acl="public-read", 46 | cache_control="max-age=86400", 47 | ) 48 | 49 | 50 | def write_noncacheable_json(path, content): 51 | write_string( 52 | path, 53 | content, 54 | content_type="application/json", 55 | acl="public-read", 56 | cache_control="no-store", 57 | ) 58 | -------------------------------------------------------------------------------- /enip_backend/export/bulk.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | from datetime import datetime, timezone 4 | 5 | from ..enip_common.pg import get_ro_cursor 6 | from .run import export_all_states, export_national 7 | 8 | # Bulk-exports a range of ingests for testing purposes. Prints out a JSON 9 | # blob describing the exports. 10 | START_TIME = datetime(2020, 10, 15, 16, 0, 0, tzinfo=timezone.utc) 11 | END_TIME = datetime(2020, 10, 15, 18, 0, 0, tzinfo=timezone.utc) 12 | 13 | 14 | def export_bulk(): 15 | ingests = [] 16 | with get_ro_cursor() as cursor: 17 | cursor.execute( 18 | "SELECT ingest_id, ingest_dt FROM ingest_run WHERE ingest_dt >= %s AND ingest_dt <= %s AND waypoint_30_dt IS NOT NULL", 19 | (START_TIME, END_TIME), 20 | ) 21 | 22 | ingests = [(res.ingest_id, res.ingest_dt) for res in cursor] 23 | 24 | logging.info(f"Running {len(ingests)} exports...") 25 | 26 | summary = [] 27 | for i, (ingest_id, ingest_dt) in enumerate(ingests): 28 | logging.info(f"[[[ INGEST {i+1} OF {len(ingests)} ]]]") 29 | 30 | summary.append( 31 | { 32 | "ingest_dt": ingest_dt.isoformat(), 33 | "exports": { 34 | "national": export_national( 35 | ingest_id, ingest_dt, ingest_dt.strftime("%Y%m%d%H%M%S") 36 | ), 37 | "states": export_all_states( 38 | ingest_id, ingest_dt, ingest_dt.strftime("%Y%m%d%H%M%S") 39 | ), 40 | }, 41 | } 42 | ) 43 | 44 | return summary 45 | 46 | 47 | if __name__ == "__main__": 48 | out_json = export_bulk() 49 | with open("./bulk.json", "w") as f: 50 | json.dump(out_json, f) 51 | -------------------------------------------------------------------------------- /enip_backend/enip_common/states.py: -------------------------------------------------------------------------------- 1 | STATES = { 2 | "AL", 3 | "AK", 4 | "AZ", 5 | "AR", 6 | "CA", 7 | "CO", 8 | "CT", 9 | "DC", 10 | "DE", 11 | "FL", 12 | "GA", 13 | "HI", 14 | "ID", 15 | "IL", 16 | "IN", 17 | "IA", 18 | "KS", 19 | "KY", 20 | "LA", 21 | "ME", 22 | "MD", 23 | "MA", 24 | "MI", 25 | "MN", 26 | "MS", 27 | "MO", 28 | "MT", 29 | "NE", 30 | "NV", 31 | "NH", 32 | "NJ", 33 | "NM", 34 | "NY", 35 | "NC", 36 | "ND", 37 | "OH", 38 | "OK", 39 | "OR", 40 | "PA", 41 | "RI", 42 | "SC", 43 | "SD", 44 | "TN", 45 | "TX", 46 | "UT", 47 | "VT", 48 | "VA", 49 | "WA", 50 | "WV", 51 | "WI", 52 | "WY", 53 | } 54 | 55 | DISTRICTS = {"ME-01", "ME-02", "NE-01", "NE-02", "NE-03"} 56 | 57 | DISTRICTS_BY_STATE = {"ME": {"ME-01", "ME-02"}, "NE": {"NE-01", "NE-02", "NE-03"}} 58 | 59 | SENATE_SPECIALS = {"GA-S"} 60 | 61 | SENATE_SPECIALS_BY_STATE = {"GA": {"GA-S"}} 62 | 63 | AT_LARGE_HOUSE_STATES = {"AK", "DE", "MT", "ND", "SD", "VT", "WY"} 64 | 65 | PRESIDENTIAL_REPORTING_UNITS = STATES | DISTRICTS 66 | ALL_REPORTING_UNITS = STATES | DISTRICTS | SENATE_SPECIALS 67 | 68 | SENATE_RACES = { 69 | "AL", 70 | "AK", 71 | "AZ", 72 | "AR", 73 | "CO", 74 | "DE", 75 | "GA", 76 | "GA-S", 77 | "ID", 78 | "IL", 79 | "IA", 80 | "KS", 81 | "KY", 82 | "LA", 83 | "ME", 84 | "MA", 85 | "MI", 86 | "MN", 87 | "MS", 88 | "MT", 89 | "NE", 90 | "NH", 91 | "NJ", 92 | "NM", 93 | "NC", 94 | "OK", 95 | "OR", 96 | "RI", 97 | "SC", 98 | "SD", 99 | "TN", 100 | "TX", 101 | "VA", 102 | "WV", 103 | "WY", 104 | } 105 | -------------------------------------------------------------------------------- /enip_backend/lambda_handlers.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from datetime import datetime, timezone 3 | 4 | import sentry_sdk 5 | from ddtrace import patch_all, tracer 6 | from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration 7 | 8 | from .calls_gsheet_sync.run import sync_calls_gsheet 9 | from .comments_gsheet_sync.run import sync_comments_gsheet 10 | from .enip_common import config 11 | from .export.run import export_all_states, export_national 12 | from .ingest.apapi import ingest_ap 13 | from .ingest.run import ingest_all 14 | 15 | logging.getLogger().setLevel(logging.INFO) 16 | patch_all() 17 | 18 | if config.SENTRY_DSN: 19 | sentry_sdk.init( 20 | config.SENTRY_DSN, 21 | environment=config.SENTRY_ENVIRONMENT, 22 | integrations=[AwsLambdaIntegration()], 23 | ) 24 | 25 | 26 | def run(event, context): 27 | with tracer.trace("enip.run_national"): 28 | with tracer.trace("enip.run_national.ingest"): 29 | ingest_id, ingest_dt, ingest_data = ingest_all() 30 | with tracer.trace("enip.run_national.export"): 31 | export_national( 32 | ingest_id, ingest_dt, ingest_dt.strftime("%Y%m%d%H%M%S"), ingest_data 33 | ) 34 | 35 | 36 | def run_states(event, context): 37 | with tracer.trace("enip.run_states"): 38 | with tracer.trace("enip.run_states.ingest"): 39 | ingest_dt = datetime.now(tz=timezone.utc) 40 | ap_data = ingest_ap( 41 | cursor=None, ingest_id=-1, save_to_db=False, return_levels={"county"} 42 | ) 43 | with tracer.trace("enip.run_states.export"): 44 | export_all_states(ap_data, ingest_dt) 45 | 46 | 47 | def run_sync_calls_gsheet(event, context): 48 | with tracer.trace("enip.run_sync_calls_gsheet"): 49 | sync_calls_gsheet() 50 | 51 | 52 | def run_sync_comments_gsheet(event, context): 53 | with tracer.trace("enip.run_sync_comments_gsheet"): 54 | sync_comments_gsheet() 55 | 56 | 57 | if __name__ == "__main__": 58 | run(None, None) 59 | # run_states(None, None) 60 | # run_sync_calls_gsheet(None, None) 61 | # run_sync_comments_gsheet(None, None) 62 | -------------------------------------------------------------------------------- /enip_backend/enip_common/gsheets.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import boto3 4 | import pygsheets 5 | from ddtrace import tracer 6 | from pygsheets.authorization import service_account 7 | 8 | from .config import GSHEET_API_CREDENTIALS_SSM_PATH 9 | 10 | 11 | @tracer.wrap("gsheets.get_gsheets_client", service="gsheets") 12 | def get_gsheets_client(): 13 | """Get a pygsheets client using service account credentials stored in SSM""" 14 | ssm_param = boto3.client("ssm").get_parameter( 15 | Name=GSHEET_API_CREDENTIALS_SSM_PATH, WithDecryption=True 16 | ) 17 | param_value = ssm_param.get("Parameter", {}).get("Value") 18 | credentials = service_account.Credentials.from_service_account_info( 19 | json.loads(param_value), 20 | scopes=("https://www.googleapis.com/auth/spreadsheets",), 21 | ) 22 | return pygsheets.authorize(custom_credentials=credentials) 23 | 24 | 25 | @tracer.wrap("gsheets.get_worksheet_data", service="gsheets") 26 | def get_worksheet_data(worksheet, data_range, expected_header=None): 27 | """Read a range of cells from a worksheet into a list of dictionaries 28 | treating the first row as a header. 29 | 30 | :param worksheet: a pygsheets Worksheet object 31 | :param data_range: cell range tuple, e.g. ('A', 'D'), ('A1', 'A17') 32 | :param expected_header: optional, if passed assert that the first row header 33 | matches the expected_header exactly 34 | :return: a list of rows as dictionaries, with the header as keys and 35 | pygsheet Cell objects as values 36 | """ 37 | sheet_data = worksheet.get_values(*data_range, returnas="cell") 38 | header = [c.value for c in sheet_data[0]] 39 | if expected_header and header != expected_header: 40 | raise Exception(f"Sheet does not have the expected header: {expected_header}") 41 | return [{header[i]: row[i] for i in range(len(header))} for row in sheet_data[1:]] 42 | 43 | 44 | @tracer.wrap("gsheets.update_cell", service="gsheets") 45 | def update_cell(cell, value): 46 | cell.set_value(value) 47 | 48 | 49 | @tracer.wrap("gsheets.worksheet_by_title", service="gsheets") 50 | def worksheet_by_title(sheet, title): 51 | return sheet.worksheet_by_title(title) 52 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Custom 2 | .env 3 | sample.csv 4 | bulk.json 5 | cov.xml 6 | 7 | # Byte-compiled / optimized / DLL files 8 | __pycache__/ 9 | *.py[cod] 10 | *$py.class 11 | 12 | # C extensions 13 | *.so 14 | 15 | # Distribution / packaging 16 | .Python 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | .eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | wheels/ 29 | pip-wheel-metadata/ 30 | share/python-wheels/ 31 | *.egg-info/ 32 | .installed.cfg 33 | *.egg 34 | MANIFEST 35 | 36 | # PyInstaller 37 | # Usually these files are written by a python script from a template 38 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 39 | *.manifest 40 | *.spec 41 | 42 | # Installer logs 43 | pip-log.txt 44 | pip-delete-this-directory.txt 45 | 46 | # Unit test / coverage reports 47 | htmlcov/ 48 | .tox/ 49 | .nox/ 50 | .coverage 51 | .coverage.* 52 | .cache 53 | nosetests.xml 54 | coverage.xml 55 | *.cover 56 | *.py,cover 57 | .hypothesis/ 58 | .pytest_cache/ 59 | 60 | # Translations 61 | *.mo 62 | *.pot 63 | 64 | # Django stuff: 65 | *.log 66 | local_settings.py 67 | db.sqlite3 68 | db.sqlite3-journal 69 | 70 | # Flask stuff: 71 | instance/ 72 | .webassets-cache 73 | 74 | # Scrapy stuff: 75 | .scrapy 76 | 77 | # Sphinx documentation 78 | docs/_build/ 79 | 80 | # PyBuilder 81 | target/ 82 | 83 | # Jupyter Notebook 84 | .ipynb_checkpoints 85 | 86 | # IPython 87 | profile_default/ 88 | ipython_config.py 89 | 90 | # pyenv 91 | .python-version 92 | 93 | # pipenv 94 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 95 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 96 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 97 | # install all needed dependencies. 98 | #Pipfile.lock 99 | 100 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 101 | __pypackages__/ 102 | 103 | # Celery stuff 104 | celerybeat-schedule 105 | celerybeat.pid 106 | 107 | # SageMath parsed files 108 | *.sage.py 109 | 110 | # Environments 111 | .env 112 | .venv 113 | env/ 114 | venv/ 115 | ENV/ 116 | env.bak/ 117 | venv.bak/ 118 | 119 | # Spyder project settings 120 | .spyderproject 121 | .spyproject 122 | 123 | # Rope project settings 124 | .ropeproject 125 | 126 | # mkdocs documentation 127 | /site 128 | 129 | # mypy 130 | .mypy_cache/ 131 | .dmypy.json 132 | dmypy.json 133 | 134 | # Pyre type checker 135 | .pyre/ 136 | 137 | # serverless 138 | .serverless/ 139 | node_modules/ 140 | 141 | # intellij 142 | .idea/ 143 | -------------------------------------------------------------------------------- /enip_backend/ingest/ingest_run.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from datetime import datetime, timedelta, timezone 3 | 4 | WAYPOINT_INTERVALS = [ 5 | ("waypoint_30_dt", timedelta(minutes=30)), 6 | ("waypoint_15_dt", timedelta(minutes=15)), 7 | ("waypoint_60_dt", timedelta(minutes=60)), 8 | ] 9 | 10 | 11 | def get_waypoint_for_dt(dt, interval): 12 | """ 13 | Rounds down to the nearest WAYPOINT_INTERVAL 14 | From: https://stackoverflow.com/a/50268328 15 | """ 16 | rounded = dt - (dt - datetime.min.replace(tzinfo=timezone.utc)) % interval 17 | return rounded 18 | 19 | 20 | def insert_ingest_run(cursor): 21 | """ 22 | Inserts a new record into ingest_run. 23 | Returns (ingest_id, ingest_dt, waypoints) 24 | Where waypoints is a list of waypoint columns (e.g. waypoint_15_dt) if this 25 | ingest corresponds to one or more waypoints 26 | """ 27 | now = datetime.now(tz=timezone.utc) 28 | 29 | # Find the most recent ingest 30 | max_sql = ", ".join( 31 | [ 32 | f"MAX({col_name}) AS waypoint{int(delta.total_seconds())}" 33 | for col_name, delta in WAYPOINT_INTERVALS 34 | ] 35 | ) 36 | 37 | cursor.execute(f"SELECT {max_sql} FROM ingest_run") 38 | 39 | waypoint_dts = [] 40 | waypoint_names = [] 41 | for (last_waypoint, (col_name, interval)) in zip( 42 | cursor.fetchone(), WAYPOINT_INTERVALS 43 | ): 44 | logging.info(f"{col_name}") 45 | logging.info(f" Previous waypoint: {last_waypoint}") 46 | 47 | # Determine if this run is a new waypoint -- if there are no runs for 48 | # the current waypoint 49 | current_waypoint = get_waypoint_for_dt(now, interval) 50 | logging.info(f" Current waypoint: {current_waypoint}") 51 | 52 | if not last_waypoint or (current_waypoint > last_waypoint): 53 | logging.info( 54 | f" -> This ingest run will be a new waypoint for {col_name}" 55 | ) 56 | waypoint_dt = current_waypoint 57 | waypoint_names.append(col_name) 58 | else: 59 | logging.info(f" -> This ingest run is not a new waypoint for {col_name}") 60 | waypoint_dt = None 61 | 62 | waypoint_dts.append(waypoint_dt) 63 | 64 | # Insert a record for this ingest 65 | waypoint_cols = ", ".join(col_name for col_name, delta in WAYPOINT_INTERVALS) 66 | waypoint_placeholders = ", ".join("%s" for col_name, delta in WAYPOINT_INTERVALS) 67 | cursor.execute( 68 | f"INSERT INTO ingest_run (ingest_dt, {waypoint_cols}) VALUES (now(), {waypoint_placeholders}) RETURNING ingest_id, ingest_dt", 69 | waypoint_dts, 70 | ) 71 | res = cursor.fetchone() 72 | return res[0], res[1], waypoint_names 73 | -------------------------------------------------------------------------------- /enip_backend/ingest/apapi.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import io 3 | import logging 4 | 5 | from ddtrace import tracer 6 | from elex.api.models import Election 7 | 8 | from ..enip_common.config import AP_API_KEY, ELECTION_DATE, INGEST_TEST_DATA 9 | from ..export.helpers import sqlrecord_from_dict 10 | 11 | OFFICE_IDS = ["P", "S", "H"] 12 | 13 | 14 | @tracer.wrap("enip.ingest.ingest_ap") 15 | def ingest_ap( 16 | cursor, ingest_id, save_to_db, return_levels={"national", "state", "district"} 17 | ): 18 | with tracer.trace("enip.ingest.ingest_ap.read_data"): 19 | # Make the API request. We need to request both reporting-unit-level results, 20 | # which gives us everything except NE/ME congressional districts, and 21 | # district results for those few results. 22 | election_ru = Election( 23 | testresults=INGEST_TEST_DATA, 24 | resultslevel="ru", 25 | officeids=OFFICE_IDS, 26 | setzerocounts=False, 27 | electiondate=ELECTION_DATE, 28 | api_key=AP_API_KEY, 29 | ) 30 | election_district = Election( 31 | testresults=INGEST_TEST_DATA, 32 | resultslevel="district", 33 | officeids=OFFICE_IDS, 34 | setzerocounts=False, 35 | electiondate=ELECTION_DATE, 36 | api_key=AP_API_KEY, 37 | ) 38 | 39 | # Convert the AP results to an in-memory CSV (much faster than a bunch of inserts) 40 | csv_file = io.StringIO() 41 | writer = csv.writer(csv_file) 42 | n_rows = 0 43 | column_headers = None 44 | return_data = [] 45 | 46 | def process_election_results(results, filter_levels=None): 47 | nonlocal n_rows 48 | nonlocal column_headers 49 | 50 | for obj in results: 51 | row = obj.serialize() 52 | row["ingest_id"] = ingest_id 53 | row["elex_id"] = row["id"] 54 | del row["id"] 55 | 56 | if column_headers is None: 57 | column_headers = row.keys() 58 | writer.writerow(column_headers) 59 | 60 | if filter_levels and row["level"] not in filter_levels: 61 | continue 62 | 63 | if save_to_db: 64 | writer.writerow(row.values()) 65 | 66 | if row["level"] in return_levels: 67 | return_data.append(sqlrecord_from_dict(row)) 68 | n_rows += 1 69 | 70 | process_election_results(election_ru.results) 71 | process_election_results(election_district.results, ["district"]) 72 | 73 | logging.info(f"Got {n_rows} rows from the AP") 74 | 75 | if save_to_db: 76 | with tracer.trace("enip.ingest.ingest_ap.save_to_db"): 77 | # Run the COPY command to insert the rows 78 | logging.info(f"Writing {n_rows} rows to Postgres...") 79 | 80 | csv_file.seek(0) 81 | 82 | cursor.copy_expert( 83 | sql=f"COPY ap_result ({','.join(column_headers)}) FROM stdin WITH DELIMITER AS ',' CSV HEADER;", 84 | file=csv_file, 85 | ) 86 | 87 | logging.info( 88 | f"Done with ingest of AP data! Returning {len(return_data)} data points for national export" 89 | ) 90 | 91 | return return_data 92 | -------------------------------------------------------------------------------- /enip_backend/comments_gsheet_sync/run.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from datetime import datetime 3 | from itertools import chain 4 | 5 | import psycopg2 6 | import sentry_sdk 7 | from pytz import timezone 8 | 9 | from ..enip_common.config import COMMENTS_GSHEET_ID, POSTGRES_URL 10 | from ..enip_common.gsheets import ( 11 | get_gsheets_client, 12 | get_worksheet_data, 13 | worksheet_by_title, 14 | ) 15 | from ..enip_common.states import SENATE_RACES 16 | 17 | RANGE = ("A", "H") 18 | 19 | TIMESTAMP_HEADER = "Timestamp" 20 | NAME_HEADER = "Your name (e.g. Luke Kastel)" 21 | OFFICE_HEADER = "Office (P for President, S for Senate, H for House)" 22 | PRESIDENT_HEADER = "Choose presidential electoral vote unit (56 choices: 50 states + DC + ME-01, ME-02, NE-01, NE-02, NE-03)" 23 | SENATE_HEADER = "Choose Senate race (35 choices: 34 states + Georgia special)" 24 | HOUSE_HEADER = "Choose House district" 25 | TITLE_HEADER = "Title" 26 | BODY_HEADER = "Body" 27 | 28 | WORKSHEET_TITLE = "Form Responses 1" 29 | 30 | TS_TZ = timezone("US/Eastern") 31 | TS_FMT = "%m/%d/%Y %H:%M:%S" 32 | 33 | 34 | def _get(row, header, validate_truthy=True): 35 | val = row[header].value 36 | if validate_truthy and not val: 37 | raise Exception( 38 | f"Missing value for {header} in row {row.get(TIMESTAMP_HEADER)}" 39 | ) 40 | return val 41 | 42 | 43 | def _map_sheet_row_to_db(r): 44 | try: 45 | office_and_race_list = [] 46 | office = _get(r, OFFICE_HEADER) 47 | if office == "P": 48 | office_and_race_list.append((office, _get(r, PRESIDENT_HEADER))) 49 | elif office == "S": 50 | office_and_race_list.append((office, _get(r, SENATE_HEADER))) 51 | elif office == "PS": 52 | state = _get(r, PRESIDENT_HEADER) 53 | office_and_race_list.append(("P", state)) 54 | if state in SENATE_RACES: 55 | office_and_race_list.append(("S", state)) 56 | if state == "GA": 57 | office_and_race_list.append(("S", "GA-S")) 58 | elif office == "H": 59 | office_and_race_list.append((office, _get(r, HOUSE_HEADER))) 60 | elif office == "N": 61 | office_and_race_list.append((office, None)) 62 | else: 63 | logging.error("Skipping. Office id must be P, S, PS, N or H") 64 | 65 | ts = datetime.strptime(_get(r, TIMESTAMP_HEADER), TS_FMT) 66 | for (office_id, race) in office_and_race_list: 67 | yield { 68 | "ts": TS_TZ.localize(ts), 69 | "submitted_by": _get(r, NAME_HEADER), 70 | "office_id": office_id, 71 | "race": race, 72 | "title": _get(r, TITLE_HEADER), 73 | "body": _get(r, BODY_HEADER), 74 | } 75 | except Exception as err: 76 | sentry_sdk.capture_exception(err) 77 | logging.error("Skipping. Exception: {}".format(err)) 78 | 79 | 80 | def sync_comments_gsheet(): 81 | client = get_gsheets_client() 82 | sheet = client.open_by_key(COMMENTS_GSHEET_ID) 83 | data = get_worksheet_data(worksheet_by_title(sheet, WORKSHEET_TITLE), RANGE) 84 | logging.info("Syncing comments gsheet") 85 | # TODO: this fails fast if any of the rows is invalid, we might want to skip instead 86 | db_rows = chain(*[_map_sheet_row_to_db(row) for row in data]) 87 | 88 | insert_stmt = """ 89 | INSERT INTO comments (ts, submitted_by, office_id, race, title, body) 90 | VALUES (%(ts)s, %(submitted_by)s, %(office_id)s, %(race)s, %(title)s, %(body)s) 91 | """ 92 | with psycopg2.connect(POSTGRES_URL) as conn, conn.cursor() as cursor: 93 | cursor.execute("DELETE FROM comments") 94 | cursor.executemany(insert_stmt, list(db_rows)) 95 | logging.info("Comments sync complete") 96 | 97 | 98 | if __name__ == "__main__": 99 | sync_comments_gsheet() 100 | -------------------------------------------------------------------------------- /enip_backend/ingest/run.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from ..enip_common.pg import get_cursor 4 | from ..enip_common.states import PRESIDENTIAL_REPORTING_UNITS, SENATE_RACES 5 | from .apapi import ingest_ap 6 | from .ingest_run import insert_ingest_run 7 | 8 | SAVE_WAYPOINT = "waypoint_15_dt" 9 | 10 | 11 | def _calls_update_stmt(table): 12 | return f""" 13 | INSERT INTO {table} (state, ap_call, ap_called_at) 14 | VALUES (%s, %s, NOW()) 15 | ON CONFLICT (state) DO UPDATE 16 | SET (ap_call, ap_called_at) = ( 17 | EXCLUDED.ap_call, 18 | CASE 19 | WHEN EXCLUDED.ap_call IS NULL THEN NULL 20 | ELSE NOW() 21 | END 22 | ) 23 | WHERE {table}.ap_call IS DISTINCT FROM EXCLUDED.ap_call OR EXCLUDED.ap_call IS NULL 24 | """ 25 | 26 | 27 | def update_senate_calls(cursor, ingest_data): 28 | def extract_state(record): 29 | if record.statepostal == "GA" and record.seatnum == 2: 30 | return "GA-S" 31 | return record.statepostal 32 | 33 | winners = {state: None for state in SENATE_RACES} 34 | for record in ingest_data: 35 | if record.officeid == "S" and record.level == "state" and record.winner: 36 | winners[extract_state(record)] = record.party 37 | 38 | rows = [(k, v) for k, v in winners.items()] 39 | rows.sort(key=lambda tup: tup[0]) 40 | cursor.executemany(_calls_update_stmt("senate_calls"), rows) 41 | 42 | 43 | def update_president_calls(cursor, ingest_data): 44 | def extract_state(record): 45 | if record.level == "district": 46 | if record.reportingunitname == "At Large": 47 | return record.statepostal 48 | elif record.reportingunitname == "District 1": 49 | return f"{record.statepostal}-01" 50 | elif record.reportingunitname == "District 2": 51 | return f"{record.statepostal}-02" 52 | elif record.reportingunitname == "District 3": 53 | return f"{record.statepostal}-03" 54 | 55 | return record.statepostal 56 | 57 | winners = {state: None for state in PRESIDENTIAL_REPORTING_UNITS} 58 | for record in ingest_data: 59 | # For state results, look at the "winner" property to determine who won 60 | if record.officeid == "P" and record.level == "state" and record.winner: 61 | winners[extract_state(record)] = record.party 62 | 63 | # For congressional districts, the AP has a bug where the "winner" 64 | # property reflects the at-large winner of the state rather than the 65 | # district winner. The electwon property correctly reflects the 66 | # district winner. 67 | if ( 68 | record.officeid == "P" 69 | and record.level == "district" 70 | and (record.electwon > 0) 71 | ): 72 | winners[extract_state(record)] = record.party 73 | 74 | rows = [(k, v) for k, v in winners.items()] 75 | rows.sort(key=lambda tup: tup[0]) 76 | cursor.executemany(_calls_update_stmt("president_calls"), rows) 77 | 78 | 79 | def ingest_all(force_save=False): 80 | with get_cursor() as cursor: 81 | # Create a record for this ingest run 82 | ingest_id, ingest_dt, waypoint_names = insert_ingest_run(cursor) 83 | logging.info(f"Ingest ID: {ingest_id}") 84 | 85 | # Fetch the AP results 86 | save_to_db = force_save or ("waypoint_15_dt" in waypoint_names) 87 | ingest_data = ingest_ap(cursor, ingest_id, save_to_db=save_to_db) 88 | # keep update order the same as calls_gsheet_run: 89 | # Senate then President, sorted by state 90 | logging.info("Updating senate calls in db") 91 | update_senate_calls(cursor, ingest_data) 92 | logging.info("Updating president calls in db") 93 | update_president_calls(cursor, ingest_data) 94 | logging.info("Comitting...") 95 | 96 | logging.info(f"All done! Completed ingest {ingest_id} at {ingest_dt}") 97 | return ingest_id, ingest_dt, ingest_data 98 | 99 | 100 | if __name__ == "__main__": 101 | ingest_all(force_save=True) 102 | -------------------------------------------------------------------------------- /serverless.yml: -------------------------------------------------------------------------------- 1 | service: enip-backend 2 | 3 | custom: 4 | stage: ${opt:stage, 'staging'} 5 | pythonRequirements: 6 | dockerizePip: true 7 | slim: false # jsonschema package fails with true: https://github.com/Julian/jsonschema/issues/628#issuecomment-564456058 8 | useDownloadCache: false 9 | noDeploy: [ 10 | 'boto3', 11 | 'botocore', 12 | 'docutils', 13 | 'jmespath', 14 | 'python-dateutil', 15 | 's3transfer', 16 | 'six', 17 | 'pip', 18 | 'setuptools' 19 | ] 20 | 21 | datadog: 22 | forwarder: "${ssm:/shared/enip/datadog_forwarder_arn~true}" 23 | flushMetricsToLogs: true 24 | addLayers: true 25 | logLevel: 'INFO' 26 | enableDDTracing: true 27 | 28 | prune: 29 | automatic: true 30 | number: 3 31 | 32 | schedule: 33 | staging: 34 | run: [] 35 | run_states: [] 36 | run_sync_calls_gsheet: [] 37 | run_sync_comments_gsheet: [] 38 | prod: 39 | run: 40 | - schedule: "rate(5 minutes)" 41 | run_states: 42 | - schedule: "rate(15 minutes)" 43 | run_sync_calls_gsheet: 44 | - schedule: "rate(5 minutes)" 45 | run_sync_comments_gsheet: 46 | - schedule: "rate(5 minutes)" 47 | 48 | provider: 49 | name: aws 50 | runtime: python3.7 51 | region: us-west-2 52 | vpc: 53 | subnetIds: 54 | - "${ssm:/vpc/subnets/private/id/0~true}" 55 | - "${ssm:/vpc/subnets/private/id/1~true}" 56 | - "${ssm:/vpc/subnets/private/id/2~true}" 57 | securityGroupIds: 58 | - "${ssm:/vpc/security-group/default/id~true}" 59 | tags: 60 | env: ${self:custom.stage} 61 | 62 | environment: 63 | SENTRY_DSN: ${ssm:/shared/enip/sentry_dsn~true} 64 | SENTRY_ENVIRONMENT: ${self:custom.stage} 65 | POSTGRES_URL: "${ssm:/${self:custom.stage}/enip/db/url~true}" 66 | POSTGRES_RO_URL: "${ssm:/${self:custom.stage}/enip/db/url_ro~true}" 67 | INGEST_TEST_DATA: "false" 68 | ELECTION_DATE: "2020-11-03" 69 | AP_API_KEY: ${ssm:/shared/enip/ap_api_key~true} 70 | S3_BUCKET: voteamerica-enip-data 71 | S3_PREFIX: ${self:custom.stage} 72 | GSHEET_API_CREDENTIALS_SSM_PATH: /shared/enip/gsheets_service_account 73 | CALLS_GSHEET_ID: ${ssm:/${self:custom.stage}/enip/calls_gsheet_id~true} 74 | COMMENTS_GSHEET_ID: ${ssm:/${self:custom.stage}/enip/comments_gsheet_id~true} 75 | HISTORICAL_START: "2020-11-03T23:00:00Z" 76 | memorySize: 3008 77 | iamRoleStatements: 78 | - Effect: 'Allow' 79 | Action: 80 | - 's3:*' 81 | Resource: 'arn:aws:s3:::voteamerica-enip-data' 82 | - Effect: 'Allow' 83 | Action: 84 | - 's3:*' 85 | Resource: 'arn:aws:s3:::voteamerica-enip-data/*' 86 | - Effect: 'Allow' 87 | Action: 88 | - 'ssm:GetParameter' 89 | Resource: 'arn:aws:ssm:us-west-2:*:parameter/shared/enip/gsheets_service_account' 90 | - Effect: 'Allow' 91 | Action: 92 | - 'kms:Decrypt' 93 | Resource: 'arn:aws:kms:us-west-2:*:key/facb0cd4-8df6-4455-9b34-27849f2b3e3f' 94 | 95 | plugins: 96 | - serverless-python-requirements 97 | - serverless-prune-plugin 98 | - serverless-plugin-datadog 99 | 100 | functions: 101 | run: 102 | handler: enip_backend.lambda_handlers.run 103 | # events: ${self:custom.schedule.${self:custom.stage}.run} 104 | timeout: 170 105 | maximumRetryAttempts: 0 106 | 107 | run_states: 108 | handler: enip_backend.lambda_handlers.run_states 109 | # events: ${self:custom.schedule.${self:custom.stage}.run_states} 110 | 111 | # Time out in a little less than 3 minutes so if we're overloaded we don't 112 | # end up stacking multiple invocations 113 | timeout: 850 114 | maximumRetryAttempts: 0 115 | run_sync_calls_gsheet: 116 | handler: enip_backend.lambda_handlers.run_sync_calls_gsheet 117 | # events: ${self:custom.schedule.${self:custom.stage}.run_sync_calls_gsheet} 118 | timeout: 170 119 | run_sync_comments_gsheet: 120 | handler: enip_backend.lambda_handlers.run_sync_comments_gsheet 121 | # events: ${self:custom.schedule.${self:custom.stage}.run_sync_comments_gsheet} 122 | timeout: 170 123 | 124 | package: 125 | exclude: 126 | - 'node_modules/**' 127 | - '.vscode/**' 128 | - '.mypy_cache/**' 129 | - 'package.json' 130 | - 'yarn.lock' 131 | -------------------------------------------------------------------------------- /enip_backend/export/state.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from typing import Any, Iterable, List 3 | 4 | from ddtrace import tracer 5 | 6 | from ..enip_common.states import AT_LARGE_HOUSE_STATES 7 | from . import structs 8 | from .helpers import ( 9 | HistoricalResults, 10 | SQLRecord, 11 | handle_candidate_results, 12 | load_election_results, 13 | load_historicals, 14 | ) 15 | 16 | 17 | class StateDataExporter: 18 | data: structs.StateData 19 | 20 | def __init__(self, ingest_run_dt: datetime, statecode: str): 21 | self.ingest_run_dt = ingest_run_dt 22 | self.historical_counts: HistoricalResults = {} 23 | self.data = structs.StateData() 24 | 25 | self.state = statecode 26 | 27 | def record_county_presidential_result(self, record: SQLRecord) -> None: 28 | """ 29 | Records a "county"-level presidential result. 30 | """ 31 | # Initialize the county 32 | county = record.fipscode 33 | if county not in self.data.counties: 34 | self.data.counties[county] = structs.County() 35 | 36 | # Add the results from this record 37 | handle_candidate_results( 38 | self.data.counties[county].P, 39 | structs.StateSummaryCandidateNamed, 40 | record, 41 | self.historical_counts, 42 | ) 43 | 44 | def record_county_senate_result(self, record: SQLRecord) -> None: 45 | """ 46 | Records a "county"-level senate result. 47 | """ 48 | state = record.statepostal 49 | if state == "GA" and record.seatnum == 2: 50 | # Georgia senate special election 51 | state = "GA-S" 52 | 53 | # Initialize the county 54 | county = record.fipscode 55 | if county not in self.data.counties: 56 | self.data.counties[county] = structs.County() 57 | 58 | # Initialize the senate result 59 | if state not in self.data.counties[county].S: 60 | self.data.counties[county].S[state] = structs.CountyCongressionalResult() 61 | 62 | # Add the results from this record 63 | handle_candidate_results( 64 | self.data.counties[county].S[state], 65 | structs.StateSummaryCandidateNamed, 66 | record, 67 | self.historical_counts, 68 | ) 69 | 70 | def record_county_house_result(self, record: SQLRecord) -> None: 71 | """ 72 | Records a "county"-level house result. 73 | """ 74 | state = record.statepostal 75 | seat = str(record.seatnum).zfill(2) 76 | if state in AT_LARGE_HOUSE_STATES: 77 | seat = "AL" 78 | 79 | # Initialize the county 80 | county = record.fipscode 81 | if county not in self.data.counties: 82 | self.data.counties[county] = structs.County() 83 | 84 | # Initialize the house result 85 | if seat not in self.data.counties[county].H: 86 | self.data.counties[county].H[seat] = structs.CountyCongressionalResult() 87 | 88 | # Add the results from this record 89 | handle_candidate_results( 90 | self.data.counties[county].H[seat], 91 | structs.StateSummaryCandidateNamed, 92 | record, 93 | self.historical_counts, 94 | ) 95 | 96 | def run_export(self, preloaded_results: Iterable[SQLRecord]) -> structs.StateData: 97 | self.data = structs.StateData() 98 | 99 | sql_filter = "level = 'county' AND statepostal = %s" 100 | filter_params: List[Any] = [self.state] 101 | 102 | with tracer.trace("enip.export.state.historicals"): 103 | self.historical_counts = load_historicals( 104 | self.ingest_run_dt, sql_filter, filter_params 105 | ) 106 | 107 | def handle_record(record): 108 | if record.officeid == "P": 109 | self.record_county_presidential_result(record) 110 | elif record.officeid == "S": 111 | self.record_county_senate_result(record) 112 | elif record.officeid == "H": 113 | self.record_county_house_result(record) 114 | else: 115 | raise RuntimeError( 116 | f"Uncategorizable result: {record.elex_id} {record.level} {record.officeid}" 117 | ) 118 | 119 | for record in preloaded_results: 120 | if record.statepostal == self.state: 121 | handle_record(record) 122 | 123 | return self.data 124 | -------------------------------------------------------------------------------- /enip_backend/calls_gsheet_sync/run.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import psycopg2 4 | from dateutil.parser import parse as parse_date 5 | from pytz import timezone 6 | 7 | from ..enip_common.config import CALLS_GSHEET_ID, POSTGRES_URL 8 | from ..enip_common.gsheets import ( 9 | get_gsheets_client, 10 | get_worksheet_data, 11 | update_cell, 12 | worksheet_by_title, 13 | ) 14 | 15 | DATA_RANGE = ("A", "D") 16 | EXPECTED_DATA_HEADER = ["State", "AP Call", "AP Called At (ET)", "Published?"] 17 | 18 | PUBLISH_DEFAULT_RANGE = ("F1", "G1") 19 | 20 | CALLED_AT_TZ = timezone("US/Eastern") 21 | CALLED_AT_FMT = "%m/%d/%Y %H:%M:%S" 22 | 23 | 24 | def _update_calls_sheet_from_db(cursor, table, worksheet): 25 | cursor.execute(f"SELECT state, ap_call, ap_called_at FROM {table}") 26 | db_calls_lookup = {r[0]: {"call": r[1], "called_at": r[2]} for r in cursor} 27 | sheet_data = get_worksheet_data(worksheet, DATA_RANGE, EXPECTED_DATA_HEADER) 28 | # Selectively update cells that don't have the same api calls from the db 29 | # Note: we intentionally don't insert new rows into the sheet for states 30 | # that exist in the db but not the sheet. The sheet will need to be seeded 31 | # with the states we want to report on. 32 | for row in sheet_data: 33 | state = row["State"].value 34 | db_call = db_calls_lookup.get(state) 35 | if db_call: 36 | call = db_call["call"] or "" 37 | ap_call_cell = row["AP Call"] 38 | if ap_call_cell.value != call: 39 | logging.info(f"Updating AP Call {state}: {call}") 40 | update_cell(ap_call_cell, call or "") 41 | 42 | called_at_fmt = ( 43 | db_call["called_at"].astimezone(CALLED_AT_TZ).strftime(CALLED_AT_FMT) 44 | if db_call["called_at"] 45 | else "" 46 | ) 47 | ap_called_at_cell = row["AP Called At (ET)"] 48 | 49 | if ( 50 | ap_called_at_cell.value 51 | and called_at_fmt 52 | and parse_date(ap_called_at_cell.value) == parse_date(called_at_fmt) 53 | ): 54 | # We compare parsed dates so we're not sensitive to Google Sheets date 55 | # formatting weirdness 56 | pass 57 | elif ap_called_at_cell.value != called_at_fmt: 58 | logging.info(f"Updating AP Called At {state}: {called_at_fmt}") 59 | update_cell(ap_called_at_cell, called_at_fmt) 60 | 61 | 62 | def _update_db_published_from_sheet(cursor, table, worksheet): 63 | publish_default_cells = worksheet.get_values(*PUBLISH_DEFAULT_RANGE)[0] 64 | if publish_default_cells[0] != "Default": 65 | raise Exception( 66 | "Worksheet {} does not match the expected format".format(worksheet.title) 67 | ) 68 | default_is_publish = publish_default_cells[1] == "Publish" 69 | 70 | def handle_published(row): 71 | value = row["Published?"].value 72 | if value == "Default": 73 | return default_is_publish 74 | return value == "Yes" 75 | 76 | sheet_data = get_worksheet_data(worksheet, DATA_RANGE, EXPECTED_DATA_HEADER) 77 | rows = [ 78 | {"state": row["State"].value, "published": handle_published(row)} 79 | for row in sheet_data 80 | ] 81 | rows.sort(key=lambda r: r["state"]) 82 | statement = f""" 83 | UPDATE {table} 84 | SET published = %(published)s 85 | WHERE state = %(state)s 86 | """ 87 | cursor.executemany(statement, rows) 88 | 89 | 90 | def sync_calls_gsheet(): 91 | client = get_gsheets_client() 92 | sheet = client.open_by_key(CALLS_GSHEET_ID) 93 | senate_sheet = worksheet_by_title(sheet, "Senate Calls") 94 | president_sheet = worksheet_by_title(sheet, "President Calls") 95 | with psycopg2.connect(POSTGRES_URL) as conn, conn.cursor() as cur: 96 | logging.info("Syncing calls from the db to google sheets") 97 | _update_calls_sheet_from_db(cur, "senate_calls", senate_sheet) 98 | _update_calls_sheet_from_db(cur, "president_calls", president_sheet) 99 | logging.info("Syncing publish settings from sheets to the db") 100 | # Keep update order the same as ingest: 101 | # Senate then President, sorted by state 102 | _update_db_published_from_sheet(cur, "senate_calls", senate_sheet) 103 | _update_db_published_from_sheet(cur, "president_calls", president_sheet) 104 | logging.info("Sync complete") 105 | 106 | 107 | if __name__ == "__main__": 108 | sync_calls_gsheet() 109 | -------------------------------------------------------------------------------- /enip_backend/export/schemas/state.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$id": "https://schema.voteamerica.com/enip/state.schema.json", 3 | "$schema": "http://json-schema.org/draft-07/schema#", 4 | "title": "ENIP State Data Schema", 5 | "type": "object", 6 | "definitions": { 7 | "county_candidate_named": { 8 | "$id": "#county_candidate_named", 9 | "type": "object", 10 | "additionalProperties": false, 11 | "required": [ 12 | "firstName", 13 | "lastName", 14 | "popVote", 15 | "popPct", 16 | "popVoteHistory" 17 | ], 18 | "properties": { 19 | "firstName": {"type": "string"}, 20 | "lastName": {"type": "string"}, 21 | "popVote": {"type": "integer"}, 22 | "popPct": {"type": "number"}, 23 | "popVoteHistory": { 24 | "type": "object", 25 | "propertyNames": { 26 | "type": "string", 27 | "format": "datetime" 28 | }, 29 | "additionalProperties": { 30 | "type": "integer" 31 | } 32 | } 33 | } 34 | }, 35 | "county_candidate_unnamed": { 36 | "$id": "#county_candidate_unnamed", 37 | "type": "object", 38 | "additionalProperties": false, 39 | "required": [ 40 | "popVote", 41 | "popPct", 42 | "popVoteHistory" 43 | ], 44 | "properties": { 45 | "popVote": {"type": "integer"}, 46 | "popPct": {"type": "number"}, 47 | "popVoteHistory": { 48 | "type": "object", 49 | "propertyNames": { 50 | "type": "string", 51 | "format": "datetime" 52 | }, 53 | "additionalProperties": { 54 | "type": "integer" 55 | } 56 | } 57 | } 58 | }, 59 | "county_congressional_result": { 60 | "$id": "#county_congressional_result", 61 | "type": "object", 62 | "additionalProperties": false, 63 | "required": ["dem", "gop", "oth", "multipleDem", "multipleGop"], 64 | "properties": { 65 | "dem": { 66 | "oneOf": [ 67 | {"type": "null" }, 68 | {"$ref": "#/definitions/county_candidate_named" } 69 | ] 70 | }, 71 | "gop": { 72 | "oneOf": [ 73 | {"type": "null" }, 74 | {"$ref": "#/definitions/county_candidate_named" } 75 | ] 76 | }, 77 | "oth": {"$ref": "#/definitions/county_candidate_unnamed" }, 78 | "multipleDem": {"type": "boolean"}, 79 | "multipleGop": {"type": "boolean"} 80 | } 81 | }, 82 | "county_presidential_result": { 83 | "$id": "#county_presidential_result", 84 | "type": "object", 85 | "additionalProperties": false, 86 | "required": ["dem", "gop", "oth"], 87 | "properties": { 88 | "dem": { 89 | "oneOf": [ 90 | {"type": "null" }, 91 | {"$ref": "#/definitions/county_candidate_named" } 92 | ] 93 | }, 94 | "gop": { 95 | "oneOf": [ 96 | {"type": "null" }, 97 | {"$ref": "#/definitions/county_candidate_named" } 98 | ] 99 | }, 100 | "oth": {"$ref": "#/definitions/county_candidate_unnamed" } 101 | } 102 | }, 103 | "county_senate_results": { 104 | "$id": "#county_senate_results", 105 | "type": "object", 106 | "additionalProperties": { 107 | "$ref": "#/definitions/county_congressional_result" 108 | }, 109 | "propertyNames": { 110 | "pattern": "^[A-Z]{2}(-S)?$" 111 | } 112 | }, 113 | "county_house_results": { 114 | "$id": "#county_house_results", 115 | "type": "object", 116 | "additionalProperties": { 117 | "$ref": "#/definitions/county_congressional_result" 118 | }, 119 | "propertyNames": { 120 | "pattern": "^(\\d\\d)|AL$" 121 | } 122 | }, 123 | "county": { 124 | "$id": "#county", 125 | "type": "object", 126 | "additionalProperties": false, 127 | "required": ["P", "H", "S"], 128 | "properties": { 129 | "P": { 130 | "$ref": "#/definitions/county_presidential_result" 131 | }, 132 | "S": { 133 | "oneOf": [ 134 | {"type": "null"}, 135 | {"$ref": "#/definitions/county_senate_results"} 136 | ] 137 | }, 138 | "H": { 139 | "$ref": "#/definitions/county_house_results" 140 | } 141 | } 142 | } 143 | }, 144 | "properties": { 145 | "counties": { 146 | "type": "object", 147 | "additionalProperties": { 148 | "$ref": "#/definitions/county" 149 | }, 150 | "propertyNames": { 151 | "pattern": "^\\d{5}$" 152 | } 153 | } 154 | }, 155 | "required": ["counties"], 156 | "additionalProperties": false 157 | } 158 | -------------------------------------------------------------------------------- /db/init.sql: -------------------------------------------------------------------------------- 1 | -- This file initializes the SQL schema. 2 | -- 3 | -- It is intended to serve as a migration that brings the database to the 4 | -- latest schema, too: when making changes, don't edit the existing code but 5 | -- rather add additional ALTER TABLE, CREATE TABLE, etc. statements to the 6 | -- bottom of this file and use IF NOT EXISTS to make it idempotent. 7 | 8 | -- ingest_runs stores one row for each time we run the ingester. 9 | CREATE TABLE IF NOT EXISTS ingest_run ( 10 | -- A unique ID for the ingestion run 11 | ingest_id SERIAL PRIMARY KEY, 12 | -- Timestamp the ingestion started at 13 | ingest_dt TIMESTAMPTZ NOT NULL, 14 | -- We provide 30-minute "waypoints" of historical data to the frontend. We 15 | -- tag the *first* ingest_run of each 30-minute interval with a waypoint_dt 16 | -- to indicate that data from this ingestion should be used to provide that 17 | -- historical data. 18 | waypoint_30_dt TIMESTAMPTZ UNIQUE, 19 | -- We're not using these waypoints *yet* but we're tracking them anyway 20 | waypoint_5_dt TIMESTAMPTZ UNIQUE, 21 | waypoint_10_dt TIMESTAMPTZ UNIQUE, 22 | waypoint_60_dt TIMESTAMPTZ UNIQUE 23 | ); 24 | 25 | CREATE INDEX IF NOT EXISTS ingest_run_ingest_dt_idx 26 | ON ingest_run (ingest_dt); 27 | 28 | CREATE INDEX IF NOT EXISTS ingest_run_waypoint_30_dt_idx 29 | ON ingest_run (waypoint_30_dt); 30 | 31 | CREATE INDEX IF NOT EXISTS ingest_run_waypoint_5_dt_idx 32 | ON ingest_run (waypoint_5_dt); 33 | 34 | CREATE INDEX IF NOT EXISTS ingest_run_waypoint_10_dt_idx 35 | ON ingest_run (waypoint_10_dt); 36 | 37 | CREATE INDEX IF NOT EXISTS ingest_run_waypoint_60_dt_idx 38 | ON ingest_run (waypoint_60_dt); 39 | 40 | 41 | -- Stores an AP result from an ingestion run 42 | CREATE TABLE IF NOT EXISTS ap_result ( 43 | -- The ID of the ingestion run this result is from 44 | ingest_id INTEGER NOT NULL REFERENCES ingest_run(ingest_id) ON DELETE CASCADE, 45 | 46 | -- The fields we get from elex 47 | elex_id TEXT NOT NULL, 48 | raceid INTEGER, 49 | racetype TEXT, 50 | racetypeid TEXT, 51 | ballotorder INTEGER, 52 | candidateid INTEGER, 53 | description TEXT, 54 | delegatecount INTEGER, 55 | electiondate DATE, 56 | electtotal INTEGER, 57 | electwon INTEGER, 58 | fipscode TEXT, 59 | first TEXT, 60 | incumbent BOOLEAN, 61 | initialization_data BOOLEAN, 62 | is_ballot_measure BOOLEAN, 63 | last TEXT, 64 | lastupdated TIMESTAMPTZ, 65 | level TEXT, 66 | national BOOLEAN, 67 | officeid TEXT, 68 | officename TEXT, 69 | party TEXT, 70 | polid INTEGER, 71 | polnum INTEGER, 72 | precinctsreporting INTEGER, 73 | precinctsreportingpct DOUBLE PRECISION, 74 | precinctstotal INTEGER, 75 | reportingunitid TEXT, 76 | reportingunitname TEXT, 77 | runoff BOOLEAN, 78 | seatname TEXT, 79 | seatnum INTEGER, 80 | statename TEXT, 81 | statepostal TEXT, 82 | test BOOLEAN, 83 | uncontested BOOLEAN, 84 | votecount INTEGER, 85 | votepct DOUBLE PRECISION, 86 | winner BOOLEAN, 87 | PRIMARY KEY (ingest_id, elex_id) 88 | ); 89 | 90 | CREATE INDEX IF NOT EXISTS ap_result_statepostal_idx 91 | ON ap_result (statepostal); 92 | 93 | CREATE INDEX IF NOT EXISTS ap_result_winner_idx 94 | ON ap_result (winner); 95 | 96 | CREATE INDEX IF NOT EXISTS ap_result_reportingunitid_idx 97 | ON ap_result (reportingunitid); 98 | 99 | CREATE INDEX IF NOT EXISTS ap_result_party_idx 100 | ON ap_result (party); 101 | 102 | CREATE INDEX IF NOT EXISTS ap_result_officeid_idx 103 | ON ap_result (officeid); 104 | 105 | CREATE INDEX IF NOT EXISTS ap_result_level_idx 106 | ON ap_result (level); 107 | 108 | CREATE INDEX IF NOT EXISTS ap_result_lastupdated_idx 109 | ON ap_result (lastupdated); 110 | 111 | 112 | CREATE INDEX IF NOT EXISTS ap_result_ingest_id_idx 113 | ON ap_result (ingest_id); 114 | 115 | -- Drop 5/10 min waypoint and just use a 15-minute one 116 | ALTER TABLE ingest_run 117 | DROP COLUMN IF EXISTS waypoint_10_dt; 118 | 119 | ALTER TABLE ingest_run 120 | DROP COLUMN IF EXISTS waypoint_5_dt; 121 | 122 | ALTER TABLE ingest_run 123 | ADD COLUMN IF NOT EXISTS waypoint_15_dt TIMESTAMPTZ UNIQUE; 124 | 125 | CREATE INDEX IF NOT EXISTS ingest_run_waypoint_15_dt_idx 126 | ON ingest_run (waypoint_15_dt); 127 | 128 | CREATE TABLE IF NOT EXISTS senate_calls ( 129 | state TEXT PRIMARY KEY, 130 | ap_call TEXT, 131 | ap_called_at TIMESTAMPTZ, 132 | published BOOLEAN NOT NULL DEFAULT FALSE 133 | ); 134 | 135 | CREATE TABLE IF NOT EXISTS president_calls ( 136 | state TEXT PRIMARY KEY, 137 | ap_call TEXT, 138 | ap_called_at TIMESTAMPTZ, 139 | published BOOLEAN NOT NULL DEFAULT FALSE 140 | ); 141 | 142 | CREATE TABLE IF NOT EXISTS comments ( 143 | ts TIMESTAMPTZ NOT NULL, 144 | submitted_by TEXT NOT NULL, 145 | office_id TEXT NOT NULL, 146 | race TEXT NOT NULL, 147 | title TEXT NOT NULL, 148 | body TEXT NOT NULL 149 | ); 150 | 151 | ALTER TABLE comments 152 | ALTER COLUMN race DROP NOT NULL; 153 | -------------------------------------------------------------------------------- /enip_backend/export/structs.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from enum import Enum 3 | from typing import Dict, List, Optional, Union 4 | 5 | from humps import camelize 6 | from pydantic import BaseModel, Field 7 | 8 | 9 | def to_camel(string): 10 | return camelize(string) 11 | 12 | 13 | # From: https://medium.com/analytics-vidhya/camel-case-models-with-fast-api-and-pydantic-5a8acb6c0eee 14 | class CamelModel(BaseModel): 15 | class Config: 16 | alias_generator = to_camel 17 | allow_population_by_field_name = True 18 | 19 | 20 | class Party(str, Enum): 21 | DEM = "dem" 22 | GOP = "gop" 23 | OTHER = "oth" 24 | 25 | @classmethod 26 | def from_ap(cls, str): 27 | if str.lower() == "dem": 28 | return cls.DEM 29 | 30 | if str.lower() == "gop": 31 | return cls.GOP 32 | 33 | return cls.OTHER 34 | 35 | 36 | class Comment(CamelModel): 37 | timestamp: datetime 38 | author: str 39 | title: str 40 | body: str 41 | 42 | 43 | class NationalSummaryPresidentCandidateNamed(CamelModel): 44 | first_name: str 45 | last_name: str 46 | pop_vote: int 47 | pop_pct: float 48 | elect_won: int = 0 49 | pop_vote_history: Dict[str, int] = {} 50 | 51 | 52 | class NationalSummaryPresidentCandidateUnnamed(CamelModel): 53 | pop_vote: int = 0 54 | pop_pct: float = 0 55 | elect_won: int = 0 56 | pop_vote_history: Dict[str, int] = {} 57 | 58 | 59 | class NationalSummaryPresident(CamelModel): 60 | dem: Optional[NationalSummaryPresidentCandidateNamed] = None 61 | gop: Optional[NationalSummaryPresidentCandidateNamed] = None 62 | oth: NationalSummaryPresidentCandidateUnnamed = Field( 63 | default_factory=NationalSummaryPresidentCandidateUnnamed 64 | ) 65 | winner: Optional[Party] 66 | 67 | 68 | class NationalSummaryWinnerCountEntry(CamelModel): 69 | won: int = 0 70 | 71 | 72 | class NationalSummaryWinnerCount(CamelModel): 73 | dem: NationalSummaryWinnerCountEntry = Field( 74 | default_factory=NationalSummaryWinnerCountEntry 75 | ) 76 | gop: NationalSummaryWinnerCountEntry = Field( 77 | default_factory=NationalSummaryWinnerCountEntry 78 | ) 79 | oth: NationalSummaryWinnerCountEntry = Field( 80 | default_factory=NationalSummaryWinnerCountEntry 81 | ) 82 | 83 | 84 | class StateSummaryCandidateNamed(CamelModel): 85 | first_name: str 86 | last_name: str 87 | pop_vote: int 88 | pop_pct: float 89 | pop_vote_history: Dict[str, int] = {} 90 | 91 | 92 | class StateSummaryCandidateUnnamed(CamelModel): 93 | pop_vote: int = 0 94 | pop_pct: float = 0 95 | pop_vote_history: Dict[str, int] = {} 96 | 97 | 98 | class StateSummaryPresident(CamelModel): 99 | dem: Optional[StateSummaryCandidateNamed] = None 100 | gop: Optional[StateSummaryCandidateNamed] = None 101 | oth: StateSummaryCandidateUnnamed = Field( 102 | default_factory=StateSummaryCandidateUnnamed 103 | ) 104 | winner: Optional[Party] = None 105 | comments: List[Comment] = [] 106 | 107 | 108 | class StateSummaryCongressionalResult(CamelModel): 109 | dem: Optional[StateSummaryCandidateNamed] = None 110 | gop: Optional[StateSummaryCandidateNamed] = None 111 | oth: StateSummaryCandidateUnnamed = Field( 112 | default_factory=StateSummaryCandidateUnnamed 113 | ) 114 | multiple_dem: bool = False 115 | multiple_gop: bool = False 116 | winner: Optional[Party] = None 117 | comments: List[Comment] = [] 118 | 119 | 120 | class StateSummary(CamelModel): 121 | P: StateSummaryPresident = Field(default_factory=StateSummaryPresident) 122 | S: Optional[StateSummaryCongressionalResult] = None 123 | H: Dict[str, StateSummaryCongressionalResult] = {} 124 | 125 | 126 | class PresidentialCDSummary(CamelModel): 127 | P: StateSummaryPresident = Field(default_factory=StateSummaryPresident) 128 | 129 | 130 | class SenateSpecialSummary(CamelModel): 131 | S: Optional[StateSummaryCongressionalResult] = None 132 | 133 | 134 | class NationalSummary(CamelModel): 135 | P: NationalSummaryPresident = Field(default_factory=NationalSummaryPresident) 136 | S: NationalSummaryWinnerCount = Field(default_factory=NationalSummaryWinnerCount) 137 | H: NationalSummaryWinnerCount = Field(default_factory=NationalSummaryWinnerCount) 138 | comments: List[Comment] = [] 139 | 140 | 141 | class NationalData(CamelModel): 142 | national_summary: NationalSummary = Field(default_factory=NationalSummary) 143 | state_summaries: Dict[ 144 | str, Union[StateSummary, PresidentialCDSummary, SenateSpecialSummary] 145 | ] = {} 146 | 147 | 148 | class CountyCongressionalResult(CamelModel): 149 | dem: Optional[StateSummaryCandidateNamed] = None 150 | gop: Optional[StateSummaryCandidateNamed] = None 151 | oth: StateSummaryCandidateUnnamed = Field( 152 | default_factory=StateSummaryCandidateUnnamed 153 | ) 154 | multiple_dem: bool = False 155 | multiple_gop: bool = False 156 | 157 | 158 | class CountyPresidentialResult(CamelModel): 159 | dem: Optional[StateSummaryCandidateNamed] = None 160 | gop: Optional[StateSummaryCandidateNamed] = None 161 | oth: StateSummaryCandidateUnnamed = Field( 162 | default_factory=StateSummaryCandidateUnnamed 163 | ) 164 | 165 | 166 | class County(CamelModel): 167 | P: CountyPresidentialResult = Field(default_factory=CountyPresidentialResult) 168 | S: Dict[str, CountyCongressionalResult] = {} 169 | H: Dict[str, CountyCongressionalResult] = {} 170 | 171 | 172 | class StateData(CamelModel): 173 | counties: Dict[str, County] = {} 174 | -------------------------------------------------------------------------------- /enip_backend/export/schemas/ga.example.json: -------------------------------------------------------------------------------- 1 | { 2 | "counties": { 3 | "10003": { 4 | "P": { 5 | "dem": { 6 | "firstName": "Joe", 7 | "lastName": "Biden", 8 | "popVote": 65853516, 9 | "popPct": 0.49, 10 | "popVoteHistory": { 11 | "2020-11-03T20:00:00Z": 34437487, 12 | "2020-11-03T20:30:00Z": 44437487, 13 | "2020-11-03T21:00:00Z": 54437487 14 | } 15 | }, 16 | "gop": { 17 | "firstName": "Donald", 18 | "lastName": "Trump", 19 | "popVote": 65853516, 20 | "popPct": 0.49, 21 | "popVoteHistory": { 22 | "2020-11-03T20:00:00Z": 34437487, 23 | "2020-11-03T20:30:00Z": 44437487, 24 | "2020-11-03T21:00:00Z": 54437487 25 | } 26 | }, 27 | "oth": { 28 | "popVote": 65853516, 29 | "popPct": 0.49, 30 | "popVoteHistory": { 31 | "2020-11-03T20:00:00Z": 34437487, 32 | "2020-11-03T20:30:00Z": 44437487, 33 | "2020-11-03T21:00:00Z": 54437487 34 | } 35 | } 36 | }, 37 | "S": { 38 | "GA": { 39 | "dem": { 40 | "firstName": "Jon", 41 | "lastName": "Ossof", 42 | "popVote": 65853516, 43 | "popPct": 0.49, 44 | "popVoteHistory": { 45 | "2020-11-03T20:00:00Z": 34437487, 46 | "2020-11-03T20:30:00Z": 44437487, 47 | "2020-11-03T21:00:00Z": 54437487 48 | } 49 | }, 50 | "gop": { 51 | "firstName": "David", 52 | "lastName": "Perdue", 53 | "popVote": 65853516, 54 | "popPct": 0.49, 55 | "popVoteHistory": { 56 | "2020-11-03T20:00:00Z": 34437487, 57 | "2020-11-03T20:30:00Z": 44437487, 58 | "2020-11-03T21:00:00Z": 54437487 59 | } 60 | }, 61 | "oth": { 62 | "popVote": 65853516, 63 | "popPct": 0.49, 64 | "popVoteHistory": { 65 | "2020-11-03T20:00:00Z": 34437487, 66 | "2020-11-03T20:30:00Z": 44437487, 67 | "2020-11-03T21:00:00Z": 54437487 68 | } 69 | }, 70 | "multipleDem": false, 71 | "multipleGop": false 72 | }, 73 | "GA-S": { 74 | "dem": { 75 | "firstName": "Raphael", 76 | "lastName": "Warnock", 77 | "popVote": 65853516, 78 | "popPct": 0.49, 79 | "popVoteHistory": { 80 | "2020-11-03T20:00:00Z": 34437487, 81 | "2020-11-03T20:30:00Z": 44437487, 82 | "2020-11-03T21:00:00Z": 54437487 83 | } 84 | }, 85 | "gop": { 86 | "firstName": "Kelly", 87 | "lastName": "Loeffler", 88 | "popVote": 65853516, 89 | "popPct": 0.49, 90 | "popVoteHistory": { 91 | "2020-11-03T20:00:00Z": 34437487, 92 | "2020-11-03T20:30:00Z": 44437487, 93 | "2020-11-03T21:00:00Z": 54437487 94 | } 95 | }, 96 | "oth": { 97 | "popVote": 65853516, 98 | "popPct": 0.49, 99 | "popVoteHistory": { 100 | "2020-11-03T20:00:00Z": 34437487, 101 | "2020-11-03T20:30:00Z": 44437487, 102 | "2020-11-03T21:00:00Z": 54437487 103 | } 104 | }, 105 | "multipleDem": true, 106 | "multipleGop": true 107 | } 108 | }, 109 | "H": { 110 | "01": { 111 | "dem": { 112 | "firstName": "Raphael", 113 | "lastName": "Warnock", 114 | "popVote": 65853516, 115 | "popPct": 0.49, 116 | "popVoteHistory": { 117 | "2020-11-03T20:00:00Z": 34437487, 118 | "2020-11-03T20:30:00Z": 44437487, 119 | "2020-11-03T21:00:00Z": 54437487 120 | } 121 | }, 122 | "gop": { 123 | "firstName": "Kelly", 124 | "lastName": "Loeffler", 125 | "popVote": 65853516, 126 | "popPct": 0.49, 127 | "popVoteHistory": { 128 | "2020-11-03T20:00:00Z": 34437487, 129 | "2020-11-03T20:30:00Z": 44437487, 130 | "2020-11-03T21:00:00Z": 54437487 131 | } 132 | }, 133 | "oth": { 134 | "popVote": 65853516, 135 | "popPct": 0.49, 136 | "popVoteHistory": { 137 | "2020-11-03T20:00:00Z": 34437487, 138 | "2020-11-03T20:30:00Z": 44437487, 139 | "2020-11-03T21:00:00Z": 54437487 140 | } 141 | }, 142 | "multipleDem": false, 143 | "multipleGop": false 144 | }, 145 | "14": { 146 | "dem": null, 147 | "gop": { 148 | "firstName": "Marjorie", 149 | "lastName": "Green", 150 | "popVote": 65853516, 151 | "popPct": 0.49, 152 | "popVoteHistory": { 153 | "2020-11-03T20:00:00Z": 34437487, 154 | "2020-11-03T20:30:00Z": 44437487, 155 | "2020-11-03T21:00:00Z": 54437487 156 | } 157 | }, 158 | "oth": { 159 | "popVote": 65853516, 160 | "popPct": 0.49, 161 | "popVoteHistory": { 162 | "2020-11-03T20:00:00Z": 34437487, 163 | "2020-11-03T20:30:00Z": 44437487, 164 | "2020-11-03T21:00:00Z": 54437487 165 | } 166 | }, 167 | "multipleDem": false, 168 | "multipleGop": false 169 | } 170 | } 171 | } 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /enip_backend/export/run.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import random 4 | from concurrent.futures import ThreadPoolExecutor 5 | 6 | import sentry_sdk 7 | from ddtrace import tracer 8 | from jsonschema import validate 9 | from jsonschema.exceptions import ValidationError 10 | 11 | from ..enip_common import s3 12 | from ..enip_common.config import CDN_URL 13 | from ..enip_common.pg import get_ro_cursor 14 | from ..enip_common.states import STATES 15 | from .national import NationalDataExporter 16 | from .schemas import national_schema, state_schema 17 | from .state import StateDataExporter 18 | 19 | THREADS = 4 20 | 21 | 22 | def export_to_s3(ingest_run_id, ingest_run_dt, json_data, schema, path, export_name): 23 | # Validate 24 | json_data_parsed = json.loads(json_data) 25 | 26 | try: 27 | validate(instance=json_data_parsed, schema=schema) 28 | except ValidationError as e: 29 | # Write the invalid JSON to s3 for diagnostics 30 | failed_cdn_url = "(failed to write to s3)" 31 | try: 32 | failed_name = f"{path}/failed/{export_name}_{ingest_run_id}.json" 33 | 34 | s3.write_cacheable_json(failed_name, json_data) 35 | failed_cdn_url = f"{CDN_URL}{failed_name}" 36 | except: 37 | logging.exception("Failed to save invalid JSON to S3") 38 | 39 | error_msg = ( 40 | f"JSON failed schema validation: {e.message} -- logged to {failed_cdn_url}" 41 | ) 42 | error_data = { 43 | "validation_message": e.message, 44 | "validator": e.validator, 45 | "validator_value": e.validator_value, 46 | "absolute_schema_path": list(e.absolute_schema_path), 47 | "bad_json_url": failed_cdn_url, 48 | } 49 | 50 | with sentry_sdk.configure_scope() as scope: 51 | scope.set_context("validation_error", error_data) 52 | sentry_sdk.capture_exception(RuntimeError(error_msg)) 53 | 54 | logging.exception(error_msg, extra=error_data) 55 | raise e 56 | 57 | # Write the JSON to s3 58 | name = f"{path}/{export_name}_{ingest_run_id}.json" 59 | s3.write_cacheable_json(name, json_data) 60 | 61 | # Load the current latest JSON 62 | latest_name = f"{path}/latest.json" 63 | latest_json = s3.read_json(latest_name) 64 | 65 | if latest_json: 66 | previous_json = s3.read_json(latest_json["path"]) 67 | else: 68 | previous_json = None 69 | 70 | # Diff 71 | was_different = previous_json != json_data_parsed 72 | 73 | # Write the new latest JSON 74 | cdn_url = f"{CDN_URL}{name}" 75 | if was_different: 76 | str(ingest_run_dt) 77 | new_latest_json = { 78 | "lastUpdated": str(ingest_run_dt), 79 | "path": name, 80 | "cdnUrl": cdn_url, 81 | } 82 | s3.write_noncacheable_json(latest_name, json.dumps(new_latest_json)) 83 | 84 | return was_different, cdn_url 85 | 86 | 87 | @tracer.wrap("enip.export.export_state", service="enip-backend-state-thread") 88 | def export_state(ingest_run_dt, state_code, ingest_data): 89 | with tracer.trace("enip.export.export_state.run_export"): 90 | data = StateDataExporter(ingest_run_dt, state_code).run_export(ingest_data) 91 | 92 | with tracer.trace("enip.export.export_state.export_to_s3"): 93 | return export_to_s3( 94 | 0, 95 | ingest_run_dt, 96 | data.json(by_alias=True), 97 | state_schema, 98 | f"states/{state_code}", 99 | ingest_run_dt.strftime("%Y%m%d%H%M%S"), 100 | ) 101 | 102 | 103 | @tracer.wrap("enip.export.export_national") 104 | def export_national(ingest_run_id, ingest_run_dt, export_name, ingest_data=None): 105 | logging.info("Running national export...") 106 | with tracer.trace("enip.export.export_ntl.run_export"): 107 | data = NationalDataExporter(ingest_run_id, ingest_run_dt).run_export( 108 | ingest_data 109 | ) 110 | 111 | with tracer.trace("enip.export.export_ntl.export_to_s3"): 112 | was_different, cdn_url = export_to_s3( 113 | ingest_run_id, 114 | ingest_run_dt, 115 | data.json(by_alias=True), 116 | national_schema, 117 | "national", 118 | export_name, 119 | ) 120 | 121 | if was_different: 122 | logging.info(f" National export completed WITH new results: {cdn_url}") 123 | else: 124 | logging.info(f" National export completed WITHOUT new results: {cdn_url}") 125 | 126 | return cdn_url 127 | 128 | 129 | def export_all_states(ap_data, ingest_run_dt): 130 | logging.info(f"Running all state exports from ingest at {str(ingest_run_dt)}...") 131 | any_failed = False 132 | results = {} 133 | 134 | with ThreadPoolExecutor(max_workers=THREADS) as executor: 135 | # Do the states in a random order so if the DB is overloaded and we're 136 | # timing out regularly, we still eventually update all of the states 137 | # (because if we time out and miss a few states, they'll probably 138 | # go earlier in the next run) 139 | states_list = list(STATES) 140 | random.shuffle(states_list) 141 | 142 | state_futures = { 143 | state_code: executor.submit( 144 | export_state, ingest_run_dt, state_code, ap_data 145 | ) 146 | for state_code in states_list 147 | } 148 | 149 | for state_code, future in state_futures.items(): 150 | try: 151 | was_different, cdn_url = future.result() 152 | if future.result(): 153 | logging.info( 154 | f" Export {state_code} completed WITH new results: {cdn_url}" 155 | ) 156 | else: 157 | logging.info( 158 | f" Export {state_code} completed WITHOUT new results: {cdn_url}" 159 | ) 160 | 161 | results[state_code] = cdn_url 162 | 163 | except Exception as e: 164 | logging.exception(f" Export {state_code} failed") 165 | sentry_sdk.capture_exception(e) 166 | 167 | if any_failed: 168 | raise RuntimeError("Some exports failed") 169 | 170 | return results 171 | 172 | 173 | def get_latest_ingest(): 174 | # Get the most recent ingest_run 175 | with get_ro_cursor() as cursor: 176 | cursor.execute( 177 | "SELECT ingest_id, ingest_dt FROM ingest_run ORDER BY ingest_dt DESC LIMIT 1" 178 | ) 179 | res = cursor.fetchone() 180 | 181 | return res.ingest_id, res.ingest_dt 182 | -------------------------------------------------------------------------------- /enip_backend/export/schemas/national.example.json: -------------------------------------------------------------------------------- 1 | { 2 | "nationalSummary": { 3 | "P": { 4 | "dem": { 5 | "firstName": "Hillary", 6 | "lastName": "Clinton", 7 | "popVote": 65853516, 8 | "popPct": 0.49, 9 | "electWon": 232, 10 | "popVoteHistory": { 11 | "2020-11-03T20:00:00Z": 34437487, 12 | "2020-11-03T20:30:00Z": 44437487, 13 | "2020-11-03T21:00:00Z": 54437487 14 | } 15 | }, 16 | "gop": { 17 | "firstName": "Hillary", 18 | "lastName": "Clinton", 19 | "popVote": 65853516, 20 | "popPct": 0.49, 21 | "electWon": 232, 22 | "popVoteHistory": { 23 | "2020-11-03T20:00:00Z": 34437487, 24 | "2020-11-03T20:30:00Z": 44437487, 25 | "2020-11-03T21:00:00Z": 54437487 26 | } 27 | }, 28 | "oth": { 29 | "popVote": 65853516, 30 | "popPct": 0.49, 31 | "electWon": 232, 32 | "popVoteHistory": { 33 | "2020-11-03T20:00:00Z": 34437487, 34 | "2020-11-03T20:30:00Z": 44437487, 35 | "2020-11-03T21:00:00Z": 54437487 36 | } 37 | }, 38 | "winner": "dem" 39 | }, 40 | "S": { 41 | "dem": { 42 | "won": 22 43 | }, 44 | "gop": { 45 | "won": 11 46 | }, 47 | "oth": { 48 | "won": 0 49 | } 50 | }, 51 | "H": { 52 | "dem": { 53 | "won": 220 54 | }, 55 | "gop": { 56 | "won": 110 57 | }, 58 | "oth": { 59 | "won": 0 60 | } 61 | }, 62 | "comments": [ 63 | { 64 | "timestamp": "2020-11-03T20:30:00Z", 65 | "author": "Jason K-B", 66 | "title": "Foo", 67 | "body": "Bar *Baz*" 68 | } 69 | ] 70 | }, 71 | "stateSummaries": { 72 | "MT": { 73 | "P": { 74 | "dem": { 75 | "firstName": "Hillary", 76 | "lastName": "Clinton", 77 | "popVote": 74437487, 78 | "popPct": 0.475899, 79 | "popVoteHistory": { 80 | "2020-11-03T20:00:00Z": 34437487, 81 | "2020-11-03T20:30:00Z": 44437487, 82 | "2020-11-03T21:00:00Z": 54437487 83 | } 84 | }, 85 | "gop": { 86 | "firstName": "Donald", 87 | "lastName": "Trump", 88 | "popVote": 74437487, 89 | "popPct": 0.475899, 90 | "popVoteHistory": { 91 | "2020-11-03T20:00:00Z": 34437487, 92 | "2020-11-03T20:30:00Z": 44437487, 93 | "2020-11-03T21:00:00Z": 54437487 94 | } 95 | }, 96 | "oth": { 97 | "popVote": 74437487, 98 | "popPct": 0.475899, 99 | "popVoteHistory": { 100 | "2020-11-03T20:00:00Z": 34437487, 101 | "2020-11-03T20:30:00Z": 44437487, 102 | "2020-11-03T21:00:00Z": 54437487 103 | } 104 | }, 105 | "winner": "dem", 106 | "comments": [ 107 | { 108 | "timestamp": "2020-11-03T20:30:00Z", 109 | "author": "Jason K-B", 110 | "title": "Foo", 111 | "body": "Bar *Baz*" 112 | } 113 | ] 114 | }, 115 | "S": { 116 | "dem": { 117 | "firstName": "Hillary", 118 | "lastName": "Clinton", 119 | "popVote": 74437487, 120 | "popPct": 0.475899, 121 | "popVoteHistory": { 122 | "2020-11-03T20:00:00Z": 34437487, 123 | "2020-11-03T20:30:00Z": 44437487, 124 | "2020-11-03T21:00:00Z": 54437487 125 | } 126 | }, 127 | "gop": { 128 | "firstName": "Donald", 129 | "lastName": "Trump", 130 | "popVote": 74437487, 131 | "popPct": 0.475899, 132 | "popVoteHistory": { 133 | "2020-11-03T20:00:00Z": 34437487, 134 | "2020-11-03T20:30:00Z": 44437487, 135 | "2020-11-03T21:00:00Z": 54437487 136 | } 137 | }, 138 | "oth": { 139 | "popVote": 74437487, 140 | "popPct": 0.475899, 141 | "popVoteHistory": { 142 | "2020-11-03T20:00:00Z": 34437487, 143 | "2020-11-03T20:30:00Z": 44437487, 144 | "2020-11-03T21:00:00Z": 54437487 145 | } 146 | }, 147 | "winner": "dem", 148 | "multipleDem": false, 149 | "multipleGop": false, 150 | "comments": [ 151 | { 152 | "timestamp": "2020-11-03T20:30:00Z", 153 | "author": "Jason K-B", 154 | "title": "Foo", 155 | "body": "Bar *Baz*" 156 | } 157 | ] 158 | }, 159 | "H": { 160 | "01": { 161 | "dem": { 162 | "firstName": "Hillary", 163 | "lastName": "Clinton", 164 | "popVote": 74437487, 165 | "popPct": 0.475899, 166 | "popVoteHistory": { 167 | "2020-11-03T20:00:00Z": 34437487, 168 | "2020-11-03T20:30:00Z": 44437487, 169 | "2020-11-03T21:00:00Z": 54437487 170 | } 171 | }, 172 | "gop": { 173 | "firstName": "Donald", 174 | "lastName": "Trump", 175 | "popVote": 74437487, 176 | "popPct": 0.475899, 177 | "popVoteHistory": { 178 | "2020-11-03T20:00:00Z": 34437487, 179 | "2020-11-03T20:30:00Z": 44437487, 180 | "2020-11-03T21:00:00Z": 54437487 181 | } 182 | }, 183 | "oth": { 184 | "popVote": 74437487, 185 | "popPct": 0.475899, 186 | "popVoteHistory": { 187 | "2020-11-03T20:00:00Z": 34437487, 188 | "2020-11-03T20:30:00Z": 44437487, 189 | "2020-11-03T21:00:00Z": 54437487 190 | } 191 | }, 192 | "winner": "dem", 193 | "multipleDem": false, 194 | "multipleGop": false, 195 | "comments": [ 196 | { 197 | "timestamp": "2020-11-03T20:30:00Z", 198 | "author": "Jason K-B", 199 | "title": "Foo", 200 | "body": "Bar *Baz*" 201 | } 202 | ] 203 | }, 204 | "AL": { 205 | "dem": { 206 | "firstName": "Hillary", 207 | "lastName": "Clinton", 208 | "popVote": 74437487, 209 | "popPct": 0.475899, 210 | "popVoteHistory": { 211 | "2020-11-03T20:00:00Z": 34437487, 212 | "2020-11-03T20:30:00Z": 44437487, 213 | "2020-11-03T21:00:00Z": 54437487 214 | } 215 | }, 216 | "gop": { 217 | "firstName": "Donald", 218 | "lastName": "Trump", 219 | "popVote": 74437487, 220 | "popPct": 0.475899, 221 | "popVoteHistory": { 222 | "2020-11-03T20:00:00Z": 34437487, 223 | "2020-11-03T20:30:00Z": 44437487, 224 | "2020-11-03T21:00:00Z": 54437487 225 | } 226 | }, 227 | "oth": { 228 | "popVote": 74437487, 229 | "popPct": 0.475899, 230 | "popVoteHistory": { 231 | "2020-11-03T20:00:00Z": 34437487, 232 | "2020-11-03T20:30:00Z": 44437487, 233 | "2020-11-03T21:00:00Z": 54437487 234 | } 235 | }, 236 | "winner": "dem", 237 | "multipleDem": false, 238 | "multipleGop": false, 239 | "comments": [ 240 | { 241 | "timestamp": "2020-11-03T20:30:00Z", 242 | "author": "Jason K-B", 243 | "title": "Foo", 244 | "body": "Bar *Baz*" 245 | } 246 | ] 247 | } 248 | } 249 | }, 250 | "GA-S": { 251 | "S": { 252 | "dem": { 253 | "firstName": "Hillary", 254 | "lastName": "Clinton", 255 | "popVote": 74437487, 256 | "popPct": 0.475899, 257 | "popVoteHistory": { 258 | "2020-11-03T20:00:00Z": 34437487, 259 | "2020-11-03T20:30:00Z": 44437487, 260 | "2020-11-03T21:00:00Z": 54437487 261 | } 262 | }, 263 | "gop": { 264 | "firstName": "Donald", 265 | "lastName": "Trump", 266 | "popVote": 74437487, 267 | "popPct": 0.475899, 268 | "popVoteHistory": { 269 | "2020-11-03T20:00:00Z": 34437487, 270 | "2020-11-03T20:30:00Z": 44437487, 271 | "2020-11-03T21:00:00Z": 54437487 272 | } 273 | }, 274 | "oth": { 275 | "popVote": 74437487, 276 | "popPct": 0.475899, 277 | "popVoteHistory": { 278 | "2020-11-03T20:00:00Z": 34437487, 279 | "2020-11-03T20:30:00Z": 44437487, 280 | "2020-11-03T21:00:00Z": 54437487 281 | } 282 | }, 283 | "winner": "dem", 284 | "multipleDem": false, 285 | "multipleGop": false, 286 | "comments": [ 287 | { 288 | "timestamp": "2020-11-03T20:30:00Z", 289 | "author": "Jason K-B", 290 | "title": "Foo", 291 | "body": "Bar *Baz*" 292 | } 293 | ] 294 | } 295 | }, 296 | "NE-01": { 297 | "P": { 298 | "dem": { 299 | "firstName": "Hillary", 300 | "lastName": "Clinton", 301 | "popVote": 74437487, 302 | "popPct": 0.475899, 303 | "popVoteHistory": { 304 | "2020-11-03T20:00:00Z": 34437487, 305 | "2020-11-03T20:30:00Z": 44437487, 306 | "2020-11-03T21:00:00Z": 54437487 307 | } 308 | }, 309 | "gop": { 310 | "firstName": "Donald", 311 | "lastName": "Trump", 312 | "popVote": 74437487, 313 | "popPct": 0.475899, 314 | "popVoteHistory": { 315 | "2020-11-03T20:00:00Z": 34437487, 316 | "2020-11-03T20:30:00Z": 44437487, 317 | "2020-11-03T21:00:00Z": 54437487 318 | } 319 | }, 320 | "oth": { 321 | "popVote": 74437487, 322 | "popPct": 0.475899, 323 | "popVoteHistory": { 324 | "2020-11-03T20:00:00Z": 34437487, 325 | "2020-11-03T20:30:00Z": 44437487, 326 | "2020-11-03T21:00:00Z": 54437487 327 | } 328 | }, 329 | "winner": "dem", 330 | "comments": [ 331 | { 332 | "timestamp": "2020-11-03T20:30:00Z", 333 | "author": "Jason K-B", 334 | "title": "Foo", 335 | "body": "Bar *Baz*" 336 | } 337 | ] 338 | } 339 | } 340 | } 341 | } 342 | -------------------------------------------------------------------------------- /enip_backend/export/helpers.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from typing import Any, Dict, Generator, List, NamedTuple, Optional, Union 3 | 4 | from ..enip_common.config import HISTORICAL_START 5 | from ..enip_common.pg import get_cursor, get_ro_cursor 6 | from . import structs 7 | 8 | SQLRecord = NamedTuple( 9 | "SQLRecord", 10 | [ 11 | ("ingest_id", int), 12 | ("elex_id", str), 13 | ("statepostal", str), 14 | ("fipscode", str), 15 | ("level", str), 16 | ("reportingunitname", Optional[str]), 17 | ("officeid", str), 18 | ("seatnum", Optional[int]), 19 | ("party", str), 20 | ("first", str), 21 | ("last", str), 22 | ("electtotal", int), 23 | ("electwon", int), 24 | ("votecount", int), 25 | ("votepct", float), 26 | ("winner", bool), 27 | ], 28 | ) 29 | 30 | 31 | def sqlrecord_from_dict(dict: Dict[str, Any]) -> SQLRecord: 32 | return SQLRecord( 33 | ingest_id=int(dict["ingest_id"]), 34 | elex_id=str(dict["elex_id"]), 35 | statepostal=str(dict["statepostal"]), 36 | fipscode=str(dict["fipscode"]), 37 | level=str(dict["level"]), 38 | reportingunitname=str(dict["reportingunitname"]) 39 | if dict["reportingunitname"] is not None 40 | else None, 41 | officeid=str(dict["officeid"]), 42 | seatnum=int(dict["seatnum"]) if dict["seatnum"] is not None else None, 43 | party=str(dict["party"]), 44 | first=str(dict["first"]), 45 | last=str(dict["last"]), 46 | electtotal=int(dict["electtotal"]), 47 | electwon=int(dict["electwon"]), 48 | votecount=int(dict["votecount"]), 49 | votepct=float(dict["votepct"]), 50 | winner=bool(dict["winner"]), 51 | ) 52 | 53 | 54 | # map of (elex id -> { waypoint_dt -> count}) 55 | HistoricalResults = Dict[str, Dict[str, int]] 56 | 57 | 58 | def handle_candidate_results( 59 | data: Union[ 60 | structs.NationalSummaryPresident, 61 | structs.StateSummaryPresident, 62 | structs.StateSummaryCongressionalResult, 63 | structs.CountyCongressionalResult, 64 | structs.CountyPresidentialResult, 65 | ], 66 | named_candidate_factory: Any, 67 | record: SQLRecord, 68 | historical_counts: HistoricalResults, 69 | ) -> None: 70 | """ 71 | Helper function that adds a result to dem/gop/other. This function: 72 | 73 | - Populates the GOP/Dem candidate if there isn't already one 74 | - If there's already a GOP/Dem candidate, or if this record is for 75 | a third-party candidate, adds the results to the "other" bucket 76 | - Gets the historical vote counts and populates those as well 77 | """ 78 | party = structs.Party.from_ap(record.party) 79 | if party == structs.Party.GOP: 80 | if data.gop: 81 | # There is already a GOP candidate. We process in order of number 82 | # of votes, so this is a secondary GOP candidate 83 | if hasattr(data, "multiple_gop"): 84 | data.multiple_gop = True 85 | else: 86 | # This is the leading GOP candidate 87 | data.gop = named_candidate_factory( 88 | first_name=record.first, 89 | last_name=record.last, 90 | pop_vote=record.votecount, 91 | pop_pct=record.votepct, 92 | pop_vote_history=historical_counts.get(record.elex_id, {}), 93 | ) 94 | return 95 | elif party == structs.Party.DEM: 96 | if data.dem: 97 | # There is already a Dem candidate. We process in order of number 98 | # of votes, so this is a secondary Dem candidate 99 | if hasattr(data, "multiple_dem"): 100 | data.multiple_dem = True 101 | else: 102 | # This is the leading Dem candidate 103 | data.dem = named_candidate_factory( 104 | first_name=record.first, 105 | last_name=record.last, 106 | pop_vote=record.votecount, 107 | pop_pct=record.votepct, 108 | pop_vote_history=historical_counts.get(record.elex_id, {}), 109 | ) 110 | return 111 | 112 | # Third-party candidate or non-leading gop/dem 113 | data.oth.pop_vote += record.votecount 114 | data.oth.pop_pct += record.votepct 115 | 116 | # Merge the candidate's historical counts into the overall historical 117 | # counts 118 | for datetime_str, count in historical_counts.get(record.elex_id, {}).items(): 119 | if datetime_str in data.oth.pop_vote_history: 120 | data.oth.pop_vote_history[datetime_str] += count 121 | else: 122 | data.oth.pop_vote_history[datetime_str] = count 123 | 124 | 125 | def load_historicals( 126 | ingest_run_dt: datetime, filter_sql: str, filter_params: List[Any] 127 | ) -> HistoricalResults: 128 | historical_counts: HistoricalResults = {} 129 | 130 | with get_ro_cursor() as cursor: 131 | # Fetch historical results and produce a map of (elex id -> { waypoint_dt -> count}) 132 | cursor.execute( 133 | f""" 134 | SELECT 135 | -- If the vote count did not change for a particular election 136 | -- between two waypoints, we don't need to report the intermediate 137 | -- values. We report only the first value (as per the ORDER BY 138 | -- below) 139 | DISTINCT ON (elex_id, votecount) 140 | ingest_run.waypoint_60_dt, 141 | ap_result.elex_id, 142 | ap_result.votecount 143 | FROM ap_result 144 | JOIN ingest_run ON ingest_run.ingest_id = ap_result.ingest_id 145 | WHERE ingest_run.waypoint_60_dt IS NOT NULL 146 | AND racetypeid = 'G' 147 | AND ingest_dt < %s 148 | AND ingest_dt > %s 149 | AND {filter_sql} 150 | ORDER BY elex_id, votecount, waypoint_60_dt ASC 151 | """, 152 | [ingest_run_dt, HISTORICAL_START] + filter_params, 153 | ) 154 | for record in cursor: 155 | if record.elex_id not in historical_counts: 156 | historical_counts[record.elex_id] = {} 157 | historical_counts[record.elex_id][ 158 | str(record.waypoint_60_dt) 159 | ] = record.votecount 160 | 161 | return historical_counts 162 | 163 | 164 | def load_election_results( 165 | ingest_run_id: str, filter_sql: str, filter_params: List[Any] 166 | ) -> Generator[SQLRecord, None, None]: 167 | # Use the RW cursor to make sure we have the latest data 168 | with get_cursor() as cursor: 169 | # Iterate over every result 170 | cursor.execute( 171 | f""" 172 | SELECT 173 | ingest_id, 174 | 175 | -- A unique ID for the race that is stable across ingest 176 | -- runs. Used to correlate current data with historical data. 177 | elex_id, 178 | 179 | -- The state code for the result 180 | statepostal, 181 | 182 | -- FIPS code for county-level results 183 | fipscode, 184 | 185 | -- national, state, or district 186 | -- district is for ME and NE congressional district presidential results 187 | level, 188 | 189 | -- For ME and NE districts, this gives us which district 190 | -- (or "At Large" for statewide seats) 191 | reportingunitname, 192 | 193 | -- P, S, or H 194 | officeid, 195 | 196 | -- For house races, which house race. Also for the GA sentate, this 197 | -- is null for the regular race and 2 for the special. Note that the 198 | -- AP marks at-large districts as seatnum=1 199 | seatnum, 200 | 201 | -- Dem, GOP, or one of many others 202 | party, 203 | 204 | -- Candidate name 205 | first, 206 | last, 207 | 208 | -- Electoral votes awarded in this race 209 | electtotal, 210 | 211 | -- Electoral votes won by this candidate. Used to work around a bug 212 | -- in the AP's reporting of congressional district winners. 213 | electwon, 214 | 215 | -- Number of votes cast for this candidate 216 | votecount, 217 | 218 | -- Percent of votes cast for this candidate 219 | votepct, 220 | 221 | -- Whether the AP has called the race for this candidate 222 | winner 223 | FROM ap_result 224 | WHERE ingest_id = %s 225 | AND racetypeid = 'G' 226 | AND {filter_sql} 227 | 228 | -- Order by votes, descending. This means we can work through the 229 | -- results in order, and mark the first dem/gop candidate we see as 230 | -- the leading candidate (and fold other candidates into the "other" 231 | -- totals) 232 | ORDER BY votecount DESC 233 | """, 234 | [ingest_run_id] + filter_params, 235 | ) 236 | 237 | for record in cursor: 238 | yield record 239 | 240 | 241 | Comments = Dict[str, Dict[str, List[structs.Comment]]] 242 | 243 | 244 | def load_comments() -> Comments: 245 | comments: Comments = {"P": {}, "S": {}, "H": {}, "N": {"N": []}} 246 | with get_cursor() as cursor: 247 | cursor.execute("SELECT * FROM comments ORDER BY ts DESC") 248 | 249 | for record in cursor: 250 | office = comments[record.office_id] 251 | 252 | # "N" (national) comments don't have races 253 | race = "N" if record.office_id == "N" else record.race 254 | 255 | if race not in office: 256 | office[race] = [] 257 | 258 | office[race].append( 259 | structs.Comment( 260 | timestamp=record.ts, 261 | author=record.submitted_by, 262 | title=record.title, 263 | body=record.body, 264 | ) 265 | ) 266 | 267 | return comments 268 | 269 | 270 | Calls = Dict[str, Dict[str, bool]] 271 | 272 | 273 | def load_calls() -> Calls: 274 | calls: Calls = {"P": {}, "S": {}} 275 | 276 | with get_cursor() as cursor: 277 | cursor.execute("SELECT * FROM senate_calls") 278 | for record in cursor: 279 | calls["S"][record.state] = record.published 280 | 281 | cursor.execute("SELECT * FROM president_calls") 282 | for record in cursor: 283 | calls["P"][record.state] = record.published 284 | 285 | return calls 286 | -------------------------------------------------------------------------------- /enip_backend/export/state_test.py: -------------------------------------------------------------------------------- 1 | import json 2 | from datetime import datetime, timezone 3 | 4 | import pytest 5 | 6 | from . import structs 7 | from .helpers import HistoricalResults, SQLRecord 8 | from .state import StateDataExporter 9 | 10 | mock_historicals: HistoricalResults = {} 11 | 12 | 13 | @pytest.fixture(autouse=True) 14 | def init_mocks(mocker): 15 | global mock_historicals 16 | 17 | mock_historicals = {} 18 | 19 | mock_load_historicals = mocker.patch("enip_backend.export.state.load_historicals") 20 | mock_load_historicals.return_value = mock_historicals 21 | 22 | 23 | def exporter(state): 24 | return StateDataExporter(datetime(2020, 11, 3, 8, 0, 0, tzinfo=timezone.utc), state) 25 | 26 | 27 | def assert_result(actual, expected): 28 | # We make assertions using the JSON representation 29 | # because that prints much better in pytest 30 | assert json.loads(actual.json()) == json.loads(expected.json()) 31 | 32 | 33 | # Helpers for constructing SQLRecords 34 | def default_name(party, first, last): 35 | if party == "Dem": 36 | return (first or "Joe", last or "Biden") 37 | elif party == "GOP": 38 | return (first or "Donald", last or "Trump") 39 | else: 40 | return (first or "Foo", last or "Barson") 41 | 42 | 43 | def res_p( 44 | state, 45 | fipscode, 46 | party, 47 | votecount, 48 | votepct, 49 | winner=False, 50 | first=None, 51 | last=None, 52 | ingest_id="test_run", 53 | elex_id="test_elex", 54 | ): 55 | first, last = default_name(party, first, last) 56 | 57 | return SQLRecord( 58 | ingest_id=ingest_id, 59 | elex_id=elex_id, 60 | statepostal=state, 61 | fipscode=fipscode, 62 | level="county", 63 | reportingunitname=None, 64 | officeid="P", 65 | seatnum=None, 66 | party=party, 67 | first=first, 68 | last=last, 69 | electtotal=538, 70 | electwon=0, 71 | votecount=votecount, 72 | votepct=votepct, 73 | winner=winner, 74 | ) 75 | 76 | 77 | def res_s( 78 | state, 79 | fipscode, 80 | party, 81 | first, 82 | last, 83 | votecount, 84 | votepct, 85 | winner=False, 86 | ingest_id="test_run", 87 | elex_id="test_elex", 88 | ): 89 | seatnum = None 90 | if state == "GA-S": 91 | state = "GA" 92 | seatnum = 2 93 | 94 | return SQLRecord( 95 | ingest_id=ingest_id, 96 | elex_id=elex_id, 97 | statepostal=state, 98 | fipscode=fipscode, 99 | level="county", 100 | reportingunitname=None, 101 | officeid="S", 102 | seatnum=seatnum, 103 | party=party, 104 | first=first, 105 | last=last, 106 | electtotal=538, 107 | electwon=0, 108 | votecount=votecount, 109 | votepct=votepct, 110 | winner=winner, 111 | ) 112 | 113 | 114 | def res_h( 115 | state, 116 | seatnum, 117 | fipscode, 118 | party, 119 | first, 120 | last, 121 | votecount, 122 | votepct, 123 | winner=False, 124 | ingest_id="test_run", 125 | elex_id="test_elex", 126 | ): 127 | return SQLRecord( 128 | ingest_id=ingest_id, 129 | elex_id=elex_id, 130 | statepostal=state, 131 | fipscode=fipscode, 132 | level="county", 133 | reportingunitname=None, 134 | officeid="H", 135 | seatnum=seatnum, 136 | party=party, 137 | first=first, 138 | last=last, 139 | electtotal=538, 140 | electwon=0, 141 | votecount=votecount, 142 | votepct=votepct, 143 | winner=winner, 144 | ) 145 | 146 | 147 | def test_prez_result(): 148 | mock_historicals["test_dem"] = {"A": 1, "B": 2} 149 | mock_historicals["test_gop"] = {"B": 10, "C": 20} 150 | mock_historicals["test_lib"] = {"C": 100, "D": 200} 151 | 152 | assert_result( 153 | exporter("MA").run_export( 154 | [ 155 | res_p( 156 | "MA", 157 | "12345", 158 | "Dem", 159 | votecount=12345, 160 | votepct=0.234, 161 | elex_id="test_dem", 162 | ), 163 | res_p( 164 | "MA", 165 | "12345", 166 | "GOP", 167 | votecount=67890, 168 | votepct=0.567, 169 | elex_id="test_gop", 170 | ), 171 | res_p( 172 | "MA", 173 | "12345", 174 | "Lib", 175 | votecount=123, 176 | votepct=0.012, 177 | elex_id="test_lib", 178 | ), 179 | ] 180 | ), 181 | structs.StateData( 182 | counties={ 183 | "12345": structs.County( 184 | P=structs.CountyPresidentialResult( 185 | dem=structs.StateSummaryCandidateNamed( 186 | first_name="Joe", 187 | last_name="Biden", 188 | pop_vote=12345, 189 | pop_pct=0.234, 190 | pop_vote_history={"A": 1, "B": 2,}, 191 | ), 192 | gop=structs.StateSummaryCandidateNamed( 193 | first_name="Donald", 194 | last_name="Trump", 195 | pop_vote=67890, 196 | pop_pct=0.567, 197 | pop_vote_history={"B": 10, "C": 20,}, 198 | ), 199 | oth=structs.StateSummaryCandidateUnnamed( 200 | pop_vote=123, 201 | pop_pct=0.012, 202 | pop_vote_history={"C": 100, "D": 200,}, 203 | ), 204 | ) 205 | ) 206 | } 207 | ), 208 | ) 209 | 210 | 211 | def test_senate_result(): 212 | mock_historicals["test_loeffler"] = {"A": 1, "B": 2} 213 | mock_historicals["test_collins"] = {"B": 10, "C": 20} 214 | mock_historicals["test_hazel"] = {"C": 100, "D": 200} 215 | 216 | assert_result( 217 | exporter("GA").run_export( 218 | [ 219 | res_s( 220 | "GA-S", 221 | "12345", 222 | "GOP", 223 | "Kelly", 224 | "Loeffler", 225 | elex_id="test_loeffler", 226 | votecount=789, 227 | votepct=0.4, 228 | ), 229 | res_s( 230 | "GA-S", 231 | "12345", 232 | "GOP", 233 | "Doug", 234 | "Collins", 235 | elex_id="test_collins", 236 | votecount=456, 237 | votepct=0.3, 238 | ), 239 | res_s( 240 | "GA-S", 241 | "12345", 242 | "Lib", 243 | "Shane", 244 | "Hazel", 245 | elex_id="test_hazel", 246 | votecount=123, 247 | votepct=0.2, 248 | ), 249 | res_s( 250 | "GA", 251 | "67890", 252 | "Dem", 253 | "Jon", 254 | "Ossof", 255 | elex_id="test_ossof", 256 | votecount=1, 257 | votepct=0.2, 258 | ), 259 | ] 260 | ), 261 | structs.StateData( 262 | counties={ 263 | "12345": structs.County( 264 | S={ 265 | "GA-S": structs.CountyCongressionalResult( 266 | gop=structs.StateSummaryCandidateNamed( 267 | first_name="Kelly", 268 | last_name="Loeffler", 269 | pop_vote=789, 270 | pop_pct=0.4, 271 | pop_vote_history={"A": 1, "B": 2}, 272 | ), 273 | oth=structs.StateSummaryCandidateUnnamed( 274 | pop_vote=123 + 456, 275 | pop_pct=0.2 + 0.3, 276 | pop_vote_history={"B": 10, "C": 20 + 100, "D": 200}, 277 | ), 278 | multiple_gop=True, 279 | ), 280 | } 281 | ), 282 | "67890": structs.County( 283 | S={ 284 | "GA": structs.CountyCongressionalResult( 285 | dem=structs.StateSummaryCandidateNamed( 286 | first_name="Jon", 287 | last_name="Ossof", 288 | pop_vote=1, 289 | pop_pct=0.2, 290 | ) 291 | ), 292 | } 293 | ), 294 | } 295 | ), 296 | ) 297 | 298 | 299 | def test_house_result(): 300 | mock_historicals["test_pelosi"] = {"A": 1, "B": 2} 301 | mock_historicals["test_buttar"] = {"B": 10, "C": 20} 302 | 303 | assert_result( 304 | exporter("CA").run_export( 305 | [ 306 | res_h( 307 | "CA", 308 | 12, 309 | "12345", 310 | "Dem", 311 | "Nancy", 312 | "Pelosi", 313 | votecount=789, 314 | votepct=0.4, 315 | elex_id="test_pelosi", 316 | ), 317 | res_h( 318 | "CA", 319 | 12, 320 | "12345", 321 | "Dem", 322 | "Shahid", 323 | "Buttar", 324 | votecount=456, 325 | votepct=0.3, 326 | elex_id="test_buttar", 327 | ), 328 | ], 329 | ), 330 | structs.StateData( 331 | counties={ 332 | "12345": structs.County( 333 | H={ 334 | "12": structs.StateSummaryCongressionalResult( 335 | dem=structs.StateSummaryCandidateNamed( 336 | first_name="Nancy", 337 | last_name="Pelosi", 338 | pop_vote=789, 339 | pop_pct=0.4, 340 | pop_vote_history={"A": 1, "B": 2}, 341 | ), 342 | oth=structs.StateSummaryCandidateUnnamed( 343 | pop_vote=456, 344 | pop_pct=0.3, 345 | pop_vote_history={"B": 10, "C": 20}, 346 | ), 347 | multiple_dem=True, 348 | ) 349 | } 350 | ) 351 | } 352 | ), 353 | ) 354 | 355 | 356 | def test_house_at_large_result(): 357 | assert_result( 358 | exporter("AK").run_export( 359 | [ 360 | res_h( 361 | "AK", 362 | 1, 363 | "10002", 364 | "GOP", 365 | "Don", 366 | "Young", 367 | votecount=12345, 368 | votepct=0.234, 369 | ) 370 | ], 371 | ), 372 | structs.StateData( 373 | counties={ 374 | "10002": structs.County( 375 | H={ 376 | "AL": structs.StateSummaryCongressionalResult( 377 | gop=structs.StateSummaryCandidateNamed( 378 | first_name="Don", 379 | last_name="Young", 380 | pop_vote=12345, 381 | pop_pct=0.234, 382 | ), 383 | ) 384 | } 385 | ) 386 | } 387 | ), 388 | ) 389 | -------------------------------------------------------------------------------- /enip_backend/export/national.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from datetime import datetime 3 | from typing import Any, Iterable, List 4 | 5 | from ddtrace import tracer 6 | 7 | from ..enip_common.states import AT_LARGE_HOUSE_STATES, DISTRICTS_BY_STATE 8 | from . import structs 9 | from .helpers import ( 10 | HistoricalResults, 11 | SQLRecord, 12 | handle_candidate_results, 13 | load_calls, 14 | load_comments, 15 | load_election_results, 16 | load_historicals, 17 | ) 18 | 19 | 20 | class NationalDataExporter: 21 | def __init__(self, ingest_run_id: str, ingest_run_dt: datetime): 22 | self.ingest_run_id = ingest_run_id 23 | self.ingest_run_dt = ingest_run_dt 24 | self.historical_counts: HistoricalResults = {} 25 | self.data = structs.NationalData() 26 | 27 | def grant_electoral_votes(self, party: structs.Party, count: int) -> None: 28 | """ 29 | Helper function to add electoral votes to the national summary when 30 | we've called a state for a candidate 31 | """ 32 | race = self.data.national_summary.P 33 | if party == structs.Party.DEM: 34 | assert race.dem is not None 35 | race.dem.elect_won += count 36 | elif party == structs.Party.GOP: 37 | assert race.gop is not None 38 | race.gop.elect_won += count 39 | else: 40 | race.oth.elect_won += count 41 | 42 | def grant_congressional_seat( 43 | self, data: structs.NationalSummaryWinnerCount, party: structs.Party 44 | ): 45 | """ 46 | Helper function to add one to the senate/house national summary 47 | """ 48 | if party == structs.Party.DEM: 49 | data.dem.won += 1 50 | elif party == structs.Party.GOP: 51 | data.gop.won += 1 52 | else: 53 | data.oth.won += 1 54 | 55 | def record_ntl_result(self, record: SQLRecord) -> None: 56 | """ 57 | Records a "national"-level result. These results are just the national-level 58 | results for a presidential candidate 59 | """ 60 | assert record.officeid == "P" 61 | assert record.statepostal == "US" 62 | handle_candidate_results( 63 | self.data.national_summary.P, 64 | structs.NationalSummaryPresidentCandidateNamed, 65 | record, 66 | self.historical_counts, 67 | ) 68 | 69 | def record_district_result(self, record: SQLRecord) -> None: 70 | """ 71 | Records a "district"-level result. These results are presidential results 72 | for congressional district and at-large NE and ME districts. 73 | """ 74 | assert record.officeid == "P" 75 | 76 | # map reporting unit name to district to get the effective state name 77 | # (for NE and ME, we treat NE-1, ME-2, etc. as their own states for 78 | # the purposes of the presidential) 79 | if record.reportingunitname == "At Large": 80 | state = record.statepostal 81 | elif record.reportingunitname == "District 1": 82 | state = f"{record.statepostal}-01" 83 | elif record.reportingunitname == "District 2": 84 | state = f"{record.statepostal}-02" 85 | elif record.reportingunitname == "District 3": 86 | state = f"{record.statepostal}-03" 87 | else: 88 | raise RuntimeError( 89 | f"Invalid {record.statepostal} district: {record.reportingunitname}" 90 | ) 91 | 92 | # Initialize the state summaries 93 | if state not in self.data.state_summaries: 94 | if record.reportingunitname == "At Large": 95 | self.data.state_summaries[state] = structs.StateSummary() 96 | else: 97 | self.data.state_summaries[state] = structs.PresidentialCDSummary() 98 | 99 | # Add the results from this record 100 | handle_candidate_results( 101 | self.data.state_summaries[state].P, 102 | structs.StateSummaryCandidateNamed, 103 | record, 104 | self.historical_counts, 105 | ) 106 | 107 | # Handle a winner call 108 | # IMPORTANT: The AP has a bug where the `winner` property of congressional 109 | # districts is always set to the at-large winner of that state. We implement 110 | # the AP's suggested workaround: to ignore the `winner` property of 111 | # congressional districts and instead look at whether electwon is > 0. 112 | if record.electwon > 0 and self.calls["P"].get(state): 113 | logging.info(f"Calling a presidential winner for CD {state}") 114 | party = structs.Party.from_ap(record.party) 115 | 116 | # mark them as the winner of the race 117 | self.data.state_summaries[state].P.winner = party 118 | 119 | # Give the candidate the electoral votes 120 | self.grant_electoral_votes(party, record.electwon) 121 | 122 | def record_state_presidential_result(self, record: SQLRecord) -> None: 123 | """ 124 | Records a "state"-level presidential result. 125 | """ 126 | state = record.statepostal 127 | if state in DISTRICTS_BY_STATE: 128 | # Ignore state-level results for ME and NE -- we use district-level 129 | # results (with the results for the At Large district reported as the 130 | # statewide results) 131 | return 132 | 133 | # Initialize the state summary 134 | if state not in self.data.state_summaries: 135 | self.data.state_summaries[state] = structs.StateSummary() 136 | 137 | # Add the results from this record 138 | handle_candidate_results( 139 | self.data.state_summaries[state].P, 140 | structs.StateSummaryCandidateNamed, 141 | record, 142 | self.historical_counts, 143 | ) 144 | 145 | # Handle a winner call 146 | if record.winner and self.calls["P"].get(state): 147 | logging.info(f"Calling a presidential winner for {state}") 148 | 149 | party = structs.Party.from_ap(record.party) 150 | 151 | # mark them as the winner of the race 152 | self.data.state_summaries[state].P.winner = party 153 | 154 | # Give the candidate the electoral votes 155 | self.grant_electoral_votes(party, record.electtotal) 156 | 157 | def record_state_senate_result(self, record: SQLRecord) -> None: 158 | """ 159 | Records a Senate result 160 | """ 161 | # Get the effective state name. We treat GA-S as a state for the 162 | # purposes of the Georgia special election 163 | state = record.statepostal 164 | if state == "GA" and record.seatnum == 2: 165 | # Georgia senate special election 166 | state = "GA-S" 167 | 168 | # Initialize the state summary, and the senate component of the state 169 | # summary 170 | if state not in self.data.state_summaries: 171 | if state == "GA-S": 172 | self.data.state_summaries[state] = structs.SenateSpecialSummary() 173 | else: 174 | self.data.state_summaries[state] = structs.StateSummary() 175 | 176 | state_summary = self.data.state_summaries[state].S 177 | if not state_summary: 178 | # No results for this senate race yet; initialize it 179 | state_summary = structs.StateSummaryCongressionalResult() 180 | 181 | self.data.state_summaries[state].S = state_summary 182 | 183 | # Add the results from this record 184 | handle_candidate_results( 185 | state_summary, 186 | structs.StateSummaryCandidateNamed, 187 | record, 188 | self.historical_counts, 189 | ) 190 | 191 | # Handle a winner call 192 | if record.winner and self.calls["S"].get(state): 193 | logging.info(f"Calling a senate winner for {state}") 194 | 195 | party = structs.Party.from_ap(record.party) 196 | 197 | # mark them as the winner of the race 198 | state_summary.winner = party 199 | 200 | # Give the party a win in the national summary 201 | self.grant_congressional_seat(self.data.national_summary.S, party) 202 | 203 | def record_state_house_result(self, record: SQLRecord) -> None: 204 | """ 205 | Records a House result 206 | """ 207 | # Get the effective seat name. For states with a single seat, we 208 | # use the designation "AL" (At-Large) instead of a seat number. 209 | state = record.statepostal 210 | seat = str(record.seatnum).zfill(2) 211 | if state in AT_LARGE_HOUSE_STATES: 212 | seat = "AL" 213 | 214 | # Initialize the state summary, and this house race's summary 215 | if state not in self.data.state_summaries: 216 | self.data.state_summaries[state] = structs.StateSummary() 217 | 218 | if seat not in self.data.state_summaries[state].H: 219 | self.data.state_summaries[state].H[ 220 | seat 221 | ] = structs.StateSummaryCongressionalResult() 222 | 223 | seat_results = self.data.state_summaries[state].H[seat] 224 | 225 | # Add the results from this record 226 | handle_candidate_results( 227 | seat_results, 228 | structs.StateSummaryCandidateNamed, 229 | record, 230 | self.historical_counts, 231 | ) 232 | 233 | # Handle a winner call 234 | # For the house, we just use AP results with no editorializing 235 | if record.winner: 236 | party = structs.Party.from_ap(record.party) 237 | 238 | # mark them as the winner of the race 239 | seat_results.winner = party 240 | 241 | # Give the party a win in the national summary 242 | self.grant_congressional_seat(self.data.national_summary.H, party) 243 | 244 | def run_export( 245 | self, preloaded_results: Iterable[SQLRecord] 246 | ) -> structs.NationalData: 247 | self.data = structs.NationalData() 248 | 249 | sql_filter = "level IN ('national', 'state', 'district')" 250 | filter_params: List[Any] = [] 251 | 252 | with tracer.trace("enip.export.national.historicals"): 253 | self.historical_counts = load_historicals( 254 | self.ingest_run_dt, sql_filter, filter_params 255 | ) 256 | 257 | with tracer.trace("enip.export.national.load_comments"): 258 | self.comments = load_comments() 259 | 260 | with tracer.trace("enip.export.national.load_calls"): 261 | self.calls = load_calls() 262 | 263 | def handle_record(record): 264 | if record.level == "national": 265 | self.record_ntl_result(record) 266 | elif record.level == "district": 267 | self.record_district_result(record) 268 | elif record.level == "state" and record.officeid == "P": 269 | self.record_state_presidential_result(record) 270 | elif record.level == "state" and record.officeid == "S": 271 | self.record_state_senate_result(record) 272 | elif record.level == "state" and record.officeid == "H": 273 | self.record_state_house_result(record) 274 | else: 275 | raise RuntimeError( 276 | f"Uncategorizable result: {record.elex_id} {record.level} {record.officeid}" 277 | ) 278 | 279 | if preloaded_results: 280 | for record in preloaded_results: 281 | handle_record(record) 282 | else: 283 | for record in load_election_results( 284 | self.ingest_run_id, sql_filter, filter_params 285 | ): 286 | handle_record(record) 287 | 288 | # Add commentary 289 | for race, comments in self.comments["P"].items(): 290 | for comment in comments: 291 | self.data.state_summaries[race].P.comments.append(comment) 292 | 293 | for race, comments in self.comments["S"].items(): 294 | for comment in comments: 295 | senate_data = self.data.state_summaries[race].S 296 | if senate_data is None: 297 | raise RuntimeError( 298 | f"Got a comment for nonexistant senate race {race}" 299 | ) 300 | 301 | senate_data.comments.append(comment) 302 | 303 | for race, comments in self.comments["H"].items(): 304 | for comment in comments: 305 | state, seat = race.split("-") 306 | self.data.state_summaries[state].H[seat].comments.append(comment) 307 | 308 | for comment in self.comments["N"]["N"]: 309 | self.data.national_summary.comments.append(comment) 310 | 311 | # Call the presidential winner 312 | pres_summary = self.data.national_summary.P 313 | if pres_summary.dem and pres_summary.dem.elect_won >= 270: 314 | pres_summary.winner = structs.Party.DEM 315 | elif pres_summary.gop and pres_summary.gop.elect_won >= 270: 316 | pres_summary.winner = structs.Party.GOP 317 | 318 | return self.data 319 | -------------------------------------------------------------------------------- /enip_backend/export/schemas/national.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$id": "https://schema.voteamerica.com/enip/national.schema.json", 3 | "$schema": "http://json-schema.org/draft-07/schema#", 4 | "title": "ENIP National Data Schema", 5 | "type": "object", 6 | "definitions": { 7 | "winner": { 8 | "$id": "#winner", 9 | "oneOf": [ 10 | {"type": "string", "enum": ["dem", "gop", "oth"]}, 11 | {"type": "null"} 12 | ] 13 | }, 14 | "comments": { 15 | "$id": "#comments", 16 | "type": "array", 17 | "items": { 18 | "type": "object", 19 | "additionalProperties": false, 20 | "required": [ 21 | "timestamp", 22 | "author", 23 | "title", 24 | "body" 25 | ], 26 | "properties": { 27 | "timestamp": { 28 | "type": "string", 29 | "format": "datetime" 30 | }, 31 | "author": { "type": "string" }, 32 | "title": { "type": "string" }, 33 | "body": { "type": "string" } 34 | } 35 | } 36 | }, 37 | "national_summary_p_candidate_named": { 38 | "$id": "#national_summary_p_candidate_named", 39 | "type": "object", 40 | "additionalProperties": false, 41 | "required": [ 42 | "firstName", 43 | "lastName", 44 | "popVote", 45 | "popPct", 46 | "electWon", 47 | "popVoteHistory" 48 | ], 49 | "properties": { 50 | "firstName": {"type": "string"}, 51 | "lastName": {"type": "string"}, 52 | "popVote": {"type": "integer"}, 53 | "popPct": {"type": "number"}, 54 | "electWon": {"type": "integer"}, 55 | "popVoteHistory": { 56 | "type": "object", 57 | "propertyNames": { 58 | "type": "string", 59 | "format": "datetime" 60 | }, 61 | "additionalProperties": { 62 | "type": "integer" 63 | } 64 | } 65 | } 66 | }, 67 | "national_summary_p_candidate_unnamed": { 68 | "$id": "#national_summary_p_candidate_unnamed", 69 | "type": "object", 70 | "additionalProperties": false, 71 | "required": [ 72 | "popVote", 73 | "popPct", 74 | "electWon" 75 | ], 76 | "properties": { 77 | "popVote": {"type": "integer"}, 78 | "popPct": {"type": "number"}, 79 | "electWon": {"type": "integer"}, 80 | "popVoteHistory": { 81 | "type": "object", 82 | "propertyNames": { 83 | "type": "string", 84 | "format": "datetime" 85 | }, 86 | "additionalProperties": { 87 | "type": "integer" 88 | } 89 | } 90 | } 91 | }, 92 | "national_summary_p": { 93 | "$id": "#national_summary_p", 94 | "type": "object", 95 | "additionalProperties": false, 96 | "required": [ 97 | "dem", 98 | "gop", 99 | "oth", 100 | "winner" 101 | ], 102 | "properties": { 103 | "dem": {"$ref": "#/definitions/national_summary_p_candidate_named"}, 104 | "gop": {"$ref": "#/definitions/national_summary_p_candidate_named"}, 105 | "oth": {"$ref": "#/definitions/national_summary_p_candidate_unnamed"}, 106 | "winner": {"$ref": "#/definitions/winner"} 107 | } 108 | }, 109 | "national_summary_winner_count": { 110 | "$id": "#national_summary_winner_count", 111 | "type": "object", 112 | "additionalProperties": false, 113 | "required": ["dem", "gop", "oth"], 114 | "properties": { 115 | "dem": { 116 | "type": "object", 117 | "additionalProperties": false, 118 | "required": ["won"], 119 | "properties": { 120 | "won": { "type": "integer" } 121 | } 122 | }, 123 | "gop": { 124 | "type": "object", 125 | "additionalProperties": false, 126 | "required": ["won"], 127 | "properties": { 128 | "won": { "type": "integer" } 129 | } 130 | }, 131 | "oth": { 132 | "type": "object", 133 | "additionalProperties": false, 134 | "required": ["won"], 135 | "properties": { 136 | "won": { "type": "integer" } 137 | } 138 | } 139 | } 140 | }, 141 | "state_summary_candidate_named": { 142 | "$id": "#state_summary_candidate_named", 143 | "type": "object", 144 | "additionalProperties": false, 145 | "required": [ 146 | "firstName", 147 | "lastName", 148 | "popVote", 149 | "popPct", 150 | "popVoteHistory" 151 | ], 152 | "properties": { 153 | "firstName": {"type": "string"}, 154 | "lastName": {"type": "string"}, 155 | "popVote": {"type": "integer"}, 156 | "popPct": {"type": "number"}, 157 | "popVoteHistory": { 158 | "type": "object", 159 | "propertyNames": { 160 | "type": "string", 161 | "format": "datetime" 162 | }, 163 | "additionalProperties": { 164 | "type": "integer" 165 | } 166 | } 167 | } 168 | }, 169 | "state_summary_candidate_unnamed": { 170 | "$id": "#state_summary_candidate_unnamed", 171 | "type": "object", 172 | "additionalProperties": false, 173 | "required": [ 174 | "popVote", 175 | "popPct", 176 | "popVoteHistory" 177 | ], 178 | "properties": { 179 | "popVote": {"type": "integer"}, 180 | "popPct": {"type": "number"}, 181 | "popVoteHistory": { 182 | "type": "object", 183 | "propertyNames": { 184 | "type": "string", 185 | "format": "datetime" 186 | }, 187 | "additionalProperties": { 188 | "type": "integer" 189 | } 190 | } 191 | } 192 | }, 193 | "state_summary_p": { 194 | "$id": "#state_summary_p", 195 | "type": "object", 196 | "additionalProperties": false, 197 | "required": [ 198 | "dem", 199 | "gop", 200 | "oth", 201 | "winner", 202 | "comments" 203 | ], 204 | "properties": { 205 | "dem": {"$ref": "#/definitions/state_summary_candidate_named" }, 206 | "gop": {"$ref": "#/definitions/state_summary_candidate_named" }, 207 | "oth": {"$ref": "#/definitions/state_summary_candidate_unnamed" }, 208 | "winner": {"$ref": "#/definitions/winner"}, 209 | "comments": {"$ref": "#/definitions/comments"} 210 | } 211 | }, 212 | "state_summary_congressional_result": { 213 | "$id": "#state_summary_congressional_result", 214 | "type": "object", 215 | "additionalProperties": false, 216 | "required": ["dem", "gop", "oth", "multipleDem", "multipleGop", "comments"], 217 | "properties": { 218 | "dem": { 219 | "oneOf": [ 220 | {"type": "null" }, 221 | {"$ref": "#/definitions/state_summary_candidate_named" } 222 | ] 223 | }, 224 | "gop": { 225 | "oneOf": [ 226 | {"type": "null" }, 227 | {"$ref": "#/definitions/state_summary_candidate_named" } 228 | ] 229 | }, 230 | "oth": {"$ref": "#/definitions/state_summary_candidate_unnamed" }, 231 | "multipleDem": {"type": "boolean"}, 232 | "multipleGop": {"type": "boolean"}, 233 | "winner": {"$ref": "#/definitions/winner"}, 234 | "comments": {"$ref": "#/definitions/comments"} 235 | } 236 | }, 237 | "state_summary": { 238 | "$id": "#state_summary", 239 | "type": "object", 240 | "additionalProperties": false, 241 | "required": ["P", "H", "S"], 242 | "properties": { 243 | "P": { 244 | "$ref": "#/definitions/state_summary_p" 245 | }, 246 | "S": { 247 | "oneOf": [ 248 | {"type": "null"}, 249 | {"$ref": "#/definitions/state_summary_congressional_result"} 250 | ] 251 | }, 252 | "H": { 253 | "type": "object", 254 | "additionalProperties": { 255 | "$ref": "#/definitions/state_summary_congressional_result" 256 | }, 257 | "propertyNames": { 258 | "pattern": "^(\\d\\d)|(AL)$" 259 | } 260 | } 261 | } 262 | }, 263 | "presidential_cd_summary": { 264 | "$id": "#presidential_cd_summary", 265 | "type": "object", 266 | "additionalProperties": false, 267 | "required": ["P"], 268 | "properties": { 269 | "P": { 270 | "$ref": "#/definitions/state_summary_p" 271 | } 272 | } 273 | }, 274 | "senate_special_summary": { 275 | "$id": "#senate_special_summary", 276 | "type": "object", 277 | "additionalProperties": false, 278 | "required": ["S"], 279 | "properties": { 280 | "S": { 281 | "$ref": "#/definitions/state_summary_congressional_result" 282 | } 283 | } 284 | } 285 | }, 286 | "properties": { 287 | "nationalSummary": { 288 | "type": "object", 289 | "description": "National summary", 290 | "additionalProperties": false, 291 | "required": ["P", "H", "S", "comments"], 292 | "properties": { 293 | "P": { 294 | "$ref": "#/definitions/national_summary_p" 295 | }, 296 | "S": { 297 | "$ref": "#/definitions/national_summary_winner_count" 298 | }, 299 | "H": { 300 | "$ref": "#/definitions/national_summary_winner_count" 301 | }, 302 | "comments": {"$ref": "#/definitions/comments"} 303 | } 304 | }, 305 | "stateSummaries": { 306 | "type": "object", 307 | "description": "National summary", 308 | "additionalProperties": false, 309 | "required": [ 310 | "AL", 311 | "AK", 312 | "AZ", 313 | "AR", 314 | "CA", 315 | "CO", 316 | "CT", 317 | "DC", 318 | "DE", 319 | "FL", 320 | "GA", 321 | "HI", 322 | "ID", 323 | "IL", 324 | "IN", 325 | "IA", 326 | "KS", 327 | "KY", 328 | "LA", 329 | "ME", 330 | "MD", 331 | "MA", 332 | "MI", 333 | "MN", 334 | "MS", 335 | "MO", 336 | "MT", 337 | "NE", 338 | "NV", 339 | "NH", 340 | "NJ", 341 | "NM", 342 | "NY", 343 | "NC", 344 | "ND", 345 | "OH", 346 | "OK", 347 | "OR", 348 | "PA", 349 | "RI", 350 | "SC", 351 | "SD", 352 | "TN", 353 | "TX", 354 | "UT", 355 | "VT", 356 | "VA", 357 | "WA", 358 | "WV", 359 | "WI", 360 | "WY", 361 | "NE-01", 362 | "NE-02", 363 | "NE-03", 364 | "ME-01", 365 | "ME-02", 366 | "GA-S" 367 | ], 368 | "properties": { 369 | "AL": { "$ref": "#/definitions/state_summary" }, 370 | "AK": { "$ref": "#/definitions/state_summary" }, 371 | "AZ": { "$ref": "#/definitions/state_summary" }, 372 | "AR": { "$ref": "#/definitions/state_summary" }, 373 | "CA": { "$ref": "#/definitions/state_summary" }, 374 | "CO": { "$ref": "#/definitions/state_summary" }, 375 | "CT": { "$ref": "#/definitions/state_summary" }, 376 | "DC": { "$ref": "#/definitions/state_summary" }, 377 | "DE": { "$ref": "#/definitions/state_summary" }, 378 | "FL": { "$ref": "#/definitions/state_summary" }, 379 | "GA": { "$ref": "#/definitions/state_summary" }, 380 | "HI": { "$ref": "#/definitions/state_summary" }, 381 | "ID": { "$ref": "#/definitions/state_summary" }, 382 | "IL": { "$ref": "#/definitions/state_summary" }, 383 | "IN": { "$ref": "#/definitions/state_summary" }, 384 | "IA": { "$ref": "#/definitions/state_summary" }, 385 | "KS": { "$ref": "#/definitions/state_summary" }, 386 | "KY": { "$ref": "#/definitions/state_summary" }, 387 | "LA": { "$ref": "#/definitions/state_summary" }, 388 | "ME": { "$ref": "#/definitions/state_summary" }, 389 | "MD": { "$ref": "#/definitions/state_summary" }, 390 | "MA": { "$ref": "#/definitions/state_summary" }, 391 | "MI": { "$ref": "#/definitions/state_summary" }, 392 | "MN": { "$ref": "#/definitions/state_summary" }, 393 | "MS": { "$ref": "#/definitions/state_summary" }, 394 | "MO": { "$ref": "#/definitions/state_summary" }, 395 | "MT": { "$ref": "#/definitions/state_summary" }, 396 | "NE": { "$ref": "#/definitions/state_summary" }, 397 | "NV": { "$ref": "#/definitions/state_summary" }, 398 | "NH": { "$ref": "#/definitions/state_summary" }, 399 | "NJ": { "$ref": "#/definitions/state_summary" }, 400 | "NM": { "$ref": "#/definitions/state_summary" }, 401 | "NY": { "$ref": "#/definitions/state_summary" }, 402 | "NC": { "$ref": "#/definitions/state_summary" }, 403 | "ND": { "$ref": "#/definitions/state_summary" }, 404 | "OH": { "$ref": "#/definitions/state_summary" }, 405 | "OK": { "$ref": "#/definitions/state_summary" }, 406 | "OR": { "$ref": "#/definitions/state_summary" }, 407 | "PA": { "$ref": "#/definitions/state_summary" }, 408 | "RI": { "$ref": "#/definitions/state_summary" }, 409 | "SC": { "$ref": "#/definitions/state_summary" }, 410 | "SD": { "$ref": "#/definitions/state_summary" }, 411 | "TN": { "$ref": "#/definitions/state_summary" }, 412 | "TX": { "$ref": "#/definitions/state_summary" }, 413 | "UT": { "$ref": "#/definitions/state_summary" }, 414 | "VT": { "$ref": "#/definitions/state_summary" }, 415 | "VA": { "$ref": "#/definitions/state_summary" }, 416 | "WA": { "$ref": "#/definitions/state_summary" }, 417 | "WV": { "$ref": "#/definitions/state_summary" }, 418 | "WI": { "$ref": "#/definitions/state_summary" }, 419 | "WY": { "$ref": "#/definitions/state_summary" }, 420 | "NE-01": { "$ref": "#/definitions/presidential_cd_summary" }, 421 | "NE-02": { "$ref": "#/definitions/presidential_cd_summary" }, 422 | "NE-03": { "$ref": "#/definitions/presidential_cd_summary" }, 423 | "ME-01": { "$ref": "#/definitions/presidential_cd_summary" }, 424 | "ME-02": { "$ref": "#/definitions/presidential_cd_summary" }, 425 | "GA-S": { "$ref": "#/definitions/senate_special_summary" } 426 | } 427 | } 428 | }, 429 | "required": ["nationalSummary", "stateSummaries"], 430 | "additionalProperties": false 431 | } 432 | -------------------------------------------------------------------------------- /enip_backend/export/national_test.py: -------------------------------------------------------------------------------- 1 | import json 2 | from datetime import datetime, timezone 3 | 4 | import pytest 5 | 6 | from . import structs 7 | from .helpers import Calls, Comments, HistoricalResults, SQLRecord 8 | from .national import NationalDataExporter 9 | 10 | mock_calls: Calls = {} 11 | mock_comments: Comments = {} 12 | mock_historicals: HistoricalResults = {} 13 | 14 | 15 | @pytest.fixture(autouse=True) 16 | def init_mocks(mocker): 17 | global mock_calls 18 | global mock_comments 19 | global mock_historicals 20 | 21 | mock_calls = {"P": {}, "S": {}} 22 | mock_comments = {"P": {}, "S": {}, "H": {}, "N": {"N": []}} 23 | mock_historicals = {} 24 | 25 | mock_load_calls = mocker.patch("enip_backend.export.national.load_calls") 26 | mock_load_calls.return_value = mock_calls 27 | 28 | mock_load_comments = mocker.patch("enip_backend.export.national.load_comments") 29 | mock_load_comments.return_value = mock_comments 30 | 31 | mock_load_historicals = mocker.patch( 32 | "enip_backend.export.national.load_historicals" 33 | ) 34 | mock_load_historicals.return_value = mock_historicals 35 | 36 | 37 | @pytest.fixture 38 | def exporter(): 39 | return NationalDataExporter( 40 | "test_run", datetime(2020, 11, 3, 8, 0, 0, tzinfo=timezone.utc) 41 | ) 42 | 43 | 44 | def assert_result(actual, expected): 45 | # We make assertions using the JSON representation 46 | # because that prints much better in pytest 47 | assert json.loads(actual.json()) == json.loads(expected.json()) 48 | 49 | 50 | # Helpers for constructing results 51 | def state_summary(state, summary): 52 | r = structs.NationalData() 53 | r.state_summaries[state] = summary 54 | 55 | return r 56 | 57 | 58 | def national_summary(summary, *state_summaries): 59 | r = structs.NationalData() 60 | r.national_summary = summary 61 | 62 | for summary_struct in state_summaries: 63 | for state, summary in summary_struct.state_summaries.items(): 64 | r.state_summaries[state] = summary 65 | 66 | return r 67 | 68 | 69 | # Helpers for constructing SQLRecords 70 | def default_name(party, first, last): 71 | if party == "Dem": 72 | return (first or "Joe", last or "Biden") 73 | elif party == "GOP": 74 | return (first or "Donald", last or "Trump") 75 | else: 76 | return (first or "Foo", last or "Barson") 77 | 78 | 79 | def res_p_national( 80 | party, 81 | votecount, 82 | votepct, 83 | winner=False, 84 | first=None, 85 | last=None, 86 | ingest_id="test_run", 87 | elex_id="test_elex", 88 | ): 89 | first, last = default_name(party, first, last) 90 | 91 | return SQLRecord( 92 | ingest_id=ingest_id, 93 | elex_id=elex_id, 94 | statepostal="US", 95 | fipscode="12345", 96 | level="national", 97 | reportingunitname=None, 98 | officeid="P", 99 | seatnum=None, 100 | party=party, 101 | first=first, 102 | last=last, 103 | electtotal=538, 104 | electwon=0, 105 | votecount=votecount, 106 | votepct=votepct, 107 | winner=winner, 108 | ) 109 | 110 | 111 | def res_p_state( 112 | state, 113 | party, 114 | votecount, 115 | votepct, 116 | electtotal=10, 117 | winner=False, 118 | first=None, 119 | last=None, 120 | ingest_id="test_run", 121 | elex_id="test_elex", 122 | ): 123 | first, last = default_name(party, first, last) 124 | 125 | return SQLRecord( 126 | ingest_id=ingest_id, 127 | elex_id=elex_id, 128 | statepostal=state, 129 | fipscode="12345", 130 | level="state", 131 | reportingunitname=None, 132 | officeid="P", 133 | seatnum=None, 134 | party=party, 135 | first=first, 136 | last=last, 137 | electtotal=electtotal, 138 | electwon=0, 139 | votecount=votecount, 140 | votepct=votepct, 141 | winner=winner, 142 | ) 143 | 144 | 145 | def res_p_district( 146 | state, 147 | reportingunitname, 148 | party, 149 | votecount, 150 | votepct, 151 | electtotal=1, 152 | electwon=0, 153 | winner=False, 154 | first=None, 155 | last=None, 156 | ingest_id="test_run", 157 | elex_id="test_elex", 158 | ): 159 | first, last = default_name(party, first, last) 160 | 161 | return SQLRecord( 162 | ingest_id=ingest_id, 163 | elex_id=elex_id, 164 | statepostal=state, 165 | fipscode="12345", 166 | level="district", 167 | reportingunitname=reportingunitname, 168 | officeid="P", 169 | seatnum=None, 170 | party=party, 171 | first=first, 172 | last=last, 173 | electtotal=electtotal, 174 | electwon=electwon, 175 | votecount=votecount, 176 | votepct=votepct, 177 | winner=winner, 178 | ) 179 | 180 | 181 | def res_s( 182 | state, 183 | party, 184 | first, 185 | last, 186 | votecount, 187 | votepct, 188 | winner=False, 189 | ingest_id="test_run", 190 | elex_id="test_elex", 191 | ): 192 | seatnum = None 193 | if state == "GA-S": 194 | state = "GA" 195 | seatnum = 2 196 | 197 | return SQLRecord( 198 | ingest_id=ingest_id, 199 | elex_id=elex_id, 200 | statepostal=state, 201 | fipscode="12345", 202 | level="state", 203 | reportingunitname=None, 204 | officeid="S", 205 | seatnum=seatnum, 206 | party=party, 207 | first=first, 208 | last=last, 209 | electtotal=1, 210 | electwon=0, 211 | votecount=votecount, 212 | votepct=votepct, 213 | winner=winner, 214 | ) 215 | 216 | 217 | def res_h( 218 | state, 219 | seatnum, 220 | party, 221 | first, 222 | last, 223 | votecount, 224 | votepct, 225 | winner=False, 226 | ingest_id="test_run", 227 | elex_id="test_elex", 228 | ): 229 | return SQLRecord( 230 | ingest_id=ingest_id, 231 | elex_id=elex_id, 232 | statepostal=state, 233 | fipscode="12345", 234 | level="state", 235 | reportingunitname=None, 236 | officeid="H", 237 | seatnum=seatnum, 238 | party=party, 239 | first=first, 240 | last=last, 241 | electtotal=1, 242 | electwon=0, 243 | votecount=votecount, 244 | votepct=votepct, 245 | winner=winner, 246 | ) 247 | 248 | 249 | # National result 250 | def test_national_result(exporter): 251 | assert_result( 252 | exporter.run_export([res_p_national("Dem", votecount=12345, votepct=0.234)]), 253 | structs.NationalData( 254 | national_summary=structs.NationalSummary( 255 | P=structs.NationalSummaryPresident( 256 | dem=structs.NationalSummaryPresidentCandidateNamed( 257 | first_name="Joe", 258 | last_name="Biden", 259 | pop_vote=12345, 260 | pop_pct=0.234, 261 | elect_won=0, 262 | ) 263 | ) 264 | ) 265 | ), 266 | ) 267 | 268 | 269 | # ME statewide (ignore) 270 | def test_statewide_me_prez_result(exporter): 271 | assert_result( 272 | exporter.run_export([res_p_state("ME", "Dem", votecount=12345, votepct=0.234)]), 273 | structs.NationalData(), 274 | ) 275 | 276 | 277 | # ME at large 278 | def test_atlarge_me_prez_result(exporter): 279 | assert_result( 280 | exporter.run_export( 281 | [ 282 | SQLRecord( 283 | ingest_id="test_run", 284 | elex_id="test_elex", 285 | statepostal="ME", 286 | fipscode="12345", 287 | level="district", 288 | reportingunitname="At Large", 289 | officeid="P", 290 | seatnum=None, 291 | party="GOP", 292 | first="Donald", 293 | last="Trump", 294 | electtotal=2, 295 | electwon=0, 296 | votecount=12345, 297 | votepct=0.234, 298 | winner=False, 299 | ) 300 | ] 301 | ), 302 | structs.NationalData( 303 | state_summaries={ 304 | "ME": structs.StateSummary( 305 | P=structs.StateSummaryPresident( 306 | gop=structs.StateSummaryCandidateNamed( 307 | first_name="Donald", 308 | last_name="Trump", 309 | pop_vote=12345, 310 | pop_pct=0.234, 311 | ) 312 | ) 313 | ) 314 | } 315 | ), 316 | ) 317 | 318 | 319 | # ME-01 320 | def test_me_01_prez_result(exporter): 321 | assert_result( 322 | exporter.run_export([res_p_district("ME", "District 1", "Dem", 12345, 0.234)]), 323 | state_summary( 324 | "ME-01", 325 | structs.PresidentialCDSummary( 326 | P=structs.StateSummaryPresident( 327 | dem=structs.StateSummaryCandidateNamed( 328 | first_name="Joe", 329 | last_name="Biden", 330 | pop_vote=12345, 331 | pop_pct=0.234, 332 | ) 333 | ) 334 | ), 335 | ), 336 | ) 337 | 338 | 339 | # NE-02 winner call, published 340 | def test_ne_02_prez_call_published(exporter): 341 | mock_calls["P"]["NE-02"] = True 342 | 343 | assert_result( 344 | exporter.run_export( 345 | [ 346 | res_p_national("Dem", votecount=67890, votepct=0.567), 347 | res_p_national("GOP", votecount=67891, votepct=0.568), 348 | # reproduce AP but where winner lies but electwon is correct 349 | res_p_district( 350 | "NE", 351 | "District 2", 352 | "Dem", 353 | votecount=12345, 354 | votepct=0.234, 355 | winner=False, 356 | electwon=1, 357 | ), 358 | res_p_district( 359 | "NE", 360 | "District 2", 361 | "GOP", 362 | votecount=12346, 363 | votepct=0.235, 364 | winner=True, 365 | electwon=0, 366 | ), 367 | ], 368 | ), 369 | national_summary( 370 | structs.NationalSummary( 371 | P=structs.NationalSummaryPresident( 372 | dem=structs.NationalSummaryPresidentCandidateNamed( 373 | first_name="Joe", 374 | last_name="Biden", 375 | pop_vote=67890, 376 | pop_pct=0.567, 377 | elect_won=1, 378 | ), 379 | gop=structs.NationalSummaryPresidentCandidateNamed( 380 | first_name="Donald", 381 | last_name="Trump", 382 | pop_vote=67891, 383 | pop_pct=0.568, 384 | elect_won=0, 385 | ), 386 | ) 387 | ), 388 | state_summary( 389 | "NE-02", 390 | structs.PresidentialCDSummary( 391 | P=structs.StateSummaryPresident( 392 | dem=structs.StateSummaryCandidateNamed( 393 | first_name="Joe", 394 | last_name="Biden", 395 | pop_vote=12345, 396 | pop_pct=0.234, 397 | ), 398 | gop=structs.StateSummaryCandidateNamed( 399 | first_name="Donald", 400 | last_name="Trump", 401 | pop_vote=12346, 402 | pop_pct=0.235, 403 | ), 404 | winner=structs.Party.DEM, 405 | ) 406 | ), 407 | ), 408 | ), 409 | ) 410 | 411 | 412 | # ME-01 winner call, unpublished 413 | def test_me_01_prez_call_unpublished(exporter): 414 | assert_result( 415 | exporter.run_export( 416 | [ 417 | res_p_national("Dem", votecount=67890, votepct=0.567), 418 | res_p_district( 419 | "ME", 420 | "District 1", 421 | "Dem", 422 | votecount=12345, 423 | votepct=0.234, 424 | winner=True, 425 | ), 426 | ], 427 | ), 428 | national_summary( 429 | structs.NationalSummary( 430 | P=structs.NationalSummaryPresident( 431 | dem=structs.NationalSummaryPresidentCandidateNamed( 432 | first_name="Joe", 433 | last_name="Biden", 434 | pop_vote=67890, 435 | pop_pct=0.567, 436 | ) 437 | ) 438 | ), 439 | state_summary( 440 | "ME-01", 441 | structs.PresidentialCDSummary( 442 | P=structs.StateSummaryPresident( 443 | dem=structs.StateSummaryCandidateNamed( 444 | first_name="Joe", 445 | last_name="Biden", 446 | pop_vote=12345, 447 | pop_pct=0.234, 448 | ), 449 | ) 450 | ), 451 | ), 452 | ), 453 | ) 454 | 455 | 456 | # state result 457 | def test_state_prez_result(exporter): 458 | assert_result( 459 | exporter.run_export( 460 | [res_p_state("CA", "Lib", votecount=12345, votepct=0.234)], 461 | ), 462 | state_summary( 463 | "CA", 464 | structs.StateSummary( 465 | P=structs.StateSummaryPresident( 466 | oth=structs.StateSummaryCandidateUnnamed( 467 | pop_vote=12345, pop_pct=0.234, 468 | ), 469 | ) 470 | ), 471 | ), 472 | ) 473 | 474 | 475 | # state winner call, published 476 | def test_state_prez_call_published(exporter): 477 | mock_calls["P"]["CA"] = True 478 | mock_calls["P"]["UT"] = True 479 | 480 | assert_result( 481 | exporter.run_export( 482 | [ 483 | res_p_national("GOP", votecount=67890, votepct=0.567), 484 | res_p_national("Lib", votecount=123, votepct=0.123), 485 | res_p_state( 486 | "CA", 487 | "GOP", 488 | votecount=12345, 489 | votepct=0.234, 490 | winner=True, 491 | electtotal=55, 492 | ), 493 | res_p_state( 494 | "UT", "Lib", votecount=123, votepct=0.123, winner=True, electtotal=6 495 | ), 496 | ], 497 | ), 498 | national_summary( 499 | structs.NationalSummary( 500 | P=structs.NationalSummaryPresident( 501 | gop=structs.NationalSummaryPresidentCandidateNamed( 502 | first_name="Donald", 503 | last_name="Trump", 504 | pop_vote=67890, 505 | pop_pct=0.567, 506 | elect_won=55, 507 | ), 508 | oth=structs.NationalSummaryPresidentCandidateUnnamed( 509 | pop_vote=123, pop_pct=0.123, elect_won=6, 510 | ), 511 | ) 512 | ), 513 | state_summary( 514 | "CA", 515 | structs.StateSummary( 516 | P=structs.StateSummaryPresident( 517 | gop=structs.StateSummaryCandidateNamed( 518 | first_name="Donald", 519 | last_name="Trump", 520 | pop_vote=12345, 521 | pop_pct=0.234, 522 | ), 523 | winner=structs.Party.GOP, 524 | ) 525 | ), 526 | ), 527 | state_summary( 528 | "UT", 529 | structs.StateSummary( 530 | P=structs.StateSummaryPresident( 531 | oth=structs.StateSummaryCandidateUnnamed( 532 | pop_vote=123, pop_pct=0.123, 533 | ), 534 | winner=structs.Party.OTHER, 535 | ) 536 | ), 537 | ), 538 | ), 539 | ) 540 | 541 | 542 | # state winner call, unpublished 543 | def test_state_prez_call_unpublished(exporter): 544 | mock_calls["P"]["UT"] = True 545 | 546 | assert_result( 547 | exporter.run_export( 548 | [ 549 | res_p_national("GOP", votecount=67890, votepct=0.567,), 550 | res_p_state( 551 | "CA", 552 | "GOP", 553 | electtotal=55, 554 | votecount=12345, 555 | votepct=0.234, 556 | winner=True, 557 | ), 558 | res_p_state("UT", "GOP", electtotal=6, votecount=456, votepct=0.456,), 559 | ], 560 | ), 561 | national_summary( 562 | structs.NationalSummary( 563 | P=structs.NationalSummaryPresident( 564 | gop=structs.NationalSummaryPresidentCandidateNamed( 565 | first_name="Donald", 566 | last_name="Trump", 567 | pop_vote=67890, 568 | pop_pct=0.567, 569 | ) 570 | ) 571 | ), 572 | state_summary( 573 | "CA", 574 | structs.StateSummary( 575 | P=structs.StateSummaryPresident( 576 | gop=structs.StateSummaryCandidateNamed( 577 | first_name="Donald", 578 | last_name="Trump", 579 | pop_vote=12345, 580 | pop_pct=0.234, 581 | ), 582 | ) 583 | ), 584 | ), 585 | state_summary( 586 | "UT", 587 | structs.StateSummary( 588 | P=structs.StateSummaryPresident( 589 | gop=structs.StateSummaryCandidateNamed( 590 | first_name="Donald", 591 | last_name="Trump", 592 | pop_vote=456, 593 | pop_pct=0.456, 594 | ), 595 | ) 596 | ), 597 | ), 598 | ), 599 | ) 600 | 601 | 602 | # senate result 603 | def test_state_senate_result(exporter): 604 | assert_result( 605 | exporter.run_export( 606 | [res_s("MA", "Grn", "Howie", "Hawkins", votecount=12345, votepct=0.234,),], 607 | ), 608 | state_summary( 609 | "MA", 610 | structs.StateSummary( 611 | S=structs.StateSummaryCongressionalResult( 612 | oth=structs.StateSummaryCandidateUnnamed( 613 | pop_vote=12345, pop_pct=0.234, 614 | ), 615 | ) 616 | ), 617 | ), 618 | ) 619 | 620 | 621 | # GA-S senate result 622 | def test_ga_special_senate_result(exporter): 623 | assert_result( 624 | exporter.run_export( 625 | [ 626 | res_s( 627 | "GA-S", "Dem", "Raphael", "Warnock", votecount=12345, votepct=0.234, 628 | ), 629 | ], 630 | ), 631 | state_summary( 632 | "GA-S", 633 | structs.SenateSpecialSummary( 634 | S=structs.StateSummaryCongressionalResult( 635 | dem=structs.StateSummaryCandidateNamed( 636 | first_name="Raphael", 637 | last_name="Warnock", 638 | pop_vote=12345, 639 | pop_pct=0.234, 640 | ), 641 | ) 642 | ), 643 | ), 644 | ) 645 | 646 | 647 | # sente winner call, published 648 | def test_senate_winner_call_published(exporter): 649 | mock_calls["S"]["NE"] = True 650 | 651 | assert_result( 652 | exporter.run_export( 653 | [ 654 | res_s( 655 | "NE", 656 | "GOP", 657 | "Ben", 658 | "Sasse", 659 | votecount=12345, 660 | votepct=0.234, 661 | winner=True, 662 | ), 663 | ], 664 | ), 665 | national_summary( 666 | structs.NationalSummary( 667 | S=structs.NationalSummaryWinnerCount( 668 | gop=structs.NationalSummaryWinnerCountEntry(won=1) 669 | ) 670 | ), 671 | state_summary( 672 | "NE", 673 | structs.StateSummary( 674 | S=structs.StateSummaryCongressionalResult( 675 | gop=structs.StateSummaryCandidateNamed( 676 | first_name="Ben", 677 | last_name="Sasse", 678 | pop_vote=12345, 679 | pop_pct=0.234, 680 | ), 681 | winner=structs.Party.GOP, 682 | ) 683 | ), 684 | ), 685 | ), 686 | ) 687 | 688 | 689 | # senate winner call, unpublished 690 | def test_senate_winner_call_unpublished(exporter): 691 | mock_calls["S"]["NE"] = False 692 | mock_calls["S"]["MA"] = True 693 | 694 | assert_result( 695 | exporter.run_export( 696 | [ 697 | res_s( 698 | "NE", 699 | "GOP", 700 | "Ben", 701 | "Sasse", 702 | votecount=12345, 703 | votepct=0.234, 704 | winner=True, 705 | ), 706 | res_s("MA", "Dem", "Ed", "Markey", votecount=12345, votepct=0.234), 707 | ], 708 | ), 709 | national_summary( 710 | structs.NationalSummary(), 711 | state_summary( 712 | "NE", 713 | structs.StateSummary( 714 | S=structs.StateSummaryCongressionalResult( 715 | gop=structs.StateSummaryCandidateNamed( 716 | first_name="Ben", 717 | last_name="Sasse", 718 | pop_vote=12345, 719 | pop_pct=0.234, 720 | ), 721 | ) 722 | ), 723 | ), 724 | state_summary( 725 | "MA", 726 | structs.StateSummary( 727 | S=structs.StateSummaryCongressionalResult( 728 | dem=structs.StateSummaryCandidateNamed( 729 | first_name="Ed", 730 | last_name="Markey", 731 | pop_vote=12345, 732 | pop_pct=0.234, 733 | ), 734 | ) 735 | ), 736 | ), 737 | ), 738 | ) 739 | 740 | 741 | # house result 742 | def test_state_house_result(exporter): 743 | assert_result( 744 | exporter.run_export( 745 | [res_h("GA", 4, "Dem", "Hank", "Johnson", votecount=12345, votepct=0.234)], 746 | ), 747 | state_summary( 748 | "GA", 749 | structs.StateSummary( 750 | H={ 751 | "04": structs.StateSummaryCongressionalResult( 752 | dem=structs.StateSummaryCandidateNamed( 753 | first_name="Hank", 754 | last_name="Johnson", 755 | pop_vote=12345, 756 | pop_pct=0.234, 757 | ), 758 | ) 759 | } 760 | ), 761 | ), 762 | ) 763 | 764 | 765 | # house at large result 766 | def test_state_house_at_large_result(exporter): 767 | assert_result( 768 | exporter.run_export( 769 | [res_h("AK", 1, "GOP", "Don", "Young", votecount=12345, votepct=0.234)], 770 | ), 771 | state_summary( 772 | "AK", 773 | structs.StateSummary( 774 | H={ 775 | "AL": structs.StateSummaryCongressionalResult( 776 | gop=structs.StateSummaryCandidateNamed( 777 | first_name="Don", 778 | last_name="Young", 779 | pop_vote=12345, 780 | pop_pct=0.234, 781 | ), 782 | ) 783 | } 784 | ), 785 | ), 786 | ) 787 | 788 | 789 | # house winner call 790 | def test_state_house_at_large_call(exporter): 791 | assert_result( 792 | exporter.run_export( 793 | [ 794 | res_h( 795 | "AK", 796 | 1, 797 | "Dem", 798 | "Alyse", 799 | "Galvin", 800 | votecount=12345, 801 | votepct=0.234, 802 | winner=True, 803 | ), 804 | res_h( 805 | "OH", 806 | 3, 807 | "Dem", 808 | "Joyce", 809 | "Beatty", 810 | votecount=12345, 811 | votepct=0.234, 812 | winner=True, 813 | ), 814 | res_h( 815 | "OH", 816 | 1, 817 | "Ind", 818 | "Kiumars", 819 | "Kiani", 820 | votecount=123, 821 | votepct=0.12, 822 | winner=True, 823 | ), 824 | ], 825 | ), 826 | national_summary( 827 | structs.NationalSummary( 828 | H=structs.NationalSummaryWinnerCount( 829 | dem=structs.NationalSummaryWinnerCountEntry(won=2), 830 | oth=structs.NationalSummaryWinnerCountEntry(won=1), 831 | ) 832 | ), 833 | state_summary( 834 | "AK", 835 | structs.StateSummary( 836 | H={ 837 | "AL": structs.StateSummaryCongressionalResult( 838 | dem=structs.StateSummaryCandidateNamed( 839 | first_name="Alyse", 840 | last_name="Galvin", 841 | pop_vote=12345, 842 | pop_pct=0.234, 843 | ), 844 | winner=structs.Party.DEM, 845 | ) 846 | } 847 | ), 848 | ), 849 | state_summary( 850 | "OH", 851 | structs.StateSummary( 852 | H={ 853 | "01": structs.StateSummaryCongressionalResult( 854 | oth=structs.StateSummaryCandidateUnnamed( 855 | first_name="Kiumars", 856 | last_name="Kiani", 857 | pop_vote=123, 858 | pop_pct=0.12, 859 | ), 860 | winner=structs.Party.OTHER, 861 | ), 862 | "03": structs.StateSummaryCongressionalResult( 863 | dem=structs.StateSummaryCandidateNamed( 864 | first_name="Joyce", 865 | last_name="Beatty", 866 | pop_vote=12345, 867 | pop_pct=0.234, 868 | ), 869 | winner=structs.Party.DEM, 870 | ), 871 | } 872 | ), 873 | ), 874 | ), 875 | ) 876 | 877 | 878 | # historicals, national-level 879 | def test_historical_national_counts(exporter): 880 | mock_historicals["test_dem"] = {"A": 1, "B": 2} 881 | mock_historicals["test_gop"] = {"B": 10, "C": 20} 882 | mock_historicals["test_lib"] = {"C": 100, "D": 200} 883 | 884 | assert_result( 885 | exporter.run_export( 886 | [ 887 | res_p_national( 888 | "Dem", votecount=12345, votepct=0.234, elex_id="test_dem" 889 | ), 890 | res_p_national( 891 | "GOP", votecount=67890, votepct=0.567, elex_id="test_gop" 892 | ), 893 | res_p_national("Lib", votecount=123, votepct=0.012, elex_id="test_lib"), 894 | ], 895 | ), 896 | national_summary( 897 | structs.NationalSummary( 898 | P=structs.NationalSummaryPresident( 899 | dem=structs.NationalSummaryPresidentCandidateNamed( 900 | first_name="Joe", 901 | last_name="Biden", 902 | pop_vote=12345, 903 | pop_pct=0.234, 904 | elect_won=0, 905 | pop_vote_history={"A": 1, "B": 2}, 906 | ), 907 | gop=structs.NationalSummaryPresidentCandidateNamed( 908 | first_name="Donald", 909 | last_name="Trump", 910 | pop_vote=67890, 911 | pop_pct=0.567, 912 | elect_won=0, 913 | pop_vote_history={"B": 10, "C": 20}, 914 | ), 915 | oth=structs.NationalSummaryPresidentCandidateNamed( 916 | first_name="Jo", 917 | last_name="Jorgensen", 918 | pop_vote=123, 919 | pop_pct=0.012, 920 | elect_won=0, 921 | pop_vote_history={"C": 100, "D": 200}, 922 | ), 923 | ) 924 | ) 925 | ), 926 | ) 927 | 928 | 929 | # multiple gop + historicals 930 | def test_historical_senate_counts(exporter): 931 | mock_historicals["test_loeffler"] = {"A": 1, "B": 2} 932 | mock_historicals["test_collins"] = {"B": 10, "C": 20} 933 | mock_historicals["test_hazel"] = {"C": 100, "D": 200} 934 | 935 | assert_result( 936 | exporter.run_export( 937 | [ 938 | res_s( 939 | "GA-S", 940 | "GOP", 941 | "Kelly", 942 | "Loeffler", 943 | elex_id="test_loeffler", 944 | votecount=789, 945 | votepct=0.4, 946 | ), 947 | res_s( 948 | "GA-S", 949 | "GOP", 950 | "Doug", 951 | "Collins", 952 | elex_id="test_collins", 953 | votecount=456, 954 | votepct=0.3, 955 | ), 956 | res_s( 957 | "GA-S", 958 | "Lib", 959 | "Shane", 960 | "Hazel", 961 | elex_id="test_hazel", 962 | votecount=123, 963 | votepct=0.2, 964 | ), 965 | ], 966 | ), 967 | state_summary( 968 | "GA-S", 969 | structs.SenateSpecialSummary( 970 | S=structs.StateSummaryCongressionalResult( 971 | gop=structs.StateSummaryCandidateNamed( 972 | first_name="Kelly", 973 | last_name="Loeffler", 974 | pop_vote=789, 975 | pop_pct=0.4, 976 | pop_vote_history={"A": 1, "B": 2}, 977 | ), 978 | oth=structs.StateSummaryCandidateUnnamed( 979 | pop_vote=123 + 456, 980 | pop_pct=0.2 + 0.3, 981 | pop_vote_history={"B": 10, "C": 20 + 100, "D": 200}, 982 | ), 983 | multiple_gop=True, 984 | ) 985 | ), 986 | ), 987 | ) 988 | 989 | 990 | # multiple dem + historicals 991 | def test_historical_house_counts(exporter): 992 | mock_historicals["test_pelosi"] = {"A": 1, "B": 2} 993 | mock_historicals["test_buttar"] = {"B": 10, "C": 20} 994 | 995 | assert_result( 996 | exporter.run_export( 997 | [ 998 | res_h( 999 | "CA", 1000 | 12, 1001 | "Dem", 1002 | "Nancy", 1003 | "Pelosi", 1004 | votecount=789, 1005 | votepct=0.4, 1006 | elex_id="test_pelosi", 1007 | ), 1008 | res_h( 1009 | "CA", 1010 | 12, 1011 | "Dem", 1012 | "Shahid", 1013 | "Buttar", 1014 | votecount=456, 1015 | votepct=0.3, 1016 | elex_id="test_buttar", 1017 | ), 1018 | ], 1019 | ), 1020 | state_summary( 1021 | "CA", 1022 | structs.StateSummary( 1023 | H={ 1024 | "12": structs.StateSummaryCongressionalResult( 1025 | dem=structs.StateSummaryCandidateNamed( 1026 | first_name="Nancy", 1027 | last_name="Pelosi", 1028 | pop_vote=789, 1029 | pop_pct=0.4, 1030 | pop_vote_history={"A": 1, "B": 2}, 1031 | ), 1032 | oth=structs.StateSummaryCandidateUnnamed( 1033 | pop_vote=456, 1034 | pop_pct=0.3, 1035 | pop_vote_history={"B": 10, "C": 20}, 1036 | ), 1037 | multiple_dem=True, 1038 | ) 1039 | } 1040 | ), 1041 | ), 1042 | ) 1043 | 1044 | 1045 | # presidential winner 1046 | def test_call_winner_dem(exporter): 1047 | mock_calls["P"]["CA"] = True 1048 | mock_calls["P"]["MA"] = True 1049 | mock_calls["P"]["WA"] = True 1050 | 1051 | assert_result( 1052 | exporter.run_export( 1053 | [ 1054 | res_p_national("Dem", 123, 0.123), 1055 | res_p_national("GOP", 456, 0.456), 1056 | res_p_state("CA", "Dem", 111, 0.111, electtotal=200, winner=True), 1057 | res_p_state("MA", "Dem", 222, 0.222, electtotal=70, winner=True), 1058 | res_p_state("WA", "GOP", 333, 0.333, electtotal=200, winner=True), 1059 | res_p_state("WY", "GOP", 444, 0.444, electtotal=70, winner=True), 1060 | ] 1061 | ), 1062 | national_summary( 1063 | structs.NationalSummary( 1064 | P=structs.NationalSummaryPresident( 1065 | dem=structs.NationalSummaryPresidentCandidateNamed( 1066 | first_name="Joe", 1067 | last_name="Biden", 1068 | pop_vote=123, 1069 | pop_pct=0.123, 1070 | elect_won=270, 1071 | ), 1072 | gop=structs.NationalSummaryPresidentCandidateNamed( 1073 | first_name="Donald", 1074 | last_name="Trump", 1075 | pop_vote=456, 1076 | pop_pct=0.456, 1077 | elect_won=200, 1078 | ), 1079 | winner=structs.Party.DEM, 1080 | ) 1081 | ), 1082 | state_summary( 1083 | "CA", 1084 | structs.StateSummary( 1085 | P=structs.StateSummaryPresident( 1086 | dem=structs.StateSummaryCandidateNamed( 1087 | first_name="Joe", 1088 | last_name="Biden", 1089 | pop_vote=111, 1090 | pop_pct=0.111, 1091 | ), 1092 | winner=structs.Party.DEM, 1093 | ), 1094 | ), 1095 | ), 1096 | state_summary( 1097 | "MA", 1098 | structs.StateSummary( 1099 | P=structs.StateSummaryPresident( 1100 | dem=structs.StateSummaryCandidateNamed( 1101 | first_name="Joe", 1102 | last_name="Biden", 1103 | pop_vote=222, 1104 | pop_pct=0.222, 1105 | ), 1106 | winner=structs.Party.DEM, 1107 | ), 1108 | ), 1109 | ), 1110 | state_summary( 1111 | "WA", 1112 | structs.StateSummary( 1113 | P=structs.StateSummaryPresident( 1114 | gop=structs.StateSummaryCandidateNamed( 1115 | first_name="Donald", 1116 | last_name="Trump", 1117 | pop_vote=333, 1118 | pop_pct=0.333, 1119 | ), 1120 | winner=structs.Party.GOP, 1121 | ), 1122 | ), 1123 | ), 1124 | state_summary( 1125 | "WY", 1126 | structs.StateSummary( 1127 | P=structs.StateSummaryPresident( 1128 | gop=structs.StateSummaryCandidateNamed( 1129 | first_name="Donald", 1130 | last_name="Trump", 1131 | pop_vote=444, 1132 | pop_pct=0.444, 1133 | ), 1134 | ), 1135 | ), 1136 | ), 1137 | ), 1138 | ) 1139 | 1140 | 1141 | def test_call_winner_gop(exporter): 1142 | mock_calls["P"]["CA"] = True 1143 | mock_calls["P"]["MA"] = True 1144 | mock_calls["P"]["WA"] = True 1145 | mock_calls["P"]["WY"] = True 1146 | 1147 | assert_result( 1148 | exporter.run_export( 1149 | [ 1150 | res_p_national("Dem", 123, 0.123), 1151 | res_p_national("GOP", 456, 0.456), 1152 | res_p_state("CA", "Dem", 111, 0.111, electtotal=200, winner=False), 1153 | res_p_state("MA", "Dem", 222, 0.222, electtotal=70, winner=True), 1154 | res_p_state("WA", "GOP", 333, 0.333, electtotal=200, winner=True), 1155 | res_p_state("WY", "GOP", 444, 0.444, electtotal=70, winner=True), 1156 | ] 1157 | ), 1158 | national_summary( 1159 | structs.NationalSummary( 1160 | P=structs.NationalSummaryPresident( 1161 | dem=structs.NationalSummaryPresidentCandidateNamed( 1162 | first_name="Joe", 1163 | last_name="Biden", 1164 | pop_vote=123, 1165 | pop_pct=0.123, 1166 | elect_won=70, 1167 | ), 1168 | gop=structs.NationalSummaryPresidentCandidateNamed( 1169 | first_name="Donald", 1170 | last_name="Trump", 1171 | pop_vote=456, 1172 | pop_pct=0.456, 1173 | elect_won=270, 1174 | ), 1175 | winner=structs.Party.GOP, 1176 | ) 1177 | ), 1178 | state_summary( 1179 | "CA", 1180 | structs.StateSummary( 1181 | P=structs.StateSummaryPresident( 1182 | dem=structs.StateSummaryCandidateNamed( 1183 | first_name="Joe", 1184 | last_name="Biden", 1185 | pop_vote=111, 1186 | pop_pct=0.111, 1187 | ), 1188 | ), 1189 | ), 1190 | ), 1191 | state_summary( 1192 | "MA", 1193 | structs.StateSummary( 1194 | P=structs.StateSummaryPresident( 1195 | dem=structs.StateSummaryCandidateNamed( 1196 | first_name="Joe", 1197 | last_name="Biden", 1198 | pop_vote=222, 1199 | pop_pct=0.222, 1200 | ), 1201 | winner=structs.Party.DEM, 1202 | ), 1203 | ), 1204 | ), 1205 | state_summary( 1206 | "WA", 1207 | structs.StateSummary( 1208 | P=structs.StateSummaryPresident( 1209 | gop=structs.StateSummaryCandidateNamed( 1210 | first_name="Donald", 1211 | last_name="Trump", 1212 | pop_vote=333, 1213 | pop_pct=0.333, 1214 | ), 1215 | winner=structs.Party.GOP, 1216 | ), 1217 | ), 1218 | ), 1219 | state_summary( 1220 | "WY", 1221 | structs.StateSummary( 1222 | P=structs.StateSummaryPresident( 1223 | gop=structs.StateSummaryCandidateNamed( 1224 | first_name="Donald", 1225 | last_name="Trump", 1226 | pop_vote=444, 1227 | pop_pct=0.444, 1228 | ), 1229 | winner=structs.Party.GOP, 1230 | ), 1231 | ), 1232 | ), 1233 | ), 1234 | ) 1235 | 1236 | 1237 | # presidential commentary 1238 | def test_prez_commentary(exporter): 1239 | mock_comments["P"]["CA"] = [ 1240 | structs.Comment( 1241 | timestamp=datetime(2020, 11, 3, 8, 1, 2, tzinfo=timezone.utc), 1242 | author="A", 1243 | title="B", 1244 | body="C", 1245 | ), 1246 | structs.Comment( 1247 | timestamp=datetime(2020, 11, 3, 8, 3, 4, tzinfo=timezone.utc), 1248 | author="D", 1249 | title="E", 1250 | body="F", 1251 | ), 1252 | ] 1253 | 1254 | assert_result( 1255 | exporter.run_export( 1256 | [res_p_state("CA", "Lib", votecount=12345, votepct=0.234)], 1257 | ), 1258 | state_summary( 1259 | "CA", 1260 | structs.StateSummary( 1261 | P=structs.StateSummaryPresident( 1262 | oth=structs.StateSummaryCandidateUnnamed( 1263 | pop_vote=12345, pop_pct=0.234, 1264 | ), 1265 | comments=[ 1266 | structs.Comment( 1267 | timestamp=datetime( 1268 | 2020, 11, 3, 8, 1, 2, tzinfo=timezone.utc 1269 | ), 1270 | author="A", 1271 | title="B", 1272 | body="C", 1273 | ), 1274 | structs.Comment( 1275 | timestamp=datetime( 1276 | 2020, 11, 3, 8, 3, 4, tzinfo=timezone.utc 1277 | ), 1278 | author="D", 1279 | title="E", 1280 | body="F", 1281 | ), 1282 | ], 1283 | ), 1284 | ), 1285 | ), 1286 | ) 1287 | 1288 | 1289 | # senate commentary 1290 | def test_senate_commentary(exporter): 1291 | mock_comments["S"]["GA"] = [ 1292 | structs.Comment( 1293 | timestamp=datetime(2020, 11, 3, 8, 1, 2, tzinfo=timezone.utc), 1294 | author="A", 1295 | title="B", 1296 | body="C", 1297 | ), 1298 | ] 1299 | 1300 | assert_result( 1301 | exporter.run_export( 1302 | [res_s("GA", "GOP", "David", "Perdue", votecount=12345, votepct=0.234,),], 1303 | ), 1304 | state_summary( 1305 | "GA", 1306 | structs.StateSummary( 1307 | S=structs.StateSummaryCongressionalResult( 1308 | gop=structs.StateSummaryCandidateNamed( 1309 | first_name="David", 1310 | last_name="Perdue", 1311 | pop_vote=12345, 1312 | pop_pct=0.234, 1313 | ), 1314 | comments=[ 1315 | structs.Comment( 1316 | timestamp=datetime( 1317 | 2020, 11, 3, 8, 1, 2, tzinfo=timezone.utc 1318 | ), 1319 | author="A", 1320 | title="B", 1321 | body="C", 1322 | ), 1323 | ], 1324 | ) 1325 | ), 1326 | ), 1327 | ) 1328 | 1329 | 1330 | # GA-S commentary 1331 | def test_senate_special_commentary(exporter): 1332 | mock_comments["S"]["GA-S"] = [ 1333 | structs.Comment( 1334 | timestamp=datetime(2020, 11, 3, 8, 1, 2, tzinfo=timezone.utc), 1335 | author="A", 1336 | title="B", 1337 | body="C", 1338 | ), 1339 | ] 1340 | 1341 | assert_result( 1342 | exporter.run_export( 1343 | [ 1344 | res_s( 1345 | "GA-S", "Dem", "Raphael", "Warnock", votecount=12345, votepct=0.234, 1346 | ), 1347 | ], 1348 | ), 1349 | state_summary( 1350 | "GA-S", 1351 | structs.SenateSpecialSummary( 1352 | S=structs.StateSummaryCongressionalResult( 1353 | dem=structs.StateSummaryCandidateNamed( 1354 | first_name="Raphael", 1355 | last_name="Warnock", 1356 | pop_vote=12345, 1357 | pop_pct=0.234, 1358 | ), 1359 | comments=[ 1360 | structs.Comment( 1361 | timestamp=datetime( 1362 | 2020, 11, 3, 8, 1, 2, tzinfo=timezone.utc 1363 | ), 1364 | author="A", 1365 | title="B", 1366 | body="C", 1367 | ), 1368 | ], 1369 | ) 1370 | ), 1371 | ), 1372 | ) 1373 | 1374 | 1375 | # house commentary 1376 | def test_house_commentary(exporter): 1377 | mock_comments["H"]["GA-04"] = [ 1378 | structs.Comment( 1379 | timestamp=datetime(2020, 11, 3, 8, 1, 2, tzinfo=timezone.utc), 1380 | author="A", 1381 | title="B", 1382 | body="C", 1383 | ), 1384 | ] 1385 | 1386 | assert_result( 1387 | exporter.run_export( 1388 | [res_h("GA", 4, "Dem", "Hank", "Johnson", votecount=12345, votepct=0.234)], 1389 | ), 1390 | state_summary( 1391 | "GA", 1392 | structs.StateSummary( 1393 | H={ 1394 | "04": structs.StateSummaryCongressionalResult( 1395 | dem=structs.StateSummaryCandidateNamed( 1396 | first_name="Hank", 1397 | last_name="Johnson", 1398 | pop_vote=12345, 1399 | pop_pct=0.234, 1400 | ), 1401 | comments=[ 1402 | structs.Comment( 1403 | timestamp=datetime( 1404 | 2020, 11, 3, 8, 1, 2, tzinfo=timezone.utc 1405 | ), 1406 | author="A", 1407 | title="B", 1408 | body="C", 1409 | ), 1410 | ], 1411 | ) 1412 | } 1413 | ), 1414 | ), 1415 | ) 1416 | 1417 | 1418 | # house at large commentary 1419 | def test_house_al_commentar(exporter): 1420 | mock_comments["H"]["AK-AL"] = [ 1421 | structs.Comment( 1422 | timestamp=datetime(2020, 11, 3, 8, 1, 2, tzinfo=timezone.utc), 1423 | author="A", 1424 | title="B", 1425 | body="C", 1426 | ), 1427 | ] 1428 | 1429 | assert_result( 1430 | exporter.run_export( 1431 | [res_h("AK", 1, "GOP", "Don", "Young", votecount=12345, votepct=0.234)], 1432 | ), 1433 | state_summary( 1434 | "AK", 1435 | structs.StateSummary( 1436 | H={ 1437 | "AL": structs.StateSummaryCongressionalResult( 1438 | gop=structs.StateSummaryCandidateNamed( 1439 | first_name="Don", 1440 | last_name="Young", 1441 | pop_vote=12345, 1442 | pop_pct=0.234, 1443 | ), 1444 | comments=[ 1445 | structs.Comment( 1446 | timestamp=datetime( 1447 | 2020, 11, 3, 8, 1, 2, tzinfo=timezone.utc 1448 | ), 1449 | author="A", 1450 | title="B", 1451 | body="C", 1452 | ), 1453 | ], 1454 | ) 1455 | } 1456 | ), 1457 | ), 1458 | ) 1459 | 1460 | 1461 | # national commentary 1462 | def test_national_commentary(exporter): 1463 | mock_comments["N"]["N"] = [ 1464 | structs.Comment( 1465 | timestamp=datetime(2020, 11, 3, 8, 1, 2, tzinfo=timezone.utc), 1466 | author="A", 1467 | title="B", 1468 | body="C", 1469 | ), 1470 | ] 1471 | 1472 | assert_result( 1473 | exporter.run_export( 1474 | [res_p_state("CA", "Lib", votecount=12345, votepct=0.234)], 1475 | ), 1476 | national_summary( 1477 | structs.NationalSummary( 1478 | comments=[ 1479 | structs.Comment( 1480 | timestamp=datetime(2020, 11, 3, 8, 1, 2, tzinfo=timezone.utc), 1481 | author="A", 1482 | title="B", 1483 | body="C", 1484 | ), 1485 | ] 1486 | ), 1487 | state_summary( 1488 | "CA", 1489 | structs.StateSummary( 1490 | P=structs.StateSummaryPresident( 1491 | oth=structs.StateSummaryCandidateUnnamed( 1492 | pop_vote=12345, pop_pct=0.234, 1493 | ), 1494 | ) 1495 | ), 1496 | ), 1497 | ), 1498 | ) 1499 | --------------------------------------------------------------------------------