├── mjcs ├── __init__.py ├── models │ ├── case.py │ ├── scraper.py │ ├── common.py │ └── __init__.py └── session.py ├── db ├── README ├── sql │ └── functions.sql ├── versions │ ├── 5cdea7764e93_add_url_column_to_case_database.py │ ├── cd9aee766fbc_add_fine_to_dsk8_charges.py │ ├── 56753c018bcc_add_errunknown_column_to_queue.py │ ├── cb47a092cecb_rename_charges_table_to_dscr_charges.py │ ├── 007964aa8288_add_sentence_term_to_dsk8_charges.py │ ├── 47dc91b89bfc_adding_ground_rent_to_bail_and_bond.py │ ├── 13f400fd5b59_add_filing_date_original_to_case_table.py │ ├── aa166aba00a5_add_property_value_to_bail_and_bond.py │ ├── 7ecafc40188c_add_property_address_to_bail_and_bond.py │ ├── 1d0f3c93ce1f_rename_court_schedule_table_to_dscr_.py │ ├── 466325744aaa_change_type_of_tracking_number_column_.py │ ├── ac3c66efcfeb_add_loc_and_details_loc_columns_to_.py │ ├── d4d38815e85c_change_related_person_agency_sub_code_.py │ ├── 0b545c28e0a3_add_checked_expunged_charges_to_cases.py │ ├── b0b43970cbca_update_dsk8_table_name_typo_change_.py │ ├── da858bf1a1b3_added_scrape_and_parse_timestamps_to_.py │ ├── 3359002939a6_remove_whitespace_in_odycrim_odytraf_.py │ ├── 61f1f04c4f18_add_judgement_date_to_bail_and_bond.py │ ├── e3ca8834d437_change_data_type_in_dscr.py │ ├── 04256934a246_restore_scrape_exempt.py │ ├── 3df211a638f7_add_retry_failed_column_to_runs_table.py │ ├── 1fc42302345c_add_duration_column_to_dscivil_trial.py │ ├── c6965a9b34fe_change_another_dscr_data_type.py │ ├── c72ff2a44e3d_convert_dsk8_tracking_number_to_string.py │ ├── f252364cce3d_remove_unused_field_in_pg.py │ ├── a68a233d6b44_remove_unused_field_in_cc.py │ ├── cd14501526dc_add_jail_text_to_k_charges.py │ ├── 0327e0a1ca6a_added_jail_cons_conc_to_odycrim_charges.py │ ├── 5a9fc170dd16_add_pg_dockets_docket_text.py │ ├── b1570fa6c264_add_order_to_column_metadata.py │ ├── c376bfc09245_added_allow_unparsed_data_to_cases.py │ ├── 8fada70ffb2b_fix_incorrect_data_type_for_.py │ ├── b8ca82cb953a_remove_column_metadata_width_pixels.py │ ├── 0ecfd4d83f6c_add_jail_suspended_term_column_to_.py │ ├── 20ee054f0e99_added_subsequent_offense_to_dstraf_.py │ ├── f1d79c03a365_add_pg_defendants_party_number.py │ ├── 0d1827f56503_added_servicefee_to_odycivil_judgment_.py │ ├── 1d128fd0312c_remove_unused_url_field_from_cases_table.py │ ├── 987da5298aca_add_address_3_to_k_party_addresses.py │ ├── da56ed9c2ee4_add_pop_award_to_field_to_odycivil_.py │ ├── 6d10dfbbb648_add_release_date_and_release_reason_to_.py │ ├── 950f3ef6e583_typo_in_odycrim_probation_column.py │ ├── 003e6ae1f9b5_add_redacted_bool_to_column_metadata.py │ ├── 98327ab2ad75_add_disposition_merged_text_to_k_charges.py │ ├── b1203592827f_added_trial_judgment_against_plaintiff_.py │ ├── ec14982d61de_add_site_field_to_searches_and_runs.py │ ├── baa609bd1160_add_allowed_values_array_to_column_.py │ ├── 1312fd27db62_create_index_for_cases_checked_expunged_.py │ ├── bb071f4f29f2_add_missing_fields_in_mccr.py │ ├── 633f3939b99d_odycrim_charges_case_number_non_nullable.py │ ├── b1dff9d12aba_add_scrape_exempt_and_parse_exempt_to_.py │ ├── 233cfe62cd05_add_removal_date_to_cc_attorneys.py │ ├── d74fb94c58b2_parser_fixes.py │ ├── aed623153eb1_missing_fields.py │ ├── 26b295254ad0_adding_amount_vehicle_and_tij_vehicle_.py │ ├── 5be140385f46_remove_unused_fields_in_pgv.py │ ├── bea2e05bd9fe_added_dob_to_odycivil_involved_parties.py │ ├── 23a7ddcfc553_add_renewed_field_to_mcci_judgments.py │ ├── 16eaf5eb169c_change_typos_in_defendant_table_and_add_.py │ ├── 95589adca841_add_second_address_field_to_k_party_.py │ ├── 015bf4aefeac_add_possibly_expunged_flag_to_odycrim_.py │ ├── 0baa1282aea4_add_label_to_column_metadata_rename_.py │ ├── e9d408e7b639_remove_unused_fields_in_dscr.py │ ├── 025ecad5c330_add_user_usercdate_fields_to_odycivil_.py │ ├── 1f9270433302_add_forfeit_date_forfeit_extended_date_.py │ ├── ed5fea32af69_add_column_metadata_table.py │ ├── d9fdeb7de827_remove_unused_indices.py │ ├── d2239f7d4a65_split_district_case_numbers_into_.py │ ├── d8738272f350_name_and_party_type_for_odycivil_.py │ ├── 20e6f8ac26d5_misc_field_additions.py │ ├── 30ac70fd7fe7_drop_scrape_exempt_parse_exempt_allow_.py │ ├── 2d0b5c6986f1_add_dscivil_trial_table.py │ ├── 1c01c67345db_more_indexes_for_case_table.py │ ├── d79a86c541b0_add_judge_fields_to_odycvcit_models.py │ ├── eee1ab2826c0_fix_to_cc_judgments.py │ ├── 03d75d982909_.py │ ├── 60fb014e0a0a_added_odycrim_sex_offender_.py │ ├── 443341b4957e_add_party_alias_table_to_cc.py │ ├── 193c09af9eb1_add_mccr_remitters_model.py │ ├── 5f3daa92e736_add_odytraf_services_table.py │ ├── f39753ce0f30_possibly_expunged_expunged.py │ ├── fa053738a6c7_added_filingfee_to_odycivil_judgment_.py │ ├── d66528af89f3_add_bail_bond_information_to_odytraf_.py │ ├── 575e78461360_add_hearings_table_for_dscivil.py │ ├── e81039c4f948_.py │ ├── 7ffd37d6c99d_add_satisfied_amended_renewed_fields_to_.py │ ├── 7dba91c1c6c0_add_indexes_to_scrapes_and_scrape_.py │ ├── 26cf16acf1c0_add_pg_plaintiffs.py │ ├── 4c53c17fe94c_remove_unused_fields_in_odycivil.py │ ├── b8eb352e4f9e_rename_odycvcit_documents.py │ ├── b016102d2c6a_add_error_field_to_scrapes_table_and_.py │ ├── 66e022919043_fix_to_dscp_and_k_models.py │ ├── a6437e172d13_additional_fields_for_odycivil_judgment_.py │ ├── 03d0db20af8d_add_odycivildisposition.py │ ├── b4f7a1413269_remove_unused_fields_in_odycrim.py │ ├── b279918a6f75_adding_cc_judgment_modifications_table.py │ ├── 63f37461c971_split_bondsman_out_of_bail_and_bond.py │ ├── 0be25bbbbd8c_added_some_missing_columns_related_to_.py │ ├── 05688275b38c_field_updates_for_dscivil.py │ ├── 10a283d18aa0_add_more_fields_to_k_charges.py │ ├── df0221cb4562_add_appearance_and_removal_dates_to_.py │ ├── eb90f8b96ee4_add_appearance_and_removal_dates_to_.py │ ├── c64263840760_remove_unused_fields_in_dsk8.py │ ├── 9dbb2931da09_modified_k_charges.py │ ├── db345d84fc41_remove_unused_fields_in_dstraf.py │ ├── 961f754ce080_update_odycosa_and_odycoa_models.py │ ├── bf65425d1227_add_dscr_bail_event_model.py │ ├── f3606af6398f_remove_unused_fields_in_odytraf.py │ ├── 4394875fda45_add_possibly_expunged_flag_to_odytraf_.py │ ├── 4baf21cf0e43_add_active_boolean_to_case_table.py │ ├── 67fc57997beb_remove_unused_fields_in_dscp.py │ ├── 93332e9c34d5_added_landlord_tenants_judgments_to_.py │ ├── c063d4f8fbf2_add_new_judge_related_fields.py │ ├── c3420fc7aeaf_add_mortgage_to_bail_and_bond.py │ ├── bc9d0d8db2e2_split_party_address_table_from_cc_.py │ ├── c2f35a46de03_additional_columns_for_.py │ ├── 7427734c0fc2_remove_spider_related_tables.py │ ├── aaa83181c4a5_added_jail_fields_to_odycvcitcharge.py │ ├── f8b3c9cf0c10_remove_unused_fields_in_odycvcit.py │ └── e86f479e25e4_add_scrapes_and_scrape_versions_tables.py ├── script.py.mako └── env.py ├── lambda ├── notifier │ ├── requirements.txt │ └── notifier_lambda.py └── parser │ ├── requirements.txt │ └── parser_lambda.py ├── lib └── psycopg2 │ ├── _psycopg.cpython-38-x86_64-linux-gnu.so │ ├── compat.py │ ├── errors.py │ └── _ipaddress.py ├── env └── base.env ├── .gitignore ├── Dockerfile ├── requirements.txt ├── resources ├── spider.service └── scraper.service ├── .github └── FUNDING.yml ├── secrets.json.example ├── cloudformation └── stack-docker-repo.yaml ├── alembic.ini ├── go.mod └── README.md /mjcs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /db/README: -------------------------------------------------------------------------------- 1 | Generic single-database configuration. -------------------------------------------------------------------------------- /lambda/notifier/requirements.txt: -------------------------------------------------------------------------------- 1 | requests == 2.28.2 -------------------------------------------------------------------------------- /lambda/parser/requirements.txt: -------------------------------------------------------------------------------- 1 | sqlalchemy == 2.0.6 2 | #psycopg2 >= 2.7.4 3 | beautifulsoup4 >= 4.6.0 4 | lxml >= 4.2.1 5 | -------------------------------------------------------------------------------- /lib/psycopg2/_psycopg.cpython-38-x86_64-linux-gnu.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dismantl/CaseHarvester/HEAD/lib/psycopg2/_psycopg.cpython-38-x86_64-linux-gnu.so -------------------------------------------------------------------------------- /db/sql/functions.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION public.age_days(refdate timestamp without time zone) 2 | RETURNS double precision 3 | LANGUAGE sql 4 | AS $function$ 5 | SELECT EXTRACT(EPOCH FROM age(refdate))/(60*60*24); 6 | $function$ -------------------------------------------------------------------------------- /env/base.env: -------------------------------------------------------------------------------- 1 | CASE_BATCH_SIZE=1000 2 | QUERY_TIMEOUT=135 3 | QUEUE_WAIT=5 4 | 5 | SPIDER_DAYS_PER_QUERY=16 6 | 7 | MAX_SCRAPE_AGE=14 8 | MAX_SCRAPE_AGE_INACTIVE=90 9 | RESCRAPE_COEFFICIENT=0.009582477754962 10 | SCRAPE_QUEUE_THRESHOLD=5000000 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .deploy-* 2 | .create-stack-buckets 3 | .init-* 4 | .package-* 5 | .push-docker-image-* 6 | setup.cfg 7 | docs/ 8 | pkg/ 9 | cloudformation/output 10 | __pycache__ 11 | *.pyc 12 | env/development.env 13 | env/production.env 14 | secrets.json 15 | .vscode 16 | .python-version -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3-buster 2 | 3 | WORKDIR /usr/src/app 4 | 5 | RUN mkdir -p /root/.aws && echo "[default]\nregion=us-east-1" > /root/.aws/config 6 | 7 | COPY requirements.txt . 8 | RUN pip install --no-cache-dir -r requirements.txt 9 | 10 | COPY mjcs ./mjcs 11 | COPY harvester.py . 12 | 13 | CMD ["python", "harvester.py", "--help"] 14 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | sqlalchemy == 2.0.6 2 | psycopg2 >= 2.7.4 3 | boto3 >= 1.9.216 4 | beautifulsoup4 == 4.9.0 5 | lxml >= 4.2.1 6 | alembic >= 0.9.9 7 | python-dotenv >= 0.8.2 8 | awscli >= 1.16.226 9 | multiprocessing_logging >= 0.3.1 10 | # pypasser >= 0.0.5 11 | requests == 2.28.2 12 | pypdf == 3.6.0 13 | watchtower == 3.0.1 14 | ec2_metadata == 2.11.0 15 | -------------------------------------------------------------------------------- /resources/spider.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Case Harvester spider 3 | Wants=network-online.target 4 | After=network-online.target 5 | 6 | [Service] 7 | ExecStart=+/root/CaseHarvester/harvester.py --environment production --cloudwatch caseharvester_spider_worker_prod spider --from-queue --record-metrics --shutdown 8 | 9 | [Install] 10 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /resources/scraper.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Case Harvester scraper 3 | Wants=network-online.target 4 | After=network-online.target 5 | 6 | [Service] 7 | ExecStart=+/root/CaseHarvester/harvester.py --environment production --cloudwatch caseharvester_scraper_worker_prod scraper --from-queue --record-metrics --shutdown 8 | 9 | [Install] 10 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /lib/psycopg2/compat.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | __all__ = ['string_types', 'text_type', 'lru_cache'] 4 | 5 | if sys.version_info[0] == 2: 6 | # Python 2 7 | PY2 = True 8 | PY3 = False 9 | string_types = basestring, 10 | text_type = unicode 11 | from ._lru_cache import lru_cache 12 | 13 | else: 14 | # Python 3 15 | PY2 = False 16 | PY3 = True 17 | string_types = str, 18 | text_type = str 19 | from functools import lru_cache 20 | -------------------------------------------------------------------------------- /db/versions/5cdea7764e93_add_url_column_to_case_database.py: -------------------------------------------------------------------------------- 1 | """add url column to case database 2 | 3 | Revision ID: 5cdea7764e93 4 | Revises: 56753c018bcc 5 | Create Date: 2018-04-08 14:03:03.324969 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '5cdea7764e93' 14 | down_revision = '56753c018bcc' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | def upgrade(): 19 | op.add_column('cases', sa.Column('url', sa.String)) 20 | 21 | def downgrade(): 22 | op.drop_column('cases', 'url') 23 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: dismantl 4 | patreon: danstaples 5 | open_collective: # openjusticebaltimore 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | custom: flattr.com/github/dismantl 13 | -------------------------------------------------------------------------------- /db/versions/cd9aee766fbc_add_fine_to_dsk8_charges.py: -------------------------------------------------------------------------------- 1 | """add fine to dsk8_charges 2 | 3 | Revision ID: cd9aee766fbc 4 | Revises: 1f9270433302 5 | Create Date: 2018-05-04 16:11:45.626539 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'cd9aee766fbc' 14 | down_revision = '1f9270433302' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.add_column('dsk8_charges', sa.Column('fine', sa.Numeric)) 21 | 22 | def downgrade(): 23 | op.drop_column('dsk8_charges', 'fine') 24 | -------------------------------------------------------------------------------- /db/versions/56753c018bcc_add_errunknown_column_to_queue.py: -------------------------------------------------------------------------------- 1 | """add errunknown column to queue 2 | 3 | Revision ID: 56753c018bcc 4 | Revises: ac3c66efcfeb 5 | Create Date: 2018-04-07 16:07:21.229635 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '56753c018bcc' 14 | down_revision = 'ac3c66efcfeb' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.add_column('queue', sa.Column('errunknown', sa.String)) 21 | 22 | def downgrade(): 23 | op.drop_column('queue', 'errunknown') 24 | -------------------------------------------------------------------------------- /db/versions/cb47a092cecb_rename_charges_table_to_dscr_charges.py: -------------------------------------------------------------------------------- 1 | """rename charges table to dscr_charges 2 | 3 | Revision ID: cb47a092cecb 4 | Revises: d4d38815e85c 5 | Create Date: 2018-05-02 13:41:48.112140 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'cb47a092cecb' 14 | down_revision = 'd4d38815e85c' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.rename_table('charges','dscr_charges') 21 | 22 | def downgrade(): 23 | op.rename_table('dscr_charges','charges') 24 | -------------------------------------------------------------------------------- /db/script.py.mako: -------------------------------------------------------------------------------- 1 | """${message} 2 | 3 | Revision ID: ${up_revision} 4 | Revises: ${down_revision | comma,n} 5 | Create Date: ${create_date} 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | ${imports if imports else ""} 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = ${repr(up_revision)} 14 | down_revision = ${repr(down_revision)} 15 | branch_labels = ${repr(branch_labels)} 16 | depends_on = ${repr(depends_on)} 17 | 18 | 19 | def upgrade(): 20 | ${upgrades if upgrades else "pass"} 21 | 22 | 23 | def downgrade(): 24 | ${downgrades if downgrades else "pass"} 25 | -------------------------------------------------------------------------------- /secrets.json.example: -------------------------------------------------------------------------------- 1 | { 2 | "development":{ 3 | "DatabaseMasterUsername":"root", 4 | "DatabaseMasterPassword":"badpassword", 5 | "DatabaseUsername":"db_user", 6 | "DatabasePassword":"badpassword", 7 | "DatabaseReadOnlyUsername":"ro_user", 8 | "DatabaseReadOnlyPassword":"badpassword" 9 | }, 10 | "production":{ 11 | "DatabaseMasterUsername":"root", 12 | "DatabaseMasterPassword":"badpassword", 13 | "DatabaseUsername":"db_user", 14 | "DatabasePassword":"badpassword", 15 | "DatabaseReadOnlyUsername":"ro_user", 16 | "DatabaseReadOnlyPassword":"badpassword" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /db/versions/007964aa8288_add_sentence_term_to_dsk8_charges.py: -------------------------------------------------------------------------------- 1 | """add sentence_term to dsk8_charges 2 | 3 | Revision ID: 007964aa8288 4 | Revises: b0b43970cbca 5 | Create Date: 2018-05-04 15:24:07.251836 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '007964aa8288' 14 | down_revision = 'b0b43970cbca' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | def upgrade(): 19 | op.add_column('dsk8_charges', sa.Column('sentence_term', sa.String)) 20 | 21 | def downgrade(): 22 | op.drop_column('dsk8_charges', 'sentence_term') 23 | -------------------------------------------------------------------------------- /db/versions/47dc91b89bfc_adding_ground_rent_to_bail_and_bond.py: -------------------------------------------------------------------------------- 1 | """adding ground_rent to bail_and_bond 2 | 3 | Revision ID: 47dc91b89bfc 4 | Revises: cd9aee766fbc 5 | Create Date: 2018-05-04 16:31:09.309299 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '47dc91b89bfc' 14 | down_revision = 'cd9aee766fbc' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | def upgrade(): 19 | op.add_column('bail_and_bond', sa.Column('ground_rent', sa.Numeric)) 20 | 21 | def downgrade(): 22 | op.drop_column('bail_and_bond', 'ground_rent') 23 | -------------------------------------------------------------------------------- /db/versions/13f400fd5b59_add_filing_date_original_to_case_table.py: -------------------------------------------------------------------------------- 1 | """add filing_date_original to case table 2 | 3 | Revision ID: 13f400fd5b59 4 | Revises: 5cdea7764e93 5 | Create Date: 2018-04-13 12:15:49.218584 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '13f400fd5b59' 14 | down_revision = '5cdea7764e93' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | def upgrade(): 19 | op.add_column('cases', sa.Column('filing_date_original', sa.String)) 20 | 21 | def downgrade(): 22 | op.drop_column('cases', 'filing_date_original') 23 | -------------------------------------------------------------------------------- /db/versions/aa166aba00a5_add_property_value_to_bail_and_bond.py: -------------------------------------------------------------------------------- 1 | """add property_value to bail_and_bond 2 | 3 | Revision ID: aa166aba00a5 4 | Revises: 7ecafc40188c 5 | Create Date: 2018-05-04 16:50:47.586934 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'aa166aba00a5' 14 | down_revision = '7ecafc40188c' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.add_column('bail_and_bond', sa.Column('property_value', sa.Numeric)) 21 | 22 | def downgrade(): 23 | op.drop_column('bail_and_bond', 'property_value') 24 | -------------------------------------------------------------------------------- /db/versions/7ecafc40188c_add_property_address_to_bail_and_bond.py: -------------------------------------------------------------------------------- 1 | """add property_address to bail_and_bond 2 | 3 | Revision ID: 7ecafc40188c 4 | Revises: 47dc91b89bfc 5 | Create Date: 2018-05-04 16:38:00.799736 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '7ecafc40188c' 14 | down_revision = '47dc91b89bfc' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.add_column('bail_and_bond', sa.Column('property_address', sa.String)) 21 | 22 | def downgrade(): 23 | op.drop_column('bail_and_bond', 'property_address') 24 | -------------------------------------------------------------------------------- /db/versions/1d0f3c93ce1f_rename_court_schedule_table_to_dscr_.py: -------------------------------------------------------------------------------- 1 | """rename court_schedule table to dscr_court_schedule 2 | 3 | Revision ID: 1d0f3c93ce1f 4 | Revises: cb47a092cecb 5 | Create Date: 2018-05-02 14:15:18.957017 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '1d0f3c93ce1f' 14 | down_revision = 'cb47a092cecb' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.rename_table('court_schedule','dscr_court_schedule') 21 | 22 | 23 | def downgrade(): 24 | op.rename_table('dscr_court_schedule','court_schedule') 25 | -------------------------------------------------------------------------------- /db/versions/466325744aaa_change_type_of_tracking_number_column_.py: -------------------------------------------------------------------------------- 1 | """change type of tracking_number column in DSCR 2 | 3 | Revision ID: 466325744aaa 4 | Revises: da858bf1a1b3 5 | Create Date: 2018-04-26 10:20:54.975328 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '466325744aaa' 14 | down_revision = 'da858bf1a1b3' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.alter_column('DSCR', 'tracking_number', type_=sa.BigInteger) 21 | 22 | 23 | def downgrade(): 24 | op.alter_column('DSCR', 'tracking_number', type_=sa.Integer) 25 | -------------------------------------------------------------------------------- /db/versions/ac3c66efcfeb_add_loc_and_details_loc_columns_to_.py: -------------------------------------------------------------------------------- 1 | """add loc and details_loc columns to cases db 2 | 3 | Revision ID: ac3c66efcfeb 4 | Revises: 5 | Create Date: 2018-04-06 18:23:44.687705 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = 'ac3c66efcfeb' 13 | down_revision = None 14 | branch_labels = None 15 | depends_on = None 16 | 17 | def upgrade(): 18 | op.add_column('cases', sa.Column('loc', sa.Integer)) 19 | op.add_column('cases', sa.Column('detail_loc', sa.String)) 20 | 21 | def downgrade(): 22 | op.drop_column('cases', 'loc') 23 | op.drop_column('cases', 'detail_loc') 24 | -------------------------------------------------------------------------------- /db/versions/d4d38815e85c_change_related_person_agency_sub_code_.py: -------------------------------------------------------------------------------- 1 | """change related_person agency_sub_code column to String 2 | 3 | Revision ID: d4d38815e85c 4 | Revises: 16eaf5eb169c 5 | Create Date: 2018-04-26 12:57:20.638288 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'd4d38815e85c' 14 | down_revision = '16eaf5eb169c' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.alter_column('related_persons', 'agency_sub_code', type_=sa.String) 21 | 22 | 23 | def downgrade(): 24 | op.alter_column('related_persons', 'agency_sub_code', type_=sa.Integer) 25 | -------------------------------------------------------------------------------- /db/versions/0b545c28e0a3_add_checked_expunged_charges_to_cases.py: -------------------------------------------------------------------------------- 1 | """Add checked_expunged_charges to cases 2 | 3 | Revision ID: 0b545c28e0a3 4 | Revises: e12e27cc98a6 5 | Create Date: 2021-07-22 11:38:19.928899 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '0b545c28e0a3' 14 | down_revision = 'e12e27cc98a6' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.add_column('cases', sa.Column('checked_expunged_charges', sa.Boolean(), server_default='false', nullable=False)) 21 | 22 | 23 | def downgrade(): 24 | op.drop_column('cases', 'checked_expunged_charges') 25 | -------------------------------------------------------------------------------- /db/versions/b0b43970cbca_update_dsk8_table_name_typo_change_.py: -------------------------------------------------------------------------------- 1 | """update DSK8 table name typo; change complaint_number to string 2 | 3 | Revision ID: b0b43970cbca 4 | Revises: 1d0f3c93ce1f 5 | Create Date: 2018-05-04 15:01:47.654701 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'b0b43970cbca' 14 | down_revision = '1d0f3c93ce1f' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.rename_table('DKS8','DSK8') 21 | op.alter_column('DSK8', 'complaint_number', type_=sa.String) 22 | 23 | def downgrade(): 24 | op.alter_column('DSK8', 'complaint_number', type_=sa.BigInteger) 25 | op.rename_table('DSK8','DKS8') 26 | -------------------------------------------------------------------------------- /db/versions/da858bf1a1b3_added_scrape_and_parse_timestamps_to_.py: -------------------------------------------------------------------------------- 1 | """added scrape and parse timestamps to case table 2 | 3 | Revision ID: da858bf1a1b3 4 | Revises: 13f400fd5b59 5 | Create Date: 2018-04-20 23:36:17.775501 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'da858bf1a1b3' 14 | down_revision = '13f400fd5b59' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.add_column('cases', sa.Column('last_scrape', sa.DateTime)) 21 | op.add_column('cases', sa.Column('last_parse', sa.DateTime)) 22 | 23 | 24 | def downgrade(): 25 | op.drop_column('cases', 'last_scrape') 26 | op.drop_column('cases', 'last_parse') 27 | -------------------------------------------------------------------------------- /db/versions/3359002939a6_remove_whitespace_in_odycrim_odytraf_.py: -------------------------------------------------------------------------------- 1 | """Remove whitespace in odycrim/odytraf.court_system 2 | 3 | Revision ID: 3359002939a6 4 | Revises: df0221cb4562 5 | Create Date: 2021-03-23 18:22:26.521555 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '3359002939a6' 14 | down_revision = 'df0221cb4562' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.execute("UPDATE odycrim SET court_system = trim(regexp_replace(court_system, '[\t\s]+', ' ', 'g'));") 21 | op.execute("UPDATE odytraf SET court_system = trim(regexp_replace(court_system, '[\t\s]+', ' ', 'g'));") 22 | 23 | 24 | def downgrade(): 25 | pass 26 | -------------------------------------------------------------------------------- /db/versions/61f1f04c4f18_add_judgement_date_to_bail_and_bond.py: -------------------------------------------------------------------------------- 1 | """add judgement_date to bail_and_bond 2 | 3 | Revision ID: 61f1f04c4f18 4 | Revises: aa166aba00a5 5 | Create Date: 2018-05-04 19:39:23.548178 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '61f1f04c4f18' 14 | down_revision = 'aa166aba00a5' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.add_column('bail_and_bond', sa.Column('judgment_date', sa.Date)) 21 | op.add_column('bail_and_bond', sa.Column('judgment_date_str', sa.String)) 22 | 23 | 24 | def downgrade(): 25 | op.drop_column('bail_and_bond', 'judgment_date_str') 26 | op.drop_column('bail_and_bond', 'judgment_date') 27 | -------------------------------------------------------------------------------- /db/versions/e3ca8834d437_change_data_type_in_dscr.py: -------------------------------------------------------------------------------- 1 | """change data type in dscr 2 | 3 | Revision ID: e3ca8834d437 4 | Revises: aeac150f0048 5 | Create Date: 2018-05-23 16:39:40.712771 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'e3ca8834d437' 14 | down_revision = 'aeac150f0048' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.alter_column('dscr','tracking_number',type_=sa.String) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.alter_column('dscr','tracking_number',type_=sa.BigInteger) 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/04256934a246_restore_scrape_exempt.py: -------------------------------------------------------------------------------- 1 | """Restore scrape_exempt 2 | 3 | Revision ID: 04256934a246 4 | Revises: d9fdeb7de827 5 | Create Date: 2021-10-10 12:16:07.400019 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '04256934a246' 14 | down_revision = 'd9fdeb7de827' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('cases', sa.Column('scrape_exempt', sa.Boolean(), server_default='false', nullable=False)) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.drop_column('cases', 'scrape_exempt') 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/3df211a638f7_add_retry_failed_column_to_runs_table.py: -------------------------------------------------------------------------------- 1 | """Add retry_failed column to runs table 2 | 3 | Revision ID: 3df211a638f7 4 | Revises: e86f479e25e4 5 | Create Date: 2018-08-20 15:01:43.097066 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '3df211a638f7' 14 | down_revision = 'e86f479e25e4' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('runs', sa.Column('retry_failed', sa.Boolean(), nullable=True)) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.drop_column('runs', 'retry_failed') 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/1fc42302345c_add_duration_column_to_dscivil_trial.py: -------------------------------------------------------------------------------- 1 | """add duration column to dscivil_trial 2 | 3 | Revision ID: 1fc42302345c 4 | Revises: 2d0b5c6986f1 5 | Create Date: 2018-05-08 14:07:38.639083 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '1fc42302345c' 14 | down_revision = '2d0b5c6986f1' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('dscivil_trial', sa.Column('duration', sa.Integer(), nullable=True)) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.drop_column('dscivil_trial', 'duration') 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/c6965a9b34fe_change_another_dscr_data_type.py: -------------------------------------------------------------------------------- 1 | """change another dscr data type 2 | 3 | Revision ID: c6965a9b34fe 4 | Revises: e3ca8834d437 5 | Create Date: 2018-05-23 16:56:39.189270 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'c6965a9b34fe' 14 | down_revision = 'e3ca8834d437' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.alter_column('dscr_charges','credit_time_served',type_=sa.String) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.alter_column('dscr_charges','credit_time_served',type_=sa.Integer) 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/c72ff2a44e3d_convert_dsk8_tracking_number_to_string.py: -------------------------------------------------------------------------------- 1 | """convert DSK8 tracking number to string 2 | 3 | Revision ID: c72ff2a44e3d 4 | Revises: 3df211a638f7 5 | Create Date: 2018-11-26 17:02:49.408284 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'c72ff2a44e3d' 14 | down_revision = '3df211a638f7' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.alter_column('dsk8','tracking_number',type_=sa.String) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.alter_column('dsk8','tracking_number',type_=sa.BigInteger) 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/f252364cce3d_remove_unused_field_in_pg.py: -------------------------------------------------------------------------------- 1 | """Remove unused field in PG 2 | 3 | Revision ID: f252364cce3d 4 | Revises: c64263840760 5 | Create Date: 2021-10-12 09:16:22.085164 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'f252364cce3d' 14 | down_revision = 'c64263840760' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.drop_column('pg_attorneys', 'address_2') 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.add_column('pg_attorneys', sa.Column('address_2', sa.VARCHAR(), autoincrement=False, nullable=True)) 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/a68a233d6b44_remove_unused_field_in_cc.py: -------------------------------------------------------------------------------- 1 | """Remove unused field in CC 2 | 3 | Revision ID: a68a233d6b44 4 | Revises: 0529952f8a2b 5 | Create Date: 2021-10-12 08:48:32.384913 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'a68a233d6b44' 14 | down_revision = '0529952f8a2b' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.drop_column('cc_documents', 'sequence_number') 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.add_column('cc_documents', sa.Column('sequence_number', sa.INTEGER(), autoincrement=False, nullable=True)) 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/cd14501526dc_add_jail_text_to_k_charges.py: -------------------------------------------------------------------------------- 1 | """Add jail_text to k_charges 2 | 3 | Revision ID: cd14501526dc 4 | Revises: 95589adca841 5 | Create Date: 2021-08-23 08:13:09.871069 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'cd14501526dc' 14 | down_revision = '95589adca841' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('k_charges', sa.Column('jail_text', sa.String(), nullable=True)) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.drop_column('k_charges', 'jail_text') 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/0327e0a1ca6a_added_jail_cons_conc_to_odycrim_charges.py: -------------------------------------------------------------------------------- 1 | """Added jail_cons_conc to odycrim_charges 2 | 3 | Revision ID: 0327e0a1ca6a 4 | Revises: d056a0d60a23 5 | Create Date: 2021-07-02 14:48:26.795883 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '0327e0a1ca6a' 14 | down_revision = 'bea2e05bd9fe' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('odycrim_charges', sa.Column('jail_cons_conc', sa.String(), nullable=True)) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.drop_column('odycrim_charges', 'jail_cons_conc') 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/5a9fc170dd16_add_pg_dockets_docket_text.py: -------------------------------------------------------------------------------- 1 | """Add pg_dockets.docket_text 2 | 3 | Revision ID: 5a9fc170dd16 4 | Revises: ff6daa02db69 5 | Create Date: 2021-09-01 19:33:13.314129 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '5a9fc170dd16' 14 | down_revision = 'ff6daa02db69' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('pg_dockets', sa.Column('docket_text', sa.String(), nullable=True)) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.drop_column('pg_dockets', 'docket_text') 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/b1570fa6c264_add_order_to_column_metadata.py: -------------------------------------------------------------------------------- 1 | """Add order to column_metadata 2 | 3 | Revision ID: b1570fa6c264 4 | Revises: 003e6ae1f9b5 5 | Create Date: 2021-08-13 12:34:05.701075 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'b1570fa6c264' 14 | down_revision = '003e6ae1f9b5' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('column_metadata', sa.Column('order', sa.Integer(), nullable=True)) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.drop_column('column_metadata', 'order') 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/c376bfc09245_added_allow_unparsed_data_to_cases.py: -------------------------------------------------------------------------------- 1 | """Added allow_unparsed_data to cases 2 | 3 | Revision ID: c376bfc09245 4 | Revises: 2b1952ac2c6e 5 | Create Date: 2021-07-03 18:36:50.904598 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'c376bfc09245' 14 | down_revision = '60fb014e0a0a' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('cases', sa.Column('allow_unparsed_data', sa.Boolean(), server_default='false', nullable=False)) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.drop_column('cases', 'allow_unparsed_data') 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/8fada70ffb2b_fix_incorrect_data_type_for_.py: -------------------------------------------------------------------------------- 1 | """Fix incorrect data type for totalprincipal 2 | 3 | Revision ID: 8fada70ffb2b 4 | Revises: 0be25bbbbd8c 5 | Create Date: 2021-03-01 17:34:16.997199 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '8fada70ffb2b' 14 | down_revision = '0be25bbbbd8c' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.alter_column('odycivil_judgment_comments', 'totalprincipal', type_=sa.String) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.alter_column('odycivil_judgment_comments', 'totalprincipal', type_=sa.Numeric) 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/b8ca82cb953a_remove_column_metadata_width_pixels.py: -------------------------------------------------------------------------------- 1 | """Remove column_metadata.width_pixels 2 | 3 | Revision ID: b8ca82cb953a 4 | Revises: d00d41f1ae1f 5 | Create Date: 2021-10-13 17:29:47.246297 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'b8ca82cb953a' 14 | down_revision = 'd00d41f1ae1f' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.drop_column('column_metadata', 'width_pixels') 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.add_column('column_metadata', sa.Column('width_pixels', sa.INTEGER(), autoincrement=False, nullable=True)) 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/0ecfd4d83f6c_add_jail_suspended_term_column_to_.py: -------------------------------------------------------------------------------- 1 | """Add jail_suspended_term column to odycrim_charges 2 | 3 | Revision ID: 0ecfd4d83f6c 4 | Revises: 950f3ef6e583 5 | Create Date: 2019-08-27 17:43:21.288099 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '0ecfd4d83f6c' 14 | down_revision = '950f3ef6e583' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('odycrim_charges', sa.Column('jail_suspended_term', sa.String(), nullable=True)) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.drop_column('odycrim_charges', 'jail_suspended_term') 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/20ee054f0e99_added_subsequent_offense_to_dstraf_.py: -------------------------------------------------------------------------------- 1 | """Added subsequent_offense to dstraf_charges 2 | 3 | Revision ID: 20ee054f0e99 4 | Revises: 4394875fda45 5 | Create Date: 2021-07-23 15:25:51.062257 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '20ee054f0e99' 14 | down_revision = '4394875fda45' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('dstraf_dispositions', sa.Column('subsequent_offense', sa.String(), nullable=True)) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.drop_column('dstraf_dispositions', 'subsequent_offense') 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/f1d79c03a365_add_pg_defendants_party_number.py: -------------------------------------------------------------------------------- 1 | """Add pg_defendants.party_number 2 | 3 | Revision ID: f1d79c03a365 4 | Revises: 5a9fc170dd16 5 | Create Date: 2021-09-01 19:53:40.314268 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'f1d79c03a365' 14 | down_revision = '5a9fc170dd16' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('pg_defendants', sa.Column('party_number', sa.Integer(), nullable=True)) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.drop_column('pg_defendants', 'party_number') 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/0d1827f56503_added_servicefee_to_odycivil_judgment_.py: -------------------------------------------------------------------------------- 1 | """Added servicefee to odycivil_judgment_comments 2 | 3 | Revision ID: 0d1827f56503 4 | Revises: 20ee054f0e99 5 | Create Date: 2021-07-23 15:52:58.507218 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '0d1827f56503' 14 | down_revision = '20ee054f0e99' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('odycivil_judgment_comments', sa.Column('servicefee', sa.Numeric(), nullable=True)) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.drop_column('odycivil_judgment_comments', 'servicefee') 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/1d128fd0312c_remove_unused_url_field_from_cases_table.py: -------------------------------------------------------------------------------- 1 | """Remove unused URL field from cases table 2 | 3 | Revision ID: 1d128fd0312c 4 | Revises: 10a283d18aa0 5 | Create Date: 2021-08-30 10:07:47.163043 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '1d128fd0312c' 14 | down_revision = 'b1570fa6c264' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.drop_column('cases', 'url') 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.add_column('cases', sa.Column('url', sa.VARCHAR(), autoincrement=False, nullable=True)) 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/987da5298aca_add_address_3_to_k_party_addresses.py: -------------------------------------------------------------------------------- 1 | """Add address_3 to k_party_addresses 2 | 3 | Revision ID: 987da5298aca 4 | Revises: 10a283d18aa0 5 | Create Date: 2021-09-01 13:56:09.501879 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '987da5298aca' 14 | down_revision = '10a283d18aa0' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('k_party_addresses', sa.Column('address_3', sa.String(), nullable=True)) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.drop_column('k_party_addresses', 'address_3') 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/da56ed9c2ee4_add_pop_award_to_field_to_odycivil_.py: -------------------------------------------------------------------------------- 1 | """Add pop_award_to field to odycivil_judgment_comments 2 | 3 | Revision ID: da56ed9c2ee4 4 | Revises: a6437e172d13 5 | Create Date: 2021-03-02 12:42:36.206857 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'da56ed9c2ee4' 14 | down_revision = 'a6437e172d13' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('odycivil_judgment_comments', sa.Column('pop_award_to', sa.String(), nullable=True)) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.drop_column('odycivil_judgment_comments', 'pop_award_to') 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/6d10dfbbb648_add_release_date_and_release_reason_to_.py: -------------------------------------------------------------------------------- 1 | """add release_date and release_reason to bail_and_bond table 2 | 3 | Revision ID: 6d10dfbbb648 4 | Revises: 007964aa8288 5 | Create Date: 2018-05-04 15:30:29.619879 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '6d10dfbbb648' 14 | down_revision = '007964aa8288' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | def upgrade(): 19 | op.add_column('bail_and_bond', sa.Column('release_date', sa.Date)) 20 | op.add_column('bail_and_bond', sa.Column('release_date_str', sa.String)) 21 | op.add_column('bail_and_bond', sa.Column('release_reason', sa.String)) 22 | 23 | def downgrade(): 24 | op.drop_column('bail_and_bond', 'release_reason') 25 | op.drop_column('bail_and_bond', 'release_date_str') 26 | op.drop_column('bail_and_bond', 'release_date') 27 | -------------------------------------------------------------------------------- /db/versions/950f3ef6e583_typo_in_odycrim_probation_column.py: -------------------------------------------------------------------------------- 1 | """Typo in odycrim_probation column 2 | 3 | Revision ID: 950f3ef6e583 4 | Revises: bf65425d1227 5 | Create Date: 2019-08-27 13:47:48.576321 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '950f3ef6e583' 14 | down_revision = 'bf65425d1227' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.alter_column('odycrim_probation','probation_usupervised',new_column_name='probation_unsupervised') 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.alter_column('odycrim_probation','probation_unsupervised',new_column_name='probation_usupervised') 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/003e6ae1f9b5_add_redacted_bool_to_column_metadata.py: -------------------------------------------------------------------------------- 1 | """Add redacted bool to column_metadata 2 | 3 | Revision ID: 003e6ae1f9b5 4 | Revises: 0baa1282aea4 5 | Create Date: 2021-08-13 10:27:18.957414 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '003e6ae1f9b5' 14 | down_revision = '0baa1282aea4' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('column_metadata', sa.Column('redacted', sa.Boolean(), server_default='false', nullable=False)) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.drop_column('column_metadata', 'redacted') 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/98327ab2ad75_add_disposition_merged_text_to_k_charges.py: -------------------------------------------------------------------------------- 1 | """Add disposition_merged_text to k_charges 2 | 3 | Revision ID: 98327ab2ad75 4 | Revises: cd14501526dc 5 | Create Date: 2021-08-23 08:16:27.022116 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '98327ab2ad75' 14 | down_revision = 'cd14501526dc' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('k_charges', sa.Column('disposition_merged_text', sa.String(), nullable=True)) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.drop_column('k_charges', 'disposition_merged_text') 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/b1203592827f_added_trial_judgment_against_plaintiff_.py: -------------------------------------------------------------------------------- 1 | """Added trial_judgment_against_plaintiff to odycivil_judgments 2 | 3 | Revision ID: b1203592827f 4 | Revises: 0d1827f56503 5 | Create Date: 2021-07-25 09:14:11.535709 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'b1203592827f' 14 | down_revision = '0d1827f56503' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('odycivil_judgments', sa.Column('trial_judgment_against_plaintiff', sa.String(), nullable=True)) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.drop_column('odycivil_judgments', 'trial_judgment_against_plaintiff') 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/ec14982d61de_add_site_field_to_searches_and_runs.py: -------------------------------------------------------------------------------- 1 | """Add site field to searches and runs 2 | 3 | Revision ID: ec14982d61de 4 | Revises: f7b524db073b 5 | Create Date: 2020-03-21 16:15:02.413672 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'ec14982d61de' 14 | down_revision = 'f7b524db073b' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('queue', sa.Column('site', sa.String(), nullable=True)) 22 | op.add_column('runs', sa.Column('site', sa.String(), nullable=True)) 23 | # ### end Alembic commands ### 24 | 25 | 26 | def downgrade(): 27 | # ### commands auto generated by Alembic - please adjust! ### 28 | op.drop_column('runs', 'site') 29 | op.drop_column('queue', 'site') 30 | # ### end Alembic commands ### 31 | -------------------------------------------------------------------------------- /db/versions/baa609bd1160_add_allowed_values_array_to_column_.py: -------------------------------------------------------------------------------- 1 | """Add allowed_values array to column_metadata 2 | 3 | Revision ID: baa609bd1160 4 | Revises: b8eb352e4f9e 5 | Create Date: 2021-08-09 01:37:13.925354 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'baa609bd1160' 14 | down_revision = 'b8eb352e4f9e' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('column_metadata', sa.Column('allowed_values', postgresql.ARRAY(sa.String(), dimensions=1), nullable=True)) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.drop_column('column_metadata', 'allowed_values') 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /mjcs/models/case.py: -------------------------------------------------------------------------------- 1 | from .common import TableBase, MetaColumn as Column 2 | from sqlalchemy import Boolean, Date, Integer, String, DateTime, Index 3 | 4 | class Case(TableBase): 5 | __tablename__ = 'cases' 6 | __table_args__ = ( 7 | Index('ixh_cases_case_number', 'case_number', postgresql_using='hash'), 8 | ) 9 | 10 | case_number = Column(String, primary_key=True) 11 | court = Column(String, enum=True) 12 | query_court = Column(String) 13 | case_type = Column(String) 14 | filing_date = Column(Date, index=True) 15 | filing_date_original = Column(String) 16 | status = Column(String, enum=True) 17 | caption = Column(String) 18 | loc = Column(Integer) 19 | detail_loc = Column(String, enum=True) 20 | last_scrape = Column(DateTime) 21 | last_parse = Column(DateTime) 22 | active = Column(Boolean, nullable=False, server_default='true') 23 | scrape_exempt = Column(Boolean, nullable=False, server_default='false') -------------------------------------------------------------------------------- /db/versions/1312fd27db62_create_index_for_cases_checked_expunged_.py: -------------------------------------------------------------------------------- 1 | """Create index for cases.checked_expunged_charges 2 | 3 | Revision ID: 1312fd27db62 4 | Revises: f39753ce0f30 5 | Create Date: 2021-07-27 10:33:19.319965 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '1312fd27db62' 14 | down_revision = 'f39753ce0f30' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_index('ixh_cases_checked_expunged_charges', 'cases', ['checked_expunged_charges'], unique=False, postgresql_using='hash') 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.drop_index('ixh_cases_checked_expunged_charges', table_name='cases', postgresql_using='hash') 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /db/versions/bb071f4f29f2_add_missing_fields_in_mccr.py: -------------------------------------------------------------------------------- 1 | """Add missing fields in MCCR 2 | 3 | Revision ID: bb071f4f29f2 4 | Revises: 5be140385f46 5 | Create Date: 2021-10-12 10:05:20.555959 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'bb071f4f29f2' 14 | down_revision = '5be140385f46' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('mccr_dockets', sa.Column('status', sa.String(), nullable=True)) 22 | op.add_column('mccr_dockets', sa.Column('ruling_judge', sa.String(), nullable=True)) 23 | # ### end Alembic commands ### 24 | 25 | 26 | def downgrade(): 27 | # ### commands auto generated by Alembic - please adjust! ### 28 | op.drop_column('mccr_dockets', 'ruling_judge') 29 | op.drop_column('mccr_dockets', 'status') 30 | # ### end Alembic commands ### 31 | -------------------------------------------------------------------------------- /db/versions/633f3939b99d_odycrim_charges_case_number_non_nullable.py: -------------------------------------------------------------------------------- 1 | """odycrim_charges.case_number non-nullable 2 | 3 | Revision ID: 633f3939b99d 4 | Revises: c376bfc09245 5 | Create Date: 2021-07-06 12:22:03.692192 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '633f3939b99d' 14 | down_revision = 'c376bfc09245' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.alter_column('odycrim_charges', 'case_number', 22 | existing_type=sa.VARCHAR(), 23 | nullable=False) 24 | # ### end Alembic commands ### 25 | 26 | 27 | def downgrade(): 28 | # ### commands auto generated by Alembic - please adjust! ### 29 | op.alter_column('odycrim_charges', 'case_number', 30 | existing_type=sa.VARCHAR(), 31 | nullable=True) 32 | # ### end Alembic commands ### 33 | -------------------------------------------------------------------------------- /db/versions/b1dff9d12aba_add_scrape_exempt_and_parse_exempt_to_.py: -------------------------------------------------------------------------------- 1 | """add scrape_exempt and parse_exempt to cases table 2 | 3 | Revision ID: b1dff9d12aba 4 | Revises: eee1ab2826c0 5 | Create Date: 2018-05-21 15:26:29.131748 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'b1dff9d12aba' 14 | down_revision = 'eee1ab2826c0' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('cases', sa.Column('parse_exempt', sa.Boolean(), nullable=True)) 22 | op.add_column('cases', sa.Column('scrape_exempt', sa.Boolean(), nullable=True)) 23 | # ### end Alembic commands ### 24 | 25 | 26 | def downgrade(): 27 | # ### commands auto generated by Alembic - please adjust! ### 28 | op.drop_column('cases', 'scrape_exempt') 29 | op.drop_column('cases', 'parse_exempt') 30 | # ### end Alembic commands ### 31 | -------------------------------------------------------------------------------- /db/versions/233cfe62cd05_add_removal_date_to_cc_attorneys.py: -------------------------------------------------------------------------------- 1 | """add removal_date to cc_attorneys 2 | 3 | Revision ID: 233cfe62cd05 4 | Revises: 443341b4957e 5 | Create Date: 2018-05-20 22:05:56.886050 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '233cfe62cd05' 14 | down_revision = '443341b4957e' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('cc_attorneys', sa.Column('removal_date', sa.Date(), nullable=True)) 22 | op.add_column('cc_attorneys', sa.Column('removal_date_str', sa.String(), nullable=True)) 23 | # ### end Alembic commands ### 24 | 25 | 26 | def downgrade(): 27 | # ### commands auto generated by Alembic - please adjust! ### 28 | op.drop_column('cc_attorneys', 'removal_date_str') 29 | op.drop_column('cc_attorneys', 'removal_date') 30 | # ### end Alembic commands ### 31 | -------------------------------------------------------------------------------- /db/versions/d74fb94c58b2_parser_fixes.py: -------------------------------------------------------------------------------- 1 | """ODYCRIM and ODYTRAF fixes 2 | 3 | Revision ID: d74fb94c58b2 4 | Revises: 0ab508b55023 5 | Create Date: 2020-05-14 11:13:06.085567 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'd74fb94c58b2' 14 | down_revision = '7dba91c1c6c0' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('odytraf_charges', sa.Column('jail_suspended_term', sa.String(), nullable=True)) 22 | op.add_column('odycrim_restitutions', sa.Column('other_cost_amount', sa.Numeric(), nullable=True)) 23 | # ### end Alembic commands ### 24 | 25 | 26 | def downgrade(): 27 | # ### commands auto generated by Alembic - please adjust! ### 28 | op.drop_column('odycrim_restitutions', 'other_cost_amount') 29 | op.drop_column('odytraf_charges', 'jail_suspended_term') 30 | # ### end Alembic commands ### 31 | -------------------------------------------------------------------------------- /db/versions/aed623153eb1_missing_fields.py: -------------------------------------------------------------------------------- 1 | """Missing fields 2 | 3 | Revision ID: aed623153eb1 4 | Revises: aaa83181c4a5 5 | Create Date: 2021-07-14 17:16:42.906620 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'aed623153eb1' 14 | down_revision = 'aaa83181c4a5' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('odycivil_judgment_comments', sa.Column('val_of_prop_to', sa.String(), nullable=True)) 22 | op.add_column('odycivil_judgments', sa.Column('interest_rate_details', sa.String(), nullable=True)) 23 | # ### end Alembic commands ### 24 | 25 | 26 | def downgrade(): 27 | # ### commands auto generated by Alembic - please adjust! ### 28 | op.drop_column('odycivil_judgments', 'interest_rate_details') 29 | op.drop_column('odycivil_judgment_comments', 'val_of_prop_to') 30 | # ### end Alembic commands ### 31 | -------------------------------------------------------------------------------- /db/versions/26b295254ad0_adding_amount_vehicle_and_tij_vehicle_.py: -------------------------------------------------------------------------------- 1 | """adding amount_vehicle and tij_vehicle to cc_judgments 2 | 3 | Revision ID: 26b295254ad0 4 | Revises: d2239f7d4a65 5 | Create Date: 2018-05-21 10:28:36.068341 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '26b295254ad0' 14 | down_revision = 'd2239f7d4a65' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('cc_judgments', sa.Column('amount_vehicle', sa.Boolean(), nullable=True)) 22 | op.add_column('cc_judgments', sa.Column('tij_vehicle', sa.Boolean(), nullable=True)) 23 | # ### end Alembic commands ### 24 | 25 | 26 | def downgrade(): 27 | # ### commands auto generated by Alembic - please adjust! ### 28 | op.drop_column('cc_judgments', 'tij_vehicle') 29 | op.drop_column('cc_judgments', 'amount_vehicle') 30 | # ### end Alembic commands ### 31 | -------------------------------------------------------------------------------- /db/versions/5be140385f46_remove_unused_fields_in_pgv.py: -------------------------------------------------------------------------------- 1 | """Remove unused fields in PGV 2 | 3 | Revision ID: 5be140385f46 4 | Revises: f252364cce3d 5 | Create Date: 2021-10-12 09:38:05.786001 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '5be140385f46' 14 | down_revision = 'f252364cce3d' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.drop_column('pgv_defendants', 'address_2') 22 | op.drop_column('pgv_plaintiffs', 'address_2') 23 | # ### end Alembic commands ### 24 | 25 | 26 | def downgrade(): 27 | # ### commands auto generated by Alembic - please adjust! ### 28 | op.add_column('pgv_plaintiffs', sa.Column('address_2', sa.VARCHAR(), autoincrement=False, nullable=True)) 29 | op.add_column('pgv_defendants', sa.Column('address_2', sa.VARCHAR(), autoincrement=False, nullable=True)) 30 | # ### end Alembic commands ### 31 | -------------------------------------------------------------------------------- /db/versions/bea2e05bd9fe_added_dob_to_odycivil_involved_parties.py: -------------------------------------------------------------------------------- 1 | """Added DOB to odycivil_involved_parties 2 | 3 | Revision ID: bea2e05bd9fe 4 | Revises: f5a2f305394e 5 | Create Date: 2021-07-02 14:34:49.531911 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'bea2e05bd9fe' 14 | down_revision = 'fa053738a6c7' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('odycivil_involved_parties', sa.Column('DOB', sa.Date(), nullable=True)) 22 | op.add_column('odycivil_involved_parties', sa.Column('DOB_str', sa.String(), nullable=True)) 23 | # ### end Alembic commands ### 24 | 25 | 26 | def downgrade(): 27 | # ### commands auto generated by Alembic - please adjust! ### 28 | op.drop_column('odycivil_involved_parties', 'DOB_str') 29 | op.drop_column('odycivil_involved_parties', 'DOB') 30 | # ### end Alembic commands ### 31 | -------------------------------------------------------------------------------- /db/versions/23a7ddcfc553_add_renewed_field_to_mcci_judgments.py: -------------------------------------------------------------------------------- 1 | """Add renewed field to mcci_judgments 2 | 3 | Revision ID: 23a7ddcfc553 4 | Revises: 62f52367e30c 5 | Create Date: 2021-09-14 10:18:02.302169 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '23a7ddcfc553' 14 | down_revision = '62f52367e30c' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('mcci_judgments', sa.Column('renewed', sa.Date(), nullable=True)) 22 | op.add_column('mcci_judgments', sa.Column('renewed_str', sa.String(), nullable=True)) 23 | # ### end Alembic commands ### 24 | 25 | 26 | def downgrade(): 27 | # ### commands auto generated by Alembic - please adjust! ### 28 | op.drop_column('mcci_judgments', 'renewed_str') 29 | op.drop_column('mcci_judgments', 'renewed') 30 | # ### end Alembic commands ### 31 | -------------------------------------------------------------------------------- /lambda/notifier/notifier_lambda.py: -------------------------------------------------------------------------------- 1 | import os 2 | import requests 3 | 4 | orch_ip_addr = os.getenv('ORCHESTRATOR_IP_ADDRESS') 5 | 6 | def state_change_notifier(event, context): 7 | print(event) 8 | instance_id = event['detail']['instance-id'] 9 | state = event['detail']['state'] 10 | response = requests.get(f'http://{orch_ip_addr}/{state}/{instance_id}') 11 | 12 | def queue_not_empty_notifier(event, context): 13 | print(event) 14 | alarm_name = event['detail']['alarmName'] 15 | state = event['detail']['state']['value'] 16 | if state == 'ALARM': 17 | if 'spider' in alarm_name: 18 | response = requests.get(f'http://{orch_ip_addr}/spider/available') 19 | elif 'scraper' in alarm_name: 20 | response = requests.get(f'http://{orch_ip_addr}/scraper/available') 21 | elif state == 'OK': 22 | if 'spider' in alarm_name: 23 | response = requests.get(f'http://{orch_ip_addr}/spider/empty') 24 | elif 'scraper' in alarm_name: 25 | response = requests.get(f'http://{orch_ip_addr}/scraper/empty') -------------------------------------------------------------------------------- /db/versions/16eaf5eb169c_change_typos_in_defendant_table_and_add_.py: -------------------------------------------------------------------------------- 1 | """change typos in defendant table and add address_2 column' 2 | 3 | Revision ID: 16eaf5eb169c 4 | Revises: 466325744aaa 5 | Create Date: 2018-04-26 12:27:55.976026 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '16eaf5eb169c' 14 | down_revision = '466325744aaa' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.rename_table('defendents','defendants') 21 | op.alter_column('defendants','defendent_name',new_column_name='defendant_name') 22 | op.alter_column('defendants','street_address',new_column_name='address_1') 23 | op.add_column('defendants', sa.Column('address_2', sa.String)) 24 | 25 | def downgrade(): 26 | op.drop_column('defendants', 'address_2') 27 | op.alter_column('defendants','address_1',new_column_name='street_address') 28 | op.alter_column('defendants','defendant_name',new_column_name='defendent_name') 29 | op.rename_table('defendants','defendents') 30 | -------------------------------------------------------------------------------- /db/versions/95589adca841_add_second_address_field_to_k_party_.py: -------------------------------------------------------------------------------- 1 | """Add second address field to k_party_addresses 2 | 3 | Revision ID: 95589adca841 4 | Revises: 9dbb2931da09 5 | Create Date: 2021-08-23 07:05:28.272833 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '95589adca841' 14 | down_revision = '9dbb2931da09' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.alter_column('k_party_addresses', 'address', new_column_name='address_1') 22 | op.add_column('k_party_addresses', sa.Column('address_2', sa.String(), nullable=True)) 23 | # ### end Alembic commands ### 24 | 25 | 26 | def downgrade(): 27 | # ### commands auto generated by Alembic - please adjust! ### 28 | op.drop_column('k_party_addresses', 'address_2') 29 | op.alter_column('k_party_addresses', 'address_1', new_column_name='address') 30 | # ### end Alembic commands ### 31 | -------------------------------------------------------------------------------- /db/versions/015bf4aefeac_add_possibly_expunged_flag_to_odycrim_.py: -------------------------------------------------------------------------------- 1 | """Add possibly_expunged flag to ODYCRIM charges due to HB 1336 2 | 3 | Revision ID: 015bf4aefeac 4 | Revises: 3359002939a6 5 | Create Date: 2021-04-29 17:40:18.045682 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '015bf4aefeac' 14 | down_revision = '3359002939a6' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('odycrim_charges', sa.Column('possibly_expunged', sa.Boolean(), server_default='false', nullable=False)) 22 | op.drop_constraint('odycrim_charges_case_number_fkey', 'odycrim_charges') 23 | # ### end Alembic commands ### 24 | 25 | 26 | def downgrade(): 27 | # ### commands auto generated by Alembic - please adjust! ### 28 | op.create_foreign_key('odycrim_charges_case_number_fkey', 'odycrim_charges', 'odycrim', ['case_number'], ['case_number'], ondelete='CASCADE') 29 | op.drop_column('odycrim_charges', 'possibly_expunged') 30 | # ### end Alembic commands ### 31 | -------------------------------------------------------------------------------- /mjcs/models/scraper.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, DateTime, Integer, Numeric, String, ForeignKey, Index 2 | from .common import TableBase 3 | 4 | class ScrapeVersion(TableBase): 5 | __tablename__ = 'scrape_versions' 6 | 7 | s3_version_id = Column(String, primary_key=True) 8 | case_number = Column(String, ForeignKey('cases.case_number', ondelete='CASCADE')) 9 | length = Column(Integer) 10 | sha256 = Column(String) 11 | 12 | Index('ix_scrapes_versions_case_number_s3_version_id', ScrapeVersion.case_number, ScrapeVersion.s3_version_id, unique=True) 13 | 14 | class Scrape(TableBase): 15 | __tablename__ = 'scrapes' 16 | 17 | id = Column(Integer, primary_key=True) 18 | case_number = Column(String, ForeignKey('cases.case_number', ondelete='CASCADE')) 19 | s3_version_id = Column(String, ForeignKey('scrape_versions.s3_version_id', ondelete='CASCADE')) 20 | timestamp = Column(DateTime) 21 | duration = Column(Numeric) # seconds 22 | error = Column(String) 23 | 24 | Index('ix_scrapes_case_number_timestamp', Scrape.case_number, Scrape.timestamp.desc(), unique=True) 25 | Index('ix_scrapes_case_number_s3_version_id', Scrape.case_number, Scrape.s3_version_id, unique=True) -------------------------------------------------------------------------------- /db/versions/0baa1282aea4_add_label_to_column_metadata_rename_.py: -------------------------------------------------------------------------------- 1 | """Add label to column_metadata, rename constraint 2 | 3 | Revision ID: 0baa1282aea4 4 | Revises: baa609bd1160 5 | Create Date: 2021-08-12 08:37:55.336601 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '0baa1282aea4' 14 | down_revision = 'baa609bd1160' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('column_metadata', sa.Column('label', sa.String(), nullable=True)) 22 | op.execute('ALTER TABLE column_metadata RENAME CONSTRAINT column_metadata_column_name_width_pixels_key TO column_metadata_table_column_name_key') 23 | # ### end Alembic commands ### 24 | 25 | 26 | def downgrade(): 27 | # ### commands auto generated by Alembic - please adjust! ### 28 | op.execute('ALTER TABLE column_metadata RENAME CONSTRAINT column_metadata_table_column_name_key TO column_metadata_column_name_width_pixels_key') 29 | op.drop_column('column_metadata', 'label') 30 | # ### end Alembic commands ### 31 | -------------------------------------------------------------------------------- /db/versions/e9d408e7b639_remove_unused_fields_in_dscr.py: -------------------------------------------------------------------------------- 1 | """Remove unused fields in DSCR 2 | 3 | Revision ID: e9d408e7b639 4 | Revises: f8b3c9cf0c10 5 | Create Date: 2021-10-11 16:06:20.777095 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'e9d408e7b639' 14 | down_revision = 'f8b3c9cf0c10' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.drop_column('dscr_related_persons', 'attorney_code') 22 | op.drop_column('dscr_related_persons', 'attorney_firm') 23 | op.drop_column('dscr_trials', 'reason') 24 | # ### end Alembic commands ### 25 | 26 | 27 | def downgrade(): 28 | # ### commands auto generated by Alembic - please adjust! ### 29 | op.add_column('dscr_trials', sa.Column('reason', sa.VARCHAR(), autoincrement=False, nullable=True)) 30 | op.add_column('dscr_related_persons', sa.Column('attorney_firm', sa.VARCHAR(), autoincrement=False, nullable=True)) 31 | op.add_column('dscr_related_persons', sa.Column('attorney_code', sa.INTEGER(), autoincrement=False, nullable=True)) 32 | # ### end Alembic commands ### 33 | -------------------------------------------------------------------------------- /db/versions/025ecad5c330_add_user_usercdate_fields_to_odycivil_.py: -------------------------------------------------------------------------------- 1 | """Add user, usercdate fields to odycivil_judgment_comments 2 | 3 | Revision ID: 025ecad5c330 4 | Revises: da56ed9c2ee4 5 | Create Date: 2021-03-02 16:39:58.072336 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '025ecad5c330' 14 | down_revision = 'da56ed9c2ee4' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('odycivil_judgment_comments', sa.Column('user', sa.String(), nullable=True)) 22 | op.add_column('odycivil_judgment_comments', sa.Column('usercdate', sa.Date(), nullable=True)) 23 | op.add_column('odycivil_judgment_comments', sa.Column('usercdate_str', sa.String(), nullable=True)) 24 | # ### end Alembic commands ### 25 | 26 | 27 | def downgrade(): 28 | # ### commands auto generated by Alembic - please adjust! ### 29 | op.drop_column('odycivil_judgment_comments', 'usercdate_str') 30 | op.drop_column('odycivil_judgment_comments', 'usercdate') 31 | op.drop_column('odycivil_judgment_comments', 'user') 32 | # ### end Alembic commands ### 33 | -------------------------------------------------------------------------------- /db/versions/1f9270433302_add_forfeit_date_forfeit_extended_date_.py: -------------------------------------------------------------------------------- 1 | """add forfeit_date, forfeit_extended_date, and days_extended to bail_and_bond table 2 | 3 | Revision ID: 1f9270433302 4 | Revises: 6d10dfbbb648 5 | Create Date: 2018-05-04 15:47:20.718907 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '1f9270433302' 14 | down_revision = '6d10dfbbb648' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | def upgrade(): 19 | op.add_column('bail_and_bond', sa.Column('forfeit_date', sa.Date)) 20 | op.add_column('bail_and_bond', sa.Column('forfeit_date_str', sa.String)) 21 | op.add_column('bail_and_bond', sa.Column('forfeit_extended_date', sa.Date)) 22 | op.add_column('bail_and_bond', sa.Column('forfeit_extended_date_str', sa.String)) 23 | op.add_column('bail_and_bond', sa.Column('days_extended', sa.Integer)) 24 | 25 | def downgrade(): 26 | op.drop_column('bail_and_bond', 'days_extended') 27 | op.drop_column('bail_and_bond', 'forfeit_extended_date_str') 28 | op.drop_column('bail_and_bond', 'forfeit_extended_date') 29 | op.drop_column('bail_and_bond', 'forfeit_date_str') 30 | op.drop_column('bail_and_bond', 'forfeit_date') 31 | -------------------------------------------------------------------------------- /db/versions/ed5fea32af69_add_column_metadata_table.py: -------------------------------------------------------------------------------- 1 | """Add column_metadata table 2 | 3 | Revision ID: ed5fea32af69 4 | Revises: cbaf431ce937 5 | Create Date: 2021-08-04 07:57:19.210092 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'ed5fea32af69' 14 | down_revision = 'cbaf431ce937' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('column_metadata', 22 | sa.Column('id', sa.Integer(), nullable=False), 23 | sa.Column('table', sa.String(), nullable=False), 24 | sa.Column('column_name', sa.String(), nullable=False), 25 | sa.Column('description', sa.String(), nullable=True), 26 | sa.Column('width_pixels', sa.Integer(), nullable=True), 27 | sa.PrimaryKeyConstraint('id'), 28 | sa.UniqueConstraint('table', 'column_name', name='column_metadata_column_name_width_pixels_key') 29 | ) 30 | # ### end Alembic commands ### 31 | 32 | 33 | def downgrade(): 34 | # ### commands auto generated by Alembic - please adjust! ### 35 | op.drop_table('column_metadata') 36 | # ### end Alembic commands ### 37 | -------------------------------------------------------------------------------- /db/versions/d9fdeb7de827_remove_unused_indices.py: -------------------------------------------------------------------------------- 1 | """Remove unused indices 2 | 3 | Revision ID: d9fdeb7de827 4 | Revises: 193c09af9eb1 5 | Create Date: 2021-09-30 08:17:44.382303 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'd9fdeb7de827' 14 | down_revision = '193c09af9eb1' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.drop_index('ix_cases_detail_loc', table_name='cases') 22 | op.drop_index('ix_cases_last_parse', table_name='cases') 23 | op.drop_index('ix_cases_last_scrape', table_name='cases') 24 | op.drop_index('ixh_cases_detail_loc', table_name='cases') 25 | # ### end Alembic commands ### 26 | 27 | 28 | def downgrade(): 29 | # ### commands auto generated by Alembic - please adjust! ### 30 | op.create_index('ixh_cases_detail_loc', 'cases', ['detail_loc'], unique=False) 31 | op.create_index('ix_cases_last_scrape', 'cases', ['last_scrape'], unique=False) 32 | op.create_index('ix_cases_last_parse', 'cases', ['last_parse'], unique=False) 33 | op.create_index('ix_cases_detail_loc', 'cases', ['detail_loc'], unique=False) 34 | # ### end Alembic commands ### 35 | -------------------------------------------------------------------------------- /db/versions/d2239f7d4a65_split_district_case_numbers_into_.py: -------------------------------------------------------------------------------- 1 | """split district case numbers into separate table from CC 2 | 3 | Revision ID: d2239f7d4a65 4 | Revises: b279918a6f75 5 | Create Date: 2018-05-21 09:02:25.532060 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'd2239f7d4a65' 14 | down_revision = 'b279918a6f75' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('cc_district_case_numbers', 22 | sa.Column('id', sa.Integer(), nullable=False), 23 | sa.Column('district_case_number', sa.String(), nullable=True), 24 | sa.Column('case_number', sa.String(), nullable=True), 25 | sa.ForeignKeyConstraint(['case_number'], ['cc.case_number'], ondelete='CASCADE'), 26 | sa.PrimaryKeyConstraint('id') 27 | ) 28 | op.drop_column('cc', 'district_case_number') 29 | # ### end Alembic commands ### 30 | 31 | 32 | def downgrade(): 33 | # ### commands auto generated by Alembic - please adjust! ### 34 | op.add_column('cc', sa.Column('district_case_number', sa.VARCHAR(), autoincrement=False, nullable=True)) 35 | op.drop_table('cc_district_case_numbers') 36 | # ### end Alembic commands ### 37 | -------------------------------------------------------------------------------- /db/versions/d8738272f350_name_and_party_type_for_odycivil_.py: -------------------------------------------------------------------------------- 1 | """name and party_type for odycivil_involved_parties can be nullable 2 | 3 | Revision ID: d8738272f350 4 | Revises: 2ca606c7a07c 5 | Create Date: 2021-07-03 17:04:08.503286 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'd8738272f350' 14 | down_revision = '93332e9c34d5' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.alter_column('odycivil_involved_parties', 'name', 22 | existing_type=sa.VARCHAR(), 23 | nullable=True) 24 | op.alter_column('odycivil_involved_parties', 'party_type', 25 | existing_type=sa.VARCHAR(), 26 | nullable=True) 27 | # ### end Alembic commands ### 28 | 29 | 30 | def downgrade(): 31 | # ### commands auto generated by Alembic - please adjust! ### 32 | op.alter_column('odycivil_involved_parties', 'party_type', 33 | existing_type=sa.VARCHAR(), 34 | nullable=False) 35 | op.alter_column('odycivil_involved_parties', 'name', 36 | existing_type=sa.VARCHAR(), 37 | nullable=False) 38 | # ### end Alembic commands ### 39 | -------------------------------------------------------------------------------- /db/versions/20e6f8ac26d5_misc_field_additions.py: -------------------------------------------------------------------------------- 1 | """Misc field additions 2 | 3 | Revision ID: 20e6f8ac26d5 4 | Revises: 763fa67ee202 5 | Create Date: 2023-04-05 12:04:16.399943 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '20e6f8ac26d5' 14 | down_revision = 'b8ca82cb953a' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.add_column('odycrim_charges', sa.Column('notes', sa.String(), nullable=True)) 21 | op.add_column('dstraf_dispositions', sa.Column('notes', sa.String(), nullable=True)) 22 | op.add_column('dscr_charges', sa.Column('notes', sa.String(), nullable=True)) 23 | op.add_column('odytraf_charges', sa.Column('notes', sa.String(), nullable=True)) 24 | op.add_column('odycivil_judgment_statuses', sa.Column('comment', sa.String(), nullable=True)) 25 | op.add_column('odycivil_judgments', sa.Column('judgment_details', sa.String(), nullable=True)) 26 | 27 | 28 | def downgrade(): 29 | op.drop_column('odycivil_judgments', 'judgment_details') 30 | op.drop_column('odycivil_judgment_statuses', 'comment') 31 | op.drop_column('odytraf_charges', 'notes') 32 | op.drop_column('dscr_charges', 'notes') 33 | op.drop_column('dstraf_dispositions', 'notes') 34 | op.drop_column('odycrim_charges', 'notes') 35 | -------------------------------------------------------------------------------- /db/versions/30ac70fd7fe7_drop_scrape_exempt_parse_exempt_allow_.py: -------------------------------------------------------------------------------- 1 | """drop scrape_exempt, parse_exempt, allow_unparsed_data 2 | 3 | Revision ID: 30ac70fd7fe7 4 | Revises: 23a7ddcfc553 5 | Create Date: 2021-09-16 14:21:27.301795 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '30ac70fd7fe7' 14 | down_revision = '23a7ddcfc553' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.drop_column('cases', 'allow_unparsed_data') 22 | op.drop_column('cases', 'scrape_exempt') 23 | op.drop_column('cases', 'parse_exempt') 24 | # ### end Alembic commands ### 25 | 26 | 27 | def downgrade(): 28 | # ### commands auto generated by Alembic - please adjust! ### 29 | op.add_column('cases', sa.Column('parse_exempt', sa.BOOLEAN(), server_default=sa.text('false'), autoincrement=False, nullable=False)) 30 | op.add_column('cases', sa.Column('scrape_exempt', sa.BOOLEAN(), server_default=sa.text('false'), autoincrement=False, nullable=False)) 31 | op.add_column('cases', sa.Column('allow_unparsed_data', sa.BOOLEAN(), server_default=sa.text('false'), autoincrement=False, nullable=False)) 32 | # ### end Alembic commands ### 33 | -------------------------------------------------------------------------------- /db/versions/2d0b5c6986f1_add_dscivil_trial_table.py: -------------------------------------------------------------------------------- 1 | """add dscivil_trial table 2 | 3 | Revision ID: 2d0b5c6986f1 4 | Revises: 575e78461360 5 | Create Date: 2018-05-08 14:02:43.459772 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '2d0b5c6986f1' 14 | down_revision = '575e78461360' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('dscivil_trial', 22 | sa.Column('id', sa.Integer(), nullable=False), 23 | sa.Column('case_number', sa.String(), nullable=True), 24 | sa.Column('date', sa.Date(), nullable=True), 25 | sa.Column('date_str', sa.String(), nullable=True), 26 | sa.Column('time', sa.Time(), nullable=True), 27 | sa.Column('time_str', sa.String(), nullable=True), 28 | sa.Column('room', sa.String(), nullable=True), 29 | sa.Column('location', sa.String(), nullable=True), 30 | sa.ForeignKeyConstraint(['case_number'], ['cases.case_number'], ondelete='CASCADE'), 31 | sa.PrimaryKeyConstraint('id') 32 | ) 33 | # ### end Alembic commands ### 34 | 35 | 36 | def downgrade(): 37 | # ### commands auto generated by Alembic - please adjust! ### 38 | op.drop_table('dscivil_trial') 39 | # ### end Alembic commands ### 40 | -------------------------------------------------------------------------------- /cloudformation/stack-docker-repo.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: '2010-09-09' 2 | Transform: AWS::Serverless-2016-10-31 3 | 4 | Parameters: 5 | EnvironmentType: 6 | Description: The environment type (prod or dev) 7 | Type: String 8 | Default: dev 9 | AllowedValues: 10 | - prod 11 | - dev 12 | ConstraintDescription: must be a prod or dev 13 | DockerRepoName: 14 | Description: Name of docker repository 15 | Type: String 16 | 17 | Resources: 18 | DockerRepo: 19 | Type: AWS::ECR::Repository 20 | Properties: 21 | RepositoryName: !Ref DockerRepoName 22 | RepositoryPolicyText: 23 | Version: "2012-10-17" 24 | Statement: 25 | - Sid: AllowPushPull 26 | Effect: Allow 27 | Principal: 28 | AWS: 29 | - !Sub "arn:aws:iam::${AWS::AccountId}:root" 30 | Action: 31 | - "ecr:GetDownloadUrlForLayer" 32 | - "ecr:BatchGetImage" 33 | - "ecr:BatchCheckLayerAvailability" 34 | - "ecr:PutImage" 35 | - "ecr:InitiateLayerUpload" 36 | - "ecr:UploadLayerPart" 37 | - "ecr:CompleteLayerUpload" 38 | 39 | Outputs: 40 | DockerRepoName: 41 | Description: ECR repository name 42 | Value: !Ref DockerRepo 43 | Export: 44 | Name: !Sub ${AWS::StackName}-DockerRepoName 45 | -------------------------------------------------------------------------------- /db/versions/1c01c67345db_more_indexes_for_case_table.py: -------------------------------------------------------------------------------- 1 | """More indexes for case table 2 | 3 | Revision ID: 1c01c67345db 4 | Revises: 1312fd27db62 5 | Create Date: 2021-07-28 02:50:42.727723 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '1c01c67345db' 14 | down_revision = '1312fd27db62' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_index(op.f('ix_cases_detail_loc'), 'cases', ['detail_loc'], unique=False) 22 | op.create_index(op.f('ix_cases_last_parse'), 'cases', ['last_parse'], unique=False) 23 | op.create_index(op.f('ix_cases_last_scrape'), 'cases', ['last_scrape'], unique=False) 24 | op.create_index('ixh_cases_detail_loc', 'cases', ['detail_loc'], unique=False, postgresql_using='hash') 25 | # ### end Alembic commands ### 26 | 27 | 28 | def downgrade(): 29 | # ### commands auto generated by Alembic - please adjust! ### 30 | op.drop_index('ixh_cases_detail_loc', table_name='cases', postgresql_using='hash') 31 | op.drop_index(op.f('ix_cases_last_scrape'), table_name='cases') 32 | op.drop_index(op.f('ix_cases_last_parse'), table_name='cases') 33 | op.drop_index(op.f('ix_cases_detail_loc'), table_name='cases') 34 | # ### end Alembic commands ### 35 | -------------------------------------------------------------------------------- /db/versions/d79a86c541b0_add_judge_fields_to_odycvcit_models.py: -------------------------------------------------------------------------------- 1 | """Add judge fields to ODYCVCIT models 2 | 3 | Revision ID: d79a86c541b0 4 | Revises: 9d9dc9ba350c 5 | Create Date: 2023-04-05 17:15:50.113602 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'd79a86c541b0' 14 | down_revision = '961f754ce080' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.add_column('odycvcit_bond_settings', sa.Column('judge', sa.String(), nullable=True)) 21 | op.add_column('odycvcit_warrants', sa.Column('judge', sa.String(), nullable=True)) 22 | op.add_column('odycvcit_court_schedule', sa.Column('judge', sa.String(), nullable=True)) 23 | op.add_column('odycvcit_charges', sa.Column('plea_judge', sa.String(), nullable=True)) 24 | op.add_column('odycvcit_charges', sa.Column('disposition_judge', sa.String(), nullable=True)) 25 | op.add_column('odycvcit_charges', sa.Column('sentence_judge', sa.String(), nullable=True)) 26 | 27 | 28 | def downgrade(): 29 | op.drop_column('odycvcit_charges', 'sentence_judge') 30 | op.drop_column('odycvcit_charges', 'disposition_judge') 31 | op.drop_column('odycvcit_charges', 'plea_judge') 32 | op.drop_column('odycvcit_court_schedule', 'judge') 33 | op.drop_column('odycvcit_warrants', 'judge') 34 | op.drop_column('odycvcit_bond_settings', 'judge') 35 | -------------------------------------------------------------------------------- /db/versions/eee1ab2826c0_fix_to_cc_judgments.py: -------------------------------------------------------------------------------- 1 | """fix to cc_judgments 2 | 3 | Revision ID: eee1ab2826c0 4 | Revises: 26b295254ad0 5 | Create Date: 2018-05-21 11:00:12.699712 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'eee1ab2826c0' 14 | down_revision = '26b295254ad0' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('cc_judgment_modifications', sa.Column('amount_other', sa.String(), nullable=True)) 22 | op.add_column('cc_judgments', sa.Column('amount_other', sa.String(), nullable=True)) 23 | op.add_column('cc_judgments', sa.Column('tij_other', sa.String(), nullable=True)) 24 | op.drop_column('cc_judgments', 'tij_vehicle') 25 | op.drop_column('cc_judgments', 'amount_vehicle') 26 | # ### end Alembic commands ### 27 | 28 | 29 | def downgrade(): 30 | # ### commands auto generated by Alembic - please adjust! ### 31 | op.add_column('cc_judgments', sa.Column('amount_vehicle', sa.BOOLEAN(), autoincrement=False, nullable=True)) 32 | op.add_column('cc_judgments', sa.Column('tij_vehicle', sa.BOOLEAN(), autoincrement=False, nullable=True)) 33 | op.drop_column('cc_judgments', 'tij_other') 34 | op.drop_column('cc_judgments', 'amount_other') 35 | op.drop_column('cc_judgment_modifications', 'amount_other') 36 | # ### end Alembic commands ### 37 | -------------------------------------------------------------------------------- /db/versions/03d75d982909_.py: -------------------------------------------------------------------------------- 1 | """bugfix 2 | 3 | Revision ID: 03d75d982909 4 | Revises: aaa83181c4a5 5 | Create Date: 2021-07-14 12:51:56.865399 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '03d75d982909' 14 | down_revision = '633f3939b99d' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.execute(sa.text(""" 21 | UPDATE 22 | cases 23 | SET 24 | last_scrape = NULL 25 | WHERE 26 | case_number IN 27 | ( 28 | SELECT 29 | case_number 30 | FROM 31 | scrapes 32 | JOIN 33 | cases USING (case_number) 34 | WHERE 35 | last_scrape IS NOT NULL 36 | GROUP BY 37 | case_number 38 | HAVING 39 | COUNT(s3_version_id) = 0 40 | ) 41 | """)) 42 | op.execute(sa.text(""" 43 | UPDATE 44 | cases 45 | SET 46 | scrape_exempt = TRUE 47 | WHERE 48 | case_number IN 49 | ( 50 | SELECT 51 | case_number 52 | FROM 53 | scrapes 54 | GROUP BY 55 | case_number 56 | HAVING 57 | COUNT(s3_version_id) = 0 58 | AND COUNT(error) >= 3 59 | ) 60 | """)) 61 | 62 | 63 | def downgrade(): 64 | pass 65 | -------------------------------------------------------------------------------- /db/versions/60fb014e0a0a_added_odycrim_sex_offender_.py: -------------------------------------------------------------------------------- 1 | """Added odycrim_sex_offender_registrations table 2 | 3 | Revision ID: 60fb014e0a0a 4 | Revises: d8738272f350 5 | Create Date: 2021-07-03 17:20:14.005447 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '60fb014e0a0a' 14 | down_revision = 'd8738272f350' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('odycrim_sex_offender_registrations', 22 | sa.Column('id', sa.Integer(), nullable=False), 23 | sa.Column('type', sa.String(), nullable=True), 24 | sa.Column('case_number', sa.String(), nullable=True), 25 | sa.Column('notes', sa.String(), nullable=True), 26 | sa.ForeignKeyConstraint(['case_number'], ['odycrim.case_number'], ondelete='CASCADE'), 27 | sa.PrimaryKeyConstraint('id') 28 | ) 29 | op.create_index('ixh_odycrim_sex_offender_registrations_case_number', 'odycrim_sex_offender_registrations', ['case_number'], unique=False, postgresql_using='hash') 30 | # ### end Alembic commands ### 31 | 32 | 33 | def downgrade(): 34 | # ### commands auto generated by Alembic - please adjust! ### 35 | op.drop_index('ixh_odycrim_sex_offender_registrations_case_number', table_name='odycrim_sex_offender_registrations') 36 | op.drop_table('odycrim_sex_offender_registrations') 37 | # ### end Alembic commands ### 38 | -------------------------------------------------------------------------------- /db/versions/443341b4957e_add_party_alias_table_to_cc.py: -------------------------------------------------------------------------------- 1 | """add party alias table to CC 2 | 3 | Revision ID: 443341b4957e 4 | Revises: bc9d0d8db2e2 5 | Create Date: 2018-05-20 20:45:49.817963 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '443341b4957e' 14 | down_revision = 'bc9d0d8db2e2' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('cc_party_alias', 22 | sa.Column('id', sa.Integer(), nullable=False), 23 | sa.Column('plaintiff_id', sa.Integer(), nullable=True), 24 | sa.Column('defendant_id', sa.Integer(), nullable=True), 25 | sa.Column('related_person_id', sa.Integer(), nullable=True), 26 | sa.Column('name', sa.String(), nullable=True), 27 | sa.Column('case_number', sa.String(), nullable=True), 28 | sa.ForeignKeyConstraint(['case_number'], ['cc.case_number'], ondelete='CASCADE'), 29 | sa.ForeignKeyConstraint(['defendant_id'], ['cc_defendants.id'], ), 30 | sa.ForeignKeyConstraint(['plaintiff_id'], ['cc_plaintiffs.id'], ), 31 | sa.ForeignKeyConstraint(['related_person_id'], ['cc_related_persons.id'], ), 32 | sa.PrimaryKeyConstraint('id') 33 | ) 34 | # ### end Alembic commands ### 35 | 36 | 37 | def downgrade(): 38 | # ### commands auto generated by Alembic - please adjust! ### 39 | op.drop_table('cc_party_alias') 40 | # ### end Alembic commands ### 41 | -------------------------------------------------------------------------------- /lambda/parser/parser_lambda.py: -------------------------------------------------------------------------------- 1 | from mjcs.parser import parse_case 2 | import json 3 | 4 | def lambda_handler(event, context): 5 | for record in event['Records']: 6 | if 's3' in record: 7 | case_number = record['s3']['object']['key'] 8 | try: 9 | parse_case(case_number) 10 | except NotImplementedError: 11 | pass 12 | except Exception: 13 | print(f'Error parsing case {case_number} (https://mdcaseexplorer.com/case/{case_number})') 14 | raise 15 | elif 'Sns' in record: 16 | msg = json.loads(record['Sns']['Message']) 17 | case_number = msg['case_number'] 18 | detail_loc = msg['detail_loc'] 19 | try: 20 | parse_case(case_number, detail_loc) 21 | except NotImplementedError: 22 | pass 23 | except Exception: 24 | print(f'Error parsing case {case_number} (https://mdcaseexplorer.com/case/{case_number})') 25 | raise 26 | elif record.get('eventSource') == 'aws:sqs': 27 | subrecords = json.loads(record['body'])['Records'] 28 | for subrecord in subrecords: 29 | case_number = subrecord['manual']['case_number'] 30 | detail_loc = subrecord['manual']['detail_loc'] 31 | try: 32 | parse_case(case_number, detail_loc) 33 | except: # Ignore all parsing errors so entire batch doesn't fail 34 | pass 35 | -------------------------------------------------------------------------------- /db/versions/193c09af9eb1_add_mccr_remitters_model.py: -------------------------------------------------------------------------------- 1 | """Add mccr_remitters model 2 | 3 | Revision ID: 193c09af9eb1 4 | Revises: 7ffd37d6c99d 5 | Create Date: 2021-09-20 13:01:47.607560 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '193c09af9eb1' 14 | down_revision = '7ffd37d6c99d' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('mccr_bond_remitters', 22 | sa.Column('id', sa.Integer(), nullable=False), 23 | sa.Column('remitter', sa.String(), nullable=True), 24 | sa.Column('case_number', sa.String(), nullable=False), 25 | sa.ForeignKeyConstraint(['case_number'], ['mccr.case_number'], ondelete='CASCADE'), 26 | sa.PrimaryKeyConstraint('id') 27 | ) 28 | op.create_index('ixh_mccr_bond_remitters_case_number', 'mccr_bond_remitters', ['case_number'], unique=False, postgresql_using='hash') 29 | op.drop_column('mccr_bail_bonds', 'remitter') 30 | # ### end Alembic commands ### 31 | 32 | 33 | def downgrade(): 34 | # ### commands auto generated by Alembic - please adjust! ### 35 | op.add_column('mccr_bail_bonds', sa.Column('remitter', sa.VARCHAR(), autoincrement=False, nullable=True)) 36 | op.drop_index('ixh_mccr_bond_remitters_case_number', table_name='mccr_bond_remitters', postgresql_using='hash') 37 | op.drop_table('mccr_bond_remitters') 38 | # ### end Alembic commands ### 39 | -------------------------------------------------------------------------------- /db/versions/5f3daa92e736_add_odytraf_services_table.py: -------------------------------------------------------------------------------- 1 | """Add odytraf_services table 2 | 3 | Revision ID: 5f3daa92e736 4 | Revises: 9d335febaed3 5 | Create Date: 2019-05-16 16:00:51.656823 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '5f3daa92e736' 14 | down_revision = '9d335febaed3' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('odytraf_services', 22 | sa.Column('id', sa.Integer(), nullable=False), 23 | sa.Column('service_type', sa.String(), nullable=False), 24 | sa.Column('requested_by', sa.String(), nullable=True), 25 | sa.Column('issued_date', sa.Date(), nullable=True), 26 | sa.Column('issued_date_str', sa.String(), nullable=True), 27 | sa.Column('service_status', sa.String(), nullable=True), 28 | sa.Column('case_number', sa.String(), nullable=True), 29 | sa.ForeignKeyConstraint(['case_number'], ['odytraf.case_number'], ondelete='CASCADE'), 30 | sa.PrimaryKeyConstraint('id') 31 | ) 32 | op.create_index(op.f('ix_odytraf_services_case_number'), 'odytraf_services', ['case_number'], unique=False) 33 | # ### end Alembic commands ### 34 | 35 | 36 | def downgrade(): 37 | # ### commands auto generated by Alembic - please adjust! ### 38 | op.drop_index(op.f('ix_odytraf_services_case_number'), table_name='odytraf_services') 39 | op.drop_table('odytraf_services') 40 | # ### end Alembic commands ### 41 | -------------------------------------------------------------------------------- /lib/psycopg2/errors.py: -------------------------------------------------------------------------------- 1 | """Error classes for PostgreSQL error codes 2 | """ 3 | 4 | # psycopg/errors.py - SQLSTATE and DB-API exceptions 5 | # 6 | # Copyright (C) 2018-2019 Daniele Varrazzo 7 | # Copyright (C) 2020 The Psycopg Team 8 | # 9 | # psycopg2 is free software: you can redistribute it and/or modify it 10 | # under the terms of the GNU Lesser General Public License as published 11 | # by the Free Software Foundation, either version 3 of the License, or 12 | # (at your option) any later version. 13 | # 14 | # In addition, as a special exception, the copyright holders give 15 | # permission to link this program with the OpenSSL library (or with 16 | # modified versions of OpenSSL that use the same license as OpenSSL), 17 | # and distribute linked combinations including the two. 18 | # 19 | # You must obey the GNU Lesser General Public License in all respects for 20 | # all of the code used other than OpenSSL. 21 | # 22 | # psycopg2 is distributed in the hope that it will be useful, but WITHOUT 23 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 24 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 25 | # License for more details. 26 | 27 | # 28 | # NOTE: the exceptions are injected into this module by the C extention. 29 | # 30 | 31 | 32 | def lookup(code): 33 | """Lookup an error code and return its exception class. 34 | 35 | Raise `!KeyError` if the code is not found. 36 | """ 37 | from psycopg2._psycopg import sqlstate_errors # avoid circular import 38 | return sqlstate_errors[code] 39 | -------------------------------------------------------------------------------- /db/versions/f39753ce0f30_possibly_expunged_expunged.py: -------------------------------------------------------------------------------- 1 | """possibly_expunged -> expunged 2 | 3 | Revision ID: f39753ce0f30 4 | Revises: b1203592827f 5 | Create Date: 2021-07-26 22:12:18.459287 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'f39753ce0f30' 14 | down_revision = 'b1203592827f' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.alter_column('dscr_charges','possibly_expunged',new_column_name='expunged') 22 | op.alter_column('dsk8_charges','possibly_expunged',new_column_name='expunged') 23 | op.alter_column('odycrim_charges','possibly_expunged',new_column_name='expunged') 24 | op.alter_column('odycvcit_charges','possibly_expunged',new_column_name='expunged') 25 | op.alter_column('odytraf_charges','possibly_expunged',new_column_name='expunged') 26 | # ### end Alembic commands ### 27 | 28 | 29 | def downgrade(): 30 | # ### commands auto generated by Alembic - please adjust! ### 31 | op.alter_column('dscr_charges','expunged',new_column_name='possibly_expunged') 32 | op.alter_column('dsk8_charges','expunged',new_column_name='possibly_expunged') 33 | op.alter_column('odycrim_charges','expunged',new_column_name='possibly_expunged') 34 | op.alter_column('odycvcit_charges','expunged',new_column_name='possibly_expunged') 35 | op.alter_column('odytraf_charges','expunged',new_column_name='possibly_expunged') 36 | # ### end Alembic commands ### 37 | -------------------------------------------------------------------------------- /db/versions/fa053738a6c7_added_filingfee_to_odycivil_judgment_.py: -------------------------------------------------------------------------------- 1 | """Added filingfee and totalcosts to odycivil_judgment_comments 2 | 3 | Revision ID: fa053738a6c7 4 | Revises: 015bf4aefeac 5 | Create Date: 2021-07-02 14:18:46.776731 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'fa053738a6c7' 14 | down_revision = '015bf4aefeac' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('odycivil_judgment_comments', sa.Column('filingfee', sa.Numeric(), nullable=True)) 22 | op.add_column('odycivil_judgment_comments', sa.Column('totalcosts', sa.Numeric(), nullable=True)) 23 | op.add_column('odycivil_judgment_comments', sa.Column('postjinterest', sa.Numeric(), nullable=True)) 24 | op.add_column('odycivil_judgment_comments', sa.Column('otherfee', sa.Numeric(), nullable=True)) 25 | op.add_column('odycivil_judgment_comments', sa.Column('judg_ccjs_ind', sa.String(), nullable=True)) 26 | # ### end Alembic commands ### 27 | 28 | 29 | def downgrade(): 30 | # ### commands auto generated by Alembic - please adjust! ### 31 | op.drop_column('odycivil_judgment_comments', 'judg_ccjs_ind') 32 | op.drop_column('odycivil_judgment_comments', 'otherfee') 33 | op.drop_column('odycivil_judgment_comments', 'postjinterest') 34 | op.drop_column('odycivil_judgment_comments', 'totalcosts') 35 | op.drop_column('odycivil_judgment_comments', 'filingfee') 36 | # ### end Alembic commands ### 37 | -------------------------------------------------------------------------------- /db/versions/d66528af89f3_add_bail_bond_information_to_odytraf_.py: -------------------------------------------------------------------------------- 1 | """Add bail bond information to ODYTRAF tables 2 | 3 | Revision ID: d66528af89f3 4 | Revises: 795bdb9f58e3 5 | Create Date: 2019-04-11 15:57:25.917635 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'd66528af89f3' 14 | down_revision = '795bdb9f58e3' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('odytraf_bail_bonds', 22 | sa.Column('id', sa.Integer(), nullable=False), 23 | sa.Column('bond_type', sa.String(), nullable=True), 24 | sa.Column('bond_amount_set', sa.String(), nullable=True), 25 | sa.Column('bond_status_date', sa.Date(), nullable=True), 26 | sa.Column('bond_status_date_str', sa.String(), nullable=True), 27 | sa.Column('bond_status', sa.String(), nullable=True), 28 | sa.Column('case_number', sa.String(), nullable=True), 29 | sa.ForeignKeyConstraint(['case_number'], ['odytraf.case_number'], ondelete='CASCADE'), 30 | sa.PrimaryKeyConstraint('id') 31 | ) 32 | op.create_index(op.f('ix_odytraf_bail_bonds_case_number'), 'odytraf_bail_bonds', ['case_number'], unique=False) 33 | # ### end Alembic commands ### 34 | 35 | 36 | def downgrade(): 37 | # ### commands auto generated by Alembic - please adjust! ### 38 | op.drop_index(op.f('ix_odytraf_bail_bonds_case_number'), table_name='odytraf_bail_bonds') 39 | op.drop_table('odytraf_bail_bonds') 40 | # ### end Alembic commands ### 41 | -------------------------------------------------------------------------------- /db/versions/575e78461360_add_hearings_table_for_dscivil.py: -------------------------------------------------------------------------------- 1 | """add hearings table for DSCIVIL 2 | 3 | Revision ID: 575e78461360 4 | Revises: 284bc3e20590 5 | Create Date: 2018-05-07 21:37:59.021935 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '575e78461360' 14 | down_revision = '284bc3e20590' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('hearings', 22 | sa.Column('id', sa.Integer(), nullable=False), 23 | sa.Column('case_number', sa.String(), nullable=True), 24 | sa.Column('complaint_id', sa.Integer(), nullable=True), 25 | sa.Column('date', sa.Date(), nullable=True), 26 | sa.Column('date_str', sa.String(), nullable=True), 27 | sa.Column('time', sa.Time(), nullable=True), 28 | sa.Column('time_str', sa.String(), nullable=True), 29 | sa.Column('room', sa.String(), nullable=True), 30 | sa.Column('location', sa.String(), nullable=True), 31 | sa.Column('duration', sa.String(), nullable=True), 32 | sa.Column('hearing_type', sa.String(), nullable=True), 33 | sa.ForeignKeyConstraint(['case_number'], ['cases.case_number'], ondelete='CASCADE'), 34 | sa.ForeignKeyConstraint(['complaint_id'], ['complaints.id'], ), 35 | sa.PrimaryKeyConstraint('id') 36 | ) 37 | # ### end Alembic commands ### 38 | 39 | 40 | def downgrade(): 41 | # ### commands auto generated by Alembic - please adjust! ### 42 | op.drop_table('hearings') 43 | # ### end Alembic commands ### 44 | -------------------------------------------------------------------------------- /db/versions/e81039c4f948_.py: -------------------------------------------------------------------------------- 1 | """Restore foreign key relationships bw charges and 2nd tier models 2 | 3 | Revision ID: e81039c4f948 4 | Revises: 1c01c67345db 5 | Create Date: 2021-07-30 17:25:35.370361 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'e81039c4f948' 14 | down_revision = '1c01c67345db' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_foreign_key(None, 'dscr_charges', 'dscr', ['case_number'], ['case_number'], ondelete='CASCADE') 22 | op.create_foreign_key(None, 'dsk8_charges', 'dsk8', ['case_number'], ['case_number'], ondelete='CASCADE') 23 | op.create_foreign_key(None, 'odycrim_charges', 'odycrim', ['case_number'], ['case_number'], ondelete='CASCADE') 24 | op.create_foreign_key(None, 'odycvcit_charges', 'odycvcit', ['case_number'], ['case_number'], ondelete='CASCADE') 25 | op.create_foreign_key(None, 'odytraf_charges', 'odytraf', ['case_number'], ['case_number'], ondelete='CASCADE') 26 | # ### end Alembic commands ### 27 | 28 | 29 | def downgrade(): 30 | # ### commands auto generated by Alembic - please adjust! ### 31 | op.drop_constraint(None, 'odytraf_charges', type_='foreignkey') 32 | op.drop_constraint(None, 'odycvcit_charges', type_='foreignkey') 33 | op.drop_constraint(None, 'odycrim_charges', type_='foreignkey') 34 | op.drop_constraint(None, 'dsk8_charges', type_='foreignkey') 35 | op.drop_constraint(None, 'dscr_charges', type_='foreignkey') 36 | # ### end Alembic commands ### 37 | -------------------------------------------------------------------------------- /db/versions/7ffd37d6c99d_add_satisfied_amended_renewed_fields_to_.py: -------------------------------------------------------------------------------- 1 | """Add satisfied, amended, renewed fields to mccr_judgments 2 | 3 | Revision ID: 7ffd37d6c99d 4 | Revises: 8c8e3bfb2073 5 | Create Date: 2021-09-20 07:45:27.050692 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '7ffd37d6c99d' 14 | down_revision = '8c8e3bfb2073' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('mccr_judgments', sa.Column('satisfied', sa.Date(), nullable=True)) 22 | op.add_column('mccr_judgments', sa.Column('satisfied_str', sa.String(), nullable=True)) 23 | op.add_column('mccr_judgments', sa.Column('amended', sa.Date(), nullable=True)) 24 | op.add_column('mccr_judgments', sa.Column('amended_str', sa.String(), nullable=True)) 25 | op.add_column('mccr_judgments', sa.Column('renewed', sa.Date(), nullable=True)) 26 | op.add_column('mccr_judgments', sa.Column('renewed_str', sa.String(), nullable=True)) 27 | # ### end Alembic commands ### 28 | 29 | 30 | def downgrade(): 31 | # ### commands auto generated by Alembic - please adjust! ### 32 | op.drop_column('mccr_judgments', 'renewed_str') 33 | op.drop_column('mccr_judgments', 'renewed') 34 | op.drop_column('mccr_judgments', 'amended_str') 35 | op.drop_column('mccr_judgments', 'amended') 36 | op.drop_column('mccr_judgments', 'satisfied_str') 37 | op.drop_column('mccr_judgments', 'satisfied') 38 | # ### end Alembic commands ### 39 | -------------------------------------------------------------------------------- /db/versions/7dba91c1c6c0_add_indexes_to_scrapes_and_scrape_.py: -------------------------------------------------------------------------------- 1 | """Add indexes to scrapes and scrape_versions tables 2 | 3 | Revision ID: 7dba91c1c6c0 4 | Revises: 0ab508b55023 5 | Create Date: 2020-05-07 19:01:38.380489 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '7dba91c1c6c0' 14 | down_revision = '0ab508b55023' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.execute('UPDATE scrapes SET duration = (-duration)') 21 | 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.create_index('ix_scrapes_versions_case_number_s3_version_id', 'scrape_versions', ['case_number', 's3_version_id'], unique=True) 24 | op.create_index('ix_scrapes_case_number_s3_version_id', 'scrapes', ['case_number', 's3_version_id'], unique=True) 25 | op.create_index('ix_scrapes_case_number_timestamp', 'scrapes', ['case_number', sa.text('timestamp DESC')], unique=True) 26 | op.drop_index('ix_scrape_timestamp', table_name='scrapes') 27 | # ### end Alembic commands ### 28 | 29 | 30 | def downgrade(): 31 | # ### commands auto generated by Alembic - please adjust! ### 32 | op.create_index('ix_scrape_timestamp', 'scrapes', ['case_number', 'timestamp'], unique=False) 33 | op.drop_index('ix_scrapes_case_number_timestamp', table_name='scrapes') 34 | op.drop_index('ix_scrapes_case_number_s3_version_id', table_name='scrapes') 35 | op.drop_index('ix_scrapes_versions_case_number_s3_version_id', table_name='scrape_versions') 36 | # ### end Alembic commands ### 37 | 38 | op.execute('UPDATE scrapes SET duration = (-duration)') 39 | -------------------------------------------------------------------------------- /db/versions/26cf16acf1c0_add_pg_plaintiffs.py: -------------------------------------------------------------------------------- 1 | """Add pg_plaintiffs 2 | 3 | Revision ID: 26cf16acf1c0 4 | Revises: f1d79c03a365 5 | Create Date: 2021-09-02 02:33:42.222573 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '26cf16acf1c0' 14 | down_revision = 'f1d79c03a365' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('pg_plaintiffs', 22 | sa.Column('id', sa.Integer(), nullable=False), 23 | sa.Column('party_number', sa.Integer(), nullable=True), 24 | sa.Column('name', sa.String(), nullable=True), 25 | sa.Column('address_1', sa.String(), nullable=True), 26 | sa.Column('address_2', sa.String(), nullable=True), 27 | sa.Column('city', sa.String(), nullable=True), 28 | sa.Column('state', sa.String(), nullable=True), 29 | sa.Column('zip_code', sa.String(), nullable=True), 30 | sa.Column('case_number', sa.String(), nullable=False), 31 | sa.ForeignKeyConstraint(['case_number'], ['pg.case_number'], ondelete='CASCADE'), 32 | sa.PrimaryKeyConstraint('id') 33 | ) 34 | op.create_index('ixh_pg_plaintiffs_case_number', 'pg_plaintiffs', ['case_number'], unique=False, postgresql_using='hash') 35 | # ### end Alembic commands ### 36 | 37 | 38 | def downgrade(): 39 | # ### commands auto generated by Alembic - please adjust! ### 40 | op.drop_index('ixh_pg_plaintiffs_case_number', table_name='pg_plaintiffs', postgresql_using='hash') 41 | op.drop_table('pg_plaintiffs') 42 | # ### end Alembic commands ### 43 | -------------------------------------------------------------------------------- /mjcs/models/common.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, Date, Numeric, Integer, String, Boolean, ForeignKey, Time, UniqueConstraint 2 | from sqlalchemy.dialects.postgresql import ARRAY 3 | from sqlalchemy.orm import relationship, declarative_base 4 | from sqlalchemy.ext.hybrid import hybrid_property 5 | from sqlalchemy.ext.declarative import declared_attr 6 | import re 7 | from datetime import datetime 8 | 9 | TableBase = declarative_base() 10 | 11 | def date_from_str(date_str): 12 | if date_str: 13 | try: 14 | return datetime.strptime(date_str,"%m/%d/%Y") 15 | except: 16 | return None 17 | return None 18 | 19 | class MetaColumn(Column): 20 | inherit_cache = True 21 | def __init__(self, *args, **kwargs): 22 | self.enum = kwargs.pop('enum', False) 23 | self.redacted = kwargs.pop('redacted', False) 24 | super(MetaColumn, self).__init__(*args, **kwargs) 25 | Column = MetaColumn 26 | 27 | class ColumnMetadata(TableBase): 28 | __tablename__ = 'column_metadata' 29 | __table_args__ = (UniqueConstraint('table', 'column_name', name='column_metadata_table_column_name_key'),) 30 | 31 | id = Column(Integer, primary_key=True) 32 | table = Column(String, nullable=False) 33 | column_name = Column(String, nullable=False) 34 | label = Column(String) 35 | description = Column(String) 36 | allowed_values = Column(ARRAY(String, dimensions=1)) 37 | redacted = Column(Boolean, nullable=False, server_default='false') 38 | order = Column(Integer) 39 | 40 | class CaseTable: 41 | @declared_attr 42 | def case_number(cls): 43 | return Column(String, ForeignKey('cases.case_number', ondelete='CASCADE'), unique=True) 44 | -------------------------------------------------------------------------------- /db/versions/4c53c17fe94c_remove_unused_fields_in_odycivil.py: -------------------------------------------------------------------------------- 1 | """Remove unused fields in ODYCIVIL 2 | 3 | Revision ID: 4c53c17fe94c 4 | Revises: f3606af6398f 5 | Create Date: 2021-10-11 15:36:35.851693 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '4c53c17fe94c' 14 | down_revision = 'f3606af6398f' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.drop_column('odycivil_defendants', 'race') 22 | op.drop_column('odycivil_defendants', 'height') 23 | op.drop_column('odycivil_defendants', 'sex') 24 | op.drop_column('odycivil_defendants', 'weight') 25 | op.drop_column('odycivil_documents', 'filed_by') 26 | op.drop_column('odycivil_services', 'service_status') 27 | # ### end Alembic commands ### 28 | 29 | 30 | def downgrade(): 31 | # ### commands auto generated by Alembic - please adjust! ### 32 | op.add_column('odycivil_services', sa.Column('service_status', sa.VARCHAR(), autoincrement=False, nullable=True)) 33 | op.add_column('odycivil_documents', sa.Column('filed_by', sa.VARCHAR(), autoincrement=False, nullable=True)) 34 | op.add_column('odycivil_defendants', sa.Column('weight', sa.INTEGER(), autoincrement=False, nullable=True)) 35 | op.add_column('odycivil_defendants', sa.Column('sex', sa.VARCHAR(), autoincrement=False, nullable=True)) 36 | op.add_column('odycivil_defendants', sa.Column('height', sa.INTEGER(), autoincrement=False, nullable=True)) 37 | op.add_column('odycivil_defendants', sa.Column('race', sa.VARCHAR(), autoincrement=False, nullable=True)) 38 | # ### end Alembic commands ### 39 | -------------------------------------------------------------------------------- /db/versions/b8eb352e4f9e_rename_odycvcit_documents.py: -------------------------------------------------------------------------------- 1 | """Rename ODYCVCIT_documents 2 | 3 | Revision ID: b8eb352e4f9e 4 | Revises: ed5fea32af69 5 | Create Date: 2021-08-04 09:42:16.643979 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'b8eb352e4f9e' 14 | down_revision = 'ed5fea32af69' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.rename_table('ODYCVCIT_documents', 'odycvcit_documents') 22 | op.execute('ALTER INDEX "ixh_ODYCVCIT_documents_case_number" RENAME TO ixh_odycvcit_documents_case_number') 23 | op.execute('ALTER INDEX "ODYCVCIT_documents_pkey" RENAME TO odycvcit_documents_pkey') 24 | op.execute('ALTER TABLE odycvcit_documents RENAME CONSTRAINT "ODYCVCIT_documents_case_number_fkey" TO odycvcit_documents_case_number_fkey') 25 | op.execute('ALTER SEQUENCE "ODYCVCIT_documents_id_seq" RENAME TO odycvcit_documents_id_seq') 26 | # ### end Alembic commands ### 27 | 28 | 29 | def downgrade(): 30 | # ### commands auto generated by Alembic - please adjust! ### 31 | op.execute('ALTER SEQUENCE odycvcit_documents_id_seq RENAME TO "ODYCVCIT_documents_id_seq"') 32 | op.execute('ALTER TABLE odycvcit_documents RENAME CONSTRAINT odycvcit_documents_case_number_fkey TO "ODYCVCIT_documents_case_number_fkey"') 33 | op.execute('ALTER INDEX odycvcit_documents_pkey RENAME TO "ODYCVCIT_documents_pkey"') 34 | op.execute('ALTER INDEX ixh_odycvcit_documents_case_number RENAME TO "ixh_ODYCVCIT_documents_case_number"') 35 | op.rename_table('odycvcit_documents', 'ODYCVCIT_documents') 36 | # ### end Alembic commands ### 37 | -------------------------------------------------------------------------------- /db/versions/b016102d2c6a_add_error_field_to_scrapes_table_and_.py: -------------------------------------------------------------------------------- 1 | """Add error field to scrapes table and add server-side defaults for boolean fields 2 | 3 | Revision ID: b016102d2c6a 4 | Revises: 4baf21cf0e43 5 | Create Date: 2020-07-09 14:03:56.174294 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'b016102d2c6a' 14 | down_revision = '4baf21cf0e43' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.alter_column('cases', 'active', 22 | existing_type=sa.BOOLEAN(), 23 | nullable=False, 24 | server_default='true') 25 | op.alter_column('cases', 'parse_exempt', 26 | existing_type=sa.BOOLEAN(), 27 | nullable=False, 28 | server_default='false') 29 | op.alter_column('cases', 'scrape_exempt', 30 | existing_type=sa.BOOLEAN(), 31 | nullable=False, 32 | server_default='false') 33 | op.add_column('scrapes', sa.Column('error', sa.String(), nullable=True)) 34 | # ### end Alembic commands ### 35 | 36 | 37 | def downgrade(): 38 | # ### commands auto generated by Alembic - please adjust! ### 39 | op.drop_column('scrapes', 'error') 40 | op.alter_column('cases', 'scrape_exempt', 41 | existing_type=sa.BOOLEAN(), 42 | nullable=True) 43 | op.alter_column('cases', 'parse_exempt', 44 | existing_type=sa.BOOLEAN(), 45 | nullable=True) 46 | op.alter_column('cases', 'active', 47 | existing_type=sa.BOOLEAN(), 48 | nullable=True) 49 | # ### end Alembic commands ### 50 | -------------------------------------------------------------------------------- /db/versions/66e022919043_fix_to_dscp_and_k_models.py: -------------------------------------------------------------------------------- 1 | """Fix to DSCP and K models 2 | 3 | Revision ID: 66e022919043 4 | Revises: d5cb6fe00424 5 | Create Date: 2021-08-23 05:21:15.530640 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '66e022919043' 14 | down_revision = 'd5cb6fe00424' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('k_defendant_aliases', 22 | sa.Column('id', sa.Integer(), nullable=False), 23 | sa.Column('alias_name', sa.String(), nullable=True), 24 | sa.Column('address_1', sa.String(), nullable=True), 25 | sa.Column('address_2', sa.String(), nullable=True), 26 | sa.Column('city', sa.String(), nullable=True), 27 | sa.Column('state', sa.String(), nullable=True), 28 | sa.Column('zip_code', sa.String(), nullable=True), 29 | sa.Column('case_number', sa.String(), nullable=False), 30 | sa.ForeignKeyConstraint(['case_number'], ['k.case_number'], ondelete='CASCADE'), 31 | sa.PrimaryKeyConstraint('id') 32 | ) 33 | op.create_index('ixh_k_defendant_aliases_case_number', 'k_defendant_aliases', ['case_number'], unique=False, postgresql_using='hash') 34 | op.alter_column('dscp_charges', 'victim_age', type_=sa.String) 35 | # ### end Alembic commands ### 36 | 37 | 38 | def downgrade(): 39 | # ### commands auto generated by Alembic - please adjust! ### 40 | op.alter_column('dscp_charges', 'victim_age', type_=sa.Integer) 41 | op.drop_index('ixh_k_defendant_aliases_case_number', table_name='k_defendant_aliases', postgresql_using='hash') 42 | op.drop_table('k_defendant_aliases') 43 | # ### end Alembic commands ### 44 | -------------------------------------------------------------------------------- /db/versions/a6437e172d13_additional_fields_for_odycivil_judgment_.py: -------------------------------------------------------------------------------- 1 | """Additional fields for odycivil_judgment_comments 2 | 3 | Revision ID: a6437e172d13 4 | Revises: 8fada70ffb2b 5 | Create Date: 2021-03-02 11:35:55.848812 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'a6437e172d13' 14 | down_revision = '8fada70ffb2b' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('odycivil_judgment_comments', sa.Column('renewed_date', sa.Date(), nullable=True)) 22 | op.add_column('odycivil_judgment_comments', sa.Column('renewed_date_end_str', sa.String(), nullable=True)) 23 | op.add_column('odycivil_judgment_comments', sa.Column('renewed_date_ent', sa.Date(), nullable=True)) 24 | op.add_column('odycivil_judgment_comments', sa.Column('renewed_date_str', sa.String(), nullable=True)) 25 | op.add_column('odycivil_judgment_comments', sa.Column('renewed_lien_date', sa.Date(), nullable=True)) 26 | op.add_column('odycivil_judgment_comments', sa.Column('renewed_lien_date_str', sa.String(), nullable=True)) 27 | # ### end Alembic commands ### 28 | 29 | 30 | def downgrade(): 31 | # ### commands auto generated by Alembic - please adjust! ### 32 | op.drop_column('odycivil_judgment_comments', 'renewed_lien_date_str') 33 | op.drop_column('odycivil_judgment_comments', 'renewed_lien_date') 34 | op.drop_column('odycivil_judgment_comments', 'renewed_date_str') 35 | op.drop_column('odycivil_judgment_comments', 'renewed_date_ent') 36 | op.drop_column('odycivil_judgment_comments', 'renewed_date_end_str') 37 | op.drop_column('odycivil_judgment_comments', 'renewed_date') 38 | # ### end Alembic commands ### 39 | -------------------------------------------------------------------------------- /db/versions/03d0db20af8d_add_odycivildisposition.py: -------------------------------------------------------------------------------- 1 | """Add ODYCIVILDisposition 2 | 3 | Revision ID: 03d0db20af8d 4 | Revises: c53bda737bbe 5 | Create Date: 2021-03-01 09:23:41.055781 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '03d0db20af8d' 14 | down_revision = 'c53bda737bbe' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('odycivil_dispositions', 22 | sa.Column('id', sa.Integer(), nullable=False), 23 | sa.Column('disp_date', sa.Date(), nullable=True), 24 | sa.Column('disp_date_str', sa.String(), nullable=True), 25 | sa.Column('disp_code', sa.String(), nullable=True), 26 | sa.Column('disp_description', sa.String(), nullable=True), 27 | sa.Column('disp_stage', sa.String(), nullable=True), 28 | sa.Column('disp_stage_description', sa.String(), nullable=True), 29 | sa.Column('user', sa.String(), nullable=True), 30 | sa.Column('usercdate', sa.Date(), nullable=True), 31 | sa.Column('usercdate_str', sa.String(), nullable=True), 32 | sa.Column('case_number', sa.String(), nullable=True), 33 | sa.ForeignKeyConstraint(['case_number'], ['odycivil.case_number'], ondelete='CASCADE'), 34 | sa.PrimaryKeyConstraint('id') 35 | ) 36 | op.create_index('ixh_odycivil_dispositions_case_number', 'odycivil_dispositions', ['case_number'], unique=False, postgresql_using='hash') 37 | # ### end Alembic commands ### 38 | 39 | 40 | def downgrade(): 41 | # ### commands auto generated by Alembic - please adjust! ### 42 | op.drop_index('ixh_odycivil_dispositions_case_number', table_name='odycivil_dispositions') 43 | op.drop_table('odycivil_dispositions') 44 | # ### end Alembic commands ### 45 | -------------------------------------------------------------------------------- /db/versions/b4f7a1413269_remove_unused_fields_in_odycrim.py: -------------------------------------------------------------------------------- 1 | """Remove unused fields in ODYCRIM 2 | 3 | Revision ID: b4f7a1413269 4 | Revises: 04256934a246 5 | Create Date: 2021-10-11 15:16:05.739117 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'b4f7a1413269' 14 | down_revision = '04256934a246' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.drop_column('odycrim_documents', 'filed_by') 22 | op.drop_column('odycrim_involved_parties', 'removal_date_str') 23 | op.drop_column('odycrim_involved_parties', 'appearance_date') 24 | op.drop_column('odycrim_involved_parties', 'appearance_date_str') 25 | op.drop_column('odycrim_involved_parties', 'removal_date') 26 | op.drop_column('odycrim_services', 'service_status') 27 | # ### end Alembic commands ### 28 | 29 | 30 | def downgrade(): 31 | # ### commands auto generated by Alembic - please adjust! ### 32 | op.add_column('odycrim_services', sa.Column('service_status', sa.VARCHAR(), autoincrement=False, nullable=True)) 33 | op.add_column('odycrim_involved_parties', sa.Column('removal_date', sa.DATE(), autoincrement=False, nullable=True)) 34 | op.add_column('odycrim_involved_parties', sa.Column('appearance_date_str', sa.VARCHAR(), autoincrement=False, nullable=True)) 35 | op.add_column('odycrim_involved_parties', sa.Column('appearance_date', sa.DATE(), autoincrement=False, nullable=True)) 36 | op.add_column('odycrim_involved_parties', sa.Column('removal_date_str', sa.VARCHAR(), autoincrement=False, nullable=True)) 37 | op.add_column('odycrim_documents', sa.Column('filed_by', sa.VARCHAR(), autoincrement=False, nullable=True)) 38 | # ### end Alembic commands ### 39 | -------------------------------------------------------------------------------- /db/versions/b279918a6f75_adding_cc_judgment_modifications_table.py: -------------------------------------------------------------------------------- 1 | """adding cc_judgment_modifications table 2 | 3 | Revision ID: b279918a6f75 4 | Revises: 233cfe62cd05 5 | Create Date: 2018-05-21 06:21:55.476908 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'b279918a6f75' 14 | down_revision = '233cfe62cd05' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('cc_judgment_modifications', 22 | sa.Column('id', sa.Integer(), nullable=False), 23 | sa.Column('judgment_id', sa.Integer(), nullable=True), 24 | sa.Column('judgment_against', sa.String(), nullable=True), 25 | sa.Column('judgment_for', sa.String(), nullable=True), 26 | sa.Column('entered_date', sa.Date(), nullable=True), 27 | sa.Column('entered_date_str', sa.String(), nullable=True), 28 | sa.Column('amount', sa.Numeric(), nullable=True), 29 | sa.Column('status_date', sa.Date(), nullable=True), 30 | sa.Column('status_date_str', sa.String(), nullable=True), 31 | sa.Column('status', sa.String(), nullable=True), 32 | sa.Column('comments', sa.String(), nullable=True), 33 | sa.Column('case_number', sa.String(), nullable=True), 34 | sa.ForeignKeyConstraint(['case_number'], ['cc.case_number'], ondelete='CASCADE'), 35 | sa.ForeignKeyConstraint(['judgment_id'], ['cc_judgments.id'], ), 36 | sa.PrimaryKeyConstraint('id') 37 | ) 38 | op.add_column('cc_judgments', sa.Column('judgment_type', sa.String(), nullable=True)) 39 | # ### end Alembic commands ### 40 | 41 | 42 | def downgrade(): 43 | # ### commands auto generated by Alembic - please adjust! ### 44 | op.drop_column('cc_judgments', 'judgment_type') 45 | op.drop_table('cc_judgment_modifications') 46 | # ### end Alembic commands ### 47 | -------------------------------------------------------------------------------- /db/versions/63f37461c971_split_bondsman_out_of_bail_and_bond.py: -------------------------------------------------------------------------------- 1 | """split bondsman out of bail_and_bond 2 | 3 | Revision ID: 63f37461c971 4 | Revises: 61f1f04c4f18 5 | Create Date: 2018-05-04 22:45:26.859394 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = '63f37461c971' 13 | down_revision = '61f1f04c4f18' 14 | branch_labels = None 15 | depends_on = None 16 | 17 | # from sqlalchemy.orm import sessionmaker 18 | # from mjcs.models import DSK8BailAndBond, DSK8Bondsman 19 | # 20 | # def upgrade(): 21 | # connection = op.get_bind() 22 | # db_factory = sessionmaker(bind = connection) 23 | # db = db_factory() 24 | # try: 25 | # for b in db.query(BailAndBond): 26 | # print('bail_and_bond',b.id) 27 | # if b.bail_bondsman or b.bondsman_address_1 or b.bondsman_city \ 28 | # or b.bondsman_state or b.bondsman_zip_code: 29 | # bondsman = Bondsman(b.case_number, b.id) 30 | # bondsman.name = b.bail_bondsman 31 | # bondsman.address_1 = b.bondsman_address_1 32 | # bondsman.city = b.bondsman_city 33 | # bondsman.state = b.bondsman_state 34 | # bondsman.zip_code = b.bondsman_zip_code 35 | # db.add(bondsman) 36 | # db.commit() 37 | # except: 38 | # db.rollback() 39 | # raise 40 | # finally: 41 | # db.close() 42 | # print('Finished splitting out bondsman') 43 | # op.drop_column('bail_and_bond', 'bail_bondsman') 44 | # op.drop_column('bail_and_bond', 'bondsman_address_1') 45 | # op.drop_column('bail_and_bond', 'bondsman_address_2') 46 | # op.drop_column('bail_and_bond', 'bondsman_city') 47 | # op.drop_column('bail_and_bond', 'bondsman_state') 48 | # op.drop_column('bail_and_bond', 'bondsman_zip_code') 49 | # 50 | # def downgrade(): 51 | # pass 52 | -------------------------------------------------------------------------------- /alembic.ini: -------------------------------------------------------------------------------- 1 | # A generic, single database configuration. 2 | 3 | [alembic] 4 | # path to migration scripts 5 | script_location = db 6 | 7 | # template used to generate migration files 8 | # file_template = %%(rev)s_%%(slug)s 9 | 10 | # timezone to use when rendering the date 11 | # within the migration file as well as the filename. 12 | # string value is passed to dateutil.tz.gettz() 13 | # leave blank for localtime 14 | # timezone = 15 | 16 | # max length of characters to apply to the 17 | # "slug" field 18 | #truncate_slug_length = 40 19 | 20 | # set to 'true' to run the environment during 21 | # the 'revision' command, regardless of autogenerate 22 | # revision_environment = false 23 | 24 | # set to 'true' to allow .pyc and .pyo files without 25 | # a source .py file to be detected as revisions in the 26 | # versions/ directory 27 | # sourceless = false 28 | 29 | # version location specification; this defaults 30 | # to db/versions. When using multiple version 31 | # directories, initial revisions must be specified with --version-path 32 | # version_locations = %(here)s/bar %(here)s/bat db/versions 33 | 34 | # the output encoding used when revision files 35 | # are written from script.py.mako 36 | # output_encoding = utf-8 37 | 38 | sqlalchemy.url = driver://user:pass@localhost/dbname 39 | 40 | 41 | # Logging configuration 42 | [loggers] 43 | keys = root,sqlalchemy,alembic 44 | 45 | [handlers] 46 | keys = console 47 | 48 | [formatters] 49 | keys = generic 50 | 51 | [logger_root] 52 | level = WARN 53 | handlers = console 54 | qualname = 55 | 56 | [logger_sqlalchemy] 57 | level = WARN 58 | handlers = 59 | qualname = sqlalchemy.engine 60 | 61 | [logger_alembic] 62 | level = INFO 63 | handlers = 64 | qualname = alembic 65 | 66 | [handler_console] 67 | class = StreamHandler 68 | args = (sys.stderr,) 69 | level = NOTSET 70 | formatter = generic 71 | 72 | [formatter_generic] 73 | format = %(levelname)-5.5s [%(name)s] %(message)s 74 | datefmt = %H:%M:%S 75 | -------------------------------------------------------------------------------- /db/versions/0be25bbbbd8c_added_some_missing_columns_related_to_.py: -------------------------------------------------------------------------------- 1 | """Added some missing columns related to ODYCIVIL 2 | 3 | Revision ID: 0be25bbbbd8c 4 | Revises: c2f35a46de03 5 | Create Date: 2021-03-01 17:30:00.832647 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '0be25bbbbd8c' 14 | down_revision = 'c2f35a46de03' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('odycivil_involved_parties', sa.Column('appearance_date', sa.Date(), nullable=True)) 22 | op.add_column('odycivil_involved_parties', sa.Column('appearance_date_str', sa.String(), nullable=True)) 23 | op.add_column('odycivil_involved_parties', sa.Column('removal_date', sa.Date(), nullable=True)) 24 | op.add_column('odycivil_involved_parties', sa.Column('removal_date_str', sa.String(), nullable=True)) 25 | op.add_column('odycivil_judgment_comments', sa.Column('dis_with_prej', sa.String(), nullable=True)) 26 | op.add_column('odycivil_judgment_comments', sa.Column('in_favor_of_def', sa.String(), nullable=True)) 27 | op.add_column('odycivil_judgment_comments', sa.Column('j_and_s_code', sa.String(), nullable=True)) 28 | # ### end Alembic commands ### 29 | 30 | 31 | def downgrade(): 32 | # ### commands auto generated by Alembic - please adjust! ### 33 | op.drop_column('odycivil_judgment_comments', 'j_and_s_code') 34 | op.drop_column('odycivil_judgment_comments', 'in_favor_of_def') 35 | op.drop_column('odycivil_judgment_comments', 'dis_with_prej') 36 | op.drop_column('odycivil_involved_parties', 'removal_date_str') 37 | op.drop_column('odycivil_involved_parties', 'removal_date') 38 | op.drop_column('odycivil_involved_parties', 'appearance_date_str') 39 | op.drop_column('odycivil_involved_parties', 'appearance_date') 40 | # ### end Alembic commands ### 41 | -------------------------------------------------------------------------------- /db/versions/05688275b38c_field_updates_for_dscivil.py: -------------------------------------------------------------------------------- 1 | """Field updates for DSCIVIL 2 | 3 | Revision ID: 05688275b38c 4 | Revises: e9d408e7b639 5 | Create Date: 2021-10-11 16:10:20.604520 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '05688275b38c' 14 | down_revision = 'e9d408e7b639' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('dscivil_judgments', sa.Column('possession_awardee', sa.String(), nullable=True)) 22 | op.add_column('dscivil_judgments', sa.Column('awardee', sa.String(), nullable=True)) 23 | op.drop_column('dscivil_related_persons', 'officer_id') 24 | op.drop_column('dscivil_related_persons', 'agency_sub_code') 25 | op.drop_column('dscivil_related_persons', 'agency_code') 26 | op.add_column('dscivil_trials', sa.Column('duration', sa.String(), nullable=True)) 27 | op.drop_column('dscivil_trials', 'reason') 28 | op.drop_column('dscivil_trials', 'trial_type') 29 | # ### end Alembic commands ### 30 | 31 | 32 | def downgrade(): 33 | # ### commands auto generated by Alembic - please adjust! ### 34 | op.add_column('dscivil_trials', sa.Column('trial_type', sa.VARCHAR(), autoincrement=False, nullable=True)) 35 | op.add_column('dscivil_trials', sa.Column('reason', sa.VARCHAR(), autoincrement=False, nullable=True)) 36 | op.drop_column('dscivil_trials', 'duration') 37 | op.add_column('dscivil_related_persons', sa.Column('agency_code', sa.VARCHAR(), autoincrement=False, nullable=True)) 38 | op.add_column('dscivil_related_persons', sa.Column('agency_sub_code', sa.VARCHAR(), autoincrement=False, nullable=True)) 39 | op.add_column('dscivil_related_persons', sa.Column('officer_id', sa.VARCHAR(), autoincrement=False, nullable=True)) 40 | op.drop_column('dscivil_judgments', 'awardee') 41 | op.drop_column('dscivil_judgments', 'possession_awardee') 42 | # ### end Alembic commands ### 43 | -------------------------------------------------------------------------------- /db/versions/10a283d18aa0_add_more_fields_to_k_charges.py: -------------------------------------------------------------------------------- 1 | """Add more fields to k_charges 2 | 3 | Revision ID: 10a283d18aa0 4 | Revises: 98327ab2ad75 5 | Create Date: 2021-08-23 08:24:24.002425 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '10a283d18aa0' 14 | down_revision = '98327ab2ad75' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('k_charges', sa.Column('fine_amount', sa.Numeric(), nullable=True)) 22 | op.add_column('k_charges', sa.Column('fine_suspended_amount', sa.Numeric(), nullable=True)) 23 | op.add_column('k_charges', sa.Column('fine_due', sa.String(), nullable=True)) 24 | op.add_column('k_charges', sa.Column('fine_first_payment_due', sa.String(), nullable=True)) 25 | op.add_column('k_charges', sa.Column('community_work_service_hours', sa.Numeric(), nullable=True)) 26 | op.add_column('k_charges', sa.Column('community_work_service_complete_by', sa.String(), nullable=True)) 27 | op.add_column('k_charges', sa.Column('community_work_service_report_to', sa.String(), nullable=True)) 28 | op.add_column('k_charges', sa.Column('community_work_service_report_date', sa.String(), nullable=True)) 29 | # ### end Alembic commands ### 30 | 31 | 32 | def downgrade(): 33 | # ### commands auto generated by Alembic - please adjust! ### 34 | op.drop_column('k_charges', 'community_work_service_report_date') 35 | op.drop_column('k_charges', 'community_work_service_report_to') 36 | op.drop_column('k_charges', 'community_work_service_complete_by') 37 | op.drop_column('k_charges', 'community_work_service_hours') 38 | op.drop_column('k_charges', 'fine_first_payment_due') 39 | op.drop_column('k_charges', 'fine_due') 40 | op.drop_column('k_charges', 'fine_suspended_amount') 41 | op.drop_column('k_charges', 'fine_amount') 42 | # ### end Alembic commands ### 43 | -------------------------------------------------------------------------------- /db/versions/df0221cb4562_add_appearance_and_removal_dates_to_.py: -------------------------------------------------------------------------------- 1 | """Add appearance and removal dates to ODYCRIM parties 2 | 3 | Revision ID: df0221cb4562 4 | Revises: eb90f8b96ee4 5 | Create Date: 2021-03-02 18:39:58.603330 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'df0221cb4562' 14 | down_revision = 'eb90f8b96ee4' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('odycrim_attorneys', sa.Column('appearance_date', sa.Date(), nullable=True)) 22 | op.add_column('odycrim_attorneys', sa.Column('appearance_date_str', sa.String(), nullable=True)) 23 | op.add_column('odycrim_attorneys', sa.Column('removal_date', sa.Date(), nullable=True)) 24 | op.add_column('odycrim_attorneys', sa.Column('removal_date_str', sa.String(), nullable=True)) 25 | op.add_column('odycrim_involved_parties', sa.Column('appearance_date', sa.Date(), nullable=True)) 26 | op.add_column('odycrim_involved_parties', sa.Column('appearance_date_str', sa.String(), nullable=True)) 27 | op.add_column('odycrim_involved_parties', sa.Column('removal_date', sa.Date(), nullable=True)) 28 | op.add_column('odycrim_involved_parties', sa.Column('removal_date_str', sa.String(), nullable=True)) 29 | # ### end Alembic commands ### 30 | 31 | 32 | def downgrade(): 33 | # ### commands auto generated by Alembic - please adjust! ### 34 | op.drop_column('odycrim_involved_parties', 'removal_date_str') 35 | op.drop_column('odycrim_involved_parties', 'removal_date') 36 | op.drop_column('odycrim_involved_parties', 'appearance_date_str') 37 | op.drop_column('odycrim_involved_parties', 'appearance_date') 38 | op.drop_column('odycrim_attorneys', 'removal_date_str') 39 | op.drop_column('odycrim_attorneys', 'removal_date') 40 | op.drop_column('odycrim_attorneys', 'appearance_date_str') 41 | op.drop_column('odycrim_attorneys', 'appearance_date') 42 | # ### end Alembic commands ### 43 | -------------------------------------------------------------------------------- /db/versions/eb90f8b96ee4_add_appearance_and_removal_dates_to_.py: -------------------------------------------------------------------------------- 1 | """Add appearance and removal dates to ODYTRAFF parties 2 | 3 | Revision ID: eb90f8b96ee4 4 | Revises: 025ecad5c330 5 | Create Date: 2021-03-02 18:25:50.680647 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'eb90f8b96ee4' 14 | down_revision = '025ecad5c330' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('odytraf_attorneys', sa.Column('appearance_date', sa.Date(), nullable=True)) 22 | op.add_column('odytraf_attorneys', sa.Column('appearance_date_str', sa.String(), nullable=True)) 23 | op.add_column('odytraf_attorneys', sa.Column('removal_date', sa.Date(), nullable=True)) 24 | op.add_column('odytraf_attorneys', sa.Column('removal_date_str', sa.String(), nullable=True)) 25 | op.add_column('odytraf_involved_parties', sa.Column('appearance_date', sa.Date(), nullable=True)) 26 | op.add_column('odytraf_involved_parties', sa.Column('appearance_date_str', sa.String(), nullable=True)) 27 | op.add_column('odytraf_involved_parties', sa.Column('removal_date', sa.Date(), nullable=True)) 28 | op.add_column('odytraf_involved_parties', sa.Column('removal_date_str', sa.String(), nullable=True)) 29 | # ### end Alembic commands ### 30 | 31 | 32 | def downgrade(): 33 | # ### commands auto generated by Alembic - please adjust! ### 34 | op.drop_column('odytraf_involved_parties', 'removal_date_str') 35 | op.drop_column('odytraf_involved_parties', 'removal_date') 36 | op.drop_column('odytraf_involved_parties', 'appearance_date_str') 37 | op.drop_column('odytraf_involved_parties', 'appearance_date') 38 | op.drop_column('odytraf_attorneys', 'removal_date_str') 39 | op.drop_column('odytraf_attorneys', 'removal_date') 40 | op.drop_column('odytraf_attorneys', 'appearance_date_str') 41 | op.drop_column('odytraf_attorneys', 'appearance_date') 42 | # ### end Alembic commands ### 43 | -------------------------------------------------------------------------------- /db/versions/c64263840760_remove_unused_fields_in_dsk8.py: -------------------------------------------------------------------------------- 1 | """Remove unused fields in DSK8 2 | 3 | Revision ID: c64263840760 4 | Revises: a68a233d6b44 5 | Create Date: 2021-10-12 09:06:59.559153 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'c64263840760' 14 | down_revision = 'a68a233d6b44' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.drop_column('dsk8_defendants', 'weight') 22 | op.drop_column('dsk8_defendants', 'height') 23 | op.drop_column('dsk8_related_persons', 'agency_code') 24 | op.drop_column('dsk8_related_persons', 'agency_sub_code') 25 | op.drop_column('dsk8_related_persons', 'attorney_firm') 26 | op.drop_column('dsk8_related_persons', 'attorney_code') 27 | op.drop_column('dsk8_related_persons', 'officer_id') 28 | op.drop_column('dsk8_trials', 'trial_type') 29 | # ### end Alembic commands ### 30 | 31 | 32 | def downgrade(): 33 | # ### commands auto generated by Alembic - please adjust! ### 34 | op.add_column('dsk8_trials', sa.Column('trial_type', sa.VARCHAR(), autoincrement=False, nullable=True)) 35 | op.add_column('dsk8_related_persons', sa.Column('officer_id', sa.VARCHAR(), autoincrement=False, nullable=True)) 36 | op.add_column('dsk8_related_persons', sa.Column('attorney_code', sa.INTEGER(), autoincrement=False, nullable=True)) 37 | op.add_column('dsk8_related_persons', sa.Column('attorney_firm', sa.VARCHAR(), autoincrement=False, nullable=True)) 38 | op.add_column('dsk8_related_persons', sa.Column('agency_sub_code', sa.VARCHAR(), autoincrement=False, nullable=True)) 39 | op.add_column('dsk8_related_persons', sa.Column('agency_code', sa.VARCHAR(), autoincrement=False, nullable=True)) 40 | op.add_column('dsk8_defendants', sa.Column('height', sa.INTEGER(), autoincrement=False, nullable=True)) 41 | op.add_column('dsk8_defendants', sa.Column('weight', sa.INTEGER(), autoincrement=False, nullable=True)) 42 | # ### end Alembic commands ### 43 | -------------------------------------------------------------------------------- /db/versions/9dbb2931da09_modified_k_charges.py: -------------------------------------------------------------------------------- 1 | """Modified k_charges 2 | 3 | Revision ID: 9dbb2931da09 4 | Revises: 66e022919043 5 | Create Date: 2021-08-23 06:34:54.284876 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '9dbb2931da09' 14 | down_revision = '66e022919043' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('k_charges', sa.Column('jail_unsuspended_years', sa.Integer(), nullable=True)) 22 | op.add_column('k_charges', sa.Column('jail_unsuspended_months', sa.Integer(), nullable=True)) 23 | op.add_column('k_charges', sa.Column('jail_unsuspended_days', sa.Integer(), nullable=True)) 24 | op.add_column('k_charges', sa.Column('jail_unsuspended_hours', sa.Integer(), nullable=True)) 25 | op.drop_column('k_charges', 'jail_suspend_all_but_months') 26 | op.drop_column('k_charges', 'jail_suspend_all_but_years') 27 | op.drop_column('k_charges', 'jail_suspend_all_but_days') 28 | op.drop_column('k_charges', 'jail_suspend_all_but_hours') 29 | # ### end Alembic commands ### 30 | 31 | 32 | def downgrade(): 33 | # ### commands auto generated by Alembic - please adjust! ### 34 | op.add_column('k_charges', sa.Column('jail_suspend_all_but_hours', sa.INTEGER(), autoincrement=False, nullable=True)) 35 | op.add_column('k_charges', sa.Column('jail_suspend_all_but_days', sa.INTEGER(), autoincrement=False, nullable=True)) 36 | op.add_column('k_charges', sa.Column('jail_suspend_all_but_years', sa.INTEGER(), autoincrement=False, nullable=True)) 37 | op.add_column('k_charges', sa.Column('jail_suspend_all_but_months', sa.INTEGER(), autoincrement=False, nullable=True)) 38 | op.drop_column('k_charges', 'jail_unsuspended_hours') 39 | op.drop_column('k_charges', 'jail_unsuspended_days') 40 | op.drop_column('k_charges', 'jail_unsuspended_months') 41 | op.drop_column('k_charges', 'jail_unsuspended_years') 42 | # ### end Alembic commands ### 43 | -------------------------------------------------------------------------------- /db/versions/db345d84fc41_remove_unused_fields_in_dstraf.py: -------------------------------------------------------------------------------- 1 | """Remove unused fields in DSTRAF 2 | 3 | Revision ID: db345d84fc41 4 | Revises: 67fc57997beb 5 | Create Date: 2021-10-11 16:59:16.928524 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'db345d84fc41' 14 | down_revision = '67fc57997beb' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.drop_column('dstraf', 'citation_number') 22 | op.drop_column('dstraf_charges', 'charge') 23 | op.drop_column('dstraf_dispositions', 'addition_charge') 24 | op.drop_column('dstraf_related_persons', 'attorney_firm') 25 | op.drop_column('dstraf_related_persons', 'agency_code') 26 | op.drop_column('dstraf_related_persons', 'officer_id') 27 | op.drop_column('dstraf_related_persons', 'agency_sub_code') 28 | op.drop_column('dstraf_related_persons', 'attorney_code') 29 | op.drop_column('dstraf_trials', 'trial_type') 30 | # ### end Alembic commands ### 31 | 32 | 33 | def downgrade(): 34 | # ### commands auto generated by Alembic - please adjust! ### 35 | op.add_column('dstraf_trials', sa.Column('trial_type', sa.VARCHAR(), autoincrement=False, nullable=True)) 36 | op.add_column('dstraf_related_persons', sa.Column('attorney_code', sa.INTEGER(), autoincrement=False, nullable=True)) 37 | op.add_column('dstraf_related_persons', sa.Column('agency_sub_code', sa.VARCHAR(), autoincrement=False, nullable=True)) 38 | op.add_column('dstraf_related_persons', sa.Column('officer_id', sa.VARCHAR(), autoincrement=False, nullable=True)) 39 | op.add_column('dstraf_related_persons', sa.Column('agency_code', sa.VARCHAR(), autoincrement=False, nullable=True)) 40 | op.add_column('dstraf_related_persons', sa.Column('attorney_firm', sa.VARCHAR(), autoincrement=False, nullable=True)) 41 | op.add_column('dstraf_dispositions', sa.Column('addition_charge', sa.VARCHAR(), autoincrement=False, nullable=True)) 42 | op.add_column('dstraf_charges', sa.Column('charge', sa.VARCHAR(), autoincrement=False, nullable=True)) 43 | op.add_column('dstraf', sa.Column('citation_number', sa.VARCHAR(), autoincrement=False, nullable=True)) 44 | # ### end Alembic commands ### 45 | -------------------------------------------------------------------------------- /db/versions/961f754ce080_update_odycosa_and_odycoa_models.py: -------------------------------------------------------------------------------- 1 | """Update ODYCOSA and ODYCOA models 2 | 3 | Revision ID: 961f754ce080 4 | Revises: 5d6a58de067a 5 | Create Date: 2023-04-05 12:13:57.808173 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '961f754ce080' 14 | down_revision = '5d6a58de067a' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.add_column('odycosa_involved_parties', sa.Column('DOB', sa.Date(), nullable=True)) 21 | op.add_column('odycosa_involved_parties', sa.Column('DOB_str', sa.String(), nullable=True)) 22 | op.add_column('odycosa_involved_parties', sa.Column('race', sa.String(), nullable=True)) 23 | op.add_column('odycosa_involved_parties', sa.Column('sex', sa.String(), nullable=True)) 24 | op.add_column('odycosa_involved_parties', sa.Column('height', sa.String(), nullable=True)) 25 | op.add_column('odycosa_involved_parties', sa.Column('weight', sa.Integer(), nullable=True)) 26 | op.add_column('odycosa_involved_parties', sa.Column('hair_color', sa.String(), nullable=True)) 27 | op.add_column('odycosa_involved_parties', sa.Column('eye_color', sa.String(), nullable=True)) 28 | op.drop_column('odycosa_involved_parties', 'agency_name') 29 | op.add_column('odycosa_judgments', sa.Column('judge_name', sa.String(), nullable=True)) 30 | op.add_column('odycosa_court_schedule', sa.Column('panel_judges', sa.String(), nullable=True)) 31 | op.add_column('odycosa_court_schedule', sa.Column('result', sa.String(), nullable=True)) 32 | 33 | 34 | def downgrade(): 35 | op.drop_column('odycosa_court_schedule', 'result') 36 | op.drop_column('odycosa_court_schedule', 'panel_judges') 37 | op.drop_column('odycosa_judgments', 'judge_name') 38 | op.add_column('odycosa_involved_parties', sa.Column('agency_name', sa.VARCHAR(), autoincrement=False, nullable=True)) 39 | op.drop_column('odycosa_involved_parties', 'eye_color') 40 | op.drop_column('odycosa_involved_parties', 'hair_color') 41 | op.drop_column('odycosa_involved_parties', 'weight') 42 | op.drop_column('odycosa_involved_parties', 'height') 43 | op.drop_column('odycosa_involved_parties', 'sex') 44 | op.drop_column('odycosa_involved_parties', 'race') 45 | op.drop_column('odycosa_involved_parties', 'DOB_str') 46 | op.drop_column('odycosa_involved_parties', 'DOB') 47 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/dismantl/caseharvester 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/akamensky/argparse v1.4.0 7 | github.com/aws/aws-sdk-go-v2 v1.17.7 8 | github.com/aws/aws-sdk-go-v2/config v1.18.19 9 | github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.25.7 10 | github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.17.3 11 | github.com/aws/aws-sdk-go-v2/service/ec2 v1.91.0 12 | github.com/aws/aws-sdk-go-v2/service/eventbridge v1.18.7 13 | github.com/aws/aws-sdk-go-v2/service/ssm v1.36.0 14 | github.com/gorilla/mux v1.8.0 15 | github.com/lzap/cloudwatchwriter2 v1.1.0 16 | github.com/pkg/errors v0.9.1 17 | github.com/rs/zerolog v1.29.0 18 | github.com/spf13/viper v1.15.0 19 | ) 20 | 21 | require ( 22 | github.com/aws/aws-sdk-go-v2/credentials v1.13.18 // indirect 23 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 // indirect 24 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 // indirect 25 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 // indirect 26 | github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 // indirect 27 | github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.23 // indirect 28 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 // indirect 29 | github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 // indirect 30 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 // indirect 31 | github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 // indirect 32 | github.com/aws/smithy-go v1.13.5 // indirect 33 | github.com/fsnotify/fsnotify v1.6.0 // indirect 34 | github.com/hashicorp/hcl v1.0.0 // indirect 35 | github.com/jmespath/go-jmespath v0.4.0 // indirect 36 | github.com/magiconair/properties v1.8.7 // indirect 37 | github.com/mattn/go-colorable v0.1.13 // indirect 38 | github.com/mattn/go-isatty v0.0.16 // indirect 39 | github.com/mitchellh/mapstructure v1.5.0 // indirect 40 | github.com/oleiade/lane/v2 v2.0.0 // indirect 41 | github.com/pelletier/go-toml/v2 v2.0.6 // indirect 42 | github.com/spf13/afero v1.9.3 // indirect 43 | github.com/spf13/cast v1.5.0 // indirect 44 | github.com/spf13/jwalterweatherman v1.1.0 // indirect 45 | github.com/spf13/pflag v1.0.5 // indirect 46 | github.com/subosito/gotenv v1.4.2 // indirect 47 | golang.org/x/exp v0.0.0-20220827204233-334a2380cb91 // indirect 48 | golang.org/x/sys v0.3.0 // indirect 49 | golang.org/x/text v0.5.0 // indirect 50 | gopkg.in/ini.v1 v1.67.0 // indirect 51 | gopkg.in/yaml.v3 v3.0.1 // indirect 52 | ) 53 | -------------------------------------------------------------------------------- /db/versions/bf65425d1227_add_dscr_bail_event_model.py: -------------------------------------------------------------------------------- 1 | """Add DSCR bail event model 2 | 3 | Revision ID: bf65425d1227 4 | Revises: 5f3daa92e736 5 | Create Date: 2019-05-17 20:40:42.215862 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'bf65425d1227' 14 | down_revision = '5f3daa92e736' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('dscr_bail_events', 22 | sa.Column('id', sa.Integer(), nullable=False), 23 | sa.Column('event_name', sa.String(), nullable=True), 24 | sa.Column('date', sa.Date(), nullable=True), 25 | sa.Column('date_str', sa.String(), nullable=True), 26 | sa.Column('bail_amount', sa.Numeric(), nullable=True), 27 | sa.Column('code', sa.String(), nullable=True), 28 | sa.Column('percentage_required', sa.Numeric(), nullable=True), 29 | sa.Column('type_of_bond', sa.String(), nullable=True), 30 | sa.Column('judge_id', sa.String(), nullable=True), 31 | sa.Column('case_number', sa.String(), nullable=True), 32 | sa.ForeignKeyConstraint(['case_number'], ['dscr.case_number'], ondelete='CASCADE'), 33 | sa.PrimaryKeyConstraint('id') 34 | ) 35 | op.create_index(op.f('ix_dscr_bail_events_case_number'), 'dscr_bail_events', ['case_number'], unique=False) 36 | op.execute(""" 37 | INSERT INTO 38 | dscr_bail_events ( 39 | case_number, 40 | event_name, 41 | date_str, 42 | date, 43 | bail_amount, 44 | code, 45 | percentage_required, 46 | type_of_bond, 47 | judge_id) 48 | SELECT 49 | case_number, 50 | event_name, 51 | bail[1], 52 | TO_DATE(bail[1], 'YYMMDD'), 53 | CAST(bail[2] AS NUMERIC), 54 | TRIM(bail[3]), 55 | CASE 56 | WHEN bail[4] = ' ' THEN NULL 57 | ELSE CAST(bail[4] AS NUMERIC) 58 | END, 59 | TRIM(bail[5]), 60 | TRIM(bail[6]) 61 | FROM ( 62 | SELECT 63 | case_number, 64 | event_name, 65 | regexp_split_to_array(comment, ';') AS bail 66 | FROM 67 | dscr_events 68 | WHERE 69 | event_name = 'BALR' 70 | OR event_name = 'BSET' 71 | OR event_name = 'INIT') foo 72 | """) 73 | # ### end Alembic commands ### 74 | 75 | 76 | def downgrade(): 77 | # ### commands auto generated by Alembic - please adjust! ### 78 | op.drop_index(op.f('ix_dscr_bail_events_case_number'), table_name='dscr_bail_events') 79 | op.drop_table('dscr_bail_events') 80 | # ### end Alembic commands ### 81 | -------------------------------------------------------------------------------- /db/versions/f3606af6398f_remove_unused_fields_in_odytraf.py: -------------------------------------------------------------------------------- 1 | """Remove unused fields in ODYTRAF 2 | 3 | Revision ID: f3606af6398f 4 | Revises: b4f7a1413269 5 | Create Date: 2021-10-11 15:25:04.463118 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'f3606af6398f' 14 | down_revision = 'b4f7a1413269' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.drop_column('odytraf', 'officer_name') 22 | op.drop_column('odytraf', 'citation_number') 23 | op.drop_column('odytraf', 'officer_id') 24 | op.drop_column('odytraf', 'agency_name') 25 | op.drop_column('odytraf_charges', 'jail_life_death') 26 | op.drop_column('odytraf_documents', 'filed_by') 27 | op.drop_column('odytraf_involved_parties', 'appearance_date') 28 | op.drop_column('odytraf_involved_parties', 'removal_date') 29 | op.drop_column('odytraf_involved_parties', 'removal_date_str') 30 | op.drop_column('odytraf_involved_parties', 'appearance_date_str') 31 | op.drop_column('odytraf_services', 'service_status') 32 | # ### end Alembic commands ### 33 | 34 | 35 | def downgrade(): 36 | # ### commands auto generated by Alembic - please adjust! ### 37 | op.add_column('odytraf_services', sa.Column('service_status', sa.VARCHAR(), autoincrement=False, nullable=True)) 38 | op.add_column('odytraf_involved_parties', sa.Column('appearance_date_str', sa.VARCHAR(), autoincrement=False, nullable=True)) 39 | op.add_column('odytraf_involved_parties', sa.Column('removal_date_str', sa.VARCHAR(), autoincrement=False, nullable=True)) 40 | op.add_column('odytraf_involved_parties', sa.Column('removal_date', sa.DATE(), autoincrement=False, nullable=True)) 41 | op.add_column('odytraf_involved_parties', sa.Column('appearance_date', sa.DATE(), autoincrement=False, nullable=True)) 42 | op.add_column('odytraf_documents', sa.Column('filed_by', sa.VARCHAR(), autoincrement=False, nullable=True)) 43 | op.add_column('odytraf_charges', sa.Column('jail_life_death', sa.VARCHAR(), autoincrement=False, nullable=True)) 44 | op.add_column('odytraf', sa.Column('agency_name', sa.VARCHAR(), autoincrement=False, nullable=True)) 45 | op.add_column('odytraf', sa.Column('officer_id', sa.VARCHAR(), autoincrement=False, nullable=True)) 46 | op.add_column('odytraf', sa.Column('citation_number', sa.VARCHAR(), autoincrement=False, nullable=True)) 47 | op.add_column('odytraf', sa.Column('officer_name', sa.VARCHAR(), autoincrement=False, nullable=True)) 48 | # ### end Alembic commands ### 49 | -------------------------------------------------------------------------------- /db/versions/4394875fda45_add_possibly_expunged_flag_to_odytraf_.py: -------------------------------------------------------------------------------- 1 | """Add possibly_expunged flag to odytraf_charges, dscr_charges, dsk8_charges 2 | 3 | Revision ID: 4394875fda45 4 | Revises: 0b545c28e0a3 5 | Create Date: 2021-07-22 14:09:27.681870 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '4394875fda45' 14 | down_revision = '0b545c28e0a3' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('dscr_charges', sa.Column('possibly_expunged', sa.Boolean(), server_default='false', nullable=False)) 22 | op.alter_column('dscr_charges', 'case_number', 23 | existing_type=sa.VARCHAR(), 24 | nullable=False) 25 | op.drop_constraint('dscr_charges_case_number_fkey', 'dscr_charges', type_='foreignkey') 26 | op.add_column('dsk8_charges', sa.Column('possibly_expunged', sa.Boolean(), server_default='false', nullable=False)) 27 | op.alter_column('dsk8_charges', 'case_number', 28 | existing_type=sa.VARCHAR(), 29 | nullable=False) 30 | op.drop_constraint('dsk8_charges_case_number_fkey', 'dsk8_charges', type_='foreignkey') 31 | op.add_column('odytraf_charges', sa.Column('possibly_expunged', sa.Boolean(), server_default='false', nullable=False)) 32 | op.alter_column('odytraf_charges', 'case_number', 33 | existing_type=sa.VARCHAR(), 34 | nullable=False) 35 | op.drop_constraint('odytraf_charges_case_number_fkey', 'odytraf_charges', type_='foreignkey') 36 | # ### end Alembic commands ### 37 | 38 | 39 | def downgrade(): 40 | # ### commands auto generated by Alembic - please adjust! ### 41 | op.create_foreign_key('odytraf_charges_case_number_fkey', 'odytraf_charges', 'odytraf', ['case_number'], ['case_number'], ondelete='CASCADE') 42 | op.alter_column('odytraf_charges', 'case_number', 43 | existing_type=sa.VARCHAR(), 44 | nullable=True) 45 | op.drop_column('odytraf_charges', 'possibly_expunged') 46 | op.create_foreign_key('dsk8_charges_case_number_fkey', 'dsk8_charges', 'dsk8', ['case_number'], ['case_number'], ondelete='CASCADE') 47 | op.alter_column('dsk8_charges', 'case_number', 48 | existing_type=sa.VARCHAR(), 49 | nullable=True) 50 | op.drop_column('dsk8_charges', 'possibly_expunged') 51 | op.create_foreign_key('dscr_charges_case_number_fkey', 'dscr_charges', 'dscr', ['case_number'], ['case_number'], ondelete='CASCADE') 52 | op.alter_column('dscr_charges', 'case_number', 53 | existing_type=sa.VARCHAR(), 54 | nullable=True) 55 | op.drop_column('dscr_charges', 'possibly_expunged') 56 | # ### end Alembic commands ### 57 | -------------------------------------------------------------------------------- /db/versions/4baf21cf0e43_add_active_boolean_to_case_table.py: -------------------------------------------------------------------------------- 1 | """Add active boolean to case table 2 | 3 | Revision ID: 4baf21cf0e43 4 | Revises: d74fb94c58b2 5 | Create Date: 2020-06-22 15:03:41.360516 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '4baf21cf0e43' 14 | down_revision = 'd74fb94c58b2' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('cases', sa.Column('active', sa.Boolean(), default=True, nullable=False)) 22 | print('Updating CC cases') 23 | op.execute(sa.text(""" 24 | UPDATE cases 25 | SET active = False 26 | FROM cc 27 | WHERE cases.case_number = cc.case_number 28 | AND cc.case_status = 'Closed/Inactive' 29 | """)) 30 | print('Updating DSCIVIL cases') 31 | op.execute(sa.text(""" 32 | UPDATE cases 33 | SET active = False 34 | FROM dscivil 35 | WHERE cases.case_number = dscivil.case_number 36 | AND dscivil.case_status = 'CLOSED' 37 | """)) 38 | print('Updating DSCR cases') 39 | op.execute(sa.text(""" 40 | UPDATE cases 41 | SET active = False 42 | FROM dscr 43 | WHERE cases.case_number = dscr.case_number 44 | AND dscr.case_status IN ('INACTIVE DUE TO INCOMPETENCY', 'CLOSED') 45 | """)) 46 | print('Updating DSK8 cases') 47 | op.execute(sa.text(""" 48 | UPDATE cases 49 | SET active = False 50 | FROM dsk8 51 | WHERE cases.case_number = dsk8.case_number 52 | AND dsk8.case_status IN ('INACTIVE', 'CLOSED') 53 | """)) 54 | print('Updating ODYCRIM cases') 55 | op.execute(sa.text(""" 56 | UPDATE cases 57 | SET active = False 58 | FROM odycrim 59 | WHERE cases.case_number = odycrim.case_number 60 | AND odycrim.case_status IN ( 61 | 'Citation Voided', 62 | 'Inactive / Incompetency', 63 | 'Closed / Inactive', 64 | 'Closed', 65 | 'Completed' 66 | ) 67 | """)) 68 | print('Updating ODYTRAF cases') 69 | op.execute(sa.text(""" 70 | UPDATE cases 71 | SET active = False 72 | FROM odytraf 73 | WHERE cases.case_number = odytraf.case_number 74 | AND odytraf.case_status IN ( 75 | 'Citation Voided', 76 | 'Inactive / Incompetency', 77 | 'Closed / Inactive', 78 | 'Closed', 79 | 'Expunged', 80 | 'Inactive' 81 | ) 82 | """)) 83 | # ### end Alembic commands ### 84 | 85 | 86 | def downgrade(): 87 | # ### commands auto generated by Alembic - please adjust! ### 88 | op.drop_column('cases', 'active') 89 | # ### end Alembic commands ### 90 | -------------------------------------------------------------------------------- /db/versions/67fc57997beb_remove_unused_fields_in_dscp.py: -------------------------------------------------------------------------------- 1 | """Remove unused fields in DSCP 2 | 3 | Revision ID: 67fc57997beb 4 | Revises: 05688275b38c 5 | Create Date: 2021-10-11 16:21:42.352090 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '67fc57997beb' 14 | down_revision = '05688275b38c' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.drop_column('dscp_defendant_aliases', 'state') 22 | op.drop_column('dscp_defendant_aliases', 'address_2') 23 | op.drop_column('dscp_defendant_aliases', 'address_1') 24 | op.drop_column('dscp_defendant_aliases', 'zip_code') 25 | op.drop_column('dscp_defendant_aliases', 'city') 26 | op.drop_column('dscp_related_persons', 'state') 27 | op.drop_column('dscp_related_persons', 'attorney_firm') 28 | op.drop_column('dscp_related_persons', 'attorney_code') 29 | op.drop_column('dscp_related_persons', 'address_2') 30 | op.drop_column('dscp_related_persons', 'address_1') 31 | op.drop_column('dscp_related_persons', 'zip_code') 32 | op.drop_column('dscp_related_persons', 'city') 33 | op.drop_column('dscp_trials', 'reason') 34 | # ### end Alembic commands ### 35 | 36 | 37 | def downgrade(): 38 | # ### commands auto generated by Alembic - please adjust! ### 39 | op.add_column('dscp_trials', sa.Column('reason', sa.VARCHAR(), autoincrement=False, nullable=True)) 40 | op.add_column('dscp_related_persons', sa.Column('city', sa.VARCHAR(), autoincrement=False, nullable=True)) 41 | op.add_column('dscp_related_persons', sa.Column('zip_code', sa.VARCHAR(), autoincrement=False, nullable=True)) 42 | op.add_column('dscp_related_persons', sa.Column('address_1', sa.VARCHAR(), autoincrement=False, nullable=True)) 43 | op.add_column('dscp_related_persons', sa.Column('address_2', sa.VARCHAR(), autoincrement=False, nullable=True)) 44 | op.add_column('dscp_related_persons', sa.Column('attorney_code', sa.INTEGER(), autoincrement=False, nullable=True)) 45 | op.add_column('dscp_related_persons', sa.Column('attorney_firm', sa.VARCHAR(), autoincrement=False, nullable=True)) 46 | op.add_column('dscp_related_persons', sa.Column('state', sa.VARCHAR(), autoincrement=False, nullable=True)) 47 | op.add_column('dscp_defendant_aliases', sa.Column('city', sa.VARCHAR(), autoincrement=False, nullable=True)) 48 | op.add_column('dscp_defendant_aliases', sa.Column('zip_code', sa.VARCHAR(), autoincrement=False, nullable=True)) 49 | op.add_column('dscp_defendant_aliases', sa.Column('address_1', sa.VARCHAR(), autoincrement=False, nullable=True)) 50 | op.add_column('dscp_defendant_aliases', sa.Column('address_2', sa.VARCHAR(), autoincrement=False, nullable=True)) 51 | op.add_column('dscp_defendant_aliases', sa.Column('state', sa.VARCHAR(), autoincrement=False, nullable=True)) 52 | # ### end Alembic commands ### 53 | -------------------------------------------------------------------------------- /db/env.py: -------------------------------------------------------------------------------- 1 | from __future__ import with_statement 2 | from alembic import context 3 | from sqlalchemy import engine_from_config, pool 4 | from sqlalchemy.sql import text 5 | from logging.config import fileConfig 6 | import sys 7 | import os 8 | 9 | # Add our source path to the search paths for modules 10 | src_path = os.path.join(os.path.dirname(__file__), '..') 11 | sys.path.insert(0, src_path) 12 | 13 | # Import our models and database objects 14 | from mjcs.config import config as my_config 15 | from mjcs.models.common import TableBase 16 | from mjcs.models import * 17 | if os.getenv('CASEHARVESTER_ENV') == 'production': 18 | my_config.initialize_from_environment('production') 19 | else: 20 | my_config.initialize_from_environment('development') 21 | 22 | # this is the Alembic Config object, which provides 23 | # access to the values within the .ini file in use. 24 | config = context.config 25 | 26 | # Interpret the config file for Python logging. 27 | # This line sets up loggers basically. 28 | fileConfig(config.config_file_name) 29 | 30 | # add your model's MetaData object here 31 | # for 'autogenerate' support 32 | # from myapp import mymodel 33 | # target_metadata = mymodel.Base.metadata 34 | target_metadata = TableBase.metadata 35 | 36 | # other values from the config, defined by the needs of env.py, 37 | # can be acquired: 38 | # my_important_option = config.get_main_option("my_important_option") 39 | # ... etc. 40 | 41 | 42 | def run_migrations_offline(): 43 | """Run migrations in 'offline' mode. 44 | 45 | This configures the context with just a URL 46 | and not an Engine, though an Engine is acceptable 47 | here as well. By skipping the Engine creation 48 | we don't even need a DBAPI to be available. 49 | 50 | Calls to context.execute() here emit the given string to the 51 | script output. 52 | 53 | """ 54 | url = my_config.MJCS_DATABASE_URL 55 | context.configure( 56 | url=url, target_metadata=target_metadata, literal_binds=True) 57 | 58 | with context.begin_transaction(): 59 | context.run_migrations() 60 | 61 | 62 | def run_migrations_online(): 63 | """Run migrations in 'online' mode. 64 | 65 | In this scenario we need to create an Engine 66 | and associate a connection with the context. 67 | 68 | """ 69 | connectable = my_config.db_engine 70 | 71 | with connectable.connect() as connection: 72 | context.configure( 73 | connection=connection, 74 | target_metadata=target_metadata 75 | ) 76 | 77 | with context.begin_transaction(): 78 | context.run_migrations() 79 | 80 | if context.is_offline_mode(): 81 | run_migrations_offline() 82 | else: 83 | run_migrations_online() 84 | 85 | # Run idempotent SQL scripts 86 | scripts_path = os.path.join(os.path.dirname(__file__), 'sql') 87 | for file in os.listdir(scripts_path): 88 | if file.endswith('.sql'): 89 | print(f'Running SQL initialization script {file}') 90 | with open(os.path.join(scripts_path, file), 'r') as script: 91 | commands = script.read() 92 | with my_config.db_engine.begin() as conn: 93 | conn.execute(text(commands)) -------------------------------------------------------------------------------- /lib/psycopg2/_ipaddress.py: -------------------------------------------------------------------------------- 1 | """Implementation of the ipaddres-based network types adaptation 2 | """ 3 | 4 | # psycopg/_ipaddress.py - Ipaddres-based network types adaptation 5 | # 6 | # Copyright (C) 2016-2019 Daniele Varrazzo 7 | # Copyright (C) 2020 The Psycopg Team 8 | # 9 | # psycopg2 is free software: you can redistribute it and/or modify it 10 | # under the terms of the GNU Lesser General Public License as published 11 | # by the Free Software Foundation, either version 3 of the License, or 12 | # (at your option) any later version. 13 | # 14 | # In addition, as a special exception, the copyright holders give 15 | # permission to link this program with the OpenSSL library (or with 16 | # modified versions of OpenSSL that use the same license as OpenSSL), 17 | # and distribute linked combinations including the two. 18 | # 19 | # You must obey the GNU Lesser General Public License in all respects for 20 | # all of the code used other than OpenSSL. 21 | # 22 | # psycopg2 is distributed in the hope that it will be useful, but WITHOUT 23 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 24 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 25 | # License for more details. 26 | 27 | from psycopg2.extensions import ( 28 | new_type, new_array_type, register_type, register_adapter, QuotedString) 29 | from psycopg2.compat import text_type 30 | 31 | # The module is imported on register_ipaddress 32 | ipaddress = None 33 | 34 | # The typecasters are created only once 35 | _casters = None 36 | 37 | 38 | def register_ipaddress(conn_or_curs=None): 39 | """ 40 | Register conversion support between `ipaddress` objects and `network types`__. 41 | 42 | :param conn_or_curs: the scope where to register the type casters. 43 | If `!None` register them globally. 44 | 45 | After the function is called, PostgreSQL :sql:`inet` values will be 46 | converted into `~ipaddress.IPv4Interface` or `~ipaddress.IPv6Interface` 47 | objects, :sql:`cidr` values into into `~ipaddress.IPv4Network` or 48 | `~ipaddress.IPv6Network`. 49 | 50 | .. __: https://www.postgresql.org/docs/current/static/datatype-net-types.html 51 | """ 52 | global ipaddress 53 | import ipaddress 54 | 55 | global _casters 56 | if _casters is None: 57 | _casters = _make_casters() 58 | 59 | for c in _casters: 60 | register_type(c, conn_or_curs) 61 | 62 | for t in [ipaddress.IPv4Interface, ipaddress.IPv6Interface, 63 | ipaddress.IPv4Network, ipaddress.IPv6Network]: 64 | register_adapter(t, adapt_ipaddress) 65 | 66 | 67 | def _make_casters(): 68 | inet = new_type((869,), 'INET', cast_interface) 69 | ainet = new_array_type((1041,), 'INET[]', inet) 70 | 71 | cidr = new_type((650,), 'CIDR', cast_network) 72 | acidr = new_array_type((651,), 'CIDR[]', cidr) 73 | 74 | return [inet, ainet, cidr, acidr] 75 | 76 | 77 | def cast_interface(s, cur=None): 78 | if s is None: 79 | return None 80 | # Py2 version force the use of unicode. meh. 81 | return ipaddress.ip_interface(text_type(s)) 82 | 83 | 84 | def cast_network(s, cur=None): 85 | if s is None: 86 | return None 87 | return ipaddress.ip_network(text_type(s)) 88 | 89 | 90 | def adapt_ipaddress(obj): 91 | return QuotedString(str(obj)) 92 | -------------------------------------------------------------------------------- /db/versions/93332e9c34d5_added_landlord_tenants_judgments_to_.py: -------------------------------------------------------------------------------- 1 | """Added LandLord Tenants judgments to odycivil_judgments 2 | 3 | Revision ID: 93332e9c34d5 4 | Revises: 0327e0a1ca6a 5 | Create Date: 2021-07-02 17:40:11.384169 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '93332e9c34d5' 14 | down_revision = '0327e0a1ca6a' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('odycivil_judgments', sa.Column('costs', sa.Boolean(), nullable=True)) 22 | op.add_column('odycivil_judgments', sa.Column('judgment_for', sa.String(), nullable=True)) 23 | op.add_column('odycivil_judgments', sa.Column('monetary_judgment', sa.Boolean(), nullable=True)) 24 | op.add_column('odycivil_judgments', sa.Column('party', sa.String(), nullable=True)) 25 | op.add_column('odycivil_judgments', sa.Column('possession', sa.Boolean(), nullable=True)) 26 | op.add_column('odycivil_judgments', sa.Column('premise_description', sa.String(), nullable=True)) 27 | op.add_column('odycivil_judgments', sa.Column('stay_of_execution_until', sa.Date(), nullable=True)) 28 | op.add_column('odycivil_judgments', sa.Column('stay_of_execution_until_str', sa.String(), nullable=True)) 29 | op.add_column('odycivil_judgments', sa.Column('stay_upon_filing_of_bond', sa.Boolean(), nullable=True)) 30 | op.add_column('odycivil_judgments', sa.Column('appeal_bond_amount', sa.Numeric(), nullable=True)) 31 | op.add_column('odycivil_judgments', sa.Column('costs_', sa.Numeric(), nullable=True)) 32 | op.add_column('odycivil_judgments', sa.Column('court_costs', sa.Numeric(), nullable=True)) 33 | op.add_column('odycivil_judgments', sa.Column('judgment', sa.String(), nullable=True)) 34 | op.add_column('odycivil_judgments', sa.Column('stay_details', sa.String(), nullable=True)) 35 | op.add_column('odycivil_judgments', sa.Column('judgment_expiration_date', sa.Date(), nullable=True)) 36 | op.add_column('odycivil_judgments', sa.Column('judgment_expiration_date_str', sa.String(), nullable=True)) 37 | # ### end Alembic commands ### 38 | 39 | 40 | def downgrade(): 41 | # ### commands auto generated by Alembic - please adjust! ### 42 | op.drop_column('odycivil_judgments', 'judgment_expiration_date_str') 43 | op.drop_column('odycivil_judgments', 'judgment_expiration_date') 44 | op.drop_column('odycivil_judgments', 'stay_upon_filing_of_bond') 45 | op.drop_column('odycivil_judgments', 'stay_of_execution_until_str') 46 | op.drop_column('odycivil_judgments', 'stay_of_execution_until') 47 | op.drop_column('odycivil_judgments', 'premise_description') 48 | op.drop_column('odycivil_judgments', 'possession') 49 | op.drop_column('odycivil_judgments', 'party') 50 | op.drop_column('odycivil_judgments', 'monetary_judgment') 51 | op.drop_column('odycivil_judgments', 'judgment_for') 52 | op.drop_column('odycivil_judgments', 'costs') 53 | op.drop_column('odycivil_judgments', 'stay_details') 54 | op.drop_column('odycivil_judgments', 'judgment') 55 | op.drop_column('odycivil_judgments', 'court_costs') 56 | op.drop_column('odycivil_judgments', 'costs_') 57 | op.drop_column('odycivil_judgments', 'appeal_bond_amount') 58 | # ### end Alembic commands ### 59 | -------------------------------------------------------------------------------- /db/versions/c063d4f8fbf2_add_new_judge_related_fields.py: -------------------------------------------------------------------------------- 1 | """Add new judge-related fields 2 | 3 | Revision ID: c063d4f8fbf2 4 | Revises: 20e6f8ac26d5 5 | Create Date: 2023-04-05 12:07:03.377875 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'c063d4f8fbf2' 14 | down_revision = '20e6f8ac26d5' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.add_column('odytraf', sa.Column('judicial_officer', sa.String(), nullable=True)) 21 | op.add_column('odytraf_bond_settings', sa.Column('judge', sa.String(), nullable=True)) 22 | op.add_column('odytraf_charges', sa.Column('plea_judge', sa.String(), nullable=True)) 23 | op.add_column('odytraf_charges', sa.Column('disposition_judge', sa.String(), nullable=True)) 24 | op.add_column('odytraf_charges', sa.Column('sentence_judge', sa.String(), nullable=True)) 25 | op.add_column('odytraf_court_schedule', sa.Column('judge', sa.String(), nullable=True)) 26 | op.add_column('odytraf_warrants', sa.Column('judge', sa.String(), nullable=True)) 27 | op.add_column('odycrim', sa.Column('judicial_officer', sa.String(), nullable=True)) 28 | op.add_column('odycrim_bond_settings', sa.Column('judge', sa.String(), nullable=True)) 29 | op.add_column('odycrim_charges', sa.Column('sentence_judge', sa.String(), nullable=True)) 30 | op.add_column('odycrim_charges', sa.Column('plea_judge', sa.String(), nullable=True)) 31 | op.add_column('odycrim_charges', sa.Column('disposition_judge', sa.String(), nullable=True)) 32 | op.add_column('odycrim_court_schedule', sa.Column('judge', sa.String(), nullable=True)) 33 | op.add_column('odycrim_warrants', sa.Column('judge', sa.String(), nullable=True)) 34 | op.add_column('odycivil', sa.Column('magistrate', sa.String(), nullable=True)) 35 | op.add_column('odycivil', sa.Column('judicial_officer', sa.String(), nullable=True)) 36 | op.add_column('odycivil_bond_settings', sa.Column('judge', sa.String(), nullable=True)) 37 | op.add_column('odycivil_warrants', sa.Column('judge', sa.String(), nullable=True)) 38 | op.add_column('odycivil_judgments', sa.Column('judge', sa.String(), nullable=True)) 39 | op.add_column('odycivil_court_schedule', sa.Column('judge', sa.String(), nullable=True)) 40 | 41 | 42 | def downgrade(): 43 | op.drop_column('odycivil_court_schedule', 'judge') 44 | op.drop_column('odycivil_judgments', 'judge') 45 | op.drop_column('odycivil_warrants', 'judge') 46 | op.drop_column('odycivil_bond_settings', 'judge') 47 | op.drop_column('odycivil', 'judicial_officer') 48 | op.drop_column('odycivil', 'magistrate') 49 | op.drop_column('odycrim_warrants', 'judge') 50 | op.drop_column('odycrim_court_schedule', 'judge') 51 | op.drop_column('odycrim_charges', 'disposition_judge') 52 | op.drop_column('odycrim_charges', 'plea_judge') 53 | op.drop_column('odycrim_charges', 'sentence_judge') 54 | op.drop_column('odycrim_bond_settings', 'judge') 55 | op.drop_column('odycrim', 'judicial_officer') 56 | op.drop_column('odytraf_warrants', 'judge') 57 | op.drop_column('odytraf_court_schedule', 'judge') 58 | op.drop_column('odytraf_charges', 'sentence_judge') 59 | op.drop_column('odytraf_charges', 'disposition_judge') 60 | op.drop_column('odytraf_charges', 'plea_judge') 61 | op.drop_column('odytraf_bond_settings', 'judge') 62 | op.drop_column('odytraf', 'judicial_officer') 63 | -------------------------------------------------------------------------------- /mjcs/session.py: -------------------------------------------------------------------------------- 1 | from .config import config 2 | import logging 3 | import requests 4 | import time 5 | from bs4 import BeautifulSoup 6 | # from pypasser import reCaptchaV3 7 | 8 | logger = logging.getLogger('mjcs') 9 | 10 | class RequestTimeout(Exception): 11 | pass 12 | 13 | class Forbidden(Exception): 14 | pass 15 | 16 | class MjcsSession: 17 | def __init__(self): 18 | self.new_session() 19 | self.requests = 0 20 | 21 | def new_session(self): 22 | self.session = requests.Session() 23 | # Because all it takes to bypass DataDome is a few headers... 24 | self.session.headers.update({ 25 | 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36', 26 | 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', 27 | 'Accept-Encoding': 'gzip, deflate', 28 | 'Accept-Language': 'en-US,en;q=0.9' 29 | }) 30 | 31 | def request(self, *args, i=1, **kwargs): 32 | if i > 2: 33 | raise Exception('Too many recursed requests') 34 | self.requests += 1 35 | response = self.session.request( 36 | *args, 37 | **kwargs, 38 | timeout=config.QUERY_TIMEOUT 39 | ) 40 | 41 | if ((response.history and response.history[0].status_code == 302 and 42 | response.history[0].headers['location'] == f'{config.MJCS_BASE_URL}/inquiry-index.jsp') 43 | or "Acceptance of the following agreement is" in response.text): 44 | logger.debug("Renewing session...") 45 | self.renew() 46 | return self.request(*args, i=i+1, **kwargs) 47 | return response 48 | 49 | def renew(self): 50 | self.requests += 1 51 | response = self.session.request( 52 | 'GET', 53 | f'{config.MJCS_BASE_URL}/inquiry-index.jsp' 54 | ) 55 | soup = BeautifulSoup(response.text, 'html.parser') 56 | disclaimer_token = soup.find('input',{'name':'disclaimer'}).get('value') 57 | 58 | # captcha_endpoint = 'https://www.google.com/recaptcha/api2/anchor?ar=1&k=6LeZrYYbAAAAAKAZ8DD6m9pYpfd-9-zgw7AHNX02&co=aHR0cHM6Ly9jYXNlc2VhcmNoLmNvdXJ0cy5zdGF0ZS5tZC51czo0NDM.&hl=en&v=UrRmT3mBwY326qQxUfVlHu1P&size=invisible&sa=submit&cb=y2j4jglyhuqt' 59 | # recaptcha_response = reCaptchaV3(captcha_endpoint) 60 | self.requests += 1 61 | response = self.session.request( 62 | 'POST', 63 | f'{config.MJCS_BASE_URL}/processDisclaimer.jis', 64 | data = { 65 | 'disclaimer': disclaimer_token, 66 | # 'txtReCaptchaMinScore': '0.7', 67 | # 'txtReCaptchaScoreSvc': 'https://jportal.mdcourts.gov/casesearch/resources/jisrecaptcha/score', 68 | # 'g-recaptcha-response': recaptcha_response 69 | } 70 | ) 71 | if (response.status_code != 200 or 72 | (response.history and response.history[0].status_code == 302 and 73 | response.history[0].headers['location'] == f'{config.MJCS_BASE_URL}/inquiry-index.jsp') or 74 | "Acceptance of the following agreement is" in response.text): 75 | err = f"Failed to authenticate with MJCS: code = {response.status_code}, body = {response.text}" 76 | logger.error(err) 77 | raise Exception(err) 78 | 79 | return response -------------------------------------------------------------------------------- /db/versions/c3420fc7aeaf_add_mortgage_to_bail_and_bond.py: -------------------------------------------------------------------------------- 1 | """add mortgage to bail_and_bond 2 | 3 | Revision ID: c3420fc7aeaf 4 | Revises: 63f37461c971 5 | Create Date: 2018-05-05 20:34:29.490423 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'c3420fc7aeaf' 14 | down_revision = '63f37461c971' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.drop_constraint('DSCR_case_number_fkey', 'DSCR', type_='foreignkey') 22 | op.create_foreign_key(None, 'DSCR', 'cases', ['case_number'], ['case_number'], ondelete='CASCADE') 23 | op.add_column('bail_and_bond', sa.Column('mortgage', sa.Numeric(), nullable=True)) 24 | op.drop_constraint('case_events_case_number_fkey', 'case_events', type_='foreignkey') 25 | op.create_foreign_key(None, 'case_events', 'cases', ['case_number'], ['case_number'], ondelete='CASCADE') 26 | op.drop_constraint('defendant_alias_case_number_fkey', 'defendant_alias', type_='foreignkey') 27 | op.create_foreign_key(None, 'defendant_alias', 'cases', ['case_number'], ['case_number'], ondelete='CASCADE') 28 | op.drop_constraint('defendants_case_number_fkey', 'defendants', type_='foreignkey') 29 | op.create_foreign_key(None, 'defendants', 'cases', ['case_number'], ['case_number'], ondelete='CASCADE') 30 | op.drop_constraint('charges_case_number_fkey', 'dscr_charges', type_='foreignkey') 31 | op.create_foreign_key(None, 'dscr_charges', 'cases', ['case_number'], ['case_number'], ondelete='CASCADE') 32 | op.drop_constraint('court_schedule_case_number_fkey', 'dscr_court_schedule', type_='foreignkey') 33 | op.create_foreign_key(None, 'dscr_court_schedule', 'cases', ['case_number'], ['case_number'], ondelete='CASCADE') 34 | op.drop_constraint('related_persons_case_number_fkey', 'related_persons', type_='foreignkey') 35 | op.create_foreign_key(None, 'related_persons', 'cases', ['case_number'], ['case_number'], ondelete='CASCADE') 36 | # ### end Alembic commands ### 37 | 38 | 39 | def downgrade(): 40 | # ### commands auto generated by Alembic - please adjust! ### 41 | op.drop_constraint(None, 'related_persons', type_='foreignkey') 42 | op.create_foreign_key('related_persons_case_number_fkey', 'related_persons', 'cases', ['case_number'], ['case_number']) 43 | op.drop_constraint(None, 'dscr_court_schedule', type_='foreignkey') 44 | op.create_foreign_key('court_schedule_case_number_fkey', 'dscr_court_schedule', 'cases', ['case_number'], ['case_number']) 45 | op.drop_constraint(None, 'dscr_charges', type_='foreignkey') 46 | op.create_foreign_key('charges_case_number_fkey', 'dscr_charges', 'cases', ['case_number'], ['case_number']) 47 | op.drop_constraint(None, 'defendants', type_='foreignkey') 48 | op.create_foreign_key('defendants_case_number_fkey', 'defendants', 'cases', ['case_number'], ['case_number']) 49 | op.drop_constraint(None, 'defendant_alias', type_='foreignkey') 50 | op.create_foreign_key('defendant_alias_case_number_fkey', 'defendant_alias', 'cases', ['case_number'], ['case_number']) 51 | op.drop_constraint(None, 'case_events', type_='foreignkey') 52 | op.create_foreign_key('case_events_case_number_fkey', 'case_events', 'cases', ['case_number'], ['case_number']) 53 | op.drop_column('bail_and_bond', 'mortgage') 54 | op.drop_constraint(None, 'DSCR', type_='foreignkey') 55 | op.create_foreign_key('DSCR_case_number_fkey', 'DSCR', 'cases', ['case_number'], ['case_number']) 56 | # ### end Alembic commands ### 57 | -------------------------------------------------------------------------------- /db/versions/bc9d0d8db2e2_split_party_address_table_from_cc_.py: -------------------------------------------------------------------------------- 1 | """split party address table from CC parties 2 | 3 | Revision ID: bc9d0d8db2e2 4 | Revises: a87c38b98bac 5 | Create Date: 2018-05-20 20:21:37.017875 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'bc9d0d8db2e2' 14 | down_revision = 'a87c38b98bac' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('cc_party_addresses', 22 | sa.Column('id', sa.Integer(), nullable=False), 23 | sa.Column('plaintiff_id', sa.Integer(), nullable=True), 24 | sa.Column('defendant_id', sa.Integer(), nullable=True), 25 | sa.Column('related_person_id', sa.Integer(), nullable=True), 26 | sa.Column('address', sa.String(), nullable=True), 27 | sa.Column('city', sa.String(), nullable=True), 28 | sa.Column('state', sa.String(), nullable=True), 29 | sa.Column('zip_code', sa.String(), nullable=True), 30 | sa.Column('case_number', sa.String(), nullable=True), 31 | sa.ForeignKeyConstraint(['case_number'], ['cc.case_number'], ondelete='CASCADE'), 32 | sa.ForeignKeyConstraint(['defendant_id'], ['cc_defendants.id'], ), 33 | sa.ForeignKeyConstraint(['plaintiff_id'], ['cc_plaintiffs.id'], ), 34 | sa.ForeignKeyConstraint(['related_person_id'], ['cc_related_persons.id'], ), 35 | sa.PrimaryKeyConstraint('id') 36 | ) 37 | op.drop_column('cc_defendants', 'address') 38 | op.drop_column('cc_defendants', 'state') 39 | op.drop_column('cc_defendants', 'city') 40 | op.drop_column('cc_defendants', 'zip_code') 41 | op.drop_column('cc_plaintiffs', 'address') 42 | op.drop_column('cc_plaintiffs', 'state') 43 | op.drop_column('cc_plaintiffs', 'city') 44 | op.drop_column('cc_plaintiffs', 'zip_code') 45 | op.drop_column('cc_related_persons', 'address') 46 | op.drop_column('cc_related_persons', 'state') 47 | op.drop_column('cc_related_persons', 'city') 48 | op.drop_column('cc_related_persons', 'zip_code') 49 | # ### end Alembic commands ### 50 | 51 | 52 | def downgrade(): 53 | # ### commands auto generated by Alembic - please adjust! ### 54 | op.add_column('cc_related_persons', sa.Column('zip_code', sa.VARCHAR(), autoincrement=False, nullable=True)) 55 | op.add_column('cc_related_persons', sa.Column('city', sa.VARCHAR(), autoincrement=False, nullable=True)) 56 | op.add_column('cc_related_persons', sa.Column('state', sa.VARCHAR(), autoincrement=False, nullable=True)) 57 | op.add_column('cc_related_persons', sa.Column('address', sa.VARCHAR(), autoincrement=False, nullable=True)) 58 | op.add_column('cc_plaintiffs', sa.Column('zip_code', sa.VARCHAR(), autoincrement=False, nullable=True)) 59 | op.add_column('cc_plaintiffs', sa.Column('city', sa.VARCHAR(), autoincrement=False, nullable=True)) 60 | op.add_column('cc_plaintiffs', sa.Column('state', sa.VARCHAR(), autoincrement=False, nullable=True)) 61 | op.add_column('cc_plaintiffs', sa.Column('address', sa.VARCHAR(), autoincrement=False, nullable=True)) 62 | op.add_column('cc_defendants', sa.Column('zip_code', sa.VARCHAR(), autoincrement=False, nullable=True)) 63 | op.add_column('cc_defendants', sa.Column('city', sa.VARCHAR(), autoincrement=False, nullable=True)) 64 | op.add_column('cc_defendants', sa.Column('state', sa.VARCHAR(), autoincrement=False, nullable=True)) 65 | op.add_column('cc_defendants', sa.Column('address', sa.VARCHAR(), autoincrement=False, nullable=True)) 66 | op.drop_table('cc_party_addresses') 67 | # ### end Alembic commands ### 68 | -------------------------------------------------------------------------------- /db/versions/c2f35a46de03_additional_columns_for_.py: -------------------------------------------------------------------------------- 1 | """Additional columns for ODYCIVILJudgmentComment 2 | 3 | Revision ID: c2f35a46de03 4 | Revises: 03d0db20af8d 5 | Create Date: 2021-03-01 09:41:26.339935 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'c2f35a46de03' 14 | down_revision = '03d0db20af8d' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('odycivil_judgment_comments', sa.Column('amendcorrect', sa.String(), nullable=True)) 22 | op.add_column('odycivil_judgment_comments', sa.Column('cc_recorded_date', sa.Date(), nullable=True)) 23 | op.add_column('odycivil_judgment_comments', sa.Column('cc_recorded_date_str', sa.String(), nullable=True)) 24 | op.add_column('odycivil_judgment_comments', sa.Column('defreason', sa.String(), nullable=True)) 25 | op.add_column('odycivil_judgment_comments', sa.Column('freetext', sa.String(), nullable=True)) 26 | op.add_column('odycivil_judgment_comments', sa.Column('post_interest_rate', sa.String(), nullable=True)) 27 | op.add_column('odycivil_judgment_comments', sa.Column('printdate', sa.Date(), nullable=True)) 28 | op.add_column('odycivil_judgment_comments', sa.Column('printdate_str', sa.String(), nullable=True)) 29 | op.add_column('odycivil_judgment_comments', sa.Column('satisfaction_date', sa.Date(), nullable=True)) 30 | op.add_column('odycivil_judgment_comments', sa.Column('satisfaction_date_str', sa.String(), nullable=True)) 31 | op.add_column('odycivil_judgment_comments', sa.Column('sat_date_entered', sa.Date(), nullable=True)) 32 | op.add_column('odycivil_judgment_comments', sa.Column('sat_date_entered_str', sa.String(), nullable=True)) 33 | op.add_column('odycivil_judgment_comments', sa.Column('totdecimalprin', sa.Numeric(), nullable=True)) 34 | op.add_column('odycivil_judgment_comments', sa.Column('totalinterest', sa.Numeric(), nullable=True)) 35 | op.add_column('odycivil_judgment_comments', sa.Column('totaljudg', sa.Numeric(), nullable=True)) 36 | op.add_column('odycivil_judgment_comments', sa.Column('totalprincipal', sa.Numeric(), nullable=True)) 37 | op.alter_column('odycivil_judgment_comments', 'damanges_pop', new_column_name='damages_pop') 38 | # ### end Alembic commands ### 39 | 40 | 41 | def downgrade(): 42 | # ### commands auto generated by Alembic - please adjust! ### 43 | op.alter_column('odycivil_judgment_comments', 'damages_pop', new_column_name='damanges_pop') 44 | op.drop_column('odycivil_judgment_comments', 'totalprincipal') 45 | op.drop_column('odycivil_judgment_comments', 'totaljudg') 46 | op.drop_column('odycivil_judgment_comments', 'totalinterest') 47 | op.drop_column('odycivil_judgment_comments', 'totdecimalprin') 48 | op.drop_column('odycivil_judgment_comments', 'sat_date_entered_str') 49 | op.drop_column('odycivil_judgment_comments', 'sat_date_entered') 50 | op.drop_column('odycivil_judgment_comments', 'satisfaction_date_str') 51 | op.drop_column('odycivil_judgment_comments', 'satisfaction_date') 52 | op.drop_column('odycivil_judgment_comments', 'printdate_str') 53 | op.drop_column('odycivil_judgment_comments', 'printdate') 54 | op.drop_column('odycivil_judgment_comments', 'post_interest_rate') 55 | op.drop_column('odycivil_judgment_comments', 'freetext') 56 | op.drop_column('odycivil_judgment_comments', 'defreason') 57 | op.drop_column('odycivil_judgment_comments', 'cc_recorded_date_str') 58 | op.drop_column('odycivil_judgment_comments', 'cc_recorded_date') 59 | op.drop_column('odycivil_judgment_comments', 'amendcorrect') 60 | # ### end Alembic commands ### 61 | -------------------------------------------------------------------------------- /db/versions/7427734c0fc2_remove_spider_related_tables.py: -------------------------------------------------------------------------------- 1 | """Remove spider-related tables 2 | 3 | Revision ID: 7427734c0fc2 4 | Revises: ec14982d61de 5 | Create Date: 2020-04-12 17:08:03.178304 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | import sys 12 | 13 | # revision identifiers, used by Alembic. 14 | revision = '7427734c0fc2' 15 | down_revision = 'ec14982d61de' 16 | branch_labels = None 17 | depends_on = None 18 | 19 | 20 | def upgrade(): 21 | answer = input('THIS WILL DELETE A LOT OF DATA, ARE YOU SURE? (y/N) ') 22 | if answer != 'y' and answer != 'Y': 23 | print("Goodbye!") 24 | sys.exit(0) 25 | # ### commands auto generated by Alembic - please adjust! ### 26 | op.drop_table('runs') 27 | op.drop_table('query_results') 28 | op.drop_table('queue') 29 | # ### end Alembic commands ### 30 | 31 | 32 | def downgrade(): 33 | # ### commands auto generated by Alembic - please adjust! ### 34 | op.create_table('queue', 35 | sa.Column('id', sa.VARCHAR(), autoincrement=False, nullable=False), 36 | sa.Column('search_string', sa.VARCHAR(), autoincrement=False, nullable=True), 37 | sa.Column('start_date', sa.DATE(), autoincrement=False, nullable=True), 38 | sa.Column('end_date', sa.DATE(), autoincrement=False, nullable=True), 39 | sa.Column('court', sa.VARCHAR(), autoincrement=False, nullable=True), 40 | sa.Column('status', sa.VARCHAR(), autoincrement=False, nullable=True), 41 | sa.Column('timeouts', sa.INTEGER(), autoincrement=False, nullable=True), 42 | sa.Column('err500s', sa.INTEGER(), autoincrement=False, nullable=True), 43 | sa.Column('errunknown', sa.VARCHAR(), autoincrement=False, nullable=True), 44 | sa.Column('site', sa.VARCHAR(), autoincrement=False, nullable=True), 45 | sa.PrimaryKeyConstraint('id', name='queue_pkey'), 46 | postgresql_ignore_search_path=False 47 | ) 48 | op.create_table('query_results', 49 | sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), 50 | sa.Column('search_id', sa.VARCHAR(), autoincrement=False, nullable=True), 51 | sa.Column('nresults', sa.INTEGER(), autoincrement=False, nullable=True), 52 | sa.Column('timestamp', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), 53 | sa.Column('query_seconds', sa.INTEGER(), autoincrement=False, nullable=True), 54 | sa.ForeignKeyConstraint(['search_id'], ['queue.id'], name='query_results_search_id_fkey'), 55 | sa.PrimaryKeyConstraint('id', name='query_results_pkey'), 56 | sa.UniqueConstraint('search_id', 'timestamp', name='_search_id_timestamp_uc') 57 | ) 58 | op.create_table('runs', 59 | sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), 60 | sa.Column('query_start_date', sa.DATE(), autoincrement=False, nullable=True), 61 | sa.Column('query_end_date', sa.DATE(), autoincrement=False, nullable=True), 62 | sa.Column('court', sa.VARCHAR(), autoincrement=False, nullable=True), 63 | sa.Column('run_start', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), 64 | sa.Column('run_seconds', sa.INTEGER(), autoincrement=False, nullable=True), 65 | sa.Column('queue_still_active', sa.INTEGER(), autoincrement=False, nullable=True), 66 | sa.Column('queue_finished', sa.INTEGER(), autoincrement=False, nullable=True), 67 | sa.Column('cases_added', sa.INTEGER(), autoincrement=False, nullable=True), 68 | sa.Column('results_processed', sa.INTEGER(), autoincrement=False, nullable=True), 69 | sa.Column('retry_failed', sa.BOOLEAN(), autoincrement=False, nullable=True), 70 | sa.Column('site', sa.VARCHAR(), autoincrement=False, nullable=True), 71 | sa.PrimaryKeyConstraint('id', name='runs_pkey') 72 | ) 73 | # ### end Alembic commands ### 74 | -------------------------------------------------------------------------------- /db/versions/aaa83181c4a5_added_jail_fields_to_odycvcitcharge.py: -------------------------------------------------------------------------------- 1 | """Added jail fields to ODYCVCITCharge 2 | 3 | Revision ID: aaa83181c4a5 4 | Revises: ba65d4f0e491 5 | Create Date: 2021-07-13 17:07:46.459076 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'aaa83181c4a5' 14 | down_revision = 'ba65d4f0e491' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('odycvcit_charges', sa.Column('converted_disposition', sa.String(), nullable=True)) 22 | op.add_column('odycvcit_charges', sa.Column('jail_life', sa.Boolean(), nullable=True)) 23 | op.add_column('odycvcit_charges', sa.Column('jail_death', sa.Boolean(), nullable=True)) 24 | op.add_column('odycvcit_charges', sa.Column('jail_start_date', sa.Date(), nullable=True)) 25 | op.add_column('odycvcit_charges', sa.Column('jail_start_date_str', sa.String(), nullable=True)) 26 | op.add_column('odycvcit_charges', sa.Column('jail_cons_conc', sa.String(), nullable=True)) 27 | op.add_column('odycvcit_charges', sa.Column('jail_years', sa.Integer(), nullable=True)) 28 | op.add_column('odycvcit_charges', sa.Column('jail_months', sa.Integer(), nullable=True)) 29 | op.add_column('odycvcit_charges', sa.Column('jail_days', sa.Integer(), nullable=True)) 30 | op.add_column('odycvcit_charges', sa.Column('jail_hours', sa.Integer(), nullable=True)) 31 | op.add_column('odycvcit_charges', sa.Column('jail_suspended_term', sa.String(), nullable=True)) 32 | op.add_column('odycvcit_charges', sa.Column('jail_suspended_years', sa.Integer(), nullable=True)) 33 | op.add_column('odycvcit_charges', sa.Column('jail_suspended_months', sa.Integer(), nullable=True)) 34 | op.add_column('odycvcit_charges', sa.Column('jail_suspended_days', sa.Integer(), nullable=True)) 35 | op.add_column('odycvcit_charges', sa.Column('jail_suspended_hours', sa.Integer(), nullable=True)) 36 | op.add_column('odycvcit_charges', sa.Column('jail_suspend_all_but_years', sa.Integer(), nullable=True)) 37 | op.add_column('odycvcit_charges', sa.Column('jail_suspend_all_but_months', sa.Integer(), nullable=True)) 38 | op.add_column('odycvcit_charges', sa.Column('jail_suspend_all_but_days', sa.Integer(), nullable=True)) 39 | op.add_column('odycvcit_charges', sa.Column('jail_suspend_all_but_hours', sa.Integer(), nullable=True)) 40 | # ### end Alembic commands ### 41 | 42 | 43 | def downgrade(): 44 | # ### commands auto generated by Alembic - please adjust! ### 45 | op.drop_column('odycvcit_charges', 'jail_suspend_all_but_hours') 46 | op.drop_column('odycvcit_charges', 'jail_suspend_all_but_days') 47 | op.drop_column('odycvcit_charges', 'jail_suspend_all_but_months') 48 | op.drop_column('odycvcit_charges', 'jail_suspend_all_but_years') 49 | op.drop_column('odycvcit_charges', 'jail_suspended_hours') 50 | op.drop_column('odycvcit_charges', 'jail_suspended_days') 51 | op.drop_column('odycvcit_charges', 'jail_suspended_months') 52 | op.drop_column('odycvcit_charges', 'jail_suspended_years') 53 | op.drop_column('odycvcit_charges', 'jail_suspended_term') 54 | op.drop_column('odycvcit_charges', 'jail_hours') 55 | op.drop_column('odycvcit_charges', 'jail_days') 56 | op.drop_column('odycvcit_charges', 'jail_months') 57 | op.drop_column('odycvcit_charges', 'jail_years') 58 | op.drop_column('odycvcit_charges', 'jail_cons_conc') 59 | op.drop_column('odycvcit_charges', 'jail_start_date_str') 60 | op.drop_column('odycvcit_charges', 'jail_start_date') 61 | op.drop_column('odycvcit_charges', 'jail_death') 62 | op.drop_column('odycvcit_charges', 'jail_life') 63 | op.drop_column('odycvcit_charges', 'converted_disposition') 64 | # ### end Alembic commands ### 65 | -------------------------------------------------------------------------------- /db/versions/f8b3c9cf0c10_remove_unused_fields_in_odycvcit.py: -------------------------------------------------------------------------------- 1 | """Remove unused fields in ODYCVCIT 2 | 3 | Revision ID: f8b3c9cf0c10 4 | Revises: 4c53c17fe94c 5 | Create Date: 2021-10-11 15:59:01.005157 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'f8b3c9cf0c10' 14 | down_revision = '4c53c17fe94c' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.drop_index('ixh_odycvcit_sex_offender_registrations_case_number', table_name='odycvcit_sex_offender_registrations') 22 | op.drop_table('odycvcit_sex_offender_registrations') 23 | op.drop_column('odycvcit_charges', 'jail_death') 24 | op.drop_column('odycvcit_charges', 'jail_suspend_all_but_months') 25 | op.drop_column('odycvcit_charges', 'jail_cons_conc') 26 | op.drop_column('odycvcit_charges', 'jail_life') 27 | op.drop_column('odycvcit_charges', 'jail_suspend_all_but_years') 28 | op.drop_column('odycvcit_charges', 'jail_suspended_term') 29 | op.drop_column('odycvcit_charges', 'jail_suspend_all_but_hours') 30 | op.drop_column('odycvcit_charges', 'jail_suspend_all_but_days') 31 | op.drop_column('odycvcit_documents', 'filed_by') 32 | op.drop_column('odycvcit_involved_parties', 'removal_date') 33 | op.drop_column('odycvcit_involved_parties', 'removal_date_str') 34 | op.drop_column('odycvcit_involved_parties', 'appearance_date_str') 35 | op.drop_column('odycvcit_involved_parties', 'appearance_date') 36 | op.drop_column('odycvcit_services', 'service_status') 37 | # ### end Alembic commands ### 38 | 39 | 40 | def downgrade(): 41 | # ### commands auto generated by Alembic - please adjust! ### 42 | op.add_column('odycvcit_services', sa.Column('service_status', sa.VARCHAR(), autoincrement=False, nullable=True)) 43 | op.add_column('odycvcit_involved_parties', sa.Column('appearance_date', sa.DATE(), autoincrement=False, nullable=True)) 44 | op.add_column('odycvcit_involved_parties', sa.Column('appearance_date_str', sa.VARCHAR(), autoincrement=False, nullable=True)) 45 | op.add_column('odycvcit_involved_parties', sa.Column('removal_date_str', sa.VARCHAR(), autoincrement=False, nullable=True)) 46 | op.add_column('odycvcit_involved_parties', sa.Column('removal_date', sa.DATE(), autoincrement=False, nullable=True)) 47 | op.add_column('odycvcit_documents', sa.Column('filed_by', sa.VARCHAR(), autoincrement=False, nullable=True)) 48 | op.add_column('odycvcit_charges', sa.Column('jail_suspend_all_but_days', sa.INTEGER(), autoincrement=False, nullable=True)) 49 | op.add_column('odycvcit_charges', sa.Column('jail_suspend_all_but_hours', sa.INTEGER(), autoincrement=False, nullable=True)) 50 | op.add_column('odycvcit_charges', sa.Column('jail_suspended_term', sa.VARCHAR(), autoincrement=False, nullable=True)) 51 | op.add_column('odycvcit_charges', sa.Column('jail_suspend_all_but_years', sa.INTEGER(), autoincrement=False, nullable=True)) 52 | op.add_column('odycvcit_charges', sa.Column('jail_life', sa.BOOLEAN(), autoincrement=False, nullable=True)) 53 | op.add_column('odycvcit_charges', sa.Column('jail_cons_conc', sa.VARCHAR(), autoincrement=False, nullable=True)) 54 | op.add_column('odycvcit_charges', sa.Column('jail_suspend_all_but_months', sa.INTEGER(), autoincrement=False, nullable=True)) 55 | op.add_column('odycvcit_charges', sa.Column('jail_death', sa.BOOLEAN(), autoincrement=False, nullable=True)) 56 | op.create_table('odycvcit_sex_offender_registrations', 57 | sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), 58 | sa.Column('type', sa.VARCHAR(), autoincrement=False, nullable=True), 59 | sa.Column('notes', sa.VARCHAR(), autoincrement=False, nullable=True), 60 | sa.Column('case_number', sa.VARCHAR(), autoincrement=False, nullable=False), 61 | sa.ForeignKeyConstraint(['case_number'], ['odycvcit.case_number'], name='odycvcit_sex_offender_registrations_case_number_fkey', ondelete='CASCADE'), 62 | sa.PrimaryKeyConstraint('id', name='odycvcit_sex_offender_registrations_pkey') 63 | ) 64 | op.create_index('ixh_odycvcit_sex_offender_registrations_case_number', 'odycvcit_sex_offender_registrations', ['case_number'], unique=False) 65 | # ### end Alembic commands ### 66 | -------------------------------------------------------------------------------- /db/versions/e86f479e25e4_add_scrapes_and_scrape_versions_tables.py: -------------------------------------------------------------------------------- 1 | """add scrapes and scrape_versions tables 2 | 3 | Revision ID: e86f479e25e4 4 | Revises: c6965a9b34fe 5 | Create Date: 2018-06-19 15:52:19.920577 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'e86f479e25e4' 14 | down_revision = 'c6965a9b34fe' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | # from sqlalchemy.orm import sessionmaker 19 | # from mjcs.models import Scrape, ScrapeVersion, Case 20 | # from mjcs.config import config 21 | # from datetime import * 22 | # from hashlib import sha256 23 | # 24 | # def upgrade(): 25 | # print('Running Alembic commands') 26 | # # ### commands auto generated by Alembic - please adjust! ### 27 | # op.create_table('scrape_versions', 28 | # sa.Column('s3_version_id', sa.String(), nullable=False), 29 | # sa.Column('case_number', sa.String(), nullable=True), 30 | # sa.Column('length', sa.Integer(), nullable=True), 31 | # sa.Column('sha256', sa.String(), nullable=True), 32 | # sa.ForeignKeyConstraint(['case_number'], ['cases.case_number'], ondelete='CASCADE'), 33 | # sa.PrimaryKeyConstraint('s3_version_id') 34 | # ) 35 | # op.create_index(op.f('ix_scrape_versions_case_number'), 'scrape_versions', ['case_number'], unique=False) 36 | # op.create_table('scrapes', 37 | # sa.Column('id', sa.Integer(), nullable=False), 38 | # sa.Column('case_number', sa.String(), nullable=True), 39 | # sa.Column('s3_version_id', sa.String(), nullable=True), 40 | # sa.Column('timestamp', sa.DateTime(), nullable=True), 41 | # sa.Column('duration', sa.Numeric(), nullable=True), 42 | # sa.ForeignKeyConstraint(['case_number'], ['cases.case_number'], ondelete='CASCADE'), 43 | # sa.ForeignKeyConstraint(['s3_version_id'], ['scrape_versions.s3_version_id'], ondelete='CASCADE'), 44 | # sa.PrimaryKeyConstraint('id') 45 | # ) 46 | # op.create_index('ix_scrape_timestamp', 'scrapes', ['case_number', sa.text('timestamp DESC')], unique=False) 47 | # op.create_index(op.f('ix_scrapes_case_number'), 'scrapes', ['case_number'], unique=False) 48 | # # ### end Alembic commands ### 49 | # if 0: # no need to do this for dev 50 | # print('Running migration') 51 | # connection = op.get_bind() 52 | # db_factory = sessionmaker(bind = connection) 53 | # db = db_factory() 54 | # try: 55 | # total_cases = db.query(Case).filter(Case.last_scrape.isnot(None)).count() 56 | # i = 0 57 | # for version in config.case_details_bucket.object_versions.all(): 58 | # i += 1 59 | # print('Adding version %s for case_number %s (%d of %d)' % (version.id, version.object_key, i, total_cases)) 60 | # try: 61 | # obj = version.get() 62 | # except: 63 | # print('ERROR') 64 | # continue 65 | # html = obj['Body'].read() 66 | # timestamp = datetime.strptime(obj['Metadata']['timestamp'], "%Y-%m-%dT%H:%M:%S.%f") 67 | # scrape_version = ScrapeVersion( 68 | # s3_version_id = version.id, 69 | # case_number = version.object_key, 70 | # length = len(html), 71 | # sha256 = sha256(html).hexdigest() 72 | # ) 73 | # scrape = Scrape( 74 | # case_number = version.object_key, 75 | # s3_version_id = version.id, 76 | # timestamp = timestamp, 77 | # duration = None 78 | # ) 79 | # db.add(scrape_version) 80 | # db.add(scrape) 81 | # db.commit() 82 | # except: 83 | # db.rollback() 84 | # raise 85 | # finally: 86 | # db.close() 87 | # print('Finished!') 88 | # 89 | # def downgrade(): 90 | # # ### commands auto generated by Alembic - please adjust! ### 91 | # op.drop_index(op.f('ix_scrapes_case_number'), table_name='scrapes') 92 | # op.drop_index('ix_scrape_timestamp', table_name='scrapes') 93 | # op.drop_table('scrapes') 94 | # op.drop_index(op.f('ix_scrape_versions_case_number'), table_name='scrape_versions') 95 | # op.drop_table('scrape_versions') 96 | # # ### end Alembic commands ### 97 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Case Harvester 2 | 3 | 4 | # Case Harvester 5 | Case Harvester is a project designed to scrape the [Maryland Judiciary Case Search](https://casesearch.courts.state.md.us/casesearch/inquiry-index.jsp) (MJCS) and build a near-complete database of Maryland court cases that can be queried and analyzed without the limitations of the MJCS interface. It is designed to leverage [Amazon Web Services (AWS)](https://aws.amazon.com/) for scalability and performance. 6 | 7 | Our database of cases (with criminal defendant PII redacted) is available to the public and can be found at [mdcaseexplorer.com](https://mdcaseexplorer.com), which is built using our [Case Explorer](https://github.com/dismantl/CaseExplorer) software. REST and GraphQL APIs are available. If you would like to download tables from our database exported monthly, you can find that at [exports.mdcaseexplorer.com](https://exports.mdcaseexplorer.com/). 8 | 9 | > **NOTE: Unless you are modifying Case Harvester for specific purposes, please do not run your own instance so that MJCS is spared unneccesary load. Instead, use the options described above for viewing the data, or if you have an AWS account you are also welcome to clone our database directly.** 10 | 11 | # Architecture 12 | Case Harvester is split into three main components: spider, scraper, and parser. Each component is a part of a pipeline that finds, downloads, and parses case data from the MJCS. The following diagram shows at a high level how each of these components interact: 13 | 14 | ![High level diagram](./img/main.svg) 15 | 16 | ### Spider 17 | The spider component is responsible for discovering new case numbers. It does this by submitting search queries to the MJCS and iterating through the results. Because the MJCS only returns a maximum of 500 results, the search algorithm splits queries that return 500 results into a set of more narrowed queries which are then submitted. Each of these queries is then split again if more than 500 results are returned, and so forth, until the MJCS is exhaustively searched for case numbers. 18 | 19 | ### Scraper 20 | The scraper component downloads and stores the case details for every case number discovered by the spider. The full HTML for each case is added to an S3 bucket. Version information is kept for each case, including a timestamp of when each version was downloaded, so changes to a case can be recorded and referenced. 21 | 22 | ### Parser 23 | The parser component is a Lambda function that parses the fields of information in the HTML case details for each case, and stores that data in the PostgreSQL database. Each new item added to the scraper S3 bucket triggers a new parser Lambda invocation, which allows for significant scaling. 24 | 25 | Case details in the MJCS are formatted differently depending on the county and type of case (e.g. district vs circuit court, criminal vs civil, etc.), and whether it is in one of the new [MDEC](https://mdcourts.gov/mdec/about)-compatible formats. MJCS [assigns a code to each of these different case types](https://www.muckrock.com/foi/maryland-154/case-search-court-classifications-56516/#comm-564971): 26 | * ODYCRIM: MDEC Criminal Cases 27 | * ODYTRAF: MDEC Traffic Cases 28 | * ODYCIVIL: MDEC Civil Cases 29 | * ODYCVCIT: MDEC Civil Citations 30 | * ODYCOSA: MDEC Appellate Court of Maryland (formerly Court of Special Appeals) 31 | * ODYCOA: MDEC Supreme Court of Maryland (formerly Court of Appeals) 32 | * DSCR: District Court Criminal Cases 33 | * DSCIVIL: District Court Civil Cases 34 | * DSCP: District Court Civil Citations 35 | * DSTRAF: District Court Traffic Cases 36 | * K: Circuit Court Criminal Cases 37 | * CC: Circuit Court Civil Cases 38 | * DV: Domestic Violence Cases 39 | * DSK8: Baltimore City Criminal Cases 40 | * PG: Prince George's County Circuit Court Criminal Cases 41 | * PGV: Prince George's County Circuit Court Civil Cases 42 | * MCCI: Montgomery County Civil Cases 43 | * MCCR: Montgomery County Criminal Cases 44 | 45 | Each different parser breaks down the case details to a granular level and stores the data in a number of database tables. This [schematic diagram](https://disman.tl/caseharvester/relationships.html) illustrates how this data is represented in the database. 46 | 47 | # Questions 48 | For questions or more information, email [dan@acab.enterprises](mailto:dan@acab.enterprises). 49 | -------------------------------------------------------------------------------- /mjcs/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .common import ColumnMetadata 2 | from .case import Case 3 | from .scraper import Scrape, ScrapeVersion 4 | from .DSCR import (DSCR, DSCRCharge, DSCRDefendant, DSCRDefendantAlias, 5 | DSCRRelatedPerson, DSCREvent, DSCRTrial, DSCRBailEvent) 6 | from .DSCP import (DSCP, DSCPCharge, DSCPDefendant, DSCPDefendantAlias, 7 | DSCPRelatedPerson, DSCPEvent, DSCPTrial, DSCPBailEvent) 8 | from .DSK8 import (DSK8, DSK8Charge, DSK8BailAndBond, DSK8Bondsman, DSK8Defendant, 9 | DSK8DefendantAlias, DSK8RelatedPerson, DSK8Event, DSK8Trial) 10 | from .DSCIVIL import (DSCIVIL, DSCIVILComplaint, DSCIVILHearing, DSCIVILJudgment, 11 | DSCIVILRelatedPerson, DSCIVILEvent, DSCIVILTrial) 12 | from .CC import (CC, CCDistrictCaseNumber, CCPlaintiff, CCDefendant, 13 | CCRelatedPerson, CCPartyAlias, CCPartyAddress, CCAttorney, 14 | CCCourtSchedule, CCJudgment, CCJudgmentModification, 15 | CCJudgmentAgainst, CCJudgmentInFavor, CCSupportOrder, CCDocument) 16 | from .ODYTRAF import (ODYTRAF, ODYTRAFReferenceNumber, ODYTRAFDefendant, 17 | ODYTRAFInvolvedParty, ODYTRAFAttorney, ODYTRAFCourtSchedule, 18 | ODYTRAFCharge, ODYTRAFWarrant, ODYTRAFBailBond, 19 | ODYTRAFBondSetting, ODYTRAFDocument, ODYTRAFAlias, 20 | ODYTRAFService) 21 | from .ODYCRIM import (ODYCRIM, ODYCRIMReferenceNumber, ODYCRIMDefendant, 22 | ODYCRIMInvolvedParty, ODYCRIMAlias, ODYCRIMAttorney, 23 | ODYCRIMCourtSchedule, ODYCRIMCharge, ODYCRIMProbation, 24 | ODYCRIMRestitution, ODYCRIMWarrant, ODYCRIMBailBond, 25 | ODYCRIMBondSetting, ODYCRIMDocument, ODYCRIMService, 26 | ODYCRIMSexOffenderRegistration) 27 | from .ODYCIVIL import (ODYCIVIL, ODYCIVILReferenceNumber, ODYCIVILCause, 28 | ODYCIVILCauseRemedy, ODYCIVILDefendant, ODYCIVILInvolvedParty, 29 | ODYCIVILAlias, ODYCIVILAttorney, ODYCIVILJudgment, 30 | ODYCIVILJudgmentStatus, ODYCIVILCourtSchedule, ODYCIVILWarrant, 31 | ODYCIVILDocument, ODYCIVILService, ODYCIVILJudgmentComment, 32 | ODYCIVILBondSetting, ODYCIVILBailBond, ODYCIVILDisposition) 33 | from .ODYCVCIT import (ODYCVCIT, ODYCVCITReferenceNumber, ODYCVCITDefendant, 34 | ODYCVCITInvolvedParty, ODYCVCITAlias, ODYCVCITAttorney, 35 | ODYCVCITCourtSchedule, ODYCVCITCharge, ODYCVCITProbation, 36 | ODYCVCITRestitution, ODYCVCITWarrant, ODYCVCITBailBond, 37 | ODYCVCITBondSetting, ODYCVCITDocument, ODYCVCITService) 38 | from .DSTRAF import (DSTRAF, DSTRAFCharge, DSTRAFDisposition, DSTRAFDefendant, 39 | DSTRAFEvent, DSTRAFTrial, DSTRAFRelatedPerson) 40 | from .K import (K, KDefendant, KCharge, 41 | KRelatedPerson, KPartyAlias, KPartyAddress, KAttorney, 42 | KCourtSchedule, KJudgment, KJudgmentModification, 43 | KJudgmentAgainst, KJudgmentInFavor, KSupportOrder, KDocument, 44 | KSentencingNetTools) 45 | from .PG import (PG, PGCharge, PGDefendant, PGDefendantAlias, PGOtherParty, 46 | PGAttorney, PGCourtSchedule, PGDocket, PGPlaintiff) 47 | from .DV import DV, DVDefendant, DVHearing, DVEvent, DVDefendantAttorney 48 | from .MCCR import (MCCR, MCCRAttorney, MCCRCharge, MCCRCourtSchedule, 49 | MCCRDefendant, MCCRDocket, MCCRBailBond, MCCRAudioMedia, 50 | MCCRJudgment, MCCRProbationOfficer, MCCRAlias, MCCRBondRemitter, 51 | MCCRDistrictCourtNumber, MCCRTrackingNumber, MCCRDWIMonitor) 52 | from .MCCI import (MCCI, MCCIAttorney, MCCICourtSchedule, MCCIDefendant, MCCIDocket, 53 | MCCIInterestedParty, MCCIIssue, MCCIJudgment, MCCIPlaintiff, 54 | MCCIAlias, MCCIWard, MCCIAudioMedia, MCCIGarnishee, MCCIResidentAgent) 55 | from .PGV import (PGV, PGVDefendant, PGVPlaintiff, PGVOtherParty, PGVAttorney, 56 | PGVJudgment, PGVDocket, PGVCourtSchedule, PGVDefendantAlias) 57 | from .ODYCOSA import (ODYCOSA, ODYCOSAAttorney, ODYCOSADocument, ODYCOSAJudgment, 58 | ODYCOSAInvolvedParty, ODYCOSAReferenceNumber, ODYCOSACourtSchedule) 59 | from .ODYCOA import (ODYCOA, ODYCOAAttorney, ODYCOADocument, ODYCOAJudgment, 60 | ODYCOAInvolvedParty, ODYCOAReferenceNumber, ODYCOACourtSchedule) --------------------------------------------------------------------------------