├── .editorconfig ├── .env.template ├── .gitattributes ├── .github ├── dependabot.yml └── workflows │ ├── backend.yml │ ├── docker.yml │ ├── frontend.yml │ └── scheduled.yml ├── .gitignore ├── LICENSE ├── README.md ├── backend ├── .coveragerc ├── .dockerignore ├── .editorconfig ├── .gitignore ├── Dockerfile ├── Makefile ├── howtheyvote │ ├── __init__.py │ ├── alembic │ │ ├── alembic.ini │ │ ├── env.py │ │ ├── script.py.mako │ │ └── versions │ │ │ ├── 064daf473f9a_add_procedure_stage_column_to_votes_.py │ │ │ ├── 1deea9ed7c52_add_oeil_subjects_column_to_votes_table.py │ │ │ ├── 1f516b18c4f6_rename_result_column_to_status.py │ │ │ ├── 2f958a6f147d_add_checksum_column_to_pipeline_runs_.py │ │ │ ├── 2fb74cc3d20c_add_text_and_position_counts_column_to_press_releases_table.py │ │ │ ├── 3895af031aea_add_responsible_committee_column_to_.py │ │ │ ├── 3eec8050f4d5_add_eurovoc_concepts_column.py │ │ │ ├── 47cf729ded96_add_dlv_title_and_result_columns_to_.py │ │ │ ├── 7c9e86276c6d_remove_is_featured_column_from_votes_.py │ │ │ ├── 848ef24718dd_add_press_release_column_to_votes_table.py │ │ │ ├── 8d1995cb0bed_make_responsible_committee_column_json_.py │ │ │ ├── 9200b9028b93_create_timestamp_index_on_votes_table.py │ │ │ ├── 9a4972f3a768_init.py │ │ │ └── 9b35d19b64c4_add_term_column_to_votes_table.py │ ├── analysis │ │ ├── __init__.py │ │ ├── helpers.py │ │ ├── press_releases.py │ │ └── votes.py │ ├── api │ │ ├── __init__.py │ │ ├── openapi_helpers.py │ │ ├── openapi_spec.py │ │ ├── query.py │ │ ├── serializers.py │ │ ├── sessions_api.py │ │ ├── static_api.py │ │ ├── stats_api.py │ │ ├── util.py │ │ └── votes_api.py │ ├── cli │ │ ├── __init__.py │ │ ├── aggregate.py │ │ ├── dev.py │ │ ├── pipeline.py │ │ ├── system.py │ │ └── temp.py │ ├── config.py │ ├── data.py │ ├── data │ │ ├── committees.json │ │ ├── countries.json │ │ ├── eurovoc.json │ │ ├── groups.json │ │ └── oeil_subjects.json │ ├── db.py │ ├── export │ │ ├── __init__.py │ │ └── csvw_helpers.py │ ├── files.py │ ├── helpers.py │ ├── json.py │ ├── metrics.py │ ├── models │ │ ├── __init__.py │ │ ├── committee.py │ │ ├── common.py │ │ ├── country.py │ │ ├── eurovoc.py │ │ ├── group.py │ │ ├── member.py │ │ ├── oeil.py │ │ ├── press_release.py │ │ ├── session.py │ │ ├── types.py │ │ └── vote.py │ ├── pipelines │ │ ├── __init__.py │ │ ├── common.py │ │ ├── members.py │ │ ├── press.py │ │ ├── rcv_list.py │ │ ├── sessions.py │ │ └── vot_list.py │ ├── pushover.py │ ├── query.py │ ├── scrapers │ │ ├── __init__.py │ │ ├── common.py │ │ ├── helpers.py │ │ ├── members.py │ │ ├── press_releases.py │ │ ├── sessions.py │ │ └── votes.py │ ├── search.py │ ├── sharepics │ │ ├── __init__.py │ │ └── cdp.py │ ├── store │ │ ├── __init__.py │ │ ├── aggregator.py │ │ ├── index.py │ │ ├── mappings.py │ │ └── writer.py │ ├── vote_stats.py │ ├── worker │ │ ├── __init__.py │ │ └── worker.py │ └── wsgi.py ├── poetry.lock ├── poetry.toml ├── pyproject.toml ├── stubs │ └── xapian │ │ └── __init__.pyi └── tests │ ├── __init__.py │ ├── analysis │ ├── test_helpers.py │ ├── test_press_releases.py │ └── test_votes.py │ ├── api │ ├── test_openapi_helpers.py │ ├── test_query.py │ ├── test_sessions_api.py │ └── test_votes_api.py │ ├── conftest.py │ ├── export │ ├── test_csvw_helpers.py │ └── test_init.py │ ├── helpers.py │ ├── models │ ├── test_country.py │ ├── test_eurovoc.py │ ├── test_group.py │ ├── test_member.py │ ├── test_types.py │ └── test_vote.py │ ├── pipelines │ ├── data │ │ ├── rcv-list_pv-9-2024-04-24-rcv-fr-evening.xml │ │ └── rcv-list_pv-9-2024-04-24-rcv-fr-noon.xml │ └── test_rcv_list.py │ ├── scrapers │ ├── __init__.py │ ├── data │ │ ├── members │ │ │ ├── member_groups_adinolfi_term_8.html │ │ │ ├── member_groups_sonneborn_term_8.html │ │ │ ├── member_groups_sonneborn_term_9.html │ │ │ ├── member_groups_weber_term_10.html │ │ │ ├── member_info_aaltola_home.html │ │ │ ├── member_info_adinolfi_home.html │ │ │ ├── member_info_weber_home.html │ │ │ └── members_directory_term_9.xml │ │ ├── press_releases │ │ │ ├── press-release_20190712IPR56948.html │ │ │ ├── press-release_20221014IPR43206.html │ │ │ ├── press-release_20241212IPR25960.html │ │ │ └── press-release_20250204IPR26689.html │ │ ├── sessions │ │ │ └── odp_mtg-pl-2024-07-16.xml │ │ └── votes │ │ │ ├── eurlex-document_p9-a-2021-0270.html │ │ │ ├── eurlex-document_p9-a-2023-0369.html │ │ │ ├── eurlex-procedure_2021-106.html │ │ │ ├── eurlex-procedure_2023-102.html │ │ │ ├── oeil-procedure-file_2022-2201-ini.html │ │ │ ├── oeil-procedure-file_2022-2852-rsp.html │ │ │ ├── oeil-procedure-file_2023-2019-ini.html │ │ │ ├── oeil-procedure-file_2024-0258-cod.html │ │ │ ├── rcv_list_incorrect_pv-9-2020-07-23-rcv-fr.xml │ │ │ ├── rcv_list_p9-pv(2020)07-23(rcv)_xc.xml │ │ │ ├── rcv_list_pv-9-2019-07-15-rcv-fr.xml │ │ │ ├── rcv_list_pv-9-2020-07-23-rcv-fr.xml │ │ │ ├── rcv_list_pv-9-2020-09-15-rcv-fr.xml │ │ │ ├── rcv_list_pv-9-2023-12-12-rcv-fr.xml │ │ │ ├── vot-list_pv-10-2024-11-14-vot-en.xml │ │ │ ├── vot-list_pv-10-2024-11-27-vot-en.xml │ │ │ ├── vot-list_pv-10-2024-11-28-vot-en.xml │ │ │ ├── vot-list_pv-10-2025-05-08-vot-en.xml │ │ │ └── vot-list_pv-9-2024-04-24-vot-en.xml │ ├── test_common.py │ ├── test_helpers.py │ ├── test_members.py │ ├── test_press_releases.py │ ├── test_sessions.py │ └── test_votes.py │ ├── store │ ├── test_aggregator.py │ ├── test_mappings.py │ └── test_writer.py │ ├── test_data.py │ ├── test_helpers.py │ ├── test_pushover.py │ ├── test_query.py │ ├── test_vote_stats.py │ └── worker │ └── test_worker.py ├── caddy └── Caddyfile ├── docker-compose.override.yml ├── docker-compose.yml ├── docs ├── architecture.md ├── data-sources.md ├── design-premises.md ├── fragments.md ├── funders.png └── pipelines.md ├── frontend ├── .dockerignore ├── .editorconfig ├── .gitignore ├── Dockerfile ├── Makefile ├── biome.json ├── openapi-ts.config.ts ├── package-lock.json ├── package.json ├── scripts │ ├── manifest │ └── watch ├── src │ ├── api │ │ ├── generated │ │ │ ├── index.ts │ │ │ ├── sdk.gen.ts │ │ │ └── types.gen.ts │ │ └── index.ts │ ├── client.entry.ts │ ├── components │ │ ├── App.tsx │ │ ├── Avatar.css │ │ ├── Avatar.tsx │ │ ├── Banner.css │ │ ├── Banner.tsx │ │ ├── BaseLayout.css │ │ ├── BaseLayout.tsx │ │ ├── Button.css │ │ ├── Button.tsx │ │ ├── CCLogo.css │ │ ├── CCLogo.tsx │ │ ├── CountriesFilterSelect.tsx │ │ ├── CountryFlag.tsx │ │ ├── CountryStatsList.tsx │ │ ├── Disclosure.css │ │ ├── Disclosure.tsx │ │ ├── EmptyState.css │ │ ├── EmptyState.tsx │ │ ├── Eyes.tsx │ │ ├── Footer.css │ │ ├── Footer.tsx │ │ ├── GroupStatsList.tsx │ │ ├── GroupsFilterSelect.tsx │ │ ├── Header.css │ │ ├── Header.tsx │ │ ├── Hero.css │ │ ├── Hero.tsx │ │ ├── Input.css │ │ ├── Input.tsx │ │ ├── List.css │ │ ├── List.tsx │ │ ├── ListItem.css │ │ ├── ListItem.tsx │ │ ├── MemberVotesList.css │ │ ├── MemberVotesList.test.tsx │ │ ├── MemberVotesList.tsx │ │ ├── Pagination.css │ │ ├── Pagination.tsx │ │ ├── PositionFilterSelect.tsx │ │ ├── SearchForm.css │ │ ├── SearchForm.tsx │ │ ├── Select.css │ │ ├── Select.test.tsx │ │ ├── Select.tsx │ │ ├── ShareButton.css │ │ ├── ShareButton.tsx │ │ ├── SortSelect.tsx │ │ ├── Sources.tsx │ │ ├── Stack.css │ │ ├── Stack.tsx │ │ ├── Stats.css │ │ ├── Stats.tsx │ │ ├── StatsCard.css │ │ ├── StatsCard.tsx │ │ ├── Tabs.css │ │ ├── Tabs.tsx │ │ ├── Tag.css │ │ ├── Tag.tsx │ │ ├── Thumb.css │ │ ├── Thumb.tsx │ │ ├── VoteCard.css │ │ ├── VoteCard.tsx │ │ ├── VoteCards.css │ │ ├── VoteCards.tsx │ │ ├── VoteHeader.css │ │ ├── VoteHeader.tsx │ │ ├── VoteResultChart.css │ │ ├── VoteResultChart.tsx │ │ ├── VoteSharepic.css │ │ ├── VoteSharepic.tsx │ │ ├── VoteTabs.tsx │ │ ├── Wrapper.css │ │ └── Wrapper.tsx │ ├── config.ts │ ├── css │ │ ├── base.css │ │ ├── fonts.css │ │ ├── typography.css │ │ ├── utils.css │ │ └── variables.css │ ├── cssLoader.js │ ├── lib │ │ ├── bem.ts │ │ ├── bots.test.ts │ │ ├── bots.ts │ │ ├── caching.ts │ │ ├── dates.ts │ │ ├── http.ts │ │ ├── islands.test.tsx │ │ ├── islands.tsx │ │ ├── links.ts │ │ ├── logging.ts │ │ ├── normalization.test.ts │ │ ├── normalization.ts │ │ ├── serialization.ts │ │ ├── server.ts │ │ ├── validation.test.ts │ │ └── validation.ts │ ├── pages │ │ ├── AboutPage.tsx │ │ ├── DevelopersPage.tsx │ │ ├── ErrorPage.css │ │ ├── ErrorPage.tsx │ │ ├── HomePage.css │ │ ├── HomePage.tsx │ │ ├── ImprintPage.tsx │ │ ├── SearchPage.css │ │ ├── SearchPage.tsx │ │ ├── ShowVotePage.tsx │ │ └── VoteSharepicPage.tsx │ ├── server.entry.ts │ └── setupTests.js ├── static │ ├── bg-dark.jpg │ ├── favicon-32px.png │ ├── fonts │ │ ├── ibm-plex-sans-condensed-semibold.woff │ │ ├── ibm-plex-sans-condensed-semibold.woff2 │ │ ├── ibm-plex-sans-medium.woff │ │ ├── ibm-plex-sans-medium.woff2 │ │ ├── ibm-plex-sans-regular.woff │ │ ├── ibm-plex-sans-regular.woff2 │ │ ├── ibm-plex-sans-semibold.woff │ │ └── ibm-plex-sans-semibold.woff2 │ ├── funders.png │ ├── groups │ │ ├── alde.svg │ │ ├── ecr.svg │ │ ├── efd.svg │ │ ├── enf.svg │ │ ├── epp.svg │ │ ├── esn.svg │ │ ├── green_efa.svg │ │ ├── gue_ngl.svg │ │ ├── id.svg │ │ ├── ni.svg │ │ ├── pfe.svg │ │ ├── renew.svg │ │ └── sd.svg │ ├── icons.svg │ ├── manifest.json │ ├── notifications-icon.png │ ├── sharepic-default.png │ ├── spotlight │ │ └── 8.1.0 │ │ │ ├── styles.min.css │ │ │ ├── web-components.min.js │ │ │ └── web-components.min.js.LICENSE.txt │ └── touch-icon-180px.png └── tsconfig.json └── storage ├── database └── .gitignore ├── files └── .gitignore └── index └── .gitignore /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | insert_final_newline = true 7 | indent_style = space 8 | indent_size = 4 9 | trim_trailing_whitespace = true 10 | 11 | [*.md] 12 | trim_trailing_whitespace = false 13 | 14 | [*.{yml,yaml}] 15 | indent_size = 2 16 | 17 | [*.{js,json}] 18 | indent_size = 2 19 | 20 | [*.{html,html.j2}] 21 | indent_size = 2 22 | -------------------------------------------------------------------------------- /.env.template: -------------------------------------------------------------------------------- 1 | HTV_BACKEND_PUBLIC_URL=https://localhost/api 2 | HTV_FRONTEND_PUBLIC_URL=https://localhost 3 | CADDY_SITE_ADDRESS=localhost 4 | 5 | # Optional 6 | PUSHOVER_API_TOKEN= 7 | PUSHOVER_USER_KEY= 8 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | backend/tests/scrapers/data/* linguist-vendored 2 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "monthly" 7 | 8 | - package-ecosystem: "npm" 9 | directory: "/frontend" 10 | schedule: 11 | interval: "monthly" 12 | open-pull-requests-limit: 50 13 | groups: 14 | dev-dependencies: 15 | applies-to: "version-updates" 16 | dependency-type: "development" 17 | exclude-patterns: 18 | # These dependencies are responsible for code generation. Updating them 19 | # likely also requires regenerating code, so it’s a good idea to update 20 | # them separately from other dev dependencies. 21 | - "@hey-api/*" 22 | minor-patch: 23 | applies-to: "version-updates" 24 | dependency-type: "production" 25 | update-types: 26 | - "minor" 27 | - "patch" 28 | 29 | - package-ecosystem: "docker" 30 | directory: "/frontend" 31 | schedule: 32 | interval: "monthly" 33 | 34 | - package-ecosystem: "pip" 35 | directory: "/backend" 36 | schedule: 37 | interval: "monthly" 38 | open-pull-requests-limit: 50 39 | groups: 40 | dev-dependencies: 41 | applies-to: "version-updates" 42 | dependency-type: "development" 43 | minor-patch: 44 | applies-to: "version-updates" 45 | dependency-type: "production" 46 | update-types: 47 | - "minor" 48 | - "patch" 49 | 50 | - package-ecosystem: "docker" 51 | directory: "/backend" 52 | schedule: 53 | interval: "monthly" 54 | 55 | - package-ecosystem: "docker-compose" 56 | directory: "/" 57 | schedule: 58 | interval: "monthly" 59 | -------------------------------------------------------------------------------- /.github/workflows/backend.yml: -------------------------------------------------------------------------------- 1 | name: Backend CI 2 | 3 | on: 4 | push: 5 | branches: ["main"] 6 | pull_request: {} 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | 12 | defaults: 13 | run: 14 | working-directory: ./backend 15 | 16 | env: 17 | # Make Python system packages (such as Xapian) accessible 18 | PYTHONPATH: /usr/lib/python3/dist-packages 19 | 20 | steps: 21 | - name: Checkout repo 22 | uses: actions/checkout@v4 23 | 24 | - name: Install Xapian 25 | run: sudo apt-get -y -q install python3-xapian 26 | 27 | # Stopwords are included in the Alpine Xapian package, but for some reason they are not included 28 | # in the Ubuntu package. This downloads them from the same source as the Alpine package: 29 | # https://gitlab.alpinelinux.org/alpine/aports/-/blob/3.19-stable/community/xapian-core/APKBUILD#L12 30 | - name: Download stopwords 31 | working-directory: ${{ runner.temp }} 32 | run: | 33 | curl -o xapian-core-1.4.26.tar.xz https://oligarchy.co.uk/xapian/1.4.26/xapian-core-1.4.26.tar.xz 34 | tar -xf xapian-core-1.4.26.tar.xz 35 | mkdir -p /usr/share/xapian-core/stopwords 36 | cp xapian-core-1.4.26/languages/stopwords/english.list /usr/share/xapian-core/stopwords/english.list 37 | 38 | - name: Install poetry 39 | run: pipx install poetry 40 | 41 | - name: Set up Python 42 | uses: actions/setup-python@v5 43 | with: 44 | python-version: "3.12" 45 | cache: "poetry" 46 | cache-dependency-path: "./backend/poetry.lock" 47 | 48 | - name: Install dependencies 49 | run: poetry install 50 | 51 | - name: Check formatting 52 | run: make format-check 53 | 54 | - name: Run linter 55 | run: make lint 56 | 57 | - name: Run types 58 | run: make typecheck 59 | 60 | - name: Run tests 61 | run: make test 62 | env: 63 | HTV_BACKEND_DATABASE_URI: "sqlite:///${{ github.workspace }}/storage/database/database.sqlite3" 64 | HTV_SEARCH_INDEX_DIR: "${{ github.workspace }}/storage/index" 65 | -------------------------------------------------------------------------------- /.github/workflows/docker.yml: -------------------------------------------------------------------------------- 1 | name: Docker images 2 | 3 | on: 4 | workflow_dispatch: {} 5 | push: 6 | branches: ["main"] 7 | pull_request: {} 8 | 9 | permissions: 10 | contents: read 11 | packages: write 12 | 13 | jobs: 14 | docker: 15 | runs-on: ubuntu-latest 16 | 17 | strategy: 18 | matrix: 19 | image: 20 | - frontend 21 | - backend 22 | 23 | steps: 24 | - name: Checkout repo 25 | uses: actions/checkout@v4 26 | 27 | # Required to build multi-platform images 28 | - name: Set up Docker Buildx 29 | uses: docker/setup-buildx-action@v3 30 | 31 | # Automatically tags Docker images based on Git tags, branches etc. 32 | - name: Extract metadata 33 | id: meta 34 | uses: docker/metadata-action@v5 35 | with: 36 | images: ghcr.io/${{ github.repository }}-${{ matrix.image }} 37 | 38 | - name: Log in to the Container registry 39 | uses: docker/login-action@v3 40 | if: ${{ github.event_name == 'workflow_dispatch' }} 41 | with: 42 | registry: ghcr.io 43 | username: ${{ github.actor }} 44 | password: ${{ secrets.GITHUB_TOKEN }} 45 | 46 | - name: Build and push 47 | uses: docker/build-push-action@v6 48 | with: 49 | context: ./${{ matrix.image }} 50 | platforms: linux/arm64 51 | push: ${{ github.event_name == 'workflow_dispatch' }} 52 | tags: ${{ steps.meta.outputs.tags }} 53 | cache-from: type=gha 54 | cache-to: type=gha,mode=max 55 | -------------------------------------------------------------------------------- /.github/workflows/frontend.yml: -------------------------------------------------------------------------------- 1 | name: Frontend CI 2 | 3 | on: 4 | push: 5 | branches: ["main"] 6 | pull_request: {} 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | 12 | defaults: 13 | run: 14 | working-directory: ./frontend 15 | 16 | steps: 17 | - name: Checkout repo 18 | uses: actions/checkout@v4 19 | 20 | - name: Set up Node.js 21 | uses: actions/setup-node@v4 22 | with: 23 | node-version: "20" 24 | cache: "npm" 25 | cache-dependency-path: "./frontend/package-lock.json" 26 | 27 | - name: Install dependencies 28 | run: npm install 29 | 30 | - name: Run linter 31 | run: make lint 32 | 33 | - name: Run types 34 | run: make typecheck 35 | 36 | - name: Run tests 37 | run: make test 38 | -------------------------------------------------------------------------------- /.github/workflows/scheduled.yml: -------------------------------------------------------------------------------- 1 | name: Scheduled scraper tests 2 | 3 | on: 4 | workflow_dispatch: {} 5 | schedule: 6 | - cron: "0 0 * * 2" # every Tuesday at 00:00 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | 12 | defaults: 13 | run: 14 | working-directory: ./backend 15 | 16 | env: 17 | # Make Python system packages (such as Xapian) accessible 18 | PYTHONPATH: /usr/lib/python3/dist-packages 19 | 20 | steps: 21 | - name: Checkout repo 22 | uses: actions/checkout@v4 23 | 24 | - name: Install Xapian 25 | run: sudo apt-get -y -q install python3-xapian 26 | 27 | # Stopwords are included in the Alpine Xapian package, but for some reason they are not included 28 | # in the Ubuntu package. This downloads them from the same source as the Alpine package: 29 | # https://gitlab.alpinelinux.org/alpine/aports/-/blob/3.19-stable/community/xapian-core/APKBUILD#L12 30 | - name: Download stopwords 31 | working-directory: ${{ runner.temp }} 32 | run: | 33 | curl -o xapian-core-1.4.26.tar.xz https://oligarchy.co.uk/xapian/1.4.26/xapian-core-1.4.26.tar.xz 34 | tar -xf xapian-core-1.4.26.tar.xz 35 | mkdir -p /usr/share/xapian-core/stopwords 36 | cp xapian-core-1.4.26/languages/stopwords/english.list /usr/share/xapian-core/stopwords/english.list 37 | 38 | - name: Install poetry 39 | run: pipx install poetry 40 | 41 | - name: Set up Python 42 | uses: actions/setup-python@v5 43 | with: 44 | python-version: "3.12" 45 | cache: "poetry" 46 | cache-dependency-path: "./backend/poetry.lock" 47 | 48 | - name: Install dependencies 49 | run: poetry install 50 | 51 | - name: Run tests against live data sources 52 | run: make test 53 | env: 54 | HTV_TEST_MOCK_REQUESTS: "false" 55 | HTV_BACKEND_DATABASE_URI: "sqlite:///${{ github.workspace }}/storage/database/database.sqlite3" 56 | HTV_SEARCH_INDEX_DIR: "${{ github.workspace }}/storage/index" 57 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_STORE 2 | .env 3 | -------------------------------------------------------------------------------- /backend/.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | source = ep_votes 4 | 5 | [report] 6 | exclude_lines = 7 | if self.debug: 8 | pragma: no cover 9 | raise NotImplementedError 10 | if __name__ == .__main__.: 11 | ignore_errors = True 12 | omit = 13 | tests/* 14 | -------------------------------------------------------------------------------- /backend/.editorconfig: -------------------------------------------------------------------------------- 1 | [*.py] 2 | indent_style = space 3 | indent_size = 4 4 | 5 | [*.xml] 6 | indent_style = space 7 | indent_size = 4 8 | 9 | [Makefile] 10 | indent_style = tab 11 | -------------------------------------------------------------------------------- /backend/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.12-alpine3.21 2 | 3 | RUN apk --update add \ 4 | build-base \ 5 | libffi-dev \ 6 | # TODO: Switch back to LibreSSL once conflicts on Alpine 3.17 are resolved. 7 | openssl-dev \ 8 | libxml2-dev \ 9 | libxslt-dev \ 10 | python3-dev \ 11 | bash \ 12 | less \ 13 | make \ 14 | cargo \ 15 | sqlite \ 16 | tmux \ 17 | xapian-core \ 18 | xapian-bindings-python3 19 | 20 | RUN pip install poetry 21 | 22 | WORKDIR /howtheyvote/backend 23 | 24 | # Copy files required to install dependencies 25 | COPY pyproject.toml poetry.toml poetry.lock . 26 | 27 | RUN poetry env use python3.12 28 | 29 | # Install only dependencies, but not the package itself (because that 30 | # would fail because we haven't yet copied the code at this point) 31 | RUN poetry install --no-root 32 | 33 | COPY . . 34 | 35 | # Install again in order to make the `htv` CLI script available, this 36 | # time without the dependencies (which have been installed before) 37 | RUN poetry install --only-root 38 | 39 | ENV TZ=UTC 40 | ENV ALEMBIC_CONFIG=./howtheyvote/alembic/alembic.ini 41 | ENV PATH="/howtheyvote/backend/.venv/bin:$PATH" 42 | 43 | # Make Python system packages (such as Xapian) accessible 44 | ENV PYTHONPATH=/usr/lib/python3.12/site-packages 45 | 46 | CMD gunicorn -b [::]:5000 --workers=5 --forwarded-allow-ips=* howtheyvote.wsgi:app 47 | -------------------------------------------------------------------------------- /backend/Makefile: -------------------------------------------------------------------------------- 1 | default: format-check lint typecheck test 2 | 3 | test: 4 | poetry run pytest --cov=. --cov=!./howtheyvote/alembic --cov-report=html:tests/htmlcov 5 | 6 | coverage: 7 | poetry run coverage xml 8 | 9 | lint: 10 | poetry run ruff check . 11 | 12 | lint-fix: 13 | poetry run ruff check --fix . 14 | 15 | format: 16 | poetry run ruff format . 17 | 18 | format-check: 19 | poetry run ruff format --check . 20 | 21 | typecheck: 22 | poetry run mypy --strict ./howtheyvote/ 23 | 24 | dev: 25 | poetry run flask --app howtheyvote.wsgi run --debug --host=0.0.0.0 26 | -------------------------------------------------------------------------------- /backend/howtheyvote/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | 4 | import structlog 5 | 6 | logging.basicConfig( 7 | format="%(message)s", 8 | stream=sys.stdout, 9 | level=logging.INFO, 10 | ) 11 | 12 | structlog.configure( 13 | processors=[ 14 | structlog.contextvars.merge_contextvars, 15 | structlog.stdlib.filter_by_level, 16 | structlog.stdlib.add_logger_name, 17 | structlog.processors.add_log_level, 18 | structlog.processors.TimeStamper(fmt="iso"), 19 | structlog.processors.format_exc_info, 20 | structlog.processors.JSONRenderer(), 21 | ], 22 | logger_factory=structlog.stdlib.LoggerFactory(), 23 | wrapper_class=structlog.stdlib.BoundLogger, 24 | ) 25 | -------------------------------------------------------------------------------- /backend/howtheyvote/alembic/env.py: -------------------------------------------------------------------------------- 1 | from alembic import context 2 | 3 | from howtheyvote.config import DATABASE_URI 4 | 5 | config = context.config 6 | config.set_main_option("sqlalchemy.url", DATABASE_URI) 7 | 8 | 9 | def run_migrations_online() -> None: 10 | connection = config.attributes.get("connection", None) 11 | context.configure(connection=connection, target_metadata=None) 12 | 13 | with context.begin_transaction(): 14 | context.run_migrations() 15 | 16 | 17 | run_migrations_online() 18 | -------------------------------------------------------------------------------- /backend/howtheyvote/alembic/script.py.mako: -------------------------------------------------------------------------------- 1 | """${message} 2 | 3 | Revision ID: ${up_revision} 4 | Revises: ${down_revision | comma,n} 5 | Create Date: ${create_date} 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | ${imports if imports else ""} 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = ${repr(up_revision)} 14 | down_revision = ${repr(down_revision)} 15 | branch_labels = ${repr(branch_labels)} 16 | depends_on = ${repr(depends_on)} 17 | 18 | 19 | def upgrade() -> None: 20 | ${upgrades if upgrades else "pass"} 21 | 22 | 23 | def downgrade() -> None: 24 | ${downgrades if downgrades else "pass"} 25 | -------------------------------------------------------------------------------- /backend/howtheyvote/alembic/versions/064daf473f9a_add_procedure_stage_column_to_votes_.py: -------------------------------------------------------------------------------- 1 | """Add procedure_stage column to votes table 2 | 3 | Revision ID: 064daf473f9a 4 | Revises: 47cf729ded96 5 | Create Date: 2025-04-11 14:53:28.949263 6 | 7 | """ 8 | 9 | import sqlalchemy as sa 10 | from alembic import op 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "064daf473f9a" 14 | down_revision = "47cf729ded96" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | op.add_column("votes", sa.Column("procedure_stage", sa.Unicode)) 21 | 22 | 23 | def downgrade() -> None: 24 | op.drop_column("votes", "procedure_stage") 25 | -------------------------------------------------------------------------------- /backend/howtheyvote/alembic/versions/1deea9ed7c52_add_oeil_subjects_column_to_votes_table.py: -------------------------------------------------------------------------------- 1 | """Add oeil_subjects column to votes table 2 | 3 | Revision ID: 1deea9ed7c52 4 | Revises: 8d1995cb0bed 5 | Create Date: 2025-05-11 20:30:46.675280 6 | 7 | """ 8 | 9 | import sqlalchemy as sa 10 | from alembic import op 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "1deea9ed7c52" 14 | down_revision = "8d1995cb0bed" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | op.add_column("votes", sa.Column("oeil_subjects", sa.JSON)) 21 | 22 | 23 | def downgrade() -> None: 24 | op.drop_column("votes", "oeil_subjects") 25 | -------------------------------------------------------------------------------- /backend/howtheyvote/alembic/versions/1f516b18c4f6_rename_result_column_to_status.py: -------------------------------------------------------------------------------- 1 | """Rename result column to status 2 | 3 | Revision ID: 1f516b18c4f6 4 | Revises: 9b35d19b64c4 5 | Create Date: 2024-12-08 11:25:26.051408 6 | 7 | """ 8 | 9 | from alembic import op 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = "1f516b18c4f6" 13 | down_revision = "9b35d19b64c4" 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade() -> None: 19 | op.alter_column("pipeline_runs", column_name="result", new_column_name="status") 20 | 21 | 22 | def downgrade() -> None: 23 | op.alter_column("pipeline_runs", column_name="status", new_column_name="result") 24 | -------------------------------------------------------------------------------- /backend/howtheyvote/alembic/versions/2f958a6f147d_add_checksum_column_to_pipeline_runs_.py: -------------------------------------------------------------------------------- 1 | """Add checksum column to pipeline_runs table 2 | 3 | Revision ID: 2f958a6f147d 4 | Revises: 1f516b18c4f6 5 | Create Date: 2024-12-07 17:12:10.792707 6 | 7 | """ 8 | 9 | import sqlalchemy as sa 10 | from alembic import op 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "2f958a6f147d" 14 | down_revision = "1f516b18c4f6" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | op.add_column("pipeline_runs", sa.Column("checksum", sa.Unicode)) 21 | 22 | 23 | def downgrade() -> None: 24 | op.drop_column("pipeline_runs", "checksum") 25 | -------------------------------------------------------------------------------- /backend/howtheyvote/alembic/versions/2fb74cc3d20c_add_text_and_position_counts_column_to_press_releases_table.py: -------------------------------------------------------------------------------- 1 | """Add text and position_counts column to press_releases table 2 | 3 | Revision ID: 2fb74cc3d20c 4 | Revises: 3895af031aea 5 | Create Date: 2025-02-23 13:13:56.054579 6 | 7 | """ 8 | 9 | import sqlalchemy as sa 10 | from alembic import op 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "2fb74cc3d20c" 14 | down_revision = "3895af031aea" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | op.add_column("press_releases", sa.Column("text", sa.Unicode)) 21 | op.add_column("press_releases", sa.Column("position_counts", sa.JSON)) 22 | 23 | 24 | def downgrade() -> None: 25 | op.drop_column("press_releases", "text") 26 | op.drop_column("press_releases", "position_counts") 27 | -------------------------------------------------------------------------------- /backend/howtheyvote/alembic/versions/3895af031aea_add_responsible_committee_column_to_.py: -------------------------------------------------------------------------------- 1 | """Add responsible_committee column to votes table 2 | 3 | Revision ID: 3895af031aea 4 | Revises: 2f958a6f147d 5 | Create Date: 2025-02-21 11:31:59.044124 6 | 7 | """ 8 | 9 | import sqlalchemy as sa 10 | from alembic import op 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "3895af031aea" 14 | down_revision = "2f958a6f147d" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | op.add_column("votes", sa.Column("responsible_committee", sa.Unicode)) 21 | 22 | 23 | def downgrade() -> None: 24 | op.drop_column("votes", "responsible_committee") 25 | -------------------------------------------------------------------------------- /backend/howtheyvote/alembic/versions/3eec8050f4d5_add_eurovoc_concepts_column.py: -------------------------------------------------------------------------------- 1 | """Add eurovoc_concepts column 2 | 3 | Revision ID: 3eec8050f4d5 4 | Revises: 9a4972f3a768 5 | Create Date: 2024-03-31 17:25:34.058229 6 | 7 | """ 8 | 9 | import sqlalchemy as sa 10 | from alembic import op 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "3eec8050f4d5" 14 | down_revision = "9a4972f3a768" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | op.add_column("votes", sa.Column("eurovoc_concepts", sa.JSON)) 21 | 22 | 23 | def downgrade() -> None: 24 | op.drop_column("votes", "eurovoc_concepts") 25 | -------------------------------------------------------------------------------- /backend/howtheyvote/alembic/versions/47cf729ded96_add_dlv_title_and_result_columns_to_.py: -------------------------------------------------------------------------------- 1 | """Add dlv_title and result columns to votes table 2 | 3 | Revision ID: 47cf729ded96 4 | Revises: 7c9e86276c6d 5 | Create Date: 2025-04-11 10:23:00.867950 6 | 7 | """ 8 | 9 | import sqlalchemy as sa 10 | from alembic import op 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "47cf729ded96" 14 | down_revision = "7c9e86276c6d" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | op.add_column("votes", sa.Column("dlv_title", sa.Unicode)) 21 | op.add_column("votes", sa.Column("result", sa.Unicode)) 22 | 23 | 24 | def downgrade() -> None: 25 | op.drop_column("votes", "dlv_title") 26 | op.drop_column("votes", "result") 27 | -------------------------------------------------------------------------------- /backend/howtheyvote/alembic/versions/7c9e86276c6d_remove_is_featured_column_from_votes_.py: -------------------------------------------------------------------------------- 1 | """Remove is_featured column from votes table 2 | 3 | Revision ID: 7c9e86276c6d 4 | Revises: 848ef24718dd 5 | Create Date: 2025-03-14 22:46:51.712125 6 | 7 | """ 8 | 9 | import sqlalchemy as sa 10 | from alembic import op 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "7c9e86276c6d" 14 | down_revision = "848ef24718dd" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | op.drop_column("votes", "is_featured") 21 | 22 | 23 | def downgrade() -> None: 24 | op.add_column("votes", sa.Column("is_featured", sa.Boolean)) 25 | -------------------------------------------------------------------------------- /backend/howtheyvote/alembic/versions/848ef24718dd_add_press_release_column_to_votes_table.py: -------------------------------------------------------------------------------- 1 | """Add press_release column to votes table 2 | 3 | Revision ID: 848ef24718dd 4 | Revises: 2fb74cc3d20c 5 | Create Date: 2025-03-01 21:27:44.628151 6 | 7 | """ 8 | 9 | import sqlalchemy as sa 10 | from alembic import op 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "848ef24718dd" 14 | down_revision = "2fb74cc3d20c" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | op.add_column("votes", sa.Column("press_release", sa.Unicode)) 21 | 22 | 23 | def downgrade() -> None: 24 | op.drop_column("votes", "press_release") 25 | -------------------------------------------------------------------------------- /backend/howtheyvote/alembic/versions/8d1995cb0bed_make_responsible_committee_column_json_.py: -------------------------------------------------------------------------------- 1 | """Make responsible_committee column JSON in votes table 2 | 3 | Revision ID: 8d1995cb0bed 4 | Revises: 064daf473f9a 5 | Create Date: 2025-03-16 17:13:08.000602 6 | 7 | """ 8 | 9 | import sqlalchemy as sa 10 | from alembic import op 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "8d1995cb0bed" 14 | down_revision = "064daf473f9a" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | op.drop_column("votes", "responsible_committee") 21 | op.add_column("votes", sa.Column("responsible_committees", sa.JSON)) 22 | 23 | 24 | def downgrade() -> None: 25 | op.drop_column("votes", "responsible_committees") 26 | op.add_column("votes", sa.Column("responsible_committee", sa.Unicode)) 27 | -------------------------------------------------------------------------------- /backend/howtheyvote/alembic/versions/9200b9028b93_create_timestamp_index_on_votes_table.py: -------------------------------------------------------------------------------- 1 | """create timestamp index on votes table 2 | 3 | Revision ID: 9200b9028b93 4 | Revises: 3eec8050f4d5 5 | Create Date: 2024-05-06 15:49:49.074361 6 | 7 | """ 8 | 9 | from alembic import op 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = "9200b9028b93" 13 | down_revision = "3eec8050f4d5" 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade() -> None: 19 | op.create_index("ix_votes_timestamp", table_name="votes", columns=["timestamp"]) 20 | 21 | 22 | def downgrade() -> None: 23 | op.drop_index("ix_votes_timestamp", table_name="votes") 24 | -------------------------------------------------------------------------------- /backend/howtheyvote/alembic/versions/9b35d19b64c4_add_term_column_to_votes_table.py: -------------------------------------------------------------------------------- 1 | """add term column to votes table 2 | 3 | Revision ID: 9b35d19b64c4 4 | Revises: 9200b9028b93 5 | Create Date: 2024-08-06 18:52:06.033551 6 | 7 | """ 8 | 9 | import sqlalchemy as sa 10 | from alembic import op 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "9b35d19b64c4" 14 | down_revision = "9200b9028b93" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | op.add_column("votes", sa.Column("term", sa.Integer)) 21 | op.create_index("ix_votes_term", table_name="votes", columns=["term"]) 22 | 23 | 24 | def downgrade() -> None: 25 | op.drop_column("votes", "term") 26 | op.drop_index("ix_votes_term", table_name="votes") 27 | -------------------------------------------------------------------------------- /backend/howtheyvote/analysis/__init__.py: -------------------------------------------------------------------------------- 1 | from .press_releases import VotePositionCountsAnalyzer 2 | from .votes import ( 3 | MainVoteAnalyzer, 4 | PressReleaseAnalyzer, 5 | VoteDataIssuesAnalyzer, 6 | VoteGroupsAnalyzer, 7 | VoteGroupsDataIssuesAnalyzer, 8 | ) 9 | 10 | __all__ = [ 11 | "PressReleaseAnalyzer", 12 | "MainVoteAnalyzer", 13 | "VoteDataIssuesAnalyzer", 14 | "VoteGroupsAnalyzer", 15 | "VoteGroupsDataIssuesAnalyzer", 16 | "VotePositionCountsAnalyzer", 17 | ] 18 | -------------------------------------------------------------------------------- /backend/howtheyvote/analysis/press_releases.py: -------------------------------------------------------------------------------- 1 | from ..models import Fragment 2 | from .helpers import extract_vote_results 3 | 4 | 5 | class VotePositionCountsAnalyzer: 6 | """Extracts mentioned vote position counts from press release text.""" 7 | 8 | def __init__(self, release_id: str, text: str): 9 | self.release_id = release_id 10 | self.text = text 11 | 12 | def run(self) -> Fragment | None: 13 | if not self.text: 14 | return None 15 | 16 | position_counts = extract_vote_results(self.text) 17 | 18 | if not position_counts: 19 | return None 20 | 21 | return Fragment( 22 | model="PressRelease", 23 | source_id=self.release_id, 24 | source_name=type(self).__name__, 25 | group_key=self.release_id, 26 | data={"position_counts": position_counts}, 27 | ) 28 | -------------------------------------------------------------------------------- /backend/howtheyvote/api/__init__.py: -------------------------------------------------------------------------------- 1 | from flask import Blueprint, current_app 2 | from flask.typing import ResponseReturnValue 3 | 4 | from .openapi_spec import spec 5 | from .sessions_api import bp as sessions_bp 6 | from .static_api import bp as static_bp 7 | from .stats_api import bp as stats_bp 8 | from .votes_api import bp as votes_bp 9 | 10 | bp = Blueprint("api", __name__) 11 | 12 | bp.register_blueprint(static_bp) 13 | bp.register_blueprint(votes_bp) 14 | bp.register_blueprint(sessions_bp) 15 | bp.register_blueprint(stats_bp) 16 | 17 | 18 | @bp.route("/") 19 | def openapi() -> ResponseReturnValue: 20 | for name, view in current_app.view_functions.items(): 21 | if not name.startswith("api.static."): 22 | spec.path(view=view) 23 | 24 | return spec.to_dict() 25 | -------------------------------------------------------------------------------- /backend/howtheyvote/api/static_api.py: -------------------------------------------------------------------------------- 1 | from flask import Blueprint, abort, send_file 2 | from flask.typing import ResponseValue 3 | 4 | from ..files import member_photo_path, vote_sharepic_path 5 | 6 | CACHE_MAX_AGE = 180 * 24 * 60 * 60 7 | bp = Blueprint("static_api", __name__) 8 | 9 | 10 | @bp.route("/static/members/-.jpg") 11 | @bp.route("/static/members/.jpg") 12 | def member_photo(member_id: int, size: int | None = None) -> ResponseValue: 13 | path = member_photo_path(member_id, size) 14 | try: 15 | return send_file(path, max_age=CACHE_MAX_AGE) 16 | except FileNotFoundError: 17 | return abort(404) 18 | 19 | 20 | @bp.route("/static/votes/sharepic-.png") 21 | def vote_sharepic(vote_id: int) -> ResponseValue: 22 | path = vote_sharepic_path(vote_id) 23 | try: 24 | return send_file(path, max_age=CACHE_MAX_AGE) 25 | except FileNotFoundError: 26 | return abort(404) 27 | -------------------------------------------------------------------------------- /backend/howtheyvote/api/stats_api.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from typing import cast 3 | 4 | from flask import Blueprint, jsonify 5 | from flask.typing import ResponseValue 6 | from sqlalchemy import func, select 7 | 8 | from ..db import Session 9 | from ..models import Member, PipelineRun, PlenarySession, Vote 10 | from .serializers import Statistics 11 | 12 | bp = Blueprint("stats_api", __name__) 13 | 14 | 15 | @bp.route("/stats") 16 | def index() -> ResponseValue: 17 | """ 18 | --- 19 | get: 20 | operationId: getStats 21 | summary: List general stats 22 | tags: 23 | - Miscellaneous 24 | responses: 25 | '200': 26 | description: Ok 27 | content: 28 | application/json: 29 | schema: 30 | type: object 31 | allOf: 32 | - $ref: '#/components/schemas/Statistics' 33 | """ 34 | 35 | query = select(func.count()).select_from(Vote) 36 | votes_total = cast(int, Session.execute(query).scalar()) 37 | 38 | query = select(func.count()).select_from(Member) 39 | members_total = cast(int, Session.execute(query).scalar()) 40 | 41 | start_year = func.min(func.strftime("%Y", PlenarySession.start_date)) 42 | end_year = func.max(func.strftime("%Y", PlenarySession.end_date)) 43 | query = select(end_year - start_year) 44 | years_total = cast(int, Session.execute(query).scalar()) 45 | 46 | query = select(func.max(PipelineRun.finished_at)) 47 | last_update_date = cast(datetime.datetime, Session.execute(query).scalar()) 48 | 49 | stats: Statistics = { 50 | "votes_total": votes_total, 51 | "members_total": members_total, 52 | "years_total": years_total, 53 | "last_update_date": last_update_date, 54 | } 55 | 56 | return jsonify(stats) 57 | -------------------------------------------------------------------------------- /backend/howtheyvote/api/util.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Callable 2 | 3 | 4 | def one_of(*args: str) -> Callable[[str], str]: 5 | """Returns a function that validates if a given value is one of the provided 6 | values and returns a `ValueError` otherwise.""" 7 | 8 | def convert(value: str) -> str: 9 | if value in args: 10 | return value 11 | raise ValueError(f"Invalid value {value}") 12 | 13 | return convert 14 | -------------------------------------------------------------------------------- /backend/howtheyvote/cli/__init__.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | import shutil 3 | import tempfile 4 | 5 | import click 6 | 7 | from ..db import Session 8 | from ..export import generate_export 9 | from ..files import file_path 10 | from ..sharepics import generate_vote_sharepic 11 | from ..worker import worker as worker_ 12 | from .aggregate import aggregate 13 | from .dev import dev 14 | from .pipeline import pipeline 15 | from .system import system 16 | from .temp import temp 17 | 18 | 19 | @click.group() 20 | def cli() -> None: 21 | pass 22 | 23 | 24 | @cli.command() 25 | def worker() -> None: 26 | """Start a worker process to execute scheduled pipeline runs.""" 27 | worker_.run() 28 | 29 | 30 | @cli.command() 31 | def export() -> None: 32 | """Generate a CSV data export.""" 33 | archive_path = file_path("export/export") 34 | generate_export(archive_path) 35 | 36 | 37 | cli.add_command(system) 38 | cli.add_command(aggregate) 39 | cli.add_command(pipeline) 40 | cli.add_command(dev) 41 | cli.add_command(temp) 42 | -------------------------------------------------------------------------------- /backend/howtheyvote/cli/aggregate.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Collection 2 | from typing import TypeVar 3 | 4 | import click 5 | 6 | from ..models import BaseWithId, Member, PlenarySession, PressRelease, Vote 7 | from ..store import ( 8 | Aggregator, 9 | MapFunc, 10 | index_records, 11 | map_member, 12 | map_plenary_session, 13 | map_press_release, 14 | map_vote, 15 | ) 16 | 17 | 18 | @click.group() 19 | def aggregate() -> None: 20 | """Aggregate fragments into the database and the search index.""" 21 | pass 22 | 23 | 24 | RecordType = TypeVar("RecordType", bound=BaseWithId) 25 | 26 | 27 | def _aggregate( 28 | model_cls: type[RecordType], 29 | map_func: MapFunc[RecordType], 30 | group_keys: Collection[str] | None, 31 | chunk_size: int | None = None, 32 | ) -> None: 33 | aggregator = Aggregator(model_cls) 34 | 35 | if group_keys is not None and len(group_keys) == 0: 36 | group_keys = None 37 | 38 | mapped_records = aggregator.mapped_records(map_func, group_keys) 39 | index_records(model_cls, mapped_records, chunk_size) 40 | 41 | 42 | @aggregate.command() 43 | @click.option("--id", "id_", multiple=True) 44 | def members(id_: Collection[str] | None = None) -> None: 45 | _aggregate(Member, map_member, group_keys=id_) 46 | 47 | 48 | @aggregate.command() 49 | @click.option("--id", "id_", multiple=True) 50 | def sessions(id_: Collection[str] | None = None) -> None: 51 | _aggregate(PlenarySession, map_plenary_session, group_keys=id_) 52 | 53 | 54 | @aggregate.command() 55 | @click.option("--id", "id_", multiple=True) 56 | def votes(id_: Collection[str] | None = None) -> None: 57 | _aggregate(Vote, map_vote, group_keys=id_, chunk_size=1000) 58 | 59 | 60 | @aggregate.command() 61 | @click.option("--id", "id_", multiple=True) 62 | def press_releases(id_: Collection[str] | None = None) -> None: 63 | _aggregate(PressRelease, map_press_release, group_keys=id_) 64 | -------------------------------------------------------------------------------- /backend/howtheyvote/cli/system.py: -------------------------------------------------------------------------------- 1 | import click 2 | 3 | from ..db import migrate as _migrate 4 | from ..search import delete_indexes as _delete_indexes 5 | 6 | 7 | @click.group() 8 | def system() -> None: 9 | """Initialize or migrate the database and/or search index.""" 10 | pass 11 | 12 | 13 | @system.command() 14 | def delete_indexes() -> None: 15 | """Delete search indexes.""" 16 | _delete_indexes() 17 | 18 | 19 | @system.command() 20 | def migrate() -> None: 21 | """Run database migrations.""" 22 | _migrate() 23 | 24 | 25 | @system.command() 26 | def upgrade() -> None: 27 | """Equivalent of running the `migrate` and `configure-indexes` subcommands.""" 28 | _migrate() 29 | -------------------------------------------------------------------------------- /backend/howtheyvote/config.py: -------------------------------------------------------------------------------- 1 | from os import environ as env 2 | 3 | # URLs 4 | PUBLIC_URL = env.get("HTV_BACKEND_PUBLIC_URL", "") 5 | FRONTEND_PUBLIC_URL = env.get("HTV_FRONTEND_PUBLIC_URL", "") 6 | FRONTEND_PRIVATE_URL = env.get("HTV_FRONTEND_PRIVATE_URL", "") 7 | 8 | # Database 9 | DATABASE_URI = env.get( 10 | "HTV_BACKEND_DATABASE_URI", "sqlite:////howtheyvote/database/database.sqlite3" 11 | ) 12 | 13 | # File storage 14 | FILES_DIR = env.get("HTV_BACKEND_FILES_DIR", "/howtheyvote/files") 15 | 16 | # Request configuration 17 | REQUEST_TIMEOUT = 20 18 | REQUEST_SLEEP = 0.25 19 | 20 | # Misc 21 | CURRENT_TERM = 10 22 | TIMEZONE = "Europe/Brussels" 23 | WORKER_PROMETHEUS_PORT = 3000 24 | SEARCH_INDEX_PREFIX = env.get("HTV_SEARCH_INDEX_PREFIX", None) 25 | SEARCH_INDEX_DIR = env.get("HTV_SEARCH_INDEX_DIR", "/howtheyvote/index") 26 | 27 | # The Alpine package `xapian-core` installs stop word lists in this location 28 | SEARCH_STOPWORDS_PATH = "/usr/share/xapian-core/stopwords/english.list" 29 | 30 | # Pushover 31 | PUSHOVER_API_TOKEN = env.get("PUSHOVER_API_TOKEN") 32 | PUSHOVER_USER_KEY = env.get("PUSHOVER_USER_KEY") 33 | -------------------------------------------------------------------------------- /backend/howtheyvote/data.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from collections.abc import Iterator 3 | from pathlib import Path 4 | from typing import Any, Generic, Self, TypeVar 5 | 6 | from .json import json_dumps, json_loads 7 | 8 | DATA_DIR = Path(__file__).resolve().parent / "data" 9 | 10 | 11 | class DeserializableDataclass: 12 | @classmethod 13 | def from_dict(cls, data: dict[Any, Any]) -> Self: 14 | return cls(**data) 15 | 16 | 17 | DataclassType = TypeVar("DataclassType", bound=DeserializableDataclass) 18 | 19 | 20 | class DataclassContainer(Generic[DataclassType]): 21 | """A convenience class to write and load dataclasses from a JSON file and retrieve 22 | individual dataclass instances by key.""" 23 | 24 | def __init__(self, dataclass: type[DataclassType], file_path: Path | str, key_attr: str): 25 | self.dataclass = dataclass 26 | self.file_path = Path(file_path) 27 | self.key_attr = key_attr 28 | self.index: dict[str, DataclassType] = {} 29 | 30 | def load(self) -> None: 31 | """Load data from file.""" 32 | text = self.file_path.read_text() 33 | records = json_loads(text) 34 | 35 | for record in records: 36 | self.add(self.dataclass.from_dict(record)) 37 | 38 | def save(self) -> None: 39 | """Save data to file.""" 40 | records = [dataclasses.asdict(r) for r in self.index.values()] # type: ignore 41 | records = sorted(records, key=lambda r: r[self.key_attr]) 42 | text = json_dumps(records, indent=2) 43 | self.file_path.write_text(text) 44 | 45 | def add(self, record: DataclassType) -> None: 46 | """Add an individual record.""" 47 | key = getattr(record, self.key_attr) 48 | 49 | if not isinstance(key, str): 50 | raise TypeError("Key value must be a string") 51 | 52 | self.index[key.lower()] = record 53 | 54 | def get(self, key: str | None) -> DataclassType | None: 55 | """Get a record by key.""" 56 | if not key: 57 | return None 58 | 59 | return self.index.get(key.lower()) 60 | 61 | def __iter__(self) -> Iterator[DataclassType]: 62 | return iter(self.index.values()) 63 | -------------------------------------------------------------------------------- /backend/howtheyvote/db.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | from alembic import command as alembic_command 4 | from alembic.config import Config 5 | from sqlalchemy import create_engine, event 6 | from sqlalchemy.engine.interfaces import DBAPIConnection 7 | from sqlalchemy.orm import scoped_session, sessionmaker 8 | from structlog import get_logger 9 | 10 | from . import __path__, config 11 | from .json import json_dumps, json_loads 12 | 13 | log = get_logger(__name__) 14 | 15 | 16 | engine = create_engine( 17 | config.DATABASE_URI, 18 | # We use a custom JSON serializer/deserializer to handle enums, 19 | # custom serialization of dataclasses (e.g. `Country`) etc. 20 | json_serializer=json_dumps, 21 | json_deserializer=json_loads, 22 | ) 23 | 24 | session_factory = sessionmaker(engine) 25 | Session = scoped_session(session_factory) 26 | 27 | 28 | @event.listens_for(engine, "connect") 29 | def on_connect(connection: DBAPIConnection, _: Any) -> None: 30 | cursor = connection.cursor() 31 | 32 | # Set up Write-Ahead Log to ensure that reads do not block writes and 33 | # writes do not block reads 34 | cursor.execute("PRAGMA journal_mode=WAL;") 35 | 36 | cursor.close() 37 | 38 | 39 | def migrate() -> None: 40 | """Run Alembic database migrations.""" 41 | with engine.connect() as connection: 42 | alembic_config = Config() 43 | root = list(__path__)[0] 44 | alembic_config.set_main_option("script_location", f"{root}/alembic") 45 | alembic_config.attributes["connection"] = connection 46 | log.info("Running database migrations.") 47 | alembic_command.upgrade(alembic_config, "head") 48 | -------------------------------------------------------------------------------- /backend/howtheyvote/files.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import requests 4 | from PIL import Image 5 | from structlog import get_logger 6 | 7 | from . import config 8 | 9 | log = get_logger(__name__) 10 | 11 | 12 | def file_path(path: str | Path) -> Path: 13 | absolute_path = config.FILES_DIR / Path(path) 14 | relative_path = absolute_path.relative_to(config.FILES_DIR) 15 | return config.FILES_DIR / relative_path 16 | 17 | 18 | def member_photo_path(member_id: int, size: int | None = None) -> Path: 19 | if size: 20 | return file_path(f"members/{member_id}-{size}.jpg") 21 | 22 | return file_path(f"members/{member_id}.jpg") 23 | 24 | 25 | def vote_sharepic_path(vote_id: int) -> Path: 26 | return file_path(f"votes/sharepic-{vote_id}.png") 27 | 28 | 29 | def download_file(url: str, path: str | Path) -> Path | None: 30 | # Ensure that the download path is inside the files directory 31 | path = file_path(path) 32 | ensure_parent(path) 33 | 34 | with requests.get(url, stream=True, timeout=config.REQUEST_TIMEOUT) as res: 35 | res.raise_for_status() 36 | 37 | with open(path, "wb") as f: 38 | for chunk in res.iter_content(): 39 | f.write(chunk) 40 | 41 | return path 42 | 43 | 44 | def ensure_parent(path: str | Path) -> None: 45 | path = file_path(path) 46 | parent_dir = Path(path).parent 47 | 48 | if not parent_dir.exists(): 49 | parent_dir.mkdir(parents=True) 50 | 51 | 52 | def image_thumb( 53 | input_path: str | Path, 54 | output_path: str | Path, 55 | format: str | None = None, 56 | size: int | None = None, 57 | ) -> Path: 58 | output_path = file_path(output_path) 59 | 60 | with Image.open(input_path) as image: 61 | thumb = image.copy() 62 | 63 | if size: 64 | thumb.thumbnail((104, 104)) 65 | 66 | if not format: 67 | format = image.format 68 | 69 | thumb.save(output_path, format=format, optimize=True) 70 | 71 | return output_path 72 | -------------------------------------------------------------------------------- /backend/howtheyvote/json.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import datetime 3 | import enum 4 | import json 5 | from typing import Any 6 | 7 | from flask.json.provider import DefaultJSONProvider 8 | 9 | 10 | class JSONEncoder(json.JSONEncoder): 11 | """A JSON encoder that handles additional built-in types such as dates, sets, enums.""" 12 | 13 | def default(self, o: Any) -> Any: 14 | if isinstance(o, datetime.date): 15 | return o.isoformat() 16 | 17 | if isinstance(o, datetime.datetime): 18 | return o.isoformat() 19 | 20 | if isinstance(o, enum.Enum): 21 | return o.value 22 | 23 | if isinstance(o, set): 24 | return sorted(o) 25 | 26 | if dataclasses.is_dataclass(o) and not isinstance(o, type): 27 | return dataclasses.asdict(o) 28 | 29 | return json.JSONEncoder.default(self, o) 30 | 31 | 32 | def json_loads(string: str | bytes) -> Any: 33 | return json.loads(string) 34 | 35 | 36 | def json_dumps(obj: Any, indent: int | None = None) -> Any: 37 | return JSONEncoder(indent=indent).encode(obj) 38 | 39 | 40 | class JSONProvider(DefaultJSONProvider): 41 | def dumps(self, obj: Any, **kwargs: Any) -> Any: 42 | return json_dumps(obj) 43 | -------------------------------------------------------------------------------- /backend/howtheyvote/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .committee import Committee, CommitteeType 2 | from .common import Base, BaseWithId, DataIssue, Fragment, PipelineRun, PipelineStatus 3 | from .country import Country, CountryType 4 | from .eurovoc import EurovocConcept, EurovocConceptType 5 | from .group import Group 6 | from .member import ( 7 | GroupMembership, 8 | Member, 9 | deserialize_group_membership, 10 | serialize_group_membership, 11 | ) 12 | from .oeil import OEILSubject, OEILSubjectType 13 | from .press_release import PressRelease 14 | from .session import PlenarySession, PlenarySessionLocation, PlenarySessionStatus 15 | from .vote import ( 16 | DocumentType, 17 | MemberVote, 18 | ProcedureStage, 19 | ProcedureType, 20 | Vote, 21 | VoteGroup, 22 | VotePosition, 23 | VotePositionCounts, 24 | VoteResult, 25 | deserialize_member_vote, 26 | serialize_member_vote, 27 | ) 28 | 29 | __all__ = [ 30 | "Base", 31 | "BaseWithId", 32 | "Fragment", 33 | "PipelineRun", 34 | "PipelineStatus", 35 | "DataIssue", 36 | "Country", 37 | "CountryType", 38 | "Group", 39 | "EurovocConcept", 40 | "EurovocConceptType", 41 | "OEILSubject", 42 | "OEILSubjectType", 43 | "Committee", 44 | "CommitteeType", 45 | "PlenarySession", 46 | "PlenarySessionLocation", 47 | "PlenarySessionStatus", 48 | "GroupMembership", 49 | "Member", 50 | "VotePosition", 51 | "VotePositionCounts", 52 | "VoteResult", 53 | "MemberVote", 54 | "DocumentType", 55 | "ProcedureStage", 56 | "ProcedureType", 57 | "Vote", 58 | "VoteGroup", 59 | "PressRelease", 60 | "serialize_group_membership", 61 | "deserialize_group_membership", 62 | "serialize_member_vote", 63 | "deserialize_member_vote", 64 | ] 65 | -------------------------------------------------------------------------------- /backend/howtheyvote/models/committee.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import datetime 3 | 4 | import sqlalchemy as sa 5 | from sqlalchemy.engine import Dialect 6 | from sqlalchemy.types import TypeDecorator 7 | 8 | from ..data import DATA_DIR, DataclassContainer, DeserializableDataclass 9 | 10 | 11 | class CommitteeMeta(type): 12 | def __getitem__(cls, key: str) -> "Committee": 13 | committee = committees.get(key) 14 | 15 | if not committee: 16 | raise KeyError() 17 | 18 | return committee 19 | 20 | 21 | @dataclasses.dataclass(frozen=True) 22 | class Committee(DeserializableDataclass, metaclass=CommitteeMeta): 23 | code: str 24 | label: str 25 | abbreviation: str 26 | start_date: datetime.date 27 | end_date: datetime.date | None 28 | 29 | def __hash__(self) -> int: 30 | return hash(self.code) 31 | 32 | @classmethod 33 | def get(cls, key: str) -> "Committee | None": 34 | try: 35 | return cls[key] 36 | except KeyError: 37 | return None 38 | 39 | 40 | committees = DataclassContainer( 41 | dataclass=Committee, 42 | file_path=DATA_DIR.joinpath("committees.json"), 43 | key_attr="code", 44 | ) 45 | committees.load() 46 | 47 | 48 | class CommitteeType(TypeDecorator[Committee]): 49 | impl = sa.Unicode 50 | cache_ok = True 51 | 52 | def process_bind_param(self, value: Committee | None, dialect: Dialect) -> str | None: 53 | if not value: 54 | return None 55 | 56 | return value.code 57 | 58 | def process_result_value(self, value: str | None, dialect: Dialect) -> Committee | None: 59 | if not value: 60 | return None 61 | 62 | return committees.get(value) 63 | -------------------------------------------------------------------------------- /backend/howtheyvote/models/common.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from enum import Enum 3 | from typing import Any 4 | 5 | import sqlalchemy as sa 6 | from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column 7 | 8 | 9 | class Base(DeclarativeBase): 10 | pass 11 | 12 | 13 | class BaseWithId(Base): 14 | __abstract__ = True 15 | 16 | id: Mapped[str | int] 17 | 18 | 19 | class Fragment(Base): 20 | __tablename__ = "fragments" 21 | 22 | model: Mapped[str] = mapped_column(sa.Unicode, primary_key=True) 23 | source_name: Mapped[str] = mapped_column(sa.Unicode, primary_key=True) 24 | source_id: Mapped[str] = mapped_column(sa.Unicode, primary_key=True) 25 | source_url: Mapped[str | None] = mapped_column(sa.Unicode) 26 | timestamp: Mapped[datetime.datetime] = mapped_column( 27 | sa.DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now 28 | ) 29 | group_key: Mapped[str] = mapped_column(sa.Unicode) 30 | data: Mapped[dict[Any, Any]] = mapped_column(sa.JSON) 31 | 32 | 33 | class DataIssue(Enum): 34 | MEMBER_VOTES_COUNT_MISMATCH = "MEMBER_VOTES_COUNT_MISMATCH" 35 | EMPTY_TITLES = "EMPTY_TITLES" 36 | VOTE_GROUP_NO_MAIN_VOTE = "VOTE_GROUP_NO_MAIN_VOTE" 37 | 38 | 39 | class PipelineStatus(Enum): 40 | SUCCESS = "SUCCESS" 41 | FAILURE = "FAILURE" 42 | DATA_UNAVAILABLE = "DATA_UNAVAILABLE" 43 | DATA_UNCHANGED = "DATA_UNCHANGED" 44 | 45 | 46 | class PipelineRun(Base): 47 | __tablename__ = "pipeline_runs" 48 | 49 | id: Mapped[int] = mapped_column(sa.Integer, primary_key=True, autoincrement=True) 50 | started_at: Mapped[sa.DateTime] = mapped_column(sa.DateTime) 51 | finished_at: Mapped[sa.DateTime] = mapped_column(sa.DateTime) 52 | pipeline: Mapped[str] = mapped_column(sa.Unicode) 53 | status: Mapped[PipelineStatus] = mapped_column(sa.Enum(PipelineStatus)) 54 | checksum: Mapped[str] = mapped_column(sa.Unicode) 55 | -------------------------------------------------------------------------------- /backend/howtheyvote/models/eurovoc.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | 3 | import sqlalchemy as sa 4 | from sqlalchemy.engine import Dialect 5 | from sqlalchemy.types import TypeDecorator 6 | 7 | from ..data import DATA_DIR, DataclassContainer, DeserializableDataclass 8 | from .country import Country 9 | 10 | 11 | class EurovocConceptMeta(type): 12 | def __getitem__(cls, key: str) -> "EurovocConcept": 13 | concept = eurovoc_concepts.get(key) 14 | 15 | if not concept: 16 | raise KeyError() 17 | 18 | return concept 19 | 20 | 21 | @dataclasses.dataclass(frozen=True) 22 | class EurovocConcept(DeserializableDataclass, metaclass=EurovocConceptMeta): 23 | id: str 24 | label: str 25 | alt_labels: list[str] 26 | related_ids: list[str] 27 | broader_ids: list[str] 28 | geo_area_code: str | None 29 | 30 | def __hash__(self) -> int: 31 | return hash(self.id) 32 | 33 | @classmethod 34 | def get(cls, key: str) -> "EurovocConcept | None": 35 | try: 36 | return cls[key] 37 | except KeyError: 38 | return None 39 | 40 | @property 41 | def related(self) -> set["EurovocConcept"]: 42 | return {EurovocConcept[id_] for id_ in self.related_ids} 43 | 44 | @property 45 | def broader(self) -> set["EurovocConcept"]: 46 | return {EurovocConcept[id_] for id_ in self.broader_ids} 47 | 48 | @property 49 | def geo_area(self) -> Country | None: 50 | if not self.geo_area_code: 51 | return None 52 | 53 | return Country.get(self.geo_area_code) 54 | 55 | 56 | eurovoc_concepts = DataclassContainer( 57 | dataclass=EurovocConcept, 58 | file_path=DATA_DIR.joinpath("eurovoc.json"), 59 | key_attr="id", 60 | ) 61 | eurovoc_concepts.load() 62 | 63 | 64 | class EurovocConceptType(TypeDecorator[EurovocConcept]): 65 | impl = sa.Unicode 66 | cache_ok = True 67 | 68 | def process_bind_param(self, value: EurovocConcept | None, dialect: Dialect) -> str | None: 69 | if not value: 70 | return None 71 | 72 | return value.id 73 | 74 | def process_result_value( 75 | self, value: str | None, dialect: Dialect 76 | ) -> EurovocConcept | None: 77 | if not value: 78 | return None 79 | 80 | return eurovoc_concepts.get(value) 81 | -------------------------------------------------------------------------------- /backend/howtheyvote/models/oeil.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | 3 | import sqlalchemy as sa 4 | from sqlalchemy.engine import Dialect 5 | from sqlalchemy.types import TypeDecorator 6 | 7 | from ..data import DATA_DIR, DataclassContainer, DeserializableDataclass 8 | 9 | 10 | class OEILSubjectMeta(type): 11 | def __getitem__(cls, key: str) -> "OEILSubject": 12 | subject = oeil_subjects.get(key) 13 | 14 | if not subject: 15 | raise KeyError() 16 | 17 | return subject 18 | 19 | 20 | @dataclasses.dataclass(frozen=True) 21 | class OEILSubject(DeserializableDataclass, metaclass=OEILSubjectMeta): 22 | code: str 23 | label: str 24 | parent_code: str 25 | 26 | def __hash__(self) -> int: 27 | return hash(self.code) 28 | 29 | @classmethod 30 | def get(cls, key: str) -> "OEILSubject | None": 31 | try: 32 | return cls[key] 33 | except KeyError: 34 | return None 35 | 36 | @property 37 | def parent(self) -> "OEILSubject | None": 38 | if not self.parent_code: 39 | return None 40 | 41 | return OEILSubject[self.parent_code] 42 | 43 | 44 | oeil_subjects = DataclassContainer( 45 | dataclass=OEILSubject, 46 | file_path=DATA_DIR.joinpath("oeil_subjects.json"), 47 | key_attr="code", 48 | ) 49 | oeil_subjects.load() 50 | 51 | 52 | class OEILSubjectType(TypeDecorator[OEILSubject]): 53 | impl = sa.Unicode 54 | cache_ok = True 55 | 56 | def process_bind_param(self, value: OEILSubject | None, dialect: Dialect) -> str | None: 57 | if not value: 58 | return None 59 | 60 | return value.code 61 | 62 | def process_result_value(self, value: str | None, dialect: Dialect) -> OEILSubject | None: 63 | if not value: 64 | return None 65 | 66 | return oeil_subjects.get(value) 67 | -------------------------------------------------------------------------------- /backend/howtheyvote/models/press_release.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | import sqlalchemy as sa 4 | from sqlalchemy.orm import Mapped, mapped_column 5 | 6 | from .common import BaseWithId 7 | from .vote import VotePositionCounts 8 | 9 | 10 | class PressRelease(BaseWithId): 11 | __tablename__ = "press_releases" 12 | 13 | id: Mapped[str] = mapped_column(sa.Unicode, primary_key=True) 14 | published_at: Mapped[datetime.datetime] = mapped_column(sa.DateTime) 15 | title: Mapped[str] = mapped_column(sa.Unicode) 16 | term: Mapped[int] = mapped_column(sa.Integer) 17 | references: Mapped[list[str]] = mapped_column(sa.JSON) 18 | procedure_references: Mapped[list[str]] = mapped_column(sa.JSON) 19 | facts: Mapped[str] = mapped_column(sa.Unicode) 20 | text: Mapped[str] = mapped_column(sa.Unicode) 21 | position_counts: Mapped[list[VotePositionCounts]] = mapped_column( 22 | sa.JSON(none_as_null=True) 23 | ) 24 | -------------------------------------------------------------------------------- /backend/howtheyvote/models/session.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from enum import Enum 3 | 4 | import sqlalchemy as sa 5 | from sqlalchemy.orm import Mapped, mapped_column 6 | 7 | from .common import BaseWithId 8 | 9 | 10 | class PlenarySessionLocation(Enum): 11 | """ 12 | Locality of a plenary session 13 | """ 14 | 15 | SXB = "SXB" 16 | BRU = "BRU" 17 | 18 | 19 | class PlenarySessionStatus(Enum): 20 | CURRENT = "CURRENT" 21 | UPCOMING = "UPCOMING" 22 | PAST = "PAST" 23 | 24 | 25 | class PlenarySession(BaseWithId): 26 | __tablename__ = "plenary_sessions" 27 | 28 | id: Mapped[str] = mapped_column(sa.Unicode, primary_key=True) 29 | term: Mapped[int] = mapped_column(sa.Integer) 30 | start_date: Mapped[datetime.date] = mapped_column(sa.Date) 31 | end_date: Mapped[datetime.date] = mapped_column(sa.Date) 32 | location: Mapped[PlenarySessionLocation | None] = mapped_column( 33 | sa.Enum(PlenarySessionLocation) 34 | ) 35 | 36 | @property 37 | def status(self) -> PlenarySessionStatus: 38 | today = datetime.date.today() 39 | 40 | if self.start_date > today: 41 | return PlenarySessionStatus.UPCOMING 42 | 43 | if self.end_date < today: 44 | return PlenarySessionStatus.PAST 45 | 46 | return PlenarySessionStatus.CURRENT 47 | -------------------------------------------------------------------------------- /backend/howtheyvote/pipelines/__init__.py: -------------------------------------------------------------------------------- 1 | from .common import PipelineResult 2 | from .members import MembersPipeline 3 | from .press import PressPipeline 4 | from .rcv_list import RCVListPipeline 5 | from .sessions import SessionsPipeline 6 | from .vot_list import VOTListPipeline 7 | 8 | __all__ = [ 9 | "PipelineResult", 10 | "RCVListPipeline", 11 | "VOTListPipeline", 12 | "PressPipeline", 13 | "MembersPipeline", 14 | "SessionsPipeline", 15 | ] 16 | -------------------------------------------------------------------------------- /backend/howtheyvote/pipelines/common.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | from abc import ABC, abstractmethod 3 | from dataclasses import dataclass 4 | from typing import Any 5 | 6 | from requests import Response 7 | from structlog import get_logger 8 | 9 | from ..models import PipelineStatus 10 | from ..scrapers import ScrapingError 11 | 12 | log = get_logger(__name__) 13 | 14 | 15 | @dataclass 16 | class PipelineResult: 17 | status: PipelineStatus 18 | checksum: str | None 19 | 20 | 21 | class DataUnavailable(Exception): # noqa: N818 22 | pass 23 | 24 | 25 | class DataUnchanged(Exception): # noqa: N818 26 | pass 27 | 28 | 29 | class BasePipeline(ABC): 30 | last_run_checksum: str | None 31 | checksum: str | None 32 | 33 | def __init__(self, last_run_checksum: str | None = None, **kwargs: Any) -> None: 34 | self.last_run_checksum = last_run_checksum 35 | self.checksum = None 36 | self._log = log.bind(pipeline=type(self).__name__, **kwargs) 37 | 38 | def run(self) -> PipelineResult: 39 | self._log.info("Running pipeline") 40 | 41 | try: 42 | self._run() 43 | status = PipelineStatus.SUCCESS 44 | except DataUnavailable: 45 | status = PipelineStatus.DATA_UNAVAILABLE 46 | except DataUnchanged: 47 | status = PipelineStatus.DATA_UNCHANGED 48 | except ScrapingError: 49 | status = PipelineStatus.FAILURE 50 | self._log.exception("Failed running pipeline") 51 | 52 | return PipelineResult( 53 | status=status, 54 | checksum=self.checksum, 55 | ) 56 | 57 | @abstractmethod 58 | def _run(self) -> None: 59 | raise NotImplementedError 60 | 61 | 62 | def compute_response_checksum(response: Response) -> str: 63 | """Compute the SHA256 hash of the response contents.""" 64 | return hashlib.sha256(response.content).hexdigest() 65 | -------------------------------------------------------------------------------- /backend/howtheyvote/pipelines/vot_list.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from collections.abc import Iterator 3 | 4 | from structlog import get_logger 5 | 6 | from ..models import Vote 7 | from ..scrapers import NoWorkingUrlError, VOTListScraper 8 | from ..store import Aggregator, BulkWriter, index_records, map_vote 9 | from .common import BasePipeline, DataUnavailable 10 | 11 | log = get_logger(__name__) 12 | 13 | 14 | class VOTListPipeline(BasePipeline): 15 | def __init__(self, term: int, date: datetime.date): 16 | super().__init__(term=term, date=date) 17 | self.term = term 18 | self.date = date 19 | self._vote_ids: set[str] = set() 20 | 21 | def _run(self) -> None: 22 | self._scrape_vot_list() 23 | self._index_votes() 24 | 25 | def _scrape_vot_list(self) -> None: 26 | self._log.info("Scraping VOT list") 27 | scraper = VOTListScraper(term=self.term, date=self.date) 28 | 29 | try: 30 | fragments = list(scraper.run()) 31 | except NoWorkingUrlError as exc: 32 | raise DataUnavailable("Pipeline data source is not available") from exc 33 | 34 | writer = BulkWriter() 35 | writer.add(fragments) 36 | writer.flush() 37 | 38 | self._vote_ids = writer.get_touched() 39 | 40 | def _index_votes(self) -> None: 41 | self._log.info("Indexing votes") 42 | index_records(Vote, self._votes()) 43 | 44 | def _votes(self) -> Iterator[Vote]: 45 | aggregator = Aggregator(Vote) 46 | return aggregator.mapped_records(map_func=map_vote, group_keys=self._vote_ids) 47 | -------------------------------------------------------------------------------- /backend/howtheyvote/pushover.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from structlog import get_logger 3 | 4 | from . import config 5 | 6 | API_URL = "https://api.pushover.net/1/messages.json" 7 | log = get_logger(__name__) 8 | 9 | 10 | def send_notification(message: str, title: str | None = None, url: str | None = None) -> None: 11 | if not config.PUSHOVER_API_TOKEN or not config.PUSHOVER_USER_KEY: 12 | log.warn( 13 | "Pushover API token or user key not configured. No push notification will be sent." 14 | ) 15 | return 16 | 17 | # Pushover limits the size of titles and messages, but apparently automatically truncates 18 | # strings longer than that, so we don’t need to handle that on our side: 19 | # https://jcs.org/2023/07/12/api 20 | 21 | try: 22 | response = requests.post( 23 | API_URL, 24 | data={ 25 | "token": config.PUSHOVER_API_TOKEN, 26 | "user": config.PUSHOVER_USER_KEY, 27 | "title": title, 28 | "message": message, 29 | "url": url, 30 | }, 31 | ) 32 | response.raise_for_status() 33 | except requests.RequestException: 34 | # Sending push notifications isn't critical, so we don't want request exceptions 35 | # to bubble up, logging them is enough. 36 | log.exception( 37 | "Failed to send push notification", 38 | message=message, 39 | title=title, 40 | url=url, 41 | ) 42 | -------------------------------------------------------------------------------- /backend/howtheyvote/query.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from collections.abc import Iterable 3 | 4 | import sqlalchemy as sa 5 | from sqlalchemy import ColumnElement, Exists, exists, func 6 | 7 | from .models import BaseWithId, Fragment, Member, PlenarySession 8 | 9 | 10 | def member_has_term(term: int) -> Exists: 11 | """Returns an expression that can be used to select only members for the given term.""" 12 | exp = func.json_each(Member.terms).table_valued("value") 13 | return exists().select_from(exp).where(exp.c.value == term) 14 | 15 | 16 | def member_active_at(date: datetime.date) -> Exists: 17 | """Returns an expression that can be used to select only members with an active group 18 | membership on the given date.""" 19 | exp = func.json_each(Member.group_memberships).table_valued("value") 20 | start_date = func.date(func.json_extract(exp.c.value, "$.start_date")) 21 | end_date = func.date(func.json_extract(exp.c.value, "$.end_date")) 22 | 23 | return ( 24 | exists() 25 | .select_from(exp) 26 | .where( 27 | sa.and_( 28 | start_date <= date, 29 | sa.or_(end_date == None, end_date >= date), # noqa: E711 30 | ) 31 | ) 32 | ) 33 | 34 | 35 | def session_is_current_at(date: datetime.date) -> ColumnElement[bool]: 36 | """Returns an expression that can be used to select the plenary session for 37 | the given date.""" 38 | return sa.and_( 39 | func.date(PlenarySession.start_date) <= func.date(date), 40 | func.date(PlenarySession.end_date) >= func.date(date), 41 | ) 42 | 43 | 44 | def fragments_for_records(records: Iterable[BaseWithId | None]) -> ColumnElement[bool]: 45 | """Returns an expression that can be used to select fragments for the given records.""" 46 | filters: list[ColumnElement[bool]] = [] 47 | 48 | for record in records: 49 | if not record: 50 | continue 51 | 52 | filters.append( 53 | sa.and_( 54 | Fragment.model == record.__class__.__name__, 55 | Fragment.group_key == record.id, 56 | ), 57 | ) 58 | 59 | return sa.or_(*filters) 60 | -------------------------------------------------------------------------------- /backend/howtheyvote/scrapers/__init__.py: -------------------------------------------------------------------------------- 1 | from .common import NoWorkingUrlError, RequestCache, ScrapingError 2 | from .members import MemberGroupsScraper, MemberInfoScraper, MembersScraper 3 | from .press_releases import ( 4 | PressReleaseScraper, 5 | PressReleasesIndexScraper, 6 | PressReleasesRSSScraper, 7 | ) 8 | from .sessions import CalendarSessionsScraper, ODPSessionScraper 9 | from .votes import ( 10 | DocumentScraper, 11 | EurlexDocumentScraper, 12 | EurlexProcedureScraper, 13 | ProcedureScraper, 14 | RCVListEnglishScraper, 15 | RCVListScraper, 16 | VOTListScraper, 17 | ) 18 | 19 | __all__ = [ 20 | "ScrapingError", 21 | "NoWorkingUrlError", 22 | "RequestCache", 23 | "MembersScraper", 24 | "MemberGroupsScraper", 25 | "MemberInfoScraper", 26 | "CalendarSessionsScraper", 27 | "ODPSessionScraper", 28 | "ProcedureScraper", 29 | "RCVListScraper", 30 | "RCVListEnglishScraper", 31 | "VOTListScraper", 32 | "DocumentScraper", 33 | "EurlexDocumentScraper", 34 | "EurlexProcedureScraper", 35 | "PressReleasesIndexScraper", 36 | "PressReleasesRSSScraper", 37 | "PressReleaseScraper", 38 | ] 39 | -------------------------------------------------------------------------------- /backend/howtheyvote/sharepics/__init__.py: -------------------------------------------------------------------------------- 1 | import base64 2 | from urllib.parse import urljoin 3 | 4 | from structlog import get_logger 5 | 6 | from .. import config 7 | from .cdp import Client 8 | 9 | log = get_logger(__name__) 10 | 11 | 12 | def generate_vote_sharepic(vote_id: int) -> bytes: 13 | """Generate a share picture for the given vote.""" 14 | url = urljoin(config.FRONTEND_PRIVATE_URL, f"/votes/{vote_id}/sharepic") 15 | return capture_screenshot(url) 16 | 17 | 18 | def capture_screenshot(url: str) -> bytes: 19 | """Uses a headless Chromium browser to take a screenshot of the given `url` 20 | and writes the resulting PNG file to `path`.""" 21 | log.info("Capturing screenshot", url=url) 22 | 23 | client = Client(host="chromium", port=9222, timeout=5) 24 | 25 | with client.connect() as session: 26 | session.send( 27 | "Emulation.setDeviceMetricsOverride", 28 | { 29 | "width": 600, 30 | "height": 315, 31 | "deviceScaleFactor": 2, 32 | "mobile": False, 33 | }, 34 | ) 35 | session.send("Page.enable") 36 | session.send("Page.navigate", {"url": url, "format": "png"}) 37 | session.wait_event("Page.loadEventFired") 38 | res = session.send("Page.captureScreenshot") 39 | 40 | return base64.b64decode(res["data"]) 41 | -------------------------------------------------------------------------------- /backend/howtheyvote/store/__init__.py: -------------------------------------------------------------------------------- 1 | from .aggregator import Aggregator, CompositeRecord, MapFunc 2 | from .index import index_db, index_records, index_search 3 | from .mappings import ( 4 | map_member, 5 | map_plenary_session, 6 | map_press_release, 7 | map_vote, 8 | map_vote_group, 9 | ) 10 | from .writer import BulkWriter 11 | 12 | __all__ = [ 13 | "BulkWriter", 14 | "Aggregator", 15 | "CompositeRecord", 16 | "MapFunc", 17 | "index_records", 18 | "map_member", 19 | "map_plenary_session", 20 | "map_vote", 21 | "map_vote_group", 22 | "map_press_release", 23 | ] 24 | -------------------------------------------------------------------------------- /backend/howtheyvote/vote_stats.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | from collections.abc import Callable 3 | from typing import TypeVar 4 | 5 | from .models import MemberVote, VotePositionCounts 6 | 7 | GroupBy = TypeVar("GroupBy") 8 | 9 | 10 | def count_vote_positions(member_votes: list[MemberVote]) -> VotePositionCounts: 11 | """Calculates the number of MEPs that voted for/against/abstention for the given 12 | list of member votes.""" 13 | counts: VotePositionCounts = { 14 | "FOR": 0, 15 | "AGAINST": 0, 16 | "ABSTENTION": 0, 17 | "DID_NOT_VOTE": 0, 18 | } 19 | 20 | for member_vote in member_votes: 21 | counts[member_vote.position.value] += 1 22 | 23 | return counts 24 | 25 | 26 | def count_vote_positions_by_group( 27 | member_votes: list[MemberVote], 28 | group_by: Callable[[MemberVote], GroupBy], 29 | ) -> dict[GroupBy, VotePositionCounts]: 30 | """Calculates the number of MEPs that voted for/against/abstention for the given 31 | list of member votes for each group. The second argument is a function that returns 32 | the group for each member vote.""" 33 | grouped_member_votes: dict[GroupBy, list[MemberVote]] = defaultdict(list) 34 | 35 | for member_vote in member_votes: 36 | group = group_by(member_vote) 37 | grouped_member_votes[group].append(member_vote) 38 | 39 | # Sort by group size 40 | grouped_member_votes = dict( 41 | sorted( 42 | grouped_member_votes.items(), 43 | key=lambda item: len(item[1]), 44 | reverse=True, 45 | ) 46 | ) 47 | 48 | grouped_counts: dict[GroupBy, VotePositionCounts] = {} 49 | 50 | for group, member_votes in grouped_member_votes.items(): 51 | grouped_counts[group] = count_vote_positions(member_votes) 52 | 53 | return grouped_counts 54 | -------------------------------------------------------------------------------- /backend/poetry.toml: -------------------------------------------------------------------------------- 1 | [virtualenvs] 2 | in-project = true 3 | 4 | [virtualenvs.options] 5 | # Xapian is installed as a system package, make it accessible in venv 6 | system-site-packages = true 7 | -------------------------------------------------------------------------------- /backend/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "howtheyvote" 3 | description = "" 4 | version = "0.1.0" 5 | requires-python = ">=3.12" 6 | dependencies = [ 7 | "requests>=2.32.3", 8 | "unidecode>=1.3.8", 9 | "beautifulsoup4>=4.12.3", 10 | "lxml>=5.3.0", 11 | "alembic>=1.14.1", 12 | "click>=8.1.8", 13 | "structlog>=25.1.0", 14 | "flask>=3.1.0", 15 | "cachetools>=5.5.1", 16 | "pillow>=11.1.0", 17 | "gunicorn>=23.0.0", 18 | "apispec>=6.8.1", 19 | "apispec-webframeworks>=1.2.0", 20 | "prometheus-client>=0.21.1", 21 | "sqlalchemy>=2.0.37", 22 | "schedule>=1.2.2", 23 | "pytz>=2025.1", 24 | "websocket-client>=1.8.0", 25 | "tabulate>=0.9.0", 26 | ] 27 | 28 | [tool.poetry] 29 | packages = [ 30 | { include = "./howtheyvote" } 31 | ] 32 | 33 | [project.scripts] 34 | htv = "howtheyvote.cli:cli" 35 | 36 | [tool.poetry.group.dev.dependencies] 37 | coverage = "^7.8" 38 | pytest = "^8.4.0" 39 | pytest-cov = "^6.1.1" 40 | mypy = "^1.16.0" 41 | ruff = "^0.11.13" 42 | types-pillow = "^10.2.0.20240822" 43 | types-requests = "^2.32.0.20250602" 44 | types-beautifulsoup4 = "^4.12.0.20250516" 45 | types-cachetools = "^6.0.0.20250525" 46 | setuptools = "^80.9.0" 47 | pytest-env = "^1.1.5" 48 | responses = "^0.25.7" 49 | time-machine = "^2.16.0" 50 | types-tabulate = "^0.9.0.20241207" 51 | pytest-mock = "^3.14.1" 52 | 53 | [tool.ruff] 54 | line-length = 95 55 | 56 | [tool.ruff.lint] 57 | select = ["E", "F", "W", "I", "N", "B", "UP", "UP006"] 58 | 59 | [tool.ruff.lint.per-file-ignores] 60 | "tests/*" = ["E501"] 61 | "__init__.py" = ["F401"] 62 | 63 | [tool.mypy] 64 | exclude = ["alembic", ".venv"] 65 | mypy_path = "$MYPY_CONFIG_FILE_DIR/stubs" 66 | 67 | [tool.pytest.ini_options] 68 | env = [ 69 | "D:HTV_BACKEND_DATABASE_URI=sqlite:////howtheyvote/database/database.test.sqlite3", 70 | "HTV_BACKEND_PUBLIC_URL=https://example.org/api", 71 | "HTV_SEARCH_INDEX_PREFIX=test", 72 | # Disable pushover notifications in test env even when an API token and user key 73 | # is configured in development environment 74 | "PUSHOVER_API_TOKEN=", 75 | "PUSHOVER_USER_KEY=", 76 | ] 77 | markers = [ 78 | "always_mock_requests: Always mock HTTP requests, even when request mocks are disabled globally" 79 | ] 80 | addopts = [ 81 | "--import-mode=importlib", 82 | ] 83 | pythonpath = [ 84 | "." 85 | ] -------------------------------------------------------------------------------- /backend/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HowTheyVote/howtheyvote/76e0b997aab9071504b5121e372e364bbc3b830e/backend/tests/__init__.py -------------------------------------------------------------------------------- /backend/tests/analysis/test_helpers.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from howtheyvote.analysis.helpers import extract_vote_results, parse_int 4 | 5 | 6 | def test_parse_int(): 7 | assert parse_int("1") == 1 8 | assert parse_int("one") == 1 9 | assert parse_int("One") == 1 10 | assert parse_int("12") == 12 11 | assert parse_int("twelve") == 12 12 | 13 | with pytest.raises(ValueError): 14 | assert parse_int("") 15 | 16 | with pytest.raises(ValueError): 17 | assert parse_int("invalid") 18 | 19 | 20 | def test_extract_vote_results(): 21 | for example in [ 22 | "400 votes in favour, 63 against with 81 abstentions", 23 | "400 votes in favor, 63 against with 81 abstentions", 24 | "400 votes in favour, 63 against, with 81 abstentions", 25 | "400 votes in favour, 63 against and 81 abstentions", 26 | "400 votes in favour, 63 against, and 81 abstentions", 27 | "400 MEPs voted in favour, 63 against and 81 abstained", 28 | "400 MEPs voted in favour, 63 against, and 81 abstained", 29 | "400 MEPs voted in favor, 63 against, and 81 abstained", 30 | ]: 31 | expected = [{"FOR": 400, "AGAINST": 63, "ABSTENTION": 81, "DID_NOT_VOTE": 0}] 32 | actual = extract_vote_results(example) 33 | assert actual == expected 34 | 35 | expected = [{"FOR": 500, "AGAINST": 143, "ABSTENTION": 9, "DID_NOT_VOTE": 0}] 36 | actual = extract_vote_results("500 votes in favour, 143 against, and nine abstentions") 37 | assert actual == expected 38 | 39 | expected = [{"FOR": 1, "AGAINST": 1, "ABSTENTION": 1, "DID_NOT_VOTE": 0}] 40 | actual = extract_vote_results("one vote in favour, one against, and one abstention") 41 | assert actual == expected 42 | 43 | expected = [{"FOR": 455, "AGAINST": 85, "ABSTENTION": 105, "DID_NOT_VOTE": 0}] 44 | actual = extract_vote_results("adopted with 455 votes to 85 and 105 abstentions") 45 | assert actual == expected 46 | 47 | expected = [{"FOR": 531, "AGAINST": 69, "ABSTENTION": 17, "DID_NOT_VOTE": 0}] 48 | actual = extract_vote_results( 49 | "With 531 votes for, 69 against and 17 abstentions, MEPs supported the Commission proposal" 50 | ) 51 | assert actual == expected 52 | -------------------------------------------------------------------------------- /backend/tests/helpers.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import Any 3 | 4 | from sqlalchemy.orm import DeclarativeBase 5 | 6 | 7 | def record_to_dict(record: DeclarativeBase) -> dict[str, Any]: 8 | return {c.name: getattr(record, c.name) for c in record.__table__.columns} 9 | 10 | 11 | def load_fixture(path: str) -> str: 12 | base = Path(__file__).resolve().parent 13 | return base.joinpath(path).read_text() 14 | -------------------------------------------------------------------------------- /backend/tests/models/test_country.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from sqlalchemy import Column, MetaData, Table, create_engine, select, text 3 | 4 | from howtheyvote.models import Country, CountryType 5 | 6 | 7 | def test_country_getitem(): 8 | austria = Country["AUT"] 9 | assert isinstance(austria, Country) 10 | assert austria.code == "AUT" 11 | 12 | with pytest.raises(KeyError): 13 | Country["invalid"] 14 | 15 | 16 | def test_country_get(): 17 | austria = Country.get("AUT") 18 | assert isinstance(austria, Country) 19 | assert austria.code == "AUT" 20 | 21 | invalid = Country.get("invalid") 22 | assert invalid is None 23 | 24 | 25 | def test_country_from_label(): 26 | assert Country.from_label("Austria") == Country["AUT"] 27 | 28 | 29 | def test_country_from_label_alternates(): 30 | assert Country.from_label("Czechia") == Country["CZE"] 31 | assert Country.from_label("Czech Republic") == Country["CZE"] 32 | 33 | 34 | def test_country_from_label_fuzzy(): 35 | # Sanity check: exact match 36 | assert Country.from_label("Kosovo*") == Country["XKX"] 37 | 38 | # Sanity check: By default, only exact matches are returned 39 | assert Country.from_label("Kosovo") is None 40 | assert Country.from_label("Kosovo under UNSCR 1244/1999") is None 41 | 42 | # If fuzzy=True, countries that start with the same substring are matched, too. 43 | assert Country.from_label("Kosovo under UNSCR 1244/1999", fuzzy=True) == Country["XKX"] 44 | assert Country.from_label("Kosovo", fuzzy=True) == Country["XKX"] 45 | 46 | 47 | def test_country_type(): 48 | engine = create_engine("sqlite://") 49 | metadata = MetaData() 50 | 51 | table = Table( 52 | "countries", 53 | metadata, 54 | Column("country", CountryType()), 55 | ) 56 | 57 | with engine.connect() as connection: 58 | metadata.create_all(connection) 59 | stmt = table.insert().values(country=Country["FRA"]) 60 | connection.execute(stmt) 61 | connection.commit() 62 | 63 | unprocessed = connection.execute(text("SELECT country from countries")).scalar() 64 | assert unprocessed == "FRA" 65 | 66 | processed = connection.execute(select(table)).scalar() 67 | assert isinstance(processed, Country) 68 | assert processed.code == "FRA" 69 | -------------------------------------------------------------------------------- /backend/tests/models/test_eurovoc.py: -------------------------------------------------------------------------------- 1 | from howtheyvote.models.country import Country 2 | from howtheyvote.models.eurovoc import EurovocConcept 3 | 4 | 5 | def test_eurovoc_concept_related(): 6 | concept = EurovocConcept["3030"] 7 | related = concept.related 8 | 9 | expected = {EurovocConcept["3740"], EurovocConcept["3293"], EurovocConcept["c_5a195ffd"]} 10 | assert related == expected 11 | 12 | 13 | def test_eurovoc_concept_broader(): 14 | concept = EurovocConcept["c_f7430876"] 15 | assert concept.broader == {EurovocConcept["3074"]} 16 | 17 | 18 | def test_eurovoc_concept_geo_area(): 19 | concept = EurovocConcept["245"] 20 | assert concept.geo_area_code is None 21 | assert concept.geo_area is None 22 | 23 | concept = EurovocConcept["3774"] 24 | assert concept.geo_area_code == "GBR" 25 | assert concept.geo_area == Country["GBR"] 26 | -------------------------------------------------------------------------------- /backend/tests/models/test_group.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from howtheyvote.models import Group 4 | 5 | 6 | def test_group_getitem(): 7 | greens = Group["GREEN_EFA"] 8 | assert isinstance(greens, Group) 9 | assert greens.code == "GREEN_EFA" 10 | 11 | with pytest.raises(KeyError): 12 | Group["invalid"] 13 | 14 | 15 | def test_group_get(): 16 | greens = Group.get("GREEN_EFA") 17 | assert isinstance(greens, Group) 18 | assert greens.code == "GREEN_EFA" 19 | 20 | invalid = Group.get("invalid") 21 | assert invalid is None 22 | 23 | 24 | def test_group_from_label(): 25 | assert Group.from_label("Group of the Greens/European Free Alliance") == Group["GREEN_EFA"] 26 | 27 | 28 | def test_group_from_label_short(): 29 | assert Group.from_label("Greens/European Free Alliance") == Group["GREEN_EFA"] 30 | 31 | 32 | def test_group_from_label_case_insensitive(): 33 | assert Group.from_label("greens/european free alliance") == Group["GREEN_EFA"] 34 | 35 | 36 | def test_group_from_label_normalized(): 37 | # Note that this doesn't use a proper apostrophe in "People's" 38 | epp = Group.from_label("Group of the European People's Party (Christian Democrats)") 39 | assert epp == Group["EPP"] 40 | 41 | guengl = Group.from_label("Group of the European United Left - Nordic Green Left") 42 | assert guengl == Group["GUE_NGL"] 43 | -------------------------------------------------------------------------------- /backend/tests/models/test_member.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from sqlalchemy import Column, MetaData, Table, create_engine, select, text 4 | 5 | from howtheyvote.models import Group, GroupMembership 6 | from howtheyvote.models.member import GroupMembershipType 7 | from howtheyvote.models.types import ListType 8 | 9 | 10 | def test_group_membership_type(): 11 | engine = create_engine("sqlite://") 12 | metadata = MetaData() 13 | 14 | table = Table( 15 | "members", 16 | metadata, 17 | Column("group_memberships", ListType(GroupMembershipType)), 18 | ) 19 | 20 | group_memberships = [ 21 | GroupMembership( 22 | term=9, 23 | group=Group["EPP"], 24 | start_date=datetime.date(2023, 1, 1), 25 | end_date=None, 26 | ) 27 | ] 28 | 29 | with engine.connect() as connection: 30 | metadata.create_all(connection) 31 | stmt = table.insert().values(group_memberships=group_memberships) 32 | connection.execute(stmt) 33 | connection.commit() 34 | 35 | stmt = text("SELECT group_memberships from members") 36 | unprocessed = connection.execute(stmt).scalar() 37 | assert ( 38 | unprocessed 39 | == '[{"term": 9, "start_date": "2023-01-01", "end_date": null, "group": "EPP"}]' 40 | ) 41 | 42 | stmt = select(table) 43 | processed = connection.execute(stmt).scalar() 44 | assert processed == group_memberships 45 | -------------------------------------------------------------------------------- /backend/tests/models/test_vote.py: -------------------------------------------------------------------------------- 1 | from howtheyvote.models import Vote 2 | 3 | 4 | def test_vote_display_title(): 5 | # Only RCV title present 6 | vote = Vote(title="Title from RCV list", procedure_title=None) 7 | assert vote.display_title == "Title from RCV list" 8 | 9 | # Only procedure title present 10 | vote = Vote(title=None, procedure_title="Title from procedure file") 11 | assert vote.display_title == "Title from procedure file" 12 | 13 | # RCV title and procedure title present 14 | vote = Vote(title="Title from RCV list", procedure_title="Title from procedure file") 15 | assert vote.display_title == "Title from RCV list" 16 | 17 | # If the RCV title is very long, prefer the procedure title if it is shorter 18 | vote = Vote( 19 | title="Establishment of 'Eurodac' for the comparison of fingerprints for the effective application of Regulation (EU) No 604/2013, for identifying an illegally staying third-country national or stateless person and on requests for the comparison with Eurodac data by Member States' law enforcement authorities and Europol for law enforcement purposes (recast)", 20 | procedure_title="Eurodac Regulation", 21 | ) 22 | assert vote.display_title == "Eurodac Regulation" 23 | 24 | # If the RCV title is very long, but the procedure title is even longer, prefer the RCV title 25 | vote = Vote( 26 | title="Setting up a special committee on foreign interference in all democratic processes in the European Union, including disinformation", 27 | procedure_title="Decision on setting up a special committee on foreign interference in all democratic processes in the European Union, including disinformation (INGE 2), and defining its responsibilities, numerical strength and term of office", 28 | ) 29 | assert ( 30 | vote.display_title 31 | == "Setting up a special committee on foreign interference in all democratic processes in the European Union, including disinformation" 32 | ) 33 | -------------------------------------------------------------------------------- /backend/tests/scrapers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HowTheyVote/howtheyvote/76e0b997aab9071504b5121e372e364bbc3b830e/backend/tests/scrapers/__init__.py -------------------------------------------------------------------------------- /backend/tests/scrapers/data/votes/rcv_list_incorrect_pv-9-2020-07-23-rcv-fr.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | B9-0229/2020 - § 1/1 6 | 7 | 8 | 9 | Aguilar 10 | 11 | 12 | 13 | 14 | de Graaff 15 | 16 | 17 | 18 | 19 | Kolakušić 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /backend/tests/scrapers/data/votes/rcv_list_p9-pv(2020)07-23(rcv)_xc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | B9-0229/2020 - § 1/1 7 | 8 | 9 | 10 | Aguilar 11 | 12 | 13 | 14 | 15 | de Graaff 16 | 17 | 18 | 19 | 20 | Kolakušić 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /backend/tests/scrapers/data/votes/rcv_list_pv-9-2019-07-15-rcv-fr.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Mardi - demande du groupe GUE/NGL 15/07/2019 17:09:37.000 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /backend/tests/scrapers/data/votes/rcv_list_pv-9-2020-07-23-rcv-fr.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | B9-0229/2020 - § 1/1 6 | 7 | 8 | 9 | Aguilar 10 | 11 | 12 | 13 | 14 | de Graaff 15 | 16 | 17 | 18 | 19 | Kolakušić 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /backend/tests/scrapers/data/votes/rcv_list_pv-9-2020-09-15-rcv-fr.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | A9-0148/2020 - Nikos Androulakis - Am 1-9, 11-16, 18-52, 55-87 6 | 7 | 8 | 9 | Adinolfi Matteo 10 | 11 | 12 | Adinolfi Isabella 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /backend/tests/scrapers/data/votes/rcv_list_pv-9-2023-12-12-rcv-fr.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | The political disqualifications in Venezuela – 5 | RC-B9-0331/2023 – Motion for a resolution (as a whole) 6 | 7 | 8 | 9 | Glueck 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /backend/tests/scrapers/test_common.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import requests 3 | from cachetools import LRUCache 4 | 5 | from howtheyvote.scrapers.common import get_url 6 | 7 | 8 | @pytest.mark.always_mock_requests 9 | def test_get_url(responses): 10 | mock = responses.get("https://example.org", body="Hello World") 11 | 12 | response = get_url("https://example.org", headers={}) 13 | assert response and response.status_code == 200 14 | assert mock.call_count == 1 15 | 16 | response = get_url("https://example.org", headers={}) 17 | assert response and response.status_code == 200 18 | assert mock.call_count == 2 19 | 20 | 21 | @pytest.mark.always_mock_requests 22 | def test_get_url_timeout(responses): 23 | responses.get("https://example.org", body=requests.RequestException()) 24 | 25 | response = get_url("https://example.org", headers={}) 26 | assert response is None 27 | 28 | 29 | @pytest.mark.always_mock_requests 30 | def test_get_url_cache(responses): 31 | mock = responses.get("https://example.org", body="Hello World") 32 | cache = LRUCache(maxsize=10) 33 | 34 | response = get_url("https://example.org", headers={}, request_cache=cache) 35 | assert response and response.status_code == 200 36 | assert mock.call_count == 1 37 | 38 | response = get_url("https://example.org", headers={}, request_cache=cache) 39 | assert response and response.status_code == 200 40 | assert mock.call_count == 1 41 | 42 | 43 | @pytest.mark.always_mock_requests 44 | def test_get_url_retries(responses): 45 | error = responses.get("https://example.org", body="Internal Server Error", status=500) 46 | success = responses.get("https://example.org", body="Hello World") 47 | 48 | response = get_url("https://example.org", headers={}, max_retries=1) 49 | assert response and response.status_code == 200 50 | assert error.call_count == 1 51 | assert success.call_count == 1 52 | -------------------------------------------------------------------------------- /backend/tests/scrapers/test_sessions.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from howtheyvote.models import PlenarySessionLocation 4 | from howtheyvote.scrapers import ODPSessionScraper 5 | 6 | from ..helpers import load_fixture 7 | 8 | 9 | def test_odp_session_scraper(responses): 10 | responses.get( 11 | "https://data.europarl.europa.eu/api/v1/meetings/MTG-PL-2024-07-16", 12 | body=load_fixture("scrapers/data/sessions/odp_mtg-pl-2024-07-16.xml"), 13 | ) 14 | 15 | scraper = ODPSessionScraper(start_date=datetime.date(2024, 7, 16)) 16 | fragment = scraper.run() 17 | 18 | assert fragment.data["location"] == PlenarySessionLocation.SXB 19 | -------------------------------------------------------------------------------- /backend/tests/store/test_mappings.py: -------------------------------------------------------------------------------- 1 | from howtheyvote.models import Country, EurovocConcept 2 | from howtheyvote.store import CompositeRecord 3 | from howtheyvote.store.mappings import map_vote 4 | 5 | 6 | def test_map_vote_eurovoc_concepts_deduplication(): 7 | record = CompositeRecord( 8 | group_key="12345", 9 | data={ 10 | "timestamp": ["2024-01-01T00:00:00Z"], 11 | "eurovoc_concepts": [["4514", "5585", "4514"]], 12 | "member_votes": [[]], 13 | }, 14 | ) 15 | 16 | vote = map_vote(record) 17 | expected = {EurovocConcept["4514"], EurovocConcept["5585"]} 18 | 19 | assert len(vote.eurovoc_concepts) == 2 20 | assert set(vote.eurovoc_concepts) == expected 21 | 22 | 23 | def test_map_vote_eurovoc_concepts_geo_areas(): 24 | record = CompositeRecord( 25 | group_key="12345", 26 | data={ 27 | "timestamp": ["2024-01-01T00:00:00Z"], 28 | "geo_areas": [["XXI"], ["GBR"]], 29 | "member_votes": [[]], 30 | }, 31 | ) 32 | 33 | vote = map_vote(record) 34 | expected = {Country["XXI"], Country["GBR"]} 35 | 36 | assert len(vote.geo_areas) == 2 37 | assert set(vote.geo_areas) == expected 38 | 39 | 40 | def test_map_vote_geo_areas_dedpuplication(): 41 | record = CompositeRecord( 42 | group_key="12345", 43 | data={ 44 | "timestamp": ["2024-01-01T00:00:00Z"], 45 | "geo_areas": [["BEL", "BEL"]], 46 | "member_votes": [[]], 47 | }, 48 | ) 49 | 50 | vote = map_vote(record) 51 | 52 | assert len(vote.geo_areas) == 1 53 | assert vote.geo_areas == {Country["BEL"]} 54 | -------------------------------------------------------------------------------- /backend/tests/test_pushover.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from responses import matchers 3 | 4 | from howtheyvote.pushover import send_notification 5 | 6 | 7 | def test_send_notification(responses, mocker): 8 | mocker.patch("howtheyvote.config.PUSHOVER_API_TOKEN", "api-token-123") 9 | mocker.patch("howtheyvote.config.PUSHOVER_USER_KEY", "user-key-123") 10 | 11 | res = responses.post( 12 | url="https://api.pushover.net/1/messages.json", 13 | match=[ 14 | matchers.urlencoded_params_matcher( 15 | { 16 | "token": "api-token-123", 17 | "user": "user-key-123", 18 | "title": "Title", 19 | "message": "Message", 20 | "url": "https://howtheyvote.eu", 21 | } 22 | ) 23 | ], 24 | body="1", 25 | ) 26 | 27 | send_notification( 28 | title="Title", 29 | message="Message", 30 | url="https://howtheyvote.eu", 31 | ) 32 | 33 | assert res.call_count == 1 34 | 35 | 36 | def test_send_notification_connection_error(responses, mocker, logs): 37 | mocker.patch("howtheyvote.config.PUSHOVER_API_TOKEN", "api-token-123") 38 | mocker.patch("howtheyvote.config.PUSHOVER_USER_KEY", "user-key-123") 39 | 40 | responses.post( 41 | url="https://api.pushover.net/1/messages.json", 42 | body=requests.ConnectionError("Could not connect to api.pushover.net"), 43 | ) 44 | 45 | send_notification( 46 | title="Title", 47 | message="Message", 48 | url="https://howtheyvote.eu", 49 | ) 50 | 51 | assert len(logs) == 1 52 | assert logs[0]["event"] == "Failed to send push notification" 53 | assert logs[0]["log_level"] == "error" 54 | assert logs[0]["title"] == "Title" 55 | assert logs[0]["message"] == "Message" 56 | assert logs[0]["url"] == "https://howtheyvote.eu" 57 | 58 | 59 | def test_send_notification_no_api_creds(logs): 60 | send_notification(message="Lorem ipsum") 61 | assert len(logs) == 1 62 | assert ( 63 | logs[0]["event"] 64 | == "Pushover API token or user key not configured. No push notification will be sent." 65 | ) 66 | -------------------------------------------------------------------------------- /backend/tests/test_vote_stats.py: -------------------------------------------------------------------------------- 1 | from howtheyvote.models import MemberVote, VotePosition 2 | from howtheyvote.vote_stats import count_vote_positions, count_vote_positions_by_group 3 | 4 | 5 | def test_count_vote_positions(): 6 | assert count_vote_positions([]) == { 7 | "FOR": 0, 8 | "AGAINST": 0, 9 | "ABSTENTION": 0, 10 | "DID_NOT_VOTE": 0, 11 | } 12 | 13 | member_votes = [ 14 | MemberVote(web_id=1, position=VotePosition.FOR), 15 | MemberVote(web_id=2, position=VotePosition.FOR), 16 | MemberVote(web_id=3, position=VotePosition.DID_NOT_VOTE), 17 | MemberVote(web_id=4, position=VotePosition.ABSTENTION), 18 | ] 19 | 20 | assert count_vote_positions(member_votes) == { 21 | "FOR": 2, 22 | "AGAINST": 0, 23 | "ABSTENTION": 1, 24 | "DID_NOT_VOTE": 1, 25 | } 26 | 27 | 28 | def test_count_vote_positions_by_group(): 29 | member_countries = { 30 | 1: "FRA", 31 | 2: "DEU", 32 | 3: "ITA", 33 | 4: "DEU", 34 | } 35 | 36 | member_votes = [ 37 | MemberVote(web_id=1, position=VotePosition.FOR), 38 | MemberVote(web_id=2, position=VotePosition.FOR), 39 | MemberVote(web_id=3, position=VotePosition.ABSTENTION), 40 | MemberVote(web_id=4, position=VotePosition.DID_NOT_VOTE), 41 | ] 42 | 43 | grouped_counts = count_vote_positions_by_group( 44 | member_votes=member_votes, group_by=lambda mv: member_countries[mv.web_id] 45 | ) 46 | 47 | expected = { 48 | "DEU": { 49 | "FOR": 1, 50 | "AGAINST": 0, 51 | "ABSTENTION": 0, 52 | "DID_NOT_VOTE": 1, 53 | }, 54 | "FRA": { 55 | "FOR": 1, 56 | "AGAINST": 0, 57 | "ABSTENTION": 0, 58 | "DID_NOT_VOTE": 0, 59 | }, 60 | "ITA": { 61 | "FOR": 0, 62 | "AGAINST": 0, 63 | "ABSTENTION": 1, 64 | "DID_NOT_VOTE": 0, 65 | }, 66 | } 67 | 68 | assert grouped_counts == expected 69 | assert list(grouped_counts.keys()) == ["DEU", "FRA", "ITA"] # Sorted by group size 70 | -------------------------------------------------------------------------------- /docker-compose.override.yml: -------------------------------------------------------------------------------- 1 | services: 2 | frontend: 3 | build: 4 | context: "./frontend" 5 | command: "make dev" 6 | ports: 7 | - "8000:8000" 8 | volumes: 9 | - "./frontend:/howtheyvote/frontend" 10 | stdin_open: true 11 | tty: true 12 | 13 | backend: 14 | build: 15 | context: "./backend" 16 | command: "make dev" 17 | volumes: 18 | - "./backend:/howtheyvote/backend" 19 | stdin_open: true 20 | tty: true 21 | 22 | worker: 23 | build: 24 | context: "./backend" 25 | volumes: 26 | - "./backend:/howtheyvote/backend" 27 | stdin_open: true 28 | tty: true 29 | 30 | caddy: 31 | stdin_open: true 32 | tty: true 33 | environment: 34 | CADDY_SITE_ADDRESS: "localhost" 35 | 36 | chromium: 37 | ports: 38 | - "9222:9222" 39 | -------------------------------------------------------------------------------- /docs/funders.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HowTheyVote/howtheyvote/76e0b997aab9071504b5121e372e364bbc3b830e/docs/funders.png -------------------------------------------------------------------------------- /docs/pipelines.md: -------------------------------------------------------------------------------- 1 | # Data Pipelines 2 | 3 | ## Once a week 4 | 5 | ```mermaid 6 | flowchart TB 7 | 8 | subgraph "SessionsPipeline" 9 | direction TB 10 | PlenarySesssionsScraper --> ODPSessionScraper 11 | end 12 | 13 | subgraph "MembersPipeline" 14 | direction TB 15 | MembersScraper --> MemberInfoScraper 16 | MemberInfoScraper --> MemberGroupsScraper 17 | end 18 | ``` 19 | 20 | ## Every day during a session 21 | 22 | ```mermaid 23 | flowchart TB 24 | subgraph "RCVListPipeline" 25 | direction TB 26 | RCVListScraper --> ProcedureScraper 27 | ProcedureScraper --> MainVoteAnalyzer 28 | MainVoteAnalyzer --> VoteGroupsAnalyzer 29 | VoteGroupsAnalyzer --> VoteDataIssuesAnalyzer 30 | VoteDataIssuesAnalyzer --> VoteGroupsDataIssuesAnalyzer 31 | end 32 | 33 | subgraph "PressPipeline" 34 | direction TB 35 | PressReleasesRSSScraper --> PressReleasesIndexScraper 36 | PressReleasesIndexScraper --> PressReleaseScraper 37 | PressReleaseScraper --> FeaturedVotesAnalyzer 38 | end 39 | ``` 40 | -------------------------------------------------------------------------------- /frontend/.dockerignore: -------------------------------------------------------------------------------- 1 | **/dist 2 | **/node_modules 3 | -------------------------------------------------------------------------------- /frontend/.editorconfig: -------------------------------------------------------------------------------- 1 | [*.{js,mjs,ts,jsx,tsx,css,svg}] 2 | indent_style = space 3 | indent_size = 2 4 | 5 | [Makefile] 6 | indent_style = tab 7 | -------------------------------------------------------------------------------- /frontend/.gitignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | node_modules/ 3 | -------------------------------------------------------------------------------- /frontend/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:22-alpine3.19 as dev 2 | 3 | WORKDIR /howtheyvote/frontend 4 | 5 | RUN apk add make bash entr 6 | 7 | COPY package.json package-lock.json . 8 | RUN npm ci 9 | 10 | COPY . . 11 | RUN make build 12 | 13 | CMD node dist/server.entry.mjs 14 | -------------------------------------------------------------------------------- /frontend/Makefile: -------------------------------------------------------------------------------- 1 | default: lint typecheck test 2 | 3 | dev: 4 | ./scripts/watch 5 | 6 | build: build-server build-client 7 | 8 | lint: 9 | ./node_modules/.bin/biome check src static 10 | 11 | lint-fix: 12 | ./node_modules/.bin/biome check --write src static 13 | 14 | typecheck: 15 | ./node_modules/.bin/tsc --noEmit 16 | 17 | typegen: 18 | ./node_modules/.bin/openapi-ts 19 | 20 | test: 21 | node \ 22 | --import ./src/setupTests.js \ 23 | --import ./src/cssLoader.js \ 24 | --import tsx/esm \ 25 | --test \ 26 | ./src/**/*.test.tsx ./src/**/*.test.ts 27 | 28 | clean: 29 | rm -rf ./dist 30 | 31 | build-server: 32 | ./node_modules/.bin/esbuild \ 33 | --bundle \ 34 | --platform=node \ 35 | --format=esm \ 36 | --packages=external \ 37 | --external:*jpg \ 38 | --external:*.woff \ 39 | --external:*.woff2 \ 40 | --sourcemap=inline \ 41 | --outfile=dist/server.entry.mjs \ 42 | --keep-names \ 43 | src/server.entry.ts 44 | 45 | build-client: 46 | ./node_modules/.bin/esbuild \ 47 | --bundle \ 48 | --external:*.jpg \ 49 | --outfile=dist/client.entry.js \ 50 | --minify \ 51 | --keep-names \ 52 | src/client.entry.ts 53 | -------------------------------------------------------------------------------- /frontend/biome.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://biomejs.dev/schemas/1.4.1-nightly.570d680/schema.json", 3 | "files": { 4 | "ignore": [ 5 | "src/api/generated/**", 6 | "static/**" 7 | ] 8 | }, 9 | "organizeImports": { 10 | "enabled": true 11 | }, 12 | "formatter": { 13 | "indentStyle": "space" 14 | }, 15 | "linter": { 16 | "enabled": true, 17 | "rules": { 18 | "recommended": true, 19 | "a11y": { 20 | "noLabelWithoutControl": { 21 | "options": { 22 | "inputComponents": [ 23 | "Input", 24 | "Select", 25 | "GroupsFilterSelect", 26 | "CountriesFilterSelect", 27 | "PositionFilterSelect", 28 | "SortSelect" 29 | ] 30 | } 31 | } 32 | } 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /frontend/openapi-ts.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig, defaultPlugins } from "@hey-api/openapi-ts"; 2 | 3 | export default defineConfig({ 4 | client: "@hey-api/client-fetch", 5 | input: "http://backend:5000/api", 6 | output: "./src/api/generated", 7 | plugins: [ 8 | ...defaultPlugins, 9 | { 10 | name: "@hey-api/sdk", 11 | throwOnError: true, 12 | }, 13 | ], 14 | }); 15 | -------------------------------------------------------------------------------- /frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "howtheyvote", 3 | "version": "0.1.0", 4 | "type": "module", 5 | "description": "", 6 | "author": "Till Prochaska ", 7 | "license": "GPL-3.0-only", 8 | "dependencies": { 9 | "@hey-api/client-fetch": "^0.6.0", 10 | "@tinyhttp/app": "^2.4.0", 11 | "isbot": "^5.1.28", 12 | "pino": "^9.7.0", 13 | "preact": "^10.26.8", 14 | "preact-render-to-string": "^6.5.13", 15 | "sirv": "^3.0.1" 16 | }, 17 | "devDependencies": { 18 | "@biomejs/biome": "^1.9.4", 19 | "@hey-api/openapi-ts": "^0.61.2", 20 | "@testing-library/preact": "^3.2.4", 21 | "@testing-library/user-event": "^14.6.1", 22 | "@types/jsdom": "^21.1.7", 23 | "@types/node": "^22.15.30", 24 | "esbuild": "^0.25.5", 25 | "jsdom": "^26.1.0", 26 | "tsx": "^4.19.4", 27 | "typescript": "^5.8.3" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /frontend/scripts/manifest: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | filter=" 4 | [ 5 | .outputs 6 | | to_entries 7 | | .[] 8 | | select(.value.entryPoint) 9 | | { key: (.value.entryPoint), value: { js: .key, css: .value.cssBundle } } 10 | ] | from_entries 11 | " 12 | 13 | jq "$filter" dist/meta.json > dist/client/manifest.json 14 | -------------------------------------------------------------------------------- /frontend/scripts/watch: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | trap exit SIGINT SIGTERM 4 | export ENTR_INOTIFY_WORKAROUND=1 5 | 6 | while true; do 7 | find src -type f | entr -r -d -s 'make build && node dist/server.entry.mjs' 8 | done 9 | -------------------------------------------------------------------------------- /frontend/src/api/generated/index.ts: -------------------------------------------------------------------------------- 1 | // This file is auto-generated by @hey-api/openapi-ts 2 | export * from './sdk.gen'; 3 | export * from './types.gen'; -------------------------------------------------------------------------------- /frontend/src/api/index.ts: -------------------------------------------------------------------------------- 1 | import { BACKEND_PRIVATE_URL } from "../config"; 2 | import { HTTPException } from "../lib/http"; 3 | import { client } from "./generated/sdk.gen"; 4 | 5 | client.setConfig({ 6 | baseUrl: BACKEND_PRIVATE_URL, 7 | }); 8 | 9 | client.interceptors.response.use((response) => { 10 | if (response.status === 404) { 11 | throw new HTTPException(404); 12 | } 13 | 14 | return response; 15 | }); 16 | 17 | export * from "./generated/sdk.gen"; 18 | export * from "./generated/types.gen"; 19 | -------------------------------------------------------------------------------- /frontend/src/client.entry.ts: -------------------------------------------------------------------------------- 1 | import Banner from "./components/Banner"; 2 | import CountryStatsList from "./components/CountryStatsList"; 3 | import Eyes from "./components/Eyes"; 4 | import GroupStatsList from "./components/GroupStatsList"; 5 | import MemberVotesList from "./components/MemberVotesList"; 6 | import ShareButton from "./components/ShareButton"; 7 | import SortSelect from "./components/SortSelect"; 8 | import VoteTabs from "./components/VoteTabs"; 9 | import { hydrateIslands } from "./lib/islands"; 10 | 11 | hydrateIslands([ 12 | MemberVotesList, 13 | GroupStatsList, 14 | CountryStatsList, 15 | VoteTabs, 16 | Eyes, 17 | ShareButton, 18 | Banner, 19 | SortSelect, 20 | ]); 21 | -------------------------------------------------------------------------------- /frontend/src/components/App.tsx: -------------------------------------------------------------------------------- 1 | import type { ComponentChildren } from "preact"; 2 | 3 | import "../css/base.css"; 4 | import "../css/fonts.css"; 5 | import "../css/typography.css"; 6 | import "../css/utils.css"; 7 | import "../css/variables.css"; 8 | 9 | import { assetUrl } from "../lib/caching"; 10 | 11 | type AppProps = { 12 | children: ComponentChildren; 13 | title: string | Array; 14 | head?: ComponentChildren; 15 | defaultCss?: boolean; 16 | defaultJs?: boolean; 17 | }; 18 | 19 | export default function App({ 20 | children, 21 | title, 22 | head, 23 | defaultCss = true, 24 | defaultJs = true, 25 | }: AppProps) { 26 | const metaTitle = Array.isArray(title) ? title : [title]; 27 | metaTitle.push("HowTheyVote.eu"); 28 | 29 | return ( 30 | 31 | 32 | {metaTitle.filter(Boolean).join(" · ")} 33 | 34 | {defaultCss && ( 35 | 36 | )} 37 | 38 | 39 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 53 | 54 | 55 | {head} 56 | 57 | 58 | {children} 59 | {defaultJs &&