├── site ├── guides │ ├── campaigns.md │ ├── elections.md │ ├── expenditures.md │ ├── independent-expenditures.md │ ├── contributions.md │ ├── cache.md │ ├── actblue-winred.md │ └── fastfec.md ├── .gitignore ├── getting-started │ ├── intro-campfin.md │ ├── installation.md │ └── basic-usage.md ├── examples │ ├── filing-deadlines.md │ └── index.md ├── .vitepress │ ├── theme │ │ ├── index.js │ │ └── custom.css │ └── config.mts ├── reference │ ├── cli-fastfec-help.txt │ ├── cli.md │ ├── cli-help.txt │ ├── cli-info-help.txt │ ├── filings.sql │ ├── cli-export-help.txt │ └── sql.md ├── package.json ├── Makefile └── index.md ├── crates ├── fec-py │ ├── .gitignore │ ├── .python-version │ ├── tests │ │ ├── __init__.py │ │ ├── test_foo.py │ │ ├── conftest.py │ │ ├── README.md │ │ └── test_parser.py │ ├── src │ │ ├── foo.rs │ │ ├── lib.rs │ │ ├── libfec_parser │ │ │ └── __init__.py │ │ └── parser.rs │ ├── Cargo.toml │ ├── pyproject.toml │ ├── Makefile │ ├── demo.py │ ├── demo-fecfile.py │ └── README.md ├── fec-parser-macros │ ├── build.rs │ ├── Makefile │ ├── Cargo.toml │ ├── date_columns.txt │ └── src │ │ └── lib.rs ├── fec-cli │ ├── pyproject.toml │ ├── src │ │ ├── commands │ │ │ ├── mod.rs │ │ │ ├── export │ │ │ │ ├── sqlite │ │ │ │ │ ├── snapshots │ │ │ │ │ │ ├── libfec__commands__export__sqlite__test__tests__basic_f99-2.snap │ │ │ │ │ │ ├── libfec__commands__export__sqlite__test__tests__basic_f99-3.snap │ │ │ │ │ │ ├── libfec__commands__export__sqlite__test__tests__8.4 1st.snap │ │ │ │ │ │ ├── libfec__commands__export__sqlite__test__tests__F1S 8.4.snap │ │ │ │ │ │ ├── libfec__commands__export__sqlite__test__tests__F1S schema.snap │ │ │ │ │ │ ├── libfec__commands__export__sqlite__test__tests__8.5 2nd.snap │ │ │ │ │ │ ├── libfec__commands__export__sqlite__test__tests__basic_f99.snap │ │ │ │ │ │ ├── libfec__commands__export__sqlite__test__tests__F1S 8.5 data.snap │ │ │ │ │ │ └── libfec__commands__export__sqlite__test__tests__F1S 8.4 data.snap │ │ │ │ │ └── test.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── dir_csv.rs │ │ │ │ ├── single.rs │ │ │ │ └── excel.rs │ │ │ ├── search.rs │ │ │ ├── cache.rs │ │ │ ├── bulk.rs │ │ │ ├── fastfec.rs │ │ │ └── info.rs │ │ ├── main.rs │ │ ├── cache │ │ │ ├── bulk_candidate_committee_linkage.rs │ │ │ ├── bulk_committee.rs │ │ │ ├── bulk_opexp.rs │ │ │ ├── bulk_candidates.rs │ │ │ └── bulk_utils.rs │ │ └── cli.rs │ └── Cargo.toml ├── fec-wasm │ ├── test.ts │ ├── Cargo.toml │ ├── example.deno.ts │ ├── example.node.mjs │ ├── Makefile │ ├── src │ │ └── lib.rs │ ├── README.md │ └── index.html ├── fec-api │ └── Cargo.toml └── fec-parser │ ├── Cargo.toml │ └── src │ ├── covers │ └── mod.rs │ ├── mappings.rs │ └── schedules.rs ├── examples ├── battleground-2026 │ ├── README.md │ ├── cook-political.txt │ ├── nrcc-targets.txt │ └── dnc-targets.txt ├── latimes-harris-new │ └── actblue-202301-202407.txt └── wapo-2024-10-harris-trump │ ├── winred-filings.txt │ ├── actblue-filings.txt │ └── README.md ├── benchmarks ├── Makefile ├── bench.sh └── README.md ├── uv.lock ├── Makefile ├── .gitignore ├── Cargo.toml ├── pyproject.toml ├── .github └── workflows │ ├── test.yaml │ ├── publish-pypi.yml │ ├── test-wasm.yaml │ ├── test-python.yml │ ├── site.yaml │ └── build-pypi.yml ├── dist-workspace.toml └── README.md /site/guides/campaigns.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /site/guides/elections.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /crates/fec-py/.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc -------------------------------------------------------------------------------- /crates/fec-py/.python-version: -------------------------------------------------------------------------------- 1 | 3.13 2 | -------------------------------------------------------------------------------- /examples/battleground-2026/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /site/.gitignore: -------------------------------------------------------------------------------- 1 | .vitepress/cache 2 | node_modules -------------------------------------------------------------------------------- /crates/fec-py/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Tests for libfec_parser 2 | -------------------------------------------------------------------------------- /site/guides/expenditures.md: -------------------------------------------------------------------------------- 1 | # Expenditures and Disbursements -------------------------------------------------------------------------------- /site/guides/independent-expenditures.md: -------------------------------------------------------------------------------- 1 | # Independent Expenditures -------------------------------------------------------------------------------- /site/getting-started/intro-campfin.md: -------------------------------------------------------------------------------- 1 | # Introduction to Campaign Finance -------------------------------------------------------------------------------- /examples/battleground-2026/cook-political.txt: -------------------------------------------------------------------------------- 1 | # https://www.cookpolitical.com/ratings/house-race-ratings -------------------------------------------------------------------------------- /site/examples/filing-deadlines.md: -------------------------------------------------------------------------------- 1 | # Reporting FEC Filing Deadlines with `libfec` 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /crates/fec-parser-macros/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | println!("cargo:rerun-if-changed=mappings2.json"); 3 | } 4 | -------------------------------------------------------------------------------- /benchmarks/Makefile: -------------------------------------------------------------------------------- 1 | LIBFEC_CLI=../target/release/libfec 2 | 3 | data/actblue-oct24.fec: 4 | $(LIBFEC_CLI) download FEC-1833806 $@ -------------------------------------------------------------------------------- /site/.vitepress/theme/index.js: -------------------------------------------------------------------------------- 1 | import DefaultTheme from 'vitepress/theme' 2 | import "./custom.css" 3 | export default DefaultTheme -------------------------------------------------------------------------------- /uv.lock: -------------------------------------------------------------------------------- 1 | version = 1 2 | requires-python = ">=3.7" 3 | 4 | [[package]] 5 | name = "fec-py" 6 | version = "0.1.0" 7 | source = { editable = "." } 8 | -------------------------------------------------------------------------------- /site/examples/index.md: -------------------------------------------------------------------------------- 1 | # Example `libfec` Projects 2 | 3 | `libfec` is designed to help you build data piplines, news apps, and reporting tools with FEC filings.n -------------------------------------------------------------------------------- /site/.vitepress/theme/custom.css: -------------------------------------------------------------------------------- 1 | :root { 2 | --vp-c-brand-1: #40a02b; 3 | --vp-c-brand-2: #a6d189; 4 | /* dd7878 */ 5 | 6 | --vp-button-brand-bg: var(--vp-c-brand-1); 7 | } -------------------------------------------------------------------------------- /examples/latimes-harris-new/actblue-202301-202407.txt: -------------------------------------------------------------------------------- 1 | FEC-1812188 2 | FEC-1805179 3 | FEC-1791562 4 | FEC-1785179 5 | FEC-1779040 6 | FEC-1765652 7 | FEC-1758569 8 | FEC-1752852 9 | FEC-1720554 10 | -------------------------------------------------------------------------------- /crates/fec-cli/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["maturin>=1.0,<2.0"] 3 | build-backend = "maturin" 4 | 5 | [project] 6 | name = "libfec" 7 | description = "A command line interface for working with FEC filings" 8 | -------------------------------------------------------------------------------- /crates/fec-wasm/test.ts: -------------------------------------------------------------------------------- 1 | /* 2 | import init, {greet} from './pkg/fec_wasm.js'; 3 | await init(); 4 | console.log(greet('asdf')); 5 | */ 6 | 7 | import {greet} from './pkg/fec_wasm.js'; 8 | const res = greet('asdf'); 9 | console.log(res.fec_version); -------------------------------------------------------------------------------- /site/guides/contributions.md: -------------------------------------------------------------------------------- 1 | # Contributions & Receipts 2 | 3 | Inidividuals, corporations, and committees can contribute to federal campaigns and committees. 4 | 5 | 6 | ## Finding all donors to a specific committee, campaign, or candidate 7 | 8 | 9 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | .PHONY: test-files 3 | 4 | .test-files/1885517.fec: 5 | echo cargo run download $(basename $@) 6 | 7 | .test-files/1913493.fec: 8 | echo cargo run download $(basename $@) 9 | 10 | test-files: .test-files/1885517.fec .test-files/1913493.fec 11 | -------------------------------------------------------------------------------- /crates/fec-cli/src/commands/mod.rs: -------------------------------------------------------------------------------- 1 | mod cache; 2 | pub mod export; 3 | mod fastfec; 4 | mod info; 5 | mod search; 6 | mod bulk; 7 | 8 | pub use cache::cache; 9 | pub use export::export; 10 | pub use fastfec::fastfec; 11 | pub use info::info; 12 | pub use search::search; 13 | pub use bulk::bulk; -------------------------------------------------------------------------------- /crates/fec-py/src/foo.rs: -------------------------------------------------------------------------------- 1 | use pyo3::prelude::*; 2 | 3 | #[pyfunction] 4 | pub fn bar() -> i32 { 5 | 42 6 | } 7 | 8 | /// Foo submodule - example module 9 | #[pymodule] 10 | pub fn foo(m: &Bound<'_, PyModule>) -> PyResult<()> { 11 | m.add_function(wrap_pyfunction!(bar, m)?)?; 12 | Ok(()) 13 | } 14 | -------------------------------------------------------------------------------- /crates/fec-api/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fec-api" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | anyhow = "1.0" 8 | derive_builder = "0.20.2" 9 | serde = {version="1", features = ["derive"]} 10 | serde_json = "1.0" 11 | ureq = {version="3.0.10", features = ["json"]} 12 | url = "2.5.4" 13 | -------------------------------------------------------------------------------- /crates/fec-cli/src/commands/export/sqlite/snapshots/libfec__commands__export__sqlite__test__tests__basic_f99-2.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: crates/fec-cli/src/commands/export/sqlite/test.rs 3 | expression: "query(&db, \"select count(*) from libfec_f99\")" 4 | snapshot_kind: text 5 | --- 6 | count(*) 7 | ==== Row 0 ==== 8 | [count(*)]:1 9 | -------------------------------------------------------------------------------- /crates/fec-parser-macros/Makefile: -------------------------------------------------------------------------------- 1 | #cat fec-parser-macros/src/mappings2.json | rg '^\s+".*",?$' | sort | uniq 2 | 3 | column_names.txt: src/mappings2.json 4 | cat $< | rg '^\s+".*",?$$' | awk -F'"' '/"/ {print $$2}' | sort | uniq > $@ 5 | 6 | 7 | date_columns.txt: column_names.txt 8 | cat $< | rg '(^date_)|(_date$$)' > $@ 9 | -------------------------------------------------------------------------------- /benchmarks/bench.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eiuo xtrace 3 | #fastfec_path=$1 4 | #libfec_path=$2 5 | fec_path=$1 6 | 7 | # /Users/alex/projects/ 8 | 9 | hyperfine \ 10 | --prepare 'rm -rf output || true' \ 11 | "../../FastFEC/zig-out/bin/fastfec -x $fec_path" \ 12 | --prepare 'rm -rf output || true' \ 13 | "libfec-release fastfec $fec_path" 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | *.db 3 | *.db-journal 4 | mappings.json 5 | 6 | notes/ 7 | output/ 8 | june/ 9 | 10 | *.fec 11 | *.fec.part 12 | *.fec.gz 13 | *.fec.zst 14 | *.db.gz 15 | *.csv 16 | *.zip 17 | *.xlsx 18 | .env 19 | 20 | 21 | .test-files 22 | .envrc 23 | 24 | .DS_Store 25 | 26 | # temporary!!! 27 | /examples/ 28 | 29 | .vscode/ 30 | 31 | dist/ 32 | *.so -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | resolver = "2" 3 | 4 | members = [ 5 | "crates/fec-parser-macros", 6 | "crates/fec-parser", 7 | "crates/fec-cli", 8 | "crates/fec-api", 9 | "crates/fec-wasm", 10 | "crates/fec-py", 11 | ] 12 | 13 | # The profile that 'cargo dist' will build with 14 | [profile.dist] 15 | inherits = "release" 16 | lto = "thin" 17 | -------------------------------------------------------------------------------- /examples/battleground-2026/nrcc-targets.txt: -------------------------------------------------------------------------------- 1 | # https://www.nrcc.org/2025/03/17/nrcc-targets-26-offensive-seats-to-expand-house-majority/ 2 | CA09 3 | CA13 4 | CA27 5 | CA45 6 | CA47 7 | FL09 8 | FL23 9 | IN01 10 | ME02 11 | MI08 12 | NC01 13 | NH01 14 | NJ09 15 | NM02 16 | NV01 17 | NV03 18 | NV04 19 | NY03 20 | NY04 21 | NY19 22 | OH09 23 | OH13 24 | TX28 25 | TX34 26 | VA07 27 | WA03 -------------------------------------------------------------------------------- /crates/fec-parser-macros/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fec-parser-macros" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | proc-macro2 = "1.0.39" 8 | quote = "1.0.21" 9 | syn = { version = "1.0.95", features = [ "extra-traits", "full", "fold", "parsing" ] } 10 | serde_json = {version="1.0.104", features=["preserve_order"]} 11 | 12 | [lib] 13 | proc-macro = true 14 | -------------------------------------------------------------------------------- /crates/fec-wasm/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fec-wasm" 3 | version = "0.1.0" 4 | edition = "2024" 5 | 6 | [dependencies] 7 | wasm-bindgen = {version="0.2.100", features = ["serde-serialize"] } 8 | fec-parser = { path = "../fec-parser" } 9 | serde = { version = "1.0", features = ["derive"] } 10 | serde-wasm-bindgen = "0.4" 11 | tsify = "0.5.5" 12 | 13 | [lib] 14 | crate-type = ["cdylib"] 15 | -------------------------------------------------------------------------------- /examples/wapo-2024-10-harris-trump/winred-filings.txt: -------------------------------------------------------------------------------- 1 | https://docquery.fec.gov/dcdev/posted/1828951.fec 2 | https://docquery.fec.gov/dcdev/posted/1800865.fec 3 | https://docquery.fec.gov/dcdev/posted/1774913.fec 4 | https://docquery.fec.gov/dcdev/posted/1751463.fec 5 | https://docquery.fec.gov/dcdev/posted/1720509.fec 6 | https://docquery.fec.gov/dcdev/posted/1686511.fec 7 | https://docquery.fec.gov/dcdev/posted/1672789.fec -------------------------------------------------------------------------------- /crates/fec-parser/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fec-parser" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | csv = "1.2.2" 8 | indexmap = {version="2.9.0", features=["serde"]} 9 | lazy_static = "1.4.0" 10 | regex = "1.9.3" 11 | serde_json = "1.0.104" 12 | thiserror = "1.0.44" 13 | fec-parser-macros = {path="../fec-parser-macros"} 14 | bstr = "1.10.0" 15 | jiff = "0.2.5" 16 | anyhow = "1.0.99" 17 | -------------------------------------------------------------------------------- /crates/fec-cli/src/commands/search.rs: -------------------------------------------------------------------------------- 1 | use crate::{cli::SearchArgs, sourcer::FilingSourcer}; 2 | 3 | pub fn search(mut sourcer: FilingSourcer, args: &SearchArgs) -> anyhow::Result<()> { 4 | Ok(()) 5 | /* 6 | let results = sourcer.cache.search_candidates(args.cycle, &args.query)?; 7 | for (candidate_id, name) in results { 8 | println!("{}: {}", candidate_id, name); 9 | } 10 | Ok(()) 11 | */ 12 | } 13 | -------------------------------------------------------------------------------- /site/reference/cli-fastfec-help.txt: -------------------------------------------------------------------------------- 1 | FastFEC compatible export 2 | 3 | Usage: libfec fastfec [OPTIONS] [OUTPUT_DIRECTORY] 4 | 5 | Arguments: 6 | FEC filing id OR path to a .fec file 7 | [OUTPUT_DIRECTORY] Output directory [default: output] 8 | 9 | Options: 10 | -h, --help Print help 11 | 12 | Global options: 13 | --cache-directory [env: LIBFEC_CACHE_DIRECTORY=] 14 | -------------------------------------------------------------------------------- /examples/battleground-2026/dnc-targets.txt: -------------------------------------------------------------------------------- 1 | # https://dccc.org/dccc-announces-2026-districts-in-play/ 2 | 3 | AK00 4 | AZ01 5 | AZ02 6 | AZ06 7 | CA22 8 | CA40 9 | CA41 10 | CO08 11 | FL07 12 | FL13 13 | FL27 14 | IA01 15 | IA02 16 | IA03 17 | KY06 18 | MI04 19 | MI07 20 | MI10 21 | MO02 22 | NE02 23 | NJ07 24 | NY17 25 | OH07 26 | OH10 27 | OH15 28 | PA01 29 | PA07 30 | PA08 31 | PA10 32 | TN05 33 | TX15 34 | VA01 35 | VA02 36 | WI01 37 | WI03 -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["maturin>=1,<2"] 3 | build-backend = "maturin" 4 | 5 | [project] 6 | name = "fec-py" 7 | requires-python = ">=3.7" 8 | version = "0.1.0" 9 | classifiers = [ 10 | "Programming Language :: Rust", 11 | "Programming Language :: Python :: Implementation :: CPython", 12 | "Programming Language :: Python :: Implementation :: PyPy", 13 | ] 14 | 15 | [tool.uv.workspace] 16 | members = ["fec-py/fec-py"] 17 | -------------------------------------------------------------------------------- /site/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "site", 3 | "version": "1.0.0", 4 | "main": "index.js", 5 | "scripts": { 6 | "test": "echo \"Error: no test specified\" && exit 1", 7 | "dev": "vitepress dev", 8 | "build": "vitepress build", 9 | "preview": "vitepress preview" 10 | }, 11 | "keywords": [], 12 | "author": "", 13 | "license": "ISC", 14 | "description": "", 15 | "devDependencies": { 16 | "vitepress": "^1.6.3" 17 | } 18 | } -------------------------------------------------------------------------------- /crates/fec-wasm/example.deno.ts: -------------------------------------------------------------------------------- 1 | // Deno example for fec-wasm 2 | // Run with: deno run --allow-read=pkg,1926611.fec example.deno.ts 3 | 4 | import init, { header } from "./pkg/fec_wasm.js"; 5 | 6 | // Auto-loads the WASM file via import.meta.url 7 | await init(); 8 | 9 | const fecData = await Deno.readFile("./1926611.fec"); 10 | 11 | const filingHeader = header(fecData); 12 | 13 | filingHeader.record_type; 14 | 15 | console.log(JSON.stringify(filingHeader, null, 2)); 16 | -------------------------------------------------------------------------------- /.github/workflows/test.yaml: -------------------------------------------------------------------------------- 1 | name: "build" 2 | on: 3 | push: 4 | branches: 5 | - main 6 | permissions: 7 | contents: read 8 | jobs: 9 | test: 10 | strategy: 11 | matrix: 12 | os: [ubuntu-latest, macos-latest, macos-15-intel, ubuntu-22.04-arm, windows-latest] 13 | fail-fast: false 14 | runs-on: ${{ matrix.os }} 15 | steps: 16 | - uses: actions/checkout@v4 17 | - run: rustup toolchain install stable --profile minimal 18 | - uses: Swatinem/rust-cache@v2 19 | - run: cargo build -p fec-cli -------------------------------------------------------------------------------- /crates/fec-wasm/example.node.mjs: -------------------------------------------------------------------------------- 1 | // Node.js example for fec-wasm 2 | // Run with: node example.node.mjs 3 | 4 | import { readFile } from "fs/promises"; 5 | import init, { header } from "./pkg/fec_wasm.js"; 6 | 7 | // Initialize the WASM module by loading the .wasm file 8 | const wasmBuffer = await readFile("./pkg/fec_wasm_bg.wasm"); 9 | await init({ module_or_path: wasmBuffer }); 10 | 11 | // Read the FEC file 12 | const fecData = await readFile("./1926611.fec"); 13 | 14 | // Parse the header 15 | const filingHeader = header(fecData); 16 | 17 | console.log(JSON.stringify(filingHeader, null, 2)); -------------------------------------------------------------------------------- /site/reference/cli.md: -------------------------------------------------------------------------------- 1 | # `libfec` CLI Reference 2 | 3 | > [!WARNING] 4 | > This documentation is incomplete! 5 | 6 | All commands and flags for the `libfec` CLI. 7 | 8 | 9 | 10 | ## `libfec export` {#export} 11 | 12 | Export FEC filings into various formats like SQLite, CSV, JSON, and Excel. 13 | 14 | ```bash 15 | 16 | ``` 17 | 18 | ## `libfec fastfec` {#fastfec} 19 | 20 | 21 | 22 | ```bash 23 | 24 | ``` 25 | 26 | 27 | ## `libfec info` {#info} 28 | 29 | 30 | 31 | ```bash 32 | 33 | ``` -------------------------------------------------------------------------------- /crates/fec-py/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod parser; 2 | mod foo; 3 | mod fecfile; 4 | 5 | use pyo3::prelude::*; 6 | use pyo3::wrap_pymodule; 7 | 8 | /// A Python module implemented in Rust. The name of this function must match 9 | /// the `lib.name` setting in the `Cargo.toml`, else Python will not be able to 10 | /// import the module. 11 | #[pymodule] 12 | fn libfec_parser(m: &Bound<'_, PyModule>) -> PyResult<()> { 13 | // Add submodules 14 | m.add_wrapped(wrap_pymodule!(parser::parser))?; 15 | m.add_wrapped(wrap_pymodule!(foo::foo))?; 16 | m.add_wrapped(wrap_pymodule!(fecfile::fecfile))?; 17 | Ok(()) 18 | } 19 | -------------------------------------------------------------------------------- /crates/fec-py/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "libfec_parser" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [lib] 7 | name = "libfec_parser" 8 | # "cdylib" is necessary to produce a shared library for Python to import from. 9 | crate-type = ["cdylib"] 10 | 11 | [dependencies] 12 | # "extension-module" tells pyo3 we want to build an extension module (skips linking against libpython.so) 13 | # "abi3-py39" tells pyo3 (and maturin) to build using the stable ABI with minimum Python version 3.9 14 | pyo3 = { version = "0.22.4", features = ["extension-module", "abi3-py39"] } 15 | fec-parser = { path="../fec-parser" } 16 | csv = "1.3" 17 | -------------------------------------------------------------------------------- /site/reference/cli-help.txt: -------------------------------------------------------------------------------- 1 | libfec CLI 2 | 3 | Usage: libfec [OPTIONS] [COMMAND] 4 | 5 | Commands: 6 | export Export FEC filings into SQLite, Excel, CSV, or JSON 7 | cache Cache .fec files from fec.gov to your filesystem 8 | info Print debug information about a FEC filing, committee, or candidate 9 | fastfec FastFEC compatible export 10 | search 11 | help Print this message or the help of the given subcommand(s) 12 | 13 | Options: 14 | -h, --help Print help 15 | -V, --version Print version 16 | 17 | Global options: 18 | --cache-directory [env: LIBFEC_CACHE_DIRECTORY=] 19 | -------------------------------------------------------------------------------- /.github/workflows/publish-pypi.yml: -------------------------------------------------------------------------------- 1 | name: "Publish PyPi wheels" 2 | on: 3 | workflow_call: 4 | inputs: 5 | plan: 6 | required: true 7 | type: string 8 | 9 | jobs: 10 | upload-libfec-wheels: 11 | runs-on: ubuntu-latest 12 | environment: release 13 | permissions: 14 | id-token: write 15 | steps: 16 | - uses: astral-sh/setup-uv@v6 17 | - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 18 | with: 19 | pattern: wheels_libfec-* 20 | path: wheels_libfec 21 | merge-multiple: true 22 | - run: uv publish -v wheels_libfec/* -------------------------------------------------------------------------------- /crates/fec-py/src/libfec_parser/__init__.py: -------------------------------------------------------------------------------- 1 | # Import the native extension and expose submodules 2 | from . import libfec_parser as _libfec_parser 3 | import sys 4 | 5 | # Make submodules accessible as libfec_parser.parser, libfec_parser.foo, and libfec_parser.fecfile 6 | parser = _libfec_parser.parser 7 | foo = _libfec_parser.foo 8 | fecfile = _libfec_parser.fecfile 9 | 10 | # Also register them in sys.modules for "from libfec_parser.parser import ..." 11 | sys.modules['libfec_parser.parser'] = parser 12 | sys.modules['libfec_parser.foo'] = foo 13 | sys.modules['libfec_parser.fecfile'] = fecfile 14 | 15 | __all__ = ["parser", "foo", "fecfile"] 16 | -------------------------------------------------------------------------------- /site/reference/cli-info-help.txt: -------------------------------------------------------------------------------- 1 | Print debug information about a FEC filing, committee, or candidate 2 | 3 | Usage: libfec info [OPTIONS] [FILINGS]... 4 | 5 | Arguments: 6 | [FILINGS]... 7 | 8 | Options: 9 | -i, --input-file .txt files of FEC filing IDs to fetch, 1 line per filing ID 10 | -f, --format Format to output information to [default: human] [possible values: human, json] 11 | --full Calculate stats on all itemizations in the provided filings 12 | -h, --help Print help 13 | 14 | Global options: 15 | --cache-directory [env: LIBFEC_CACHE_DIRECTORY=] 16 | -------------------------------------------------------------------------------- /site/Makefile: -------------------------------------------------------------------------------- 1 | reference/filings.sql: 2 | solite-dev q ca01.db \ 3 | "select sql from sqlite_master where type = 'table' and tbl_name = 'libfec_filings';" \ 4 | -f value \ 5 | -o $@ 6 | 7 | reference/cli-help.txt: 8 | ../target/debug/libfec --help > $@ 9 | 10 | reference/cli-export-help.txt: 11 | ../target/debug/libfec export --help > $@ 12 | 13 | reference/cli-info-help.txt: 14 | ../target/debug/libfec info --help > $@ 15 | 16 | reference/cli-fastfec-help.txt: 17 | ../target/debug/libfec fastfec --help > $@ 18 | 19 | 20 | sql: reference/filings.sql 21 | cli: reference/cli-help.txt reference/cli-export-help.txt reference/cli-info-help.txt reference/cli-fastfec-help.txt 22 | 23 | all: cli sql -------------------------------------------------------------------------------- /crates/fec-py/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "libfec-parser" 3 | version = "0.1.0" 4 | description = "Add your description here" 5 | readme = "README.md" 6 | authors = [ 7 | { name = "Alex Garcia", email = "alexsebastian.garcia@gmail.com" } 8 | ] 9 | requires-python = ">=3.7" 10 | dependencies = [] 11 | 12 | [project.optional-dependencies] 13 | dev = [ 14 | "pytest>=7.0.0", 15 | "pytest-cov>=4.0.0", 16 | ] 17 | 18 | [project.scripts] 19 | libfec-parser = "libfec_parser:main" 20 | 21 | [tool.maturin] 22 | module-name = "libfec_parser" 23 | python-packages = ["libfec_parser"] 24 | python-source = "src" 25 | 26 | [build-system] 27 | requires = ["maturin>=1.0,<2.0"] 28 | build-backend = "maturin" 29 | -------------------------------------------------------------------------------- /examples/wapo-2024-10-harris-trump/actblue-filings.txt: -------------------------------------------------------------------------------- 1 | https://docquery.fec.gov/dcdev/posted/1864088.fec 2 | https://docquery.fec.gov/dcdev/posted/1833806.fec 3 | https://docquery.fec.gov/dcdev/posted/1817884.fec 4 | https://docquery.fec.gov/dcdev/posted/1805179.fec 5 | https://docquery.fec.gov/dcdev/posted/1791562.fec 6 | https://docquery.fec.gov/dcdev/posted/1785179.fec 7 | https://docquery.fec.gov/dcdev/posted/1779040.fec 8 | https://docquery.fec.gov/dcdev/posted/1765652.fec 9 | https://docquery.fec.gov/dcdev/posted/1758569.fec 10 | https://docquery.fec.gov/dcdev/posted/1752852.fec 11 | https://docquery.fec.gov/dcdev/posted/1720554.fec 12 | https://docquery.fec.gov/dcdev/posted/1686601.fec 13 | https://docquery.fec.gov/dcdev/posted/1671120.fec -------------------------------------------------------------------------------- /site/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | # https://vitepress.dev/reference/default-theme-home-page 3 | layout: home 4 | 5 | hero: 6 | name: "libfec" 7 | text: "" 8 | tagline: A tool for wranging campaign finance data from the FEC 9 | actions: 10 | - theme: brand 11 | text: Installing 12 | link: /getting-started/installation 13 | - theme: alt 14 | text: Examples 15 | link: /examples 16 | 17 | features: 18 | - title: Parse FEC filings! 19 | details: Convert .fec files into SQLite, Excel, CSVs, etc. 20 | - title: Works with the OpenFEC API 21 | details: Automatically download all filings for a given committee/candidate in one command 22 | - title: Really fast! 23 | details: Parses large ActBlue + WinRed Filings in 30 seconds or less 24 | --- 25 | 26 | -------------------------------------------------------------------------------- /.github/workflows/test-wasm.yaml: -------------------------------------------------------------------------------- 1 | name: "test-wasm" 2 | on: 3 | push: 4 | branches: 5 | - main 6 | pull_request: 7 | permissions: 8 | contents: read 9 | jobs: 10 | test-wasm: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v4 14 | - run: rustup toolchain install stable --profile minimal 15 | - run: rustup target add wasm32-unknown-unknown 16 | - uses: Swatinem/rust-cache@v2 17 | - name: Install wasm-bindgen-cli 18 | run: cargo install wasm-bindgen-cli 19 | - name: Build WASM package 20 | run: make build 21 | working-directory: crates/fec-wasm 22 | - name: Upload WASM package 23 | uses: actions/upload-artifact@v4 24 | with: 25 | name: wasm-pkg 26 | path: crates/fec-wasm/pkg/ 27 | -------------------------------------------------------------------------------- /examples/wapo-2024-10-harris-trump/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ```bash 4 | cargo run filings \ 5 | --committee=C00694323 \ 6 | --form-type=F3X \ 7 | --coverage-after=2022-11-15 \ 8 | --coverage-before=2024-09-30 \ 9 | --filing-urls-only 10 | https://docquery.fec.gov/dcdev/posted/1828951.fec 11 | https://docquery.fec.gov/dcdev/posted/1800865.fec 12 | https://docquery.fec.gov/dcdev/posted/1774913.fec 13 | https://docquery.fec.gov/dcdev/posted/1751463.fec 14 | https://docquery.fec.gov/dcdev/posted/1720509.fec 15 | https://docquery.fec.gov/dcdev/posted/1686511.fec 16 | https://docquery.fec.gov/dcdev/posted/1672789.fec 17 | ``` 18 | 19 | ```bash 20 | cargo run filings \ 21 | --committee=C00401224 \ 22 | --form-type=F3X \ 23 | --coverage-after=2022-11-15 \ 24 | --coverage-before=2024-09-30\ 25 | --filing-urls-only 26 | 27 | ``` 28 | 29 | 30 | ``` 31 | 32 | ``` -------------------------------------------------------------------------------- /.github/workflows/test-python.yml: -------------------------------------------------------------------------------- 1 | name: "test-python" 2 | on: 3 | push: 4 | branches: 5 | - main 6 | pull_request: 7 | permissions: 8 | contents: read 9 | jobs: 10 | test-python: 11 | strategy: 12 | matrix: 13 | os: [ubuntu-latest, macos-latest, macos-15-intel, windows-latest] 14 | fail-fast: false 15 | runs-on: ${{ matrix.os }} 16 | steps: 17 | - uses: actions/checkout@v4 18 | - run: rustup toolchain install stable --profile minimal 19 | - uses: Swatinem/rust-cache@v2 20 | - uses: astral-sh/setup-uv@v6 21 | - run: uv tool install maturin 22 | - name: Build Python wheels 23 | run: make build 24 | working-directory: crates/fec-py 25 | - name: Upload wheels 26 | uses: actions/upload-artifact@v4 27 | with: 28 | name: wheels-${{ matrix.os }} 29 | path: crates/fec-py/dist/*.whl 30 | -------------------------------------------------------------------------------- /crates/fec-cli/src/commands/export/sqlite/snapshots/libfec__commands__export__sqlite__test__tests__basic_f99-3.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: crates/fec-cli/src/commands/export/sqlite/test.rs 3 | expression: "query(&db, \"select * from libfec_filings\")" 4 | snapshot_kind: text 5 | --- 6 | filing_id | fec_version | software_name | software_version | report_id | report_number | comment | cover_record_form | cover_record_form_amendment_indicator | filer_id | filer_name | report_code | coverage_from_date | coverage_through_date 7 | ==== Row 0 ==== 8 | [filing_id]:1913493 9 | [fec_version]:8.4 10 | [software_name]:FEC WebForms 11 | [software_version]:8.4.0.0 12 | [report_id]:NULL 13 | [report_number]:NULL 14 | [comment]:NULL 15 | [cover_record_form]:F99 16 | [cover_record_form_amendment_indicator]:NULL 17 | [filer_id]:C00917203 18 | [filer_name]:RIGHTS OF AMERICANS ASSOCIATION 19 | [report_code]:NULL 20 | [coverage_from_date]:NULL 21 | [coverage_through_date]:NULL 22 | -------------------------------------------------------------------------------- /.github/workflows/site.yaml: -------------------------------------------------------------------------------- 1 | name: Deploy Site 2 | on: 3 | workflow_dispatch: {} 4 | push: 5 | branches: 6 | - main 7 | paths: 8 | - "site/**" 9 | - ".github/**" 10 | jobs: 11 | deploy: 12 | runs-on: ubuntu-latest 13 | permissions: 14 | pages: write 15 | id-token: write 16 | environment: 17 | name: github-pages 18 | url: ${{ steps.deployment.outputs.page_url }} 19 | steps: 20 | - uses: actions/checkout@v5 21 | - uses: actions/setup-node@v6 22 | with: 23 | cache: npm 24 | cache-dependency-path: site/package-lock.json 25 | - run: npm ci 26 | working-directory: site/ 27 | - run: npm run build 28 | working-directory: site/ 29 | - uses: actions/configure-pages@v5 30 | - uses: actions/upload-pages-artifact@v3 31 | with: 32 | path: site/.vitepress/dist 33 | - id: deployment 34 | uses: actions/deploy-pages@v4 35 | -------------------------------------------------------------------------------- /crates/fec-py/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: build build-release demo demo-fecfile clean 2 | 3 | # Build development wheel 4 | build: 5 | maturin build -m Cargo.toml --out dist 6 | 7 | # Build release (optimized) wheel 8 | build-release: 9 | maturin build -m Cargo.toml --release --out dist 10 | 11 | # Run demo.py with sample FEC files from cache2 12 | demo: build 13 | uv run --no-cache --no-project --isolated \ 14 | --with 'libfec_parser @ file://dist/libfec_parser-0.1.0-cp39-abi3-macosx_11_0_arm64.whl' \ 15 | demo.py ../../cache2/*.fec 16 | 17 | # Run demo-fecfile.py with sample FEC files from cache2 18 | demo-fecfile: build 19 | uv run --no-cache --no-project --isolated \ 20 | --with 'libfec_parser @ file://dist/libfec_parser-0.1.0-cp39-abi3-macosx_11_0_arm64.whl' \ 21 | demo-fecfile.py ../../cache2/*.fec 22 | 23 | test-pytest: 24 | uv run --no-project --isolated \ 25 | --with pytest \ 26 | --with 'libfec_parser @ file://dist/libfec_parser-0.1.0-cp39-abi3-macosx_11_0_arm64.whl' \ 27 | pytest tests/ -------------------------------------------------------------------------------- /crates/fec-wasm/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: build test test-deno test-node clean help 2 | 3 | all: build 4 | 5 | build: 6 | @echo "Building fec-wasm..." 7 | cargo build --release --target wasm32-unknown-unknown 8 | wasm-bindgen --target web --out-dir pkg ../../target/wasm32-unknown-unknown/release/fec_wasm.wasm 9 | 10 | test: test-deno test-node 11 | 12 | test-deno: 13 | @echo "Running Deno example..." 14 | deno run --allow-read example.deno.ts 15 | 16 | test-node: 17 | @echo "Running Node.js example..." 18 | node example.node.mjs 19 | 20 | clean: 21 | @echo "Cleaning build artifacts..." 22 | rm -rf pkg/ 23 | cargo clean 24 | 25 | # Show help 26 | help: 27 | @echo "Available targets:" 28 | @echo " make build - Build the WASM module" 29 | @echo " make test - Run all tests (Deno + Node.js)" 30 | @echo " make test-deno - Run Deno example" 31 | @echo " make test-node - Run Node.js example" 32 | @echo " make clean - Clean build artifacts" 33 | @echo " make help - Show this help message" 34 | -------------------------------------------------------------------------------- /crates/fec-py/tests/test_foo.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for libfec_parser.foo module 3 | """ 4 | import pytest 5 | from libfec_parser.foo import bar 6 | 7 | 8 | class TestBar: 9 | """Tests for bar function""" 10 | 11 | def test_bar_returns_42(self): 12 | """Test that bar() returns 42""" 13 | result = bar() 14 | assert result == 42 15 | 16 | def test_bar_returns_int(self): 17 | """Test that bar() returns an integer""" 18 | result = bar() 19 | assert isinstance(result, int) 20 | 21 | def test_bar_is_callable(self): 22 | """Test that bar is callable""" 23 | assert callable(bar) 24 | 25 | def test_bar_no_arguments(self): 26 | """Test that bar() accepts no arguments""" 27 | # This should not raise an error 28 | result = bar() 29 | assert result is not None 30 | 31 | def test_bar_with_arguments_fails(self): 32 | """Test that bar() with arguments raises TypeError""" 33 | with pytest.raises(TypeError): 34 | bar(123) 35 | -------------------------------------------------------------------------------- /crates/fec-cli/src/main.rs: -------------------------------------------------------------------------------- 1 | mod api_flags; 2 | mod cache; 3 | mod cli; 4 | mod commands; 5 | mod sourcer; 6 | use crate::cli::Commands; 7 | use clap::Parser; 8 | use std::env; 9 | 10 | fn main() { 11 | let args: Vec = env::args().collect(); 12 | let cli = cli::Cli::parse_from(args.clone()); 13 | let sourcer = sourcer::FilingSourcer::new(cli.top_level.cache_directory.clone()); 14 | let result = match *cli.command { 15 | Commands::Info(args) => commands::info(sourcer, args), 16 | Commands::Export(args) => commands::export(sourcer, args), 17 | Commands::Fastfec(args) => commands::fastfec(sourcer, args), 18 | Commands::Cache(ref args) => commands::cache(sourcer, args), 19 | Commands::Search(ref args) => commands::search(sourcer, args), 20 | Commands::Bulk(ref args) => commands::bulk(sourcer, args), 21 | }; 22 | 23 | match result { 24 | Ok(()) => { 25 | std::process::exit(0); 26 | } 27 | Err(_) => { 28 | std::process::exit(1); 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /crates/fec-cli/src/commands/export/sqlite/snapshots/libfec__commands__export__sqlite__test__tests__8.4 1st.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: crates/fec-cli/src/commands/export/sqlite/test.rs 3 | expression: "query(&db, \"select * from libfec_f99\")" 4 | snapshot_kind: text 5 | --- 6 | filing_id | form_type | filer_committee_id_number | committee_name | street_1 | street_2 | city | state | zip_code | treasurer_last_name | treasurer_first_name | treasurer_middle_name | treasurer_prefix | treasurer_suffix | date_signed | text_code | filing_frequency | pdf_attachment | text 7 | ==== Row 0 ==== 8 | [filing_id]:1913493 9 | [form_type]:F99 10 | [filer_committee_id_number]:C00917203 11 | [committee_name]:RIGHTS OF AMERICANS ASSOCIATION 12 | [street_1]:2444 W. TOUHY AVENUE 13 | [street_2]: 14 | [city]:CHICAGO 15 | [state]:IL 16 | [zip_code]:60645 17 | [treasurer_last_name]:Hill 18 | [treasurer_first_name]:Tamika 19 | [treasurer_middle_name]:La'Shon 20 | [treasurer_prefix]:Mrs. 21 | [treasurer_suffix]: 22 | [date_signed]:2025-09-01 23 | [text_code]: 24 | [filing_frequency]: 25 | [pdf_attachment]: 26 | [text]: 27 | -------------------------------------------------------------------------------- /crates/fec-parser-macros/date_columns.txt: -------------------------------------------------------------------------------- 1 | amendment_date 2 | authorized_date 3 | cash_on_hand_as_of_date 4 | communication_date 5 | contribution_date 6 | coverage_from_date 7 | coverage_through_date 8 | date_day_after_general_election 9 | date_general_election 10 | date_incurred 11 | date_notarized 12 | date_notary_commission_expires 13 | date_of_election 14 | date_public_distribution 15 | date_signed 16 | disbursement_date 17 | dissemination_date 18 | donation_date 19 | effective_date 20 | election_date 21 | established_date 22 | event_year_to_date 23 | expenditure_date 24 | expenditure_total_cycle_to_date 25 | fifth_candidate_contribution_date 26 | fifty_first_contributor_date 27 | first_candidate_contribution_date 28 | fourth_candidate_contribution_date 29 | item_contribution_aquired_date 30 | loan_due_date 31 | loan_incurred_date 32 | loan_payment_to_date 33 | original_amendment_date 34 | original_registration_date 35 | planned_termination_report_date 36 | receipt_date 37 | received_date 38 | refund_date 39 | requirements_met_date 40 | second_candidate_contribution_date 41 | third_candidate_contribution_date 42 | -------------------------------------------------------------------------------- /site/guides/cache.md: -------------------------------------------------------------------------------- 1 | # The `libfec` Cache 2 | 3 | By default, the `libfec` will cache FEC files as they are downloaded. Individual `.fec` files never change, and downloading filings on every export can take a long time. 4 | 5 | 6 | The `libfec export` command will automatically save downloaded FEC filings into your cache directory. You can use the `libfec cache --print` command to see where this default `libfec` cache directory is located on your machine: 7 | 8 | ```bash 9 | $ libfec cache --print 10 | /Users/alex/.cache/libfec/cache 11 | ``` 12 | 13 | ## Changing the cache directory 14 | 15 | You can configure the cache directory in two ways — first with the `LIBFEC_CACHE_DIRECTORY` environment variable: 16 | 17 | ```bash 18 | LIBFEC_CACHE_DIRECTORY=$PWD/cache libfec export \ 19 | --election 2024 CA40 \ 20 | -o ca40.db 21 | ``` 22 | 23 | This will create a new `cache/` directory and cache 30 matching filings into this folder. 24 | 25 | Alternatively, you can use the `--cache-directory` flag: 26 | 27 | ```bash 28 | libfec export \ 29 | --cache-directory=cache \ 30 | --election 2024 CA40 \ 31 | -o ca40.db 32 | ``` -------------------------------------------------------------------------------- /crates/fec-cli/src/cache/bulk_candidate_committee_linkage.rs: -------------------------------------------------------------------------------- 1 | /** 2 | * > "TODO" 3 | * https://TODO 4 | * 5 | * Sample: https:TODO 6 | * 7 | */ 8 | use crate::cache::bulk_utils::{sync_item, BulkDataItem}; 9 | use anyhow::Result; 10 | use rusqlite::Transaction; 11 | use std::sync::LazyLock; 12 | 13 | static ITEM: LazyLock = LazyLock::new(|| BulkDataItem { 14 | table_name: "libfec_candidate_committee_linkages".to_owned(), 15 | url_scheme: "https://www.fec.gov/files/bulk-downloads/$YEAR/ccl$YEAR2.zip".to_string(), 16 | schema: SCHEMA.to_string(), 17 | data_file_name: "ccl.txt".to_string(), 18 | column_count: 7, 19 | }); 20 | 21 | static SCHEMA: &str = r#" 22 | CREATE TABLE IF NOT EXISTS libfec_candidate_committee_linkages( 23 | cycle INTEGER, 24 | candidate_id, 25 | candidate_election_year, 26 | fec_election_year, 27 | committee_id, 28 | committee_type, 29 | committee_designation, 30 | linkage_id, 31 | UNIQUE(cycle, linkage_id) 32 | ); 33 | "#; 34 | 35 | pub fn export(tx: &mut Transaction<'_>, year: u16) -> Result<()> { 36 | sync_item(tx, year, &ITEM)?; 37 | Ok(()) 38 | } 39 | -------------------------------------------------------------------------------- /site/reference/filings.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE libfec_filings( 2 | /** 3 | * 4 | * 5 | */ 6 | 7 | --- Unique numeric identifier for this filing, assigned by the FEC, ex 1884420 8 | filing_id TEXT PRIMARY KEY NOT NULL, 9 | 10 | --- Version of the FEC filing format, ex '8.4' 11 | fec_version TEXT NOT NULL, 12 | 13 | --- Name of the software that produced this filing, ex 'NetFile' 14 | software_name TEXT NOT NULL, 15 | 16 | --- Version of the software that produced this filing, ex '2022451' 17 | software_version TEXT NOT NULL, 18 | 19 | --- If this filing is an amendment, the report_id of the original filing, otherwise null. ex 1884419 20 | report_id INTEGER, 21 | 22 | --- Sequential number of amendments 23 | report_number TEXT, 24 | 25 | --- Any header comments provided by the filer 26 | comment TEXT, 27 | 28 | --- Form type of the cover record, ex 'F3' 29 | cover_record_form TEXT NOT NULL, 30 | cover_record_form_amendment_indicator TEXT, 31 | 32 | filer_id TEXT NOT NULL, 33 | filer_name TEXT NOT NULL, 34 | report_code TEXT, 35 | coverage_from_date TEXT, 36 | coverage_through_date TEXT 37 | ) -------------------------------------------------------------------------------- /crates/fec-py/demo.py: -------------------------------------------------------------------------------- 1 | # uv run --no-project --isolated --with 'libfec_parser @ file://../../dist/libfec_parser-0.1.0-cp39-abi3-macosx_11_0_arm64.whl' demo.py ... 2 | 3 | from libfec_parser.parser import fec_header 4 | from libfec_parser.foo import bar 5 | from pathlib import Path 6 | import sys 7 | 8 | from libfec_parser.parser import Filing 9 | 10 | # input can be 1) path or 2) bytes, or 3) a file-like/url response object 11 | f = Filing(sys.argv[1]) 12 | print(f.header) 13 | print(f.cover) 14 | for itemization in f.itemizations: 15 | print(itemization) 16 | 17 | def main() -> None: 18 | # Test the foo module 19 | print(f"bar() returns: {bar()}") 20 | 21 | # Get file paths from command line arguments 22 | if len(sys.argv) < 2: 23 | print("Usage: demo.py [fec_file2] [fec_file3] ...") 24 | print("Example: demo.py 1887722.fec") 25 | sys.exit(1) 26 | 27 | fec_files = sys.argv[1:] 28 | 29 | for fec_file in fec_files: 30 | file_path = Path(fec_file) 31 | result = fec_header(file_path.read_bytes()) 32 | print(f"{fec_file}: {result}") 33 | 34 | if __name__ == "__main__": 35 | main() -------------------------------------------------------------------------------- /dist-workspace.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = ["cargo:."] 3 | packages = ["fec-cli"] 4 | 5 | # Config for 'dist' 6 | [dist] 7 | # The preferred dist version to use in CI (Cargo.toml SemVer syntax) 8 | cargo-dist-version = "0.30.3" 9 | # CI backends to support 10 | ci = "github" 11 | # The installers to generate for each app 12 | installers = ["shell"] 13 | # Target platforms to build apps for (Rust target-triple syntax) 14 | targets = ["aarch64-apple-darwin", "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"] 15 | # The archive format to use for windows builds (defaults .zip) 16 | windows-archive = ".tar.gz" 17 | # The archive format to use for non-windows builds (defaults .tar.xz) 18 | unix-archive = ".tar.gz" 19 | # Path that installers should place binaries in 20 | install-path = "CARGO_HOME" 21 | # Whether to install an updater program 22 | install-updater = false 23 | # Local artifacts jobs to run in CI 24 | local-artifacts-jobs = ["./build-pypi"] 25 | # Publish jobs to run in CI 26 | publish-jobs = ["./publish-pypi"] 27 | 28 | [dist.github-custom-runners] 29 | aarch64-apple-darwin = "macos-latest" 30 | aarch64-unknown-linux-gnu = "ubuntu-22.04-arm" 31 | global = "ubuntu-latest" 32 | x86_64-unknown-linux-gnu = "ubuntu-22.04" 33 | -------------------------------------------------------------------------------- /crates/fec-cli/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fec-cli" 3 | version = "0.0.16" 4 | edition = "2021" 5 | authors = ["Alex Garcia "] 6 | description = "A fast FEC filing parser and toolkit" 7 | repository = "https://github.com/asg017/libfec" 8 | homepage = "https://github.com/asg017/libfec" 9 | 10 | [package.metadata.wix] 11 | upgrade-guid = "BA74C7A9-7590-48BD-B8B4-E502C5DD79B6" 12 | path-guid = "EC65EB40-397C-4215-A391-CD42F22B75C9" 13 | license = false 14 | eula = false 15 | 16 | 17 | [[bin]] 18 | name = "libfec" 19 | path = "src/main.rs" 20 | 21 | [dependencies] 22 | fec-parser = { path = "../fec-parser" } 23 | fec-api = { path = "../fec-api" } 24 | clap = { version = "4.1.8", features = ["derive", "cargo", "env"] } 25 | anyhow = {version="1.0", features=["backtrace"]} 26 | csv = "1.2.2" 27 | colored = "2.1.0" 28 | rusqlite = { version = "0.34", features = ["bundled", "jiff"] } 29 | thiserror = "1.0.63" 30 | indicatif = "0.17.8" 31 | tabled = "0.20.0" 32 | ureq = "3.0.10" 33 | quick-xml = "=0.36.1" 34 | serde_json = "1.0.125" 35 | zip = "4.2.0" 36 | url = "2.5.2" 37 | jiff = "0.2.5" 38 | rust_xlsxwriter = "0.89.0" 39 | num-format = "0.4.4" 40 | derive_builder = "0.20.2" 41 | tempfile = "3.20.0" 42 | etcetera = "0.10.0" 43 | 44 | [dev-dependencies] 45 | insta = { version = "1.45.0", features = ["yaml"] } 46 | #rusqlite = {version="0.32.1", features = ["bundled"] } -------------------------------------------------------------------------------- /crates/fec-cli/src/commands/cache.rs: -------------------------------------------------------------------------------- 1 | use colored::Colorize; 2 | use indicatif::HumanBytes; 3 | use jiff::Timestamp; 4 | 5 | use crate::{ 6 | cli::CacheArgs, 7 | sourcer::{FecFilingId, FilingSourcer}, 8 | }; 9 | 10 | pub fn cache(sourcer: FilingSourcer, args: &CacheArgs) -> anyhow::Result<()> { 11 | if args.print { 12 | println!("{}", sourcer.cache.cache_directory.display()); 13 | return Ok(()); 14 | } 15 | let t0 = jiff::Timestamp::now(); 16 | todo!("Fix cache command"); 17 | /* 18 | let stats = sourcer 19 | .cache 20 | .cache_all( 21 | &sourcer, 22 | match args.filings.clone() { 23 | Some(filings) => filings 24 | .iter() 25 | .map(|x| FecFilingId::from_str(x).unwrap()) 26 | .collect(), 27 | None => vec![], 28 | }, 29 | &args.api, 30 | ) 31 | .unwrap(); 32 | 33 | let duration = Timestamp::now() - t0; 34 | println!( 35 | "{} Cached {} filings in {:#}", 36 | "✓".green(), 37 | stats.number_downloaded + stats.number_preexisting, 38 | duration, 39 | ); 40 | println!( 41 | " {} filings downloaded ({})", 42 | stats.number_downloaded, 43 | HumanBytes(stats.downloaded_bytes as u64) 44 | ); 45 | if stats.number_preexisting > 0 { 46 | println!(" {} pre-existing", stats.number_preexisting); 47 | } 48 | */ 49 | 50 | Ok(()) 51 | } 52 | -------------------------------------------------------------------------------- /crates/fec-wasm/src/lib.rs: -------------------------------------------------------------------------------- 1 | use std::io::BufReader; 2 | use serde::{Deserialize, Serialize}; 3 | use wasm_bindgen::prelude::*; 4 | use fec_parser::{Filing, FilingHeader}; 5 | use tsify::Tsify; 6 | 7 | /// FEC Filing header 8 | #[derive(Tsify, Serialize, Deserialize)] 9 | #[tsify(into_wasm_abi, from_wasm_abi)] 10 | pub struct FilingHeaderJs { 11 | 12 | /// Record type XXX 13 | pub record_type: String, 14 | pub ef_type: String, 15 | pub fec_version: String, 16 | pub software_name: String, 17 | pub software_version: String, 18 | pub report_id: Option, 19 | pub report_number: Option, 20 | pub comment: Option, 21 | } 22 | 23 | impl FilingHeaderJs { 24 | fn from(filing_header: &FilingHeader) -> Self { 25 | Self { 26 | record_type: filing_header.record_type.clone(), 27 | ef_type: filing_header.ef_type.clone(), 28 | fec_version: filing_header.fec_version.clone(), 29 | software_name: filing_header.software_name.clone(), 30 | software_version: filing_header.software_version.clone(), 31 | report_id: filing_header.report_id.clone(), 32 | report_number: filing_header.report_number.clone(), 33 | comment: filing_header.comment.clone(), 34 | } 35 | } 36 | } 37 | 38 | /// greet bro 39 | #[wasm_bindgen] 40 | pub fn header(body: &[u8]) -> FilingHeaderJs { 41 | let f = Filing::from_reader(body, "123".to_string(), body.len()).unwrap(); 42 | FilingHeaderJs::from(&f.header) 43 | } -------------------------------------------------------------------------------- /crates/fec-wasm/README.md: -------------------------------------------------------------------------------- 1 | # fec-wasm 2 | 3 | WebAssembly bindings for the FEC parser. 4 | 5 | ## Building 6 | 7 | 1. Build the WASM module: 8 | ```bash 9 | cargo build --target wasm32-unknown-unknown --release 10 | ``` 11 | 12 | 2. Generate JavaScript bindings: 13 | ```bash 14 | wasm-bindgen --target web --out-dir pkg ../../target/wasm32-unknown-unknown/release/fec_wasm.wasm 15 | ``` 16 | 17 | Or use the combined command: 18 | ```bash 19 | cargo build --target wasm32-unknown-unknown --release && \ 20 | wasm-bindgen --target web --out-dir pkg ../../target/wasm32-unknown-unknown/release/fec_wasm.wasm 21 | ``` 22 | 23 | ## Examples 24 | 25 | ### Deno 26 | 27 | ```bash 28 | deno run --allow-read example.deno.ts 29 | ``` 30 | 31 | ### Node.js 32 | 33 | ```bash 34 | node example.node.mjs 35 | ``` 36 | 37 | ### Browser 38 | 39 | Open `index.html` in a browser (requires a local web server): 40 | 41 | ```bash 42 | # Using Python 43 | python -m http.server 8000 44 | 45 | # Using Deno 46 | deno serve --port 8000 . 47 | 48 | # Then open http://localhost:8000 in your browser 49 | ``` 50 | 51 | All examples parse the header from `1926611.fec` and display the FEC version. 52 | 53 | ## Requirements 54 | 55 | - Rust toolchain with `wasm32-unknown-unknown` target 56 | - `wasm-bindgen-cli` version 0.2.105 (must match version in Cargo.toml) 57 | 58 | Install wasm-bindgen-cli: 59 | ```bash 60 | cargo install wasm-bindgen-cli --version 0.2.105 61 | ``` 62 | 63 | Add the wasm32 target: 64 | ```bash 65 | rustup target add wasm32-unknown-unknown 66 | ``` 67 | -------------------------------------------------------------------------------- /crates/fec-cli/src/commands/export/sqlite/snapshots/libfec__commands__export__sqlite__test__tests__F1S 8.4.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: crates/fec-cli/src/commands/export/sqlite/test.rs 3 | expression: "query(&db, \"select sql from sqlite_master where name = 'libfec_F1S'\")" 4 | snapshot_kind: text 5 | --- 6 | sql 7 | ==== Row 0 ==== 8 | [sql]:CREATE TABLE [libfec_F1S]( 9 | filing_id text references libfec_filings(filing_id), 10 | form_type text, 11 | filer_committee_id_number text, 12 | joint_fund_participant_committee_name text, 13 | joint_fund_participant_committee_id_number text, 14 | joint_fund_participant_committee_type text, 15 | affiliated_committee_id_number text, 16 | affiliated_committee_name text, 17 | affiliated_candidate_id_number text, 18 | affiliated_last_name text, 19 | affiliated_first_name text, 20 | affiliated_middle_name text, 21 | affiliated_prefix text, 22 | affiliated_suffix text, 23 | affiliated_street_1 text, 24 | affiliated_street_2 text, 25 | affiliated_city text, 26 | affiliated_state text, 27 | affiliated_zip_code text, 28 | affiliated_relationship_code text, 29 | agent_last_name text, 30 | agent_first_name text, 31 | agent_middle_name text, 32 | agent_prefix text, 33 | agent_suffix text, 34 | agent_street_1 text, 35 | agent_street_2 text, 36 | agent_city text, 37 | agent_state text, 38 | agent_zip_code text, 39 | agent_title text, 40 | agent_telephone text, 41 | bank_name text, 42 | bank_street_1 text, 43 | bank_street_2 text, 44 | bank_city text, 45 | bank_state text, 46 | bank_zip_code text 47 | ) 48 | -------------------------------------------------------------------------------- /crates/fec-cli/src/commands/export/sqlite/snapshots/libfec__commands__export__sqlite__test__tests__F1S schema.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: crates/fec-cli/src/commands/export/sqlite/test.rs 3 | expression: "query(&db, \"select sql from sqlite_master where name = 'libfec_F1S'\")" 4 | snapshot_kind: text 5 | --- 6 | sql 7 | ==== Row 0 ==== 8 | [sql]:CREATE TABLE [libfec_F1S]( 9 | filing_id text references libfec_filings(filing_id), 10 | form_type text, 11 | filer_committee_id_number text, 12 | joint_fund_participant_committee_name text, 13 | joint_fund_participant_committee_id_number text, 14 | joint_fund_participant_committee_type text, 15 | affiliated_committee_id_number text, 16 | affiliated_committee_name text, 17 | affiliated_candidate_id_number text, 18 | affiliated_last_name text, 19 | affiliated_first_name text, 20 | affiliated_middle_name text, 21 | affiliated_prefix text, 22 | affiliated_suffix text, 23 | affiliated_street_1 text, 24 | affiliated_street_2 text, 25 | affiliated_city text, 26 | affiliated_state text, 27 | affiliated_zip_code text, 28 | affiliated_relationship_code text, 29 | agent_last_name text, 30 | agent_first_name text, 31 | agent_middle_name text, 32 | agent_prefix text, 33 | agent_suffix text, 34 | agent_street_1 text, 35 | agent_street_2 text, 36 | agent_city text, 37 | agent_state text, 38 | agent_zip_code text, 39 | agent_title text, 40 | agent_telephone text, 41 | bank_name text, 42 | bank_street_1 text, 43 | bank_street_2 text, 44 | bank_city text, 45 | bank_state text, 46 | bank_zip_code text 47 | ) 48 | -------------------------------------------------------------------------------- /crates/fec-cli/src/commands/export/sqlite/snapshots/libfec__commands__export__sqlite__test__tests__8.5 2nd.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: crates/fec-cli/src/commands/export/sqlite/test.rs 3 | expression: "query(&db, \"select * from libfec_f99\")" 4 | snapshot_kind: text 5 | --- 6 | filing_id | form_type | filer_committee_id_number | committee_name | street_1 | street_2 | city | state | zip_code | treasurer_last_name | treasurer_first_name | treasurer_middle_name | treasurer_prefix | treasurer_suffix | date_signed | text_code | filing_frequency | pdf_attachment | text 7 | ==== Row 0 ==== 8 | [filing_id]:1913493 9 | [form_type]:F99 10 | [filer_committee_id_number]:C00917203 11 | [committee_name]:RIGHTS OF AMERICANS ASSOCIATION 12 | [street_1]:2444 W. TOUHY AVENUE 13 | [street_2]: 14 | [city]:CHICAGO 15 | [state]:IL 16 | [zip_code]:60645 17 | [treasurer_last_name]:Hill 18 | [treasurer_first_name]:Tamika 19 | [treasurer_middle_name]:La'Shon 20 | [treasurer_prefix]:Mrs. 21 | [treasurer_suffix]: 22 | [date_signed]:2025-09-01 23 | [text_code]: 24 | [filing_frequency]: 25 | [pdf_attachment]: 26 | [text]: 27 | ==== Row 1 ==== 28 | [filing_id]:1913595 29 | [form_type]:F99 30 | [filer_committee_id_number]:C00776393 31 | [committee_name]:COMMITTEE TO ELECT MIKE EZELL 32 | [street_1]:P.O. BOX 17784 33 | [street_2]: 34 | [city]:HATTIESBURG 35 | [state]:MS 36 | [zip_code]:39404 37 | [treasurer_last_name]:HOBBS 38 | [treasurer_first_name]:CABELL 39 | [treasurer_middle_name]: 40 | [treasurer_prefix]: 41 | [treasurer_suffix]: 42 | [date_signed]:2025-09-02 43 | [text_code]:MST 44 | [filing_frequency]: 45 | [pdf_attachment]: 46 | [text]: 47 | -------------------------------------------------------------------------------- /crates/fec-py/tests/conftest.py: -------------------------------------------------------------------------------- 1 | """ 2 | Pytest configuration and shared fixtures for libfec_parser tests 3 | """ 4 | import pytest 5 | from pathlib import Path 6 | 7 | 8 | def pytest_configure(config): 9 | """Configure pytest with custom markers""" 10 | config.addinivalue_line( 11 | "markers", "slow: marks tests as slow (deselect with '-m \"not slow\"')" 12 | ) 13 | config.addinivalue_line( 14 | "markers", "network: marks tests that require network access" 15 | ) 16 | 17 | 18 | @pytest.fixture(scope="session") 19 | def project_root(): 20 | """Get the project root directory""" 21 | return Path(__file__).parent.parent.parent.parent 22 | 23 | 24 | @pytest.fixture(scope="session") 25 | def cache_dir(project_root): 26 | """Get the cache directory with FEC files""" 27 | return project_root / "cache" 28 | 29 | 30 | @pytest.fixture(scope="session") 31 | def benchmarks_dir(project_root): 32 | """Get the benchmarks directory with FEC files""" 33 | return project_root / "benchmarks" 34 | 35 | 36 | @pytest.fixture(scope="session") 37 | def available_fec_files(cache_dir, benchmarks_dir): 38 | """Get list of all available FEC test files""" 39 | files = [] 40 | 41 | if cache_dir.exists(): 42 | files.extend(list(cache_dir.glob("*.fec"))) 43 | 44 | if benchmarks_dir.exists(): 45 | files.extend(list(benchmarks_dir.glob("*.fec"))) 46 | 47 | return files 48 | 49 | 50 | @pytest.fixture 51 | def first_fec_file(available_fec_files): 52 | """Get the first available FEC file for testing""" 53 | if not available_fec_files: 54 | pytest.skip("No FEC test files available") 55 | return available_fec_files[0] 56 | -------------------------------------------------------------------------------- /crates/fec-cli/src/cache/bulk_committee.rs: -------------------------------------------------------------------------------- 1 | /** 2 | * > "The committee master file contains one record for each committee registered with the Federal Election Commission. This includes federal political action committees and party committees, campaign committees for presidential, house and senate candidates, as well as groups or organizations who are spending money for or against candidates for federal office." 3 | * https://www.fec.gov/campaign-finance-data/committee-master-file-description/ 4 | * 5 | * Sample: https://www.fec.gov/files/bulk-downloads/2026/cm26.zip 6 | * 7 | */ 8 | use crate::cache::bulk_utils::{sync_item, BulkDataItem}; 9 | use anyhow::{Context, Result}; 10 | use derive_builder::Builder; 11 | use fec_api::Office; 12 | use rusqlite::{Connection, Transaction}; 13 | use std::sync::LazyLock; 14 | 15 | static ITEM: LazyLock = LazyLock::new(|| BulkDataItem { 16 | table_name: "libfec_committees".to_owned(), 17 | url_scheme: "https://www.fec.gov/files/bulk-downloads/$YEAR/cm$YEAR2.zip".to_string(), 18 | schema: SCHEMA.to_string(), 19 | data_file_name: "cm.txt".to_string(), 20 | column_count: 15, 21 | }); 22 | 23 | static SCHEMA: &str = r#" 24 | CREATE TABLE IF NOT EXISTS libfec_committees( 25 | cycle INTEGER, 26 | committee_id TEXT, 27 | name, 28 | treasurer_name, 29 | address_street1, 30 | address_street2, 31 | address_city, 32 | address_state, 33 | address_zip, 34 | designation, 35 | committee_type, 36 | party_affiliation, 37 | filing_frequency, 38 | interest_group_category, 39 | connected_org_name, 40 | candidate_id, 41 | UNIQUE(cycle, committee_id) 42 | ); 43 | "#; 44 | 45 | pub fn export(tx: &mut Transaction<'_>, year: u16) -> Result<()> { 46 | sync_item(tx, year, &ITEM)?; 47 | Ok(()) 48 | } 49 | -------------------------------------------------------------------------------- /benchmarks/README.md: -------------------------------------------------------------------------------- 1 | # `libfec` Benchmarks 2 | 3 | The `libfec` tool is very fast! Along with the [builtin cache](./guides/cache), `libfec` is likely the fastest tool for working with FEC filings. 4 | 5 | For example, the largest single FEC filing I could find is [`FEC-1909062`](https://docquery.fec.gov/cgi-bin/forms/C00401224/1909062/), the `10GB` Mid-Year 2025 report for ActBlue. To export all reported contributions in that filing, you can run: 6 | 7 | ```bash 8 | libfec export 1909062.fec \ 9 | --target receipts \ 10 | -o ab-2025-h1.csv 11 | ``` 12 | 13 | On my machine (2024 MacBook Pro M4 Pro) this takes 31 seconds, exporting a `5.8 GB` CSV with 25 million rows! 14 | 15 | Other formats are supported to, like SQLite: 16 | 17 | ```bash 18 | libfec export 1909062.fec \ 19 | -o ab-2025-h1.db 20 | ``` 21 | 22 | This is slower, clocking in at 2 minutes 10 seconds, generating a `11GB` file. Exporting to SQLite will import all itemizations, including receipts and disbursements (while the above CSV export only export receipts). Also inserting into a SQLite table is always slower than writing a CSV. 23 | 24 | ## Compared with `fastfec` 25 | 26 | The most similar tool to `libfec` is [`fastfec` from the Washington Post](https://github.com/washingtonpost/FastFEC). They both do very different things, so drawing a direct comparison is hard to do. But `libfec` has a [`libfec fastfec`](https://alexgarcia.xyz/libfec/guides/fastfec.html) subcommand that aims to do the same thing as `fastfec` (convert a single `.fec` file to a directory of CSVs by form type). 27 | 28 | For example, a sample `fastfec` command would be: 29 | 30 | ```bash 31 | fastfec 1805248.fec output/ 32 | ``` 33 | 34 | And the `libfec fastfec` version would look like: 35 | 36 | ```bash 37 | libfec fastfec 1805248.fec output/ 38 | ``` 39 | 40 | Now, 41 | 42 | 43 | -------------------------------------------------------------------------------- /crates/fec-cli/src/cache/bulk_opexp.rs: -------------------------------------------------------------------------------- 1 | /** 2 | * > "This file contains disbursements reported on FEC Form 3 Line 17, 3 | * FEC Form 3P Line 23, and FEC Form 3X Lines 21(a)(i), 21(a)(ii) and 21(b)." 4 | * https://www.fec.gov/campaign-finance-data/operating-expenditures-file-description/ 5 | * 6 | * Sample: https://www.fec.gov/files/bulk-downloads/2026/oppexp26.zip 7 | * 8 | */ 9 | use crate::cache::bulk_utils::{sync_item, BulkDataItem}; 10 | use anyhow::{Context, Result}; 11 | use derive_builder::Builder; 12 | use fec_api::Office; 13 | use rusqlite::{Connection, Transaction}; 14 | use std::sync::LazyLock; 15 | 16 | static ITEM: LazyLock = LazyLock::new(|| BulkDataItem { 17 | table_name: "operating_expenses".to_owned(), 18 | url_scheme: "https://www.fec.gov/files/bulk-downloads/$YEAR/oppexp$YEAR2.zip".to_string(), 19 | schema: SCHEMA.to_string(), 20 | data_file_name: "oppexp.txt".to_string(), 21 | column_count: 25, 22 | }); 23 | 24 | static SCHEMA: &str = r#" 25 | CREATE TABLE IF NOT EXISTS operating_expenses( 26 | cycle INTEGER, 27 | committee_id, 28 | amendment_indicator, 29 | report_year INTEGER, 30 | report_type, 31 | image_number, 32 | line_number, 33 | form_type_code, 34 | schedule_type_code, 35 | name, 36 | city, 37 | state, 38 | zip_code, 39 | transaction_date, 40 | transaction_amount float, 41 | transaction_pgi, 42 | purpose, 43 | category, 44 | category_description, 45 | memo_code, 46 | memo_text, 47 | entity_type, 48 | sub_id integer, 49 | filing_id integer, 50 | transaction_id, 51 | back_reference_transaction_id, 52 | UNIQUE(cycle, sub_id) 53 | ); 54 | 55 | "#; 56 | 57 | pub fn export(tx: &mut Transaction<'_>, year: u16) -> Result<()> { 58 | let result = sync_item(tx, year, &ITEM)?; 59 | println!("opeexp {year} {result:?}"); 60 | Ok(()) 61 | } 62 | -------------------------------------------------------------------------------- /site/guides/actblue-winred.md: -------------------------------------------------------------------------------- 1 | # Analyzing ActBlue & WinRed filings with libfec 2 | 3 | [ActBlue](https://www.actblue.com/) ([`C00401224`](https://www.fec.gov/data/committee/C00401224/)) and [WinRed](https://winred.com/) ([`C00694323`](https://www.fec.gov/data/committee/C00694323/)) are the two largest federal campaign fundraising platforms for Democrats and Republicans, respectively. 4 | 5 | They act as [conduits](https://moneyinpolitics.wtf/conduit/) between individual donors and campaigns/committees. People donate *through* these online platforms (typically with credit cards) to whichever campaign or committee they care about. These [earmarked contributions](https://moneyinpolitics.wtf/earmarked-contribution/) are noted as such within FEC filings, and are reported in both ActBlue/WinRed filings (as disbursements) and committee filings (as receipts). 6 | 7 | Because they are conduits, ActBlue and WinRed are required to report *all* earmarked contributions they receive. This is unique! Typically, campaign and PACs are only required to report individuals who donated $200 or more for a specific election, meaning small-dollar donors don't appear in committee filings. However, those small-dollar donors will appear in ActBlue/WinRed filings. 8 | 9 | ## ActBlue and WinRed filings are *big* 10 | 11 | A vast majority of PACs submit very small FEC filings, < `10KB`. Most well-funded House or Senate campaigns may include hundreds or thousands of contributors, but they filings will still be between `500KB` or `5MB` large. Presidential campaigns or political parties may have filings up to the 10's or 100's of MB, like Kamala Harris's `571 MB` [Post-General 2024 report](https://docquery.fec.gov/cgi-bin/forms/C00703975/1890563/). 12 | 13 | But ActBlue and WinRed filings are on another level, consistently between `2GB` and `8GB`. Their [Mid-Year 2025 report](https://docquery.fec.gov/dcdev/posted/1909062.fec) alone clocks in at `10.3 GB`. 14 | 15 | -------------------------------------------------------------------------------- /site/reference/cli-export-help.txt: -------------------------------------------------------------------------------- 1 | Export FEC filings into SQLite, Excel, CSV, or JSON 2 | 3 | Usage: libfec export [OPTIONS] [FILINGS]... 4 | 5 | Arguments: 6 | [FILINGS]... 7 | 8 | Options: 9 | -f, --format Output file [possible values: sqlite, excel, csv, json] 10 | --target Output file [possible values: schedule-a, schedule-b] 11 | -o, --output Output file 12 | --output-directory Output directory 13 | --clobber Overwrite existing files 14 | --cover-only Only export cover records, not itemizations 15 | -h, --help Print help 16 | 17 | FEC API options: 18 | --committee 19 | Filter filings to only these committees 20 | --candidate 21 | Filter filings to only these candidates 22 | --form-type 23 | Form type to filter filings by (e.g. 'F3', 'F3X', 'F1', 'F2', etc) 24 | --committee-type 25 | one-letter type code of the organization 26 | --cycle 27 | Election cycle 28 | --coverage-before 29 | Only filings that cover dates before this date 30 | --coverage-after 31 | Only filings that cover dates after this date 32 | --coverage-between 33 | Only filings container coverage between these dates 34 | --election 35 | 36 | --state 37 | 38 | --district 39 | 40 | --office 41 | 42 | --bulk-daily-between 43 | 44 | --api-key 45 | API key to use for OpenFEC API requests. If not provided, the DEMO_KEY will be used. 46 | 47 | Global options: 48 | --cache-directory [env: LIBFEC_CACHE_DIRECTORY=] 49 | -------------------------------------------------------------------------------- /.github/workflows/build-pypi.yml: -------------------------------------------------------------------------------- 1 | # Build libfec wheels. 2 | # 3 | # Assumed to run as a subworkflow of .github/workflows/release.yml; specifically, as a local 4 | # artifacts job within `cargo-dist`. 5 | # 6 | # courtesy: https://github.com/astral-sh/uv/blob/34b5afcba6c89046cf8eb97efb15ae3d18015570/.github/workflows/build-binaries.yml#L57 7 | name: "Build PyPi wheels" 8 | 9 | on: 10 | workflow_call: 11 | inputs: 12 | plan: 13 | required: true 14 | type: string 15 | 16 | jobs: 17 | macos-x86_64: 18 | runs-on: macos-14 19 | steps: 20 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 21 | - uses: astral-sh/setup-uv@v6 22 | - run: rustup target add x86_64-apple-darwin 23 | - run: uvx maturin build -b bin --release --locked --out dist/ --target x86_64-apple-darwin 24 | working-directory: crates/fec-cli 25 | - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 26 | with: 27 | name: wheels_libfec-macos-x86_64 28 | path: crates/fec-cli/dist 29 | macos-aarch64: 30 | runs-on: macos-latest 31 | steps: 32 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 33 | - uses: astral-sh/setup-uv@v6 34 | - run: uvx maturin build -b bin --release --locked --out dist/ 35 | working-directory: crates/fec-cli 36 | - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 37 | with: 38 | name: wheels_libfec-macos-aarch64 39 | path: crates/fec-cli/dist 40 | linux-x86_64: 41 | runs-on: ubuntu-latest 42 | steps: 43 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 44 | - uses: astral-sh/setup-uv@v6 45 | - run: uvx maturin build -b bin --release --locked --out dist/ 46 | working-directory: crates/fec-cli 47 | - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 48 | with: 49 | name: wheels_libfec-linux-x86_64 50 | path: crates/fec-cli/dist -------------------------------------------------------------------------------- /site/.vitepress/config.mts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vitepress' 2 | 3 | // https://vitepress.dev/reference/site-config 4 | export default defineConfig({ 5 | title: "libfec", 6 | description: "Tools to work with campaign finance data from the FEC", 7 | appearance: false, 8 | base: "/libfec/", 9 | themeConfig: { 10 | // https://vitepress.dev/reference/default-theme-config 11 | nav: [ 12 | { text: 'Examples', link: '/examples' } 13 | ], 14 | 15 | sidebar: [ 16 | { 17 | text: 'Getting Started', 18 | items: [ 19 | /*{ text: 'Intro to Campaign Finance', link: '/getting-started/intro-campfin' },*/ 20 | { text: 'Installation', link: '/getting-started/installation' }, 21 | { text: 'Basic Usage', link: '/getting-started/basic-usage' }, 22 | ] 23 | }, 24 | /* 25 | { 26 | text: 'Examples', 27 | items: [ 28 | {text: "Filing Deadlines", link: '/examples/filing-deadlines'}, 29 | ], 30 | },*/ 31 | { 32 | text: 'Guides', 33 | items: [ 34 | /* 35 | { text: 'Contributions & Receipts', link: '/guides/contributions' }, 36 | { text: 'Expenditures & Disbursements', link: '/guides/expenditures' }, 37 | { text: 'Elections', link: '/guides/elections' }, 38 | { text: 'ActBlue & WinRed', link: '/guides/actblue-winred' }, 39 | { text: 'Independent Expenditures', link: '/guides/independent-expenditures' }, 40 | */ 41 | //{ text: 'Campaigns', link: '/guides/campaigns' }, 42 | { text: 'Caching', link: '/guides/cache' }, 43 | { text: 'FastFEC Compatibility', link: '/guides/fastfec' }, 44 | ] 45 | }, 46 | { 47 | text: 'Reference', 48 | items: [ 49 | { text: 'CLI Reference', link: '/reference/cli' }, 50 | { text: 'SQL Reference', link: '/reference/sql' }, 51 | ], 52 | 53 | }, 54 | ], 55 | 56 | socialLinks: [ 57 | { icon: 'github', link: 'https://github.com/asg017/libfec' } 58 | ] 59 | } 60 | }) 61 | -------------------------------------------------------------------------------- /site/getting-started/installation.md: -------------------------------------------------------------------------------- 1 | # Installing `libfec` 2 | 3 | `libfec` is a command line interface (CLI) for interacting with federal campaign finance data from the FEC. While it's written in Rust, you personally don't need to know or even have Rust installed to use it. Pre-compiled binaries are made available for every `libfec` release for Windows, MacOS, and Linux users. 4 | 5 | Use one of the methods detailed below to install the `libfec` CLI on your machine. 6 | 7 | ## Recommended install script 8 | 9 | The easiest way to install the `libfec` CLI is with the following installation script: 10 | 11 | ```bash 12 | curl --proto '=https' --tlsv1.2 -LsSf https://github.com/asg017/libfec/releases/download/0.0.12/fec-cli-installer.sh | sh 13 | ``` 14 | 15 | Once ran, you can run the `libfec` command line by directly calling: 16 | 17 | ```sh 18 | libfec --help 19 | ``` 20 | 21 | ## With the `libfec` PyPi Package 22 | 23 | If you're a Python user, the [`libfec` PyPi package](https://pypi.org/project/libfec/) includes pre-built binaries, so you can `pip install libfec` the CLI. 24 | 25 | If you use [uv](https://github.com/astral-sh/uv), you can use `uvx` for quick-and-dirty one-off scripts: 26 | 27 | ```bash 28 | uvx libfec --help 29 | ``` 30 | 31 | Or use [`uv tool install`](https://docs.astral.sh/uv/reference/cli/#uv-tool-install) for an alternative global install: 32 | 33 | ```bash 34 | uv tool install libfec 35 | libfec --help 36 | ``` 37 | 38 | ## Manually 39 | 40 | Find a [recent release of `libfec`](https://github.com/asg017/libfec/releases) on the project's Releases page and manually download the correct package for your platform. 41 | 42 | ## Building yourself 43 | 44 | Ensure you have Rust installed, then run: 45 | 46 | ```bash 47 | cargo install --git https://github.com/asg017/libfec.git fec-cli 48 | ``` 49 | 50 | 51 | 52 | Alternatively you can manually clone the repository and run `cargo build --release`: 53 | 54 | ```bash 55 | git clone git@github.com:asg017/libfec.git 56 | cd libfec 57 | cargo build --release 58 | ``` 59 | 60 | That will build the `libfec` CLI on your computer, and will be available at `target/release/libfec`. -------------------------------------------------------------------------------- /crates/fec-cli/src/commands/export/sqlite/snapshots/libfec__commands__export__sqlite__test__tests__basic_f99.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: crates/fec-cli/src/commands/export/sqlite/test.rs 3 | expression: "query(&db, \"select name, sql from sqlite_master where type='table'\")" 4 | snapshot_kind: text 5 | --- 6 | name | sql 7 | ==== Row 0 ==== 8 | [name]:libfec_filings 9 | [sql]:CREATE TABLE libfec_filings( 10 | /** 11 | * 12 | * 13 | */ 14 | 15 | --- Unique numeric identifier for this filing, assigned by the FEC, ex 1884420 16 | filing_id TEXT PRIMARY KEY NOT NULL, 17 | 18 | --- Version of the FEC filing format, ex '8.4' 19 | fec_version TEXT NOT NULL, 20 | 21 | --- Name of the software that produced this filing, ex 'NetFile' 22 | software_name TEXT NOT NULL, 23 | 24 | --- Version of the software that produced this filing, ex '2022451' 25 | software_version TEXT NOT NULL, 26 | 27 | --- If this filing is an amendment, the report_id of the original filing, otherwise null. ex 1884419 28 | report_id TEXT, 29 | 30 | --- Sequential number of amendments 31 | report_number TEXT, 32 | 33 | --- Any header comments provided by the filer 34 | comment TEXT, 35 | 36 | --- Form type of the cover record, ex 'F3' 37 | cover_record_form TEXT NOT NULL, 38 | cover_record_form_amendment_indicator TEXT, 39 | 40 | filer_id TEXT NOT NULL, 41 | filer_name TEXT NOT NULL, 42 | report_code TEXT, 43 | coverage_from_date TEXT, 44 | coverage_through_date TEXT 45 | ) 46 | ==== Row 1 ==== 47 | [name]:libfec_F99 48 | [sql]:CREATE TABLE [libfec_F99]( 49 | filing_id text references libfec_filings(filing_id), 50 | form_type text, 51 | filer_committee_id_number text, 52 | committee_name text, 53 | street_1 text, 54 | street_2 text, 55 | city text, 56 | state text, 57 | zip_code text, 58 | treasurer_last_name text, 59 | treasurer_first_name text, 60 | treasurer_middle_name text, 61 | treasurer_prefix text, 62 | treasurer_suffix text, 63 | date_signed date, 64 | text_code text, 65 | filing_frequency text, 66 | pdf_attachment text, 67 | text text 68 | ) 69 | -------------------------------------------------------------------------------- /crates/fec-parser/src/covers/mod.rs: -------------------------------------------------------------------------------- 1 | mod form3p; 2 | pub use crate::covers::form3p::{Form3P, Form3PSummary}; 3 | use indexmap::IndexMap; 4 | 5 | pub enum Cover { 6 | Form3P(Form3P), 7 | } 8 | 9 | pub(crate) fn cover_from_form_type( 10 | cover_record_form_type: &str, 11 | data: &IndexMap, 12 | ) -> Option { 13 | // TODO collides with F3PS? 14 | if cover_record_form_type.starts_with("F3P") { 15 | return Form3P::from_data(data).map(Cover::Form3P); 16 | } 17 | None 18 | } 19 | 20 | pub struct Treasurer { 21 | pub first_name: String, 22 | pub last_name: String, 23 | pub middle_name: Option, 24 | pub prefix: Option, 25 | pub suffix: Option, 26 | } 27 | impl ToString for Treasurer { 28 | fn to_string(&self) -> String { 29 | let mut name = String::new(); 30 | if let Some(prefix) = &self.prefix { 31 | name.push_str(prefix.trim()); 32 | name.push(' '); 33 | } 34 | name.push_str(&self.first_name.trim()); 35 | if let Some(middle_name) = &self.middle_name { 36 | name.push(' '); 37 | name.push_str(middle_name.trim()); 38 | } 39 | name.push(' '); 40 | name.push_str(&self.last_name.trim()); 41 | if let Some(suffix) = &self.suffix { 42 | name.push(' '); 43 | name.push_str(suffix.trim()); 44 | } 45 | name.trim().to_string() 46 | } 47 | } 48 | impl Treasurer { 49 | pub(crate) fn from_data(data: &IndexMap) -> Self { 50 | let first_name = data 51 | .get("treasurer_first_name") 52 | .cloned() 53 | .unwrap_or_default(); 54 | let last_name = data.get("treasurer_last_name").cloned().unwrap_or_default(); 55 | let middle_name = data 56 | .get("treasurer_middle_name") 57 | .cloned() 58 | .filter(|s| !s.is_empty()); 59 | let prefix = data 60 | .get("treasurer_prefix") 61 | .cloned() 62 | .filter(|s| !s.is_empty()); 63 | let suffix = data 64 | .get("treasurer_suffix") 65 | .cloned() 66 | .filter(|s| !s.is_empty()); 67 | 68 | Self { 69 | first_name, 70 | last_name, 71 | middle_name, 72 | prefix, 73 | suffix, 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /crates/fec-cli/src/commands/bulk.rs: -------------------------------------------------------------------------------- 1 | use crate::{ 2 | cache::{bulk_candidate_committee_linkage, bulk_candidates, bulk_committee, bulk_opexp}, 3 | cli::{BulkArgs, BulkSource, CycleArg}, 4 | sourcer::FilingSourcer, 5 | }; 6 | use indicatif::{ProgressBar, ProgressStyle}; 7 | 8 | pub fn bulk(sourcer: FilingSourcer, args: &BulkArgs) -> anyhow::Result<()> { 9 | println!("Running bulk command with args: {:?}", args); 10 | let mut db = rusqlite::Connection::open(&args.output)?; 11 | 12 | // Convert cycle argument to a vector of even years 13 | let cycles: Vec = match args.cycle { 14 | CycleArg::Single(year) => vec![year], 15 | CycleArg::Range(start, end) => { 16 | (start..=end) 17 | .filter(|y| y % 2 == 0) 18 | .collect() 19 | } 20 | }; 21 | 22 | let mut tx = db.transaction()?; 23 | 24 | // Calculate total steps for progress bar 25 | let mut total_steps = cycles.len() * args.source.len(); 26 | if args.source.contains(&BulkSource::Candidates) && args.source.contains(&BulkSource::Committees) { 27 | total_steps += cycles.len(); // Add linkage steps 28 | } 29 | 30 | let pb = ProgressBar::new(total_steps as u64); 31 | pb.set_style( 32 | ProgressStyle::default_bar() 33 | .template("{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} {msg}") 34 | .unwrap() 35 | .progress_chars("#>-") 36 | ); 37 | 38 | for year in cycles { 39 | for source in &args.source { 40 | pb.set_message(format!("{} - {:?}", year, source)); 41 | let result = match source { 42 | BulkSource::Opex => bulk_opexp::export(&mut tx, year), 43 | BulkSource::Committees => bulk_committee::export(&mut tx, year), 44 | BulkSource::Candidates => bulk_candidates::export(&mut tx, year), 45 | }; 46 | result.unwrap(); 47 | pb.inc(1); 48 | } 49 | if args.source.contains(&BulkSource::Candidates) && args.source.contains(&BulkSource::Committees) { 50 | pb.set_message(format!("{} - Candidate-Committee Linkage", year)); 51 | bulk_candidate_committee_linkage::export(&mut tx, year)?; 52 | pb.inc(1); 53 | } 54 | } 55 | 56 | pb.finish_with_message("Complete!"); 57 | tx.commit()?; 58 | Ok(()) 59 | } 60 | -------------------------------------------------------------------------------- /crates/fec-parser/src/mappings.rs: -------------------------------------------------------------------------------- 1 | use regex::{RegexSet, RegexSetBuilder}; 2 | use std::collections::HashSet; 3 | 4 | use fec_parser_macros::{ 5 | gen_column_names, gen_date_columns, gen_float_columns, gen_form_type_version_set, 6 | gen_form_types, 7 | }; 8 | 9 | lazy_static::lazy_static! { 10 | pub static ref DATE_COLUMNS: HashSet = HashSet::from(gen_date_columns!("")); 11 | } 12 | lazy_static::lazy_static! { 13 | pub static ref FLOAT_COLUMNS: HashSet = HashSet::from(gen_float_columns!("")); 14 | } 15 | 16 | pub static FORM_TYPES: &[&str] = &gen_form_types!(""); 17 | 18 | lazy_static::lazy_static! { 19 | pub static ref FORM_TYPES_SET: RegexSet = RegexSetBuilder::new(FORM_TYPES) 20 | .case_insensitive(true) 21 | .build() 22 | .expect("Static regex set to compile for FORM_TYPES"); 23 | } 24 | lazy_static::lazy_static! { 25 | pub static ref FORM_TYPE_VERSIONS_SET: Vec = gen_form_type_version_set!(""); 26 | } 27 | 28 | lazy_static::lazy_static! { 29 | pub static ref COLUMN_NAMES: Vec>> = gen_column_names!(); 30 | } 31 | 32 | pub fn field_idx(field: &str) -> Option { 33 | let matches = FORM_TYPES_SET.matches(field); 34 | matches.iter().next() 35 | } 36 | 37 | pub fn column_names_for_field<'a>( 38 | form_type: &str, 39 | fec_version: &str, 40 | ) -> anyhow::Result<&'a Vec> { 41 | let idx = field_idx(form_type).ok_or_else(|| { 42 | anyhow::anyhow!(format!( 43 | "Unknown form type '{}'; cannot determine filed idx", 44 | form_type 45 | )) 46 | })?; 47 | let idx2 = FORM_TYPE_VERSIONS_SET 48 | .get(idx) 49 | .ok_or_else(|| { 50 | anyhow::anyhow!(format!( 51 | "No form type versions regex set for form type '{}'", 52 | form_type 53 | )) 54 | })? 55 | .matches(fec_version) 56 | .iter() 57 | .next() 58 | .ok_or_else(|| { 59 | anyhow::anyhow!(format!( 60 | "Unknown FEC version '{}' for form type '{}'; cannot determine filed idx2", 61 | fec_version, form_type 62 | )) 63 | })?; 64 | let columns = COLUMN_NAMES 65 | .get(idx) 66 | .ok_or_else(|| anyhow::anyhow!(format!("No column names for form type '{}'", form_type)))? 67 | .get(idx2) 68 | .ok_or_else(|| { 69 | anyhow::anyhow!(format!( 70 | "No column names for form type '{}' and FEC version '{}'", 71 | form_type, fec_version 72 | )) 73 | })?; 74 | Ok(columns) 75 | } 76 | -------------------------------------------------------------------------------- /crates/fec-cli/src/commands/export/mod.rs: -------------------------------------------------------------------------------- 1 | mod dir_csv; 2 | mod excel; 3 | mod single; 4 | pub mod sqlite; 5 | use anyhow::anyhow; 6 | use excel::cmd_export_excel; 7 | use sqlite::cmd_export_sqlite; 8 | 9 | use crate::{ 10 | cli::{ExportArgs, ExportFormat}, 11 | sourcer::FilingSourcer, 12 | }; 13 | 14 | pub fn export(sourcer: FilingSourcer, args: ExportArgs) -> anyhow::Result<()> { 15 | let result = match (args.output.clone(), args.output_directory.clone()) { 16 | (None, None) => { 17 | Err(anyhow!("Must specify either --output (output to a file) or --output-directory (output multiple files to a directory)")) 18 | } 19 | (Some(_), Some(_)) => { 20 | Err(anyhow!("Cannot specify both --output and --output-directory")) 21 | } 22 | (Some(output), None) => { 23 | if args.clobber && output.exists() { 24 | std::fs::remove_file(&output)?; 25 | } 26 | match output.extension().and_then(|s| s.to_str()) { 27 | Some("db") => cmd_export_sqlite(sourcer, output, args), 28 | Some("xlsx") => cmd_export_excel(sourcer, output, args), 29 | Some("csv") => { 30 | if let Some(target) = args.target { 31 | single::cmd_export_single(sourcer, output, args, target, single::SingleOutput::Csv) 32 | }else { 33 | Err(anyhow!("Must specify --target when exporting to a single CSV file")) 34 | } 35 | }, 36 | Some("json") => { 37 | if let Some(target) = args.target { 38 | single::cmd_export_single(sourcer, output, args, target, single::SingleOutput::Json) 39 | }else { 40 | Err(anyhow!("Must specify --target when exporting to a single JSON file")) 41 | } 42 | }, 43 | Some(_) | None => Err(anyhow!("Unsupported output format")), 44 | } 45 | } 46 | (None, Some(output_directory)) => { 47 | std::fs::create_dir_all(&output_directory)?; 48 | match args.format { 49 | None => Err(anyhow!("Must specify --format when using --output-directory")), 50 | Some(ExportFormat::Sqlite) => todo!(), 51 | Some(ExportFormat::Excel) => todo!(), 52 | Some(ExportFormat::Csv) => { 53 | dir_csv::export(sourcer, args, output_directory) 54 | }, 55 | Some(ExportFormat::Json) => todo!(), 56 | } 57 | } 58 | }; 59 | /* 60 | let result = match args.output.extension().map(|s| s.to_str()).flatten() { 61 | Some("db") => cmd_export_sqlite(sourcer, args), 62 | Some("xlsx") => cmd_export_excel(sourcer, args), 63 | Some(_) | None => Err(anyhow!("asdf").into()), 64 | }; 65 | */ 66 | match result { 67 | Ok(_) => Ok(()), 68 | Err(e) => { 69 | eprintln!("Error exporting: {:?}", e); 70 | Err(e) 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /site/reference/sql.md: -------------------------------------------------------------------------------- 1 | 2 | # `libfec` SQL Reference 3 | 4 | > [!WARNING] 5 | > This documentation is incomplete! 6 | 7 | The [`libfec export`](./cli.md#export) command can export FEC filings into a SQLite database. All header, cover, and itemizations records from a FEC filing are inserted into various SQLite tables, allowing you to write SQL to extract out the exact information you need. 8 | 9 | 10 | ```bash 11 | libfec export --committee= 12 | 13 | ``` 14 | 15 | The SQL tables that are created are based on 16 | 17 | 18 | ## `libfec_filings` 19 | 20 |
21 | See libfec_filings SQL schema 22 | 23 | ```sql 24 | 25 | ``` 26 | 27 |
28 | 29 | 30 | The `libfec_filings` table contains a single row for every FEC filing export. That rows contains the 31 | (["header record"](https://docs.google.com/spreadsheets/d/1ZoUxmMw-X_I8DGMBYh-857hqgAzyYX2z/edit?gid=1892984062#gid=1892984062)) and "cover record" 32 | for a given filing. For example: 33 | 34 | ```bash 35 | libfec export FEC-1848680 FEC-1870171 -o sanchez.db 36 | ``` 37 | 38 | The `libfec_filings` table inside of `sanchez.db` will have 2 rows in the table: 39 | 40 | ```sql 41 | select 42 | filing_id, 43 | cover_record_form, 44 | filer_id, 45 | filer_name, 46 | report_code 47 | from libfec_filings; 48 | ``` 49 | 50 | ``` 51 | ┌───────────┬───────────────────┬───────────┬────────────────────┬─────────────┐ 52 | │ filing_id │ cover_record_form │ filer_id │ filer_name │ report_code │ 53 | ├───────────┼───────────────────┼───────────┼────────────────────┼─────────────┤ 54 | │ 1870171 │ F3 │ C00384057 │ Stand With Sanchez │ YE │ 55 | ├───────────┼───────────────────┼───────────┼────────────────────┼─────────────┤ 56 | │ 1848680 │ F1 │ C00384057 │ Stand With Sanchez │ │ 57 | └───────────┴───────────────────┴───────────┴────────────────────┴─────────────┘ 58 | ``` 59 | 60 | The first filing, 61 | [`FEC-1870171`](https://docquery.fec.gov/cgi-bin/forms/C00384057/1870171/) 62 | , is the 63 | [FEC Form 3 "Year-End"](https://www.fec.gov/resources/cms-content/documents/policy-guidance/fecfrm3.pdf) 64 | financial report filed by 65 | [Stand with Sanchez](https://www.fec.gov/data/committee/C00384057) 66 | , the principal campaign committee for Linda Sanchez (CA-38). The second filing, 67 | [`FEC-1848680`](https://docquery.fec.gov/cgi-bin/forms/C00384057/1848680/) 68 | , is the 69 | [FEC Form 1 "Statement of Organization"](https://www.fec.gov/resources/cms-content/documents/fecfrm1sf.pdf) 70 | that the Sanchez campaign submitted in the 2024 election cycle. 71 | 72 | The `libfec_filings` table is the core table for all the other exported itemizations. Every other table in the `sanchez.db` database (`libfec_F1`, `libfec_schedule_a`, etc.) all have foreign keys that point to the `libfec_filings.filing_id` primary key column. 73 | 74 | ## Cover Tables 75 | 76 | "Cover records" refer to the second line in a `.fec` file, which specifies which form the candidate or committee submitted. When exporting to a SQLite database, A every form type will have it's own table. 77 | 78 | In the `sanchez.db` example from above, there are two cover record tables: `libfec_F1` and `libfec_F3`. 79 | 80 | T 81 | ```sql 82 | 83 | ``` 84 | 85 | -------------------------------------------------------------------------------- /crates/fec-py/tests/README.md: -------------------------------------------------------------------------------- 1 | # Tests for libfec_parser 2 | 3 | This directory contains pytest-based tests for the `libfec_parser` Python package. 4 | 5 | ## Structure 6 | 7 | - `test_parser.py` - Tests for the `libfec_parser.parser` module 8 | - Tests for `fec_header()` function 9 | - Tests for `Filing` class 10 | - Tests for `Header`, `Cover`, and `Itemization` classes 11 | 12 | - `test_foo.py` - Tests for the `libfec_parser.foo` module 13 | - Tests for the example `bar()` function 14 | 15 | - `test_fecfile.py` - Tests for the `libfec_parser.fecfile` module 16 | - Tests for `loads()` function 17 | - Tests for `from_file()` function 18 | - Tests for `from_http()` function 19 | - Tests for `parse_header()` function 20 | - Tests for `parse_line()` function 21 | - Tests for `print_example()` function 22 | - Integration tests 23 | 24 | - `conftest.py` - Shared pytest configuration and fixtures 25 | 26 | ## Running Tests 27 | 28 | Make sure you have pytest installed: 29 | 30 | ```bash 31 | # Using uv (recommended) 32 | uv pip install pytest 33 | 34 | # Or using pip 35 | pip install pytest 36 | ``` 37 | 38 | Run all tests: 39 | 40 | ```bash 41 | # From the crates/fec-py directory 42 | pytest tests/ 43 | 44 | # Or with verbose output 45 | pytest -v tests/ 46 | 47 | # Run specific test file 48 | pytest tests/test_parser.py 49 | 50 | # Run specific test class 51 | pytest tests/test_parser.py::TestFiling 52 | 53 | # Run specific test 54 | pytest tests/test_parser.py::TestFiling::test_filing_from_path_string 55 | ``` 56 | 57 | ## Test Options 58 | 59 | Skip slow tests: 60 | ```bash 61 | pytest -m "not slow" tests/ 62 | ``` 63 | 64 | Skip network tests: 65 | ```bash 66 | pytest -m "not network" tests/ 67 | ``` 68 | 69 | Run with coverage: 70 | ```bash 71 | pytest --cov=libfec_parser --cov-report=html tests/ 72 | ``` 73 | 74 | ## Test Data 75 | 76 | The tests look for sample FEC files in: 77 | - `../../cache/` - Cached FEC files 78 | - `../../benchmarks/` - Benchmark FEC files 79 | 80 | If no sample files are found, tests that require them will be skipped. 81 | 82 | ## Adding New Tests 83 | 84 | When adding new functionality to `libfec_parser`: 85 | 86 | 1. Add tests to the appropriate `test_*.py` file 87 | 2. Use descriptive test names starting with `test_` 88 | 3. Add docstrings to explain what each test does 89 | 4. Use fixtures from `conftest.py` for common setup 90 | 5. Add markers for slow or network-dependent tests 91 | 92 | Example: 93 | ```python 94 | @pytest.mark.slow 95 | def test_large_file_parsing(first_fec_file): 96 | """Test parsing a very large FEC file""" 97 | # Test implementation 98 | pass 99 | ``` 100 | 101 | ## Test Coverage 102 | 103 | Current coverage includes: 104 | - ✅ Parser module (Filing, Header, Cover, Itemization classes) 105 | - ✅ Foo module (example module) 106 | - ✅ Fecfile module (fecfile compatibility layer) 107 | - ✅ Error handling and edge cases 108 | - ✅ Integration tests 109 | 110 | ## CI/CD 111 | 112 | These tests can be integrated into CI/CD pipelines: 113 | 114 | ```bash 115 | # Example GitHub Actions workflow 116 | - name: Run tests 117 | run: | 118 | uv pip install pytest pytest-cov 119 | pytest tests/ --cov=libfec_parser --cov-report=xml 120 | ``` 121 | -------------------------------------------------------------------------------- /crates/fec-wasm/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | FEC WASM Example 7 | 37 | 38 | 39 |

FEC WASM Parser

40 |

Select an FEC file to parse its header:

41 | 42 | 43 | 44 |
45 | 46 | 107 | 108 | 109 | -------------------------------------------------------------------------------- /crates/fec-cli/src/commands/export/dir_csv.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::HashMap, fs::File, path::PathBuf}; 2 | 3 | use fec_parser::schedules::{form_type_schedule_type, ScheduleType}; 4 | 5 | use crate::{ 6 | cli::ExportArgs, commands::export::sqlite::form_type_parse, sourcer::{FilingSourcer, ItemizationProgressBar} 7 | }; 8 | 9 | #[derive(Debug, Clone, PartialEq, Eq, Hash)] 10 | enum ItemizationKey { 11 | Schedule(ScheduleType), 12 | FormType(String), 13 | } 14 | 15 | struct ItemizationValue { 16 | writer: csv::Writer, 17 | } 18 | 19 | pub fn export( 20 | mut sourcer: FilingSourcer, 21 | args: ExportArgs, 22 | output_directory: PathBuf, 23 | ) -> anyhow::Result<()> { 24 | 25 | let mut cover_writers: std::collections::HashMap = HashMap::new(); 26 | let mut itemization_writers: std::collections::HashMap = 27 | HashMap::new(); 28 | let mb = indicatif::MultiProgress::new(); 29 | let iter = sourcer 30 | .resolve_iterator_from_flags(args.filings, args.api, Some(&mb))? 31 | .1; 32 | 33 | for filing in iter { 34 | let mut filing = match filing { 35 | Ok(f) => f, 36 | Err(_) => todo!(), 37 | }; 38 | 39 | { 40 | let (form_type, _amendment_indicator) = form_type_parse(&filing.cover.form_type); 41 | cover_writers.entry(form_type.to_owned()) 42 | .or_insert_with(|| { 43 | let path = output_directory.join(format!("cover_{}.csv", form_type)); 44 | let f = File::create_new(path).unwrap(); 45 | let mut w = csv::WriterBuilder::new() 46 | .flexible(true) 47 | .has_headers(true) 48 | .from_writer(f); 49 | w.write_record( 50 | filing.cover.record_column_names.clone() 51 | ).unwrap(); 52 | ItemizationValue { writer: w } 53 | }) 54 | .writer 55 | .write_record(&filing.cover.record).unwrap(); 56 | } 57 | 58 | 59 | 60 | let pb = ItemizationProgressBar::new(&mb, &filing); 61 | while let Some(r) = filing.next_row() { 62 | let row = r?; 63 | pb.update(&row); 64 | 65 | let key = match form_type_schedule_type(row.row_type.as_str()) { 66 | Some(schedule_type) => ItemizationKey::Schedule(schedule_type), 67 | None => ItemizationKey::FormType(row.row_type.as_str().to_owned()), 68 | }; 69 | let writer = &mut itemization_writers 70 | .entry(key.clone()) 71 | .or_insert_with(|| { 72 | let path = match key.clone() { 73 | ItemizationKey::Schedule(schedule_type) => output_directory 74 | .join(format!("{}.csv", schedule_type.to_sqlite_tablename())), 75 | ItemizationKey::FormType(form_type) => { 76 | output_directory.join(format!("form_{}.csv", form_type)) 77 | } 78 | }; 79 | let f = File::create_new(path).unwrap(); 80 | let mut w = csv::WriterBuilder::new() 81 | .flexible(true) 82 | .has_headers(true) 83 | .from_writer(f); 84 | let mut column_names = fec_parser::mappings::column_names_for_field( 85 | &row.row_type, 86 | &filing.header.fec_version, 87 | ) 88 | .unwrap() 89 | .to_owned(); 90 | column_names.insert(0, "filing_id".to_owned()); 91 | w.write_record(column_names).expect("Writing CSV header"); 92 | ItemizationValue { writer: w } 93 | }) 94 | .writer; 95 | 96 | writer.write_field(filing.filing_id.as_str())?; 97 | for field in &row.record { 98 | writer.write_field(field)?; 99 | } 100 | writer.write_record(None::<&[u8]>)?; 101 | } 102 | } 103 | Ok(()) 104 | } 105 | -------------------------------------------------------------------------------- /crates/fec-cli/src/commands/export/sqlite/snapshots/libfec__commands__export__sqlite__test__tests__F1S 8.5 data.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: crates/fec-cli/src/commands/export/sqlite/test.rs 3 | expression: "query(&db, \"select * from libfec_F1S where filing_id = '1923816'\")" 4 | snapshot_kind: text 5 | --- 6 | filing_id | form_type | filer_committee_id_number | joint_fund_participant_committee_name | joint_fund_participant_committee_id_number | joint_fund_participant_committee_type | affiliated_committee_id_number | affiliated_committee_name | affiliated_candidate_id_number | affiliated_last_name | affiliated_first_name | affiliated_middle_name | affiliated_prefix | affiliated_suffix | affiliated_street_1 | affiliated_street_2 | affiliated_city | affiliated_state | affiliated_zip_code | affiliated_relationship_code | agent_last_name | agent_first_name | agent_middle_name | agent_prefix | agent_suffix | agent_street_1 | agent_street_2 | agent_city | agent_state | agent_zip_code | agent_title | agent_telephone | bank_name | bank_street_1 | bank_street_2 | bank_city | bank_state | bank_zip_code 7 | ==== Row 0 ==== 8 | [filing_id]:1923816 9 | [form_type]:F1S 10 | [filer_committee_id_number]:C00925099 11 | [joint_fund_participant_committee_name]:Dodo Committee 12 | [joint_fund_participant_committee_id_number]:C00853135 13 | [joint_fund_participant_committee_type]:P 14 | [affiliated_committee_id_number]: 15 | [affiliated_committee_name]: 16 | [affiliated_candidate_id_number]: 17 | [affiliated_last_name]: 18 | [affiliated_first_name]: 19 | [affiliated_middle_name]: 20 | [affiliated_prefix]: 21 | [affiliated_suffix]: 22 | [affiliated_street_1]: 23 | [affiliated_street_2]: 24 | [affiliated_city]: 25 | [affiliated_state]: 26 | [affiliated_zip_code]: 27 | [affiliated_relationship_code]: 28 | [agent_last_name]: 29 | [agent_first_name]: 30 | [agent_middle_name]: 31 | [agent_prefix]: 32 | [agent_suffix]: 33 | [agent_street_1]: 34 | [agent_street_2]: 35 | [agent_city]: 36 | [agent_state]: 37 | [agent_zip_code]: 38 | [agent_title]: 39 | [agent_telephone]: 40 | [bank_name]: 41 | [bank_street_1]: 42 | [bank_street_2]: 43 | [bank_city]: 44 | [bank_state]: 45 | [bank_zip_code]: 46 | ==== Row 1 ==== 47 | [filing_id]:1923816 48 | [form_type]:F1S 49 | [filer_committee_id_number]:C00925099 50 | [joint_fund_participant_committee_name]:Dodo Congress and lobbying committee 51 | [joint_fund_participant_committee_id_number]:C00925065 52 | [joint_fund_participant_committee_type]:N 53 | [affiliated_committee_id_number]: 54 | [affiliated_committee_name]: 55 | [affiliated_candidate_id_number]: 56 | [affiliated_last_name]: 57 | [affiliated_first_name]: 58 | [affiliated_middle_name]: 59 | [affiliated_prefix]: 60 | [affiliated_suffix]: 61 | [affiliated_street_1]: 62 | [affiliated_street_2]: 63 | [affiliated_city]: 64 | [affiliated_state]: 65 | [affiliated_zip_code]: 66 | [affiliated_relationship_code]: 67 | [agent_last_name]: 68 | [agent_first_name]: 69 | [agent_middle_name]: 70 | [agent_prefix]: 71 | [agent_suffix]: 72 | [agent_street_1]: 73 | [agent_street_2]: 74 | [agent_city]: 75 | [agent_state]: 76 | [agent_zip_code]: 77 | [agent_title]: 78 | [agent_telephone]: 79 | [bank_name]: 80 | [bank_street_1]: 81 | [bank_street_2]: 82 | [bank_city]: 83 | [bank_state]: 84 | [bank_zip_code]: 85 | ==== Row 2 ==== 86 | [filing_id]:1923816 87 | [form_type]:F1S 88 | [filer_committee_id_number]:C00925099 89 | [joint_fund_participant_committee_name]:Dodo Commonwealth PAC 90 | [joint_fund_participant_committee_id_number]:C00925073 91 | [joint_fund_participant_committee_type]:N 92 | [affiliated_committee_id_number]: 93 | [affiliated_committee_name]: 94 | [affiliated_candidate_id_number]: 95 | [affiliated_last_name]: 96 | [affiliated_first_name]: 97 | [affiliated_middle_name]: 98 | [affiliated_prefix]: 99 | [affiliated_suffix]: 100 | [affiliated_street_1]: 101 | [affiliated_street_2]: 102 | [affiliated_city]: 103 | [affiliated_state]: 104 | [affiliated_zip_code]: 105 | [affiliated_relationship_code]: 106 | [agent_last_name]: 107 | [agent_first_name]: 108 | [agent_middle_name]: 109 | [agent_prefix]: 110 | [agent_suffix]: 111 | [agent_street_1]: 112 | [agent_street_2]: 113 | [agent_city]: 114 | [agent_state]: 115 | [agent_zip_code]: 116 | [agent_title]: 117 | [agent_telephone]: 118 | [bank_name]: 119 | [bank_street_1]: 120 | [bank_street_2]: 121 | [bank_city]: 122 | [bank_state]: 123 | [bank_zip_code]: 124 | -------------------------------------------------------------------------------- /crates/fec-parser-macros/src/lib.rs: -------------------------------------------------------------------------------- 1 | extern crate proc_macro; 2 | 3 | use proc_macro::TokenStream; 4 | use quote::quote; 5 | 6 | const DATE_COLUMNS_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/date_columns.txt"); 7 | #[proc_macro] 8 | pub fn gen_date_columns(_: TokenStream) -> TokenStream { 9 | let keys: Vec = std::fs::read_to_string(DATE_COLUMNS_PATH) 10 | .expect("unable to read date-columns.txt") 11 | .lines() 12 | .map(|v| v.to_string()) 13 | .collect(); 14 | 15 | let output = quote! { 16 | [ 17 | #( #keys.to_string() ),* 18 | ] 19 | }; 20 | 21 | output.into() 22 | } 23 | const FLOAT_COLUMNS_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/float-columns.txt"); 24 | #[proc_macro] 25 | pub fn gen_float_columns(_: TokenStream) -> TokenStream { 26 | let keys: Vec = std::fs::read_to_string(FLOAT_COLUMNS_PATH) 27 | .expect("unable to read date-columns.txt") 28 | .lines() 29 | .map(|v| v.to_string()) 30 | .collect(); 31 | 32 | let output = quote! { 33 | [ 34 | #( #keys.to_string() ),* 35 | ] 36 | }; 37 | 38 | output.into() 39 | } 40 | 41 | const MAPPINGS_JSON_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/src/mappings2.json"); 42 | 43 | #[proc_macro] 44 | pub fn gen_form_types(_: TokenStream) -> TokenStream { 45 | let json_data: serde_json::Value = { 46 | let contents = 47 | std::fs::read_to_string(MAPPINGS_JSON_PATH).expect("Unable to read the JSON file"); 48 | serde_json::from_str(&contents).expect("JSON parsing error") 49 | }; 50 | let keys: Vec = json_data 51 | .as_object() 52 | .expect("JSON is not an object") 53 | .keys() 54 | .map(|key| key.to_string()) 55 | .collect(); 56 | 57 | let output = quote! { 58 | [ 59 | #( #keys ),* 60 | ] 61 | }; 62 | 63 | output.into() 64 | } 65 | 66 | #[proc_macro] 67 | pub fn gen_form_type_version_set(_: TokenStream) -> TokenStream { 68 | let json_data: serde_json::Value = { 69 | let contents = 70 | std::fs::read_to_string(MAPPINGS_JSON_PATH).expect("Unable to read the JSON file"); 71 | serde_json::from_str(&contents).expect("JSON parsing error") 72 | }; 73 | let values = json_data 74 | .as_object() 75 | .expect("JSON is not an object") 76 | .values(); 77 | 78 | let mut result = Vec::new(); 79 | for value in values { 80 | let keys: Vec = value.as_object().unwrap().keys().cloned().collect(); 81 | 82 | let item = quote! { 83 | RegexSetBuilder::new([ 84 | #( #keys ),* 85 | ]) 86 | .case_insensitive(true) 87 | .build() 88 | .unwrap() 89 | }; 90 | result.push(item); 91 | } 92 | 93 | let output = quote! { 94 | vec![ 95 | #( #result ),* 96 | 97 | ] 98 | }; 99 | 100 | output.into() 101 | } 102 | #[proc_macro] 103 | pub fn gen_column_names(_: TokenStream) -> TokenStream { 104 | let json_data: serde_json::Value = { 105 | let contents = 106 | std::fs::read_to_string(MAPPINGS_JSON_PATH).expect("Unable to read the JSON file"); 107 | serde_json::from_str(&contents).expect("JSON parsing error") 108 | }; 109 | let mut form_types = vec![]; 110 | 111 | for (_, value) in json_data.as_object().unwrap().iter() { 112 | let mut list_of_columns = vec![]; 113 | 114 | for (_, item) in value.as_object().unwrap().iter() { 115 | let column_names: Vec = item 116 | .as_array() 117 | .unwrap() 118 | .into_iter() 119 | .map(|value| value.as_str().unwrap().to_owned()) 120 | .collect(); 121 | 122 | list_of_columns.push(quote! { 123 | vec![ 124 | #( #column_names.to_string() ),* 125 | ] 126 | }) 127 | } 128 | 129 | form_types.push(quote! { 130 | vec![ 131 | #( #list_of_columns ),* 132 | ] 133 | }) 134 | } 135 | 136 | let output = quote! { 137 | vec![ 138 | #( #form_types ),* 139 | ] 140 | }; 141 | 142 | output.into() 143 | } 144 | -------------------------------------------------------------------------------- /site/guides/fastfec.md: -------------------------------------------------------------------------------- 1 | # FastFEC Compatability 2 | 3 | [FastFEC](https://github.com/washingtonpost/FastFEC/) is popular a CLI tool for converting FEC filings to CSV files, created by the Washington Post in 2021. The `libfec` CLI contains a compatible CLI command to the `fastfec` CLI, aptly named `libfec fastfec`. 4 | 5 | The `libfec fastfec` command is meant for users who already use `fastfec` to try out `libfec` in their workflows, without completely re-writing their pipelines. 6 | 7 | For example, Say we are working with [`FEC-1781583`, Nikki Haley's March 2024 FEC filing](https://docquery.fec.gov/cgi-bin/forms/C00833392/1781583/). We can download the raw `.fec` file like so: 8 | 9 | ```bash 10 | curl -o 1781583.fec 'https://docquery.fec.gov/dcdev/posted/1781583.fec' 11 | ``` 12 | 13 | We can convert that `23MB` `.fec` file into a directory of CSVs with `fastfec` like so: 14 | 15 | ```bash 16 | $ fastfec 1781583.fec output/ 17 | About to parse filing ID 1781583 18 | Trying filename: 1781583.fec 19 | $ tree --du -h output 20 | [ 19M] output 21 | └── [ 19M] 1781583 22 | ├── [5.4K] F3PA.csv 23 | ├── [ 12M] SA17A.csv 24 | ├── [1.3K] SA17C.csv 25 | ├── [6.4M] SA18.csv 26 | ├── [1.3K] SA20A.csv 27 | ├── [177K] SB23.csv 28 | ├── [3.2K] SB28A.csv 29 | └── [ 132] header.csv 30 | ``` 31 | 32 | That creates a new directory `output/1781583` with 8 CSVs, containing the itemization data from that filing. 33 | 34 | Now with `libfec`, we can do a similar transformation with the `libfec fastfec` command: 35 | 36 | ```bash 37 | $ libfec fastfec 1781583.fec output2 38 | $ tree --du -h output2 39 | [ 18M] output2 40 | └── [ 18M] 1781583 41 | ├── [5.4K] F3PA.csv 42 | ├── [ 12M] SA17A.csv 43 | ├── [1.3K] SA17C.csv 44 | ├── [6.4M] SA18.csv 45 | ├── [1.3K] SA20A.csv 46 | ├── [176K] SB23.csv 47 | ├── [3.2K] SB28A.csv 48 | └── [ 133] header.csv 49 | ``` 50 | 51 | ## Additional `libfec fastfec` features 52 | 53 | `fastfec` requires that the `.fec` file be pre-downloaded, but `libfec fastfec` can download FEC filings on the fly: 54 | 55 | 56 | ```bash 57 | libfec fastfec FEC-1779004 output2/ 58 | ``` 59 | 60 | This will download [`FEC-1779004`](https://docquery.fec.gov/cgi-bin/forms/C00833392/1779004/) file and write results to `output2/1779004` directly — no `curl` required! 61 | 62 | 63 | ## Benchmarks 64 | 65 | Benchmarking `fastfec` is a bit complicated. Using the pre-compiled build from the [official `fastfec` Release](https://github.com/washingtonpost/FastFEC/releases/tag/0.4.1) seems to show that `fastfec` is ~11x slower than `libfec`: 66 | 67 | ``` 68 | Benchmark 1: ./fastfec -x 1805248.fec 69 | Time (mean ± σ): 4.754 s ± 0.024 s [User: 4.686 s, System: 0.034 s] 70 | Range (min … max): 4.728 s … 4.803 s 10 runs 71 | 72 | Benchmark 2: libfec fastfec 1805248.fec 73 | Time (mean ± σ): 411.9 ms ± 10.9 ms [User: 347.0 ms, System: 52.7 ms] 74 | Range (min … max): 400.7 ms … 439.2 ms 10 runs 75 | 76 | Summary 77 | libfec fastfec 1805248.fec ran 78 | 11.54 ± 0.31 times faster than ./fastfec -x 1805248.fec 79 | ``` 80 | 81 | But this isn't 100% accurate. I believe the offical builds of `fastfec` use the debug compilation arguments, and could be faster if compiled with the `--release=fast` flag. 82 | 83 | As an experiment, I built `fastfec` on my own machine, on top [of this PR](https://github.com/washingtonpost/FastFEC/pull/76), which gave faster results: 84 | 85 | 86 | ``` 87 | Benchmark 1: ../../FastFEC/zig-out/bin/fastfec -x 1805248.fec 88 | Time (mean ± σ): 714.4 ms ± 32.5 ms [User: 678.2 ms, System: 23.3 ms] 89 | Range (min … max): 688.7 ms … 800.7 ms 10 runs 90 | 91 | Benchmark 2: libfec fastfec 1805248.fec 92 | Time (mean ± σ): 394.4 ms ± 5.0 ms [User: 339.8 ms, System: 50.1 ms] 93 | Range (min … max): 389.4 ms … 406.5 ms 10 runs 94 | 95 | Summary 96 | libfec fastfec 1805248.fec ran 97 | 1.81 ± 0.09 times faster than ../../FastFEC/zig-out/bin/fastfec -x 1805248.fec 98 | ``` 99 | 100 | Now this version `fastfec` is faster than the older version, but `libfec` is still `~1.8x` faster. 101 | 102 | So I'd say in general, `libfec` is faster than `fastfec` in a head-to-head competition. But they're also very different tools, where `libfec` can resolve filings for committees and contests, while `fastfeec` just converts raw `.fec` files to a single CSV format. -------------------------------------------------------------------------------- /crates/fec-cli/src/commands/export/sqlite/snapshots/libfec__commands__export__sqlite__test__tests__F1S 8.4 data.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: crates/fec-cli/src/commands/export/sqlite/test.rs 3 | expression: "query(&db, \"select * from libfec_F1S\")" 4 | snapshot_kind: text 5 | --- 6 | filing_id | form_type | filer_committee_id_number | joint_fund_participant_committee_name | joint_fund_participant_committee_id_number | joint_fund_participant_committee_type | affiliated_committee_id_number | affiliated_committee_name | affiliated_candidate_id_number | affiliated_last_name | affiliated_first_name | affiliated_middle_name | affiliated_prefix | affiliated_suffix | affiliated_street_1 | affiliated_street_2 | affiliated_city | affiliated_state | affiliated_zip_code | affiliated_relationship_code | agent_last_name | agent_first_name | agent_middle_name | agent_prefix | agent_suffix | agent_street_1 | agent_street_2 | agent_city | agent_state | agent_zip_code | agent_title | agent_telephone | bank_name | bank_street_1 | bank_street_2 | bank_city | bank_state | bank_zip_code 7 | ==== Row 0 ==== 8 | [filing_id]:1913562 9 | [form_type]:F1S 10 | [filer_committee_id_number]:C00677898 11 | [joint_fund_participant_committee_name]: 12 | [joint_fund_participant_committee_id_number]: 13 | [joint_fund_participant_committee_type]: 14 | [affiliated_committee_id_number]:C00873331 15 | [affiliated_committee_name]:HAYES VICTORY FUND 16 | [affiliated_candidate_id_number]: 17 | [affiliated_last_name]: 18 | [affiliated_first_name]: 19 | [affiliated_middle_name]: 20 | [affiliated_prefix]: 21 | [affiliated_suffix]: 22 | [affiliated_street_1]:PO BOX 65322 23 | [affiliated_street_2]: 24 | [affiliated_city]:Washington 25 | [affiliated_state]:DC 26 | [affiliated_zip_code]:20035 27 | [affiliated_relationship_code]:JFR 28 | [agent_last_name]: 29 | [agent_first_name]: 30 | [agent_middle_name]: 31 | [agent_prefix]: 32 | [agent_suffix]: 33 | [agent_street_1]: 34 | [agent_street_2]: 35 | [agent_city]: 36 | [agent_state]: 37 | [agent_zip_code]: 38 | [agent_title]: 39 | [agent_telephone]: 40 | [bank_name]: 41 | [bank_street_1]: 42 | [bank_street_2]: 43 | [bank_city]: 44 | [bank_state]: 45 | [bank_zip_code]: 46 | ==== Row 1 ==== 47 | [filing_id]:1913562 48 | [form_type]:F1S 49 | [filer_committee_id_number]:C00677898 50 | [joint_fund_participant_committee_name]: 51 | [joint_fund_participant_committee_id_number]: 52 | [joint_fund_participant_committee_type]: 53 | [affiliated_committee_id_number]:C00902270 54 | [affiliated_committee_name]:FRONTLINE PROTECTION FUND 55 | [affiliated_candidate_id_number]: 56 | [affiliated_last_name]: 57 | [affiliated_first_name]: 58 | [affiliated_middle_name]: 59 | [affiliated_prefix]: 60 | [affiliated_suffix]: 61 | [affiliated_street_1]:PO Box 65322 62 | [affiliated_street_2]: 63 | [affiliated_city]:Washington 64 | [affiliated_state]:DC 65 | [affiliated_zip_code]:20035 66 | [affiliated_relationship_code]:JFR 67 | [agent_last_name]: 68 | [agent_first_name]: 69 | [agent_middle_name]: 70 | [agent_prefix]: 71 | [agent_suffix]: 72 | [agent_street_1]: 73 | [agent_street_2]: 74 | [agent_city]: 75 | [agent_state]: 76 | [agent_zip_code]: 77 | [agent_title]: 78 | [agent_telephone]: 79 | [bank_name]: 80 | [bank_street_1]: 81 | [bank_street_2]: 82 | [bank_city]: 83 | [bank_state]: 84 | [bank_zip_code]: 85 | ==== Row 2 ==== 86 | [filing_id]:1913562 87 | [form_type]:F1S 88 | [filer_committee_id_number]:C00677898 89 | [joint_fund_participant_committee_name]: 90 | [joint_fund_participant_committee_id_number]: 91 | [joint_fund_participant_committee_type]: 92 | [affiliated_committee_id_number]:C00916429 93 | [affiliated_committee_name]:JEFFRIES BATTLEGROUND PROTECTION FUND 94 | [affiliated_candidate_id_number]: 95 | [affiliated_last_name]: 96 | [affiliated_first_name]: 97 | [affiliated_middle_name]: 98 | [affiliated_prefix]: 99 | [affiliated_suffix]: 100 | [affiliated_street_1]:430 SOUTH CAPITOL STREET SE 101 | [affiliated_street_2]:2ND FL 102 | [affiliated_city]:Washington 103 | [affiliated_state]:DC 104 | [affiliated_zip_code]:20003 105 | [affiliated_relationship_code]:JFR 106 | [agent_last_name]: 107 | [agent_first_name]: 108 | [agent_middle_name]: 109 | [agent_prefix]: 110 | [agent_suffix]: 111 | [agent_street_1]: 112 | [agent_street_2]: 113 | [agent_city]: 114 | [agent_state]: 115 | [agent_zip_code]: 116 | [agent_title]: 117 | [agent_telephone]: 118 | [bank_name]: 119 | [bank_street_1]: 120 | [bank_street_2]: 121 | [bank_city]: 122 | [bank_state]: 123 | [bank_zip_code]: 124 | -------------------------------------------------------------------------------- /crates/fec-parser/src/schedules.rs: -------------------------------------------------------------------------------- 1 | use crate::mappings; 2 | 3 | #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] 4 | pub enum ScheduleType { 5 | // Itemized Receipts 6 | // https://github.com/fecgov/fecfile-validate/blob/develop/schema/backlog/schedules/SchA.json 7 | ScheduleA, 8 | // Itemized Disbursements 9 | // https://github.com/fecgov/fecfile-validate/blob/develop/schema/backlog/schedules/SchB.json 10 | ScheduleB, 11 | // Loans 12 | // https://github.com/fecgov/fecfile-validate/blob/develop/schema/backlog/schedules/SchC.json 13 | ScheduleC, 14 | 15 | // Loans And Lines Of Credit From Lending Institutions 16 | // https://github.com/fecgov/fecfile-validate/blob/develop/schema/backlog/schedules/SchC1.json 17 | ScheduleC1, 18 | 19 | // Loan Guarantor Name & Address Information 20 | // https://github.com/fecgov/fecfile-validate/blob/develop/schema/backlog/schedules/SchC2.json 21 | ScheduleC2, 22 | 23 | // DEBTS AND OBLIGATIONS (Itemized for each one) 24 | // https://github.com/fecgov/fecfile-validate/blob/develop/schema/backlog/schedules/SchD.json 25 | ScheduleD, 26 | 27 | // ITEMIZED INDEPENDENT EXPENDITURES 28 | // https://github.com/fecgov/fecfile-validate/blob/develop/schema/backlog/schedules/SchE.json 29 | ScheduleE, 30 | 31 | // ITEMIZED COORDINATED EXPENDITURES MADE BY POLITICAL PARTY COMMITTEES OR DESIGNATED AGENT(S) ON BEHALF OF CANDIDATES FOR FEDERAL OFFICE 32 | // https://github.com/fecgov/fecfile-validate/blob/develop/schema/backlog/schedules/SchF.json 33 | ScheduleF, 34 | //ScheduleH1, 35 | //ScheduleH2, 36 | //ScheduleH3, 37 | //ScheduleH4, 38 | //ScheduleH5, 39 | //ScheduleH6, 40 | //ScheduleI, 41 | } 42 | 43 | pub fn form_type_schedule_type(form_type: &str) -> Option { 44 | if form_type.starts_with("SA") { 45 | Some(ScheduleType::ScheduleA) 46 | } else if form_type.starts_with("SB") { 47 | Some(ScheduleType::ScheduleB) 48 | } else if form_type.starts_with("SC1") { 49 | Some(ScheduleType::ScheduleC1) 50 | } else if form_type.starts_with("SC2") { 51 | Some(ScheduleType::ScheduleC2) 52 | } else if form_type.starts_with("SC") { 53 | Some(ScheduleType::ScheduleC) 54 | } else if form_type.starts_with("SD") { 55 | Some(ScheduleType::ScheduleD) 56 | } else if form_type.starts_with("SE") { 57 | Some(ScheduleType::ScheduleE) 58 | } else if form_type.starts_with("SF") { 59 | Some(ScheduleType::ScheduleF) 60 | } else { 61 | None 62 | } 63 | } 64 | 65 | impl ScheduleType { 66 | pub fn column_names(&self, fec_version: &str) -> anyhow::Result> { 67 | match self { 68 | ScheduleType::ScheduleA => { 69 | Ok(mappings::column_names_for_field("SAx", fec_version)?.to_owned()) 70 | } 71 | ScheduleType::ScheduleB => { 72 | Ok(mappings::column_names_for_field("SBx", fec_version)?.to_owned()) 73 | } 74 | _ => todo!(), 75 | } 76 | } 77 | } 78 | 79 | impl ToString for ScheduleType { 80 | fn to_string(&self) -> String { 81 | match self { 82 | ScheduleType::ScheduleA => "SA".to_string(), 83 | ScheduleType::ScheduleB => "SB".to_string(), 84 | ScheduleType::ScheduleC => "SC".to_string(), 85 | ScheduleType::ScheduleC1 => "SC1".to_string(), 86 | ScheduleType::ScheduleC2 => "SC2".to_string(), 87 | ScheduleType::ScheduleD => "SD".to_string(), 88 | ScheduleType::ScheduleE => "SE".to_string(), 89 | ScheduleType::ScheduleF => "SF".to_string(), 90 | } 91 | } 92 | } 93 | 94 | impl ScheduleType { 95 | pub fn to_sqlite_tablename(self) -> String { 96 | match self { 97 | ScheduleType::ScheduleA => "schedule_a".to_string(), 98 | ScheduleType::ScheduleB => "schedule_b".to_string(), 99 | ScheduleType::ScheduleC => "schedule_c".to_string(), 100 | ScheduleType::ScheduleC1 => "schedule_c1".to_string(), 101 | ScheduleType::ScheduleC2 => "schedule_c2".to_string(), 102 | ScheduleType::ScheduleD => "schedule_d".to_string(), 103 | ScheduleType::ScheduleE => "schedule_e".to_string(), 104 | ScheduleType::ScheduleF => "schedule_f".to_string(), 105 | } 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /crates/fec-cli/src/commands/fastfec.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Context; 2 | /** 3 | * Export FEC filing data in a format compatible with FastFEC. 4 | * FastFEC reads in single FEC file and writes multiple CSV files 5 | * in the following directory structure: 6 | * 7 | * / 8 | * header.csv 9 | * .csv 10 | * <...form_type.>.csv 11 | * 12 | * where <...form_type> are all the itemization form types for the filing (" SA15A", "F3X", etc.) 13 | * 14 | * Reference: https://github.com/washingtonpost/FastFEC 15 | */ 16 | use fec_parser::Filing; 17 | use indicatif::{ProgressBar, ProgressStyle}; 18 | use std::{collections::HashMap, fs::File, io::Read, path::Path, sync::LazyLock}; 19 | 20 | use crate::{cli::FastFecArgs, sourcer::FilingSourcer}; 21 | 22 | static STYLE: LazyLock = LazyLock::new(|| { 23 | ProgressStyle::with_template( 24 | "{msg}.fec:\t[{elapsed_precise}] {bar:40.cyan/blue} {eta} {decimal_bytes_per_sec} {decimal_total_bytes} total", 25 | ) 26 | .expect("valid progress style") 27 | }); 28 | 29 | fn write_header_csv(filing: &Filing, header_csv_path: &Path) -> anyhow::Result<()> { 30 | let f = File::create_new(header_csv_path)?; 31 | let mut w = csv::WriterBuilder::new() 32 | .flexible(true) 33 | .has_headers(false) 34 | .from_writer(f); 35 | 36 | w.write_record(&[ 37 | "record_type", 38 | "ef_type", 39 | "fec_version", 40 | "soft_name", 41 | "soft_ver", 42 | "report_id", 43 | "report_number", 44 | "comment", 45 | ])?; 46 | w.write_record(vec![ 47 | filing.header.record_type.clone(), 48 | filing.header.ef_type.clone(), 49 | filing.header.fec_version.clone(), 50 | filing.header.software_name.clone(), 51 | filing.header.software_version.clone(), 52 | filing.header.report_id.clone().unwrap_or_default(), 53 | filing.header.report_number.clone().unwrap_or("".to_owned()), 54 | filing.header.comment.clone().unwrap_or_default(), 55 | ])?; 56 | Ok(()) 57 | } 58 | 59 | fn write_cover_csv(filing: &Filing, cover_csv_path: &Path) -> anyhow::Result<()> { 60 | let f = File::create_new(cover_csv_path)?; 61 | let mut w = csv::WriterBuilder::new() 62 | .flexible(true) 63 | .has_headers(false) 64 | .from_writer(f); 65 | w.write_record(&filing.cover.record_column_names)?; 66 | w.write_record(&filing.cover.record.clone())?; 67 | Ok(()) 68 | } 69 | 70 | fn write_fastfec_compat(mut filing: Filing, directory: &Path) -> anyhow::Result<()> { 71 | let mut csv_writers: HashMap> = HashMap::new(); 72 | let pb = ProgressBar::new(filing.source_length as u64).with_style(STYLE.clone()); 73 | pb.set_message(filing.filing_id.to_owned()); 74 | 75 | let filing_directory = directory.join(filing.filing_id.to_string()); 76 | std::fs::create_dir_all(&filing_directory)?; 77 | 78 | write_header_csv(&filing, &filing_directory.join("header.csv"))?; 79 | write_cover_csv( 80 | &filing, 81 | &filing_directory.join(format!("{}.csv", filing.cover.form_type)), 82 | )?; 83 | 84 | while let Some(r) = filing.next_row() { 85 | let r = r.context("Error reading next row")?; 86 | pb.set_position( 87 | r.record 88 | .position() 89 | .expect("CSV position to be available") 90 | .byte(), 91 | ); 92 | 93 | if let Some(w) = csv_writers.get_mut(&r.row_type) { 94 | w.write_record(&r.record.clone())?; 95 | } else { 96 | let f = File::create_new(filing_directory.join(format!("{}.csv", r.row_type)))?; 97 | let mut w = csv::WriterBuilder::new() 98 | .flexible(true) 99 | .has_headers(false) 100 | .from_writer(f); 101 | 102 | let column_names = fec_parser::mappings::column_names_for_field( 103 | &r.row_type, 104 | &filing.header.fec_version, 105 | )?; 106 | w.write_record(column_names)?; 107 | w.write_record(&r.record.clone())?; 108 | csv_writers.insert(r.row_type, w); 109 | } 110 | } 111 | Ok(()) 112 | } 113 | 114 | pub fn fastfec(sourcer: FilingSourcer, args: FastFecArgs) -> anyhow::Result<()> { 115 | let filing = sourcer.resolve_from_user_argument(&args.filing_id)?; 116 | std::fs::create_dir_all(&args.output_directory)?; 117 | write_fastfec_compat(filing, &args.output_directory)?; 118 | Ok(()) 119 | } 120 | -------------------------------------------------------------------------------- /crates/fec-cli/src/commands/export/sqlite/test.rs: -------------------------------------------------------------------------------- 1 | #[cfg(test)] 2 | mod tests { 3 | use fec_parser::Filing; 4 | 5 | use super::super::{export_itemizations, init, insert_filing_metadata}; 6 | 7 | /// Macro to easily load test filings by ID 8 | macro_rules! filing { 9 | ($id:expr) => {{ 10 | let bytes: &[u8] = include_bytes!(concat!("../../../../../../.test-files/", $id, ".fec")); 11 | Filing::from_reader(bytes, $id.to_string(), bytes.len()).unwrap() 12 | }}; 13 | } 14 | #[test] 15 | fn test_basic() { 16 | assert_eq!(1 + 1, 2); 17 | } 18 | 19 | fn query(db: &rusqlite::Connection, query: &str) -> String { 20 | let mut out = String::new(); 21 | let mut stmt = db.prepare(query).unwrap(); 22 | let columns = stmt 23 | .column_names() 24 | .iter() 25 | .map(|s| s.to_string()) 26 | .collect::>(); 27 | out.push_str(&columns.join(" | ")); 28 | out.push('\n'); 29 | let mut rows = stmt.query([]).unwrap(); 30 | let mut idx = 0; 31 | while let Some(row) = rows.next().unwrap() { 32 | let col_count = row.as_ref().column_count(); 33 | out.push_str(format!("==== Row {} ====\n", idx).as_str()); 34 | idx += 1; 35 | 36 | for i in 0..col_count { 37 | let value: rusqlite::types::Value = row.get(i).unwrap(); 38 | out.push_str(&format!("[{}]:", columns[i])); 39 | match value { 40 | rusqlite::types::Value::Null => out.push_str("NULL"), 41 | rusqlite::types::Value::Integer(i) => out.push_str(&i.to_string()), 42 | rusqlite::types::Value::Real(f) => out.push_str(&f.to_string()), 43 | rusqlite::types::Value::Text(s) => out.push_str(&s), 44 | rusqlite::types::Value::Blob(b) => out.push_str(&format!("{:?}", b)), 45 | } 46 | out.push('\n'); 47 | } 48 | } 49 | out 50 | } 51 | 52 | #[test] 53 | #[ignore] 54 | fn test_basic_f99() { 55 | let f99_84 = filing!("1913493"); 56 | let mut db = rusqlite::Connection::open_in_memory().unwrap(); 57 | let mut tx = db.transaction().unwrap(); 58 | init(&mut tx).unwrap(); 59 | insert_filing_metadata(&mut tx, &f99_84).unwrap(); 60 | tx.commit().unwrap(); 61 | 62 | insta::assert_snapshot!(query( 63 | &db, 64 | "select name, sql from sqlite_master where type='table'" 65 | ),); 66 | insta::assert_snapshot!(query(&db, "select count(*) from libfec_f99"),); 67 | insta::assert_snapshot!(query(&db, "select * from libfec_filings"),); 68 | } 69 | #[test] 70 | #[ignore] 71 | fn test_f99_84_and_85() { 72 | let f99_84 = filing!("1913493"); 73 | let f99_85 = filing!("1913595"); 74 | 75 | let mut db = rusqlite::Connection::open_in_memory().unwrap(); 76 | let mut tx = db.transaction().unwrap(); 77 | 78 | init(&mut tx).unwrap(); 79 | 80 | insert_filing_metadata(&mut tx, &f99_84).unwrap(); 81 | tx.commit().unwrap(); 82 | 83 | insta::assert_snapshot!("8.4 1st", query(&db, "select * from libfec_f99"),); 84 | 85 | let mut tx = db.transaction().unwrap(); 86 | insert_filing_metadata(&mut tx, &f99_85).unwrap(); 87 | tx.commit().unwrap(); 88 | insta::assert_snapshot!("8.5 2nd", query(&db, "select * from libfec_f99"),); 89 | 90 | 91 | } 92 | 93 | #[test] 94 | fn test_f1s_84_and_85() -> anyhow::Result<()> { 95 | let f1s_84 = filing!("1913562"); 96 | let f1s_85 = filing!("1923816"); 97 | let mut db = rusqlite::Connection::open_in_memory()?; 98 | let mut tx = db.transaction()?; 99 | init(&mut tx)?; 100 | tx.commit()?; 101 | 102 | let mut tx = db.transaction()?; 103 | insert_filing_metadata(&mut tx, &f1s_84).unwrap(); 104 | export_itemizations(&mut tx, f1s_84, None)?; 105 | tx.commit()?; 106 | 107 | insta::assert_snapshot!("F1S schema", query(&db, "select sql from sqlite_master where name = 'libfec_F1S'"),); 108 | insta::assert_snapshot!("F1S 8.4 data", query(&db, "select * from libfec_F1S"),); 109 | 110 | let mut tx = db.transaction()?; 111 | insert_filing_metadata(&mut tx, &f1s_85).unwrap(); 112 | export_itemizations(&mut tx, f1s_85, None)?; 113 | tx.commit()?; 114 | insta::assert_snapshot!("F1S 8.5 data", query(&db, "select * from libfec_F1S where filing_id = '1923816'"),); 115 | 116 | Ok(()) 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /crates/fec-cli/src/commands/export/single.rs: -------------------------------------------------------------------------------- 1 | use std::{fs::File, io::Write, path::PathBuf}; 2 | 3 | use fec_parser::schedules::{form_type_schedule_type, ScheduleType}; 4 | 5 | use crate::{ 6 | cli::{ExportArgs, ExportTarget}, 7 | sourcer::{FilingSourcer, ItemizationProgressBar}, 8 | }; 9 | 10 | pub enum SingleOutput { 11 | Csv, 12 | Json, 13 | } 14 | 15 | fn target_matches_form_type(target: &ExportTarget, form_type: &str) -> bool { 16 | match form_type_schedule_type(form_type) { 17 | Some(ScheduleType::ScheduleA) => matches!(target, ExportTarget::ScheduleA), 18 | Some(ScheduleType::ScheduleB) => matches!(target, ExportTarget::ScheduleB), 19 | _ => false, 20 | } 21 | } 22 | 23 | enum Writer { 24 | Csv{ 25 | writer: csv::Writer, 26 | nrecords: usize, 27 | }, 28 | Json { 29 | file: File, 30 | column_names: Vec, 31 | }, 32 | } 33 | pub fn cmd_export_single( 34 | mut sourcer: FilingSourcer, 35 | output_path: PathBuf, 36 | args: ExportArgs, 37 | target: ExportTarget, 38 | output_type: SingleOutput, 39 | ) -> anyhow::Result<()> { 40 | let t0 = jiff::Timestamp::now(); 41 | let mut output = match output_type { 42 | SingleOutput::Csv => { 43 | let mut writer = csv::WriterBuilder::new() 44 | .flexible(true) 45 | .has_headers(true) 46 | .from_writer(std::fs::File::create_new(&output_path)?); 47 | let mut columns: Vec = 48 | Into::::into(target).column_names("8.5")?; 49 | columns.insert(0, "filing_id".to_owned()); 50 | let nrecords = columns.len(); 51 | writer.write_record(columns).expect("Writing CSV header"); 52 | Writer::Csv{ 53 | writer, 54 | nrecords, 55 | } 56 | } 57 | SingleOutput::Json => { 58 | let mut f = File::create_new(&output_path)?; 59 | f.write_all(b"[")?; 60 | let mut columns: Vec = 61 | Into::::into(target).column_names("8.5")?; 62 | columns.insert(0, "filing_id".to_owned()); 63 | Writer::Json { 64 | file: f, 65 | column_names: columns, 66 | } 67 | } 68 | }; 69 | 70 | let mb = indicatif::MultiProgress::new(); 71 | let iter = sourcer 72 | .resolve_iterator_from_flags(args.filings, args.api, Some(&mb))? 73 | .1; 74 | let mut nrows = 0; 75 | 76 | for filing in iter { 77 | let mut filing = match filing { 78 | Ok(f) => f, 79 | Err(_) => todo!(), 80 | }; 81 | let pb = ItemizationProgressBar::new(&mb, &filing); 82 | let mut first = true; 83 | while let Some(r) = filing.next_row() { 84 | let row = r?; 85 | pb.update(&row); 86 | 87 | if target_matches_form_type(&target, row.row_type.as_str()) { 88 | nrows += 1; 89 | match &mut output { 90 | Writer::Csv{writer, nrecords} => { 91 | writer.write_field(&filing.filing_id)?; 92 | // TODO: check if theres non-empty rows beyond nrecords - 1 93 | for field in row.record.iter().take(*nrecords - 1) { 94 | writer.write_field(field)?; 95 | } 96 | writer.write_record(None::<&[u8]>)?; 97 | } 98 | Writer::Json { file, column_names } => { 99 | if first { 100 | first = false; 101 | } else { 102 | file.write_all(b",")?; 103 | } 104 | let mut record = serde_json::map::Map::new(); 105 | record.insert( 106 | "filing_id".to_owned(), 107 | serde_json::Value::String(filing.filing_id.clone()), 108 | ); 109 | for (i, field) in row.record.iter().enumerate() { 110 | record.insert( 111 | column_names[i + 1].clone(), 112 | serde_json::Value::String(field.to_string()), 113 | ); 114 | } 115 | let value = serde_json::Value::Object(record); 116 | let s = serde_json::to_string(&value)?; 117 | file.write_all(s.as_bytes())?; 118 | } 119 | } 120 | } 121 | } 122 | } 123 | if let Writer::Json { file, .. } = &mut output { 124 | file.write_all(b"]")?; 125 | } 126 | let duration = jiff::Timestamp::now() - t0; 127 | eprintln!( 128 | "Exported {} rows to {} in {:.2?}", 129 | nrows, 130 | output_path.display(), 131 | duration 132 | ); 133 | Ok(()) 134 | } 135 | -------------------------------------------------------------------------------- /site/getting-started/basic-usage.md: -------------------------------------------------------------------------------- 1 | # Basic `libfec` Usage 2 | 3 | Once you have [installed the `libfec` CLI](./installation.md), you can begin to export FEC filings into various file formats like CSVs, JSON, SQLite, or Excel. 4 | 5 | ## Basic exports 6 | 7 | Let's export all data from Nikki Haley's 2023 Q1 presidential campaign's filing, [`FEC-1721616`](https://docquery.fec.gov/cgi-bin/forms/C00833392/1721616): 8 | 9 | ```bash 10 | libfec export \ 11 | FEC-1721616 \ 12 | -o haley-2023-Q1.db 13 | ``` 14 | ```output 15 | Exporting filings to SQLite database at "haley-2023-Q1.db" 16 | Finished exporting 1 filings into haley-2023-Q1.db, in 0 seconds 17 | ``` 18 | 19 | Let's break down this command: 20 | 21 | - `libfec export`: We're using the `libfec` program to call the `export` command, which exports FEC filings into different file formats. 22 | - `FEC-1721616`: This is the positional argument for the `export` subcommand, the ID of the FEC filing we care about (Nikki Haley's 2023 Q1 financial report). 23 | - `-o haley-2023-Q1.db`: The "output" flag, meaning we want to output data into the `haley-2023-Q1.db` file. The `.db` file extension designates this as a SQLite export. 24 | 25 | So the result is a single `haley-2023-Q1.db` SQLite database file, with various generated tables for each itemization: 26 | 27 | ```bash 28 | › sqlite3 haley-2023-Q1.db 29 | sqlite> .tables 30 | libfec_F3P libfec_filings libfec_schedule_a libfec_schedule_b 31 | ``` 32 | 33 | You can then use SQL to query the exact into you're looking for, using your programming language's SQLite client library. See [SQL Reference](../reference/sql.md) for more info about these tables. 34 | 35 | ## Exporting to CSV 36 | 37 | There are two types of CSV exports that `libfec` supports: **CSV directory exports** and **target specific" CSV exports**. 38 | 39 | CSV directory exports are made with the `--output-directory` and `--format csv` flags: 40 | 41 | ```bash 42 | $ libfec export \ 43 | FEC-1721616 \ 44 | --format csv \ 45 | --output-directory haley-2023-Q1 46 | $ tree --du -h haley-2023-Q1/ 47 | ``` 48 | 49 | ```output 50 | [5.4M] haley-2023-Q1/ 51 | ├── [5.3K] cover_F3P.csv 52 | ├── [5.4M] schedule_a.csv 53 | └── [ 43K] schedule_b.csv 54 | ``` 55 | 56 | Itemizations from the `FEC-1721616` filing will be exported by itemization type into various CSVs inside of `haley-2023-Q1`. In this example, `cover_F3P.csv` will have the `F3P` summary page information from the filing. The `schedule_a.csv` file contains receipt information including individual contributors and PAC contributions. The `schedule_b.csv` file contains disbursement information, like operating expenses, transfers to other PACs, refunds, and more. 57 | 58 | If instead you only care about receipts, you can instead perform a target-sepcfic CSV export like so: 59 | 60 | ```bash 61 | $ libfec export \ 62 | FEC-1721616 \ 63 | --target receipts \ 64 | -o haley-2023-Q1-receipts.csv 65 | ``` 66 | 67 | The `--target receipts` flag will export only the Schedule A itemizations records from `FEC-1721616` into a single CSV file at `haley-2023-Q1-receipts.csv`. 68 | 69 | Alternatively use `--target disbursements` for Schedule B itemizations: 70 | 71 | ```bash 72 | $ libfec export \ 73 | FEC-1721616 \ 74 | --target disbursements \ 75 | -o haley-2023-Q1-disbursements.csv 76 | ``` 77 | 78 | ## Exporting a candidate or committee 79 | 80 | Copy+pasting FEC filing IDs can get old after a while! Instead of exporting a single FEC filing, you can also provide a candidate ID or committee ID, and `libfec` will find all relevent filings. 81 | 82 | For example, let's export all filings from the 2024 cycle of Congresswoman Young Kim's principal campaign committee, [Young Kim for Congress (`C00665638`)](https://www.fec.gov/data/committee/C00665638/): 83 | 84 | ```bash 85 | libfec export \ 86 | --cycle 2024 \ 87 | C00665638 \ 88 | -o kim.db 89 | ``` 90 | 91 | Now `kim.db` contains data from 12 filings submitted by Kim's committee, relevant to the 2024 election cycle. `libfec` will automatically use the [OpenFEC API](https://api.open.fec.gov/developers/) to find relevant filings. 92 | 93 | 94 | ## Exporting all candidates in an election 95 | 96 | `libfec` can also resolve filings from a specific presidential, senate, or house race. Say we wanted to get all filings from all canddiate committees who ran in the [California 40th district](https://en.wikipedia.org/wiki/California%27s_40th_congressional_district) 2024 race, which is the current district Kim represents. We can get that like so: 97 | 98 | ```bash 99 | libfec export \ 100 | --election 2024 \ 101 | CA40 \ 102 | -o ca40-2024.db 103 | ``` 104 | 105 | The `--election` flag will find all candidates who ran in the 2024 CA40 election, and export all of their filings into `ca40-2024.db`. 106 | 107 | You can do the same for senate elections: 108 | 109 | ```bash 110 | # Iowa senate candidates for 2026 111 | libfec export \ 112 | --election 2026 \ 113 | senate-IA \ 114 | -o ia-2026.db 115 | ``` 116 | 117 | Or presidents: 118 | 119 | ```bash 120 | libfec export \ 121 | --election 2024 \ 122 | president \ 123 | -o prez-2024.db 124 | ``` 125 | 126 | ## Form Type Filtering 127 | 128 | Candidates and committees file several different types of forms and reports to the FEC. You can filter for exact forms that you care about with the `--form-type` flag. 129 | 130 | For example, candidates file F2 forms to declare their candidacy. To export all F2 forms filed by people who ran for the California senate seat in 2024, you can run: 131 | 132 | ```bash 133 | libfec export \ 134 | --cycle 2024 \ 135 | --form-type F2 \ 136 | --state CA \ 137 | --office S \ 138 | -o ca-senate-2024.db 139 | ``` -------------------------------------------------------------------------------- /crates/fec-cli/src/cache/bulk_candidates.rs: -------------------------------------------------------------------------------- 1 | /** 2 | * > "The all candidate summary file contains one record including summary financial 3 | * > information for all candidates who raised or spent money during the period 4 | * > no matter when they are up for election." 5 | * https://www.fec.gov/campaign-finance-data/candidate-master-file-description/ 6 | * 7 | * Sample: https://www.fec.gov/files/bulk-downloads/2026/weball26.zip 8 | * 9 | */ 10 | use crate::cache::bulk_utils::{BulkDataItem, SyncResult, sync_item}; 11 | use anyhow::{Context, Result}; 12 | use derive_builder::Builder; 13 | use fec_api::Office; 14 | use rusqlite::{Connection, Transaction}; 15 | use std::{path::PathBuf, sync::LazyLock}; 16 | 17 | static ITEM: LazyLock = LazyLock::new(|| BulkDataItem { 18 | table_name: "libfec_candidates".to_owned(), 19 | url_scheme: "https://www.fec.gov/files/bulk-downloads/$YEAR/cn$YEAR2.zip".to_string(), 20 | schema: SCHEMA.to_string(), 21 | data_file_name: "cn.txt".to_string(), 22 | column_count: 15, 23 | }); 24 | 25 | static SCHEMA: &str = r#" 26 | CREATE TABLE IF NOT EXISTS libfec_candidates( 27 | cycle INTEGER, -- REFERENCES candidate_cycles(year) ON DELETE CASCADE, 28 | candidate_id TEXT, 29 | name, 30 | party_affiliation, 31 | election_year INTEGER, 32 | state, 33 | office, 34 | district, 35 | incumbent_challenger_status, 36 | status, 37 | principal_campaign_committee, 38 | address_street1, 39 | address_street2, 40 | address_city, 41 | address_state, 42 | address_zip, 43 | 44 | UNIQUE(cycle, candidate_id) 45 | ); 46 | "#; 47 | 48 | #[derive(Debug, Clone, Builder, Default)] 49 | pub struct ResolveCandidateParams { 50 | cycle: u16, 51 | office: Option, 52 | state: Option, 53 | district: Option, 54 | } 55 | 56 | pub(crate) fn include( 57 | tx: &mut Transaction, 58 | bulk_db_path: PathBuf, 59 | params: &ResolveCandidateParams, 60 | ) -> Result<()> { 61 | tx.execute_batch(SCHEMA)?; 62 | 63 | if !tx 64 | .prepare_cached("select 1 from pragma_database_list where name = 'bulk_db'")? 65 | .exists([])? 66 | { 67 | tx.execute( 68 | "ATTACH DATABASE ? AS bulk_db", 69 | [bulk_db_path.to_str().unwrap()], 70 | )?; 71 | } 72 | 73 | let sql = r#" 74 | INSERT OR REPLACE INTO libfec_candidates 75 | SELECT * 76 | FROM bulk_db.libfec_candidates 77 | WHERE cycle = :cycle 78 | AND election_year = cast(:cycle as text) 79 | AND principal_campaign_committee != '' 80 | AND if(:office is null, true, office = :office) 81 | AND if(:state is null, true, state = :state) 82 | AND if(:district is null, true, cast(district as text) = :district) 83 | "#; 84 | let params = rusqlite::named_params! { 85 | ":cycle": params.cycle, 86 | ":office": params.office.clone().map(|o| match o { 87 | Office::House => "H", 88 | Office::Senate => "S", 89 | Office::President => "P", 90 | }), 91 | ":state": params.state, 92 | ":district": params.district 93 | }; 94 | let mut stmt = tx.prepare(sql)?; 95 | stmt.execute(params)?; 96 | drop(stmt); 97 | //tx.execute("DETACH DATABASE bulk_db", [])?; 98 | Ok(()) 99 | } 100 | 101 | fn query_candidate_principal_campaign_committees( 102 | db: &Connection, 103 | params: ResolveCandidateParams, 104 | ) -> Result> { 105 | let sql = r#" 106 | SELECT 107 | principal_campaign_committee 108 | FROM libfec_candidates 109 | WHERE cycle = :cycle 110 | AND election_year = cast(:cycle as text) 111 | AND principal_campaign_committee != '' 112 | AND if(:office is null, true, office = :office) 113 | AND if(:state is null, true, state = :state) 114 | AND if(:district is null, true, cast(district as text) = :district) 115 | "#; 116 | let params = rusqlite::named_params! { 117 | ":cycle": params.cycle, 118 | ":office": params.office.map(|o| match o { 119 | Office::House => "H", 120 | Office::Senate => "S", 121 | Office::President => "P", 122 | }), 123 | ":state": params.state, 124 | ":district": params.district 125 | }; 126 | let mut stmt = db.prepare(sql)?; 127 | let committee_ids = stmt 128 | .query_map(params, |row| { 129 | let committee_id: String = row.get(0)?; 130 | Ok(committee_id) 131 | })? 132 | .collect::, _>>()?; 133 | Ok(committee_ids) 134 | } 135 | 136 | pub fn resolve_candidate_principal_campaign_committees( 137 | mut bulk_db: Connection, 138 | params: ResolveCandidateParams, 139 | ) -> Result> { 140 | bulk_db.execute_batch(SCHEMA)?; 141 | let mut tx = bulk_db 142 | .transaction() 143 | .context("Could not start a transaction on the .bulk-data.db database")?; 144 | sync_item(&mut tx, params.cycle, &*ITEM)?; 145 | tx.commit()?; 146 | query_candidate_principal_campaign_committees(&bulk_db, params) 147 | } 148 | 149 | pub fn search_candidates( 150 | bulk_db: &mut Connection, 151 | cycle: u16, 152 | name_query: &str, 153 | ) -> Result> { 154 | bulk_db.execute_batch(SCHEMA)?; 155 | let mut tx = bulk_db 156 | .transaction() 157 | .context("Could not start a transaction on the .bulk-data.db database")?; 158 | sync_item(&mut tx, cycle, &*ITEM)?; 159 | tx.commit()?; 160 | 161 | let sql = r#" 162 | SELECT 163 | candidate_id, 164 | name 165 | FROM libfec_candidates 166 | WHERE cycle = :cycle 167 | AND name LIKE '%' || :name_query || '%' 168 | "#; 169 | let params = rusqlite::named_params! { 170 | ":cycle": cycle, 171 | ":name_query": name_query, 172 | }; 173 | let mut stmt = bulk_db.prepare(sql)?; 174 | let results = stmt 175 | .query_map(params, |row| { 176 | let candidate_id: String = row.get(0)?; 177 | let name: String = row.get(1)?; 178 | Ok((candidate_id, name)) 179 | })? 180 | .collect::, _>>()?; 181 | Ok(results) 182 | } 183 | 184 | 185 | pub fn export(tx: &mut Transaction<'_>, year: u16) -> Result<()> { 186 | sync_item(tx, year, &ITEM)?; 187 | Ok(()) 188 | } 189 | -------------------------------------------------------------------------------- /crates/fec-py/demo-fecfile.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Demo script for libfec_parser.fecfile - a compatibility layer for the fecfile PyPI package. 4 | 5 | This demonstrates the API that mimics the original fecfile library: 6 | - loads() - Parse FEC content from string/bytes 7 | - from_file() - Load from a file path 8 | - from_http() - Load from FEC website (if available) 9 | - parse_header() - Parse just the header 10 | - parse_line() - Parse a single line 11 | - print_example() - Print a sample of the parsed data 12 | 13 | Usage: 14 | python demo-fecfile.py [file2.fec] ... 15 | """ 16 | 17 | import sys 18 | from pathlib import Path 19 | from libfec_parser.fecfile import loads, from_file, parse_header, parse_line, print_example 20 | 21 | def demo_loads(file_path): 22 | """Demo the loads() function""" 23 | print(f"\n{'='*70}") 24 | print(f"Demo: loads() with file: {file_path}") 25 | print('='*70) 26 | 27 | # Read file content 28 | content = Path(file_path).read_text(encoding='utf-8', errors='ignore') 29 | 30 | # Parse with loads() 31 | parsed = loads(content) 32 | 33 | print(f"\n📋 Header:") 34 | print(f" FEC Version: {parsed['header']['fec_version']}") 35 | print(f" Software: {parsed['header']['software_name']} v{parsed['header']['software_version']}") 36 | 37 | print(f"\n📄 Filing:") 38 | print(f" Form Type: {parsed['filing']['form_type']}") 39 | print(f" Filer ID: {parsed['filing'].get('filer_committee_id_number', 'N/A')}") 40 | if 'committee_name' in parsed['filing']: 41 | print(f" Committee: {parsed['filing']['committee_name']}") 42 | 43 | print(f"\n📊 Itemizations:") 44 | for schedule, items in parsed['itemizations'].items(): 45 | print(f" {schedule}: {len(items)} items") 46 | 47 | if parsed['text']: 48 | print(f"\n📝 Text records: {len(parsed['text'])} items") 49 | 50 | return parsed 51 | 52 | def demo_from_file(file_path): 53 | """Demo the from_file() function""" 54 | print(f"\n{'='*70}") 55 | print(f"Demo: from_file() with: {file_path}") 56 | print('='*70) 57 | 58 | parsed = from_file(file_path) 59 | 60 | print(f"\n📋 Header:") 61 | print(f" FEC Version: {parsed['header']['fec_version']}") 62 | 63 | print(f"\n📄 Filing:") 64 | print(f" Form Type: {parsed['filing']['form_type']}") 65 | 66 | print(f"\n📊 Itemizations summary:") 67 | total_items = sum(len(items) for items in parsed['itemizations'].values()) 68 | print(f" Total itemization records: {total_items}") 69 | 70 | return parsed 71 | 72 | def demo_filter_itemizations(file_path): 73 | """Demo the filter_itemizations option""" 74 | print(f"\n{'='*70}") 75 | print(f"Demo: loads() with filter_itemizations=['SA', 'SB']") 76 | print('='*70) 77 | 78 | content = Path(file_path).read_text(encoding='utf-8', errors='ignore') 79 | 80 | # Parse with filter 81 | parsed = loads(content, options={'filter_itemizations': ['SA', 'SB']}) 82 | 83 | print(f"\n📊 Filtered Itemizations (only SA and SB schedules):") 84 | for schedule, items in parsed['itemizations'].items(): 85 | print(f" {schedule}: {len(items)} items") 86 | 87 | def demo_parse_header(file_path): 88 | """Demo the parse_header() function""" 89 | print(f"\n{'='*70}") 90 | print(f"Demo: parse_header()") 91 | print('='*70) 92 | 93 | # Read just the first line (header) 94 | with open(file_path, 'r', encoding='utf-8', errors='ignore') as f: 95 | header_line = f.readline() 96 | 97 | header, version, lines_consumed = parse_header(header_line) 98 | 99 | print(f"\n📋 Parsed Header:") 100 | print(f" Version: {version}") 101 | print(f" Lines consumed: {lines_consumed}") 102 | print(f" Record type: {header['record_type']}") 103 | print(f" Software: {header['software_name']} v{header['software_version']}") 104 | 105 | def demo_parse_line(file_path): 106 | """Demo the parse_line() function""" 107 | print(f"\n{'='*70}") 108 | print(f"Demo: parse_line()") 109 | print('='*70) 110 | 111 | # Read first few lines 112 | with open(file_path, 'r', encoding='utf-8', errors='ignore') as f: 113 | lines = [f.readline() for _ in range(3)] 114 | 115 | # Parse header to get version 116 | header, version, _ = parse_header(lines[0]) 117 | 118 | # Parse the cover line (second line) 119 | cover = parse_line(lines[1], version, line_num=2) 120 | 121 | print(f"\n📄 Parsed Cover Line (line 2):") 122 | print(f" Form type: {cover.get('form_type', 'N/A')}") 123 | print(f" Fields in cover: {len(cover)}") 124 | print(f" Sample fields: {list(cover.keys())[:5]}...") 125 | 126 | def demo_print_example(file_path): 127 | """Demo the print_example() function""" 128 | print(f"\n{'='*70}") 129 | print(f"Demo: print_example()") 130 | print('='*70) 131 | 132 | parsed = from_file(file_path) 133 | 134 | print("\n📋 Example output (first item of each type):") 135 | print_example(parsed) 136 | 137 | def main(): 138 | fec_files = sys.argv[1:] 139 | 140 | # Run demos on the first file 141 | first_file = fec_files[0] 142 | 143 | print("\n" + "="*70) 144 | print(f"FEC FILE COMPATIBILITY API DEMO") 145 | print(f"File: {first_file}") 146 | print("="*70) 147 | 148 | # Demo each function 149 | try: 150 | demo_from_file(first_file) 151 | demo_loads(first_file) 152 | demo_parse_header(first_file) 153 | demo_parse_line(first_file) 154 | demo_filter_itemizations(first_file) 155 | demo_print_example(first_file) 156 | except Exception as e: 157 | print(f"\n❌ Error: {e}") 158 | import traceback 159 | traceback.print_exc() 160 | sys.exit(1) 161 | 162 | # Process remaining files with brief output 163 | if len(fec_files) > 1: 164 | print(f"\n{'='*70}") 165 | print(f"Processing remaining {len(fec_files) - 1} files...") 166 | print('='*70) 167 | 168 | for fec_file in fec_files[1:]: 169 | try: 170 | parsed = from_file(fec_file) 171 | total = sum(len(items) for items in parsed['itemizations'].values()) 172 | print(f"\n✓ {Path(fec_file).name}: " 173 | f"v{parsed['header']['fec_version']}, " 174 | f"{parsed['filing']['form_type']}, " 175 | f"{total} itemizations") 176 | except Exception as e: 177 | print(f"\n✗ {Path(fec_file).name}: Error - {e}") 178 | 179 | print(f"\n{'='*70}") 180 | print("✨ Demo complete!") 181 | print('='*70) 182 | 183 | if __name__ == "__main__": 184 | main() 185 | -------------------------------------------------------------------------------- /crates/fec-cli/src/cli.rs: -------------------------------------------------------------------------------- 1 | pub use crate::api_flags::FilingsApiFlags; 2 | use clap::{Args, Parser, Subcommand, ValueEnum}; 3 | use core::str; 4 | use fec_parser::schedules::ScheduleType; 5 | use std::{ 6 | env, 7 | path::PathBuf, 8 | }; 9 | 10 | #[derive( 11 | Debug, 12 | Default, 13 | Clone, 14 | Copy, 15 | PartialEq, 16 | Eq, 17 | //serde::Serialize, 18 | //serde::Deserialize, 19 | clap::ValueEnum, 20 | )] 21 | pub enum CmdInfoFormat { 22 | #[default] 23 | Human, 24 | Json, 25 | } 26 | 27 | #[derive(Args, Debug)] 28 | pub struct InfoArgs { 29 | pub filings: Vec, 30 | 31 | #[arg( 32 | long, 33 | short = 'i', 34 | help = ".txt files of FEC filing IDs to fetch, 1 line per filing ID" 35 | )] 36 | pub input_file: Option, 37 | 38 | #[arg( 39 | long, 40 | short = 'f', 41 | value_enum, 42 | help = "Format to output information to", 43 | default_value_t = CmdInfoFormat::Human)] 44 | pub format: CmdInfoFormat, 45 | //#[arg(long, short = 'f', value_enum)] 46 | //pub format: Option, 47 | #[arg( 48 | long, 49 | help = "Calculate stats on all itemizations in the provided filings" 50 | )] 51 | pub full: bool, 52 | } 53 | 54 | #[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] 55 | pub enum ExportFormat { 56 | Sqlite, 57 | Excel, 58 | Csv, 59 | Json, 60 | } 61 | 62 | #[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] 63 | pub enum ExportTarget { 64 | #[value(alias = "receipts")] 65 | ScheduleA, 66 | #[value(alias = "disbursements")] 67 | ScheduleB, 68 | } 69 | 70 | impl Into for ExportTarget { 71 | fn into(self) -> ScheduleType { 72 | match self { 73 | ExportTarget::ScheduleA => ScheduleType::ScheduleA, 74 | ExportTarget::ScheduleB => ScheduleType::ScheduleB, 75 | } 76 | } 77 | } 78 | 79 | #[derive(Args, Debug)] 80 | pub struct ExportArgs { 81 | #[arg(required = false)] 82 | pub filings: Vec, 83 | 84 | #[arg(long, short = 'f', help = "Output file")] 85 | pub format: Option, 86 | 87 | #[arg(long, help = "Output file")] 88 | pub target: Option, 89 | 90 | #[arg(long, short = 'o', help = "Output file")] 91 | pub output: Option, 92 | 93 | #[arg(long, alias = "outdir", help = "Output directory")] 94 | pub output_directory: Option, 95 | 96 | #[arg(long, action, help = "Overwrite existing files")] 97 | pub clobber: bool, 98 | 99 | #[arg(long, action, help = "Only export cover records, not itemizations")] 100 | pub cover_only: bool, 101 | 102 | //#[arg(long, short = 'f', help = "Format to export to")] 103 | //pub format: Option, 104 | #[command(flatten)] 105 | pub api: FilingsApiFlags, 106 | } 107 | 108 | #[derive(Args, Debug)] 109 | pub struct FastFecArgs { 110 | /// FEC filing id OR path to a .fec file 111 | pub filing_id: String, 112 | 113 | /// Output directory 114 | 115 | #[arg(default_value = "output")] 116 | pub output_directory: PathBuf, 117 | } 118 | 119 | #[derive(Args, Debug)] 120 | pub struct CacheArgs { 121 | /// FEC filing id, ex `FEC-C00606962` 122 | pub filings: Option>, 123 | 124 | #[arg(long, action, help = "Print cache directory location")] 125 | pub print: bool, 126 | 127 | #[arg( 128 | long, 129 | alias = "concurrent", 130 | help = "Number of concurrent downloads", 131 | default_value_t = 8 132 | )] 133 | pub number_concurrent: usize, 134 | 135 | #[command(flatten)] 136 | pub api: FilingsApiFlags, 137 | } 138 | 139 | #[derive(Args, Debug)] 140 | pub struct SearchArgs { 141 | pub query: String, 142 | #[arg(long, default_value_t = 2024, help = "Election cycle year to search")] 143 | pub cycle: u16, 144 | } 145 | 146 | #[derive(Debug, Clone, PartialEq, Eq)] 147 | pub enum CycleArg { 148 | Single(u16), 149 | Range(u16, u16), 150 | } 151 | 152 | impl std::str::FromStr for CycleArg { 153 | type Err = String; 154 | 155 | fn from_str(s: &str) -> Result { 156 | if let Some((start, end)) = s.split_once('-') { 157 | let start_year = start.trim().parse::() 158 | .map_err(|_| format!("Invalid start year: {}", start))?; 159 | let end_year = end.trim().parse::() 160 | .map_err(|_| format!("Invalid end year: {}", end))?; 161 | 162 | if start_year > end_year { 163 | return Err(format!("Start year {} cannot be greater than end year {}", start_year, end_year)); 164 | } 165 | 166 | Ok(CycleArg::Range(start_year, end_year)) 167 | } else { 168 | let year = s.trim().parse::() 169 | .map_err(|_| format!("Invalid year: {}", s))?; 170 | Ok(CycleArg::Single(year)) 171 | } 172 | } 173 | } 174 | 175 | impl std::fmt::Display for CycleArg { 176 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 177 | match self { 178 | CycleArg::Single(year) => write!(f, "{}", year), 179 | CycleArg::Range(start, end) => write!(f, "{}-{}", start, end), 180 | } 181 | } 182 | } 183 | 184 | #[derive(Args, Debug)] 185 | pub struct BulkArgs { 186 | #[arg(long, short = 'o', help = "Output file path")] 187 | pub output: PathBuf, 188 | #[arg(long, help = "Election cycle year (e.g., 2024) or range (e.g., 2024-2026)")] 189 | pub cycle: CycleArg, 190 | 191 | #[arg(long, value_delimiter = ',', value_enum, help = "Bulk data source(s) to export (comma-separated, e.g., candidates,committees)")] 192 | pub source: Vec, 193 | 194 | } 195 | 196 | #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] 197 | pub enum BulkSource { 198 | Opex, 199 | Committees, 200 | Candidates, 201 | } 202 | 203 | 204 | #[derive(Subcommand, Debug)] 205 | pub enum Commands { 206 | /// Export FEC filings into SQLite, Excel, CSV, or JSON 207 | Export(ExportArgs), 208 | 209 | /// Cache .fec files from fec.gov to your filesystem 210 | Cache(CacheArgs), 211 | 212 | /// Print debug information about a FEC filing, committee, or candidate 213 | Info(InfoArgs), 214 | 215 | //Feed(FeedArgs), 216 | /// FastFEC compatible export 217 | Fastfec(FastFecArgs), 218 | 219 | Search(SearchArgs), 220 | 221 | // Export bulk datasets from fec.gov 222 | Bulk(BulkArgs), 223 | } 224 | 225 | #[derive(Parser)] 226 | #[command( 227 | name = "libfec", 228 | author, 229 | long_version = env!("CARGO_PKG_VERSION"), 230 | about = "libfec CLI", 231 | version, 232 | subcommand_required = false, 233 | arg_required_else_help = false, 234 | )] 235 | pub struct Cli { 236 | #[command(subcommand)] 237 | pub command: Box, 238 | 239 | #[command(flatten)] 240 | pub top_level: TopLevelArgs, 241 | } 242 | 243 | #[derive(Parser)] 244 | pub struct TopLevelArgs { 245 | // TODO: api-key, api-url 246 | #[arg( 247 | global = true, 248 | long, 249 | env = "LIBFEC_CACHE_DIRECTORY", 250 | help_heading = "Global options" 251 | )] 252 | pub cache_directory: Option, 253 | } 254 | -------------------------------------------------------------------------------- /crates/fec-py/README.md: -------------------------------------------------------------------------------- 1 | # libfec_parser 2 | 3 | Python bindings for the FEC parser library. 4 | 5 | ## Installation 6 | 7 | ```bash 8 | # Install from wheel (after building) 9 | pip install dist/libfec_parser-0.1.0-cp39-abi3-macosx_11_0_arm64.whl 10 | ``` 11 | 12 | ## Building 13 | 14 | **Note:** Don't use `cargo build` directly - use `maturin` to build PyO3 extension modules. 15 | 16 | ### Quick Start with Makefile 17 | 18 | The easiest way to build is using the Makefile: 19 | 20 | ```bash 21 | cd /Users/alex/projects/libfec/crates/fec-py 22 | 23 | # Build development wheel 24 | make build 25 | 26 | # Build release (optimized) wheel 27 | make build-release 28 | 29 | # Run demo scripts 30 | make demo 31 | make demo-fecfile 32 | ``` 33 | 34 | ### Development Build 35 | 36 | For local development and testing, use `maturin develop` to build and install in-place: 37 | 38 | ```bash 39 | cd /Users/alex/projects/libfec 40 | maturin develop -m crates/fec-py/Cargo.toml 41 | ``` 42 | 43 | This installs the package in editable mode in your current Python environment. 44 | 45 | ### Manual Build 46 | 47 | To build a distributable wheel package manually: 48 | 49 | ```bash 50 | cd /Users/alex/projects/libfec 51 | maturin build -m crates/fec-py/Cargo.toml --out dist 52 | ``` 53 | 54 | For a release (optimized) build: 55 | 56 | ```bash 57 | cd /Users/alex/projects/libfec 58 | maturin build -m crates/fec-py/Cargo.toml --release --out dist 59 | ``` 60 | 61 | ## Usage 62 | 63 | ### fecfile Compatibility API 64 | 65 | The `libfec_parser.fecfile` module provides a compatibility layer that mimics the API of the [fecfile](https://pypi.org/project/fecfile/) PyPI package: 66 | 67 | ```python 68 | from libfec_parser.fecfile import loads, from_file, parse_header, parse_line, print_example 69 | 70 | # Load and parse a filing from a file 71 | parsed = from_file("./path/to/filing.fec") 72 | 73 | # Access parsed data 74 | print(parsed['header']['fec_version']) 75 | print(parsed['filing']['form_type']) 76 | print(parsed['itemizations']['Schedule A'][0]) 77 | 78 | # Parse from string/bytes 79 | content = open("filing.fec", "rb").read() 80 | parsed = loads(content) 81 | 82 | # Filter specific schedules 83 | parsed = loads(content, options={'filter_itemizations': ['SA', 'SB']}) 84 | 85 | # Parse just the header 86 | header, version, lines_consumed = parse_header(header_line) 87 | 88 | # Parse a single line 89 | line_dict = parse_line(line, version) 90 | 91 | # Print example (first item of each type) 92 | print_example(parsed) 93 | ``` 94 | 95 | See [demo-fecfile.py](demo-fecfile.py) for a complete demonstration. 96 | 97 | ### Native Python API 98 | 99 | ```python 100 | from libfec_parser.parser import Filing 101 | 102 | # Create a Filing from a file path 103 | f = Filing("./path/to/filing.fec") 104 | 105 | # Access header information 106 | print(f.header.fec_version) # FEC file version 107 | print(f.header.software_name) # Software used to create filing 108 | print(f.header.software_version) # Software version 109 | 110 | # Access cover page information 111 | print(f.cover.form_type) # Form type (e.g., "F3P") 112 | print(f.cover.filer_id) # Committee/filer ID 113 | print(f.cover.filer_name) # Committee/filer name 114 | print(f.cover.report_code) # Report code (e.g., "Q1", "M10") 115 | print(f.cover.coverage_from_date) # Coverage period start 116 | print(f.cover.coverage_through_date) # Coverage period end 117 | 118 | # Iterate through itemizations (schedules) 119 | for itemization in f.itemizations: 120 | print(itemization.row_type) # Row type (e.g., "SA11AI", "SB21B") 121 | print(itemization.fields()) # All fields as a list 122 | print(itemization[0]) # Access specific field by index 123 | 124 | # Alternative: Create from bytes 125 | with open("./path/to/filing.fec", "rb") as file: 126 | f = Filing(file.read()) 127 | 128 | # Alternative: Create from file-like object 129 | import urllib.request 130 | with urllib.request.urlopen("https://example.com/filing.fec") as response: 131 | f = Filing(response) 132 | ``` 133 | 134 | ### Legacy Function 135 | 136 | The `fec_header` function is also available for quick header parsing: 137 | 138 | ```python 139 | from libfec_parser.parser import fec_header 140 | from pathlib import Path 141 | 142 | contents = Path("./filing.fec").read_bytes() 143 | version = fec_header(contents) # Returns FEC version string 144 | print(version) # e.g., "8.4" 145 | ``` 146 | 147 | ## Running the Demo 148 | 149 | The easiest way to run the demos is using the Makefile: 150 | 151 | ```bash 152 | cd /Users/alex/projects/libfec/crates/fec-py 153 | 154 | # Run demo.py (will build if needed) 155 | make demo 156 | 157 | # Run demo-fecfile.py 158 | make demo-fecfile 159 | ``` 160 | 161 | Or run manually with specific files: 162 | 163 | ```bash 164 | # Run with the built wheel using uv 165 | uv run --no-cache --no-project --isolated \ 166 | --with 'libfec_parser @ file://dist/libfec_parser-0.1.0-cp39-abi3-macosx_11_0_arm64.whl' \ 167 | demo.py ../../cache2/1461586.fec ../../cache2/1478292.fec 168 | 169 | # Or if installed locally 170 | python demo.py path/to/filing1.fec path/to/filing2.fec 171 | ``` 172 | 173 | The demos will parse the provided FEC files and demonstrate the API usage 174 | 175 | ## API Reference 176 | 177 | ### Classes 178 | 179 | #### `Filing` 180 | 181 | Main class for parsing FEC filing files. 182 | 183 | **Constructor:** `Filing(source)` 184 | - `source`: Can be a file path (str), bytes, or file-like object with a `read()` method 185 | 186 | **Attributes:** 187 | - `header`: `Header` object with filing header information 188 | - `cover`: `Cover` object with cover page information 189 | - `itemizations`: List of `Itemization` objects representing all schedule rows 190 | 191 | #### `Header` 192 | 193 | Contains FEC filing header information. 194 | 195 | **Attributes:** 196 | - `record_type`: str - Record type (always "HDR") 197 | - `ef_type`: str - Electronic filing type 198 | - `fec_version`: str - FEC version (e.g., "8.4", "8.5") 199 | - `software_name`: str - Software name used to create filing 200 | - `software_version`: str - Software version 201 | - `report_id`: Optional[str] - Report ID if present 202 | - `report_number`: Optional[str] - Report number if present 203 | - `comment`: Optional[str] - Comment if present 204 | 205 | #### `Cover` 206 | 207 | Contains FEC filing cover page information. 208 | 209 | **Attributes:** 210 | - `form_type`: str - Form type (e.g., "F3P", "F3X") 211 | - `filer_id`: str - Committee or filer ID 212 | - `filer_name`: str - Committee or filer name 213 | - `report_code`: Optional[str] - Report code (e.g., "Q1", "M10", "YE") 214 | - `coverage_from_date`: Optional[str] - Coverage period start date 215 | - `coverage_through_date`: Optional[str] - Coverage period end date 216 | 217 | **Methods:** 218 | - `fields()`: Returns a dictionary of all cover record fields 219 | 220 | #### `Itemization` 221 | 222 | Represents a single itemization (schedule) row in the filing. 223 | 224 | **Attributes:** 225 | - `row_type`: str - Row type identifier (e.g., "SA11AI", "SB21B") 226 | 227 | **Methods:** 228 | - `fields()`: Returns list of all field values 229 | - `__len__()`: Returns number of fields 230 | - `__getitem__(idx)`: Access field by index (supports negative indexing) 231 | 232 | ## Development 233 | 234 | After making changes to the Rust code: 235 | 236 | 1. Rebuild and test: 237 | ```bash 238 | cd /Users/alex/projects/libfec/crates/fec-py 239 | make build 240 | make demo 241 | ``` 242 | 243 | Or manually: 244 | 245 | 1. Rebuild the wheel: 246 | ```bash 247 | cd /Users/alex/projects/libfec/crates/fec-py 248 | maturin build -m Cargo.toml --out dist 249 | ``` 250 | 251 | 2. Test with the new wheel: 252 | ```bash 253 | uv run --no-cache --no-project --isolated \ 254 | --with 'libfec_parser @ file://dist/libfec_parser-0.1.0-cp39-abi3-macosx_11_0_arm64.whl' \ 255 | demo.py ../../cache2/*.fec 256 | ``` 257 | 258 | Note: Use `--no-cache` with `uv` to ensure it uses the newly built wheel and doesn't cache an old version. 259 | -------------------------------------------------------------------------------- /crates/fec-py/src/parser.rs: -------------------------------------------------------------------------------- 1 | use pyo3::prelude::*; 2 | use pyo3::types::PyDict; 3 | use std::io::Cursor; 4 | use std::path::PathBuf; 5 | 6 | /// Python wrapper for FilingHeader 7 | #[pyclass] 8 | #[derive(Clone)] 9 | pub struct Header { 10 | #[pyo3(get)] 11 | pub record_type: String, 12 | #[pyo3(get)] 13 | pub ef_type: String, 14 | #[pyo3(get)] 15 | pub fec_version: String, 16 | #[pyo3(get)] 17 | pub software_name: String, 18 | #[pyo3(get)] 19 | pub software_version: String, 20 | #[pyo3(get)] 21 | pub report_id: Option, 22 | #[pyo3(get)] 23 | pub report_number: Option, 24 | #[pyo3(get)] 25 | pub comment: Option, 26 | } 27 | 28 | #[pymethods] 29 | impl Header { 30 | fn __repr__(&self) -> String { 31 | format!( 32 | "Header(fec_version='{}', software_name='{}', software_version='{}')", 33 | self.fec_version, self.software_name, self.software_version 34 | ) 35 | } 36 | } 37 | 38 | /// Python wrapper for FilingCover 39 | #[pyclass] 40 | #[derive(Clone)] 41 | pub struct Cover { 42 | #[pyo3(get)] 43 | pub form_type: String, 44 | #[pyo3(get)] 45 | pub filer_id: String, 46 | #[pyo3(get)] 47 | pub filer_name: String, 48 | #[pyo3(get)] 49 | pub report_code: Option, 50 | #[pyo3(get)] 51 | pub coverage_from_date: Option, 52 | #[pyo3(get)] 53 | pub coverage_through_date: Option, 54 | } 55 | 56 | #[pymethods] 57 | impl Cover { 58 | fn __repr__(&self) -> String { 59 | format!( 60 | "Cover(form_type='{}', filer_id='{}', filer_name='{}')", 61 | self.form_type, self.filer_id, self.filer_name 62 | ) 63 | } 64 | 65 | /// Get all cover record fields as a dictionary 66 | fn fields<'py>(&self, py: Python<'py>) -> PyResult> { 67 | let dict = PyDict::new_bound(py); 68 | dict.set_item("form_type", &self.form_type)?; 69 | dict.set_item("filer_id", &self.filer_id)?; 70 | dict.set_item("filer_name", &self.filer_name)?; 71 | dict.set_item("report_code", &self.report_code)?; 72 | dict.set_item("coverage_from_date", &self.coverage_from_date)?; 73 | dict.set_item("coverage_through_date", &self.coverage_through_date)?; 74 | Ok(dict) 75 | } 76 | } 77 | 78 | /// Python wrapper for FilingRow (itemization) 79 | #[pyclass] 80 | #[derive(Clone)] 81 | pub struct Itemization { 82 | #[pyo3(get)] 83 | pub row_type: String, 84 | fields: Vec, 85 | } 86 | 87 | #[pymethods] 88 | impl Itemization { 89 | fn __repr__(&self) -> String { 90 | format!("Itemization(row_type='{}', {} fields)", self.row_type, self.fields.len()) 91 | } 92 | 93 | fn __len__(&self) -> usize { 94 | self.fields.len() 95 | } 96 | 97 | fn __getitem__(&self, idx: isize) -> PyResult { 98 | let len = self.fields.len() as isize; 99 | let actual_idx = if idx < 0 { 100 | (len + idx) as usize 101 | } else { 102 | idx as usize 103 | }; 104 | 105 | self.fields.get(actual_idx) 106 | .cloned() 107 | .ok_or_else(|| pyo3::exceptions::PyIndexError::new_err("Index out of range")) 108 | } 109 | 110 | /// Get all fields as a list 111 | fn fields(&self) -> Vec { 112 | self.fields.clone() 113 | } 114 | } 115 | 116 | /// Main Filing class 117 | #[pyclass] 118 | pub struct Filing { 119 | header: Header, 120 | cover: Cover, 121 | itemizations: Vec, 122 | } 123 | 124 | #[pymethods] 125 | impl Filing { 126 | #[new] 127 | #[pyo3(signature = (source))] 128 | pub fn new(source: &Bound<'_, PyAny>) -> PyResult { 129 | // Handle different input types: path (str), bytes, or file-like object 130 | let (reader, source_length): (Box, usize) = if let Ok(path_str) = source.extract::() { 131 | // It's a file path 132 | let path = PathBuf::from(path_str); 133 | let file = std::fs::File::open(&path) 134 | .map_err(|e| pyo3::exceptions::PyIOError::new_err(format!("Failed to open file: {}", e)))?; 135 | let len = file.metadata() 136 | .map_err(|e| pyo3::exceptions::PyIOError::new_err(format!("Failed to get file metadata: {}", e)))? 137 | .len() as usize; 138 | (Box::new(file), len) 139 | } else if let Ok(bytes) = source.extract::>() { 140 | // It's bytes 141 | let len = bytes.len(); 142 | (Box::new(Cursor::new(bytes)), len) 143 | } else if let Ok(bytes_like) = source.call_method0("read") { 144 | // It's a file-like object with read() method 145 | let bytes: Vec = bytes_like.extract()?; 146 | let len = bytes.len(); 147 | (Box::new(Cursor::new(bytes)), len) 148 | } else { 149 | return Err(pyo3::exceptions::PyTypeError::new_err( 150 | "Source must be a file path (str), bytes, or file-like object with read() method" 151 | )); 152 | }; 153 | 154 | // Parse the filing 155 | let mut filing = fec_parser::Filing::from_reader(reader, "filing".to_string(), source_length) 156 | .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("Failed to parse filing: {}", e)))?; 157 | 158 | // Convert header 159 | let header = Header { 160 | record_type: filing.header.record_type.clone(), 161 | ef_type: filing.header.ef_type.clone(), 162 | fec_version: filing.header.fec_version.clone(), 163 | software_name: filing.header.software_name.clone(), 164 | software_version: filing.header.software_version.clone(), 165 | report_id: filing.header.report_id.clone(), 166 | report_number: filing.header.report_number.clone(), 167 | comment: filing.header.comment.clone(), 168 | }; 169 | 170 | // Convert cover 171 | let cover = Cover { 172 | form_type: filing.cover.form_type.clone(), 173 | filer_id: filing.cover.filer_id.clone(), 174 | filer_name: filing.cover.filer_name.clone(), 175 | report_code: filing.cover.report_code.clone(), 176 | coverage_from_date: filing.cover.coverage_from_date.map(|d| d.to_string()), 177 | coverage_through_date: filing.cover.coverage_through_date.map(|d| d.to_string()), 178 | }; 179 | 180 | // Collect all itemizations 181 | let mut itemizations = Vec::new(); 182 | while let Some(row_result) = filing.next_row() { 183 | let row = row_result 184 | .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("Failed to read row: {}", e)))?; 185 | 186 | let fields: Vec = row.record.iter().map(|s| s.to_string()).collect(); 187 | itemizations.push(Itemization { 188 | row_type: row.row_type, 189 | fields, 190 | }); 191 | } 192 | 193 | Ok(Filing { 194 | header, 195 | cover, 196 | itemizations, 197 | }) 198 | } 199 | 200 | #[getter] 201 | fn header(&self) -> Header { 202 | self.header.clone() 203 | } 204 | 205 | #[getter] 206 | fn cover(&self) -> Cover { 207 | self.cover.clone() 208 | } 209 | 210 | #[getter] 211 | fn itemizations(&self) -> Vec { 212 | self.itemizations.clone() 213 | } 214 | 215 | fn __repr__(&self) -> String { 216 | format!( 217 | "Filing(form_type='{}', filer_id='{}', {} itemizations)", 218 | self.cover.form_type, self.cover.filer_id, self.itemizations.len() 219 | ) 220 | } 221 | } 222 | 223 | #[pyfunction] 224 | pub fn fec_header(contents: &[u8]) -> PyResult { 225 | let f = fec_parser::Filing::from_reader(contents, "123".to_string(), contents.len()) 226 | .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("Failed to parse filing: {}", e)))?; 227 | Ok(f.header.fec_version) 228 | } 229 | 230 | /// Parser submodule for FEC file parsing 231 | #[pymodule] 232 | pub fn parser(m: &Bound<'_, PyModule>) -> PyResult<()> { 233 | m.add_function(wrap_pyfunction!(fec_header, m)?)?; 234 | m.add_class::()?; 235 | m.add_class::
()?; 236 | m.add_class::()?; 237 | m.add_class::()?; 238 | Ok(()) 239 | } 240 | -------------------------------------------------------------------------------- /crates/fec-cli/src/cache/bulk_utils.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Context; 2 | use jiff::civil::DateTime; 3 | use rusqlite::{types::ToSqlOutput, OptionalExtension, Transaction}; 4 | /** 5 | * Most of the "bulk data" files found in https://www.fec.gov/data/browse-data/?tab=bulk-data 6 | * are zip files with a single '|' delimited text file. 7 | * These utility function works with those files. 8 | */ 9 | use std::{ 10 | io::{BufWriter, Cursor, Read}, 11 | str::FromStr, 12 | }; 13 | use ureq::{http::Response, Body}; 14 | 15 | pub(crate) struct BulkDataItem { 16 | pub(crate) table_name: String, 17 | pub(crate) schema: String, 18 | pub(crate) url_scheme: String, 19 | pub(crate) data_file_name: String, 20 | pub(crate) column_count: usize, 21 | } 22 | 23 | pub(crate) fn csv_reader_from_response( 24 | response: Response, 25 | name: &str, 26 | ) -> anyhow::Result>>> { 27 | let mut buffer = Cursor::new(Vec::new()); 28 | std::io::copy( 29 | &mut response.into_body().into_reader(), 30 | &mut BufWriter::new(&mut buffer), 31 | )?; 32 | 33 | let mut archive = zip::ZipArchive::new(buffer)?; 34 | let mut txt_file = archive.by_name(name)?; 35 | let mut cn_contents = Vec::new(); 36 | txt_file.read_to_end(&mut cn_contents)?; 37 | 38 | Ok(csv::ReaderBuilder::new() 39 | .has_headers(false) 40 | .delimiter(b'|') 41 | .from_reader(Cursor::new(cn_contents))) 42 | } 43 | 44 | pub(crate) fn insert_rows( 45 | tx: &mut Transaction, 46 | cycle_year: u16, 47 | csv_reader: &mut csv::Reader>>, 48 | table_name: &str, 49 | number_of_columns: usize, 50 | ) -> anyhow::Result<()> { 51 | let sql = format!( 52 | "INSERT INTO {table_name} VALUES(?, {})", 53 | (0..number_of_columns) 54 | .map(|_| "?") 55 | .collect::>() 56 | .join(",") 57 | ); 58 | let mut stmt = tx.prepare(&sql)?; 59 | 60 | for result in csv_reader.records() { 61 | let record = result?; 62 | let mut params = vec![ToSqlOutput::from(cycle_year)]; 63 | params.extend( 64 | record 65 | .iter() 66 | .take(number_of_columns) 67 | .map(|s| ToSqlOutput::from(s)), 68 | ); 69 | stmt.execute(rusqlite::params_from_iter(params))?; 70 | } 71 | Ok(()) 72 | } 73 | 74 | #[derive(Debug)] 75 | pub(crate) enum SyncResult { 76 | //New, 77 | Updated, 78 | SkippedRecent, 79 | SkippedNotModified, 80 | } 81 | 82 | struct CycleRow { 83 | modified_at: String, 84 | last_checked_at: String, 85 | } 86 | 87 | pub(crate) fn sync_item( 88 | tx: &mut Transaction, 89 | year: u16, 90 | item: &BulkDataItem, 91 | ) -> anyhow::Result { 92 | tx.execute_batch(&item.schema)?; 93 | tx.execute( 94 | &format!( 95 | r#" 96 | CREATE TABLE IF NOT EXISTS {}_cycles ( 97 | year INTEGER PRIMARY KEY, 98 | modified_at TEXT NOT NULL, 99 | last_checked_at TEXT NOT NULL 100 | ) 101 | "#, 102 | item.table_name 103 | ), 104 | [], 105 | )?; 106 | let result: Option = tx 107 | .query_row( 108 | &format!( 109 | r#" 110 | SELECT 111 | modified_at, 112 | last_checked_at 113 | FROM {}_cycles 114 | WHERE year = ? 115 | "#, 116 | item.table_name 117 | ), 118 | [year], 119 | |row| { 120 | Ok(CycleRow { 121 | modified_at: row.get::(0)?, 122 | last_checked_at: row.get::(1)?, 123 | }) 124 | }, 125 | ) 126 | .optional()?; 127 | 128 | // if there is already data for the given year, and the Last-Modified header 129 | // is recent (within the last 30 minutes), skip the update 130 | if let Some(row) = &result { 131 | // TODO use this somewhere? 132 | let _last_modified = jiff::fmt::rfc2822::parse(&row.modified_at)?; 133 | let last_checked_at = DateTime::from_str(&row.last_checked_at) 134 | .with_context(|| { 135 | format!( 136 | "failed to parse last_checked_at datetime from {}", 137 | &row.last_checked_at 138 | ) 139 | })? 140 | .in_tz("UTC") 141 | .with_context(|| { 142 | format!( 143 | "failed to convert last_checked_at datetime to UTC timezone for {}", 144 | &row.last_checked_at 145 | ) 146 | })? 147 | .timestamp(); 148 | let minutes_since_last_check = jiff::Timestamp::now() 149 | .since(last_checked_at) 150 | .with_context(|| { 151 | format!( 152 | "failed to compute minutes since last_checked_at datetime from {}", 153 | &row.last_checked_at 154 | ) 155 | })? 156 | .total(jiff::Unit::Minute) 157 | .expect("Span total calculations for minutes should never overflow"); 158 | 159 | if minutes_since_last_check < 30.0 { 160 | tx.execute( 161 | &format!( 162 | r#" 163 | UPDATE {}_cycles 164 | SET last_checked_at = datetime('now') 165 | WHERE year = ? 166 | "#, 167 | item.table_name 168 | ), 169 | [year], 170 | ) 171 | .with_context(|| { 172 | format!( 173 | "failed to update last_checked_at for {} {}", 174 | item.table_name, year 175 | ) 176 | })?; 177 | return Ok(SyncResult::SkippedRecent); 178 | } 179 | } 180 | 181 | let config = ureq::Agent::config_builder() 182 | .timeout_global(Some(std::time::Duration::from_secs(30))) 183 | .build(); 184 | let url = item 185 | .url_scheme 186 | .replace("$YEAR2", &year.to_string()[year.to_string().len() - 2..]) 187 | .replace("$YEAR", &year.to_string()); 188 | let request = config.new_agent().get( 189 | &url, //"https://www.fec.gov/files/bulk-downloads/{}/cn{}.zip", 190 | ); 191 | 192 | let request = if let Some(row) = &result { 193 | request.header("If-Modified-Since", &row.modified_at) 194 | } else { 195 | request 196 | }; 197 | 198 | let response = request 199 | .call() 200 | .with_context(|| format!("Failed to fetch bulk data zipfile at {}", url.clone()))?; 201 | if response.status() == 304 { 202 | tx.execute( 203 | &format!( 204 | r#" 205 | UPDATE {}_cycles 206 | SET last_checked_at = datetime('now') 207 | WHERE year = ?; 208 | "#, 209 | item.table_name 210 | ), 211 | rusqlite::params![year], 212 | )?; 213 | 214 | return Ok(SyncResult::SkippedNotModified); 215 | } else if response.status() != 200 { 216 | panic!( 217 | "Failed to fetch candidates for {year}: {}", 218 | response.status() 219 | ); 220 | } 221 | 222 | let last_modified = response 223 | .headers() 224 | .get("Last-Modified") 225 | .ok_or_else(|| { 226 | anyhow::anyhow!( 227 | "response for {} did not include Last-Modified header", 228 | url.clone() 229 | ) 230 | })? 231 | .to_str() 232 | .with_context(|| format!("failed to parse Last-Modified header from {}", url.clone()))?; 233 | tx.execute( 234 | &format!( 235 | r#" 236 | INSERT OR REPLACE INTO {}_cycles (year, modified_at, last_checked_at) 237 | VALUES (?, ?, datetime('now')) 238 | "#, 239 | item.table_name 240 | ), 241 | rusqlite::params![year, last_modified], 242 | ) 243 | .with_context(|| { 244 | format!( 245 | "failed to insert or replace cycle row for {} {}", 246 | item.table_name, year 247 | ) 248 | })?; 249 | 250 | tx.execute( 251 | &format!("DELETE FROM {} WHERE cycle = ?", item.table_name), 252 | [year.to_string()], 253 | ) 254 | .with_context(|| { 255 | format!( 256 | "failed to delete existing rows for {} {}", 257 | item.table_name, year 258 | ) 259 | })?; 260 | 261 | let mut rdr = csv_reader_from_response(response, &item.data_file_name)?; 262 | insert_rows(tx, year, &mut rdr, &item.table_name, item.column_count)?; 263 | return Ok(SyncResult::Updated); 264 | } 265 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # `libfec` 2 | 3 | A CLI for working with [FEC filings](https://www.fec.gov/help-candidates-and-committees/filing-reports/fecfile-software/), for building campaign finance data pipelines, news applications, and reporting tools. 4 | 5 | - A single-binary CLI for MacOS, Linux, and Windows 6 | - Outputs FEC filings to CSVs, JSON, Excel, or SQLite 7 | - Only recent FEC filings version are support, ~2018-present (including the latest `8.5` version) 8 | - Possible Python/Node.js/Ruby/WASM bindings in the future 9 | - Really really fast! 10 | 11 | ```bash 12 | # All financial activity from Elon Musk's America PAC from 2025-2026 to a SQLite database 13 | libfec export C00879510 \ 14 | --cycle 2026 \ 15 | -o america-pac.db 16 | 17 | # ActBlue's 2024 Post-General report as a directory of CSVs 18 | libfec export \ 19 | FEC-1857001 \ 20 | --format csv \ 21 | --output-directory output-actblue 22 | 23 | # George Santos' 2022 House campaign disbursements as a CSV 24 | libfec export C00721365 \ 25 | --target disbursements \ 26 | --cycle 2022 \ 27 | -o santos22.csv 28 | ``` 29 | 30 | In the United States, all candidates running for federal office (House of Representatives, Senate, President, etc.), political action commitees (PACs), and political parties must periodically report financial information to the Federal Election Commission (FEC). These filings are publicly available, and gives the public insight into the people and groups funding federal election campaigns. 31 | 32 | But these filings are extremely complex and hard to analyze. The [FEC website](https://www.fec.gov) offers different APIs and pre-packaged slices of all this data, but it can be hard to navigate or not up-to-date. So, `libfec` allows you to parse and export data directly from raw filings themselves, from the original `.fec` file format. 33 | 34 | There are already many open-source FEC parsers out there (see [Prior Art](#prior-art) for more info). So, `libfec` aims to be a fast, easy-to-use alternative that natively supports CSV, JSON, and SQLite exports! 35 | 36 | ## Installation 37 | 38 | The recommended no-fuss installation method is with our installation script: 39 | 40 | ```sh 41 | curl --proto '=https' --tlsv1.2 -LsSf https://github.com/asg017/libfec/releases/latest/download/fec-cli-installer.sh | sh 42 | ``` 43 | 44 | Alternatively, use the [`libfec` PyPi package](https://pypi.org/project/libfec/) with [`uv`](https://github.com/astral-sh/uv): 45 | 46 | ```sh 47 | uvx libfec --help 48 | 49 | # or use `uv tool` for a global install 50 | uv tool install libfec 51 | libfec --help 52 | ``` 53 | 54 | See [Installing `libfec`](https://alexgarcia.xyz/libfec/getting-started/installation) for all installation options. 55 | 56 | ## Usage 57 | 58 | To get all the itemizations from Nikki Haley's presidential campaign's [March 2024 FEC filing](https://docquery.fec.gov/cgi-bin/forms/C00833392/1781583/), you can run: 59 | 60 | ```bash 61 | libfec export FEC-1781583 -o haley.db 62 | ``` 63 | 64 | And you'll have a SQLite database with all itemizations from that filing, including individual donors (above $200), committee transfers, operating expenses, loan information, and more. 65 | 66 | Under the hood, `libfec` will directly download and stream [the raw `1781583.fec` filing](https://docquery.fec.gov/dcdev/posted/1781583.fec), exporting the individual itemizations rows into SQLite tables. You can then use SQL to extract out the data you need, for visualizations, analysis, or data pipelines. 67 | 68 | Or if you prefer an Excel workbook: 69 | 70 | ```bash 71 | libfec export FEC-1781583 -o haley.xlsx 72 | ``` 73 | 74 | ### All filings for a committee or campaign 75 | 76 | Sometimes you don't care about a single filing — maybe you want *all* filings for a given candidate or committee. 77 | 78 | For example, for all filings made by Congresswoman Young Kim's (CA-39) [principal campaign committee (`C00665638`)](https://www.fec.gov/data/committee/C00665638/) in the 2024 election cycle (Jan 2023 - Dec 2024), in [Form F3](https://www.fec.gov/resources/cms-content/documents/policy-guidance/fecfrm3.pdf), you could run: 79 | 80 | 81 | ```bash 82 | libfec export C00665638 \ 83 | --form-type=F3 \ 84 | --cycle=2024 \ 85 | -o kim.db 86 | ``` 87 | 88 | This will use [the OpenFEC API](https://api.open.fec.gov/developers/) find all relevant filings and export them to `kim.db`. The result is a `34MB` SQLite database with 130k receipt itemizations (individual contributions, transfers, etc.) and nearly 5k disbursements itemizations. 89 | 90 | Using this method, `libfec` will only fetch the most recent filings, and NOT outdated filings that were later amended. 91 | 92 | ### Analyzing parsed FEC filings 93 | 94 | `libfec`'s SQLite support is the best way to analyze a candidate's or committee's financial activity. Using just SQL, you can extract out the exact data you want from a committee's filings, or analyze data directly in a standard way. 95 | 96 | 99 | 100 | Using the `kim.db` example from above — which cities had the highest total donations from individual contributors to Kim’s campaign in 2024? 101 | 102 | 103 | ```sql 104 | select 105 | contributor_city, 106 | contributor_state, 107 | sum(contribution_amount) 108 | from libfec_schedule_a 109 | where form_type = 'SA11AI' -- itemized donations from individuals 110 | and entity_type = 'IND' -- exclude ActBlue memoized items 111 | group by 1, 2 112 | order by 3 desc 113 | limit 10; 114 | ``` 115 | 116 | ``` 117 | contributor_city contributor_state sum(contribution_amount) 118 | ------------------ ------------------- -------------------------- 119 | Los Angeles CA 111615.0 120 | Irvine CA 93362.9 121 | Newport Beach CA 78302.3 122 | New York NY 75881.7 123 | Washington DC 66831.4 124 | Anaheim CA 62406.7 125 | Fullerton CA 59160.4 126 | Santa Ana CA 49312.7 127 | Dallas TX 45014.1 128 | Houston TX 42849.8 129 | ``` 130 | 131 | How much cash did Kim's campaign have throughout the cycle? 132 | 133 | ```sql 134 | select 135 | coverage_from_date, 136 | coverage_through_date, 137 | -- format as USD currency 138 | format('$%,.2f', col_a_cash_on_hand_close_of_period) as cash_on_hand_end 139 | from libfec_F3 140 | order by 1; 141 | ``` 142 | 143 | ``` 144 | coverage_from_date coverage_through_date cash_on_hand_end 145 | -------------------- ----------------------- ------------------ 146 | 2023-01-01 2023-03-31 $902,615.71 147 | 2023-04-01 2023-06-30 $1,653,724.42 148 | 2023-07-01 2023-09-30 $2,223,485.38 149 | 2023-10-01 2023-12-31 $2,536,056.45 150 | 2024-01-01 2024-02-14 $2,509,006.41 151 | 2024-02-15 2024-03-31 $2,947,460.75 152 | 2024-04-01 2024-06-30 $3,610,109.15 153 | 2024-07-01 2024-09-30 $3,313,760.94 154 | 2024-10-01 2024-10-16 $2,822,770.30 155 | 2024-10-17 2024-11-25 $1,755,623.90 156 | 2024-11-26 2024-12-31 $1,737,498.43 157 | 158 | ``` 159 | 160 | How much did the campaign spend on WinRed fees? 161 | 162 | ```sql 163 | select 164 | format('$%,.2f', sum(expenditure_amount)) as total_spent 165 | from libfec_schedule_b 166 | where payee_organization_name = 'WinRed Technical Services' 167 | ``` 168 | 169 | ``` 170 | total_spent 171 | ------------- 172 | $116,433.69 173 | ``` 174 | 175 | ## Prior Art 176 | 177 | There has been nearly 15 years of open source development on various FEC parsers, created by newsroom developers across the nation. Every new parser and tool has learned from it's predecessors, and `libfec` is no exception. 178 | 179 | Specifically, `libfec` adopted many features and configuration from the [FastFEC](https://github.com/washingtonpost/FastFEC) and [fecfile](https://github.com/esonderegger/fecfile) projects. 180 | 181 | Below are all the open source FEC file parsers and tools that I could readily find. Many haven't been updated in a while, but most still work! 182 | 183 | 184 | | Repo | Language | Release date | 185 | | ----------------------------------------- | ------------- | ------------ | 186 | | https://github.com/cschnaars/FEC-Scraper | Python+SQLite | ~2011 | 187 | | https://github.com/dwillis/Fech | Ruby | ~2012? | 188 | | https://github.com/PublicI/fec-parse | Node.js | ~2015 | 189 | | https://github.com/newsdev/fec2json | Python | ~2018 | 190 | | https://github.com/esonderegger/fecfile | Python | ~2018 | 191 | | https://github.com/washingtonpost/FastFEC | C/Python/WASM | ~2021 | 192 | | https://github.com/NickCrews/feco3 | Rust | ~2023 | -------------------------------------------------------------------------------- /crates/fec-cli/src/commands/export/excel.rs: -------------------------------------------------------------------------------- 1 | use crate::{cli::ExportArgs, sourcer::FilingSourcer}; 2 | use fec_parser::{ 3 | mappings::{column_names_for_field, DATE_COLUMNS, FLOAT_COLUMNS}, 4 | schedules::{form_type_schedule_type, ScheduleType}, 5 | Filing, FilingRow, 6 | }; 7 | use rust_xlsxwriter::{worksheet::Worksheet, ExcelDateTime, Format, Workbook}; 8 | use std::{collections::HashMap, io::Read, path::PathBuf}; 9 | 10 | struct ScheduleSheetState { 11 | worksheet: Worksheet, 12 | row_idx: u32, 13 | column_types: Vec, 14 | } 15 | 16 | #[derive(Clone, Copy)] 17 | enum FieldFormat { 18 | Text, 19 | Float, 20 | Date, 21 | } 22 | 23 | fn write_schedule_row( 24 | sheets: &mut HashMap, 25 | schedule: ScheduleType, 26 | filing_fec_version: &str, 27 | row: &FilingRow, 28 | ) -> anyhow::Result<()> { 29 | let state = sheets.entry(schedule.clone()).or_insert_with(|| { 30 | let mut new_ws = Worksheet::new(); 31 | new_ws 32 | .set_name(match schedule { 33 | ScheduleType::ScheduleA => "Schedule A", 34 | ScheduleType::ScheduleB => "Schedule B", 35 | ScheduleType::ScheduleC => "Schedule C", 36 | ScheduleType::ScheduleC1 => "Schedule C1", 37 | ScheduleType::ScheduleC2 => "Schedule C2", 38 | ScheduleType::ScheduleD => "Schedule D", 39 | ScheduleType::ScheduleE => "Schedule E", 40 | ScheduleType::ScheduleF => "Schedule F", 41 | }) 42 | .unwrap(); 43 | 44 | let column_names = column_names_for_field(&row.row_type, filing_fec_version).unwrap(); 45 | let column_types: Vec = column_names 46 | .iter() 47 | .map(|c| { 48 | if DATE_COLUMNS.contains(c) { 49 | FieldFormat::Date 50 | } else if FLOAT_COLUMNS.contains(c) { 51 | FieldFormat::Float 52 | } else { 53 | FieldFormat::Text 54 | } 55 | }) 56 | .collect(); 57 | 58 | for (idx, column_name) in column_names.iter().enumerate() { 59 | // never more than 65535 columns, so u16 is fine 60 | new_ws.write_string(0, idx as u16, column_name).unwrap(); 61 | } 62 | 63 | ScheduleSheetState { 64 | row_idx: 1, 65 | worksheet: new_ws, 66 | column_types, 67 | } 68 | }); 69 | 70 | for (idx, v) in row.record.iter().enumerate() { 71 | state.worksheet.write_string(state.row_idx, idx as u16, v)?; 72 | match state.column_types.get(idx) { 73 | Some(FieldFormat::Float) => { 74 | if let Ok(num) = v.parse::() { 75 | state.worksheet.write_number_with_format( 76 | state.row_idx, 77 | idx as u16, 78 | num, 79 | &Format::new().set_num_format("$,0.00"), 80 | )?; 81 | } 82 | } 83 | Some(FieldFormat::Date) => match jiff::civil::Date::strptime("%Y%m%d", &v) { 84 | Ok(date) => { 85 | let d = ExcelDateTime::from_ymd( 86 | date.year().try_into().unwrap(), 87 | date.month().try_into().unwrap(), 88 | date.day().try_into().unwrap(), 89 | ) 90 | .unwrap(); 91 | state.worksheet.write_datetime_with_format( 92 | state.row_idx, 93 | idx as u16, 94 | d, 95 | &Format::new().set_num_format("yyyy-mm-dd"), 96 | )?; 97 | } 98 | Err(_) => { 99 | state.worksheet.write_string(state.row_idx, idx as u16, v)?; 100 | } 101 | }, 102 | None | Some(FieldFormat::Text) => { 103 | state.worksheet.write_string(state.row_idx, idx as u16, v)?; 104 | } 105 | } 106 | } 107 | state.row_idx += 1; 108 | 109 | /* 110 | let state:&mut ScheduleSheetState = match sheets.get_mut(&schedule) { 111 | 112 | // schedule worksheet already exists, so just append row 113 | //Some(ScheduleSheetState{row_idx, worksheet}) => {} 114 | Some(state) => state, 115 | // schedule worksheet does not exist, so create it 116 | None => { 117 | let mut new_ws = Worksheet::new(); 118 | new_ws.set_name(match schedule { 119 | ScheduleType::ScheduleA => "Schedule A", 120 | ScheduleType::ScheduleB => "Schedule B", 121 | ScheduleType::ScheduleC => "Schedule C", 122 | ScheduleType::ScheduleC1 => "Schedule C1", 123 | ScheduleType::ScheduleC2 => "Schedule C2", 124 | ScheduleType::ScheduleD => "Schedule D", 125 | ScheduleType::ScheduleE => "Schedule E", 126 | ScheduleType::ScheduleF => "Schedule F", 127 | })?; 128 | 129 | let cols = column_names_for_field(&row.row_type, filing_fec_version).unwrap(); 130 | for (idx, col) in cols.iter().enumerate() { 131 | let col_idx = (idx).try_into().unwrap(); 132 | new_ws.write_string(0, col_idx, col)?; 133 | } 134 | for (idx, v) in row.record.iter().enumerate() { 135 | new_ws.write_string(1, (idx).try_into().unwrap(), v)?; 136 | } 137 | sheets.entry(schedule) 138 | sheets.insert(schedule, ScheduleSheetState {row_idx: 2, worksheet: new_ws}); 139 | } 140 | }; 141 | 142 | for (idx, v) in row.record.iter().enumerate() { 143 | worksheet.write_string(*row_idx, (idx).try_into().unwrap(), v)?; 144 | } 145 | (*row_idx) += 1; 146 | */ 147 | Ok(()) 148 | } 149 | 150 | fn write_form_type_row( 151 | sheets: &mut HashMap, 152 | filing_fec_version: &str, 153 | row: &FilingRow, 154 | ) -> anyhow::Result<()> { 155 | match sheets.get_mut(&row.row_type) { 156 | // schedule worksheet already exists, so just append row 157 | Some((idx, worksheet)) => { 158 | let row_idx = (*idx).try_into().unwrap(); 159 | 160 | let mut col_idx = 0; 161 | worksheet.write_string(row_idx, col_idx, "TODO")?; 162 | col_idx += 1; 163 | 164 | for v in &row.record { 165 | worksheet.write_string(row_idx, col_idx, v)?; 166 | //worksheet.write_number_with_format(row, col, number, format) 167 | col_idx += 1 168 | } 169 | (*idx) += 1; 170 | } 171 | None => { 172 | let mut new_ws = Worksheet::new(); 173 | new_ws.set_name(row.row_type.clone())?; 174 | 175 | // header row 176 | let cols = column_names_for_field(&row.row_type, filing_fec_version).unwrap(); 177 | let mut col_idx: u16 = 0; 178 | new_ws.write_string(0, col_idx, "filing_id")?; 179 | col_idx += 1; 180 | for col in cols { 181 | new_ws.write_string(0, col_idx, col)?; 182 | col_idx += 1; 183 | } 184 | 185 | let mut col_idx = 0; 186 | new_ws.write_string(1, col_idx, "TODO")?; 187 | col_idx += 1; 188 | for v in &row.record { 189 | new_ws.write_string(1, col_idx, v)?; 190 | col_idx += 1; 191 | } 192 | 193 | sheets.insert(row.row_type.clone(), (2, new_ws)); 194 | } 195 | }; 196 | Ok(()) 197 | } 198 | 199 | fn add_summary_worksheet( 200 | workbook: &mut Workbook, 201 | filing: &Filing>, 202 | ) -> anyhow::Result<()> { 203 | let ws = workbook.add_worksheet(); 204 | ws.set_name("Summary")?; 205 | ws.write_string(0, 0, &filing.filing_id)?; 206 | 207 | for (idx, (k, v)) in filing.cover.cover_record_kv.iter().enumerate() { 208 | ws.write_string((2 + idx).try_into().unwrap(), 0, k)?; 209 | ws.write_string((2 + idx).try_into().unwrap(), 1, v)?; 210 | } 211 | Ok(()) 212 | } 213 | 214 | pub fn cmd_export_excel( 215 | mut sourcer: FilingSourcer, 216 | path: PathBuf, 217 | args: ExportArgs, 218 | ) -> anyhow::Result<()> { 219 | let mb = indicatif::MultiProgress::new(); 220 | let mut iter = sourcer 221 | .resolve_iterator_from_flags(args.filings, args.api, Some(&mb))? 222 | .1; 223 | let mut filing = iter.next().unwrap().unwrap(); 224 | if iter.next().is_some() { 225 | return Err(anyhow::anyhow!( 226 | "Only one filing supported for Excel export" 227 | )); 228 | } 229 | 230 | let mut workbook = Workbook::new(); 231 | add_summary_worksheet(&mut workbook, &filing)?; 232 | 233 | let mut schedule_sheets: HashMap = HashMap::new(); 234 | let mut rowtype_sheets: HashMap = HashMap::new(); 235 | 236 | while let Some(Ok(row)) = filing.next_row() { 237 | match form_type_schedule_type(&row.row_type) { 238 | Some(schedule) => { 239 | write_schedule_row( 240 | &mut schedule_sheets, 241 | schedule, 242 | &filing.header.fec_version, 243 | &row, 244 | )?; 245 | } 246 | None => { 247 | write_form_type_row(&mut rowtype_sheets, &filing.header.fec_version, &row)?; 248 | } 249 | }; 250 | } 251 | 252 | let mut schedule_sheets: Vec<(ScheduleType, ScheduleSheetState)> = 253 | schedule_sheets.into_iter().collect(); 254 | schedule_sheets.sort_by_key(|(_, ws)| ws.worksheet.name()); 255 | 256 | for ( 257 | _, 258 | ScheduleSheetState { 259 | worksheet: mut ws, .. 260 | }, 261 | ) in schedule_sheets 262 | { 263 | ws.autofit_to_max_width(300); 264 | workbook.push_worksheet(ws); 265 | } 266 | 267 | for (_, (_, ws)) in rowtype_sheets.into_iter() { 268 | workbook.push_worksheet(ws); 269 | } 270 | 271 | workbook.save(path)?; 272 | Ok(()) 273 | } 274 | -------------------------------------------------------------------------------- /crates/fec-py/tests/test_parser.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for libfec_parser.parser module 3 | """ 4 | import pytest 5 | from pathlib import Path 6 | from libfec_parser.parser import fec_header, Filing, Header, Cover, Itemization 7 | 8 | 9 | @pytest.fixture 10 | def sample_fec_file(): 11 | """Get path to a sample FEC file""" 12 | # Look for a sample file in the cache or benchmarks directory 13 | cache_dir = Path(__file__).parent.parent.parent.parent / "cache" 14 | if cache_dir.exists(): 15 | fec_files = list(cache_dir.glob("*.fec")) 16 | if fec_files: 17 | return fec_files[0] 18 | 19 | # Try benchmarks 20 | bench_dir = Path(__file__).parent.parent.parent.parent / "benchmarks" 21 | if bench_dir.exists(): 22 | fec_files = list(bench_dir.glob("*.fec")) 23 | if fec_files: 24 | return fec_files[0] 25 | 26 | pytest.skip("No sample FEC files found") 27 | 28 | 29 | @pytest.fixture 30 | def sample_fec_bytes(sample_fec_file): 31 | """Read sample FEC file as bytes""" 32 | return sample_fec_file.read_bytes() 33 | 34 | 35 | class TestFecHeader: 36 | """Tests for fec_header function""" 37 | 38 | def test_fec_header_returns_version(self, sample_fec_bytes): 39 | """Test that fec_header returns a version string""" 40 | version = fec_header(sample_fec_bytes) 41 | assert isinstance(version, str) 42 | assert len(version) > 0 43 | 44 | def test_fec_header_with_invalid_data(self): 45 | """Test fec_header with invalid data""" 46 | with pytest.raises(ValueError): 47 | fec_header(b"invalid fec data") 48 | 49 | def test_fec_header_with_empty_bytes(self): 50 | """Test fec_header with empty bytes""" 51 | with pytest.raises(ValueError): 52 | fec_header(b"") 53 | 54 | 55 | class TestHeader: 56 | """Tests for Header class""" 57 | 58 | def test_header_attributes(self, sample_fec_file): 59 | """Test that Header has expected attributes""" 60 | filing = Filing(str(sample_fec_file)) 61 | header = filing.header 62 | 63 | assert isinstance(header, Header) 64 | assert hasattr(header, 'record_type') 65 | assert hasattr(header, 'ef_type') 66 | assert hasattr(header, 'fec_version') 67 | assert hasattr(header, 'software_name') 68 | assert hasattr(header, 'software_version') 69 | assert hasattr(header, 'report_id') 70 | assert hasattr(header, 'report_number') 71 | assert hasattr(header, 'comment') 72 | 73 | def test_header_repr(self, sample_fec_file): 74 | """Test Header __repr__""" 75 | filing = Filing(str(sample_fec_file)) 76 | header = filing.header 77 | repr_str = repr(header) 78 | 79 | assert isinstance(repr_str, str) 80 | assert 'Header' in repr_str 81 | assert header.fec_version in repr_str 82 | 83 | 84 | class TestCover: 85 | """Tests for Cover class""" 86 | 87 | def test_cover_attributes(self, sample_fec_file): 88 | """Test that Cover has expected attributes""" 89 | filing = Filing(str(sample_fec_file)) 90 | cover = filing.cover 91 | 92 | assert isinstance(cover, Cover) 93 | assert hasattr(cover, 'form_type') 94 | assert hasattr(cover, 'filer_id') 95 | assert hasattr(cover, 'filer_name') 96 | assert hasattr(cover, 'report_code') 97 | assert hasattr(cover, 'coverage_from_date') 98 | assert hasattr(cover, 'coverage_through_date') 99 | 100 | def test_cover_repr(self, sample_fec_file): 101 | """Test Cover __repr__""" 102 | filing = Filing(str(sample_fec_file)) 103 | cover = filing.cover 104 | repr_str = repr(cover) 105 | 106 | assert isinstance(repr_str, str) 107 | assert 'Cover' in repr_str 108 | assert cover.form_type in repr_str 109 | 110 | def test_cover_fields_method(self, sample_fec_file): 111 | """Test Cover.fields() returns a dictionary""" 112 | filing = Filing(str(sample_fec_file)) 113 | cover = filing.cover 114 | fields = cover.fields() 115 | 116 | assert isinstance(fields, dict) 117 | assert 'form_type' in fields 118 | assert 'filer_id' in fields 119 | assert 'filer_name' in fields 120 | 121 | 122 | class TestItemization: 123 | """Tests for Itemization class""" 124 | 125 | def test_itemization_attributes(self, sample_fec_file): 126 | """Test that Itemization has expected attributes""" 127 | filing = Filing(str(sample_fec_file)) 128 | 129 | if len(filing.itemizations) > 0: 130 | itemization = filing.itemizations[0] 131 | assert isinstance(itemization, Itemization) 132 | assert hasattr(itemization, 'row_type') 133 | assert isinstance(itemization.row_type, str) 134 | 135 | def test_itemization_repr(self, sample_fec_file): 136 | """Test Itemization __repr__""" 137 | filing = Filing(str(sample_fec_file)) 138 | 139 | if len(filing.itemizations) > 0: 140 | itemization = filing.itemizations[0] 141 | repr_str = repr(itemization) 142 | 143 | assert isinstance(repr_str, str) 144 | assert 'Itemization' in repr_str 145 | assert itemization.row_type in repr_str 146 | 147 | def test_itemization_len(self, sample_fec_file): 148 | """Test Itemization __len__""" 149 | filing = Filing(str(sample_fec_file)) 150 | 151 | if len(filing.itemizations) > 0: 152 | itemization = filing.itemizations[0] 153 | length = len(itemization) 154 | 155 | assert isinstance(length, int) 156 | assert length >= 0 157 | 158 | def test_itemization_getitem_positive_index(self, sample_fec_file): 159 | """Test Itemization __getitem__ with positive index""" 160 | filing = Filing(str(sample_fec_file)) 161 | 162 | if len(filing.itemizations) > 0: 163 | itemization = filing.itemizations[0] 164 | if len(itemization) > 0: 165 | field = itemization[0] 166 | assert isinstance(field, str) 167 | 168 | def test_itemization_getitem_negative_index(self, sample_fec_file): 169 | """Test Itemization __getitem__ with negative index""" 170 | filing = Filing(str(sample_fec_file)) 171 | 172 | if len(filing.itemizations) > 0: 173 | itemization = filing.itemizations[0] 174 | if len(itemization) > 0: 175 | field = itemization[-1] 176 | assert isinstance(field, str) 177 | 178 | def test_itemization_getitem_out_of_bounds(self, sample_fec_file): 179 | """Test Itemization __getitem__ with out of bounds index""" 180 | filing = Filing(str(sample_fec_file)) 181 | 182 | if len(filing.itemizations) > 0: 183 | itemization = filing.itemizations[0] 184 | with pytest.raises(IndexError): 185 | _ = itemization[9999] 186 | 187 | def test_itemization_fields_method(self, sample_fec_file): 188 | """Test Itemization.fields() returns a list""" 189 | filing = Filing(str(sample_fec_file)) 190 | 191 | if len(filing.itemizations) > 0: 192 | itemization = filing.itemizations[0] 193 | fields = itemization.fields() 194 | 195 | assert isinstance(fields, list) 196 | assert all(isinstance(f, str) for f in fields) 197 | 198 | 199 | class TestFiling: 200 | """Tests for Filing class""" 201 | 202 | def test_filing_from_path_string(self, sample_fec_file): 203 | """Test Filing initialization with file path string""" 204 | filing = Filing(str(sample_fec_file)) 205 | 206 | assert isinstance(filing, Filing) 207 | assert isinstance(filing.header, Header) 208 | assert isinstance(filing.cover, Cover) 209 | assert isinstance(filing.itemizations, list) 210 | 211 | def test_filing_from_bytes(self, sample_fec_bytes): 212 | """Test Filing initialization with bytes""" 213 | filing = Filing(sample_fec_bytes) 214 | 215 | assert isinstance(filing, Filing) 216 | assert isinstance(filing.header, Header) 217 | assert isinstance(filing.cover, Cover) 218 | 219 | def test_filing_from_file_object(self, sample_fec_file): 220 | """Test Filing initialization with file-like object""" 221 | with open(sample_fec_file, 'rb') as f: 222 | filing = Filing(f) 223 | 224 | assert isinstance(filing, Filing) 225 | assert isinstance(filing.header, Header) 226 | assert isinstance(filing.cover, Cover) 227 | 228 | def test_filing_repr(self, sample_fec_file): 229 | """Test Filing __repr__""" 230 | filing = Filing(str(sample_fec_file)) 231 | repr_str = repr(filing) 232 | 233 | assert isinstance(repr_str, str) 234 | assert 'Filing' in repr_str 235 | assert filing.cover.form_type in repr_str 236 | 237 | def test_filing_header_property(self, sample_fec_file): 238 | """Test Filing.header property""" 239 | filing = Filing(str(sample_fec_file)) 240 | header = filing.header 241 | 242 | assert isinstance(header, Header) 243 | assert header.fec_version 244 | 245 | def test_filing_cover_property(self, sample_fec_file): 246 | """Test Filing.cover property""" 247 | filing = Filing(str(sample_fec_file)) 248 | cover = filing.cover 249 | 250 | assert isinstance(cover, Cover) 251 | assert cover.form_type 252 | assert cover.filer_id 253 | 254 | def test_filing_itemizations_property(self, sample_fec_file): 255 | """Test Filing.itemizations property""" 256 | filing = Filing(str(sample_fec_file)) 257 | itemizations = filing.itemizations 258 | 259 | assert isinstance(itemizations, list) 260 | assert all(isinstance(item, Itemization) for item in itemizations) 261 | 262 | def test_filing_with_invalid_path(self): 263 | """Test Filing with non-existent file path""" 264 | with pytest.raises(IOError): 265 | Filing("/path/that/does/not/exist.fec") 266 | 267 | def test_filing_with_invalid_type(self): 268 | """Test Filing with invalid input type""" 269 | with pytest.raises(TypeError): 270 | Filing(12345) # Invalid type 271 | 272 | def test_filing_with_invalid_data(self): 273 | """Test Filing with invalid FEC data""" 274 | with pytest.raises(ValueError): 275 | Filing(b"invalid fec data") 276 | -------------------------------------------------------------------------------- /crates/fec-cli/src/commands/info.rs: -------------------------------------------------------------------------------- 1 | use colored::Colorize; 2 | use fec_parser::{ 3 | covers::{Cover, Form3PSummary}, 4 | report_code_label, Filing, 5 | }; 6 | use indicatif::{HumanBytes, ProgressBar}; 7 | use serde_json::Value; 8 | use std::{collections::HashMap, error::Error, io::Read, time::Duration}; 9 | 10 | use tabled::{ 11 | builder::Builder as TableBuilder, 12 | settings::{object::Columns as TableColumns, Alignment as TableAlignment, Style as TableStyle}, 13 | }; 14 | 15 | use crate::{ 16 | cli::{CmdInfoFormat, InfoArgs}, 17 | sourcer::FilingSourcer, 18 | }; 19 | struct FilingFormMetadata { 20 | count: usize, 21 | bytes: usize, 22 | } 23 | 24 | fn form_name(form_type: &str) -> &str { 25 | let base_form_type = if form_type.ends_with('A') || form_type.ends_with('N') { 26 | &form_type[..form_type.len() - 1] 27 | } else { 28 | form_type 29 | }; 30 | 31 | match base_form_type { 32 | "F1" => "Statement of Organization", 33 | "F1M" => "Notification of Multicandidate Status", 34 | "F2" => "Statement of Candidacy", 35 | "F24" => "24/48 Hour Report of Independent Expenditures", 36 | "F3" => "Report of Receipts and Disbursements for an Authorized Committee", 37 | "F3P" => "Report of Receipts and Disbursements by an Authorized Committee of a Candidate for The Office of President or Vice President", 38 | "F3L" => "Report of Contributions Bundled by Lobbyists/Registrants and Lobbyist/Registrant PACs", 39 | "F3X" => "Report of Receipts and Disbursements for other than an Authorized Committee", 40 | "F4" => "Report of Receipts and Disbursements for a Committee or Organization Supporting a Nomination Convention", 41 | "F5" => "Report of Independent Expenditures Made and Contributions Received", 42 | "F6" => "48 Hour Notice of Contributions/Loans Received", 43 | "F7" => "Report of Communication Costs by Corporations and Membership Organizations", 44 | "F8" => "Debt Settlement Plan", 45 | "F9" => "24 Hour Notice of Disbursements for Electioneering Communications", 46 | "F13" => "Report of Donations Accepted for Inaugural Committee", 47 | "F99" => "Miscellaneous Text", 48 | "FRQ" => "Request for Additional Information", 49 | _ => "", 50 | } 51 | } 52 | use num_format::{Locale, ToFormattedString}; 53 | use tabled::{builder::Builder, settings::Style}; 54 | 55 | fn format_usd(amount: f64) -> String { 56 | let rounded = (amount * 100.0).round() as i64; // convert to cents 57 | let dollars = rounded / 100; 58 | let cents = (rounded % 100).abs(); // handle negative cents correctly 59 | 60 | format!("${}.{:02}", dollars.to_formatted_string(&Locale::en), cents) 61 | } 62 | 63 | fn print_summary(summary: &Form3PSummary) { 64 | let mut b = Builder::with_capacity(3, 0); 65 | b.push_record(["Summary"]); 66 | 67 | let items = vec![ 68 | ( 69 | "6. Cash on Hand at BEGINNING of the Reporting Period", 70 | summary.line6_cash_on_hand_beginning_period, 71 | ), 72 | ( 73 | "7. Total Receipts This Period", 74 | summary.line7_total_receipts, 75 | ), 76 | ("8. Subtotal (6 + 7)", summary.line8_subtotal), 77 | ( 78 | "9. Total Disbursements This Period", 79 | summary.line9_total_disbursements, 80 | ), 81 | ( 82 | "10. Cash on Hand at CLOSE of the Reporting Period", 83 | summary.line10_cash_on_hand_end_period, 84 | ), 85 | ( 86 | "11. Debts and Obligations Owed TO the Committee", 87 | summary.line11_debts_owed_to_committee, 88 | ), 89 | ( 90 | "12. Debts and Obligations Owed BY the Committee", 91 | summary.line12_debts_owed_by_committee, 92 | ), 93 | ( 94 | "13. Expenditures Subject To Limitation", 95 | summary.line13_expenditures_subject_to_limits, 96 | ), 97 | ( 98 | "14. NET Contributions (Other than Loans)", 99 | summary.line14_net_contributions_other_than_loans, 100 | ), 101 | ( 102 | "15. NET Operating Expenditures", 103 | summary.line15_net_operating_expenditures, 104 | ), 105 | ]; 106 | for (label, value) in items { 107 | b.push_record([label.to_string(), format_usd(value)]); 108 | } 109 | 110 | let mut table = b.build(); 111 | table.with(Style::modern()); 112 | table.modify( 113 | tabled::settings::object::Columns::last(), 114 | tabled::settings::Alignment::right(), 115 | ); 116 | 117 | // make 1st row (title) span entire width 118 | table 119 | .modify((0, 0), tabled::settings::Span::column(2)) 120 | .modify((0, 0), tabled::settings::Alignment::center()); 121 | // border correct bc header row does weird stuff 122 | table.with(tabled::settings::themes::BorderCorrection::span()); 123 | println!("{}", table) 124 | } 125 | 126 | fn process_filing( 127 | filing: &mut Filing, 128 | format: &CmdInfoFormat, 129 | spinner: &Option, 130 | full: bool, 131 | ) { 132 | if matches!(format, CmdInfoFormat::Human) { 133 | if !full { 134 | if let Some(ref spinner) = spinner { 135 | spinner.finish_and_clear(); 136 | } 137 | } 138 | 139 | println!( 140 | "{} {} {} by {} ({})", 141 | format!("FEC-{}", filing.filing_id).bold(), 142 | filing.cover.form_type, 143 | filing 144 | .cover 145 | .report_code 146 | .as_ref() 147 | .map(|report_code| report_code_label(report_code.as_str())) 148 | .unwrap_or(""), 149 | filing.cover.filer_name.bold(), 150 | filing.cover.filer_id, 151 | ); 152 | if let (Some(from), Some(through)) = ( 153 | filing.cover.coverage_from_date, 154 | filing.cover.coverage_through_date, 155 | ) { 156 | println!("Covering {} to {}", from.to_string(), through.to_string(),); 157 | } 158 | println!(); 159 | 160 | println!("{}", form_name(&filing.cover.form_type).dimmed()); 161 | 162 | if let Some(ref cover) = filing.cover.cover_data { 163 | match cover { 164 | Cover::Form3P(form) => { 165 | println!( 166 | "Signed by {} on {}", 167 | form.treasurer.to_string().bold(), 168 | form.signed.to_string().bold() 169 | ); 170 | print_summary(&form.summary); 171 | } 172 | } 173 | } 174 | 175 | println!( 176 | "{}", 177 | format!( 178 | "https://docquery.fec.gov/cgi-bin/forms/{}/{}", 179 | filing.cover.filer_id, filing.filing_id 180 | ) 181 | .blue() 182 | ); 183 | 184 | println!( 185 | "v{} {} filed with {} {}", 186 | filing.header.fec_version, 187 | HumanBytes(filing.source_length as u64), 188 | filing.header.software_name, 189 | filing.header.software_version 190 | ); 191 | 192 | if let Some(ref report_id) = filing.header.report_id { 193 | println!("{}: '{}'", "Report ID".bold(), report_id); 194 | } 195 | if let Some(ref report_number) = filing.header.report_number { 196 | println!("Report #{}", report_number); 197 | } 198 | if let Some(ref comment) = filing.header.comment { 199 | println!("{}: '{}'", "Comment".bold(), comment); 200 | } 201 | } 202 | if !full { 203 | return; 204 | } 205 | 206 | if let Some(spinner) = spinner { 207 | spinner.set_message("Summarizing rows…"); 208 | } 209 | 210 | let mut status: HashMap = HashMap::new(); 211 | while let Some(row) = filing.next_row() { 212 | let row = row.unwrap(); 213 | if let Some(x) = status.get_mut(&row.row_type) { 214 | x.count += 1; 215 | x.bytes += row.original_size; 216 | } else { 217 | status.insert( 218 | row.row_type.clone(), 219 | FilingFormMetadata { 220 | count: 1, 221 | bytes: row.original_size, 222 | }, 223 | ); 224 | } 225 | } 226 | 227 | if let Some(ref spinner) = spinner { 228 | spinner.finish_and_clear(); 229 | } 230 | 231 | let mut x: Vec<_> = status.iter().collect(); 232 | x.sort_by(|a, b| b.1.count.cmp(&a.1.count)); 233 | match format { 234 | CmdInfoFormat::Human => { 235 | let mut tbl = TableBuilder::new(); 236 | tbl.push_record(["Form Type", "# Rows", "Size"]); 237 | for (x, y) in x { 238 | tbl.push_record([ 239 | x, 240 | &indicatif::HumanCount(y.count as u64).to_string(), 241 | &indicatif::HumanBytes(y.bytes as u64).to_string(), 242 | ]); 243 | } 244 | let tbl = tbl 245 | .build() 246 | .with(TableStyle::modern_rounded()) 247 | .modify(TableColumns::new(1..3), TableAlignment::right()) 248 | .to_string(); 249 | 250 | println!("{tbl}"); 251 | } 252 | CmdInfoFormat::Json => { 253 | let v = Value::Null; 254 | println!("{}", v); 255 | } 256 | } 257 | } 258 | 259 | enum InfoInput { 260 | Filing(String), 261 | Commitee(String), 262 | //Canddate(String), 263 | } 264 | pub fn info(sourcer: FilingSourcer, args: InfoArgs) -> anyhow::Result<()> { 265 | let spinner = match args.format { 266 | CmdInfoFormat::Human => { 267 | let s = ProgressBar::new_spinner(); 268 | s.enable_steady_tick(Duration::from_millis(100)); 269 | Some(s) 270 | } 271 | _ => None, 272 | }; 273 | let inputs = args.filings.iter().map(|v| { 274 | if v.starts_with("C") { 275 | InfoInput::Commitee(v.clone()) 276 | } else { 277 | InfoInput::Filing(v.clone()) 278 | } 279 | }); 280 | for input in inputs { 281 | match input { 282 | InfoInput::Filing(filing) => { 283 | let mut filing = sourcer.resolve_from_user_argument(&filing)?; 284 | process_filing(&mut filing, &args.format, &spinner, args.full); 285 | } 286 | InfoInput::Commitee(commitee_id) => { 287 | // TODO print info about the committee 288 | println!("{commitee_id}"); 289 | } 290 | } 291 | } 292 | 293 | Ok(()) 294 | } 295 | --------------------------------------------------------------------------------