congress-analyzer/legacy/fetch_votes.py

#!/usr/bin/env python3
"""Fetch all House roll call votes from 119th Congress and analyze Massie's record."""
import urllib.request
import xml.etree.ElementTree as ET
import json
import time
import os
import sys

MASSIE_ID = "M001184"
YEARS = {2025: 362, 2026: 191}
CACHE_DIR = "/home/user/polisci/vote_cache"
os.makedirs(CACHE_DIR, exist_ok=True)

UA = "Mozilla/5.0 (research; polisci-analysis)"

def fetch(year, roll):
    path = f"{CACHE_DIR}/{year}_{roll:03d}.xml"
    if os.path.exists(path) and os.path.getsize(path) > 200:
        with open(path, "rb") as f:
            return f.read()
    url = f"https://clerk.house.gov/evs/{year}/roll{roll:03d}.xml"
    req = urllib.request.Request(url, headers={"User-Agent": UA})
    try:
        with urllib.request.urlopen(req, timeout=30) as r:
            data = r.read()
        with open(path, "wb") as f:
            f.write(data)
        time.sleep(0.35)  # throttle
        return data
    except Exception as e:
        print(f"FAIL {year}/{roll}: {e}", file=sys.stderr)
        return None

def parse(data, year, roll):
    try:
        root = ET.fromstring(data)
    except Exception as e:
        return None
    meta = root.find("vote-metadata")
    if meta is None:
        return None
    def t(tag):
        el = meta.find(tag)
        return (el.text or "").strip() if el is not None else ""
    info = {
        "year": year, "roll": roll,
        "date": t("action-date"),
        "question": t("vote-question"),
        "result": t("vote-result"),
        "legis_num": t("legis-num"),
        "desc": t("vote-desc"),
        "majority": t("majority"),
    }
    # party totals
    party_totals = {}
    for pt in meta.findall("vote-totals/totals-by-party"):
        party = pt.findtext("party", "").strip()
        party_totals[party] = {
            "yea": int(pt.findtext("yea-total", "0") or 0),
            "nay": int(pt.findtext("nay-total", "0") or 0),
            "present": int(pt.findtext("present-total", "0") or 0),
            "nv": int(pt.findtext("not-voting-total", "0") or 0),
        }
    info["R"] = party_totals.get("Republican", {"yea":0,"nay":0,"present":0,"nv":0})
    info["D"] = party_totals.get("Democratic", {"yea":0,"nay":0,"present":0,"nv":0})
    info["I"] = party_totals.get("Independent", {"yea":0,"nay":0,"present":0,"nv":0})
    # Massie's vote
    massie = None
    for rv in root.iter("recorded-vote"):
        leg = rv.find("legislator")
        if leg is not None and leg.get("name-id") == MASSIE_ID:
            v = rv.find("vote")
            massie = (v.text or "").strip() if v is not None else None
            break
    info["massie"] = massie
    return info

def classify(v):
    """Given parsed vote, return (alignment, blocked_side)."""
    r_yea, r_nay = v["R"]["yea"], v["R"]["nay"]
    d_yea, d_nay = v["D"]["yea"], v["D"]["nay"]
    # Determine each party's majority position
    r_pos = "Yea" if r_yea > r_nay else ("Nay" if r_nay > r_yea else "Split")
    d_pos = "Yea" if d_yea > d_nay else ("Nay" if d_nay > d_yea else "Split")
    m = v["massie"]
    if m not in ("Yea", "Nay", "Aye", "No"):
        return ("N/A: " + (m or "absent"), None, r_pos, d_pos)
    # Normalize Aye/No to Yea/Nay
    m_norm = "Yea" if m in ("Yea", "Aye") else "Nay"
    helped_r = (r_pos != "Split" and m_norm == r_pos)
    helped_d = (d_pos != "Split" and m_norm == d_pos)
    if helped_r and helped_d:
        align = "Helped Both"
    elif helped_r:
        align = "Helped Republicans"
    elif helped_d:
        align = "Helped Democrats"
    else:
        align = "Helped Neither"

    # Blocking analysis: Massie voted against [side]'s majority position, AND that side lost the vote
    result = v["result"].lower()
    measure_passed = result in ("passed", "agreed to", "adopted")
    measure_failed = "fail" in result or "reject" in result or "not agreed" in result or "not passed" in result

    blocked = None
    # "Dem-backed measure": D majority was Yea -> they wanted it to pass.
    # If D wanted Yea, measure failed, and Massie voted Nay -> Massie helped block a Dem-backed measure.
    if d_pos == "Yea" and measure_failed and m_norm == "Nay":
        if r_pos != "Yea":  # only count if Rs didn't also back it -> actually a partisan block
            blocked = "Democrat"
    # Also if D wanted Nay (to defeat it) but it passed and Massie voted Yea... that's not blocking, that's helping pass
    if r_pos == "Yea" and measure_failed and m_norm == "Nay":
        if d_pos != "Yea":
            blocked = "Republican"
    return (align, blocked, r_pos, d_pos)

def main():
    all_votes = []
    total = sum(YEARS.values())
    done = 0
    for year, max_roll in YEARS.items():
        for roll in range(1, max_roll + 1):
            data = fetch(year, roll)
            done += 1
            if done % 25 == 0:
                print(f"  fetched {done}/{total}", file=sys.stderr)
            if not data:
                continue
            v = parse(data, year, roll)
            if not v:
                continue
            align, blocked, r_pos, d_pos = classify(v)
            v["alignment"] = align
            v["blocked"] = blocked
            v["r_pos"] = r_pos
            v["d_pos"] = d_pos
            all_votes.append(v)
    with open("/home/user/polisci/votes.json", "w") as f:
        json.dump(all_votes, f)
    print(f"Saved {len(all_votes)} votes", file=sys.stderr)

if __name__ == "__main__":
    main()