"""Who votes in a New York City governor's race: the analysis behind a SparkyData article. py src/build_article.py who-votes-in-a-midterm The question: a governor's race draws far fewer New Yorkers than a presidential one; who are the ones who stay home, and has that changed in thirty years? Datasets (cards in datasets/): - ``cps_voting``: the Census Bureau's November Voting and Registration Supplements, 1996-2024, the city's adult citizens (about 1,000 to 1,900 a year): whether each said they registered and voted, with age, schooling, birthplace and tenure. The reader reproduces the Bureau's Table 4b for New York State in 2018 and 2024 exactly. - ``nyc_boe_turnout``: the Board of Elections' ballots for the same elections, set against the survey's own count of adult citizens, to show how far the reported rate runs above the count. Survey answers are reported votes. Elections of a kind are pooled (the four midterms 2010-2022, the five presidential elections 2008-2024, and the earlier ones) so the groups carry enough people; margins are 90%, clustered on the household in each year, and approximate. Deterministic: no sampling. """ from __future__ import annotations import numpy as np import pandas as pd from article_kit import Article, lines, r, strip from nycdata import cps, cps_voting, elections from sparkylib.stats import moe_difference SLUG = "who-votes-in-a-midterm" SEED = 20261030 ERAS = {"mid_recent": ("Midterms 2010-2022", [2010, 2014, 2018, 2022]), "pres_recent": ("Presidential 2008-2024", [2008, 2012, 2016, 2020, 2024]), "mid_early": ("Midterms 1998-2006", [1998, 2002, 2006]), "pres_early": ("Presidential 1996-2004", [1996, 2000, 2004])} AGES = [("18_29", "18 to 29", 18, 29), ("30_44", "30 to 44", 30, 44), ("45_64", "45 to 64", 45, 64), ("65_plus", "65 and over", 65, 200)] SPLITS = { "age": [("18_29", "Under 30", lambda d: d["age"] <= 29), ("65_plus", "65 and over", lambda d: d["age"] >= 65)], "school": [("no_degree", "No bachelor's degree", lambda d: d["educ"] < 43), ("degree", "Bachelor's degree or more", lambda d: d["educ"] >= 43)], "born": [("naturalized", "Naturalized citizens", lambda d: d["cit"] == 4), ("native", "Native-born", lambda d: d["cit"] <= 3)], "tenure": [("renter", "Renters", lambda d: d["tenure"].isin([2, 3])), ("owner", "Owners", lambda d: d["tenure"] == 1)], } DATES = ("nyc_boe_important_dates_2026", "nyc_boe/all_important_dates_2026.html") SPLIT_LABEL = {"age": "Age: 65 and over against under 30", "school": "Schooling: a degree against none", "born": "Birthplace: native-born against naturalized", "tenure": "Home: owners against renters"} SPLIT_SHORT = {"age": "Old vs young", "school": "Degree vs none", "born": "Native vs naturalized", "tenure": "Owners vs renters"} def city_records(years: list[int]) -> pd.DataFrame: d = pd.concat([cps_voting.load(y) for y in years], ignore_index=True) return d[d["borough"].notna()].reset_index(drop=True) def voted_share(d: pd.DataFrame, des, mask, flag: str = "voted1") -> dict: m = np.asarray(mask, bool) v, moe = cps.ratio_moe(d, y=d[flag].to_numpy(), x=np.ones(len(d)), weight="weight", mask=m, design=des) w = d["weight"].to_numpy() return {"share": r(100 * v, 2), "moe": r(100 * moe, 2), "records": int(m.sum()), "weighted_yes": r(float((w * d[flag].to_numpy())[m].sum()), 0), "weighted_base": r(float(w[m].sum()), 0)} def gap_between(d: pd.DataFrame, des, a, b, flag: str = "voted1") -> dict: """The share in ``b`` minus the share in ``a``, with the margin of the difference of the two ratios from the same design (influence values subtracted).""" w = d["weight"].to_numpy() y = d[flag].to_numpy() ra, za = cps.ratio(y, np.ones(len(d)), w, np.asarray(a, bool)) rb, zb = cps.ratio(y, np.ones(len(d)), w, np.asarray(b, bool)) dv, mv = 100 * (rb - ra), 100 * des.moe(zb - za) return {"pts": r(dv, 2), "lo": r(dv - mv, 2), "hi": r(dv + mv, 2)} def main(argv: list[str] | None = None) -> int: """Run the analysis and write every file (no options).""" A = Article(SLUG, __file__, seed=SEED) P = A.printed years = cps_voting.years() assert years[0] == 1996 and years[-1] == 2024, "the page is written for the 1996-2024 files" # ---- every column read from a fixed-width year against its layout, the household number included lay = cps_voting.check_layouts()[0] assert lay.passed, lay.detail A.results["checks"] = {"cps_voting_layouts": lay.detail} # ---- the reproduction: the Bureau's own table for New York State for y in (2018, 2024): res = cps_voting.check_published(y)[0] assert res.passed, res.detail d = cps_voting.load(y) A.reproduce(f"New York State, adult citizens who said they voted, November {y} (%)", published=float(res.target), published_moe90=None, reproduced=r(cps_voting.reported_rate(d), 1), kind="share", tolerance=0.05) A.results["reproduction_source"] = "U.S. Census Bureau, Table 4b (P20-583 and P20-590), New York State, total citizen population" A.results["reproduction_note"] = "The Bureau's published percentage voted, reproduced from the public-use file to one decimal." # ---- every year: the city's reported rate, and the Board's ballots per hundred adult citizens (the census's # count, elections.adult_citizens: 2005 left out, years between files a straight line, assumed), with the # survey's own weighted count beside it t = elections.turnout().set_index("year") ac = elections.adult_citizens(1996, 2024) ac = ac[ac["area"] == "New York City"].set_index("year") yearly = {} for y in years: d = city_records([y]) des = cps_voting.design(d) tot = voted_share(d, des, np.ones(len(d), bool)) cit = float(d["weight"].sum()) counted = 100 * t.loc[y, "ballots"] / ac.loc[y, "count"] blk = {"reported": tot["share"], "reported_moe": tot["moe"], "records": tot["records"], "weighted_yes": tot["weighted_yes"], "weighted_base": tot["weighted_base"], "ballots": int(t.loc[y, "ballots"]), "counted_per_100": r(counted, 2), "over_report": r(tot["share"] - counted, 2), "denominator": ac.loc[y, "basis"], "census_adult_citizens": r(float(ac.loc[y, "count"]), -3), "survey_adult_citizens": r(cit, -3), "counted_per_100_survey_count": r(100 * t.loc[y, "ballots"] / cit, 2), "kind": t.loc[y, "kind"], "midterm": y % 4 == 2} # who gave no answer to the vote question (counted as not voting, as the Bureau counts them) noans = (~d["voted"].isin([1, 2])).astype(float).to_numpy() for k, lo, hi in (("young", 18, 29), ("old", 65, 200)): m = d["age"].between(lo, hi).to_numpy() blk[f"no_answer_{k}"] = r(100 * np.sum(d["weight"].to_numpy()[m] * noans[m]) / d["weight"].to_numpy()[m].sum(), 1) for k, lab, lo, hi in AGES: blk[f"age_{k}"] = voted_share(d, des, d["age"].between(lo, hi))["share"] blk["degree"] = voted_share(d, des, d["educ"] >= 43)["share"] blk["no_degree"] = voted_share(d, des, d["educ"] < 43)["share"] yearly[str(y)] = blk A.results["yearly"] = yearly ov = {y: v["over_report"] for y, v in yearly.items()} mid_ov = [v for y, v in ov.items() if int(y) % 4 == 2] pres_ov = [v for y, v in ov.items() if int(y) % 4 == 0] below = [y for y, v in ov.items() if v <= 0] A.results["over_report"] = {"lo": min(ov.values()), "hi": max(ov.values()), "2022": ov["2022"], "2024": ov["2024"], "years_above": len(ov) - len(below), "years": len(ov), "years_not_above": below, "midterm_mean": r(float(np.mean(mid_ov)), 1), "presidential_mean": r(float(np.mean(pres_ov)), 1), "midterm_lo": min(mid_ov), "midterm_hi": max(mid_ov), "presidential_lo": min(pres_ov), "presidential_hi": max(pres_ov)} assert not below, f"the page says the survey's rate ran above the Board's count in every election: {below}" # the difference is a comparison of two rates on different bases: name the years it is inside the survey's own # margin, and the years the Board's ballots per hundred of the survey's own count of adult citizens is the higher within = [y for y, v in yearly.items() if v["over_report"] < v["reported_moe"]] on_survey_count = [y for y, v in yearly.items() if v["counted_per_100_survey_count"] > v["reported"]] A.results["over_report"]["years_within_margin"] = within A.results["over_report"]["years_counted_higher_on_survey_count"] = on_survey_count assert within == ["2004", "2008"] and on_survey_count == ["2008"], (within, on_survey_count) # the same screen on the survey's own count (a screen against the reported rate's margin alone: the census # denominator's margin is not propagated, and the survey's own denominator is shared with the numerator) within_survey = [y for y, v in yearly.items() if abs(v["reported"] - v["counted_per_100_survey_count"]) < v["reported_moe"]] A.results["over_report"]["years_within_margin_survey_base"] = within_survey assert within_survey == ["2000", "2008", "2024"], within_survey # the same difference on the survey's own weighted count of adult citizens, which runs below the census's by up to # a tenth before 2004 (the CPS's 1990-based controls) and leaves out people in institutions ovs = {y: r(v["reported"] - v["counted_per_100_survey_count"], 2) for y, v in yearly.items()} mid_s = [v for y, v in ovs.items() if int(y) % 4 == 2] pres_s = [v for y, v in ovs.items() if int(y) % 4 == 0] A.results["over_report_survey_base"] = {"midterm_lo": min(mid_s), "midterm_hi": max(mid_s), "midterm_mean": r(float(np.mean(mid_s)), 1), "presidential_lo": min(pres_s), "presidential_hi": max(pres_s), "presidential_mean": r(float(np.mean(pres_s)), 1)} gapb = {y: r(100 * (v["survey_adult_citizens"] / v["census_adult_citizens"] - 1), 1) for y, v in yearly.items()} A.results["survey_base_vs_census_pct"] = gapb assert min(gapb.values()) > -11 and max(gapb.values()) < 4, gapb # 2020's base is the Census Bureau's experimental release: the presidential average without it pres_x = [v for y, v in ov.items() if int(y) % 4 == 0 and y != "2020"] A.results["over_report"]["presidential_mean_without_2020"] = r(float(np.mean(pres_x)), 1) assert round(A.results["over_report"]["presidential_mean_without_2020"]) == round(A.results["over_report"]["presidential_mean"]), \ "the page says about the same with or without 2020" # ---- the eras: each split, its two groups and the gap eras = {} for key, (label, ys) in ERAS.items(): d = city_records(ys) des = cps_voting.design(d) blk = {"label": label, "years": ys, "all": voted_share(d, des, np.ones(len(d), bool)), "registered": voted_share(d, des, np.ones(len(d), bool), "registered1")} for k, lab, lo, hi in AGES: m = d["age"].between(lo, hi).to_numpy() blk[f"age_{k}"] = voted_share(d, des, m) blk[f"age_{k}_registered"] = voted_share(d, des, m, "registered1") # turnout among those who said they registered reg = m & (d["registered1"] == 1).to_numpy() blk[f"age_{k}_voted_if_registered"] = voted_share(d, des, reg) # each age group's share of adult citizens and of reported voters w = d["weight"].to_numpy() blk[f"age_{k}_share_of_citizens"] = r(100 * w[m].sum() / w.sum(), 1) v = (d["voted1"] == 1).to_numpy() blk[f"age_{k}_share_of_voters"] = r(100 * w[m & v].sum() / w[v].sum(), 1) for split, groups in SPLITS.items(): (ka, la, fa), (kb, lb, fb) = groups ma, mb = fa(d).to_numpy(), fb(d).to_numpy() blk[split] = {ka: voted_share(d, des, ma), kb: voted_share(d, des, mb), "gap": gap_between(d, des, ma, mb)} # the gap between every two age groups (the second's rate minus the first's), from the same households blk["age_pairs"] = {} for i, (ka, _, loa, hia) in enumerate(AGES): for kb, _, lob, hib in AGES[i + 1:]: blk["age_pairs"][f"{ka}__{kb}"] = gap_between(d, des, d["age"].between(loa, hia), d["age"].between(lob, hib)) eras[key] = blk A.results["eras"] = eras # ---- the drop from a presidential year to a midterm, by group, in the recent eras (pooled) M, Pr = eras["mid_recent"], eras["pres_recent"] drop = {} for k, lab, *_ in AGES: drop[k] = r(Pr[f"age_{k}"]["share"] - M[f"age_{k}"]["share"], 1) A.results["drop_by_age"] = drop # does one age group's drop exceed another's? drop_a - drop_b is the midterm gap (b minus a) less the # presidential one; the two pools are different elections (independent samples) dc = {} for pair in M["age_pairs"]: gm_, gp_ = M["age_pairs"][pair], Pr["age_pairs"][pair] mm_ = moe_difference((gm_["hi"] - gm_["lo"]) / 2, (gp_["hi"] - gp_["lo"]) / 2) ch = gm_["pts"] - gp_["pts"] dc[pair] = {"pts": r(ch, 1), "lo": r(ch - mm_, 1), "hi": r(ch + mm_, 1)} A.results["drop_contrasts"] = dc # the page: both groups under 45 drop by more than both groups over 45; the two youngest are not ranked for pair in ("18_29__45_64", "18_29__65_plus", "30_44__45_64", "30_44__65_plus"): A.claim(f"drop_contrasts.{pair}", "up") assert dc["18_29__30_44"]["lo"] < 0 < dc["18_29__30_44"]["hi"], dc["18_29__30_44"] gaps = {s: {"midterm": M[s]["gap"], "presidential": Pr[s]["gap"]} for s in SPLITS} A.results["gaps"] = gaps # is the age gap wider in midterms? the two gaps come from different elections (independent samples) gm, gp = M["age"]["gap"], Pr["age"]["gap"] mm = moe_difference((gm["hi"] - gm["lo"]) / 2, (gp["hi"] - gp["lo"]) / 2) widen = gm["pts"] - gp["pts"] A.results["age_gap_wider_in_midterms"] = {"pts": r(widen, 1), "lo": r(widen - mm, 1), "hi": r(widen + mm, 1), "ratio": r(gm["pts"] / gp["pts"], 1)} # each gap, recent era against early era, for midterms and presidential years (independent samples) def era_change(split, a, b): ga, gb = eras[a][split]["gap"], eras[b][split]["gap"] mm_ = moe_difference((ga["hi"] - ga["lo"]) / 2, (gb["hi"] - gb["lo"]) / 2) ch = gb["pts"] - ga["pts"] return {"pts": r(ch, 1), "lo": r(ch - mm_, 1), "hi": r(ch + mm_, 1)} A.results["since_early"] = {f"{split}_{kind}": era_change(split, f"{kind}_early", f"{kind}_recent") for split in SPLITS for kind in ("mid", "pres")} # in midterms, does age divide more than schooling? (the same elections: a paired difference) d_m = city_records(ERAS["mid_recent"][1]) des_m = cps_voting.design(d_m) w_, y_ = d_m["weight"].to_numpy(), d_m["voted1"].to_numpy() one = np.ones(len(d_m)) parts = {} for split in SPLITS: (ka, la, fa), (kb, lb, fb) = SPLITS[split] ra, za = cps.ratio(y_, one, w_, fa(d_m).to_numpy()) rb, zb = cps.ratio(y_, one, w_, fb(d_m).to_numpy()) parts[split] = (rb - ra, zb - za) for other in ("school", "born", "tenure"): dv = 100 * (parts["age"][0] - parts[other][0]) mv = 100 * des_m.moe(parts["age"][1] - parts[other][1]) A.results[f"age_minus_{other}_midterm"] = {"pts": r(dv, 1), "lo": r(dv - mv, 1), "hi": r(dv + mv, 1)} A.claim("age_minus_born_midterm", "up") A.claim("age_minus_tenure_midterm", "up") A.claim("age_gap_wider_in_midterms", "up") A.claim("since_early.age_mid", "down") # the age divide is cut at its extremes (under 30 against 65 and over) and schooling once (a degree against # none): the same contrasts with both cut the same way, so the page can say what its ranking depends on def paired(fa_, fb_, fc_, fd_): """gap(b minus a) less gap(d minus c), the same elections, with its margin.""" ra, za = cps.ratio(y_, one, w_, fa_) rb, zb = cps.ratio(y_, one, w_, fb_) rc, zc = cps.ratio(y_, one, w_, fc_) rd, zd = cps.ratio(y_, one, w_, fd_) dv_ = 100 * ((rb - ra) - (rd - rc)) mv_ = 100 * des_m.moe((zb - za) - (zd - zc)) return {"pts": r(dv_, 1), "lo": r(dv_ - mv_, 1), "hi": r(dv_ + mv_, 1)} age_lo, age_hi = (d_m["age"] <= 29).to_numpy(), (d_m["age"] >= 65).to_numpy() age_30 = (d_m["age"] >= 30).to_numpy() deg, nodeg = (d_m["educ"] >= 43).to_numpy(), (d_m["educ"] < 43).to_numpy() nohs, grad = (d_m["educ"] < 39).to_numpy(), (d_m["educ"] >= 44).to_numpy() # no diploma; a graduate degree cut = {"age_binary_minus_school": paired(age_lo, age_30, nodeg, deg), "age_extremes_minus_school_extremes": paired(age_lo, age_hi, nohs, grad), "school_extremes_gap": gap_between(d_m, des_m, nohs, grad)} A.results["cut_check_midterm"] = cut # the page: cut the same way, about the same; schooling at its extremes, wider than age at its extremes assert cut["age_binary_minus_school"]["lo"] < 0 < cut["age_binary_minus_school"]["hi"], cut assert cut["age_extremes_minus_school_extremes"]["hi"] < 0, cut # the same cut applied to every other comparison the page makes with the age gap: the widening in midterms, # the narrowing since 1998-2006 (independent pools) and the contrasts with birthplace and renting (paired) def gap_binary(key): d_ = city_records(ERAS[key][1]) return gap_between(d_, cps_voting.design(d_), (d_["age"] <= 29).to_numpy(), (d_["age"] >= 30).to_numpy()) def ind(a, b): mm_ = moe_difference((a["hi"] - a["lo"]) / 2, (b["hi"] - b["lo"]) / 2) ch = b["pts"] - a["pts"] return {"pts": r(ch, 1), "lo": r(ch - mm_, 1), "hi": r(ch + mm_, 1)} gb = {k: gap_binary(k) for k in ERAS} cut["age_binary_gaps"] = gb cut["widening_age_binary"] = ind(gb["pres_recent"], gb["mid_recent"]) cut["narrowing_age_binary"] = ind(gb["mid_early"], gb["mid_recent"]) for other in ("born", "tenure"): (kc, lc, fc), (kd, ld, fd) = SPLITS[other] cut[f"age_binary_minus_{other}"] = paired(age_lo, age_30, fc(d_m).to_numpy(), fd(d_m).to_numpy()) # in presidential years schooling divides more than the oldest-youngest age gap, and the schooling gap is wider # in presidential years than in midterms (the page says so: registered) d_p = city_records(ERAS["pres_recent"][1]) des_p = cps_voting.design(d_p) wp, yp = d_p["weight"].to_numpy(), d_p["voted1"].to_numpy() onep = np.ones(len(d_p)) (ka, la, fa), (kb, lb, fb) = SPLITS["age"] (kc, lc, fc), (kd, ld, fd) = SPLITS["school"] ra, za = cps.ratio(yp, onep, wp, fa(d_p).to_numpy()); rb, zb = cps.ratio(yp, onep, wp, fb(d_p).to_numpy()) rc, zc = cps.ratio(yp, onep, wp, fc(d_p).to_numpy()); rd, zd = cps.ratio(yp, onep, wp, fd(d_p).to_numpy()) dv = 100 * ((rb - ra) - (rd - rc)) mv = 100 * des_p.moe((zb - za) - (zd - zc)) A.results["age_minus_school_presidential"] = {"pts": r(dv, 1), "lo": r(dv - mv, 1), "hi": r(dv + mv, 1)} A.claim("age_minus_school_presidential", "down") sg_m, sg_p = M["school"]["gap"], Pr["school"]["gap"] ms = moe_difference((sg_m["hi"] - sg_m["lo"]) / 2, (sg_p["hi"] - sg_p["lo"]) / 2) A.results["school_gap_pres_minus_mid"] = {"pts": r(sg_p["pts"] - sg_m["pts"], 1), "lo": r(sg_p["pts"] - sg_m["pts"] - ms, 1), "hi": r(sg_p["pts"] - sg_m["pts"] + ms, 1)} A.claim("school_gap_pres_minus_mid", "up") for other in ("born", "tenure"): (kc, lc, fc), (kd, ld, fd) = SPLITS[other] rc, zc = cps.ratio(yp, onep, wp, fc(d_p).to_numpy()); rd, zd = cps.ratio(yp, onep, wp, fd(d_p).to_numpy()) dv = 100 * ((rb - ra) - (rd - rc)) mv = 100 * des_p.moe((zb - za) - (zd - zc)) A.results[f"age_minus_{other}_presidential"] = {"pts": r(dv, 1), "lo": r(dv - mv, 1), "hi": r(dv + mv, 1)} # ---- the same age comparisons among people who answered the vote question (a check: the Bureau counts a # missing answer as not voting, and the young leave it blank more often) resp = {} for key, (label, ys) in ERAS.items(): d = city_records(ys) d = d[d["voted"].isin([1, 2])].reset_index(drop=True) des = cps_voting.design(d) (ka, la, fa), (kb, lb, fb) = SPLITS["age"] ma, mb = fa(d).to_numpy(), fb(d).to_numpy() resp[key] = {"young": voted_share(d, des, ma), "old": voted_share(d, des, mb), "gap": gap_between(d, des, ma, mb)} dfull = city_records(ys) noans = (~dfull["voted"].isin([1, 2])).to_numpy() w = dfull["weight"].to_numpy() for nm, (lo, hi) in (("young", (18, 29)), ("old", (65, 200))): m = dfull["age"].between(lo, hi).to_numpy() resp[key][f"no_answer_{nm}"] = r(100 * w[m & noans].sum() / w[m].sum(), 1) rg = {k: v["gap"] for k, v in resp.items()} def indep(a, b): mm_ = moe_difference((a["hi"] - a["lo"]) / 2, (b["hi"] - b["lo"]) / 2) ch = b["pts"] - a["pts"] return {"pts": r(ch, 1), "lo": r(ch - mm_, 1), "hi": r(ch + mm_, 1)} resp["wider_in_midterms"] = indep(rg["pres_recent"], rg["mid_recent"]) resp["since_early_mid"] = indep(rg["mid_early"], rg["mid_recent"]) A.results["respondents_only"] = resp # the page keeps a claim only if its direction holds among respondents too assert resp["wider_in_midterms"]["lo"] > 0, resp["wider_in_midterms"] assert resp["since_early_mid"]["hi"] < 0, resp["since_early_mid"] # ---- the young in each midterm, 1998-2022, with margins (one election each) young = {} for y in years: d = city_records([y]) des = cps_voting.design(d) young[str(y)] = {"young": voted_share(d, des, d["age"] <= 29), "old": voted_share(d, des, d["age"] >= 65), "gap": gap_between(d, des, (d["age"] <= 29).to_numpy(), (d["age"] >= 65).to_numpy())} A.results["young_by_year"] = young # the midterms whose own gap clears zero, and 2018's A.results["midterm_gaps_clear_zero"] = [y for y, v in young.items() if int(y) % 4 == 2 and v["gap"]["lo"] > 0] A.results["gap_2018"] = young["2018"]["gap"] # the pools again with every election weighted equally (the CPS's population controls change between 2002 and # 2004, so a pool of raw weights gives later elections more weight): the mean of the yearly gaps, margins by # quadrature over independent years def equal_weight(ys): g = [young[str(y)]["gap"] for y in ys] m = float(np.mean([x["pts"] for x in g])) e = float(np.sqrt(sum(((x["hi"] - x["lo"]) / 2) ** 2 for x in g))) / len(g) return {"pts": r(m, 1), "lo": r(m - e, 1), "hi": r(m + e, 1)} ew = {k: equal_weight(ys) for k, (_, ys) in ERAS.items()} A.results["equal_weight"] = {"gaps": ew, "widening": ind(ew["pres_recent"], ew["mid_recent"]), "narrowing": ind(ew["mid_early"], ew["mid_recent"])} A.latest("2024-11", "CPS November 2024 Voting and Registration Supplement (the latest; the 2026 supplement is fielded " "in November 2026 and published in 2027)", why_older="The supplement is asked once every two years, in the week after a November election; no 2026 " "reading exists until the Bureau publishes the November 2026 file in 2027.") A.long_view(1996, 2024, "CPS November Voting and Registration Supplements, every election 1996-2024, the city's adult citizens", why_shorter="The supplement's vote question (PES1, asked of every adult citizen) has run in its present form " "since November 1996; the 1994 file asks it differently and has no PES1, so 1996-2024, fifteen " "elections over 29 years, is the longest series under one question.") # ---- the facts the page states about single elections and the files gov = t[t["kind"] == "governor"]["ballots"] A.results["fewest_governor_ballots_year"] = int(gov.idxmin()) assert int(gov.idxmin()) == 2014, "the page says 2014 drew the fewest ballots of any governor's race in the Board's table" mid_gaps = {y: v["old"]["share"] - v["young"]["share"] for y, v in young.items() if int(y) % 4 == 2} A.results["closest_midterm"] = min(mid_gaps, key=mid_gaps.get) assert A.results["closest_midterm"] == "2018", mid_gaps # the page: New York elects its governor in the midterm years (the Board's table, every midterm since 1990) assert all(t.loc[y, "kind"] == "governor" for y in range(1990, 2023, 4)), t.loc[1990:2022, "kind"] A.results["governor_in_every_midterm_since_1990"] = True # the page: 2018 was also the midterm with the most ballots per adult citizen since 1996, by the Board's count mid_counted = {y: v["counted_per_100"] for y, v in yearly.items() if int(y) % 4 == 2} A.results["fullest_midterm_by_count"] = max(mid_counted, key=mid_counted.get) assert A.results["fullest_midterm_by_count"] == "2018", mid_counted recs = [v["records"] for v in yearly.values()] A.results["records_per_year"] = {"lo": int(round(min(recs), -2)), "hi": int(round(max(recs), -2))} A.results["published_ny"] = {str(y): x["published"] for y, x in zip((2018, 2024), A.results["reproduction"])} A.claim("age_minus_school_midterm", "up") mg = {k: M[k]["gap"]["pts"] for k in SPLITS} assert max(mg, key=mg.get) == "age", mg tg_m, tg_p = M["tenure"]["gap"], Pr["tenure"]["gap"] mt = moe_difference((tg_m["hi"] - tg_m["lo"]) / 2, (tg_p["hi"] - tg_p["lo"]) / 2) A.results["tenure_gap_pres_minus_mid"] = {"pts": r(tg_p["pts"] - tg_m["pts"], 1), "lo": r(tg_p["pts"] - tg_m["pts"] - mt, 1), "hi": r(tg_p["pts"] - tg_m["pts"] + mt, 1)} assert A.results["tenure_gap_pres_minus_mid"]["lo"] < 0 < A.results["tenure_gap_pres_minus_mid"]["hi"], "the page says similar" Q = A.quotes() q = Q.quote(*DATES, "between October 24 and November 1, 2026") A.results["quotes"] = {"early_2026": {"quote": q["quote"], "page": q.get("page"), "source": q.get("source")}} for k, (src, fname, needle) in { "offices_2026": ("nyc_boe_annual_report_2025", "nyc_boe/BOE_Annual_Report_2025.pdf", "voters will elect members of the New York State Legislature, the Governor"), "acs_2005_gq": ("census_acs_comparing_2006", "census_acs_comparing_2006.html", "not included in the 2005 ACS"), "proxy_2024": ("cps_nov2024_voting_techdoc", "CPS_nov2024_voting_techdoc.pdf", "a single respondent could provide answers for themselves or provide answers for all eligible " "households members")}.items(): qq = Q.quote(src, fname, needle) A.results["quotes"][k] = {"quote": qq["quote"], "page": qq.get("page"), "source": qq.get("source")} # the margins against the Bureau's own approximation: its generalized-variance parameter for New York State # (b = 8,171 in the 2024 techdoc), SE of a percentage = sqrt(b / base * p * (100 - p)), base the weighted count gq = Q.quote("cps_nov2024_voting_techdoc", "CPS_nov2024_voting_techdoc.pdf", "8,171") b_ny = 8171.0 d24 = city_records([2024]) des24 = cps_voting.design(d24) gvf = {} for k, m in (("all", np.ones(len(d24), bool)), ("young", (d24["age"] <= 29).to_numpy()), ("old", (d24["age"] >= 65).to_numpy())): ours = voted_share(d24, des24, m) base = float(d24["weight"].to_numpy()[m].sum()) pshare = ours["share"] gvf[k] = {"share": pshare, "ours_moe": ours["moe"], "gvf_moe": r(1.645 * (b_ny / base * pshare * (100 - pshare)) ** 0.5, 2)} A.results["gvf_check_2024"] = gvf A.results["quotes"]["gvf_ny_2024"] = {"quote": gq["quote"], "page": gq.get("page"), "source": gq.get("source")} assert all(abs(v["ours_moe"] - v["gvf_moe"]) / v["gvf_moe"] < 0.35 for v in gvf.values()), gvf # ---- the margin headroom of every registered claim: the factor by which its margin could grow before the # range touches zero (the margins are clustered on the household only, so a reader can see which calls are close) def node_at(path): n = A.results for k in path.split("."): n = n[k] return n head = {} for path in ("age_gap_wider_in_midterms", "since_early.age_mid", "age_minus_school_midterm", "age_minus_born_midterm", "age_minus_tenure_midterm", "age_minus_school_presidential", "school_gap_pres_minus_mid", "drop_contrasts.18_29__45_64", "drop_contrasts.18_29__65_plus", "drop_contrasts.30_44__45_64", "drop_contrasts.30_44__65_plus"): nd = node_at(path) half = (nd["hi"] - nd["lo"]) / 2 head[path] = r(abs(nd["pts"]) / half, 2) A.results["claims_headroom"] = head closest = sorted(head, key=head.get)[:2] A.results["closest_calls"] = {"paths": closest, "wider_by_pct": r(100 * (head[closest[-1]] - 1), 0), "widening_pct": r(100 * (head["age_gap_wider_in_midterms"] - 1), 0), "narrowing_pct": r(100 * (head["since_early.age_mid"] - 1), 0)} assert closest == ["drop_contrasts.18_29__45_64", "age_minus_school_midterm"], closest # ---- printed figures (the page's strings) def pct(path): P(path, f"{get_(path):.0f}%") def pts(path): P(path, f"{abs(get_(path)):.0f}") def get_(path): node = A.results for k in path.split("."): node = node[k] return node for path in ("eras.pres_recent.all.share", "eras.mid_recent.all.share", "eras.mid_recent.age_18_29.share", "eras.mid_recent.age_65_plus.share", "eras.pres_recent.age_18_29.share", "eras.mid_recent.age_18_29_share_of_citizens", "eras.mid_recent.age_18_29_share_of_voters", "eras.mid_recent.age_18_29_registered.share", "eras.mid_recent.age_65_plus_registered.share", "eras.mid_recent.age_18_29_voted_if_registered.share", "eras.mid_recent.age_65_plus_voted_if_registered.share", "eras.mid_recent.school.degree.share", "eras.mid_recent.school.no_degree.share", "eras.pres_recent.school.degree.share", "eras.pres_recent.school.no_degree.share", "eras.mid_recent.born.naturalized.share", "eras.mid_recent.born.native.share", "eras.mid_recent.tenure.renter.share", "eras.mid_recent.tenure.owner.share", "eras.mid_early.age_18_29.share", "eras.mid_early.age_65_plus.share", "young_by_year.2018.young.share", "young_by_year.2018.old.share", "yearly.2022.reported", "respondents_only.mid_recent.no_answer_young", "respondents_only.mid_recent.no_answer_old", "respondents_only.mid_recent.young.share", "respondents_only.mid_recent.old.share"): pct(path) for path in ("eras.mid_recent.age.gap.pts", "eras.pres_recent.age.gap.pts", "eras.mid_early.age.gap.pts", "drop_by_age.18_29", "drop_by_age.65_plus", "yearly.2022.counted_per_100", "over_report.midterm_lo", "over_report.midterm_hi", "over_report.midterm_mean", "over_report.presidential_mean", "respondents_only.wider_in_midterms.pts", "respondents_only.since_early_mid.pts", "since_early.age_mid.pts"): pts(path) # the page says the respondents-only gap is about as wide as the Bureau's count's, and narrowed by about as much as it widens ro = A.results["respondents_only"] assert abs(ro["mid_recent"]["gap"]["pts"] - M["age"]["gap"]["pts"]) < 1.5 assert abs(abs(ro["since_early_mid"]["pts"]) - ro["wider_in_midterms"]["pts"]) < 1.5 P("published_ny.2018", f"{A.results['published_ny']['2018']:.1f}%") P("published_ny.2024", f"{A.results['published_ny']['2024']:.1f}%") P("gvf_check_2024.young.gvf_moe", f"{A.results['gvf_check_2024']['young']['gvf_moe']:.1f}") P("gvf_check_2024.young.ours_moe", f"{A.results['gvf_check_2024']['young']['ours_moe']:.1f}") for path in ("age_gap_wider_in_midterms.lo", "age_gap_wider_in_midterms.hi", "age_minus_school_midterm.lo", "age_minus_school_midterm.hi", "cut_check_midterm.age_binary_minus_school.lo", "cut_check_midterm.age_binary_minus_school.hi"): P(path, f"{get_(path):.1f}") for path in ("since_early.age_mid.lo", "since_early.age_mid.hi", "age_minus_school_presidential.lo", "age_minus_school_presidential.hi"): P(path, f"{abs(get_(path)):.1f}") pts("cut_check_midterm.school_extremes_gap.pts") for path in ("over_report_survey_base.midterm_lo", "over_report_survey_base.midterm_hi", "over_report_survey_base.presidential_mean"): pts(path) pct("young_by_year.2022.young.share") pct("young_by_year.2022.old.share") P("closest_calls.wider_by_pct", f"{A.results['closest_calls']['wider_by_pct']:.0f}%") P("closest_calls.widening_pct", f"{A.results['closest_calls']['widening_pct']:.0f}%") P("closest_calls.narrowing_pct", f"{A.results['closest_calls']['narrowing_pct']:.0f}%") pts("gap_2018.pts") for path in ("cut_check_midterm.widening_age_binary.lo", "cut_check_midterm.widening_age_binary.hi", "equal_weight.narrowing.lo", "equal_weight.narrowing.hi"): P(path, f"{abs(get_(path)):.1f}") P("records_per_year.lo", f"{A.results['records_per_year']['lo']:,}") P("records_per_year.hi", f"{A.results['records_per_year']['hi']:,}") # ---- charts A.chart("by_age", strip( title="In a governor's race, adult citizens under 45 drop off the most", subtitle="Adult citizens in New York City who said they voted, by age: the four midterms 2010-2022 and the five presidential elections 2008-2024, taken together", x=[["pres", "Presidential years", 1], ["mid", "Governor's races (midterms)", 0]], series=[[k, lab, lab] for k, lab, *_ in AGES], cells={"": {k: {"pres": Pr[f"age_{k}"]["share"], "pres_moe": Pr[f"age_{k}"]["moe"], "mid": M[f"age_{k}"]["share"], "mid_moe": M[f"age_{k}"]["moe"]} for k, *_ in AGES}}, format="pct", axis="Said they voted", order="series", value_x="mid", value_label="midterms", moe_label="90% margin", note="Reported votes, which run above the Board of Elections' count in every year; a missing answer counts as not " "voting, as the Census Bureau counts it. Margins are clustered on the household and approximate.", provenance="U.S. Census Bureau, CPS November Voting and Registration Supplements")) yrs = [str(y) for y in years] A.chart("young_and_old", lines( title="Under-30s said they voted less often than people 65 and over in every midterm but 2018, when the gap was smallest", subtitle="Adult citizens in New York City who said they voted, under 30 and 65 and over, every even-year November election 1996-2024", x=[[y, y + (" (midterm)" if int(y) % 4 == 2 else " (presidential)")] for y in yrs], series=[["young", "18 to 29", "Under 30"], ["old", "65 and over", "65 and over"]], cells={"": {k: {**{y: young[y][k]["share"] for y in yrs}, **{f"{y}_moe": young[y][k]["moe"] for y in yrs}} for k in ("young", "old")}}, format="pct", axis="Said they voted", labels="legend", x_ticks=[y for y in yrs if int(y) % 4 == 0], x_tick_labels={y: y for y in yrs if int(y) % 4 == 0}, x_axis="Year (even-year November elections)", moe_note="ticks are approximate 90% margins, clustered on the household", note=f"One election per point, so each carries a wide margin (about {min(v['young']['moe'] for v in young.values()):.0f} " f"to {max(v['young']['moe'] for v in young.values()):.0f} points either way for the young); midterms are the " "dips. Reported votes.", provenance="U.S. Census Bureau, CPS November Voting and Registration Supplements")) A.chart("gaps", strip( title="In a midterm the gap between the oldest and youngest adult citizens is the widest of four; in a presidential year, the gap between a degree and none is", subtitle="Gap in the share who said they voted between two groups of adult citizens, percentage points, New York City, midterms 2010-2022 and presidential years 2008-2024", x=[["presidential", "Presidential years", 1], ["midterm", "Midterms", 0]], series=[[s, SPLIT_LABEL[s], SPLIT_SHORT[s]] for s in SPLITS], cells={"": {s: {"presidential": gaps[s]["presidential"]["pts"], "midterm": gaps[s]["midterm"]["pts"]} for s in SPLITS}}, format="num", axis="Gap (points)", order="series", value_x="midterm", value_label="midterms", note="Each gap is the second group's rate minus the first's (65 and over minus under 30; a degree minus none; " "native-born minus naturalized; owners minus renters). The ranges of the comparisons are in the questions below.", provenance="U.S. Census Bureau, CPS November Voting and Registration Supplements")) A.chart("reported_and_counted", lines( title="The share who said they voted ran above the Board's ballots per 100 adult citizens in every election since 1996, on the census's count", subtitle="Adult citizens in New York City: the share who said they voted, and the Board of Elections' ballots per 100 adult citizens, 1996-2024", x=[[y, y] for y in yrs], series=[["reported", "Said they voted (survey)", "Said they voted"], ["counted", "Ballots counted per 100 adult citizens (Board)", "Ballots counted"]], cells={"": {"reported": {y: yearly[y]["reported"] for y in yrs}, "counted": {y: yearly[y]["counted_per_100"] for y in yrs}}}, format="pct", axis="Per 100 adult citizens", labels="legend", x_ticks=[y for y in yrs if int(y) % 4 == 0], note="The Board counts every ballot cast; adult citizens are the census's count (the 1990 and 2000 censuses and the " "yearly survey from 2006, the years between a straight line, assumed; 2005 left out, since that year's survey " "didn't count group quarters; 2020's is the experimental release); the division is this analysis's, not the " "Board's. The two rates have different bases: the survey's own count of the city's adult citizens is a sample " "estimate controlled to the state's population, ran up to a tenth below the census's before 2004, and leaves " "out people in institutions. So the space between the lines mixes reporting error with the survey's sampling " "error and the difference in bases; in 2004 and 2008 it is inside the survey's margin.", provenance="U.S. Census Bureau, CPS November supplements; Board of Elections in the City of New York")) rows = [] for y, v in yearly.items(): rows.append({"table": "yearly", "period": y, "group": "all adult citizens", "share": v["reported"], "moe": v["reported_moe"], "records": v["records"], "weighted_yes": v["weighted_yes"], "weighted_base": v["weighted_base"], "ballots": v["ballots"], "census_adult_citizens": r(float(ac.loc[int(y), "count"]), 0), "census_basis": v["denominator"], "counted_per_100": v["counted_per_100"]}) for k, lab in (("young", "18 to 29"), ("old", "65 and over")): e = young[y][k] rows.append({"table": "voted_by_age_yearly", "period": y, "group": lab, "share": e["share"], "moe": e["moe"], "records": e["records"], "weighted_yes": e["weighted_yes"], "weighted_base": e["weighted_base"], "ballots": None, "census_adult_citizens": None, "census_basis": None, "counted_per_100": None}) for key, blk in eras.items(): for k, lab, *_ in AGES: for table, suffix in (("voted_by_age", ""), ("registered_by_age", "_registered"), ("voted_if_registered_by_age", "_voted_if_registered")): e = blk[f"age_{k}{suffix}"] rows.append({"table": table, "period": blk["label"], "group": lab, "share": e["share"], "moe": e["moe"], "records": e["records"], "weighted_yes": e["weighted_yes"], "weighted_base": e["weighted_base"], "ballots": None, "census_adult_citizens": None, "census_basis": None, "counted_per_100": None}) for k, lab in (("young", "18 to 29"), ("old", "65 and over")): e = resp[key][k] rows.append({"table": "voted_by_age_answer_recorded", "period": blk["label"], "group": lab, "share": e["share"], "moe": e["moe"], "records": e["records"], "weighted_yes": e["weighted_yes"], "weighted_base": e["weighted_base"], "ballots": None, "census_adult_citizens": None, "census_basis": None, "counted_per_100": None}) for split, groups in SPLITS.items(): if split == "age": # the age groups are in voted_by_age already continue for g, lab, _ in groups: e = blk[split][g] rows.append({"table": f"voted_by_{split}", "period": blk["label"], "group": lab, "share": e["share"], "moe": e["moe"], "records": e["records"], "weighted_yes": e["weighted_yes"], "weighted_base": e["weighted_base"], "ballots": None, "census_adult_citizens": None, "census_basis": None, "counted_per_100": None}) data = pd.DataFrame(rows) assert not data.duplicated(["table", "period", "group"]).any(), "one row per table, period and group" A.write(data=data) print(f"{SLUG}: wrote {A.out}") return 0 if __name__ == "__main__": raise SystemExit(main())