"""How New York City votes since early voting began: the analysis behind a SparkyData article.

    py src/build_article.py how-new-york-votes

The question: until 2019 New York had no early voting, and until 2024 its only mail ballot was the absentee ballot,
which needs an excuse; how did the city's voters cast their ballots before, how do they now, who votes before
Election Day, and how is 2026's early voting going against the governor's race of 2022?

Datasets (cards in datasets/):
- ``cps_voting``: the Census Bureau's November Voting and Registration Supplements, 1996-2024, the city's
  adult citizens who said they voted, and how: in person on Election Day, in person before it, or by mail
  (one question to 2002, two from 2004; ``nycdata.cps_voting``, ``how``).
- ``nyc_boe_turnout``: the Board of Elections' records: its early-voting check-ins at the close of each day
  for every general election since 2020 (the Internet Archive's copies of its page) and 2026's so far
  (``elections.early_checkins``), its ballots by kind and borough for 2022-2025 (``borough_summaries``), and
  its ballots in every general election (``turnout``).
The survey gives the long view and who; the Board's counts give the shares of ballots and the boroughs, with no
sampling error. The two are set side by side, never mixed: the survey's shares are of people who said they
voted, the Board's of ballots counted.

Deterministic: no sampling.
"""

from __future__ import annotations

import numpy as np
import pandas as pd

from article_kit import Article, lines, r, strip
from nycdata import cps, cps_voting, elections
from sparkylib.stats import direction, moe_difference

SLUG = "how-new-york-votes"
SEED = 20261030
BOROUGHS = ["Bronx", "Brooklyn", "Manhattan", "Queens", "Staten Island"]
METHODS = [("election_day", "In person on Election Day"), ("early", "In person before Election Day"), ("mail", "By mail")]
BEFORE = [1996, 1998, 2000, 2002, 2004, 2006, 2008, 2010, 2012, 2014, 2016, 2018]     # no early voting in New York
AGES = [("18_29", "18 to 29", 18, 29), ("30_44", "30 to 44", 30, 44), ("45_64", "45 to 64", 45, 64), ("65_plus", "65 and over", 65, 200)]
KIND_LABEL = {"president": "presidential", "governor": "governor's", "mayor": "mayoral", "council": "Council"}
BOE19 = ("nyc_boe_annual_report_2019", "nyc_boe/BOE_Annual_Report_2019.pdf")
BOE20 = ("nyc_boe_annual_report_2020", "nyc_boe/BOE_Annual_Report_2020.pdf")
BOE24 = ("nyc_boe_annual_report_2024", "nyc_boe/BOE_Annual_Report_2024.pdf")
BOE25 = ("nyc_boe_annual_report_2025", "nyc_boe/BOE_Annual_Report_2025.pdf")
DATES = ("nyc_boe_important_dates_2026", "nyc_boe/all_important_dates_2026.html")
#: the saved copy of the Board's 2026 check-in page this version reads (sources.yaml: nyc_boe_early_voting_checkins_<date>)
CHECKINS_AS_OF = "2026-10-25"


# ------------------------------------------------------------------ the survey: how voters voted


def voters_in_city(years: list[int]) -> pd.DataFrame:
    """The city's adult citizens who said they voted, in the given November files (one frame)."""
    d = pd.concat([cps_voting.load(y) for y in years], ignore_index=True)
    return d[d["borough"].notna() & (d["voted"] == 1)].reset_index(drop=True)


def method_share(d: pd.DataFrame, des, mask=None) -> dict:
    """The weighted share of voters (in ``mask``) by how they voted, each with its 90% margin, and the share who
    voted before Election Day (early in person or by mail)."""
    m = np.ones(len(d), bool) if mask is None else np.asarray(mask, bool)
    one = np.ones(len(d))
    out = {"records": int(m.sum())}
    for k, _ in METHODS:
        v, e = cps.ratio_moe(d, y=(d["how"] == k).to_numpy().astype(float), x=one, weight="weight", mask=m, design=des)
        out[k] = {"share": r(100 * v, 2), "moe": r(100 * e, 2)}
    v, e = cps.ratio_moe(d, y=d["how"].isin(["early", "mail"]).to_numpy().astype(float), x=one, weight="weight", mask=m,
                         design=des)
    out["before"] = {"share": r(100 * v, 2), "moe": r(100 * e, 2)}
    v, e = cps.ratio_moe(d, y=(d["how"] == "no_answer").to_numpy().astype(float), x=one, weight="weight", mask=m, design=des)
    out["no_answer"] = {"share": r(100 * v, 2), "moe": r(100 * e, 2)}
    return out


def method_gap(d: pd.DataFrame, des, how: str, a, b) -> dict:
    """The share voting by ``how`` in group ``b`` minus group ``a``, the same households (a paired margin)."""
    w, y = d["weight"].to_numpy(), (d["how"] == how).to_numpy().astype(float)
    one = np.ones(len(d))
    ra, za = cps.ratio(y, one, w, np.asarray(a, bool))
    rb, zb = cps.ratio(y, one, w, np.asarray(b, bool))
    dv, mv = 100 * (rb - ra), 100 * des.moe(zb - za)
    return {"pts": r(dv, 2), "lo": r(dv - mv, 2), "hi": r(dv + mv, 2)}


def independent_change(a: dict, b: dict) -> dict:
    """b minus a for two shares from different elections (independent samples)."""
    ch = b["share"] - a["share"]
    mm = moe_difference(a["moe"], b["moe"])
    return {"pts": r(ch, 2), "lo": r(ch - mm, 2), "hi": r(ch + mm, 2)}


# ------------------------------------------------------------------ the analysis


def main(argv: list[str] | None = None) -> int:
    """Run the analysis and write every file (no options)."""
    A = Article(SLUG, __file__, seed=SEED)
    P = A.printed

    # ---- the survey's columns against each year's layout, and the Bureau's published table for New York
    lay = cps_voting.check_layouts()[0]
    assert lay.passed, lay.detail
    ck = elections.check_checkins(CHECKINS_AS_OF)[0]
    assert ck.passed, ck.detail
    cs = elections.check_summaries()
    assert all(c.passed for c in cs), [c.detail for c in cs if not c.passed]
    A.results["checks"] = {"cps_voting_layouts": lay.detail, "boe_early_checkins": ck.detail,
                           "boe_summaries": "; ".join(c.detail for c in cs)}
    t14 = cps_voting.check_method(2024)[0]
    assert t14.passed, t14.detail
    A.results["checks"]["cps_voting_method_table14"] = t14.detail
    A.reproduce("United States, voters who said they voted by mail, November 2024 (%; Table 14)", published=float(t14.target),
                published_moe90=None, reproduced=float(t14.value), kind="share", tolerance=0.05)
    for y in (2018, 2024):
        res = cps_voting.check_published(y)[0]
        assert res.passed, res.detail
        A.reproduce(f"New York State, adult citizens who said they voted, November {y} (%)", published=float(res.target),
                    published_moe90=None, reproduced=r(cps_voting.reported_rate(cps_voting.load(y)), 1), kind="share",
                    tolerance=0.05)

    # ---- the survey, every election: how the city's voters said they voted
    years = cps_voting.years()
    assert years[0] == 1996 and years[-1] == 2024, years
    by_year = {}
    for y in years:
        d = voters_in_city([y])
        by_year[str(y)] = method_share(d, cps_voting.design(d))
    A.results["survey_by_year"] = by_year
    d_pre = voters_in_city(BEFORE)
    pre = method_share(d_pre, cps_voting.design(d_pre))
    A.results["survey_before_2019"] = pre
    # the page: from 1996 to 2018 at least 94 in 100 voters said they voted in person on Election Day, every year
    ed_pre = [by_year[str(y)]["election_day"]["share"] for y in BEFORE]
    A.results["survey_before_2019"]["election_day_min"] = min(ed_pre)
    A.results["survey_before_2019"]["election_day_max"] = max(ed_pre)
    assert min(ed_pre) >= 94, ed_pre
    # the change from the years before early voting to each election since
    A.results["survey_change"] = {str(y): independent_change(pre["election_day"], by_year[str(y)]["election_day"])
                                  for y in (2020, 2022, 2024)}
    for y in (2020, 2022, 2024):
        A.claim(f"survey_change.{y}", "down")
    # the "However": Election Day's share rose from the pandemic year to the 2022 governor's race and fell again in
    # 2024 (elections two years apart share no households: the survey's 4-8-4 rotation)
    A.results["survey_swing"] = {
        "2020_2022": independent_change(by_year["2020"]["election_day"], by_year["2022"]["election_day"]),
        "2022_2024": independent_change(by_year["2022"]["election_day"], by_year["2024"]["election_day"])}
    A.claim("survey_swing.2020_2022", "up")
    A.claim("survey_swing.2022_2024", "down")

    # ---- who votes by mail, by age, in each election since early voting began. Each year on its own: a pool of a
    # midterm and a presidential year carries whichever year a group's voters lean to (the methodologist's review of
    # 24 October: the youngest voters' mail share was about the middle-aged's in 2022, well above it in 2020 and 2024)
    ages, age_gaps = {}, {}
    for y in (2020, 2022, 2024):
        d = voters_in_city([y])
        des = cps_voting.design(d)
        ages[str(y)] = {k: method_share(d, des, d["age"].between(lo, hi)) for k, _, lo, hi in AGES}
        young, middle, old = (d["age"] <= 29).to_numpy(), d["age"].between(45, 64).to_numpy(), (d["age"] >= 65).to_numpy()
        age_gaps[str(y)] = {"mail_old_minus_middle": method_gap(d, des, "mail", middle, old),
                            "mail_young_minus_middle": method_gap(d, des, "mail", middle, young)}
    A.results["survey_age"] = ages
    A.results["survey_age_gaps"] = age_gaps
    # the household-clustered margins leave out the survey's geographic clustering; against the Bureau's own
    # approximation for New York State (its generalized-variance parameter b = 8,171 in the 2024 techdoc: SE of a
    # percentage = sqrt(b / base * p * (100 - p)), base the weighted count) the 2024 mail shares by age come out within
    # a third, as the sibling article's turnout shares did
    Qg = A.quotes()
    gq = Qg.quote("cps_nov2024_voting_techdoc", "CPS_nov2024_voting_techdoc.pdf", "8,171")
    b_ny = 8171.0
    d24 = voters_in_city([2024])
    gvf = {}
    for k, _, lo, hi in AGES:
        m = d24["age"].between(lo, hi).to_numpy()
        base = float(d24["weight"].to_numpy()[m].sum())
        pshare = ages["2024"][k]["mail"]["share"]
        gvf[k] = {"share": pshare, "ours_moe": ages["2024"][k]["mail"]["moe"],
                  "gvf_moe": r(1.645 * (b_ny / base * pshare * (100 - pshare)) ** 0.5, 2)}
    A.results["gvf_check_2024"] = gvf
    assert all(abs(v["ours_moe"] - v["gvf_moe"]) / v["gvf_moe"] < 0.35 for v in gvf.values()), gvf
    for y in ("2020", "2022", "2024"):
        A.claim(f"survey_age_gaps.{y}.mail_old_minus_middle", "up")
    for y in ("2020", "2024"):
        A.claim(f"survey_age_gaps.{y}.mail_young_minus_middle", "up")
    g22 = age_gaps["2022"]["mail_young_minus_middle"]
    assert direction(g22["lo"], g22["hi"]) == "unclear", "the page says 2022's youngest and middle-aged were about the same"

    # ---- the Board: early-voting check-ins at the close of each day, every general election since 2020
    k = elections.early_checkins(CHECKINS_AS_OF)
    city = k[k["area"] == "New York City"]
    by_day = {str(y): {int(dd): int(v) for dd, v in g.set_index("day")["checkins"].items()} for y, g in city.groupby("year")}
    A.results["checkins_by_day"] = {y: {str(dd): v for dd, v in days.items()} for y, days in by_day.items()}
    t = elections.turnout().set_index("year")
    past = [y for y in elections.CHECKIN_KIND if y < 2026]
    assert all(sorted(by_day[str(y)]) == list(range(1, 10)) for y in past), "every past election has nine days"
    A.results["early_share_of_ballots"] = {
        str(y): {"checkins": by_day[str(y)][9], "ballots": int(t.loc[y, "ballots"]), "kind": elections.CHECKIN_KIND[y],
                 "share": r(100 * by_day[str(y)][9] / t.loc[y, "ballots"], 2)} for y in past}
    # the last weekend's share of the nine days (days 8 and 9, the Saturday and Sunday before Election Day)
    A.results["last_weekend_share"] = {str(y): r(100 * (by_day[str(y)][9] - by_day[str(y)][7]) / by_day[str(y)][9], 2)
                                       for y in past}
    # 2026 so far, against the same day of each earlier election
    assert "2026" in by_day, "2026's check-ins are not saved yet"
    n26 = max(by_day["2026"])
    now = by_day["2026"][n26]
    g26 = city[city["year"] == 2026].set_index("day")
    A.results["y2026"] = {"days": n26, "date": str(g26.loc[n26, "date"]), "checkins": now,
                          "same_day": {str(y): by_day[str(y)][n26] for y in past},
                          "vs_2022_pct": r(100 * (now / by_day["2022"][n26] - 1), 1),
                          "vs_2025_pct": r(100 * (now / by_day["2025"][n26] - 1), 1),
                          "boroughs": {b: int(v) for b, v in k[(k["year"] == 2026) & (k["day"] == n26) & (k["area"] != "New York City")]
                                       .set_index("area")["checkins"].items()}}
    A.latest(str(g26.loc[n26, "date"]), f"The Board of Elections' early-voting check-ins for the 3 November 2026 general "
             f"election, unofficial, at the close of day {n26}")
    A.long_view(1996, 2024, "CPS November Voting and Registration Supplements, every even-year election 1996-2024: how the "
                "city's voters said they voted",
                why_shorter="The supplement has asked how people voted since November 1996, the first file read here; the "
                            "1994 file asks it differently, so 1996-2024, fifteen elections over 29 years, is the longest "
                            "series under the same questions.")

    # ---- the Board's ballots by kind and borough, 2022-2025, and the check-ins against its early ballots
    bs = elections.borough_summaries()
    kinds, borough_kinds, recon = {}, {}, {}
    for y, g in bs.groupby("year"):
        tot = g[["ballots", "early", "election_day", "mail"]].sum()
        kinds[str(y)] = {kk: r(100 * tot[kk] / tot["ballots"], 2) for kk in ("early", "election_day", "mail")}
        kinds[str(y)]["before"] = r(100 * (tot["early"] + tot["mail"]) / tot["ballots"], 2)
        kinds[str(y)]["ballots"] = int(tot["ballots"])
        borough_kinds[str(y)] = {b: {kk: r(100 * row[kk] / row["ballots"], 2) for kk in ("early", "election_day", "mail")}
                                 | {"before": r(100 * (row["early"] + row["mail"]) / row["ballots"], 2)}
                                 for b, row in g.set_index("borough").iterrows()}
        if str(y) in by_day:
            recon[str(y)] = {"checkins": by_day[str(y)][9], "early_ballots": int(tot["early"]),
                             "diff_pct": r(100 * (by_day[str(y)][9] / tot["early"] - 1), 2)}
            A.reproduce(f"Early ballots, New York City, November {y} (check-ins at the close of day 9 against the "
                        f"Board's count)", published=int(tot["early"]), published_moe90=None,
                        reproduced=by_day[str(y)][9], kind="count", tolerance=0.005 * int(tot["early"]))
    A.results["board_kinds"] = kinds
    # the survey's shares beside the Board's counts, where both exist (2022 and 2024): people say they voted by mail
    # more often than the Board counted mail ballots
    A.results["survey_vs_board"] = {y: {kk: {"survey": by_year[y][kk]["share"], "survey_moe": by_year[y][kk]["moe"],
                                             "board": kinds[y][kk]} for kk in ("election_day", "early", "mail")}
                                    for y in ("2022", "2024")}
    assert all(by_year[y]["mail"]["share"] - by_year[y]["mail"]["moe"] > kinds[y]["mail"] for y in ("2022", "2024")), \
        "the page says the survey's mail share runs above the Board's count beyond its margin"
    A.results["board_boroughs"] = borough_kinds
    A.results["checkins_vs_early_ballots"] = recon
    A.results["reproduction_source"] = ("U.S. Census Bureau, Table 4b (P20-583 and P20-590), New York State, and Table 14 "
                                        "(P20-590), the nation's voters by method; Board of Elections in the City of New "
                                        "York, annual reports 2022-2025")
    A.results["reproduction_note"] = ("The Bureau's published percentage voted, reproduced from the public-use file to one "
                                      "decimal; the Board's early ballots reproduced from its own check-ins within half a "
                                      "per cent (check-ins are unofficial and counted at the poll site).")
    assert all(max(("early", "election_day", "mail"), key=lambda kk: kinds[y][kk]) == "election_day" and
               min(("early", "election_day", "mail"), key=lambda kk: kinds[y][kk]) == "mail" for y in kinds), kinds
    assert [y for y in kinds if kinds[y]["before"] > kinds[y]["election_day"]] == ["2024"], kinds
    es = A.results["early_share_of_ballots"]
    assert es["2025"]["share"] > 2 * es["2021"]["share"], "the page says early voting's share more than doubled"
    b25 = borough_kinds["2025"]
    A.results["borough_order_2025"] = sorted(b25, key=lambda b: b25[b]["before"], reverse=True)
    assert A.results["borough_order_2025"][-1] == "Bronx", A.results["borough_order_2025"]
    # the page: the Bronx cast the smallest share before Election Day in every citywide race the summaries give, and
    # Manhattan the largest (in 2023, a Council year with no citywide race, Brooklyn's was smaller)
    citywide = ("2022", "2024", "2025")
    assert all(min(borough_kinds[y], key=lambda b: borough_kinds[y][b]["before"]) == "Bronx" for y in citywide), borough_kinds
    assert all(max(borough_kinds[y], key=lambda b: borough_kinds[y][b]["before"]) == "Manhattan" for y in citywide), borough_kinds

    # ---- the documents
    Q = A.quotes()
    quotes = {}
    for key, (src, needle) in {
            "law_2019": (BOE19, "in January, early voting was passed by the state legislature for the November 5th general "
                                "election; allowing voters the opportunity to cast their ballot up to nine days prior to the "
                                "regular Election Day, including two weekends"),
            "covid_excuse_2020": (BOE20, "allowed fear of COVID as an excuse to request an absentee ballot"),
            "thirds_2020": (BOE20, "one-third of the voters returning absentee ballots, one-third voting during the 9-day "
                                   "Early Voting period, and one-third voting on November 3rd, Election Day"),
            "early_mail_2024": (BOE24, "New Yorkers can now vote via early mail ballot"),
            "early_days_2025": (BOE25, "General Election – November 4, 2025 Early Voting Days: October 25 – November 2"),
            "dropoff_2026": (DATES, "Drop off your ballot to an early voting poll site in your county between October 24 "
                                    "and November 1, 2026"),
            "postmark_2026": (DATES, "Put your ballot in the mail ensuring it receives a postmark no later than November 3, "
                                     "2026"),
            "eday_dropoff_2026": (DATES, "Drop off your ballot to an election day poll site on November 3, 2026 no later "
                                         "than 9pm")}.items():
        q = Q.quote(src[0], src[1], needle)
        quotes[key] = {"quote": q["quote"], "page": q.get("page"), "source": q.get("source")}
    quotes["gvf_ny_2024"] = {"quote": gq["quote"], "page": gq.get("page"), "source": gq.get("source")}
    A.results["quotes"] = quotes

    # ---- printed figures
    def pct(path, nd=0):
        P(path, f"{get(path):.{nd}f}%")

    def get(path):
        node = A.results
        for kk in path.split("."):
            node = node[kk]
        return node

    for path in ("survey_before_2019.election_day.share", "survey_before_2019.mail.share",
                 "survey_by_year.2020.election_day.share", "survey_by_year.2022.election_day.share",
                 "survey_by_year.2024.election_day.share", "survey_by_year.2024.early.share", "survey_by_year.2024.mail.share",
                 "board_kinds.2025.election_day", "board_kinds.2024.election_day",
                 "board_kinds.2025.early", "board_kinds.2025.mail", "board_kinds.2022.early",
                 "early_share_of_ballots.2020.share", "early_share_of_ballots.2022.share", "early_share_of_ballots.2024.share",
                 "board_boroughs.2025.Bronx.before", "board_boroughs.2025.Manhattan.before",
                 "survey_age.2024.18_29.mail.share", "survey_age.2024.45_64.mail.share", "survey_age.2024.65_plus.mail.share",
                 "last_weekend_share.2022", "last_weekend_share.2025", "board_kinds.2024.before",
                 "survey_vs_board.2022.mail.survey", "survey_vs_board.2022.mail.board", "survey_vs_board.2024.mail.survey",
                 "survey_vs_board.2024.mail.board", "early_share_of_ballots.2021.share", "early_share_of_ballots.2025.share"):
        pct(path)
    for path in ("survey_change.2022.lo", "survey_change.2022.hi", "survey_change.2024.lo", "survey_change.2024.hi"):
        P(path, f"{abs(get(path)):.1f}")
    pct("survey_by_year.2018.election_day.share")
    P("survey_by_year.2024.election_day.moe", f"{get('survey_by_year.2024.election_day.moe'):.0f}")
    for path in ("survey_by_year.2022.no_answer.share", "survey_by_year.2024.no_answer.share"):
        pct(path)
    worst = max(abs(v["diff_pct"]) for v in recon.values())
    A.results["checkins_vs_early_ballots_max_diff_pct"] = r(worst, 2)
    assert worst < 0.2, recon
    P("checkins_vs_early_ballots_max_diff_pct", f"{worst:.1f}%")
    P("survey_before_2019.election_day_min", f"{A.results['survey_before_2019']['election_day_min']:.0f}%")
    P("y2026.checkins", f"{A.results['y2026']['checkins']:,}")
    P("y2026.same_day.2022", f"{A.results['y2026']['same_day']['2022']:,}")
    pct("survey_vs_board.2024.early.survey")
    pct("survey_vs_board.2024.early.board")
    P("survey_age.2022.18_29.records", f"{A.results['survey_age']['2022']['18_29']['records']}")
    P("checkins_by_day.2022.9", f"{by_day['2022'][9]:,}")
    recs = [v["records"] for v in by_year.values()]
    A.results["survey_records_min"], A.results["survey_records_max"] = min(recs), max(recs)
    P("survey_records_min", f"{min(recs):,}")
    P("survey_records_max", f"{max(recs):,}")

    # ---- charts
    yrs = [str(y) for y in years]
    A.chart("how_since_1996", lines(
        title="Until 2019 nearly every New York City voter said they voted in person on Election Day; since 2020, between four and six in ten",
        subtitle="Adult citizens in New York City who said they voted, by how they voted, every even-year November election 1996-2024",
        x=[[y, y] for y in yrs],
        series=[[kk, lab, lab] for kk, lab in METHODS],
        cells={"": {kk: {**{y: by_year[y][kk]["share"] for y in yrs}, **{f"{y}_moe": by_year[y][kk]["moe"] for y in yrs}}
                    for kk, _ in METHODS}},
        format="pct", axis="Share of voters", labels="legend",
        x_ticks=[y for y in yrs if int(y) % 4 == 0], x_axis="Year (even-year November elections)",
        moe_note="ticks are approximate 90% margins, clustered on the household",
        note="What voters told the Census Bureau after each election; a voter who did not say how counts among the voters "
             "and in none of the three lines. "
             "New York had no early voting before 2019; an absentee ballot needs an excuse, and in 2020 fear of COVID "
             "counted as one; early mail ballots began in 2024. One question to 2002, two from 2004 (in person or by mail; "
             "on Election Day or before).",
        provenance="U.S. Census Bureau, CPS November Voting and Registration Supplements"))
    days = [str(dd) for dd in range(1, 10)]
    show = [y for y in sorted(by_day) if y != "2023"]
    A.chart("checkins_by_day", lines(
        title=f"After {n26} day{'s' if n26 > 1 else ''} of early voting, {now:,} New Yorkers had checked in",
        subtitle="Early-voting check-ins at the close of each day, running total, New York City, every general election with a governor, president or mayor at the top of the ballot, 2020-2025, and 2026 so far, thousands",
        x=[[dd, f"Day {dd}"] for dd in days],
        series=[[y, f"{y} ({KIND_LABEL[elections.CHECKIN_KIND[int(y)]]})", y] for y in show],
        cells={"": {y: {dd: r(by_day[y][int(dd)] / 1000, 1) for dd in days if int(dd) in by_day[y]} for y in show}},
        format="k", axis="Check-ins (thousands)", labels="legend", x_axis="Day of early voting",
        note="The Board of Elections' unofficial check-ins at the close of each day, as its page printed them; 2020 to 2025 "
             "from the Internet Archive's copies. Early voting runs nine days, ending the Sunday before Election Day. 2023, a "
             "Council year, is left out.",
        provenance="Board of Elections in the City of New York; Internet Archive"))
    es_years = [str(y) for y in past]
    A.chart("early_share", lines(
        title="Early voting's share of the ballots rises and falls with the race: over a third in the presidential years, a quarter of the 2022 governor's race, a third of 2025's mayoral race",
        subtitle="Early-voting check-ins over the nine days as a share of every ballot cast, New York City, every general election 2020-2025",
        x=[[y, f"{y} ({KIND_LABEL[elections.CHECKIN_KIND[int(y)]]})"] for y in es_years],
        series=[["early", "Early check-ins, share of ballots", "Early"]],
        cells={"": {"early": {y: A.results["early_share_of_ballots"][y]["share"] for y in es_years}}},
        format="pct", axis="Share of ballots cast", x_axis="General election (the race at the top of the ballot)",
        x_tick_labels={y: y for y in es_years},
        note="The Board of Elections' unofficial check-ins at the close of the ninth day over the ballots it counted in the "
             "election (its turnout records); the check-ins come within 0.2% of its early ballots in 2022-2025. 2021 and 2025 "
             "are mayoral races, 2023 a Council year, 2022 a governor's race, 2020 and 2024 presidential. Counts, not a sample.",
        provenance="Board of Elections in the City of New York; Internet Archive"))
    A.chart("boroughs", strip(
        title="The Bronx votes early or by mail least and Manhattan most, in each citywide race",
        subtitle="Ballots cast early in person or by mail, share of each borough's ballots, 2022 governor's race, 2024 presidential race, 2025 mayoral race",
        x=[["2022", "2022, governor", 2], ["2024", "2024, president", 1], ["2025", "2025, mayor", 0]],
        series=[[b, b, b] for b in BOROUGHS],
        cells={"": {b: {y: borough_kinds[y][b]["before"] for y in ("2022", "2024", "2025")} for b in BOROUGHS}},
        format="pct", axis="Share of ballots", value_x="2025", value_label="2025",
        note="The Board of Elections' yearly turnout summaries: early ballots and mail ballots (the absentee and military, "
             "federal and federal write-in columns) over every ballot counted in the borough. Counts, not a sample: no margin.",
        provenance="Board of Elections in the City of New York, annual reports 2022-2025"))
    ay = ("2020", "2022", "2024")
    A.chart("by_age", strip(
        title="Voters 65 and over report voting by mail more than those 45 to 64, in each election surveyed since 2020",
        subtitle="Adult citizens in New York City who said they voted by mail, by age, 2020, 2022 and 2024 elections",
        x=[["2020", "2020, president", 2], ["2022", "2022, governor", 1], ["2024", "2024, president", 0]],
        series=[[kk, lab, lab] for kk, lab, *_ in AGES],
        cells={"": {kk: {**{y: ages[y][kk]["mail"]["share"] for y in ay}, **{f"{y}_moe": ages[y][kk]["mail"]["moe"] for y in ay}}
                    for kk, *_ in AGES}},
        format="pct", axis="Share of voters who voted by mail", order="series", value_x="2024", value_label="2024",
        moe_label="90% margin",
        note="What voters told the Census Bureau; it reports more mail voting than the Board of Elections counts. Margins "
             "clustered on the household, approximate.",
        provenance="U.S. Census Bureau, CPS November Voting and Registration Supplements"))

    # ---- data.csv
    rows = []
    for y, v in by_year.items():
        for kk, lab in METHODS:
            rows.append({"table": "survey_how_voted", "period": y, "area": "New York City", "group": lab,
                         "value": v[kk]["share"], "unit": "percent of voters", "moe90": v[kk]["moe"], "records": v["records"]})
    for y, gs in ages.items():
        for (gk, glab, *_), v in ((a, gs[a[0]]) for a in AGES):
            for kk, lab in METHODS:
                rows.append({"table": "survey_how_voted_by_age", "period": y, "area": "New York City",
                             "group": f"{glab}: {lab}", "value": v[kk]["share"], "unit": "percent of voters",
                             "moe90": v[kk]["moe"], "records": v["records"]})
    for _, row in k.iterrows():
        rows.append({"table": "early_checkins", "period": f"{row['year']} day {row['day']} ({row['date']})", "area": row["area"],
                     "group": elections.CHECKIN_KIND[row["year"]], "value": row["checkins"], "unit": "check-ins, cumulative",
                     "moe90": None, "records": None})
    for _, row in bs.iterrows():
        for kk in ("early", "election_day", "mail", "ballots"):
            rows.append({"table": "board_ballots_by_kind", "period": str(row["year"]), "area": row["borough"], "group": kk,
                         "value": int(row[kk]), "unit": "ballots", "moe90": None, "records": None})
    data = pd.DataFrame(rows)
    assert not data.duplicated(["table", "period", "area", "group"]).any()
    A.write(data=data)
    print(f"{SLUG}: wrote {A.out}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())