"""Half of New York's workers are paid under $30 an hour: the analysis behind a SparkyData article. py src/build_article.py paid-under-30-an-hour The question: the Council's Int 0757-2026 would raise the city's minimum wage to $30 by 2030 while the State's stays at $17 in 2027. How many people who work in New York are paid less than $30 an hour now, who are they, and where does $30 sit against the city's pay over the last thirty-five years? Datasets (cards in datasets/): - ``census_history``: the 1990 and 2000 census samples and the ACS 2005-2014 one-year files, city residents; hourly pay is wage and salary income over usual weekly hours times weeks worked, the 1990 census asking about 1989 and the 2000 census about 1999. - ``acs1_pums``: the ACS one-year files 2015-2024 for the New York study area, already in 2024 dollars; residents, and jobs whose workplace is in the five boroughs. - ``cps_basic``: the CPS outgoing rotation groups, January to September 2026, the current year: usual weekly earnings over usual weekly hours, in nominal dollars. - ``bls_labour``: the NY-metro CPI-U, annual and monthly, to put every year in 2024 dollars. The minimum wage comes from the State's and the federal histories, each rate checked against the saved page; the bill's schedule from its text on Legistar; EPI's projection from its post. Nothing here estimates what a $30 minimum would do to jobs, hours or prices: the article counts who is paid less than $30 now. Deterministic: no sampling. """ from __future__ import annotations from datetime import date import numpy as np import pandas as pd from article_kit import Article, lines, r, strip from nycdata import acs1, bls, cps, history, pums from nycdata.acs1 import puma_vintage from nycdata.geo import POW_NYC from nycdata.recode import (EMPLOYEE_CLASSES, OCCUPATIONS, hourly_wage, industry_from_naics, INDUSTRIES, occupation_from_code) from sparkylib.stats import direction, moe_difference, weighted_median, woodruff_median SLUG = "paid-under-30-an-hour" SEED = 20261013 THIRTY = 30.0 LATEST_ACS = 2024 COMPARE_FROM = 2019 INFLATION_MAIN = 0.025 # assumed yearly NY-metro inflation after the last CPI month INFLATION_SWEEP = (0.020, 0.025, 0.030, 0.035) MONTHS = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"] # Saved pages (data/reference/reports), registered in sources.yaml. NYS_HISTORY = ("nys_dol_minimum_wage_history", "nys_dol_minimum_wage_history_2026-10-13.html") NYS_2027 = ("nys_dol_minimum_wage_2027", "nys_dol_minimum_wage_2026-10-13.html") FEDERAL = ("us_dol_federal_minimum_wage_history", "us_dol_federal_minimum_wage_history_2026-10-13.html") GOVERNOR = ("governor_minimum_wage_2027", "governor_minimum_wage_2027_2026-10-13.html") BILL = ("nyc_council_int_0757_2026", "nyc_council_int_0757_2026_2026-10-13.html") EPI = ("epi_thirty_by_2030_nyc", "epi_thirty_by_2030_nyc_2025-08-28.html") S3952 = ("nysenate_s3952_2025", "nysenate_s3952_2026-10-13.html") # The story's history (v0.9.3; style guide, section 1): the laws that raised the floor and the one that # paused it, each quoted from its own saved document, and the reading in the coverage it checks. LAW_2016 = ("governor_cuomo_15_minimum_wage_2016", "governor_cuomo_15_minimum_wage_2016-04-04.html") LAW_2023 = ("governor_minimum_wage_reminder_2023", "governor_minimum_wage_reminder_2023-12-27.html") LAB_652 = ("ny_labor_law_652", "ny_labor_law_652_2026-10-18.html") MEMO_2026 = ("governor_gpb44_minimum_wage_memo_2026", "governor_gpb44_minimum_wage_memo_2026-10.pdf") ABC7_2026 = ("abc7_council_30_minimum_wage_2026", "abc7_council_30_minimum_wage_2026-03-10.html") # The general minimum wage in New York City: the higher of the State's rate and the federal # one, each step with the words on the saved page that give it. From 31 December 2016 the # City's rate for employers of 11 or more (most jobs; 10 or fewer lagged by a year to 2019). SCHEDULE = [ (date(1981, 1, 1), 3.35, NYS_HISTORY, "Increased from $3.10 to $3.35"), (date(1990, 4, 1), 3.80, NYS_HISTORY, "Increased from $3.35 to $3.80"), (date(1991, 4, 1), 4.25, NYS_HISTORY, "Increased from $3.80 to $4.25"), (date(1996, 10, 1), 4.75, FEDERAL, "Oct 1, 1996 $4.75 for all covered, nonexempt workers"), (date(1997, 9, 1), 5.15, FEDERAL, "Sep 1, 1997 5 $5.15 for all covered, nonexempt workers"), (date(2000, 3, 31), 5.15, NYS_HISTORY, "March 31, 2000 Increased from $4.25 to $5.15"), (date(2005, 1, 1), 6.00, NYS_HISTORY, "Increased from $5.15 to $6.00"), (date(2006, 1, 1), 6.75, NYS_HISTORY, "Increased from $6.00 to $6.75"), (date(2007, 1, 1), 7.15, NYS_HISTORY, "Increased from $6.75 to $7.15"), (date(2009, 7, 24), 7.25, NYS_HISTORY, "Increased from $7.15 to $7.25"), (date(2013, 12, 31), 8.00, NYS_HISTORY, "Increased from $7.25 to $8.00"), (date(2014, 12, 31), 8.75, NYS_HISTORY, "Increased from $8.00 to $8.75"), (date(2015, 12, 31), 9.00, NYS_HISTORY, "Increased from $8.75 to $9.00"), (date(2016, 12, 31), 11.00, NYS_HISTORY, "Large employers of 11 or more employees: $11.00 per hour on and after December 31, 2016"), (date(2017, 12, 31), 13.00, NYS_HISTORY, "$13.00 per hour on and after December 31, 2017; $15.00 per hour on and after December 31, 2018. Small employers"), (date(2018, 12, 31), 15.00, NYS_HISTORY, "$15.00 per hour on and after December 31, 2018. Small employers"), (date(2024, 1, 1), 16.00, NYS_HISTORY, "All Employers: $16.00 per hour on and after January 1, 2024"), (date(2025, 1, 1), 16.50, NYS_HISTORY, "$16.50 per hour on and after January 1, 2025"), (date(2026, 1, 1), 17.00, NYS_HISTORY, "$17.00 per hour on and after January 1, 2026"), ] RATE_2027 = (17.00, NYS_2027, "On January 1, 2027, New York's minimum wage will stay the same. The minimum wage rates will be: $17.00 per hour in New York City") # Int 0757-2026, section 20-1703: employers of more than 500 employees nationwide (schedule 1) # and of 500 or fewer (schedule 2), from 1 January of each year. BILL_LARGE = {2027: (20.0, "On or after January 1, 2027, $20 an hour"), 2028: (23.0, "On or after January 1, 2028, $23 an hour"), 2029: (26.0, "On or after January 1, 2029, $26 an hour"), 2030: (30.0, "On or after January 1, 2030, $30 an hour")} BILL_SMALL = {2027: 19.0, 2028: 21.5, 2029: 24.0, 2030: 27.0, 2031: 29.0} # EPI's Table 2: the 2023 one-year ACS projected to 2030 at 3.3% nominal wage growth a year. EPI_GROWTH, EPI_YEARS = 0.033, 7 EPI_ROWS = {25: (1_358_700, 29.7, 0.64), 28: (1_579_900, 34.5, 0.71), 30: (1_680_900, 36.7, 0.76), 32: (1_868_200, 40.8, 0.82)} # Occupation names as the page prints them (the dictionary's are title case and long). SHORT_NAMES = {"Driver/Sales Workers And Truck Drivers": "Truck and delivery drivers", "Security Guards And Gambling Surveillance Officers": "Security guards"} ACS_COLS = ["AGEP", "SEX", "CIT", "ESR", "COW", "WAGP", "WKHP", "WKWN", "JWTRNS", "POWSP", "POWPUMA", "OCCP", "NAICSP"] CPS_VARS = ["HRHHID", "HRHHID2", "PULINENO", "HRYEAR4", "HRMONTH", "HRMIS", "GESTFIPS", "GTCO", "PRTAGE", "PEMLR", "PRERELG", "PEERNHRY", "PEERNHRO", "PTERNWA", "PEHRUSL1", "PWORWGT", "PRHERNAL", "PRWERNAL", "PEIO1COW"] # ------------------------------------------------------------------ the minimum wage def minimum_wage(A: Article) -> dict: """The minimum by calendar year (the average of the rates in force each day), checked step by step against the saved pages; the 2027 rate; the bill's schedule.""" Q = A.quotes() checks = [Q.quote(src, fname, words) for _, _, (src, fname), words in SCHEDULE] rate27, (src, fname), words = RATE_2027 checks.append(Q.quote(src, fname, words)) checks.append(Q.quote(*GOVERNOR, "the minimum wage would increase to: $17.60 per hour in New York City")) for y, (v, words) in BILL_LARGE.items(): checks.append(Q.quote(*BILL, words)) checks.append(Q.quote(*BILL, "employs more than 500 employees nationwide")) checks.append(Q.quote(*BILL, "(ii) the state of New York")) checks.append(Q.quote(*BILL, "(iii) the city of New York or any local government")) checks.append(Q.quote(*BILL, "On or after January 1, 2031, $29 an hour")) checks.append(Q.quote(*S3952, "authorizes the city of New York to establish a minimum wage that is higher than the statewide minimum wage")) checks.append(Q.quote(*S3952, "Jan 07, 2026 referred to labor")) for thr, (count, share, ratio) in EPI_ROWS.items(): checks.append(Q.quote(*EPI, f"${thr} by 2030 {count:,} {share}% {ratio:.2f}")) checks.append(Q.quote(*EPI, "Assumes annual nominal wage growth rate of 3.3%")) checks.append(Q.quote(*EPI, "Source: EPI analysis of 2023 1-year ACS data")) def rate_on(d: date) -> float: return [v for start, v, _, _ in SCHEDULE if start <= d][-1] nominal = {} for y in range(1989, 2027): days = pd.date_range(f"{y}-01-01", f"{y}-12-31", freq="D") nominal[y] = float(np.mean([rate_on(d.date()) for d in days])) nominal[2027] = rate27 return {"nominal": nominal, "checks": len(checks), "quotes": {f"q{i:02d}": q for i, q in enumerate(checks)}} def history_quotes(A: Article) -> dict: """The story's history and the readings it checks (v0.9.3; owner, 14 and 15 October 2026: context from primary documents, never as a cause): every sentence the page uses, found in its saved document. None of their figures enters a comparison with the files; the page prints them from ``results["history"]``, read from these quotes.""" Q = A.quotes() return { "law_2016_date": Q.quote(*LAW_2016, "April 4, 2016"), "law_2016_city": Q.quote(*LAW_2016, "For workers in New York City employed by large businesses (those with at least 11 " "employees), the minimum wage would rise to $11 at the end of 2016, then another $2 " "each year after, reaching $15 on 12/31/2018"), "law_2016_workers": Q.quote(*LAW_2016, "will lift the earnings of more than 2.3 million New Yorkers"), "law_2023_date": Q.quote(*LAW_2023, "December 27, 2023"), "law_2023_budget": Q.quote(*LAW_2023, "As part of the FY 2024 Budget, Governor Hochul secured an historic agreement to " "increase New York's minimum wage through 2026 and index it to inflation beginning in 2027"), "law_2023_sixteen": Q.quote(*LAW_2023, "Will Increase to $16-Per-Hour in New York City"), "law_2023_index": Q.quote(*LAW_2023, "In 2027, the minimum wage will increase annually at a rate determined by the Consumer " "Price Index for Urban Wage Earners and Clerical Workers (CPI-W) for the Northeast Region"), "law_2023_offramp": Q.quote(*LAW_2023, 'An "off-ramp" is available in the event of certain economic or budget conditions'), "lab652_no_increase": Q.quote(*LAB_652, "there shall be no increase in the minimum wage in the state for the following year " "if any of the following conditions are met"), "lab652_index": Q.quote(*LAB_652, "increasing the then current year's minimum wage rate by the rate of change in the " "average of the three most recent consecutive twelve-month periods between the first of " "August and the thirty-first of July, each over their preceding twelve-month periods " "published by the United States department of labor non-seasonally adjusted consumer " "price index for northeast region urban wage earners and clerical workers (CPI-W)"), "lab652_prices": Q.quote(*LAB_652, "(i) the rate of change in the average of the most recent period of the first of August " "to the thirty-first of July over the preceding period of the first of August to the " "thirty-first of July published by the United States department of labor non-seasonally " "adjusted consumer price index for the northeast region urban wage earners and clerical " "workers (CPI-W), or any successor index as calculated by the United States department " "of labor, is negative"), "lab652_unemployment": Q.quote(*LAB_652, "(ii) the three-month moving average of the seasonally adjusted New York state " "unemployment rate as determined by the U-3 measure of labor underutilization for " "the most recent period ending the thirty-first of July as calculated by the United " "States department of labor rises by one-half percentage point or more relative to " "its low during the previous twelve months"), "lab652_jobs": Q.quote(*LAB_652, "(iii) seasonally adjusted, total non-farm employment for New York state in July, " "calculated by the United States department of labor, decreased from the seasonally " "adjusted, total non-farm employment for New York state in April, and seasonally adjusted, " "total non-farm employment for New York state in July, calculated by the United States " "department of labor, decreased from the seasonally adjusted, total non-farm employment " "for New York state in January"), "lab652_two_years": Q.quote(*LAB_652, "limited to no more than two consecutive years"), "dol_2027_offramp": Q.quote(*NYS_2027, "For 2027, the conditions for one of the off-ramps has been met"), "dol_2027_home_care": Q.quote(*NYS_2027, "The minimum wage for home care aides will increase on January 1, 2027"), "governor_2027_offramp": Q.quote(*GOVERNOR, "One of the off-ramps conditions has been met for 2027"), "governor_2027_legislature": Q.quote(*GOVERNOR, "If approved by the State Legislature, new rates would take effect on March 1, 2027"), "memo_bill": Q.quote(*MEMO_2026, "PROGRAM BILL #44"), "memo_sensitive": Q.quote(*MEMO_2026, "has turned out to be overly sensitive"), "memo_sectors": Q.quote(*MEMO_2026, "the number of private education, hospitality, and retail jobs"), "memo_decline": Q.quote(*MEMO_2026, "has declined slightly, by a monthly 0.07% from January to July"), "memo_attribution": Q.quote(*MEMO_2026, "Due in part to the extreme shifts in immigration enforcement, restrictive visa " "policies, and federal tariffs"), "memo_purpose": Q.quote(*MEMO_2026, "The off-ramps were designed to protect businesses from having increased wages in the " "event of significant economic downturns"), "memo_jobs_offramp": Q.quote(*MEMO_2026, "pertaining to the absolute number of jobs in the state"), "memo_threshold": Q.quote(*MEMO_2026, "calculate whether the median of the converted monthly rates from the twelve-, six-, " "and three-month rates of change in seasonally-adjusted, total non-farm employment are " "less than negative zero point two five percent (-0.25%)"), "memo_march": Q.quote(*MEMO_2026, "take effect on March 1, 2027"), "bill_sponsor": Q.quote(*BILL, "Sandy Nurse"), "bill_introduced": Q.quote(*BILL, "3/10/2026"), "bill_committee": Q.quote(*BILL, "Committee on Consumer and Worker Protection"), "bill_referred": Q.quote(*BILL, "Referred to Comm by Council"), "bill_sponsors": Q.quote(*BILL, "Council Member Sponsors: 15"), "bill_tipped": Q.quote(*BILL, "until December 1, 2031, the minimum wage for an employee who is a food service worker " "receiving tips shall be (1) a minimum wage of at least two-thirds of the applicable minimum wage"), "bill_tipped_tips": Q.quote(*BILL, "provided that the tips of such an employee, when added to such minimum wage, are equal to " "or exceed the applicable minimum wage"), "bill_summary_less_than_500": Q.quote(*BILL, "employers with less than 500 employees would pay their employees $29 an hour by 2031"), "bill_schedule_2_2032": Q.quote(*BILL, "On or after January 1, 2032, the minimum wage is set forth in subdivision c of this section"), "bill_franchise": Q.quote(*BILL, "Franchisee. The term"), "abc7_date": Q.quote(*ABC7_2026, "Tuesday, March 10, 2026"), "abc7_poverty": Q.quote(*ABC7_2026, "Council Member Sandra Nurse, who sponsored the bill, said the existing rate leaves " "too many New Yorkers in poverty"), "abc7_highest": Q.quote(*ABC7_2026, "This would make minimum wage the highest in the nation"), "abc7_large": Q.quote(*ABC7_2026, "Employers with more than 500 employees would pay workers $20 an hour by 2027 and $30 an hour by 2030"), "abc7_smaller": Q.quote(*ABC7_2026, "Companies with less than 500 workers would hit this mark by 2032"), } def quoted_number(A: Article, key: str, pattern: str) -> float: """A figure read from a saved quote (never typed into the page by hand).""" import re m = re.search(pattern, A.results["quotes"][key]["quote"]) if not m: raise AssertionError(f"{key}: {pattern!r} not in the quote") return float(m.group(1).replace(",", "")) def deflators_with_2026() -> tuple[dict[int, float], dict]: """Factors to 2024 dollars by calendar year: the annual NY-metro CPI-U to 2025, and for 2026 the mean of the months published so far.""" defl = dict(bls.deflators()) monthly = bls.cpi_monthly() avg24 = float(np.mean([monthly[(2024, m)] for m in range(1, 13)])) m26 = [v for (y, m), v in monthly.items() if y == 2026] defl[2026] = avg24 / float(np.mean(m26)) last = max(monthly) return defl, {"avg_2024": avg24, "last_month": last, "last_value": monthly[last], "months_2026": len(m26)} def in_todays_money(nominal: float, year: int, month: int, cpi: dict, rate: float) -> float: """A future nominal rate in 2024 dollars: the CPI to its last published month, then an ASSUMED flat yearly inflation rate to the first of ``month`` in ``year`` (not a forecast).""" (ly, lm), last = cpi["last_month"], cpi["last_value"] years = ((year * 12 + month) - (ly * 12 + lm)) / 12 return nominal / (last / cpi["avg_2024"] * (1 + rate) ** years) # ------------------------------------------------------------------ the census years def census_pay(year: int, defl: dict[int, float], band_means: dict[float, float]) -> tuple[np.ndarray, np.ndarray, object]: """Residents who work for wages (employees, at work or with a job, 16 and over) and their hourly pay in 2024 dollars; the estimator. The censuses deflate with the previous year's CPI (they asked about 1989 and 1999); the ACS years with ADJINC and the survey year's. From 2008 to 2018 the ACS recorded weeks in bands: each band's midpoint is replaced by the band's mean in the years that recorded weeks exactly (``history.weeks_band_means``).""" banded = 2008 <= year <= 2018 if year < 2015: s = history.load(year, ["age", "esr", "cow", "wages", "hours_usual", "weeks"] + (["adjinc"] if year >= 2005 else [])) d = s.data f = defl[year - 1] if year < 2005 else d["adjinc"].to_numpy(float) * defl[year] weeks = d["weeks"].replace(band_means) if banded else d["weeks"] hourly = hourly_wage(d["wages"].to_numpy(float) * f, d["hours_usual"], weeks) emp = d["esr"].isin([1, 2]).to_numpy() & d["cow"].isin(EMPLOYEE_CLASSES).to_numpy() & (d["age"] >= 16).to_numpy() return emp & np.isfinite(hourly), hourly, s.estimator(min_records=0) s = acs1.load(year, "person", ["AGEP", "ESR", "COW", "WAGP", "WKHP", "WKWN"]) d = s.data weeks = d["WKWN"].replace(band_means) if banded else d["WKWN"] hourly = hourly_wage(d["WAGP"], d["WKHP"], weeks) # the extract is in 2024 dollars already emp = s.in_nyc() & d["ESR"].isin([1, 2]).to_numpy() & d["COW"].isin(EMPLOYEE_CLASSES).to_numpy() & (d["AGEP"] >= 16).to_numpy() return emp & np.isfinite(hourly), hourly, s.estimator(min_records=0) def median_pay(R, mask, hourly) -> tuple[float, float]: """The weighted median and its Woodruff margin (the Census Bureau's method for published medians: pay heaps at round numbers, so recomputing the median under each replicate weight often gives the same value and a margin of zero). No margin for a census year or 2020.""" if R.no_margin: return weighted_median(hourly[mask], R.w[mask]), float("nan") return woodruff_median(hourly[mask], R.w[mask], np.asarray(R.rw[mask], dtype=float), bin_width=0.25) def long_series(defl: dict[int, float]) -> dict: out = {} band_means = history.weeks_band_means() for y in history.years() + acs1.years(): emp, hourly, R = census_pay(y, defl, band_means) (med, med_moe), under = median_pay(R, emp, hourly), R.share(emp, hourly < THIRTY) out[y] = {"median": r(med, 2), "median_moe": r(med_moe, 2), "under30": r(100 * under.value, 2), "under30_moe": r(100 * under.moe, 1), "records": int(emp.sum()), "pay_year": y - 1 if y < 2005 else y} return out # ------------------------------------------------------------------ 2024 in detail def acs_detail(year: int) -> tuple[pd.DataFrame, object, dict[str, np.ndarray]]: """Employees with an hourly wage who live in the city or whose work is done there.""" s = acs1.load(year, "person", ACS_COLS) d = s.data hourly = hourly_wage(d["WAGP"], d["WKHP"], d["WKWN"]) emp = d["ESR"].isin([1, 2]).to_numpy() & d["COW"].isin(EMPLOYEE_CLASSES).to_numpy() & (d["AGEP"] >= 16).to_numpy() & np.isfinite(hourly) resident = s.in_nyc() codes = set(POW_NYC[puma_vintage(year)]) home = (pd.to_numeric(d["JWTRNS"], errors="coerce") == 11).to_numpy() works_city = (pd.to_numeric(d["POWSP"], errors="coerce") == 36).to_numpy() & pd.to_numeric(d["POWPUMA"], errors="coerce").isin(codes).to_numpy() private = d["COW"].isin([1, 2]).to_numpy() away = d["ESR"].to_numpy() == 2 # with a job but not at work: the census asks no place of work U = {"residents": emp & resident, "city_jobs": emp & ((works_city & ~home) | (resident & home)), "private_city_jobs": emp & private & ((works_city & ~home) | (resident & home))} d = d.assign(hourly=hourly, away=away) return d, s.estimator(min_records=0), U def detail_block(d, R, U, cuts: dict[str, float]) -> dict: out = {} for u, mask in U.items(): tot = R.count(mask) blk = {"total": r(tot.value, -3), "total_moe": r(tot.moe, -3), "records": int(mask.sum())} med, med_moe = median_pay(R, mask, d["hourly"].to_numpy(float)) blk["median"], blk["median_moe"] = r(med, 2), r(med_moe, 2) for k, c in cuts.items(): sh, n = R.share(mask, d["hourly"].to_numpy(float) < c), R.count(mask & (d["hourly"].to_numpy(float) < c)) blk[k] = {"cut": r(c, 2), "share": r(100 * sh.value, 2), "moe": r(100 * sh.moe, 1), "count": r(n.value, -3), "count_moe": r(n.moe, -3), "records": int((mask & (d["hourly"].to_numpy(float) < c)).sum())} out[u] = blk return out def who(d, R, mask) -> dict: """Who among private-sector jobs in the city is paid under $30 (2024).""" h = d["hourly"].to_numpy(float) under = h < THIRTY def group(labels, order, names=None) -> list[dict]: rows = [] for k in order: g = mask & (labels == k) if g.sum() < acs1.MIN_RECORDS: # the dataset's small-cell rule continue sh, n, comp = R.share(g, under), R.count(g & under), R.share(mask & under, labels == k) rows.append({"key": k, "label": (names or {}).get(k, k), "share": r(100 * sh.value, 2), "moe": r(100 * sh.moe, 1), "count": r(n.value, -3), "of_all_under": r(100 * comp.value, 1), "records": int(g.sum())}) return rows occ = occupation_from_code(d["OCCP"], "acs") ind = industry_from_naics(d["NAICSP"]) cit = pd.to_numeric(d["CIT"], errors="coerce").to_numpy() citizenship = np.select([cit <= 3, cit == 4], ["us_born", "naturalized"], "noncitizen") age = pd.to_numeric(d["AGEP"], errors="coerce").to_numpy() ageband = np.select([age < 25, age < 35, age < 55], ["16_24", "25_34", "35_54"], "55_plus") sex = np.where(pd.to_numeric(d["SEX"], errors="coerce").to_numpy() == 2, "women", "men") hours = np.where(pd.to_numeric(d["WKHP"], errors="coerce").to_numpy() >= 35, "full_time", "part_time") labels = {k: SHORT_NAMES.get(v, v[0] + v[1:].lower()) for k, v in pums.code_labels("OCCP").items()} occ4 = d["OCCP"].astype(str).str.replace(r"\.0$", "", regex=True).str.zfill(4).to_numpy() w = R.w top = pd.Series(w[mask & under]).groupby(occ4[mask & under]).sum().sort_values(ascending=False).head(10) return {"occupation_group": group(occ, [k for k, _ in OCCUPATIONS], dict(OCCUPATIONS)), "industry": group(ind, [k for k, _ in INDUSTRIES], dict(INDUSTRIES)), "citizenship": group(citizenship, ["us_born", "naturalized", "noncitizen"], {"us_born": "Citizens from birth", "naturalized": "Naturalized citizens", "noncitizen": "Not citizens"}), "age": group(ageband, ["16_24", "25_34", "35_54", "55_plus"], {"16_24": "16 to 24", "25_34": "25 to 34", "35_54": "35 to 54", "55_plus": "55 and over"}), "sex": group(sex, ["women", "men"], {"women": "Women", "men": "Men"}), "hours": group(hours, ["full_time", "part_time"], {"full_time": "35 or more hours a week", "part_time": "Under 35 hours"}), "top_occupations": group(occ4, list(top.index), {k: labels.get(k, k) for k in top.index})} def epi_reproduction(d, R, U, defl) -> dict: """EPI's Table 2 from the same 2023 one-year file: 2023 pay (the extract's 2024 dollars back to 2023 dollars) grown 3.3% a year for seven years, for work done in the city.""" mask = U["city_jobs"] w23 = d["hourly"].to_numpy(float) / defl[2023] g7 = (1 + EPI_GROWTH) ** EPI_YEARS resident = R.share(U["residents"], w23 * g7 < 30) g = (1 + EPI_GROWTH) ** EPI_YEARS med = R.median(mask, w23).value out = {} for thr, (count, share, kaitz) in EPI_ROWS.items(): sh = R.share(mask, w23 * g < thr) out[str(thr)] = {"epi_share": share, "share": r(100 * sh.value, 1), "moe": r(100 * sh.moe, 1), "epi_count": count, "epi_ratio": kaitz, "ratio": r(thr / (med * g), 2)} out["residents_30"] = {"share": r(100 * resident.value, 1), "moe": r(100 * resident.moe, 1)} return out # ------------------------------------------------------------------ the CPS, 2026 def cps_pay_this_year() -> dict: """Wage and salary workers living in the city, January to the latest month of 2026, in the outgoing rotation groups (the CPS's earnings sample): usual weekly earnings over usual weekly hours, nominal dollars, final earnings weight. Margins clustered on the household.""" months = [m for m in cps.months() if m[0] == cps.months()[-1][0]] d = cps.load(CPS_VARS, months=months) d = d[cps.in_nyc(d) & d["HRMIS"].isin([4, 8]).to_numpy() & (d["PRERELG"] == 1).to_numpy() & d["PEMLR"].isin([1, 2]).to_numpy() & (d["PRTAGE"] >= 16).to_numpy() & d["PEIO1COW"].between(1, 5).to_numpy() & (d["PWORWGT"] > 0).to_numpy()].reset_index(drop=True) weekly = np.where(d["PTERNWA"] > 0, d["PTERNWA"] / 100.0, np.nan) hourly_paid = (d["PEERNHRY"] == 1).to_numpy() hrs = np.where(d["PEHRUSL1"] > 0, d["PEHRUSL1"], np.where(hourly_paid & (d["PEERNHRO"] > 0), d["PEERNHRO"], np.nan)) wage = weekly / hrs ok = np.isfinite(wage) allocated = ((d["PRWERNAL"] == 1) | (hourly_paid & (d["PRHERNAL"] == 1))).to_numpy() des = cps.Design(d) one = np.ones(len(d)) share, moe = cps.ratio_moe(d, y=(wage < THIRTY).astype(float), x=one, weight="PWORWGT", mask=ok, design=des) rep, rep_moe = cps.ratio_moe(d, y=(wage < THIRTY).astype(float), x=one, weight="PWORWGT", mask=ok & ~allocated, design=des) alloc, _ = cps.ratio_moe(d, y=allocated.astype(float), x=one, weight="PWORWGT", mask=ok, design=des) v, w = wage[ok], d["PWORWGT"].to_numpy(float)[ok] o = np.argsort(v, kind="mergesort") cw = np.cumsum(w[o]) median = float(v[o][np.searchsorted(cw, cw[-1] / 2)]) first, last = months[0], months[-1] return {"first_month": f"{first[0]}-{first[1]:02d}", "last_month": f"{last[0]}-{last[1]:02d}", "months": len(months), "under30": {"share": r(100 * share, 1), "moe": r(100 * moe, 1)}, "under30_reported": {"share": r(100 * rep, 1), "moe": r(100 * rep_moe, 1), "records": int((ok & ~allocated).sum())}, "allocated_share": r(100 * alloc, 0), "median": r(median, 2), "records": int(ok.sum()), "households": int(np.unique(des.cluster[ok]).size)} # ------------------------------------------------------------------ main def main(argv: list[str] | None = None) -> int: """Run the analysis and write every file (no options).""" A = Article(SLUG, __file__, seed=SEED) defl, cpi = deflators_with_2026() mw = minimum_wage(A) real = {y: v * defl[y] for y, v in mw["nominal"].items() if y in defl} A.results["quotes"] = {**mw["quotes"], **history_quotes(A)} A.results["history"] = { "law_2016_first_step": quoted_number(A, "law_2016_city", r"would rise to \$(\d+) at the end of 2016"), "law_2016_target": quoted_number(A, "law_2016_city", r"reaching \$(\d+) on 12/31/2018"), "law_2016_workers_millions": quoted_number(A, "law_2016_workers", r"more than ([\d.]+) million New Yorkers"), "law_2023_first_step": quoted_number(A, "law_2023_sixteen", r"Increase to \$(\d+)-Per-Hour in New York City"), "memo_monthly_decline_pct": quoted_number(A, "memo_decline", r"by a monthly ([\d.]+)% from January to July"), "memo_threshold_pct": quoted_number(A, "memo_threshold", r"\(-([\d.]+)%\)"), "bill_sponsors": quoted_number(A, "bill_sponsors", r"Council Member Sponsors: (\d+)"), } A.results["minimum"] = {"checks": mw["checks"], "nominal": {str(y): r(v, 2) for y, v in mw["nominal"].items()}, "real_2024": {str(y): r(v, 2) for y, v in real.items()}} # $30 in 2030, and the bill's steps, in 2024 dollars under the assumed inflation. t30 = {f"{k:.3f}": in_todays_money(THIRTY, 2030, 1, cpi, k) for k in INFLATION_SWEEP} main30 = t30[f"{INFLATION_MAIN:.3f}"] steps = {str(y): r(in_todays_money(v, y, 1, cpi, INFLATION_MAIN), 2) for y, (v, _) in BILL_LARGE.items()} seventeen_2027 = in_todays_money(RATE_2027[0], 2027, 1, cpi, INFLATION_MAIN) A.results["thirty_2030"] = {"main": r(main30, 2), "lo": r(min(t30.values()), 2), "hi": r(max(t30.values()), 2), "assumed_inflation": INFLATION_MAIN * 100, "sweep": {k: r(v, 2) for k, v in t30.items()}, "cpi_last_month": f"{cpi['last_month'][0]}-{cpi['last_month'][1]:02d}", "cpi_since_2024": r(100 * (cpi["last_value"] / cpi["avg_2024"] - 1), 1), "bill_large_2024usd": steps, "state_2027_2024usd": r(seventeen_2027, 2), "bill_small": {str(k): v for k, v in BILL_SMALL.items()}} # Thirty-five years of pay. ser = long_series(defl) ratio = {} for y, e in ser.items(): py = e["pay_year"] ratio[y] = real[py] / e["median"] # the minimum of the year each point sits at (the pay year for a census) A.results["series"] = {str(y): {**e, "minimum_2024usd": r(real[e["pay_year"]], 2), "ratio": r(100 * ratio[y], 0)} for y, e in ser.items()} years = sorted(ser) first, latest = years[0], years[-1] a, b = ser[COMPARE_FROM], ser[latest] pts, pmoe = b["under30"] - a["under30"], moe_difference(a["under30_moe"], b["under30_moe"]) A.results["change_under30"] = {"pts": r(pts, 1), "moe": r(pmoe, 1), "lo": r(pts - pmoe, 1), "hi": r(pts + pmoe, 1)} A.claim("change_under30", "down") # the page: a fall that holds dm, dmoe = b["median"] - a["median"], moe_difference(a["median_moe"], b["median_moe"]) A.results["change_median"] = {"usd": r(dm, 2), "moe": r(dmoe, 2), "lo": r(dm - dmoe, 2), "hi": r(dm + dmoe, 2)} assert direction(dm - dmoe, dm + dmoe) == "unclear", "the page says the median was about the same from 2019 to 2024" A.results["minimum"]["doubled_pct"] = r(100 * (real[latest] / real[first - 1] - 1), 0) # 1989 to 2024, after inflation A.results["minimum"]["since_peak_pct"] = r(100 * (real[2026] / real[2019] - 1), 1) assert A.results["minimum"]["since_peak_pct"] < 0 A.long_view(1989, latest, "Census 1990 and 2000 samples (1989 and 1999 pay) and the ACS one-year files 2005-2024") # 2024 in detail, and 2019 for the change. d24, R24, U24 = acs_detail(LATEST_ACS) cuts = {"under30": THIRTY, "under_2030": main30, "under_2030_lo": min(t30.values()), "under_2030_hi": max(t30.values()), "under_step_2027": float(steps["2027"])} floor24 = 15.0 * defl[2023] # the lowest general minimum over the year the 2024 survey asked about ($15 in 2023) cuts["under_legal_minimum"] = floor24 A.results["acs2024"] = detail_block(d24, R24, U24, cuts) A.results["who"] = who(d24, R24, U24["private_city_jobs"]) A.results["away_share"] = r(100 * R24.share(U24["residents"], d24["away"].to_numpy()).value, 1) d19, R19, U19 = acs_detail(COMPARE_FROM) A.results["acs2019"] = detail_block(d19, R19, U19, {"under30": THIRTY}) d23, R23, U23 = acs_detail(2023) A.results["epi"] = epi_reproduction(d23, R23, U23, defl) e30 = A.results["epi"]["30"] A.reproduce("EPI: share of the city's wage-earning workforce under $30 in 2030 (2023 ACS, 3.3% a year)", published=e30["epi_share"], published_moe90=None, reproduced=e30["share"], reproduced_moe90=e30["moe"], kind="pct") # Deflator check: the 1989 median under the BLS research series R-CPI-U-RS (national), which # restates the 1990s' CPI with today's methods, instead of the New York-area CPI-U. rs = bls.r_cpi_u_rs_annual() nominal_1989 = ser[1990]["median"] / defl[1989] A.results["rs_check"] = {"median_1989_metro_cpi": ser[1990]["median"], "median_1989_r_cpi_u_rs": r(nominal_1989 * rs[2024] / rs[1989], 2), "minimum_1989_r_cpi_u_rs": r(mw["nominal"][1989] * rs[2024] / rs[1989], 2)} A.results["rates"] = {"state_2026": 17.0, "state_2027": RATE_2027[0], "governor_march_2027": 17.60, "bill_large_2027": BILL_LARGE[2027][0], "bill_2030": BILL_LARGE[2030][0], "minimum_2024": mw["nominal"][2024]} A.results["ratio_2030_to_2024_median"] = r(100 * main30 / A.results["acs2024"]["private_city_jobs"]["median"], 0) # 2026. now = cps_pay_this_year() A.results["now"] = now lm = int(now["last_month"][5:]) A.latest(now["last_month"], "CPS basic monthly files, outgoing rotation groups (usual weekly earnings and hours)") # ---- printed figures (the prose's strings) def by_key(rows: list[dict]) -> dict: return {o["key"]: o for o in rows} W = A.results["who"] occ = by_key(W["top_occupations"]) A.results["story"] = {"home_health_aides": occ["3601"]["share"], "young": by_key(W["age"])["16_24"]["share"], "noncitizen": by_key(W["citizenship"])["noncitizen"]["share"], "us_born": by_key(W["citizenship"])["us_born"]["share"], "naturalized": by_key(W["citizenship"])["naturalized"]["share"], "health_of_under": by_key(W["industry"])["health"]["of_all_under"], # the beliefs tested (v0.9.3): who the workers under $30 are, as shares of them "young_of_under": by_key(W["age"])["16_24"]["of_all_under"], "middle_aged_of_under": by_key(W["age"])["35_54"]["of_all_under"], "us_born_of_under": by_key(W["citizenship"])["us_born"]["of_all_under"], "full_time_of_under": by_key(W["hours"])["full_time"]["of_all_under"], "salary_heap": {"salary": 60000, "hourly": r(60000 / 2080, 2)}} assert occ["3601"]["label"] == "Home health aides" P = A.printed st = A.results["story"] P("now.under30.share", f"{now['under30']['share']:.0f}%") P("now.last_month", f"{MONTHS[lm - 1]} {now['last_month'][:4]}") P("now.median", f"${now['median']:.2f}") P("now.allocated_share", f"{now['allocated_share']:.0f}%") P("now.under30_reported.share", f"{now['under30_reported']['share']:.0f}%") P("story.salary_heap.salary", "$60,000") assert now["median"] == st["salary_heap"]["hourly"], "the page says this year's median is the $60,000 heap" res, jobs, priv = (A.results["acs2024"][k] for k in ("residents", "city_jobs", "private_city_jobs")) P("acs2024.residents.under30.share", f"{res['under30']['share']:.0f}%") P("acs2024.city_jobs.under30.share", f"{jobs['under30']['share']:.0f}%") P("acs2024.private_city_jobs.under30.share", f"{priv['under30']['share']:.0f}%") P("acs2024.private_city_jobs.under30.count", f"{priv['under30']['count'] / 1e6:.1f} million") P("acs2024.private_city_jobs.under_2030.share", f"{priv['under_2030']['share']:.0f}%") P("acs2024.city_jobs.under_2030.share", f"{jobs['under_2030']['share']:.0f}%") P("acs2024.residents.median", f"${res['median']:.2f}") T = A.results["thirty_2030"] P("thirty_2030.main", f"${T['main']:.2f}") P("thirty_2030.lo", f"${T['lo']:.2f}") P("thirty_2030.hi", f"${T['hi']:.2f}") P("thirty_2030.cpi_since_2024", f"{T['cpi_since_2024']:.1f}%") P("thirty_2030.assumed_inflation", f"{T['assumed_inflation']:.1f}%") S = A.results["series"] P(f"series.{first}.median", f"${S[str(first)]['median']:.2f}") P(f"series.{latest}.median", f"${S[str(latest)]['median']:.2f}") P(f"series.{COMPARE_FROM}.median", f"${S[str(COMPARE_FROM)]['median']:.2f}") low = min(years, key=lambda y: S[str(y)]["median"]) assert low == 2014, "the page says the median's lowest estimate is 2014's" P(f"series.{low}.median", f"${S[str(low)]['median']:.2f}") P(f"series.{first}.under30", f"{S[str(first)]['under30']:.0f}%") P(f"series.{latest}.under30", f"{S[str(latest)]['under30']:.0f}%") P(f"series.{COMPARE_FROM}.under30", f"{S[str(COMPARE_FROM)]['under30']:.0f}%") P(f"series.{first}.ratio", f"{S[str(first)]['ratio']:.0f}%") P(f"series.{latest}.ratio", f"{S[str(latest)]['ratio']:.0f}%") P("change_under30.pts", f"{abs(A.results['change_under30']['pts']):.1f}") P("change_under30.moe", f"{A.results['change_under30']['moe']:.1f}") P("minimum.real_2024.1989", f"${A.results['minimum']['real_2024']['1989']:.2f}") P("minimum.real_2024.2026", f"${A.results['minimum']['real_2024']['2026']:.2f}") peak = max(A.results["minimum"]["real_2024"], key=lambda y: A.results["minimum"]["real_2024"][y]) assert peak == "2019", "the page says the minimum's value peaked in 2019" P("minimum.real_2024.2019", f"${A.results['minimum']['real_2024']['2019']:.2f}") for k, txt in (("state_2027", "$17"), ("governor_march_2027", "$17.60"), ("bill_large_2027", "$20"), ("bill_2030", "$30"), ("minimum_2024", "$16")): assert abs(A.results["rates"][k] - float(txt[1:])) < 1e-9, k P(f"rates.{k}", txt) P("ratio_2030_to_2024_median", f"{A.results['ratio_2030_to_2024_median']:.0f}%") P("acs2024.private_city_jobs.median", f"${priv['median']:.2f}") P("acs2024.residents.under_legal_minimum.share", f"{res['under_legal_minimum']['share']:.0f}%") P("acs2024.residents.under_legal_minimum.cut", f"${res['under_legal_minimum']['cut']:.2f}") P("rs_check.median_1989_r_cpi_u_rs", f"${A.results['rs_check']['median_1989_r_cpi_u_rs']:.2f}") P("rs_check.minimum_1989_r_cpi_u_rs", f"${A.results['rs_check']['minimum_1989_r_cpi_u_rs']:.2f}") A.results["epi_growth_pct"] = EPI_GROWTH * 100 P("epi_growth_pct", f"{EPI_GROWTH * 100:.1f}%") E = A.results["epi"]["30"] P("epi.30.epi_share", f"{E['epi_share']:.1f}%") P("epi.30.share", f"{E['share']:.1f}%") P("epi.30.ratio", f"{E['ratio']:.2f}") P("epi.30.epi_ratio", f"{E['epi_ratio']:.2f}") P("epi.residents_30.share", f"{A.results['epi']['residents_30']['share']:.1f}%") P("away_share", f"{A.results['away_share']:.0f}%") P("story.home_health_aides", f"{st['home_health_aides']:.0f}%") P("story.young", f"{st['young']:.0f}%") P("story.noncitizen", f"{st['noncitizen']:.0f}%") P("story.us_born", f"{st['us_born']:.0f}%") P("story.naturalized", f"{st['naturalized']:.0f}%") P("acs2024.private_city_jobs.under_2030.count", f"{priv['under_2030']['count'] / 1e6:.2f} million") P("story.young_of_under", f"{st['young_of_under']:.0f}%") P("story.middle_aged_of_under", f"{st['middle_aged_of_under']:.0f}%") P("story.us_born_of_under", f"{st['us_born_of_under']:.0f}%") P("story.full_time_of_under", f"{st['full_time_of_under']:.0f}%") P("change_median.usd", f"${A.results['change_median']['usd']:.2f}") P("change_median.moe", f"${A.results['change_median']['moe']:.2f}") P("minimum.doubled_pct", f"{A.results['minimum']['doubled_pct']:.0f}%") P("minimum.since_peak_pct", f"{abs(A.results['minimum']['since_peak_pct']):.0f}%") H = A.results["history"] P("history.law_2016_first_step", f"${H['law_2016_first_step']:.0f}") P("history.law_2016_target", f"${H['law_2016_target']:.0f}") P("history.law_2016_workers_millions", f"{H['law_2016_workers_millions']:.1f} million") P("history.law_2023_first_step", f"${H['law_2023_first_step']:.0f}") P("history.memo_monthly_decline_pct", f"{H['memo_monthly_decline_pct']:.2f}%") P("history.memo_threshold_pct", f"{H['memo_threshold_pct']:.2f}%") # ---- charts xs = [(y, f"{S[str(y)]['pay_year']}" + (" (1990 census)" if y == 1990 else " (2000 census)" if y == 2000 else "")) for y in years] A.chart("half_for_a_generation", lines( title="About half or more of the city's working residents have been paid under $30 an hour since 1989", subtitle="Share of New York City residents who work for wages paid under $30 an hour, in 2024 dollars", x=[[str(y), lab] for y, lab in xs], series=[["under30", "Paid under $30 an hour", "Under $30"]], cells={"": {"under30": {**{str(y): S[str(y)]["under30"] for y in years}, **{f"{y}_moe": S[str(y)]["under30_moe"] for y in years if y >= 2005}}}}, format="pct", axis="Share of residents who work for wages", x_ticks=[str(y) for y in years if y in (1990, 2000, 2005, 2010, 2015, 2020, latest)], x_tick_labels={str(y): lab.split(" (")[0] for y, lab in xs if " (" in lab}, note="Hourly pay is a year's wages over usual weekly hours times weeks worked. The 1989 and 1999 points come from " "the decennial census long form, the later ones from the American Community Survey: a change of survey, at " "the gap in the line. No margin is calculated for the censuses; 2020 was an experimental file and its margin is " "not shown; the 2005 file left out group quarters. Weeks were " "recorded in bands from 2008 to 2018 and are set at each band's mean. Dollars are 2024 dollars on the " "New York-area consumer price index.", provenance="U.S. Census Bureau, decennial census and American Community Survey microdata; BLS, CPI-U")) mx = list(range(1989, 2031)) floor_cells = {str(y): r(real[y], 2) for y in range(1989, 2027)} mid_cells = {str(S[str(y)]["pay_year"]): S[str(y)]["median"] for y in years} mid_moe = {f"{S[str(y)]['pay_year']}_moe": S[str(y)]["median_moe"] for y in years if y >= 2005} A.chart("floor_and_middle", lines( title="The floor climbed toward a middle that ended where it began", subtitle="The minimum wage in New York City and the median hourly pay of its residents who work for wages, in 2024 dollars", x=[[str(y), str(y)] for y in mx], series=[["median", "Median hourly pay of residents who work for wages", "Median pay"], ["minimum", "The minimum wage, the year's average", "Minimum wage"], ["bill", "Int 0757's rate for large employers, at an assumed 2.5% a year inflation", "The $30 bill"]], cells={"": {"median": {**mid_cells, **mid_moe}, "minimum": floor_cells, "bill": {str(y): float(v) for y, v in steps.items()}}}, format="usd2", axis="Dollars an hour, in 2024 dollars", x_ticks=[str(y) for y in mx if y % 5 == 0], note="The minimum is the higher of the State's and the federal rate on each day, averaged over the year; from " "2017 the City's rate for employers of 11 or more. Median pay is from the 1990 and 2000 censuses (1989 and " "1999 pay) and the ACS from 2005, a change of survey. The bill's steps for 2027 to 2030 are its rates for " "employers of more than 500, deflated at an assumed 2.5% a year after August 2026, not a forecast.", provenance="NYS Department of Labor; U.S. Department of Labor; NYC Council, Int 0757-2026; U.S. Census Bureau; BLS, CPI-U")) top = A.results["who"]["top_occupations"] # Row names short enough to leave a phone room for the dots; the tooltip keeps the census title. short = {"Janitors and building cleaners": "Janitors, cleaners", "Construction laborers": "Building laborers", "Truck and delivery drivers": "Truck drivers", "Retail salespersons": "Retail sales", "Customer service representatives": "Customer service"} A.chart("who_under_30", strip( title="Most care, food, cleaning and retail workers are paid under $30", subtitle="The ten occupations with the most private-sector workers in the city paid under $30 an hour in 2024: the share of each paid less", x=[["2024", "2024"]], series=[[o["key"], o["label"], short.get(o["label"], o["label"])] for o in top], cells={"": {o["key"]: {"2024": o["share"], "2024_moe": o["moe"]} for o in top}}, format="pct", axis="Share paid under $30 an hour", value_label="the share paid under $30", moe_label="90% margin", note="Private and non-profit employees whose work is done in the five boroughs, wherever they live, and city " "residents who work from home. The Council bill excludes government employers; its $30 in 2030 is for " "employers of more than 500, which the census does not identify. Ordered by the share.", provenance="U.S. Census Bureau, American Community Survey one-year microdata, 2024")) rows = [{"table": "long_view", "year": S[str(y)]["pay_year"], "survey": y, "group": "residents who work for wages", "value": S[str(y)]["under30"], "moe": S[str(y)]["under30_moe"], "measure": "share paid under $30 (2024 dollars), %"} for y in years] rows += [{"table": "long_view", "year": S[str(y)]["pay_year"], "survey": y, "group": "residents who work for wages", "value": S[str(y)]["median"], "moe": S[str(y)]["median_moe"], "measure": "median hourly pay (2024 dollars)"} for y in years] rows += [{"table": "minimum_wage", "year": y, "survey": None, "group": "New York City", "value": r(v, 2), "moe": None, "measure": "minimum wage, yearly average (nominal)"} for y, v in mw["nominal"].items()] rows += [{"table": "who_2024", "year": 2024, "survey": 2024, "group": f"{k}: {o['label']}", "value": o["share"], "moe": o["moe"], "measure": "share of private-sector jobs in the city paid under $30, %"} for k, v in A.results["who"].items() for o in v] A.write(data=pd.DataFrame(rows)) print(f"{SLUG}: wrote {A.out}") return 0 if __name__ == "__main__": raise SystemExit(main())