diff --git a/.gitignore b/.gitignore index e2da7cc59..7301e0ec2 100644 --- a/.gitignore +++ b/.gitignore @@ -92,5 +92,10 @@ cert.txt # local MCP server config (contains auth tokens) .mcp.json mcp-server/create-agent-key.ts -.gcloudignore + +# Claude CLAUDE.md + +#gcloud +.gcloudignore + diff --git a/components/Navbar.tsx b/components/Navbar.tsx index 3c68cdd44..c404b0871 100644 --- a/components/Navbar.tsx +++ b/components/Navbar.tsx @@ -15,6 +15,7 @@ import { NavbarLinkAiTools, NavbarLinkBills, NavbarLinkHearings, + NavbarLinkLobbying, NavbarLinkEditProfile, NavbarLinkEffective, NavbarLinkFAQ, @@ -79,6 +80,9 @@ const MobileNav: React.FC> = () => { ) : null} + {flags().lobbyingTable ? ( + + ) : null} {authenticated ? : <>} @@ -233,6 +237,12 @@ const DesktopNav: React.FC> = () => { + {flags().lobbyingTable ? ( +
+ +
+ ) : null} + {authenticated ? (
diff --git a/components/NavbarComponents.tsx b/components/NavbarComponents.tsx index a378acaed..e8f121e80 100644 --- a/components/NavbarComponents.tsx +++ b/components/NavbarComponents.tsx @@ -142,6 +142,27 @@ export const NavbarLinkBills: React.FC< ) } +export const NavbarLinkLobbying: React.FC< + React.PropsWithChildren<{ + handleClick?: any + other?: any + }> +> = ({ handleClick, other }) => { + const isMobile = useMediaQuery("(max-width: 768px)") + const { t } = useTranslation(["common", "auth"]) + return ( + + + {t("navigation.lobbying")} + + + ) +} + export const NavbarLinkBallotQuestions: React.FC< React.PropsWithChildren<{ handleClick?: any diff --git a/components/bill/BillDetails.tsx b/components/bill/BillDetails.tsx index d4d8a363a..65d941b49 100644 --- a/components/bill/BillDetails.tsx +++ b/components/bill/BillDetails.tsx @@ -6,7 +6,7 @@ import { Banner } from "../shared/StyledSharedComponents" import { BillNumber, Styled } from "./BillNumber" import { BillTestimonies } from "./BillTestimonies" import BillTrackerConnectedView from "./BillTracker" -import { LobbyingTable } from "./LobbyingTable" +import { LobbyingBillCard } from "components/lobbying/LobbyingBillCard" import { Committees, Hearing, Sponsors } from "./SponsorsAndCommittees" import { Status } from "./Status" import { Summary } from "./Summary" @@ -99,7 +99,11 @@ export const BillDetails = ({ bill }: BillProps) => { {flags.lobbyingTable && ( - + )} diff --git a/components/db/lobbying.ts b/components/db/lobbying.ts new file mode 100644 index 000000000..8a5ea3db9 --- /dev/null +++ b/components/db/lobbying.ts @@ -0,0 +1,228 @@ +import { + collection, + doc, + getDoc, + getDocs, + limit, + orderBy, + query, + where +} from "firebase/firestore" +import { useAsync } from "react-async-hook" +import type { + LobbyingClientSummary, + LobbyingFiling, + LobbyingRegistrant, + LobbyingStats +} from "functions/src/lobbying/types" +import { firestore } from "../firebase" + +// Mirror of constants in functions/src/lobbying/types.ts — kept here to avoid +// pulling firebase-admin (a Node-only package) into the browser bundle. +const FILINGS_COLLECTION = "lobbyingFilings" +const REGISTRANTS_COLLECTION = "lobbyingRegistrants" +const CLIENTS_COLLECTION = "lobbyingClients" +const LOBBYING_STATS_COLLECTION = "lobbyingMeta" +const LOBBYING_STATS_DOC_ID = "stats" + +// ── Internal fetchers ───────────────────────────────────────────────────────── + +async function fetchLobbyingStats(): Promise { + const snap = await getDoc( + doc(firestore, LOBBYING_STATS_COLLECTION, LOBBYING_STATS_DOC_ID) + ) + return snap.exists() ? (snap.data() as LobbyingStats) : undefined +} + +async function fetchFilingsForBill( + court: number, + billId: string +): Promise { + const snap = await getDocs( + query( + collection(firestore, FILINGS_COLLECTION), + where("generalCourt", "==", court), + where("billId", "==", billId) + ) + ) + return snap.docs.map(d => d.data() as LobbyingFiling) +} + +async function fetchRegistrants(opts: { + regType?: "Lobbyist" | "Employer" + year?: number + pageSize?: number +}): Promise { + const constraints = [ + opts.regType ? where("regType", "==", opts.regType) : undefined, + opts.year ? where("year", "==", opts.year) : undefined, + orderBy("entityNameNorm"), + limit(opts.pageSize ?? 50) + ].filter(Boolean) as Parameters[1][] + + const snap = await getDocs( + query(collection(firestore, REGISTRANTS_COLLECTION), ...constraints) + ) + return snap.docs.map(d => d.data() as LobbyingRegistrant) +} + +async function fetchRegistrant( + registrantId: string +): Promise { + const snap = await getDoc( + doc(firestore, REGISTRANTS_COLLECTION, registrantId) + ) + return snap.exists() ? (snap.data() as LobbyingRegistrant) : undefined +} + +async function fetchFilingsForRegistrant( + registrantId: string +): Promise { + const snap = await getDocs( + query( + collection(firestore, FILINGS_COLLECTION), + where("registrantId", "==", registrantId), + orderBy("year", "desc") + ) + ) + return snap.docs.map(d => d.data() as LobbyingFiling) +} + +async function fetchClients(opts: { + year?: number + pageSize?: number +}): Promise { + const constraints = [ + opts.year ? where("years", "array-contains", opts.year) : undefined, + orderBy("clientNameNorm"), + limit(opts.pageSize ?? 50) + ].filter(Boolean) as Parameters[1][] + + const snap = await getDocs( + query(collection(firestore, CLIENTS_COLLECTION), ...constraints) + ) + return snap.docs.map(d => d.data() as LobbyingClientSummary) +} + +async function fetchClient( + clientSlug: string +): Promise { + const snap = await getDoc(doc(firestore, CLIENTS_COLLECTION, clientSlug)) + return snap.exists() ? (snap.data() as LobbyingClientSummary) : undefined +} + +async function fetchAllRegistrants(): Promise { + const snap = await getDocs( + query( + collection(firestore, REGISTRANTS_COLLECTION), + orderBy("entityNameNorm"), + limit(2000) + ) + ) + return snap.docs.map(d => d.data() as LobbyingRegistrant) +} + +async function fetchRegistrantsByEntityName( + entityNameNorm: string +): Promise { + const snap = await getDocs( + query( + collection(firestore, REGISTRANTS_COLLECTION), + where("entityNameNorm", "==", entityNameNorm), + orderBy("year", "desc") + ) + ) + return snap.docs.map(d => d.data() as LobbyingRegistrant) +} + +async function fetchFilingsForEntityName( + entityNameNorm: string +): Promise { + const snap = await getDocs( + query( + collection(firestore, FILINGS_COLLECTION), + where("entityNameNorm", "==", entityNameNorm), + orderBy("year", "desc") + ) + ) + return snap.docs.map(d => d.data() as LobbyingFiling) +} + +async function fetchFilingsForCourt(court: number): Promise { + const snap = await getDocs( + query( + collection(firestore, FILINGS_COLLECTION), + where("generalCourt", "==", court) + ) + ) + return snap.docs.map(d => d.data() as LobbyingFiling) +} + +async function fetchFilingsForClient( + clientNameNorm: string +): Promise { + const snap = await getDocs( + query( + collection(firestore, FILINGS_COLLECTION), + where("clientNameNorm", "==", clientNameNorm), + orderBy("year", "desc") + ) + ) + return snap.docs.map(d => d.data() as LobbyingFiling) +} + +// ── Public hooks ────────────────────────────────────────────────────────────── + +export function useLobbyingFilingsForCourt(court: number) { + return useAsync(fetchFilingsForCourt, [court]) +} + +export function useLobbyingAllRegistrants() { + return useAsync(fetchAllRegistrants, []) +} + +export function useLobbyingRegistrantsByEntityName(entityNameNorm: string) { + return useAsync(fetchRegistrantsByEntityName, [entityNameNorm]) +} + +export function useLobbyingFilingsForEntityName(entityNameNorm: string) { + return useAsync(fetchFilingsForEntityName, [entityNameNorm]) +} + +export function useLobbyingStats() { + return useAsync(fetchLobbyingStats, []) +} + +export function useLobbyingFilingsForBill(court: number, billId: string) { + return useAsync(fetchFilingsForBill, [court, billId]) +} + +export function useLobbyingRegistrants(opts: { + regType?: "Lobbyist" | "Employer" + year?: number + pageSize?: number +}) { + return useAsync(fetchRegistrants, [opts]) +} + +export function useLobbyingRegistrant(registrantId: string) { + return useAsync(fetchRegistrant, [registrantId]) +} + +export function useLobbyingFilingsForRegistrant(registrantId: string) { + return useAsync(fetchFilingsForRegistrant, [registrantId]) +} + +export function useLobbyingClients( + opts: { year?: number; pageSize?: number } = {} +) { + return useAsync(fetchClients, [opts]) +} + +export function useLobbyingClient(clientSlug: string) { + return useAsync(fetchClient, [clientSlug]) +} + +export function useLobbyingFilingsForClient(clientNameNorm: string) { + return useAsync(fetchFilingsForClient, [clientNameNorm]) +} diff --git a/components/featureFlags.ts b/components/featureFlags.ts index 83041ebcd..b3afb0ba9 100644 --- a/components/featureFlags.ts +++ b/components/featureFlags.ts @@ -37,7 +37,7 @@ const defaults: Record = { notifications: true, billTracker: true, followOrg: true, - lobbyingTable: false, + lobbyingTable: true, hearingsAndTranscriptions: true, phoneVerificationUI: true, ballotQuestions: true, diff --git a/components/lobbying/LobbyingAttribution.tsx b/components/lobbying/LobbyingAttribution.tsx new file mode 100644 index 000000000..3271b71f1 --- /dev/null +++ b/components/lobbying/LobbyingAttribution.tsx @@ -0,0 +1,29 @@ +import React from "react" +import { useTranslation } from "next-i18next" +import { MAPLE_COLORS } from "./chartTheme" + +const SOS_URL = "https://www.sec.state.ma.us/LobbyistPublicSearch/Default.aspx" + +export const LobbyingAttribution: React.FC<{ className?: string }> = ({ + className +}) => { + const { t } = useTranslation("lobbying") + return ( +

+ + {t("attribution")} + +

+ ) +} + +const style: React.CSSProperties = { + fontSize: 12, + color: MAPLE_COLORS.textMuted, + marginTop: "1rem" +} + +const linkStyle: React.CSSProperties = { + color: MAPLE_COLORS.textMuted, + textDecoration: "underline" +} diff --git a/components/lobbying/LobbyingBillCard.tsx b/components/lobbying/LobbyingBillCard.tsx new file mode 100644 index 000000000..0cc138046 --- /dev/null +++ b/components/lobbying/LobbyingBillCard.tsx @@ -0,0 +1,254 @@ +import React from "react" +import { useTranslation } from "next-i18next" +import { useLobbyingFilingsForBill } from "components/db/lobbying" +import { LobbyingFilingsTable } from "./LobbyingFilingsTable" +import { MAPLE_COLORS } from "./chartTheme" +import { normalizePosition } from "./LobbyingPositionChip" + +interface LobbyingBillCardProps { + court: number + billId: string + className?: string +} + +export const LobbyingBillCard: React.FC = ({ + court, + billId, + className +}) => { + const { t } = useTranslation(["lobbying", "common"]) + const { + result: filings, + status, + error + } = useLobbyingFilingsForBill(court, billId) + + if (status === "loading" || status === "not-requested") { + return ( +
+

+ {t("lobbying:loading")} +

+
+ ) + } + + if (status === "error") { + if (process.env.NODE_ENV === "development") { + return ( +
+

+ Lobbying data error: {error?.message} +

+
+ ) + } + return null + } + + if (!filings || filings.length === 0) return null + + const counts = { support: 0, oppose: 0, neutral: 0, none: 0 } + for (const f of filings) { + const pos = normalizePosition(f.position) + counts[pos]++ + } + const total = filings.length + + const explorerHref = `/lobbying/bills/${court}/${billId}` + + return ( +
+
+ {t("lobbying:titles.overview")} + + {t("lobbying:billCard.filingCount_other", { count: total })} + +
+ + + +
+ + + +
+ + + + + {t("lobbying:billCard.viewAll")} + +
+ ) +} + +const PositionBar: React.FC<{ + counts: { support: number; oppose: number; neutral: number; none: number } + total: number +}> = ({ counts, total }) => { + if (total === 0) return null + const pct = (n: number) => `${((n / total) * 100).toFixed(1)}%` + return ( +
+ {counts.support > 0 && ( +
+ )} + {counts.oppose > 0 && ( +
+ )} + {counts.neutral > 0 && ( +
+ )} + {counts.none > 0 && ( +
+ )} +
+ ) +} + +const LegendItem: React.FC<{ + label: string + color: string + count: number +}> = ({ label, color, count }) => { + if (count === 0) return null + return ( + + + {label} ({count}) + + ) +} + +const cardStyle: React.CSSProperties = { + background: MAPLE_COLORS.surfaceBase, + border: `1px solid ${MAPLE_COLORS.borderDefault}`, + borderRadius: 8, + boxShadow: "0 0.25rem 1rem rgba(15, 23, 42, 0.06)", + padding: "1rem" +} + +const headerStyle: React.CSSProperties = { + display: "flex", + justifyContent: "space-between", + alignItems: "baseline", + marginBottom: "0.75rem" +} + +const titleStyle: React.CSSProperties = { + fontFamily: "Nunito, system-ui, sans-serif", + fontWeight: 700, + fontSize: 13, + color: MAPLE_COLORS.textMuted, + textTransform: "uppercase", + letterSpacing: "0.06em" +} + +const countStyle: React.CSSProperties = { + fontSize: 12, + color: MAPLE_COLORS.textMuted +} + +const barContainerStyle: React.CSSProperties = { + display: "flex", + height: 8, + borderRadius: 4, + overflow: "hidden", + marginBottom: "0.5rem", + background: MAPLE_COLORS.graySubtleBorder +} + +const barSegmentStyle: React.CSSProperties = { + height: "100%", + transition: "width 0.2s ease" +} + +const legendRowStyle: React.CSSProperties = { + display: "flex", + gap: "0.75rem", + marginBottom: "0.75rem", + flexWrap: "wrap" +} + +const legendItemStyle: React.CSSProperties = { + display: "flex", + alignItems: "center", + fontSize: 12, + color: MAPLE_COLORS.textMuted +} + +const viewAllLinkStyle: React.CSSProperties = { + display: "block", + fontSize: 13, + fontWeight: 600, + color: MAPLE_COLORS.primary, + textDecoration: "none", + marginTop: "0.5rem" +} diff --git a/components/lobbying/LobbyingFilingsTable.tsx b/components/lobbying/LobbyingFilingsTable.tsx new file mode 100644 index 000000000..e86236d0f --- /dev/null +++ b/components/lobbying/LobbyingFilingsTable.tsx @@ -0,0 +1,134 @@ +import React from "react" +import { Table } from "react-bootstrap" +import { useTranslation } from "next-i18next" +import type { LobbyingFiling } from "functions/src/lobbying/types" +import { LobbyingPositionChip } from "./LobbyingPositionChip" +import { MAPLE_COLORS } from "./chartTheme" + +interface LobbyingFilingsTableProps { + filings: LobbyingFiling[] + showBill?: boolean + showClient?: boolean + showFirm?: boolean + showAmount?: boolean + showActivity?: boolean + maxRows?: number + onViewAll?: () => void +} + +export const LobbyingFilingsTable: React.FC = ({ + filings, + showBill = false, + showClient = true, + showFirm = true, + showAmount = true, + showActivity = false, + maxRows, + onViewAll +}) => { + const { t } = useTranslation("lobbying") + const rows = maxRows ? filings.slice(0, maxRows) : filings + const truncated = maxRows != null && filings.length > maxRows + + if (filings.length === 0) { + return ( +

+ {t("noData")} +

+ ) + } + + return ( +
+ + + + {showBill && } + {showClient && } + {showFirm && } + {showActivity && } + + {showAmount && ( + + )} + + + + + {rows.map(f => ( + + {showBill && ( + + )} + {showClient && } + {showFirm && } + {showActivity && ( + + )} + + {showAmount && ( + + )} + + + ))} + +
{t("fields.bills")}{t("fields.clientName")}{t("fields.firmName")}{t("fields.activity")}{t("filters.position")}{t("fields.amount")}{t("fields.year")}
+ {f.billId ? ( + + {f.billId} + + ) : ( + — + )} + {f.clientName}{f.entityName} + {f.activityTitle || "—"} + + + + {f.amount != null + ? f.amount.toLocaleString("en-US", { + style: "currency", + currency: "USD", + maximumFractionDigits: 0 + }) + : "—"} + {f.year}
+ {truncated && onViewAll && ( + + )} +
+ ) +} + +const tableStyle: React.CSSProperties = { + fontSize: 13, + color: MAPLE_COLORS.textBody +} + +const theadRowStyle: React.CSSProperties = { + fontSize: 11, + fontWeight: 700, + textTransform: "uppercase", + letterSpacing: "0.05em", + color: MAPLE_COLORS.textMuted, + borderBottom: `2px solid ${MAPLE_COLORS.borderDefault}` +} + +const cellStyle: React.CSSProperties = { + verticalAlign: "middle", + padding: "0.4rem 0.5rem" +} + +const viewAllStyle: React.CSSProperties = { + background: "none", + border: "none", + color: MAPLE_COLORS.primary, + fontSize: 13, + fontWeight: 600, + cursor: "pointer", + padding: 0, + textDecoration: "underline" +} diff --git a/components/lobbying/LobbyingPaginationBar.tsx b/components/lobbying/LobbyingPaginationBar.tsx new file mode 100644 index 000000000..7c6254a45 --- /dev/null +++ b/components/lobbying/LobbyingPaginationBar.tsx @@ -0,0 +1,77 @@ +import React from "react" +import { useTranslation } from "next-i18next" +import { MAPLE_COLORS } from "./chartTheme" + +interface Props { + page: number + totalPages: number + totalItems: number + pageSize: number + onPage: (p: number) => void +} + +export function LobbyingPaginationBar({ + page, + totalPages, + totalItems, + pageSize, + onPage +}: Props) { + const { t } = useTranslation("lobbying") + if (totalPages <= 1) return null + const start = (page - 1) * pageSize + 1 + const end = Math.min(page * pageSize, totalItems) + + return ( +
+ + {start}–{end} of {totalItems} + +
+ + + {page} / {totalPages} + + +
+
+ ) +} + +const wrapStyle: React.CSSProperties = { + display: "flex", + justifyContent: "space-between", + alignItems: "center", + padding: "0.75rem 0 0.25rem", + flexWrap: "wrap", + gap: "0.5rem" +} + +const btnStyle = (disabled: boolean): React.CSSProperties => ({ + padding: "0.3rem 0.7rem", + border: `1px solid ${MAPLE_COLORS.borderDefault}`, + borderRadius: 4, + background: MAPLE_COLORS.surfaceBase, + color: disabled ? MAPLE_COLORS.textMuted : MAPLE_COLORS.textBody, + fontSize: 13, + cursor: disabled ? "default" : "pointer", + opacity: disabled ? 0.5 : 1 +}) diff --git a/components/lobbying/LobbyingPositionChip.tsx b/components/lobbying/LobbyingPositionChip.tsx new file mode 100644 index 000000000..c88b578b1 --- /dev/null +++ b/components/lobbying/LobbyingPositionChip.tsx @@ -0,0 +1,74 @@ +import React from "react" +import { useTranslation } from "next-i18next" +import { POSITION_COLORS } from "./chartTheme" + +type Position = "support" | "oppose" | "neutral" | "none" + +const CHIP_STYLES: Record = { + support: { + background: "#e8f6ea", + color: "#1d5d2d", + border: "1px solid #3d9922" + }, + oppose: { + background: "#fde8ef", + color: "#902141", + border: "1px solid #c71e32" + }, + neutral: { + background: "#f1f5f9", + color: "#475569", + border: "1px solid #cbd5e1" + }, + none: { + background: "#f1f5f9", + color: "#64748b", + border: "1px solid #d7e0ea" + } +} + +function normalizePosition(raw: string): Position { + const lower = raw.toLowerCase() + if (lower.includes("support")) return "support" + if (lower.includes("oppose") || lower.includes("against")) return "oppose" + if (lower.includes("neutral")) return "neutral" + return "none" +} + +interface LobbyingPositionChipProps { + position: string + className?: string +} + +export const LobbyingPositionChip: React.FC = ({ + position, + className +}) => { + const { t } = useTranslation("lobbying") + const key = normalizePosition(position) + + return ( + + {t(`position.${key}`)} + + ) +} + +const chipBaseStyle: React.CSSProperties = { + display: "inline-block", + padding: "0.15rem 0.55rem", + borderRadius: 999, + fontSize: 12, + fontWeight: 600, + letterSpacing: "0.01em", + whiteSpace: "nowrap" +} + +export { normalizePosition } +export type { Position } diff --git a/components/lobbying/LobbyingSubnav.tsx b/components/lobbying/LobbyingSubnav.tsx new file mode 100644 index 000000000..e6d6054d5 --- /dev/null +++ b/components/lobbying/LobbyingSubnav.tsx @@ -0,0 +1,73 @@ +import React from "react" +import { useRouter } from "next/router" +import { useTranslation } from "next-i18next" +import { Container } from "components/bootstrap" +import { MAPLE_COLORS } from "./chartTheme" + +export function LobbyingSubnav() { + const { t } = useTranslation("lobbying") + const { pathname } = useRouter() + + const LINKS = [ + { label: t("subnav.overview"), href: "/lobbying", exact: true }, + { label: t("sections.bills"), href: "/lobbying/bills" }, + { label: t("sections.clients"), href: "/lobbying/clients" }, + { label: t("sections.firms"), href: "/lobbying/firms" } + ] + + return ( +
+ +
+ {t("subnav.label")} + +
+ +
+ ) +} + +const barStyle: React.CSSProperties = { + background: "#fff", + borderBottom: `1px solid ${MAPLE_COLORS.borderDefault}`, + width: "100%" +} + +const innerStyle: React.CSSProperties = { + display: "flex", + alignItems: "center", + gap: "1.75rem", + padding: "0.5rem 0", + flexWrap: "wrap" +} + +const titleStyle: React.CSSProperties = { + fontSize: 12, + fontWeight: 600, + textTransform: "uppercase", + letterSpacing: "0.07em", + color: MAPLE_COLORS.textMuted, + whiteSpace: "nowrap" +} + +const linkStyle = (active: boolean): React.CSSProperties => ({ + color: active ? MAPLE_COLORS.primary : MAPLE_COLORS.textBody, + fontWeight: active ? 600 : 400, + textDecoration: "none", + fontSize: 14, + paddingBottom: "0.4rem", + borderBottom: `2px solid ${active ? MAPLE_COLORS.primary : "transparent"}`, + display: "inline-block", + transition: "color 0.1s, border-color 0.1s" +}) diff --git a/components/lobbying/chartTheme.tsx b/components/lobbying/chartTheme.tsx new file mode 100644 index 000000000..7e1aea0f8 --- /dev/null +++ b/components/lobbying/chartTheme.tsx @@ -0,0 +1,317 @@ +/** + * Recharts visual theme for MAPLE lobbying charts. + * + * All values are derived from the design tokens in styles/bootstrap.scss. + * CSS custom properties (var(--maple-*)) are not used here because Recharts + * renders into SVG/Canvas where CSS vars are not reliably resolved — hard-coded + * hex values that match the SCSS source of truth are the correct approach. + */ + +import React from "react" +import { TooltipContentProps } from "recharts" +import type { + NameType, + Payload, + ValueType +} from "recharts/types/component/DefaultTooltipContent" + +// ── Token mirrors ───────────────────────────────────────────────────────────── +// Keep in sync with styles/bootstrap.scss. + +export const MAPLE_COLORS = { + // Brand + primary: "#1a3185", + primaryStrong: "#12266f", + accent: "#ff8600", + danger: "#c71e32", + green: "#3d9922", + stagePast: "#8ebc81", + periwinkle: "#8c98c2", + + // Text + textBody: "#334155", + textMuted: "#64748b", + textStrong: "#1e293b", + + // Surfaces & borders + surfaceBase: "#ffffff", + surfacePage: "#eae7e7", + surfaceMuted: "#f8fafc", + borderDefault: "rgba(15, 23, 42, 0.15)", + borderSubtle: "rgba(15, 23, 42, 0.08)", + + // Semantic subtle tints + blueSubtleBg: "#e8efff", + blueSubtleText: "#1d3f8a", + greenSubtleBg: "#e8f6ea", + greenSubtleText: "#1d5d2d", + redSubtleBg: "#fde8ef", + redSubtleText: "#902141", + graySubtleBg: "#f1f5f9", + graySubtleText: "#475569", + graySubtleBorder: "#d7e0ea" +} as const + +// ── Data palettes ───────────────────────────────────────────────────────────── + +/** Ordered series palette: navy → orange → green → periwinkle → soft green → red */ +export const DATA_PALETTE: readonly string[] = [ + MAPLE_COLORS.primary, + MAPLE_COLORS.accent, + MAPLE_COLORS.green, + MAPLE_COLORS.periwinkle, + MAPLE_COLORS.stagePast, + MAPLE_COLORS.danger +] + +/** Lobbying position colours. Use across all position visualisations. */ +export const POSITION_COLORS = { + support: MAPLE_COLORS.green, + oppose: MAPLE_COLORS.danger, + neutral: MAPLE_COLORS.textMuted, + none: MAPLE_COLORS.graySubtleBorder +} as const + +export type LobbyingPosition = keyof typeof POSITION_COLORS + +// ── Axis / grid props ───────────────────────────────────────────────────────── +// Spread directly onto , , . + +const TICK_STYLE = { + fill: MAPLE_COLORS.textMuted, + fontSize: 12, + fontFamily: "Nunito, system-ui, sans-serif" +} + +export const X_AXIS_PROPS = { + tick: TICK_STYLE, + axisLine: { stroke: MAPLE_COLORS.borderDefault }, + tickLine: { stroke: MAPLE_COLORS.borderDefault }, + padding: { left: 4, right: 4 } as const +} + +export const Y_AXIS_PROPS = { + tick: TICK_STYLE, + axisLine: false as const, + tickLine: false as const, + width: 60 +} + +export const GRID_PROPS = { + strokeDasharray: "3 3", + stroke: MAPLE_COLORS.borderSubtle, + vertical: false +} + +// ── Line styles ─────────────────────────────────────────────────────────────── + +/** Standard line. Spread onto . */ +export const LINE_PROPS = { + strokeWidth: 2, + dot: false as const, + activeDot: { r: 5, strokeWidth: 2, stroke: MAPLE_COLORS.surfaceBase } +} + +/** Emphasis line (e.g. total / primary series on a dual-axis chart). */ +export const LINE_PROPS_EMPHASIS = { + ...LINE_PROPS, + strokeWidth: 2.5 +} + +/** Dashed reference or secondary line. */ +export const LINE_PROPS_DASHED = { + ...LINE_PROPS, + strokeDasharray: "5 3", + strokeWidth: 1.5 +} + +// ── Bar styles ──────────────────────────────────────────────────────────────── + +/** Rounded top corners matching $maple-radius-sm (4 px). Spread onto . */ +export const BAR_PROPS = { + radius: [4, 4, 0, 0] as [number, number, number, number], + maxBarSize: 48 +} + +/** Tighter bar for dense / stacked charts. */ +export const BAR_PROPS_COMPACT = { + radius: [2, 2, 0, 0] as [number, number, number, number], + maxBarSize: 32 +} + +// ── Area fill gradients ─────────────────────────────────────────────────────── + +/** + * Returns an SVG block defining a vertical gradient for each data series. + * Place inside the Recharts / and reference with + * `fill="url(#gradient-primary)"` on each . + * + * Usage: + * + * {AREA_GRADIENT_DEFS} + * + * + */ +export const AREA_GRADIENT_DEFS = ( + <> + {DATA_PALETTE.map((color, i) => ( + + + + + ))} + +) + +/** Convenience: gradient fill ID for the nth series in DATA_PALETTE. */ +export const gradientFill = (index: number) => `url(#maple-gradient-${index})` + +// ── Legend props ────────────────────────────────────────────────────────────── + +export const LEGEND_PROPS = { + wrapperStyle: { + fontSize: 12, + fontFamily: "Nunito, system-ui, sans-serif", + color: MAPLE_COLORS.textMuted, + paddingTop: 8 + }, + iconSize: 10, + iconType: "circle" as const +} + +// ── Tooltip ─────────────────────────────────────────────────────────────────── + +/** + * Drop-in replacement for Recharts' default tooltip. + * Matches MAPLE card styling: white surface, subtle border and shadow, Nunito. + * + * Usage: + * } /> + * `Year: ${d}`} />} /> + */ +export const MapleTooltip: React.FC< + TooltipContentProps & { + labelFormatter?: (label: string | number) => React.ReactNode + valueFormatter?: (value: ValueType) => string + } +> = ({ active, payload, label, labelFormatter, valueFormatter }) => { + if (!active || !payload?.length) return null + + const displayLabel = + labelFormatter && label != null ? labelFormatter(label) : label + const fmt = valueFormatter ?? ((v: ValueType | undefined) => String(v ?? "")) + + return ( +
+ {displayLabel != null && ( +
{displayLabel}
+ )} + {payload.map((entry: Payload, i: number) => ( +
+ + {entry.name} + + + {entry.value != null ? fmt(entry.value) : ""} + +
+ ))} +
+ ) +} + +const tooltipContainerStyle: React.CSSProperties = { + background: MAPLE_COLORS.surfaceBase, + border: `1px solid ${MAPLE_COLORS.borderDefault}`, + borderRadius: 8, + boxShadow: "0 0.5rem 1.5rem rgba(15, 23, 42, 0.06)", + padding: "0.5rem 0.75rem", + fontFamily: "Nunito, system-ui, sans-serif", + fontSize: 13, + color: MAPLE_COLORS.textBody, + minWidth: 140 +} + +const tooltipLabelStyle: React.CSSProperties = { + fontWeight: 700, + color: MAPLE_COLORS.textStrong, + marginBottom: "0.25rem", + fontSize: 12, + textTransform: "uppercase", + letterSpacing: "0.04em" +} + +const tooltipRowStyle: React.CSSProperties = { + display: "flex", + justifyContent: "space-between", + gap: "1rem", + lineHeight: 1.6 +} + +// ── Chart container ─────────────────────────────────────────────────────────── + +/** + * Wrapper div that gives a Recharts chart the same surface treatment as a + * MAPLE card: white background, subtle border, sm shadow, md border-radius. + * Accepts an optional title rendered above the chart area. + */ +export const ChartContainer: React.FC< + React.PropsWithChildren<{ + title?: string + height?: number | string + className?: string + style?: React.CSSProperties + }> +> = ({ title, height = 260, children, className, style }) => ( +
+ {title &&
{title}
} +
{children}
+
+) + +const chartContainerStyle: React.CSSProperties = { + background: MAPLE_COLORS.surfaceBase, + border: `1px solid ${MAPLE_COLORS.borderDefault}`, + borderRadius: 8, + boxShadow: "0 0.25rem 1rem rgba(15, 23, 42, 0.06)", + padding: "1rem 1rem 0.5rem" +} + +const chartTitleStyle: React.CSSProperties = { + fontFamily: "Nunito, system-ui, sans-serif", + fontWeight: 700, + fontSize: 13, + color: MAPLE_COLORS.textMuted, + textTransform: "uppercase", + letterSpacing: "0.06em", + marginBottom: "0.75rem" +} + +// ── Formatters ──────────────────────────────────────────────────────────────── + +const currencyFmt = new Intl.NumberFormat("en-US", { + style: "currency", + currency: "USD", + maximumFractionDigits: 0 +}) + +export const formatCurrency = (v: number): string => currencyFmt.format(v) + +const compactFmt = new Intl.NumberFormat("en-US", { + style: "currency", + currency: "USD", + notation: "compact", + maximumFractionDigits: 1 +}) + +/** Compact axis tick labels: $1.2M, $450K */ +export const formatCurrencyCompact = (v: number): string => compactFmt.format(v) + +export const formatCount = (v: number): string => v.toLocaleString("en-US") diff --git a/components/lobbying/lobbying.module.css b/components/lobbying/lobbying.module.css new file mode 100644 index 000000000..99137a73d --- /dev/null +++ b/components/lobbying/lobbying.module.css @@ -0,0 +1,5 @@ +@media (max-width: 767px) { + .mobileHide { + display: none !important; + } +} diff --git a/components/lobbying/usePagination.ts b/components/lobbying/usePagination.ts new file mode 100644 index 000000000..6e500220e --- /dev/null +++ b/components/lobbying/usePagination.ts @@ -0,0 +1,18 @@ +import { useMemo, useState } from "react" + +export function usePagination(items: T[], pageSize: number) { + const [page, setPage] = useState(1) + const totalPages = Math.max(1, Math.ceil(items.length / pageSize)) + const safePage = Math.min(page, totalPages) + const pageItems = useMemo( + () => items.slice((safePage - 1) * pageSize, safePage * pageSize), + [items, safePage, pageSize] + ) + return { + page: safePage, + setPage, + pageItems, + totalPages, + totalItems: items.length + } +} diff --git a/docs/lobbying-disclosure-ingestion.md b/docs/lobbying-disclosure-ingestion.md new file mode 100644 index 000000000..99c9652a7 --- /dev/null +++ b/docs/lobbying-disclosure-ingestion.md @@ -0,0 +1,1038 @@ +# Lobbying Disclosure Ingestion Pipeline + +## Overview + +The MA Secretary of State lobbying portal +([sec.state.ma.us/LobbyistPublicSearch](https://www.sec.state.ma.us/LobbyistPublicSearch/)) +publishes semi-annual disclosure filings for all registered lobbyists and +lobbying entities. This document describes the plan for scraping that data and +storing it in Firestore in a way that allows joining to MAPLE bill data. + +The portal has three levels of pages: + +1. **Search page** → one row per registrant per year +2. **Summary page** → registrant metadata + links to semi-annual disclosure + filings +3. **CompleteDisclosure page** → per-client compensation table + per-client bill + activity tables + +Historical data goes back to 2005. MAPLE has bill data only from ~2020 onward, +so bill joins will only resolve for filings from the 192nd General Court (2021) +and later. All historical filings are ingested regardless. + +--- + +## Terminology + +The portal has two registrant types: + +- **Lobbyist** — an individual person who lobbies directly on behalf of clients. +- **Employer** — a lobbying firm that employs individual lobbyists and is + retained by clients. Called "Lobbyist Entity" on the portal. + +In both cases, the registrant reports compensation received from each **client** +(the organization that hired them) and which bills they lobbied for that client. + +--- + +## Firestore Data Model + +Two top-level collections, normalized by registrant and by lobbying activity +record. + +### `/lobbyingRegistrants/{registrantId}` + +`registrantId` is a SHA-256 hash (first 40 hex chars) of +`{entityName}_{year}`. Hashing avoids sanitizing arbitrary Unicode and +punctuation in entity names to fit Firestore ID constraints while remaining +stable and dedup-safe across runs. + +One model covers both individual lobbyists and lobbying firms. A separate model +is not needed because the portal search returns both under the same schema, and +per-filing detail pages do not expose which individual lobbyists within a firm +worked on which bill. + +```typescript +interface LobbyingRegistrant { + registrantId: string // SHA-256 hash of "{entityName}_{year}" + entityName: string // firm name or individual lobbyist name (raw portal value) + entityNameNorm: string // normalized form; see Normalization section + year: number + generalCourt: number // computed from year + regType: "Lobbyist" | "Employer" + clients: LobbyingClient[] + disclosureUrls: string[] // source portal URLs, for audit trail + fetchedAt: Timestamp +} + +interface LobbyingClient { + clientName: string + clientNameNorm: string // normalized form + compensation: number | null +} +``` + +**Example document** (`lobbyingRegistrants/0f11970cc6f17e35cc02685b794cb63a655c13b3`): + +```json +{ + "registrantId": "0f11970cc6f17e35cc02685b794cb63a655c13b3", + "entityName": "27 South Strategies, LLC", + "entityNameNorm": "27 SOUTH STRATEGIES", + "year": 2024, + "generalCourt": 193, + "regType": "Employer", + "clients": [ + { + "clientName": "The Massachusetts International Festival of the Arts, Inc.", + "clientNameNorm": "MASSACHUSETTS INTERNATIONAL FESTIVAL OF ARTS", + "compensation": 24000.0 + }, + { + "clientName": "Veterinary Emergency Group, LLC", + "clientNameNorm": "VETERINARY EMERGENCY GROUP", + "compensation": 33000.0 + }, + { + "clientName": "Gobrands, Inc", + "clientNameNorm": "GOBRANDS", + "compensation": 60000.0 + } + ], + "disclosureUrls": [ + "https://www.sec.state.ma.us/LobbyistPublicSearch/CompleteDisclosure.aspx?sysvalue=hTpfuAXM..." + ] +} +``` + +### `/lobbyingFilings/{filingId}` + +`filingId` is a SHA-256 hash (first 40 hex chars) of the logical key +`{entityName}_{clientName}_{chamber}_{activityRef}_{generalCourt}`. + +```typescript +type LobbyingChamber = + | "House Bill" + | "Senate Bill" + | "House Docket" + | "Senate Docket" + | "Executive" // lobbying of executive branch agencies + | "Other" // catch-all for rare legacy codes (FY, CMR, etc.) + +interface LobbyingFiling { + filingId: string + entityName: string // raw portal value + entityNameNorm: string // normalized form + clientName: string // raw portal value; "_total_salary_" sentinel for pre-2013 + clientNameNorm: string // normalized form + year: number + generalCourt: number + chamber: LobbyingChamber + // For legislative chambers: the bill number string (e.g. "H1234", "HD56"). + // For Executive: the agency name. Not a bill reference. + billId: string | null + activityTitle: string // bill title (legislative) or meeting description (executive) + position: string // "Support" | "Oppose" | "Neutral" | etc.; empty for executive + amount: number | null // compensation allocated to this activity + fetchedAt: Timestamp +} +``` + +**Example documents** — query: `lobbyingFilings` where `generalCourt == 193` and `billId == "H1"`: + +```json +[ + { + "filingId": "0bac34faf2f624083daaaea57ee55f5364d2be29", + "entityName": "Christina Ascolillo", + "entityNameNorm": "CHRISTINA ASCOLILLO", + "clientName": "American Council of Engineering Companies of Massachusetts", + "clientNameNorm": "AMERICAN COUNCIL OF ENGINEERING COMPANIES OF MASSACHUSETTS", + "year": 2023, + "generalCourt": 193, + "chamber": "House Bill", + "billId": "H1", + "activityTitle": "An Act making appropriations for the Fiscal Year 2024...", + "position": "Neutral", + "amount": 0.0 + }, + { + "filingId": "109600251ca8fd279e8bb7562bc9d234c39f63a8", + "entityName": "Christina Ascolillo", + "entityNameNorm": "CHRISTINA ASCOLILLO", + "clientName": "St. Mary's Center for Women and Children, Inc.", + "clientNameNorm": "ST MARY S CENTER FOR WOMEN AND CHILDREN", + "year": 2023, + "generalCourt": 193, + "chamber": "House Bill", + "billId": "H1", + "activityTitle": "An Act making appropriations for the Fiscal Year 2024...", + "position": "Neutral", + "amount": 0.0 + } +] +``` + +### Constructing `billId` from Raw Portal Data + +The portal stores bill numbers as bare integers and records the chamber +separately. The `billId` field — which maps to `Bill.id` in MAPLE — is +constructed during ingest by combining chamber prefix and integer: + +| `chamber` | Prefix | Example raw | `billId` | +| --------------- | ------ | ----------- | -------- | +| `House Bill` | `H` | `1234` | `H1234` | +| `Senate Bill` | `S` | `1234` | `S1234` | +| `House Docket` | `HD` | `56` | `HD56` | +| `Senate Docket` | `SD` | `56` | `SD56` | +| `Executive` | — | agency name | `null` | +| `Other` | — | varies | `null` | + +Note: `H1234` and `S1234` are distinct bills even though they share the same +integer. The prefix is required to disambiguate. `billId` is `null` for +non-legislative chambers. + +#### Legacy chamber code normalization + +The portal uses short-form codes in older filings, normalized during ingest: + +| Raw value | Stored as | +| --------- | ------------- | +| `HB` | `House Bill` | +| `SB` | `Senate Bill` | + +Rare codes (`FY`, `C`, `CMR`, `HR`, etc.) are stored as `Other`. + +### Joining to Bill Data + +**The join only applies to legislative chambers** (`House Bill`, `Senate Bill`, +`House Docket`, `Senate Docket`) where `billId` is non-null. For `Executive` +and `Other`, no join should be attempted. + +```typescript +// Only valid when filing.billId !== null +db.collection(`/generalCourts/${filing.generalCourt}/bills`).doc(filing.billId) +``` + +--- + +## Entity Name Normalization + +The portal does not enforce consistent name formatting. The same client or +registrant may appear as "Acme Corp.", "ACME CORPORATION", "Acme, Inc. d/b/a +Acme Consulting", etc. across filings and years. Without normalization, +grouping by entity is unreliable. + +Both `entityName` and `clientName` are normalized using the following pipeline, +applied in order. The raw portal value is always preserved alongside the +normalized form. + +### Normalization pipeline + +1. **Uppercase** — convert the entire string to upper case. +2. **Strip d/b/a suffix** — remove everything from the first occurrence of + `D/B/A`, `D/B/A`, `DBA` (and spacing variants) onward, so the registered + name is used rather than a trade name. +3. **Hyphen → space** — replace `-` with ` ` so `LAN-TEL` and `LAN TEL` + collapse to the same key. +4. **Punctuation → space** — replace `,`, `.`, `'`, `'`, `'`, `(`, `)` with + space. Replacement with space (not empty string) prevents adjacent tokens + from concatenating (e.g. `,INC` becomes ` INC`, which is then caught by step + 5). +5. **Remove legal entity type words** — whole-word removal of: `LLC`, `LLP`, + `INC`, `INCORPORATED`, `CORPORATION`, `CORP`, `LTD`, `LIMITED`, `PC`, + `PLLC`. +6. **Remove "THE"** — whole-word removal anywhere in the string (not just as a + leading prefix). +7. **Ampersand → AND** — replace `&` with `AND`. +8. **Fix known typo** — replace `ASSICIATES` with `ASSOCIATES` (legacy portal + data). +9. **Remove professional suffix phrases** — whole-phrase removal of: `LAW +OFFICE OF`, `AND ASSOCIATES`, `& ASSOCIATES`, `AND ASSOC`, `ATTORNEY AT +LAW`, `ATTORNEY@LAW`, `ATTORNET AT LAW`, `AND PARTNERS`, `PUBLIC POLICY +GROUP`, `LEGISLATIVE SERVICES`, `POLICY GROUP`, `ASSOCIATES`, `COUNSELLORS +AT LAW`. +10. **Collapse whitespace** — replace runs of whitespace with a single space and + strip leading/trailing whitespace. + +### Usage + +`entityNameNorm` and `clientNameNorm` are stored on every document and filing. +They should be used for grouping, deduplication, and display-level matching. +Raw names are preserved for provenance and audit. + +--- + +## Deduplication and Amount Aggregation + +### Does lobbying the same bill multiple times mean we should sum amounts? + +The portal collects two semi-annual disclosure filings per registrant per year +(one for each 6-month period). In theory, a registrant could report the same +bill in both H1 and H2 filings with separate compensation amounts that should +be summed. Analysis of the actual data shows this does not occur: after +processing, zero rows share the same `(entityName, clientName, year, +generalCourt, billId, position)` — each (registrant, client, bill, year) +combination appears exactly once. The semi-annual periods report different +activity, not the same activity twice. + +The same registrant can lobby the same bill across multiple General Courts +(observed up to 6 times across years). These are stored as separate documents +per `generalCourt` and should not be summed — each court is a distinct +legislative session. + +### Null-bill row deduplication + +The one real duplication artifact in the portal data is **null-bill rows** — +entries filed when a registrant had no specific bills to report for a client in +a period. These appear in both the H1 and H2 disclosures as identical rows and +should be collapsed. During ingest, if the same `(entityName, clientName, year, +generalCourt, chamber, position)` with a null `billId` is encountered more than +once, keep the row with the highest `amount` so no spend is lost if the two +copies carry different values (in practice amounts are usually both zero). + +### Ingest strategy + +When processing multiple disclosure URLs for the same registrant+year, write +`lobbyingFilings` documents using the logical key as the document ID. A +subsequent disclosure URL that produces the same document ID will naturally +upsert (overwrite) rather than duplicate. For null-bill rows, since `billId` is +null, include `chamber` in the document ID to avoid false merges between +executive and legislative null rows. + +--- + +## Scraper Architecture + +### Why a standalone Cloud Run container + +The MA SoS portal is protected by Imperva WAF, which uses TLS fingerprinting to +classify HTTP clients at the network layer before examining any headers. Node.js +produces a TLS fingerprint that Imperva challenges with a JavaScript +verification page; Python's `requests` library produces a fingerprint that +Imperva allows through without challenge. This is a runtime-level constraint +that cannot be addressed by header configuration or cipher reordering alone. + +The scraper therefore runs as a standalone **Cloud Run container** written in +Python, deployed alongside the existing MCP server container. All data modeling, +Firestore collection/field names, and normalization logic are documented here and +kept consistent between the Python container and the TypeScript type definitions +in `functions/src/lobbying/types.ts`. + +### Cloud Run container: `lobbying-scraper/` + +**Files:** `lobbying-scraper/{scrape,portal,normalize,writer}.py` + +- Scheduled weekly by Cloud Scheduler +- Runs an incremental check: fetches the current and prior year's summary links + (one POST), compares disc URLs against the Firestore cursor, and **exits + immediately if nothing is new** (fast path, typically seconds) +- When new or updated disclosures are found, fetches and processes them +- Persists a cursor in `scrapers/lobbying`: + - `processedDiscUrls: string[]` — disc URLs already written; skipped on + re-runs + - `summaryDiscCache: {[summaryUrl]: string[]}` — maps summary page URLs to + their disc URLs so summary page GETs are skipped for prior-year registrants + whose disclosures are all already processed +- For each new disclosure URL: + - Parse registrant + client compensation rows → upsert `lobbyingRegistrants` + - Parse bill activity rows → batch-write `lobbyingFilings` +- 1s delay between requests; exponential backoff on transient failures + +### Incremental strategy + +In steady state (after the initial backfill), each weekly run: + +1. One POST to fetch all summary links for current + prior year +2. For prior-year registrants with all disc URLs in the cursor: zero GETs +3. For current-year registrants: one GET per summary page to check for new + disclosure periods +4. For any new disc URLs: one GET per disclosure page + +New filings arrive twice a year (semi-annual reporting periods). Between +periods, the run completes in under a minute. + +The backfill script (`--mode backfill`) uses a separate subcollection cursor at +`scrapers/lobbyingBackfill/processedUrls/{urlHash}` so it does not interfere +with the live scraper state. + +### Portal HTML format eras + +The `CompleteDisclosure.aspx` page has changed layout several times. The parser +handles all four eras: + +| Era | Years | Compensation source | Notes | +| -------- | ------------ | ----------------------------------------------------- | --------------------------------------------------------------------------------------------------- | +| Modern | 2019–present | `grdvClientPaidToEntity` grid | Per-client rows; both employer and individual registrant variants | +| Hybrid | 2014–2018 | `Panel1_N` div blocks | Compensation in collapsible panels; bill activity in a separate grid | +| Legacy B | 2009–2013 | "Compensation received" column in bill-activity table | Per-client amounts; deduplication required (same (client, amount) pair can appear in multiple rows) | +| Legacy A | 2005–2008 | `grdvSalaryPaid` entity-total row | No per-client breakdown; stored under sentinel `clientName: "_total_salary_"` | + +Pre-2009 **individual** lobbyist summary pages link to `RegVersionLobbyist.aspx` +rather than `CompleteDisclosure.aspx`. These produce no disclosure detail and are +skipped — expected portal behavior. Employer/entity registrants use +`CompleteDisclosure.aspx` correctly across all years. + +For Legacy A filings (2005–2008), `clientName: "_total_salary_"` signals to +callers that per-client compensation is unavailable. No bill-level compensation +amount is available for these years. + +--- + +## New Files + +``` +functions/src/lobbying/ + types.ts — Runtypes schema definitions for LobbyingRegistrant, LobbyingFiling + normalize.ts — Entity name normalization pipeline (also used client-side) + index.ts — Re-exports + +lobbying-scraper/ + scrape.py — Entry point: --mode weekly (incremental) | --mode backfill + portal.py — HTTP fetch wrappers + pure HTML parsers for all 4 format eras + normalize.py — Port of normalize.ts + writer.py — Firestore document construction + writes + archive.py — GCS raw-HTML archive (write-only; enabled by ARCHIVE_RAW=1) + reparse_archive.py — Offline driver to re-ingest archived HTML into Firestore + requirements.txt — requests, beautifulsoup4, google-cloud-firestore, google-cloud-storage + Dockerfile — Python 3.12-slim image +``` + +The TypeScript lobbying module (`functions/src/lobbying/`) contains only the +schema types and normalization logic. There is no TypeScript scraper or +Firebase Function — ingestion is handled entirely by the Cloud Run container. +This follows the same pattern as the MCP server and avoids the complexity of +running multiple language runtimes in the same Firebase Functions deployment. + +--- + +## Deploying the Cloud Run Container + +Follows the same pattern as the MCP server. The Artifact Registry repo +(`maple-lobbying`) and Cloud Run job (`maple-lobbying-scraper`) are already +created in `digital-testimony-dev`. + +```bash +cd lobbying-scraper +IMAGE=us-central1-docker.pkg.dev/digital-testimony-dev/maple-lobbying/scraper:latest +docker build -t $IMAGE . && docker push $IMAGE + +gcloud run jobs update maple-lobbying-scraper \ + --image=$IMAGE \ + --project=digital-testimony-dev \ + --region=us-central1 +``` + +For a new project (prod), create the job first: + +```bash +gcloud artifacts repositories create maple-lobbying \ + --repository-format=docker --location=us-central1 --project= + +gcloud run jobs create maple-lobbying-scraper \ + --image=$IMAGE \ + --project= \ + --region=us-central1 \ + --task-timeout=30m \ + --max-retries=0 + +# Schedule weekly (Mondays 6am UTC) +gcloud scheduler jobs create http maple-lobbying-weekly \ + --schedule="0 6 * * 1" \ + --uri="https://us-central1-run.googleapis.com/apis/run.googleapis.com/v1/namespaces//jobs/maple-lobbying-scraper:run" \ + --http-method=POST \ + --oauth-service-account-email=@.iam.gserviceaccount.com \ + --location=us-central1 +``` + +## Historical Backfill + +Runs `scrape.py --mode backfill` directly. Resumable — the subcollection +cursor at `scrapers/lobbyingBackfill/processedUrls` tracks progress. + +Always set `ARCHIVE_RAW=1` for real runs so every fetched page is preserved +for offline reparsing. Create the GCS archive bucket first if it does not +exist (see Step 7 of the test plan). + +```bash +cd lobbying-scraper + +# Test a single year with no writes +GOOGLE_CLOUD_PROJECT=digital-testimony-dev \ + GOOGLE_APPLICATION_CREDENTIALS=~/.config/gcloud/application_default_credentials.json \ + python3 scrape.py --mode backfill --year 2024 --limit 3 --dry-run + +# Run a single year for real (with archiving) +GOOGLE_CLOUD_PROJECT=digital-testimony-dev \ + GOOGLE_APPLICATION_CREDENTIALS=~/.config/gcloud/application_default_credentials.json \ + ARCHIVE_RAW=1 \ + python3 scrape.py --mode backfill --year 2024 + +# Full history (2005–present, resumable, with archiving) +GOOGLE_CLOUD_PROJECT=digital-testimony-dev \ + GOOGLE_APPLICATION_CREDENTIALS=~/.config/gcloud/application_default_credentials.json \ + ARCHIVE_RAW=1 \ + python3 scrape.py --mode backfill +``` + +--- + +## Firestore Rules + +Add read-only public rules alongside the existing `generalCourts` rule: + +``` +match /lobbyingRegistrants/{doc} { allow read: if true; } +match /lobbyingFilings/{doc} { allow read: if true; } +``` + +--- + +## Firestore Indexes + +Add composite indexes for common query patterns: + +| Collection | Fields | Use case | +| ----------------- | -------------------------------------- | ---------------------------------------- | +| `lobbyingFilings` | `generalCourt ASC, billId ASC` | Fetch all legislative filings for a bill | +| `lobbyingFilings` | `generalCourt ASC, chamber ASC` | Filter by chamber within a court | +| `lobbyingFilings` | `generalCourt ASC, entityNameNorm ASC` | Fetch all filings for a registrant | +| `lobbyingFilings` | `generalCourt ASC, clientNameNorm ASC` | Fetch all filings for a client | + +Note: bill-join queries should always filter on `chamber` (or check +`billId !== null`) to exclude `Executive` and `Other` rows before treating +`billId` as a MAPLE bill reference. + +--- + +## Implementation Status + +| File | Status | Notes | +| ------------------------------------- | ---------- | ---------------------------------------------------------------- | +| `functions/src/lobbying/types.ts` | ✅ Done | Firestore schema types; imported by future frontend code | +| `functions/src/lobbying/normalize.ts` | ✅ Done | Normalization pipeline; also ported to `normalize.py` | +| `functions/src/lobbying/index.ts` | ✅ Done | Re-exports types and normalize | +| `firestore.rules` | ✅ Done | | +| `firestore.indexes.json` | ✅ Done | | +| `lobbying-scraper/normalize.py` | ✅ Done | Port of normalize.ts | +| `lobbying-scraper/portal.py` | ✅ Done | All 4 format eras; pure parsers separated from fetch wrappers | +| `lobbying-scraper/writer.py` | ✅ Done | Firestore document construction | +| `lobbying-scraper/scrape.py` | ✅ Done | Entry point; `--mode weekly` and `--mode backfill` | +| `lobbying-scraper/archive.py` | ✅ Done | GCS write-only archive; enabled via `ARCHIVE_RAW=1` | +| `lobbying-scraper/reparse_archive.py` | ✅ Done | Offline reparse driver; looks up registrant meta from Firestore | +| `lobbying-scraper/Dockerfile` | ⚠️ Rebuild | Needs rebuild after `google-cloud-storage` added to requirements | + +### Document ID scheme + +Both `registrantId` and `filingId` are SHA-256 hashes (first 40 hex chars) of +their respective logical keys. Hashes are used rather than slugified strings +because entity names and client names contain arbitrary Unicode and punctuation +that would require aggressive sanitization to fit Firestore ID constraints. The +hash is stable across runs for the same logical record. + +--- + +## Future Work (Subsequent PRs) + +### Frontend + +- **Dedicated lobbying pages** + + - `/lobbyists` index: searchable list of registrants with total compensation, + client count, and year filter + - `/lobbyists/{registrantId}` profile: full client list, all bills lobbied, + compensation over time + - `/clients/{clientNameNorm}` profile: registrants hired, bills lobbied, + total spend per year + +- **Bill page integration** (`/bills/{court}/{billId}`) + + - "Lobbying activity" section listing registrants + clients that lobbied this + bill, with position (Support / Oppose / Neutral) and compensation where + available + - Link to registrant profile pages + +- **Organization profile page integration** + - If an organization's normalized name matches a `clientNameNorm` in + `lobbyingFilings`, surface a "Lobbying history" panel showing which bills + they lobbied and which registrants they hired + +### MCP Tools + +Expose lobbying data via the MAPLE MCP server so that AI agents and Claude can +answer questions like "who lobbied bill H1234?" or "what did Acme Corp lobby +for in 2024?". + +- **`get_lobbying_filings_for_bill`** — given `generalCourt` + `billId`, return + all `lobbyingFilings` for that bill with registrant, client, position, and + amount +- **`get_lobbying_registrant`** — given `registrantId`, return the registrant + document with client list and disclosure URLs +- **`search_lobbying_by_client`** — given a client name (raw or normalized), + return matching filings across all courts +- **`get_lobbying_summary_for_bill`** — aggregate view: unique registrant count, + unique client count, total compensation (where non-null), position breakdown + +--- + +## Incremental Test Plan + +Testing proceeds from the inside out: unit logic first, then live portal +fetches against the real site, then a small Firestore write, then a full +backfill year, then steady-state function operation. + +### Step 1 — Unit tests: parser, normalization, bill construction + +Run the pytest suite against all 4 HTML format eras using committed fixture +pages: + +```bash +cd lobbying-scraper +python -m pytest tests/ -v +``` + +Expected: 26 tests pass. Covers compensation totals and client/bill counts for +all eras (2007, 2011, 2016, 2024 — employer and individual), normalization edge +cases, bill ID construction, and specific bug regressions (semicolon bill +separator, "Total amount" artifact row, null billId for executive rows). + +### Step 2 — Live portal fetch: dry run + +Verify the portal is reachable and the parser returns valid data without writing +to Firestore: + +```bash +cd lobbying-scraper +GOOGLE_CLOUD_PROJECT=digital-testimony-dev \ + GOOGLE_APPLICATION_CREDENTIALS=~/.config/gcloud/application_default_credentials.json \ + python3 scrape.py --mode backfill --year 2024 --limit 3 --dry-run +``` + +Expected: 3 registrants fetched, compensation and bill rows printed, no +Firestore writes. + +### Step 3 — Firestore write: single year, small limit + +Write a small batch to the dev project and verify results: + +```bash +cd lobbying-scraper +GOOGLE_CLOUD_PROJECT=digital-testimony-dev \ + GOOGLE_APPLICATION_CREDENTIALS=~/.config/gcloud/application_default_credentials.json \ + python3 scrape.py --mode backfill --year 2024 --limit 3 +``` + +Verify in Firestore console: + +- `lobbyingRegistrants` has 3 documents with `entityName`, `entityNameNorm`, + `regType`, `clients`, `generalCourt` +- `lobbyingFilings` has documents with `billId` non-null for legislative rows + and `null` for Executive rows +- `scrapers/lobbyingBackfill/processedUrls` has entries with `url` and + `processedAt` fields +- Re-running skips already-processed URLs (output shows 0 new disclosures) + +### Step 4 — Spot-check: bill join + +Pick a `lobbyingFiling` document with a non-null `billId` and a `generalCourt` +≥ 192. Verify the bill exists in MAPLE: + +``` +/generalCourts/{filing.generalCourt}/bills/{filing.billId} +``` + +If the bill is found, the join key is correct. If not found, check: (a) whether +MAPLE has data for that court, (b) whether the bill number format matches +(prefix + integer, no leading zeros). + +### Step 5 — Create GCS archive bucket (once per project) + +The archive bucket name is derived from `GOOGLE_CLOUD_PROJECT`. Create it once +with the Archive storage class (written once, read rarely): + +```bash +gsutil mb -p digital-testimony-dev -l us-central1 \ + -c archive gs://digital-testimony-dev-lobbying-archive +``` + +Repeat for prod when running the prod backfill: + +```bash +gsutil mb -p digital-testimony-prod -l us-central1 \ + -c archive gs://digital-testimony-prod-lobbying-archive +``` + +Each project uses its own bucket. Dev and prod data are isolated; the Cloud Run +service account for each project only needs access to its own bucket. + +### Step 6 — Backfill: full current year (or partial across all years) + +Always run with `ARCHIVE_RAW=1` so every fetched page is preserved for offline +reparsing. For a large-scale dev test before a full prod run, `--limit N` is +applied per year — e.g. `--limit 50` across all years fetches ~10% of history: + +```bash +cd lobbying-scraper + +# Full current year +GOOGLE_CLOUD_PROJECT=digital-testimony-dev \ + GOOGLE_APPLICATION_CREDENTIALS=~/.config/gcloud/application_default_credentials.json \ + ARCHIVE_RAW=1 \ + python3 scrape.py --mode backfill --year 2024 + +# ~10% partial across all years (good large-scale dev validation) +GOOGLE_CLOUD_PROJECT=digital-testimony-dev \ + GOOGLE_APPLICATION_CREDENTIALS=~/.config/gcloud/application_default_credentials.json \ + ARCHIVE_RAW=1 \ + python3 scrape.py --mode backfill --limit 50 +``` + +Expected for full year: ~500–600 registrants, ~1,000 disclosure pages, several +thousand filing documents written. + +### Step 7 — Backfill: full history (2005–present) + +Run without `--year` to process all years. Can be interrupted and resumed: + +```bash +GOOGLE_CLOUD_PROJECT=digital-testimony-dev \ + GOOGLE_APPLICATION_CREDENTIALS=~/.config/gcloud/application_default_credentials.json \ + ARCHIVE_RAW=1 \ + python3 scrape.py --mode backfill +``` + +Expected runtime: several hours at ~1s/request. The subcollection cursor at +`scrapers/lobbyingBackfill/processedUrls` allows safe interruption and +resumption. + +### Step 8 — Deploy and verify Cloud Run scraper + +Build and push the container image, then update the Cloud Run job: + +```bash +cd lobbying-scraper +IMAGE=us-central1-docker.pkg.dev/digital-testimony-dev/maple-lobbying/scraper:latest +docker build -t $IMAGE . && docker push $IMAGE + +gcloud run jobs update maple-lobbying-scraper \ + --image=$IMAGE \ + --project=digital-testimony-dev \ + --region=us-central1 +``` + +Trigger a manual run via the Cloud Run console or: + +```bash +gcloud run jobs execute maple-lobbying-scraper \ + --project=digital-testimony-dev \ + --region=us-central1 +``` + +Verify via Cloud Run logs: the run should find near-zero new disclosures if the +backfill already completed, confirming the weekly fast-path works correctly. + +--- + +## Design Decisions + +| Decision | Choice | Rationale | +| --------------------------- | ---------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Collection placement | Top-level `/lobbyingRegistrants`, `/lobbyingFilings` | Lobbying data spans multiple General Courts and is not scoped to a single court like bills/members | +| Single registrant model | One type, `regType: "Lobbyist" \| "Employer"` | Individual lobbyists and firms share the same portal schema; per-bill individual attribution is not available | +| `billId` construction | `{chamberPrefix}{billNumber}` at ingest time | Raw portal data stores chamber and integer separately; the composite is what matches MAPLE's `Bill.id` | +| `billId` null for Executive | `null` instead of agency name | Prevents accidental bill lookups; makes join guard explicit at the type level | +| Normalized name fields | Store both raw and `*Norm` fields | Raw names preserved for provenance; normalized names used for grouping and matching | +| HTML parser | `beautifulsoup4` (Python) | Runs in the Cloud Run container alongside the scraper; no JavaScript runtime needed | +| Live scraper cursor | Array in `/scrapers/lobbying` doc | ~1,000 URLs/year fits well within the 1 MB Firestore doc limit; simple and atomic with other scraper state | +| Backfill cursor | Firestore subcollection `/scrapers/lobbyingBackfill/processedUrls/{urlHash}` | Full 2005-present history (~50,000 URLs) would exceed the 1 MB doc limit; subcollection scales without bound and is durable, inspectable, and resumable from any machine | +| Incremental strategy | Skip already-processed disclosure URLs; write docs by logical key (upsert) | Survives function restarts and re-runs without re-fetching already-scraped pages; natural upsert prevents duplicates without an explicit dedup pass | +| Legacy format (pre-2013) | Store with `clientName: "_total_salary_"` sentinel | Preserves data completeness; callers can filter on this value | +| Historical data | Admin backfill script (2005 → present) | Full history is ingested once; Cloud Function maintains current+prior year going forward | + +--- + +## Appendix: Phase 2 — OCPF Contribution Ingestion + +This appendix tracks planned additions to the lobbying pipeline in a subsequent +PR. The changes link OCPF (Office of Campaign and Political Finance) campaign +contribution records to lobbying registrants, enabling questions like "how much +did this firm contribute to politicians while lobbying on this bill?" + +### Background + +The MA Secretary of State portal (Phase 1, implemented) covers who lobbied which +bills and for how much compensation. OCPF covers who donated to which political +candidates. Linking the two surfaces the intersection: lobbying firms that also +made political contributions. + +OCPF publishes bulk data files at +`ocpf2.blob.core.windows.net/downloads/data2/` (the same host used by +`matchOcpfMembers.ts` for the filers file). The contribution records file URL +needs to be confirmed before implementation begins. + +### New Data + +#### `/lobbyingContributionRecipients/{recipientId}` + +One document per deduplicated contribution recipient (politician/PAC). The +deduplication pipeline is described below. + +```typescript +interface LobbyingContributionRecipient { + recipientId: string // SHA-256(recipientName + officeSought)[:40] + recipientName: string // canonical display name after dedup + recipientSlug: string // slugify("{recipientName} {officeSought}") + officeSought: string // canonical office value (see normalization below) + totalReceived: number + nContributions: number + years: number[] + nameVariants: string[] // all raw names that resolved to this record + manuallyMerged: boolean // true if recipient_merges.json was applied + topFirms: [string, string, number][] // [entityName, entityNameNorm, amount] + fetchedAt: Timestamp +} +``` + +#### New fields on `LobbyingRegistrant` (optional, written by contribution run) + +```typescript +nBills?: number // total filing rows across all clients and years +nContributions?: number // total OCPF contribution records from this registrant +totalContributions?: number // total dollar amount contributed +``` + +`nBills` is computed during the disclosure scrape (bill rows are already in +memory) and written as a side effect of `--mode weekly` and `--mode backfill`. +`nContributions` and `totalContributions` are written by `--mode contributions`. + +#### Pure-contribution firms (new registrant subtype) + +Firms that made OCPF contributions but have zero lobbying activity (no filings) +are written as `LobbyingRegistrant` documents with `nBills: 0`, `clients: []`, +`totalContributions ≥ 500`. These were previously invisible. The $500 threshold +filters noise; adjust after reviewing the data. + +### Contribution Recipient Deduplication + +The same politician may appear under many name variants in OCPF data ("Joe +Curtatone", "Joseph Curtatone", "Curtatone"). The dedup pipeline runs in five +passes in-memory after loading all contribution records. + +All rules are systemic (handle a class of inputs); individual special cases are +handled via `recipient_merges.json` only. + +#### New file: `lobbying-scraper/recipient_normalize.py` + +**Step 1 — `_extract_recipient_name(raw)`** + +Strips committee wrapper names using 14 regex patterns (e.g. "Cte. to Elect Ron +Mariano" → "Ron Mariano"). Then applies in order: + +- Last-first flip: `"Mariano, Ronald"` → `"Ronald Mariano"` +- Honorific strip: `"Sen. Marc Rodrigues"` → `"Marc Rodrigues"` +- Party label strip: `"Ron Mariano Democrat"` → `"Ron Mariano"` +- Initial-dot prefix strip: `"R.Mariano"` → `"Mariano"` + +**Step 2 — `_recipient_dedup_key(name, office)` → `(first, last, office)`** + +Key is a `(first_token, last_token, canonical_office)` tuple. Before keying, +`first_token` is passed through `_NICKNAME_MAP`: + +| Raw | Canonical | +| ---------------- | --------- | +| joe | joseph | +| mike | michael | +| bob | robert | +| bill | william | +| jim | james | +| tom | thomas | +| dan | daniel | +| dave | david | +| steve | steven | +| ron | ronald | +| tim | timothy | +| rick, rich, dick | richard | +| charlie, chuck | charles | +| ben | benjamin | +| tony | anthony | +| marty | martin | +| don | donald | +| ken | kenneth | +| ed | edward | +| ray | raymond | +| nick | nicholas | + +If either token is a generic word (`the`, `comm`, `committee`, `democratic`, +`republican`, `house`, `senate`, `friends`, `cte`, `pac`), the key falls back +to the full canonical text. + +**Step 3 — Manual merges (`recipient_merges.json`)** + +JSON file maps `(name, office)` alias pairs to a canonical record. Start as an +empty object `{}`; populate incrementally as data review finds misses. Applied +after key assignment. + +**Step 4 — `_merge_bare_surnames(groups)`** + +Single-name entries where `first == last` (e.g. `"Curtatone"`) are merged into +the highest-total multi-token peer with the same `(last, office)`. + +**Step 5 — `_fuzzy_merge_keys(groups)`** + +Groups keys by `(first, office)`. Pairs where last tokens differ by Levenshtein +distance ≤ 1 are merged into the higher-total key. Minimum 6 chars on both last +tokens (protects short surnames: Walsh, Jones, etc.). + +### Office Normalization — `_normalize_office(raw)` in `recipient_normalize.py` + +Canonical office values: + +`State Representative` · `State Senator` · `Governor` · `Lt. Governor` · +`Attorney General` · `Treasurer` · `Auditor` · `Secretary of State` · `Mayor` · +`City Council` · `District Attorney` · `Sheriff` · `PAC` + +Normalization pipeline (applied in order): + +1. Strip `"Candidate for [Massachusetts] X"` prefix → `"X"` +2. Strip leading `MA` / `Mass.` / `Massachusetts` +3. Strip trailing junk punctuation (`:;/`) +4. Strip parenthetical suffixes +5. Strip `"of the …"` / `"from the …"` suffixes +6. Strip after dash or slash +7. Strip after comma +8. Strip ordinal suffixes (e.g. `" 3rd Norfolk …"`) +9. Re-strip trailing junk (ordinal strip can leave dangling colons) +10. `OFFICE_MAP` lookup (canonical string → canonical value) +11. Fallback — district lookup: ordinal + MA county → `State Representative` +12. Fallback — county suffix: `"Representative Suffolk"` → strip county → retry map +13. Fallback — location qualifier: strip `"of "`, trailing word, or leading + word → retry map (handles `"Mayor of Somerville"`, `"Somerville Mayor"`, + `"Mayor Somerville"` → `"Mayor"`) + +### New Files + +``` +lobbying-scraper/ + ocpf_contributions.py — OCPF bulk data download, parse, match to registrants + recipient_normalize.py — office normalization + 5-step recipient dedup pipeline + recipient_merges.json — manual merge overrides (start as {}) +``` + +### Infrastructure Changes + +**`firestore.rules`** — add: + +``` +match /lobbyingContributionRecipients/{id} { allow read: if true; allow write: if false; } +``` + +**`firestore.indexes.json`** — add: + +| Collection | Fields | Use case | +| -------------------------------- | -------------------------------------- | --------------------------- | +| `lobbyingContributionRecipients` | `officeSought ASC, totalReceived DESC` | Office-filtered leaderboard | +| `lobbyingContributionRecipients` | `officeSought ASC, recipientName ASC` | Alphabetical browse | + +### What Is Not Implemented (Phase 2) + +**AI bill topics (`tags.json` equivalent)** — requires a bill embeddings parquet +file. MAPLE does not produce this file. The topics feature is deferred; MAPLE's +existing bill search index could be used as a substitute in a future PR. + +**`recipient_merges.json`** — starts empty. Requires domain review of the +deduplicated output to find cases the automated rules miss. Populate +incrementally. + +### Order of Work + +1. Confirm OCPF bulk contribution file URL and column schema +2. Confirm or incorporate HTML archiving pattern (see open question below) +3. Implement `recipient_normalize.py` (self-contained, testable in isolation) +4. Implement `ocpf_contributions.py` +5. Update `functions/src/lobbying/types.ts` schema +6. Update `writer.py` and `scrape.py` +7. Update `firestore.rules` and `firestore.indexes.json` + +### HTML Archiving for Fast Reparsing + +The current scraper fetches and immediately parses SoS portal HTML, storing +nothing but the parsed Firestore documents. If parsing logic changes, the full +portal must be re-scraped — slow and fragile given the Imperva TLS constraint. +Phase 2 adds GCS archiving of raw HTML as a side effect of every portal fetch. + +#### Design principles + +The archive is **write-only cold storage**, not a cache. It is fully decoupled +from both the incremental cursor (which stays Firestore-only) and the parse +path. Fetching from the portal is always driven by the Firestore cursor; the +archive is a downstream side effect of a successful fetch. + +Parsers must be **pure functions with no I/O** — `parse_summary(soup)`, +`parse_disclosure_detail(soup, year)` — so the identical code runs in the live +scrape and in the offline reparse script without modification. + +#### GCS key scheme + +One object per fetched page: `raw_html/{sha1(url)}.html`. URL-hash is +collision-free without modeling the ASP.NET URL key space. Individual `.html` +objects (not batch tarballs) are appropriate here because our scraper runs +weekly in small increments rather than in large historical sweeps; per-object +GCS is simpler and makes the reparse path a flat list + get. + +Bucket: `gs://-lobbying-archive/raw_html/` +Storage class: Archive (written once, read rarely). + +#### Write timing + +Archived immediately after `raise_for_status()` passes, before parsing, and +**not gated on parse success**: + +```python +def _get(session, url): + r = session.get(url, ...) + r.raise_for_status() + if "Summary.aspx" in url or "CompleteDisclosure" in url: + _archive_page(url, r.text) # side effect; never blocks parse + return BeautifulSoup(r.text, "html.parser") +``` + +Gating on parse success would defeat the purpose: the archive exists precisely +to recover from parser gaps later, so we want the bytes even when the current +parser does not fully understand them. The search/results page is excluded +(same URL across all years, trivially regenerable). + +#### Reparse path + +A dedicated offline script `lobbying-scraper/reparse_archive.py`: + +``` +python3 reparse_archive.py [--limit N] [--dry-run] +``` + +Lists `raw_html/*.html` objects from GCS, downloads them, resolves each +`sha1.html` back to its original URL (stored as object metadata at write time), +and runs the pure parser functions against the archived soup. Tracks completed +objects via a `processed_archive.txt` marker so reparse is resumable. + +#### Cursor interaction + +No change to the Firestore cursor. The archive is never consulted to skip a +portal fetch. "Have I processed this disclosure?" is still answered by the +Firestore cursor; the archive is a consequence of fetching, not an input to the +decision. + +#### New infrastructure required + +- GCS bucket `-lobbying-archive` (Archive class, no public access) +- `google-cloud-storage` added to `lobbying-scraper/requirements.txt` +- `GOOGLE_CLOUD_PROJECT` env var (already available on Cloud Run) used to + derive bucket name +- IAM: Cloud Run service account needs `storage.objects.create` on the bucket +- New file: `lobbying-scraper/reparse_archive.py` + +#### Order of work within Phase 2 + +This is a prerequisite for `ocpf_contributions.py` only in the sense that we +want the archive in place before running the full historical backfill so we do +not have to re-scrape. It is otherwise independent and can be implemented +alongside the contribution pipeline rather than before it. diff --git a/docs/lobbying-frontend-plan.md b/docs/lobbying-frontend-plan.md new file mode 100644 index 000000000..97601330f --- /dev/null +++ b/docs/lobbying-frontend-plan.md @@ -0,0 +1,414 @@ +# Lobbying Explorer — Frontend Plan + +_Living document. Update as implementation progresses._ + +--- + +## Overview + +A dedicated **Lobbying Explorer** section of MAPLE, surfacing OCPF lobbying disclosure +data as a first-class feature alongside bills and testimony. The explorer is a new +top-level nav item that provides four browsable entity views (overview, bills, clients, +firms) and integrates lightly into existing bill detail pages. Full visual design is +deferred to the design team; this phase establishes the data layer, routing skeleton, +i18n scaffolding, and placeholder UI. + +--- + +## Routes + +``` +/lobbying — overview: stats, search, entry cards +/lobbying/bills — browse all lobbied bills +/lobbying/clients — browse clients / sponsors +/lobbying/clients/[clientSlug] — client detail +/lobbying/firms — browse lobbying firms (registrants) +/lobbying/firms/[registrantId] — firm detail +``` + +Existing bill detail pages get one lightweight sidebar card (filing count + link to +`/lobbying/bills?bill=H1234&gc=193`). No other pages are touched. + +--- + +## Navigation + +Add **Lobbying** as a new top-level item in the primary navbar, between Bills and Learn. + +```tsx +// components/NavbarComponents.tsx — new export +export const NavbarLinkLobbying: React.FC<...> = ({ handleClick }) => { + const { t } = useTranslation("common") + return ( + + {t("navigation.lobbying")} + + ) +} +``` + +Add `"navigation.lobbying": "Lobbying"` to `public/locales/en/common.json`. + +--- + +## Data Model + +### Firestore schema (current — from `lobbying-data-pipeline`) + +| Collection | Doc ID | Key fields | +| --------------------- | ---------------- | --------------------------------------------------------------------------------------------------------------- | +| `lobbyingRegistrants` | `{registrantId}` | `entityName`, `regType`, `year`, `generalCourt`, `clients[]`, `disclosureUrls[]` | +| `lobbyingFilings` | `{filingId}` | `billId`, `generalCourt`, `entityName`, `clientName`, `clientNameNorm`, `position`, `amount`, `year`, `chamber` | + +### Required additions + +**1. Add `registrantId` to `LobbyingFiling`** + +`lobbyingFilings` currently stores `entityName` (string) but not the registrant document +ID. Without this, querying all filings for a firm requires a `where("entityName", "==", x)` +string match instead of an indexed ID lookup. Add `registrantId: string` to +`LobbyingFiling` type and populate it in the scraper. + +**2. Add `lobbyingStats` singleton document** + +Pre-computed by the ingest pipeline. Avoids expensive client-side aggregations on the +overview page. + +```ts +// functions/src/lobbying/types.ts +export const LobbyingStats = Record({ + totalFilings: Number, + totalRegistrants: Number, + totalClients: Number, // distinct clientNameNorm + totalBillsWithFilings: Number, // distinct (billId, generalCourt) pairs + courtsWithData: Array(Number), // sorted list of general court numbers + spendByYear: Record(String, Number), // { "2023": 88775472, ... } + filingsByYear: Record(String, Number) // { "2023": 42000, ... } +}) +export type LobbyingStats = Static +export const LOBBYING_STATS_DOC = "lobbyingStats" // top-level doc +``` + +**3. Add `lobbyingClients` collection** + +Clients are currently embedded in `registrants.clients[]`. A flat collection enables +efficient browse, search, and pagination. + +```ts +export const LobbyingClientSummary = Record({ + clientSlug: String, // URL-safe slug for routing + clientName: String, + clientNameNorm: String, + totalFilings: Number, + totalAmount: Null.Or(Number), + years: Array(Number), + generalCourts: Array(Number), + positionCounts: Record({ + // { support: N, oppose: N, neutral: N, none: N } + support: Number, + oppose: Number, + neutral: Number, + none: Number + }), + registrantIds: Array(String) // firms they have worked with +}) +export const CLIENTS_COLLECTION = "lobbyingClients" +``` + +**4. Required Firestore indexes** + +Add to `firestore.indexes.json`: + +```json +{ "collectionGroup": "lobbyingFilings", + "fields": [{"fieldPath": "billId"}, {"fieldPath": "generalCourt"}] }, +{ "collectionGroup": "lobbyingFilings", + "fields": [{"fieldPath": "registrantId"}, {"fieldPath": "year", "order": "DESCENDING"}] }, +{ "collectionGroup": "lobbyingFilings", + "fields": [{"fieldPath": "clientNameNorm"}, {"fieldPath": "year", "order": "DESCENDING"}] }, +{ "collectionGroup": "lobbyingFilings", + "fields": [{"fieldPath": "generalCourt"}, {"fieldPath": "year", "order": "DESCENDING"}] } +``` + +--- + +## Data hooks (`components/db/lobbying.ts`, new file) + +All hooks use `useAsync` pattern with loading/error states, following existing +conventions in `components/db/`. + +```ts +// Overview stats (reads lobbyingStats singleton) +useLobbyingStats(): AsyncResult + +// Filings for a specific bill — used in bill sidebar and lobbying/bills?bill= +useLobbyingFilingsForBill(court: number, billId: string): AsyncResult + +// Paginated registrant browse +useLobbyingRegistrants(opts: { + regType?: "Lobbyist" | "Employer" + year?: number + pageSize?: number +}): AsyncResult + +// Single registrant + their filings +useLobbyingRegistrant(registrantId: string): AsyncResult +useLobbyingFilingsForRegistrant(registrantId: string): AsyncResult + +// Client browse and detail +useLobbyingClients(opts?: { year?: number }): AsyncResult +useLobbyingClient(clientSlug: string): AsyncResult +useLobbyingFilingsForClient(clientNameNorm: string): AsyncResult +``` + +--- + +## i18n + +New namespace `lobbying` in `public/locales/en/lobbying.json`. All standalone lobbying +pages use `createGetStaticTranslationProps(["auth", "common", "footer", "lobbying"])`. + +The bill sidebar card reuses existing `common` namespace keys (`bill.lobbying_parties`, +`bill.client_name`, `bill.position`, `bill.disclosure_date`, `bill.pro`, `bill.neutral`). + +```json +{ + "title": "Lobbying Explorer", + "subtitle": "Browse Massachusetts lobbying disclosures filed with the Secretary of State.", + "stats": { + "totalBills": "Lobbied bills", + "totalClients": "Clients", + "sessionsCovered": "Sessions covered", + "totalSpend": "Total compensation reported" + }, + "sections": { + "bills": "Bills", + "clients": "Clients", + "firms": "Lobbying Firms" + }, + "filters": { + "session": "Session", + "year": "Year", + "position": "Position", + "allSessions": "All sessions", + "allPositions": "All positions", + "search": "Search…" + }, + "position": { + "support": "Support", + "oppose": "Oppose", + "neutral": "Neutral", + "none": "No position" + }, + "fields": { + "clientName": "Client", + "firmName": "Lobbying Firm", + "amount": "Compensation", + "year": "Year", + "filings": "Filings", + "bills": "Bills lobbied", + "clients": "Clients represented" + }, + "attribution": "Data from the <0>MA Secretary of State Lobbyist Public Search.", + "noData": "No lobbying filings found.", + "loading": "Loading lobbying data…", + "titles": { + "overview": "Lobbying Explorer", + "bills": "Lobbied Bills", + "clients": "Clients & Sponsors", + "firms": "Lobbying Firms", + "client": "Client Profile", + "firm": "Firm Profile" + }, + "billCard": { + "filingCount_one": "{{count}} lobbying filing", + "filingCount_other": "{{count}} lobbying filings", + "viewAll": "View all lobbying activity →" + } +} +``` + +--- + +## Components + +All in a new `components/lobbying/` directory. These are functional placeholders +designed to be restyled by the design team without changing props or data dependencies. + +### `LobbyingStatsBar` + +Overview stats (total bills, clients, sessions, spend) displayed as `TitledSectionCard` +stat boxes. Reads from `useLobbyingStats()`. + +### `LobbyingPositionChip` + +Inline pill showing position: Support (green) / Oppose (red) / Neutral (grey) / None. +Used throughout tables and detail pages. + +```tsx + // → "Support" pill +``` + +### `LobbyingFilingsTable` + +Shared table component used across bill sidebar, client detail, and firm detail. +Columns configurable via props — different views show different subsets. + +```tsx +interface LobbyingFilingsTableProps { + filings: LobbyingFiling[] + showBill?: boolean // show Bill ID + title column (off for bill-scoped view) + showClient?: boolean // show Client column + showFirm?: boolean // show Firm column + showAmount?: boolean + maxRows?: number // truncate with "View all" link +} +``` + +### `LobbyingBillCard` + +Lightweight sidebar card for bill detail pages. Shows filing count, stacked +support/oppose/neutral bar, and a "View all lobbying activity →" link. + +```tsx +interface LobbyingBillCardProps { + court: number + billId: string + className?: string +} +``` + +Replaces the dummy `LobbyingTable` component. Enabled by updating the +`lobbyingTable` feature flag to `true` in dev. + +### `LobbyingEntityCard` + +Summary card for browse lists (one card per client or firm). Shows name, filing count, +year range, and a position summary bar. Reused in both `/lobbying/clients` and +`/lobbying/firms` list views. + +--- + +## Visualizations + +These are proposed charts for detail and overview pages. Implementation uses +**Recharts** (to be added as a dependency) — lightweight, React-native, composable. + +| Chart | Location | Type | Data | +| ---------------------------------- | ------------------------------------- | ---------------------- | --------------------------------------------------- | +| Support/oppose/neutral stacked bar | Bill sidebar card, lobbied-bills list | Horizontal stacked bar | `n_supporters`, `n_opposers`, `n_neutrals` per bill | +| Position donut | Client detail, Firm detail | Doughnut | Filing position breakdown | +| Spend by year | Client detail, Firm detail | Bar | `spendByYear` from filings | +| Total spend + filings over time | Overview page | Dual-axis line | `LobbyingStats.spendByYear` + `filingsByYear` | +| Top tags horizontal bar | Client detail | Horizontal bar | MAPLE bill tags from filings for that client | +| Position heatmap by session | Client detail (optional) | Grid heatmap | Position per general court | + +All charts are conditionally rendered only when data is loaded and non-empty. The +position stacked bar on the bill sidebar is the highest-value chart — it answers +"who is on which side" at a glance and should be treated as the MVP chart. + +--- + +## Pages + +### `/lobbying` — Overview (`pages/lobbying/index.tsx`) + +- `getStaticProps` with ISR (`revalidate: 3600`) +- Reads `LobbyingStats` at build time +- Client-side: global search input querying `lobbyingFilings` for bill IDs / client names +- Sections: stat cards, three entry cards (Bills / Clients / Firms), attribution bar +- Data source attribution to MA Secretary of State required on every lobbying page + +### `/lobbying/bills` — Lobbied bills browse (`pages/lobbying/bills.tsx`) + +- `getStaticProps` (ISR) +- Client-side filterable table: session (general court), position filter, text search +- Columns: Bill ID (→ bill detail), Title, Session, Top tags (MAPLE bill tags), Position bar, Filings count +- Default sort: most filings first +- Clicking a bill row links to bill detail page (`/bills/[court]/[billId]`), not a lobbying sub-page + +### `/lobbying/clients` — Client browse (`pages/lobbying/clients/index.tsx`) + +- Client-side filterable list using `useLobbyingClients()` +- Columns: Client name, Total filings, Bills, Years active, Position breakdown +- Text search on client name + +### `/lobbying/clients/[clientSlug]` — Client detail (`pages/lobbying/clients/[clientSlug].tsx`) + +- `getServerSideProps`; fetch `LobbyingClientSummary` + filings by `clientNameNorm` +- Sections: stats bar, position donut, spend by year bar, bills lobbied table (with MAPLE tags) +- Bills table columns: Bill ID, Title, Session, MAPLE tags, Position, Year + +### `/lobbying/firms` — Firms browse (`pages/lobbying/firms/index.tsx`) + +- Same pattern as clients; filters by `regType` (Lobbyist / Employer / all) +- Columns: Firm name, Type, Clients, Bills, Years active + +### `/lobbying/firms/[registrantId]` — Firm detail (`pages/lobbying/firms/[registrantId].tsx`) + +- `getServerSideProps`; fetch `LobbyingRegistrant` + filings +- Sections: stats bar, position donut, clients represented table, bills lobbied table, compensation by year bar + +--- + +## Bill Page Integration + +In `components/bill/BillDetails.tsx`, replace the existing dummy `LobbyingTable` with +`LobbyingBillCard`. The feature flag `lobbyingTable` stays as the gate. + +```tsx +{ + flags.lobbyingTable && ( + + ) +} +``` + +The card is intentionally lightweight: one line of filing count, one stacked bar, one +link. The bill page is not the primary place for lobbying exploration. + +--- + +## Implementation Phases + +### Phase 1 — Foundation (current) + +- [ ] Add Firestore schema additions (`registrantId` on filings, `lobbyingStats`, `lobbyingClients`) +- [ ] Add Firestore indexes to `firestore.indexes.json` +- [ ] Add data hooks (`components/db/lobbying.ts`) +- [ ] Add `lobbying.json` locale file +- [ ] Add `NavbarLinkLobbying` to navbar +- [ ] Build `LobbyingBillCard` (replaces dummy `LobbyingTable` in bill detail) +- [ ] Enable `lobbyingTable` feature flag in dev +- [ ] Build `LobbyingPositionChip` and `LobbyingFilingsTable` shared components + +### Phase 2 — Browse pages (after design) + +- [ ] Install Recharts +- [ ] Overview page with stats and search +- [ ] Bills browse page +- [ ] Clients browse + detail pages +- [ ] Firms browse + detail pages +- [ ] Add position stacked bar chart to bill card + +### Phase 3 — Polish + +- [ ] Mobile responsive layouts +- [ ] Pagination and loading states +- [ ] Attribution links to SoS on every lobbying page +- [ ] Empty states and error states +- [ ] Run `firestore.indexes.json` deploy + +--- + +## Open Questions + +1. **Navbar label**: "Lobbying" or "Lobbying Explorer"? Keep short for nav. +2. **Bill page flag**: Enable `lobbyingTable` in prod when Phase 1 is ready, or wait for design sign-off? +3. **Client slug generation**: Use `clientNameNorm` directly (already normalized in pipeline) or generate a separate URL slug? Recommend reusing `clientNameNorm` with any remaining non-URL chars stripped. +4. **Recharts vs react-chartjs-2**: Recharts is recommended (pure React, no canvas imperative API). Confirm with team before adding dependency. +5. **SoS attribution link format**: The `LobbyingRegistrant` stores `disclosureUrls[]` which are direct SoS filing URLs. Use these for deep-linking from firm detail pages. diff --git a/firestore.indexes.json b/firestore.indexes.json index 83cb3fa6d..e78707bc2 100644 --- a/firestore.indexes.json +++ b/firestore.indexes.json @@ -788,25 +788,46 @@ "collectionGroup": "ballotQuestions", "queryScope": "COLLECTION", "fields": [ - { "fieldPath": "electionYear", "order": "ASCENDING" }, - { "fieldPath": "ballotStatus", "order": "ASCENDING" } + { + "fieldPath": "electionYear", + "order": "ASCENDING" + }, + { + "fieldPath": "ballotStatus", + "order": "ASCENDING" + } ] }, { "collectionGroup": "publishedTestimony", "queryScope": "COLLECTION_GROUP", "fields": [ - { "fieldPath": "ballotQuestionId", "order": "ASCENDING" }, - { "fieldPath": "publishedAt", "order": "DESCENDING" } + { + "fieldPath": "ballotQuestionId", + "order": "ASCENDING" + }, + { + "fieldPath": "publishedAt", + "order": "DESCENDING" + } ] }, { "collectionGroup": "publishedTestimony", "queryScope": "COLLECTION", "fields": [ - { "fieldPath": "billId", "order": "ASCENDING" }, - { "fieldPath": "court", "order": "ASCENDING" }, - { "fieldPath": "ballotQuestionId", "order": "ASCENDING" } + { + "fieldPath": "billId", + "order": "ASCENDING" + }, + { + "fieldPath": "court", + "order": "ASCENDING" + }, + { + "fieldPath": "ballotQuestionId", + "order": "ASCENDING" + } ] }, { @@ -898,6 +919,118 @@ } } ] + }, + { + "collectionGroup": "lobbyingFilings", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "generalCourt", + "order": "ASCENDING" + }, + { + "fieldPath": "billId", + "order": "ASCENDING" + } + ] + }, + { + "collectionGroup": "lobbyingFilings", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "generalCourt", + "order": "ASCENDING" + }, + { + "fieldPath": "chamber", + "order": "ASCENDING" + } + ] + }, + { + "collectionGroup": "lobbyingFilings", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "generalCourt", + "order": "ASCENDING" + }, + { + "fieldPath": "entityNameNorm", + "order": "ASCENDING" + } + ] + }, + { + "collectionGroup": "lobbyingFilings", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "generalCourt", + "order": "ASCENDING" + }, + { + "fieldPath": "clientNameNorm", + "order": "ASCENDING" + } + ] + }, + { + "collectionGroup": "lobbyingFilings", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "registrantId", + "order": "ASCENDING" + }, + { + "fieldPath": "year", + "order": "DESCENDING" + } + ] + }, + { + "collectionGroup": "lobbyingFilings", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "clientNameNorm", + "order": "ASCENDING" + }, + { + "fieldPath": "year", + "order": "DESCENDING" + } + ] + }, + { + "collectionGroup": "lobbyingFilings", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "entityNameNorm", + "order": "ASCENDING" + }, + { + "fieldPath": "year", + "order": "DESCENDING" + } + ] + }, + { + "collectionGroup": "lobbyingRegistrants", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "entityNameNorm", + "order": "ASCENDING" + }, + { + "fieldPath": "year", + "order": "DESCENDING" + } + ] } ], "fieldOverrides": [ diff --git a/firestore.rules b/firestore.rules index a95586279..df5659ac5 100644 --- a/firestore.rules +++ b/firestore.rules @@ -103,6 +103,18 @@ service cloud.firestore { allow read: if true; allow write: if false; } + match /lobbyingRegistrants/{id} { + allow read: if true; + allow write: if false; + } + match /lobbyingFilings/{id} { + allow read: if true; + allow write: if false; + } + match /lobbyingMeta/{id} { + allow read: if true; + allow write: if false; + } match /transcriptions/{tid} { // public, read-only allow read: if true diff --git a/functions/src/lobbying/index.ts b/functions/src/lobbying/index.ts new file mode 100644 index 000000000..6d039ae51 --- /dev/null +++ b/functions/src/lobbying/index.ts @@ -0,0 +1,2 @@ +export * from "./types" +export { normalizeEntityName } from "./normalize" diff --git a/functions/src/lobbying/normalize.ts b/functions/src/lobbying/normalize.ts new file mode 100644 index 000000000..a7beb338f --- /dev/null +++ b/functions/src/lobbying/normalize.ts @@ -0,0 +1,72 @@ +/** + * Entity name normalization pipeline. + * + * The SoS portal does not enforce consistent name formatting. The same client or + * registrant may appear as "Acme Corp.", "ACME CORPORATION", "Acme, Inc. d/b/a + * Acme Consulting", etc. across filings and years. + * + * The steps must be applied in the exact order + * listed here; changing the order produces different (incorrect) output. + */ + +// Step 2: strip d/b/a trade-name suffix before any other transforms so the +// trade name doesn't bleed into the canonical form. +const DBA_RE = /\s+D\s*\/+B\s*\/+A?\s+.*|\s+DBA\s+.*/i + +// Step 5: remove legal entity type words with whole-word matching so +// "INCORPORATED" and "CORP" are caught in addition to "LLC"/"INC". +const LEGAL_ENTITY_RE = + /\b(LLC|LLP|INC|INCORPORATED|CORPORATION|CORP|LTD|LIMITED|PC|PLLC)\b/g + +// Step 6: remove "THE" as a whole word anywhere (not just as a leading prefix). +const THE_RE = /\bTHE\b/g + +// Step 9: professional suffix phrases to remove wholesale. +const MISC_PHRASES = [ + "LAW OFFICE OF", + "AND ASSOCIATES", + "& ASSOCIATES", + "AND ASSOC", + "ATTORNEY AT LAW", + "ATTORNEY@LAW", + "ATTORNET AT LAW", // known portal typo + "AND PARTNERS", + "PUBLIC POLICY GROUP", + "LEGISLATIVE SERVICES", + "POLICY GROUP", + "ASSOCIATES", + "COUNSELLORS AT LAW" +] + +export function normalizeEntityName(raw: string | null | undefined): string { + if (!raw) return "" + + let x = raw.toUpperCase() // Step 1: uppercase + + x = x.replace(DBA_RE, "") // Step 2: strip d/b/a suffix + + x = x.replace(/-/g, " ") // Step 3: hyphen → space + + // Step 4: punctuation → space (not empty string, so ",INC" → " INC" → caught + // by step 5's whole-word removal). + for (const ch of [",", ".", "'", "‘", "’", "(", ")"]) { + x = x.split(ch).join(" ") + } + + x = x.replace(LEGAL_ENTITY_RE, " ") // Step 5: remove legal entity type words + + x = x.replace(THE_RE, " ") // Step 6: remove THE anywhere + + x = x.replace(/&/g, "AND") // Step 7: ampersand → AND + + x = x.replace("ASSICIATES", "ASSOCIATES") // Step 8: fix known portal typo + + // Step 9: remove professional suffix phrases + for (const phrase of MISC_PHRASES) { + x = x.split(phrase).join(" ") + } + + x = x.replace(/\s+/g, " ").trim() // Step 10: collapse whitespace + + return x +} diff --git a/functions/src/lobbying/types.ts b/functions/src/lobbying/types.ts new file mode 100644 index 000000000..c5e6ae936 --- /dev/null +++ b/functions/src/lobbying/types.ts @@ -0,0 +1,155 @@ +import { + Array, + Dictionary, + InstanceOf, + Literal, + Number, + Null, + Record, + Static, + String, + Union +} from "runtypes" +import { Timestamp } from "../firebase" + +export type LobbyingChamber = Static +export const LobbyingChamber = Union( + Literal("House Bill"), + Literal("Senate Bill"), + Literal("House Docket"), + Literal("Senate Docket"), + Literal("Executive"), + Literal("Other") +) + +export type LobbyingClient = Static +export const LobbyingClient = Record({ + clientName: String, + clientNameNorm: String, + compensation: Null.Or(Number) +}) + +export type LobbyingRegistrant = Static +export const LobbyingRegistrant = Record({ + registrantId: String, + entityName: String, + entityNameNorm: String, + year: Number, + generalCourt: Number, + regType: Union(Literal("Lobbyist"), Literal("Employer")), + clients: Array(LobbyingClient), + disclosureUrls: Array(String), + fetchedAt: InstanceOf(Timestamp) +}) + +export type LobbyingFiling = Static +export const LobbyingFiling = Record({ + filingId: String, + // ID of the parent lobbyingRegistrants document — enables indexed queries + // by firm without a string-equality scan on entityName. + registrantId: String, + entityName: String, + entityNameNorm: String, + clientName: String, + clientNameNorm: String, + year: Number, + generalCourt: Number, + chamber: LobbyingChamber, + // Non-null only for legislative chambers (House Bill, Senate Bill, House Docket, + // Senate Docket). For Executive and Other, no bill join should be attempted. + billId: Null.Or(String), + activityTitle: String, + position: String, + amount: Null.Or(Number), + fetchedAt: InstanceOf(Timestamp) +}) + +// ── Pre-computed aggregates ─────────────────────────────────────────────────── + +export type LobbyingPositionCounts = Static +export const LobbyingPositionCounts = Record({ + support: Number, + oppose: Number, + neutral: Number, + none: Number +}) + +/** + * Singleton document written by the ingest pipeline after each scrape run. + * Stored at /lobbyingStats (top-level document, no collection). + * Avoids expensive client-side fan-out on the Lobbying Explorer overview page. + */ +export type LobbyingStats = Static +export const LobbyingStats = Record({ + totalFilings: Number, + totalRegistrants: Number, + totalClients: Number, + totalBillsWithFilings: Number, + courtsWithData: Array(Number), + spendByYear: Dictionary(Number), + filingsByYear: Dictionary(Number) +}) + +/** + * One document per unique client/sponsor in the lobbyingClients collection. + * Written by the ingest pipeline; enables paginated client browse without + * scanning the full lobbyingFilings collection. + */ +export type LobbyingClientSummary = Static +export const LobbyingClientSummary = Record({ + clientSlug: String, + clientName: String, + clientNameNorm: String, + totalFilings: Number, + totalAmount: Null.Or(Number), + years: Array(Number), + generalCourts: Array(Number), + positionCounts: LobbyingPositionCounts, + registrantIds: Array(String) +}) + +/** Firestore path for the pre-computed stats singleton: collection/docId */ +export const LOBBYING_STATS_COLLECTION = "lobbyingMeta" +export const LOBBYING_STATS_DOC_ID = "stats" + +export const CLIENTS_COLLECTION = "lobbyingClients" + +/** Firestore path for lobbying registrant documents */ +export const REGISTRANTS_COLLECTION = "lobbyingRegistrants" + +/** Firestore path for lobbying filing documents */ +export const FILINGS_COLLECTION = "lobbyingFilings" + +/** Firestore path for the live scraper cursor document */ +export const SCRAPER_DOC = "/scrapers/lobbying" + +/** Firestore path for the backfill cursor subcollection */ +export const BACKFILL_DOC = "/scrapers/lobbyingBackfill" +export const BACKFILL_URLS_COLLECTION = "processedUrls" + +/** Earliest year with portal data */ +export const FIRST_LOBBYING_YEAR = 2005 + +/** + * Sentinel clientName used for pre-2013 legacy filings where compensation is + * reported as a single total rather than broken down per client. + */ +export const LEGACY_TOTAL_CLIENT = "_total_salary_" + +/** + * Chamber prefix map for constructing billId values that match MAPLE's Bill.id. + * Typed as a plain index signature so portal.ts can look up any LobbyingChamber + * without triggering "Property X does not exist" on the Partial. + */ +export const CHAMBER_PREFIXES: { [chamber: string]: string | undefined } = { + "House Bill": "H", + "Senate Bill": "S", + "House Docket": "HD", + "Senate Docket": "SD" +} + +/** Canonical chamber values for legacy short-form codes found in older filings */ +export const LEGACY_CHAMBER_MAP: { [raw: string]: LobbyingChamber } = { + HB: "House Bill", + SB: "Senate Bill" +} diff --git a/lobbying-scraper/.dockerignore b/lobbying-scraper/.dockerignore new file mode 100644 index 000000000..9460c99c4 --- /dev/null +++ b/lobbying-scraper/.dockerignore @@ -0,0 +1,4 @@ +__pycache__/ +*.pyc +*.pyo +.env diff --git a/lobbying-scraper/Dockerfile b/lobbying-scraper/Dockerfile new file mode 100644 index 000000000..afdf54d04 --- /dev/null +++ b/lobbying-scraper/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.12-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY archive.py normalize.py portal.py writer.py scrape.py ./ + +# Cloud Run sets PORT; we don't use it (this is a job, not a server). +# Cloud Scheduler invokes the container via HTTP POST to /; handle it minimally. +ENV PYTHONUNBUFFERED=1 + +# ENTRYPOINT is the fixed executable; CMD provides default args that --args overrides. +ENTRYPOINT ["python3", "scrape.py"] +CMD ["--mode", "weekly"] diff --git a/lobbying-scraper/__pycache__/normalize.cpython-37.pyc b/lobbying-scraper/__pycache__/normalize.cpython-37.pyc new file mode 100644 index 000000000..888500fe6 Binary files /dev/null and b/lobbying-scraper/__pycache__/normalize.cpython-37.pyc differ diff --git a/lobbying-scraper/__pycache__/portal.cpython-37.pyc b/lobbying-scraper/__pycache__/portal.cpython-37.pyc new file mode 100644 index 000000000..85442cfa8 Binary files /dev/null and b/lobbying-scraper/__pycache__/portal.cpython-37.pyc differ diff --git a/lobbying-scraper/archive.py b/lobbying-scraper/archive.py new file mode 100644 index 000000000..ede5d6656 --- /dev/null +++ b/lobbying-scraper/archive.py @@ -0,0 +1,60 @@ +"""GCS raw-HTML archive for the MA SoS lobbying scraper. + +Write-only cold storage: every fetched Summary/CompleteDisclosure page is +saved as gs://{bucket}/raw_html/{sha1(url)}.html with the original URL stored +as object metadata. This enables offline reparsing when parser logic changes +without re-scraping the portal (which is rate-limited and Imperva-protected). + +Enabled by setting ARCHIVE_RAW=1 in the environment. When disabled (default), +save_page() is a no-op so local runs and tests work without GCS credentials. + +The bucket is named {GOOGLE_CLOUD_PROJECT}-lobbying-archive and should be +created with the Archive storage class (written once, read rarely). +""" + +from __future__ import annotations + +import hashlib +import os + +_ENABLED = os.environ.get("ARCHIVE_RAW", "").lower() in ("1", "true", "yes") +_bucket_name: str | None = None +_client = None # google.cloud.storage.Client, lazily initialized + + +def _gcs(): + global _client + if _client is None: + from google.cloud import storage # noqa: PLC0415 + _client = storage.Client() + return _client + + +def _get_bucket_name() -> str: + global _bucket_name + if _bucket_name is None: + project = os.environ.get("GOOGLE_CLOUD_PROJECT") or _gcs().project + _bucket_name = f"{project}-lobbying-archive" + return _bucket_name + + +def blob_name(url: str) -> str: + """Return the GCS object name for a given URL.""" + return "raw_html/" + hashlib.sha1(url.encode()).hexdigest() + ".html" + + +def save_page(url: str, html: str) -> None: + """Write raw HTML to GCS. No-op when ARCHIVE_RAW is not set.""" + if not _ENABLED: + return + try: + bucket = _gcs().bucket(_get_bucket_name()) + blob = bucket.blob(blob_name(url)) + blob.metadata = {"source-url": url} + blob.upload_from_string( + html.encode("utf-8"), + content_type="text/html; charset=utf-8", + ) + except Exception as exc: + # Archive failures must never interrupt the live scrape path. + print(f" [archive] WARNING: failed to save {url[:80]!r}: {exc}") diff --git a/lobbying-scraper/normalize.py b/lobbying-scraper/normalize.py new file mode 100644 index 000000000..6e6f7418e --- /dev/null +++ b/lobbying-scraper/normalize.py @@ -0,0 +1,50 @@ +"""Entity name normalization pipeline. + +Direct port of functions/src/lobbying/normalize.ts. Steps must be applied in +this exact order — changing the order produces different (incorrect) output. +""" + +from __future__ import annotations + +import re + +_DBA_RE = re.compile(r"\s+D\s*/+B\s*/+A?\s+.*|\s+DBA\s+.*", re.IGNORECASE) +_LEGAL_RE = re.compile( + r"\b(LLC|LLP|INC|INCORPORATED|CORPORATION|CORP|LTD|LIMITED|PC|PLLC)\b" +) +_THE_RE = re.compile(r"\bTHE\b") +_WS_RE = re.compile(r"\s+") + +_MISC_PHRASES = [ + "LAW OFFICE OF", + "AND ASSOCIATES", + "& ASSOCIATES", + "AND ASSOC", + "ATTORNEY AT LAW", + "ATTORNEY@LAW", + "ATTORNET AT LAW", # known portal typo + "AND PARTNERS", + "PUBLIC POLICY GROUP", + "LEGISLATIVE SERVICES", + "POLICY GROUP", + "ASSOCIATES", + "COUNSELLORS AT LAW", +] + + +def normalize_entity_name(raw: str | None) -> str: + if not raw: + return "" + x = raw.upper() # 1. uppercase + x = _DBA_RE.sub("", x) # 2. strip d/b/a suffix + x = x.replace("-", " ") # 3. hyphen → space + for ch in (",", ".", "'", "‘", "’", "(", ")"): + x = x.replace(ch, " ") # 4. punctuation → space + x = _LEGAL_RE.sub(" ", x) # 5. remove legal entity words + x = _THE_RE.sub(" ", x) # 6. remove THE anywhere + x = x.replace("&", "AND") # 7. ampersand → AND + x = x.replace("ASSICIATES", "ASSOCIATES") # 8. fix known typo + for phrase in _MISC_PHRASES: # 9. remove professional suffix phrases + x = x.replace(phrase, " ") + x = _WS_RE.sub(" ", x).strip() # 10. collapse whitespace + return x diff --git a/lobbying-scraper/portal.py b/lobbying-scraper/portal.py new file mode 100644 index 000000000..3a5528898 --- /dev/null +++ b/lobbying-scraper/portal.py @@ -0,0 +1,512 @@ +"""HTTP client and HTML parser for the MA SoS lobbying portal. + +Portal: https://www.sec.state.ma.us/LobbyistPublicSearch/ + +Page flow: + 1. Search POST -> summary links table + 2. Summary.aspx -> registrant name/year/type + CompleteDisclosure links + 3. CompleteDisclosure.aspx -> per-client compensation + per-client bill activity + +Four HTML format eras for CompleteDisclosure pages (detected by table IDs): + Modern (2019+): grdvClientPaidToEntity + grdvActivitiesNew{year}_{n} + Hybrid (2014-2018): no comp table; Panel1_{n} divs + grdvActivitiesNew_{n} + Legacy (2009-2013): grdvActivities with Compensation received column + Legacy (2005-2008): grdvSalaryPaid (entity total) + grdvActivities (no comp col) + +parse_summary() and parse_disclosure_detail() are pure functions (no I/O) so +they can be called from both the live scraper and the offline reparse driver. +fetch_* are thin wrappers that handle HTTP and raw-HTML archiving. +""" + +from __future__ import annotations + +import hashlib +import re +import time +from dataclasses import dataclass, field +from typing import Optional + +import requests +from bs4 import BeautifulSoup, Tag + +import archive + +# ── Constants ───────────────────────────────────────────────────────────────── + +BASE_URL = "https://www.sec.state.ma.us/LobbyistPublicSearch/" +SEARCH_URL = BASE_URL + "Default.aspx" + +_UA = ( + "Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) " + "AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148" +) +_REQUEST_DELAY = 1.0 +_MAX_RETRIES = 6 +_RETRY_STATUS = {429, 500, 502, 503, 504} + +# Lobby disclosure data begins in 2005; GC 183 started Jan 2003. +FIRST_YEAR = 2005 +FIRST_GC = 183 +FIRST_GC_START_YEAR = 2003 + +# clientName sentinel for pre-2009 filings where compensation is a single entity total +LEGACY_TOTAL_CLIENT = "_total_salary_" + +# Maps canonical chamber names to the bill-ID prefix used in MAPLE's Bill.id +CHAMBER_PREFIXES: dict[str, str] = { + "House Bill": "H", + "Senate Bill": "S", + "House Docket": "HD", + "Senate Docket": "SD", +} + +# Legacy short-form chamber codes found in older filings +LEGACY_CHAMBER_MAP: dict[str, str] = { + "HB": "House Bill", + "SB": "Senate Bill", +} + +# ── Data types ──────────────────────────────────────────────────────────────── + + +@dataclass +class Compensation: + client_name: str + amount: Optional[float] + + +@dataclass +class BillActivity: + client_name: str + chamber: str # canonical LobbyingChamber value + raw_bill_number: str + bill_id: Optional[str] # e.g. "H1234"; null for Executive/Other + activity_title: str + position: str + amount: Optional[float] + + +@dataclass +class DisclosureMeta: + entity_name: str + year: Optional[int] + reg_type: str # "Lobbyist" | "Employer" + disclosure_urls: list[str] = field(default_factory=list) + + +@dataclass +class DisclosureDetail: + compensation: list[Compensation] = field(default_factory=list) + bills: list[BillActivity] = field(default_factory=list) + + +# ── Derived-value helpers ───────────────────────────────────────────────────── + + +def year_to_general_court(year: int) -> int: + return FIRST_GC + (year - FIRST_GC_START_YEAR) // 2 + + +def normalize_chamber(raw: str) -> str: + t = raw.strip() + if t in LEGACY_CHAMBER_MAP: + return LEGACY_CHAMBER_MAP[t] + known = {"House Bill", "Senate Bill", "House Docket", "Senate Docket", "Executive"} + return t if t in known else "Other" + + +def construct_bill_id(chamber: str, raw_bill_number: str) -> Optional[str]: + """Construct the MAPLE-compatible billId from chamber + raw integer. + + Returns None for Executive and Other chambers where no bill join is possible. + H1234 and S1234 are distinct bills even though they share the same integer; + the prefix is required to disambiguate. + """ + prefix = CHAMBER_PREFIXES.get(chamber) + if not prefix: + return None + try: + return f"{prefix}{int(raw_bill_number)}" + except (ValueError, TypeError): + return None + + +def registrant_id(entity_name: str, year: int) -> str: + key = f"{year}|{entity_name}" + return hashlib.sha256(key.encode()).hexdigest()[:40] + + +def filing_id( + entity_name: str, + client_name: str, + chamber: str, + bill_id: Optional[str], + general_court: int, + position: str, +) -> str: + key = "|".join([ + entity_name, client_name, chamber, + bill_id or "__null__", str(general_court), position, + ]) + return hashlib.sha256(key.encode()).hexdigest()[:40] + + +# ── HTTP session ────────────────────────────────────────────────────────────── + + +def make_session() -> requests.Session: + s = requests.Session() + s.headers.update({ + "User-Agent": _UA, + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate, br", + "Connection": "keep-alive", + }) + return s + + +def _get(session: requests.Session, url: str) -> BeautifulSoup: + for attempt in range(_MAX_RETRIES): + time.sleep(_REQUEST_DELAY * (2 ** attempt) if attempt else _REQUEST_DELAY) + try: + r = session.get(url, timeout=60) + except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e: + if attempt == _MAX_RETRIES - 1: + raise + print(f" GET network error (attempt {attempt + 1}): {e}") + continue + if r.status_code in _RETRY_STATUS: + print(f" GET HTTP {r.status_code} (attempt {attempt + 1}) — retrying") + if attempt == _MAX_RETRIES - 1: + r.raise_for_status() + continue + r.raise_for_status() + # Archive content pages before parsing; excludes the search page which + # shares one URL across all years and is trivially regenerable. + if "Summary.aspx" in url or "CompleteDisclosure" in url: + archive.save_page(url, r.text) + return BeautifulSoup(r.text, "html.parser") + raise RuntimeError("_get: exhausted retries") # unreachable + + +def _post(session: requests.Session, url: str, data: dict) -> BeautifulSoup: + for attempt in range(_MAX_RETRIES): + time.sleep(_REQUEST_DELAY * (2 ** attempt) if attempt else _REQUEST_DELAY) + try: + r = session.post(url, data=data, timeout=180) + except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e: + if attempt == _MAX_RETRIES - 1: + raise + print(f" POST network error (attempt {attempt + 1}): {e}") + continue + if r.status_code in _RETRY_STATUS: + print(f" POST HTTP {r.status_code} (attempt {attempt + 1}) — retrying") + if attempt == _MAX_RETRIES - 1: + r.raise_for_status() + continue + r.raise_for_status() + return BeautifulSoup(r.text, "html.parser") + raise RuntimeError("_post: exhausted retries") # unreachable + + +# ── Portal scraping ─────────────────────────────────────────────────────────── + + +def _viewstate(soup: BeautifulSoup) -> dict: + return { + inp["name"]: inp.get("value", "") + for inp in soup.find_all("input", type="hidden") + if inp.get("name") + } + + +def fetch_summary_links(session: requests.Session, year: int) -> list[str]: + """Return all Summary.aspx URLs for a given year via a single search POST.""" + soup = _get(session, SEARCH_URL) + data = { + **_viewstate(soup), + "__EVENTTARGET": "", + "__EVENTARGUMENT": "", + "ctl00$ContentPlaceHolder1$Search": "rdbSearchByType", + "ctl00$ContentPlaceHolder1$ucSearchCriteriaByType$ddlYear": str(year), + "ctl00$ContentPlaceHolder1$ucSearchCriteriaByType$txtN_ame": "", + "ctl00$ContentPlaceHolder1$ucSearchCriteriaByType$lddSearchType$DropDown": "3", + "ctl00$ContentPlaceHolder1$ucSearchCriteriaByType$drpType": "L", + "ctl00$ContentPlaceHolder1$drpPageSize": "20000", + "ctl00$ContentPlaceHolder1$btnSearch": "Search", + } + results = _post(session, SEARCH_URL, data) + table = results.find( + "table", + id=lambda x: x and "grdvSearchResultByTypeAndCategory" in x, + ) + if not table: + return [] + return [ + BASE_URL + a["href"] if not a["href"].startswith("http") else a["href"] + for a in table.find_all("a", href=True) + if "Summary.aspx" in a["href"] + ] + + +def parse_summary(soup: BeautifulSoup) -> DisclosureMeta: + """Parse a Summary.aspx page. Pure function — no I/O.""" + def text(el_id: str) -> str: + el = soup.find(id=el_id) + return el.get_text(strip=True) if el else "" + + entity_name = text("ContentPlaceHolder1_lblRegistrantName") + year_text = text("ContentPlaceHolder1_lblYear") + reg_type_raw = text("ContentPlaceHolder1_lblRegType") + + try: + year = int(year_text) + except ValueError: + year = None + + reg_type = "Employer" if "Entity" in reg_type_raw else "Lobbyist" + + disc_urls = [ + BASE_URL + a["href"] if not a["href"].startswith("http") else a["href"] + for a in soup.find_all("a", href=True) + if "CompleteDisclosure" in a["href"] + ] + + return DisclosureMeta( + entity_name=entity_name, + year=year, + reg_type=reg_type, + disclosure_urls=disc_urls, + ) + + +def fetch_disclosure_meta(session: requests.Session, summary_url: str) -> DisclosureMeta: + return parse_summary(_get(session, summary_url)) + + +def _parse_amount(text: str) -> Optional[float]: + cleaned = text.replace("$", "").replace(",", "").strip() + try: + return float(cleaned) + except ValueError: + return None + + +def _grid_rows(table: Tag) -> list: + return table.find_all("tr", class_=lambda c: c and "Grid" in c and "Header" not in c) + + +def parse_disclosure_detail(soup: BeautifulSoup, year: int) -> DisclosureDetail: + """Parse a CompleteDisclosure page. Pure function — no I/O. + + Four HTML format eras (detected by table IDs): + + Modern (2019+): grdvClientPaidToEntity holds per-client compensation; + bills in grdvActivitiesNew{year}_{n} (one table per client). + + Hybrid (2014-2018): no grdvClientPaidToEntity. Bills in grdvActivitiesNew_{n} + (no year suffix). Per-client compensation is in id-less Panel1_{n} divs + ("Total amount paid by client...: $X") indexed by lblClientName_{n} spans. + Each client reports either a Panel1 total OR activity-level amounts — + summing both sources is safe because the unused one is always zero. + Omitting this path silently drops ~99% of 2014-2018 compensation. + + Legacy (2009-2013): single grdvActivities table with a "Compensation received" + column carrying a per-client total repeated on every bill row for that client. + Must deduplicate distinct (client, amount) pairs before summing — never sum + raw rows or the total is multiplied by bill count. + + Legacy (2005-2008): grdvActivities has no compensation column; fall back to + grdvSalaryPaid entity total under the _total_salary_ placeholder client. + """ + compensation: list[Compensation] = [] + bills: list[BillActivity] = [] + gc = year_to_general_court(year) + + # ── Modern / Hybrid: per-client activity tables ─────────────────────────── + comp_table = soup.find( + "table", + id=lambda x: x and "grdvClientPaidToEntity" in (x or ""), + ) + if comp_table: + for row in _grid_rows(comp_table): + cells = [td.get_text(strip=True) for td in row.find_all("td")] + if len(cells) >= 2: + compensation.append(Compensation( + client_name=cells[0], + amount=_parse_amount(cells[1]), + )) + + # Activity tables: ID pattern grdvActivitiesNew{year}_{n} (Modern, 2019+) + # or grdvActivitiesNew_{n} with no year (Hybrid, 2014-2018). The same loop + # handles both; compensation source differs (see Hybrid block below). + activity_by_client: dict[str, float] = {} + for act_table in soup.find_all( + "table", + id=lambda x: x and re.search(r"grdvActivitiesNew(\d{4})?_\d+", x or ""), + ): + client_span = act_table.find_previous( + "span", + id=lambda x: x and "lblClientName" in (x or ""), + ) + client_name = client_span.get_text(strip=True) if client_span else "" + + for row in _grid_rows(act_table): + cells = [td.get_text(strip=True) for td in row.find_all("td")] + if len(cells) < 4: + continue + chamber = normalize_chamber(cells[0]) + raw_num = cells[1] + bill_id = construct_bill_id(chamber, raw_num) + amt = _parse_amount(cells[4]) if len(cells) > 4 else None + bills.append(BillActivity( + client_name=client_name, + chamber=chamber, + raw_bill_number=raw_num, + bill_id=bill_id, + activity_title=cells[2] if len(cells) > 2 else "", + position=cells[3] if len(cells) > 3 else "", + amount=amt, + )) + if amt: + activity_by_client[client_name] = ( + activity_by_client.get(client_name, 0.0) + amt + ) + + # Hybrid (2014-2018): no modern comp table — reconstruct per-client amounts + # from Panel1 "Total amount paid by client" divs indexed by client-name spans. + if not comp_table and bills: + client_by_idx = { + sp["id"].split("_")[-1]: sp.get_text(strip=True) + for sp in soup.find_all( + "span", id=lambda x: x and "lblClientName_" in (x or "") + ) + } + panel_by_client: dict[str, float] = {} + for div in soup.find_all("div", id=lambda x: x and "Panel1_" in (x or "")): + idx = div["id"].split("_")[-1] + cn = client_by_idx.get(idx) + if not cn: + continue + m = re.search(r"\$([\d,]+\.\d\d)", div.get_text(" ", strip=True)) + panel_by_client[cn] = float(m.group(1).replace(",", "")) if m else 0.0 + for cn in set(panel_by_client) | set(activity_by_client): + amt = panel_by_client.get(cn, 0.0) + activity_by_client.get(cn, 0.0) + if amt: + compensation.append(Compensation(client_name=cn, amount=amt)) + + if comp_table or bills: + return DisclosureDetail(compensation=compensation, bills=bills) + + # ── Legacy format (2005-2013): single grdvActivities table ─────────────── + act_table = soup.find("table", id=lambda x: x and x.endswith("grdvActivities")) + # Distinct (client, amount) pairs from the "Compensation received" column + # (2009-2013). The portal repeats the per-client total on every bill row for + # that client, so we deduplicate before summing to avoid multiplying by bill count. + legacy_comp_pairs: set = set() + comp_col: Optional[int] = None + + if act_table: + all_rows = act_table.find_all("tr") + headers = [ + th.get_text(strip=True) + for th in (all_rows[0].find_all(["th", "td"]) if all_rows else []) + ] + + if headers and "Activity" in headers[0]: + # 6-col entity layout has Lobbyist as second header + if len(headers) >= 2 and "Lobbyist" in headers[1]: + bill_col, pos_col, client_col = 0, 2, 4 + else: + bill_col, pos_col, client_col = 0, 1, 3 + else: + bill_col, pos_col, client_col = 1, None, 3 + + if any("Compensation" in h for h in headers): + comp_col = len(headers) - 1 + + chamber_map = { + "H": "House Bill", "S": "Senate Bill", + "HD": "House Docket", "SD": "Senate Docket", + } + skip = {"Activity or Bill No and Title", "N/A", "None", "", "Total amount"} + + for row in all_rows[1:]: + cells = [td.get_text(strip=True) for td in row.find_all("td")] + if len(cells) <= max(bill_col, client_col): + continue + client_name = cells[client_col] + bill_cell = cells[bill_col] + amt = ( + _parse_amount(cells[comp_col]) + if comp_col is not None and len(cells) > comp_col + else None + ) + # Exclude "Total amount" summary rows appended by legacy individual + # disclosures — these are not real clients. + if amt is not None and client_name not in ("Total amount", "Total", ""): + legacy_comp_pairs.add((client_name, amt)) + if not bill_cell or bill_cell in skip: + continue + # Bill token may use a semicolon separator ("H73; Title") or a space + parts = re.split(r"[;\s]", bill_cell, maxsplit=1) + bill_no = parts[0].rstrip(";") + m = re.match(r"^([A-Z]+)(\d+)$", bill_no) + if not m: + continue + prefix, number = m.group(1), m.group(2) + chamber = chamber_map.get(prefix, "Other") + bill_id = construct_bill_id(chamber, number) + bills.append(BillActivity( + client_name=client_name, + chamber=chamber, + raw_bill_number=number, + bill_id=bill_id, + activity_title=parts[1].strip() if len(parts) > 1 else "", + position=( + cells[pos_col] + if pos_col is not None and len(cells) > pos_col + else "" + ), + amount=amt, + )) + + # Per-client compensation from the "Compensation received" column (2009-2013). + # Fall back to grdvSalaryPaid entity total when no compensation column exists (2005-2008). + if comp_col is not None: + per_client: dict[str, float] = {} + for cn, amt in legacy_comp_pairs: + per_client[cn] = per_client.get(cn, 0.0) + amt + for cn, amt in per_client.items(): + if amt: + compensation.append(Compensation(client_name=cn, amount=amt)) + else: + salary_table = soup.find( + "table", id=lambda x: x and "grdvSalaryPaid" in (x or "") + ) + if salary_table: + total = 0.0 + for row in salary_table.find_all("tr"): + cells = [td.get_text(strip=True) for td in row.find_all("td")] + if len(cells) >= 2 and "Total" not in cells[0]: + amt = _parse_amount(cells[1]) + if amt: + total += amt + if total: + compensation.append( + Compensation(client_name=LEGACY_TOTAL_CLIENT, amount=total) + ) + + return DisclosureDetail(compensation=compensation, bills=bills) + + +def fetch_disclosure_detail( + session: requests.Session, disc_url: str, year: int +) -> DisclosureDetail: + return parse_disclosure_detail(_get(session, disc_url), year) + + +def year_from_disc_url(url: str) -> Optional[int]: + """Extract the filing year from a CompleteDisclosure URL query string.""" + m = re.search(r"[?&]FilingYear=(\d{4})", url) + return int(m.group(1)) if m else None diff --git a/lobbying-scraper/reparse_archive.py b/lobbying-scraper/reparse_archive.py new file mode 100644 index 000000000..266ff1ad6 --- /dev/null +++ b/lobbying-scraper/reparse_archive.py @@ -0,0 +1,155 @@ +"""Offline reparse driver: re-ingests raw HTML from the GCS archive. + +Downloads archived CompleteDisclosure pages from GCS, re-runs the pure parsers +against them, and writes results back to Firestore. Use this when parser logic +has changed and historical data needs to be re-ingested without re-scraping. + +For each archived disclosure page the driver looks up the corresponding +registrant document in Firestore (via the disclosureUrls array) to obtain the +entity name needed to construct filing document IDs. Registrant documents must +therefore already exist before running a reparse. + +Usage: + GOOGLE_APPLICATION_CREDENTIALS=~/.config/gcloud/application_default_credentials.json \\ + python3 reparse_archive.py [--limit N] [--dry-run] + +Progress is tracked in Firestore at /scrapers/lobbyingReparse so the run is +resumable: restarting skips blobs already marked as processed. +""" + +from __future__ import annotations + +import argparse +import os + +from bs4 import BeautifulSoup +from google.cloud import firestore, storage + +import archive +from portal import DisclosureMeta, parse_disclosure_detail, year_from_disc_url +from writer import REGISTRANTS_COLLECTION, write_filings + +REPARSE_DOC = "scrapers/lobbyingReparse" + + +def _meta_for_disc_url(db: firestore.Client, disc_url: str) -> DisclosureMeta | None: + """Look up the registrant that owns this disclosure URL.""" + results = ( + db.collection(REGISTRANTS_COLLECTION) + .where("disclosureUrls", "array_contains", disc_url) + .limit(1) + .get() + ) + if not results: + return None + data = results[0].to_dict() + return DisclosureMeta( + entity_name=data.get("entityName", ""), + year=data.get("year"), + reg_type=data.get("regType", "Lobbyist"), + disclosure_urls=data.get("disclosureUrls", []), + ) + + +def _is_processed(db: firestore.Client, blob_name: str) -> bool: + doc = db.document(REPARSE_DOC).get() + if not doc.exists: + return False + return blob_name in doc.to_dict().get("processedBlobs", []) + + +def _mark_processed(db: firestore.Client, blob_name: str) -> None: + db.document(REPARSE_DOC).set( + {"processedBlobs": firestore.ArrayUnion([blob_name])}, + merge=True, + ) + + +def run(limit: int | None, dry_run: bool) -> None: + gcs = storage.Client() + bucket_name = archive._get_bucket_name() + bucket = gcs.bucket(bucket_name) + + db: firestore.Client | None = None if dry_run else firestore.Client() + + blobs = list(bucket.list_blobs(prefix="raw_html/")) + print(f"Found {len(blobs)} archived pages") + + processed = 0 + skipped = 0 + errors = 0 + + for blob in blobs: + if limit is not None and processed >= limit: + break + + blob.reload() + url = (blob.metadata or {}).get("source-url", "") + + if "CompleteDisclosure" not in url: + skipped += 1 + continue + + if db is not None and _is_processed(db, blob.name): + skipped += 1 + continue + + year = year_from_disc_url(url) + if year is None: + print(f" SKIP {blob.name}: cannot extract year from {url!r}") + skipped += 1 + continue + + meta: DisclosureMeta | None = None + if db is not None: + meta = _meta_for_disc_url(db, url) + if meta is None: + print(f" SKIP {blob.name}: no registrant found for {url!r}") + skipped += 1 + continue + + try: + html = blob.download_as_text(encoding="utf-8") + soup = BeautifulSoup(html, "html.parser") + detail = parse_disclosure_detail(soup, year) + except Exception as exc: + print(f" ERROR parsing {url}: {exc}") + errors += 1 + continue + + print( + f" {url[:80]!r} — {len(detail.compensation)} comp," + f" {len(detail.bills)} bills" + ) + + if not dry_run and db is not None and meta is not None: + write_filings(db, meta, detail) + _mark_processed(db, blob.name) + + processed += 1 + + print( + f"\nDone: {processed} reparsed, {skipped} skipped, {errors} errors" + + (" (dry run — nothing written)" if dry_run else "") + ) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Reparse raw HTML archive into Firestore" + ) + parser.add_argument("--limit", type=int, default=None, help="Stop after N pages") + parser.add_argument( + "--dry-run", action="store_true", help="Parse but do not write to Firestore" + ) + args = parser.parse_args() + + # Ensure archive module can resolve the bucket name even in dry-run mode + if not os.environ.get("ARCHIVE_RAW"): + os.environ["ARCHIVE_RAW"] = "1" + + run(limit=args.limit, dry_run=args.dry_run) + + +if __name__ == "__main__": + main() diff --git a/lobbying-scraper/requirements.txt b/lobbying-scraper/requirements.txt new file mode 100644 index 000000000..d92c075e4 --- /dev/null +++ b/lobbying-scraper/requirements.txt @@ -0,0 +1,4 @@ +requests>=2.28 +beautifulsoup4>=4.12 +google-cloud-firestore>=2.14 +google-cloud-storage>=2.10 diff --git a/lobbying-scraper/scrape.py b/lobbying-scraper/scrape.py new file mode 100644 index 000000000..151349c8a --- /dev/null +++ b/lobbying-scraper/scrape.py @@ -0,0 +1,291 @@ +"""Lobbying disclosure scraper — Cloud Run entry point. + +Runs on a weekly Cloud Scheduler trigger. Checks for new or amended disclosures +and exits immediately if none are found (fast path). When new disclosures exist, +fetches and writes them to Firestore. + +Also serves as the library used by the TypeScript backfill admin script via +subprocess. + +Environment variables: + GOOGLE_CLOUD_PROJECT — GCP project ID (set automatically in Cloud Run) + FIRESTORE_EMULATOR_HOST — set to use the local emulator (e.g. localhost:8080) + +CLI flags (for local / backfill use): + --year YEAR Only process this year (default: current + prior) + --limit N Max registrants per year (for testing) + --dry-run Fetch and parse but do not write to Firestore +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import sys +from datetime import datetime, timezone + +from google.cloud import firestore + +from portal import ( + FIRST_YEAR, + fetch_disclosure_detail, + fetch_disclosure_meta, + fetch_summary_links, + make_session, +) +from writer import ( + BACKFILL_DOC, + BACKFILL_URLS_COLLECTION, + SCRAPER_DOC, + write_filings, + write_registrant, +) + + +# ── Cursor helpers ──────────────────────────────────────────────────────────── + + +def _load_live_cursor(db: firestore.Client) -> tuple[set[str], dict[str, list[str]]]: + """Return (processedDiscUrls, summaryDiscCache) from the live scraper doc.""" + doc = db.document(SCRAPER_DOC).get() + data = doc.to_dict() or {} + return ( + set(data.get("processedDiscUrls", [])), + data.get("summaryDiscCache", {}), + ) + + +def _save_live_cursor( + db: firestore.Client, + processed: set[str], + cache: dict[str, list[str]], +) -> None: + db.document(SCRAPER_DOC).set( + {"processedDiscUrls": list(processed), "summaryDiscCache": cache}, + merge=True, + ) + + +def _is_backfill_processed(db: firestore.Client, disc_url: str) -> bool: + h = hashlib.sha256(disc_url.encode()).hexdigest()[:40] + return db.document(BACKFILL_DOC).collection(BACKFILL_URLS_COLLECTION).document(h).get().exists + + +def _mark_backfill_processed(db: firestore.Client, disc_url: str) -> None: + h = hashlib.sha256(disc_url.encode()).hexdigest()[:40] + db.document(BACKFILL_DOC).collection(BACKFILL_URLS_COLLECTION).document(h).set( + {"url": disc_url, "processedAt": datetime.now(tz=timezone.utc).isoformat()} + ) + + +# ── Core processing ─────────────────────────────────────────────────────────── + + +def process_disclosure( + db: firestore.Client | None, + session, + summary_url: str, + disc_url: str, + year: int, + dry_run: bool = False, +) -> tuple[int, int]: + """Fetch one disclosure page and write registrant + filing documents. + + Returns (compensation_rows, filing_rows). + """ + meta = fetch_disclosure_meta(session, summary_url) + detail = fetch_disclosure_detail(session, disc_url, year) + + if dry_run or db is None: + return len(detail.compensation), len(detail.bills) + + write_registrant(db, meta, detail, disc_url) + n_filings = write_filings(db, meta, detail) + return len(detail.compensation), n_filings + + +# ── Weekly incremental run ──────────────────────────────────────────────────── + + +def run_weekly( + db: "firestore.Client | None", + years: list[int], + limit: int | None = None, + dry_run: bool = False, +) -> int: + """Incremental weekly check. Returns number of new disclosures processed.""" + current_year = datetime.now(tz=timezone.utc).year + processed, cache = _load_live_cursor(db) if db is not None else (set(), {}) + + session = make_session() + new_count = 0 + + for year in years: + print(f"\n── {year} ──") + try: + summary_urls = fetch_summary_links(session, year) + except Exception as e: + print(f" failed to fetch summary links: {e}", file=sys.stderr) + continue + + if limit: + summary_urls = summary_urls[:limit] + + print(f" {len(summary_urls)} registrants on portal") + + for summary_url in summary_urls: + # Use cached disc URLs for prior years; always re-check current year + disc_urls = cache.get(summary_url) + if disc_urls is None or year == current_year: + try: + meta = fetch_disclosure_meta(session, summary_url) + disc_urls = meta.disclosure_urls + cache[summary_url] = disc_urls + if not dry_run: + _save_live_cursor(db, processed, cache) + except Exception as e: + print(f" failed to fetch summary {summary_url}: {e}", file=sys.stderr) + continue + + new_disc_urls = [u for u in disc_urls if u not in processed] + if not new_disc_urls: + continue + + for disc_url in new_disc_urls: + try: + comp_n, filing_n = process_disclosure( + db, session, summary_url, disc_url, year, dry_run=dry_run + ) + processed.add(disc_url) + new_count += 1 + print(f" processed: {comp_n} clients, {filing_n} filings") + if not dry_run: + _save_live_cursor(db, processed, cache) + except Exception as e: + print(f" failed to process {disc_url}: {e}", file=sys.stderr) + + return new_count + + +# ── Historical backfill ─────────────────────────────────────────────────────── + + +def _completed_years(db: "firestore.Client") -> set[int]: + data = db.document(BACKFILL_DOC).get().to_dict() or {} + return set(data.get("completedYears", [])) + + +def _mark_year_complete(db: "firestore.Client", year: int) -> None: + db.document(BACKFILL_DOC).set( + {"completedYears": firestore.ArrayUnion([year])}, merge=True + ) + + +def run_backfill( + db: "firestore.Client | None", + years: list[int], + limit: int | None = None, + dry_run: bool = False, +) -> int: + """Full historical backfill using the subcollection cursor. Resumable.""" + session = make_session() + total_new = 0 + + done = _completed_years(db) if db is not None and not dry_run else set() + if done: + print(f"Skipping already-completed years: {sorted(done)}") + + for year in years: + if year in done: + continue + + print(f"\n── {year} ──") + try: + summary_urls = fetch_summary_links(session, year) + except Exception as e: + print(f" failed to fetch summary links: {e}", file=sys.stderr) + continue + + if limit: + summary_urls = summary_urls[:limit] + + print(f" {len(summary_urls)} registrants on portal") + year_new = 0 + + for i, summary_url in enumerate(summary_urls): + try: + meta = fetch_disclosure_meta(session, summary_url) + except Exception as e: + print(f" [{i+1}/{len(summary_urls)}] failed to fetch summary: {e}", file=sys.stderr) + continue + + for disc_url in meta.disclosure_urls: + if db is not None and not dry_run and _is_backfill_processed(db, disc_url): + continue + try: + comp_n, filing_n = process_disclosure( + db, session, summary_url, disc_url, year, dry_run=dry_run + ) + if not dry_run: + _mark_backfill_processed(db, disc_url) + total_new += 1 + year_new += 1 + except Exception as e: + print(f" failed to process {disc_url}: {e}", file=sys.stderr) + + if (i + 1) % 50 == 0 or i + 1 == len(summary_urls): + print(f" [{i+1}/{len(summary_urls)}] {year_new} new disclosures so far") + + print(f" {year} complete: {year_new} new disclosures") + if db is not None and not dry_run and not limit: + _mark_year_complete(db, year) + + return total_new + + +# ── Entry point ─────────────────────────────────────────────────────────────── + + +def main() -> None: + p = argparse.ArgumentParser() + p.add_argument("--year", type=int, default=None) + p.add_argument("--limit", type=int, default=None) + p.add_argument("--dry-run", action="store_true") + p.add_argument( + "--mode", + choices=["weekly", "backfill"], + default="weekly", + help="weekly: incremental check; backfill: full history with subcollection cursor", + ) + args = p.parse_args() + + current_year = datetime.now(tz=timezone.utc).year + + if args.year: + years = [args.year] + elif args.mode == "weekly": + years = [current_year, current_year - 1] + else: + years = list(range(FIRST_YEAR, current_year + 1)) + + project = os.environ.get("GOOGLE_CLOUD_PROJECT") + db = firestore.Client(project=project) if not args.dry_run else None + + if args.mode == "weekly": + n = run_weekly(db, years, limit=args.limit, dry_run=args.dry_run) + if n == 0: + print("\nNo new disclosures found.") + else: + print(f"\nDone: {n} new disclosures written.") + else: + n = run_backfill(db, years, limit=args.limit, dry_run=args.dry_run) + print(f"\nBackfill complete: {n} new disclosures written.") + + # Emit structured result for callers (e.g. TypeScript backfill script) + print(json.dumps({"newDisclosures": n}), file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/lobbying-scraper/tests/__init__.py b/lobbying-scraper/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lobbying-scraper/tests/__pycache__/__init__.cpython-37.pyc b/lobbying-scraper/tests/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 000000000..c8d566ec1 Binary files /dev/null and b/lobbying-scraper/tests/__pycache__/__init__.cpython-37.pyc differ diff --git a/lobbying-scraper/tests/__pycache__/test_portal_parser.cpython-37-pytest-7.4.4.pyc b/lobbying-scraper/tests/__pycache__/test_portal_parser.cpython-37-pytest-7.4.4.pyc new file mode 100644 index 000000000..e18d528fe Binary files /dev/null and b/lobbying-scraper/tests/__pycache__/test_portal_parser.cpython-37-pytest-7.4.4.pyc differ diff --git a/lobbying-scraper/tests/fixtures/2007e_disc.html.gz b/lobbying-scraper/tests/fixtures/2007e_disc.html.gz new file mode 100644 index 000000000..4311a97da Binary files /dev/null and b/lobbying-scraper/tests/fixtures/2007e_disc.html.gz differ diff --git a/lobbying-scraper/tests/fixtures/2007e_summ.html.gz b/lobbying-scraper/tests/fixtures/2007e_summ.html.gz new file mode 100644 index 000000000..58d40abbe Binary files /dev/null and b/lobbying-scraper/tests/fixtures/2007e_summ.html.gz differ diff --git a/lobbying-scraper/tests/fixtures/2011e_disc.html.gz b/lobbying-scraper/tests/fixtures/2011e_disc.html.gz new file mode 100644 index 000000000..2a01a7963 Binary files /dev/null and b/lobbying-scraper/tests/fixtures/2011e_disc.html.gz differ diff --git a/lobbying-scraper/tests/fixtures/2011e_summ.html.gz b/lobbying-scraper/tests/fixtures/2011e_summ.html.gz new file mode 100644 index 000000000..d7b525310 Binary files /dev/null and b/lobbying-scraper/tests/fixtures/2011e_summ.html.gz differ diff --git a/lobbying-scraper/tests/fixtures/2011i_disc.html.gz b/lobbying-scraper/tests/fixtures/2011i_disc.html.gz new file mode 100644 index 000000000..84736b4a7 Binary files /dev/null and b/lobbying-scraper/tests/fixtures/2011i_disc.html.gz differ diff --git a/lobbying-scraper/tests/fixtures/2011i_summ.html.gz b/lobbying-scraper/tests/fixtures/2011i_summ.html.gz new file mode 100644 index 000000000..430f68591 Binary files /dev/null and b/lobbying-scraper/tests/fixtures/2011i_summ.html.gz differ diff --git a/lobbying-scraper/tests/fixtures/2016e_disc.html.gz b/lobbying-scraper/tests/fixtures/2016e_disc.html.gz new file mode 100644 index 000000000..279fd59c0 Binary files /dev/null and b/lobbying-scraper/tests/fixtures/2016e_disc.html.gz differ diff --git a/lobbying-scraper/tests/fixtures/2016e_summ.html.gz b/lobbying-scraper/tests/fixtures/2016e_summ.html.gz new file mode 100644 index 000000000..a0bc28b85 Binary files /dev/null and b/lobbying-scraper/tests/fixtures/2016e_summ.html.gz differ diff --git a/lobbying-scraper/tests/fixtures/2024e_disc.html.gz b/lobbying-scraper/tests/fixtures/2024e_disc.html.gz new file mode 100644 index 000000000..9074511f5 Binary files /dev/null and b/lobbying-scraper/tests/fixtures/2024e_disc.html.gz differ diff --git a/lobbying-scraper/tests/fixtures/2024e_summ.html.gz b/lobbying-scraper/tests/fixtures/2024e_summ.html.gz new file mode 100644 index 000000000..cb6181711 Binary files /dev/null and b/lobbying-scraper/tests/fixtures/2024e_summ.html.gz differ diff --git a/lobbying-scraper/tests/fixtures/2024i_disc.html.gz b/lobbying-scraper/tests/fixtures/2024i_disc.html.gz new file mode 100644 index 000000000..44dddc2ef Binary files /dev/null and b/lobbying-scraper/tests/fixtures/2024i_disc.html.gz differ diff --git a/lobbying-scraper/tests/fixtures/2024i_summ.html.gz b/lobbying-scraper/tests/fixtures/2024i_summ.html.gz new file mode 100644 index 000000000..e6dc3b1b6 Binary files /dev/null and b/lobbying-scraper/tests/fixtures/2024i_summ.html.gz differ diff --git a/lobbying-scraper/tests/test_portal_parser.py b/lobbying-scraper/tests/test_portal_parser.py new file mode 100644 index 000000000..03f73aaee --- /dev/null +++ b/lobbying-scraper/tests/test_portal_parser.py @@ -0,0 +1,172 @@ +"""Regression tests for the MA SoS lobbying disclosure parser. + +The portal HTML has four distinct format eras; the parser is the most likely +thing to silently break when the portal changes its markup. These tests parse +committed fixture pages (one per era, employer + individual) and assert known- +correct compensation totals, client/bill counts, era detection, and specific +bug fixes (the "Total amount" summary-row artifact; the "H73;" semicolon bill +separator; hybrid-era Panel1 compensation). + +Fixtures: tests/fixtures/*.html.gz (gzipped real disclosure + summary pages). +""" + +import gzip +import sys +from pathlib import Path + +import pytest +from bs4 import BeautifulSoup + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from portal import ( + _parse_amount, + parse_disclosure_detail, + parse_summary, + year_to_general_court, +) + +FIXTURES = Path(__file__).parent / "fixtures" + + +def _soup(name: str) -> BeautifulSoup: + with gzip.open(FIXTURES / f"{name}.html.gz", "rt", encoding="utf-8") as fh: + return BeautifulSoup(fh.read(), "html.parser") + + +def _comp_total(detail) -> float: + return sum(c.amount for c in detail.compensation if c.amount) + + +# ── Disclosure parsing ──────────────────────────────────────────────────────── + +# (fixture, year, expected_comp, n_clients, n_bills, era_label) +DISCLOSURE_CASES = [ + ("2007e", 2007, 112_500.00, 1, 2, "legacy 2005-2008: entity total under _total_salary_"), + ("2011e", 2011, 641_243.00, 23, 4, "legacy 2009-2013: per-client Compensation received column"), + ("2016e", 2016, 990_474.00, 30, 1357, "hybrid 2014-2018: Panel1 div totals"), + ("2024e", 2024, 115_000.00, 5, 22, "modern 2019+: grdvClientPaidToEntity"), + ("2024i", 2024, 1_095_200.0, 17, 135, "modern 2019+ individual"), + ("2011i", 2011, 18_518.00, 1, 0, "legacy 2009-2013 individual"), +] + + +@pytest.mark.parametrize("fix,year,exp_comp,n_clients,n_bills,era", DISCLOSURE_CASES) +def test_compensation_total_and_counts(fix, year, exp_comp, n_clients, n_bills, era): + detail = parse_disclosure_detail(_soup(f"{fix}_disc"), year) + assert _comp_total(detail) == pytest.approx(exp_comp, abs=1.0), f"{fix} ({era}) comp total" + assert len(detail.compensation) == n_clients, f"{fix} ({era}) client count" + assert len(detail.bills) == n_bills, f"{fix} ({era}) bill count" + + +@pytest.mark.parametrize("fix,year,_c,_n,_b,_e", DISCLOSURE_CASES) +def test_no_total_amount_artifact(fix, year, _c, _n, _b, _e): + """The legacy individual summary row (client_name == 'Total amount') must + never be captured as a real client — that bug inflated 2010-2013 comp rows.""" + detail = parse_disclosure_detail(_soup(f"{fix}_disc"), year) + bad = [ + c for c in detail.compensation + if c.client_name in ("Total amount", "Total", "") + ] + assert not bad, f"{fix} produced summary-row artifacts: {bad}" + + +def test_legacy_2007_uses_total_salary_placeholder(): + """2005-2008 has no per-client comp column; comp falls back to the entity + salary total stored under the _total_salary_ placeholder client.""" + detail = parse_disclosure_detail(_soup("2007e_disc"), 2007) + assert [c.client_name for c in detail.compensation] == ["_total_salary_"] + + +def test_legacy_2011_is_per_client_not_placeholder(): + """2009-2013 has a per-client 'Compensation received' column, so comp is + stored under real client names — never the _total_salary_ placeholder.""" + detail = parse_disclosure_detail(_soup("2011e_disc"), 2011) + names = [c.client_name for c in detail.compensation] + assert "_total_salary_" not in names + assert len(names) == len(set(names)), "per-client compensation should be deduplicated" + + +def test_hybrid_2016_has_nonzero_compensation(): + """2014-2018 compensation comes from Panel1 divs; was silently $0 before fix.""" + detail = parse_disclosure_detail(_soup("2016e_disc"), 2016) + assert _comp_total(detail) == pytest.approx(990_474.00, abs=1.0) + assert len(detail.compensation) == 30 + + +def test_semicolon_bill_separator_parsed(): + """Legacy bill tokens may use 'H73; Title' (semicolon separator) instead of + a space; the bill number must still be parsed correctly.""" + detail = parse_disclosure_detail(_soup("2011e_disc"), 2011) + house_numbers = {b.raw_bill_number for b in detail.bills if b.chamber == "House Bill"} + assert "73" in house_numbers, "H73 (semicolon-separated) should be parsed" + + +def test_modern_individual_per_client_comp(): + """Modern individual registrants report per-client compensation in + grdvClientPaidToEntity — verify it is captured.""" + detail = parse_disclosure_detail(_soup("2024i_disc"), 2024) + assert _comp_total(detail) > 0 + assert all( + c.client_name not in ("Total amount", "") for c in detail.compensation + ) + + +# ── Summary page parsing ────────────────────────────────────────────────────── + +# (fixture, entity_name, year, reg_type, n_disc_urls) +SUMMARY_CASES = [ + ("2007e_summ", "Ventry Associates, LLP", 2007, "Employer", 2), + ("2011e_summ", "ML Strategies, LLC", 2011, "Employer", 7), + ("2024e_summ", "21c, LLC", 2024, "Employer", 2), + ("2024i_summ", "Anthony Arthur Abdelahad", 2024, "Lobbyist", 2), + ("2011i_summ", "Aaron Judd Agulnek", 2011, "Lobbyist", 4), +] + + +@pytest.mark.parametrize("fix,name,year,reg_type,n_urls", SUMMARY_CASES) +def test_summary_metadata(fix, name, year, reg_type, n_urls): + meta = parse_summary(_soup(fix)) + assert meta.entity_name == name + assert meta.year == year + assert meta.reg_type == reg_type + assert len(meta.disclosure_urls) == n_urls + assert all("CompleteDisclosure" in u for u in meta.disclosure_urls) + + +# ── Helper functions ────────────────────────────────────────────────────────── + +def test_parse_amount(): + assert _parse_amount("$1,234.56") == 1234.56 + assert _parse_amount("$0.00") == 0.0 + assert _parse_amount("") is None + assert _parse_amount("N/A") is None + + +def test_year_to_general_court(): + assert year_to_general_court(2003) == 183 + assert year_to_general_court(2004) == 183 + assert year_to_general_court(2005) == 184 + assert year_to_general_court(2023) == 193 + assert year_to_general_court(2025) == 194 + + +# ── Bill ID construction ────────────────────────────────────────────────────── + +def test_bill_ids_in_modern_disclosure(): + """Verify bill_id is correctly constructed for each chamber prefix.""" + detail = parse_disclosure_detail(_soup("2024e_disc"), 2024) + house = [b for b in detail.bills if b.chamber == "House Bill"] + senate = [b for b in detail.bills if b.chamber == "Senate Bill"] + if house: + assert all(b.bill_id and b.bill_id.startswith("H") for b in house) + if senate: + assert all(b.bill_id and b.bill_id.startswith("S") for b in senate) + + +def test_executive_rows_have_null_bill_id(): + """Executive chamber rows must produce null billId so no accidental bill join occurs.""" + detail = parse_disclosure_detail(_soup("2024i_disc"), 2024) + executive = [b for b in detail.bills if b.chamber == "Executive"] + if executive: + assert all(b.bill_id is None for b in executive) diff --git a/lobbying-scraper/writer.py b/lobbying-scraper/writer.py new file mode 100644 index 000000000..d49d12424 --- /dev/null +++ b/lobbying-scraper/writer.py @@ -0,0 +1,123 @@ +"""Firestore document construction and write helpers. + +Mirrors the data model in functions/src/lobbying/types.ts. All collection +names and field names must stay in sync with that file. +""" + +from __future__ import annotations + +from datetime import datetime, timezone + +from google.cloud import firestore +from normalize import normalize_entity_name +from portal import ( + BillActivity, + Compensation, + DisclosureDetail, + DisclosureMeta, + filing_id, + registrant_id, + year_to_general_court, +) + +REGISTRANTS_COLLECTION = "lobbyingRegistrants" +FILINGS_COLLECTION = "lobbyingFilings" +SCRAPER_DOC = "scrapers/lobbying" +BACKFILL_DOC = "scrapers/lobbyingBackfill" +BACKFILL_URLS_COLLECTION = "processedUrls" + + +def _now() -> datetime: + return datetime.now(tz=timezone.utc) + + +def write_registrant( + db: firestore.Client, + meta: DisclosureMeta, + detail: DisclosureDetail, + disc_url: str, +) -> None: + """Upsert a LobbyingRegistrant document.""" + if not meta.entity_name or meta.year is None: + return + + doc_id = registrant_id(meta.entity_name, meta.year) + ref = db.collection(REGISTRANTS_COLLECTION).document(doc_id) + + clients = [ + { + "clientName": c.client_name, + "clientNameNorm": normalize_entity_name(c.client_name), + "compensation": c.amount, + } + for c in detail.compensation + ] + + data = { + "registrantId": doc_id, + "entityName": meta.entity_name, + "entityNameNorm": normalize_entity_name(meta.entity_name), + "year": meta.year, + "generalCourt": year_to_general_court(meta.year), + "regType": meta.reg_type, + "clients": clients, + "disclosureUrls": firestore.ArrayUnion([disc_url]), + "fetchedAt": _now(), + } + ref.set(data, merge=True) + + +def write_filings( + db: firestore.Client, + meta: DisclosureMeta, + detail: DisclosureDetail, +) -> int: + """Batch-write LobbyingFiling documents. Returns the number written.""" + if not meta.entity_name or meta.year is None or not detail.bills: + return 0 + + gc = year_to_general_court(meta.year) + entity_name = meta.entity_name + entity_norm = normalize_entity_name(entity_name) + now = _now() + + batch = db.batch() + count = 0 + + for bill in detail.bills: + fid = filing_id( + entity_name, + bill.client_name, + bill.chamber, + bill.bill_id, + gc, + bill.position, + ) + ref = db.collection(FILINGS_COLLECTION).document(fid) + doc = { + "filingId": fid, + "entityName": entity_name, + "entityNameNorm": entity_norm, + "clientName": bill.client_name, + "clientNameNorm": normalize_entity_name(bill.client_name), + "year": meta.year, + "generalCourt": gc, + "chamber": bill.chamber, + "billId": bill.bill_id, + "activityTitle": bill.activity_title, + "position": bill.position, + "amount": bill.amount, + "fetchedAt": now, + } + batch.set(ref, doc) + count += 1 + + # Firestore batch limit is 500 writes + if count % 400 == 0: + batch.commit() + batch = db.batch() + + if count % 400 != 0: + batch.commit() + + return count diff --git a/package.json b/package.json index 9a0329989..6d2d70fd3 100644 --- a/package.json +++ b/package.json @@ -131,6 +131,7 @@ "react-query": "^3.39.3", "react-redux": "^8.0.2", "react-select": "^5.2.2", + "recharts": "^3.9.2", "redux-mock-store": "^1.5.4", "redux-thunk": "^3.1.0", "runtypes": "6.6.0", diff --git a/pages/bills/[court]/[billId].tsx b/pages/bills/[court]/[billId].tsx index b60308a7d..faa181aa2 100644 --- a/pages/bills/[court]/[billId].tsx +++ b/pages/bills/[court]/[billId].tsx @@ -40,6 +40,7 @@ export const getServerSideProps: GetServerSideProps = async ctx => { "auth", "common", "footer", + "lobbying", "testimony", "profile" ])) diff --git a/pages/lobbying/bills/[court]/[billId].tsx b/pages/lobbying/bills/[court]/[billId].tsx new file mode 100644 index 000000000..b95c170d5 --- /dev/null +++ b/pages/lobbying/bills/[court]/[billId].tsx @@ -0,0 +1,407 @@ +import React, { useEffect, useMemo, useState } from "react" +import { useTranslation } from "next-i18next" +import { useRouter } from "next/router" +import { Col, Container, Row } from "components/bootstrap" +import { createPage } from "components/page" +import { createGetStaticTranslationProps } from "components/translations" +import { useLobbyingFilingsForBill } from "components/db/lobbying" +import { LobbyingFilingsTable } from "components/lobbying/LobbyingFilingsTable" +import { LobbyingSubnav } from "components/lobbying/LobbyingSubnav" +import { LobbyingAttribution } from "components/lobbying/LobbyingAttribution" +import { usePagination } from "components/lobbying/usePagination" +import { LobbyingPaginationBar } from "components/lobbying/LobbyingPaginationBar" +import { normalizePosition } from "components/lobbying/LobbyingPositionChip" +import { MAPLE_COLORS } from "components/lobbying/chartTheme" +import type { LobbyingFiling } from "functions/src/lobbying/types" +import styles from "components/lobbying/lobbying.module.css" + +const PAGE_SIZE = 50 + +type SortKey = "client" | "firm" | "position" | "amount" | "year" +type SortDir = "asc" | "desc" + +function sortFilings( + filings: LobbyingFiling[], + key: SortKey, + dir: SortDir +): LobbyingFiling[] { + const mul = dir === "asc" ? 1 : -1 + return [...filings].sort((a, b) => { + switch (key) { + case "client": + return mul * a.clientName.localeCompare(b.clientName) + case "firm": + return mul * a.entityName.localeCompare(b.entityName) + case "position": + return ( + mul * + normalizePosition(a.position).localeCompare( + normalizePosition(b.position) + ) + ) + case "amount": + return mul * ((a.amount ?? -1) - (b.amount ?? -1)) + case "year": + return mul * (a.year - b.year) + default: + return 0 + } + }) +} + +function SortTh({ + label, + sortKey, + current, + dir, + onSort, + style, + className +}: { + label: string + sortKey: SortKey + current: SortKey + dir: SortDir + onSort: (k: SortKey) => void + style?: React.CSSProperties + className?: string +}) { + const active = sortKey === current + const indicator = active ? (dir === "asc" ? " ↑" : " ↓") : "" + return ( + onSort(sortKey)} + className={className} + style={{ + ...thStyle, + ...style, + cursor: "pointer", + color: active ? MAPLE_COLORS.primary : MAPLE_COLORS.textMuted, + userSelect: "none" + }} + > + {label} + {indicator} + + ) +} + +function BillFilingsPage() { + const { t } = useTranslation("lobbying") + const { query } = useRouter() + const court = Number(query.court) + const billId = query.billId as string | undefined + + const { + result: filings, + status, + error + } = useLobbyingFilingsForBill(court, billId ?? "") + + const [sortKey, setSortKey] = useState("year") + const [sortDir, setSortDir] = useState("desc") + const [posFilter, setPosFilter] = useState< + "all" | "support" | "oppose" | "neutral" + >("all") + + function handleSort(key: SortKey) { + if (key === sortKey) { + setSortDir(d => (d === "asc" ? "desc" : "asc")) + } else { + setSortKey(key) + setSortDir(key === "client" || key === "firm" ? "asc" : "desc") + } + } + + const filtered = useMemo(() => { + if (!filings) return [] + if (posFilter === "all") return filings + return filings.filter(f => normalizePosition(f.position) === posFilter) + }, [filings, posFilter]) + + const sorted = useMemo( + () => sortFilings(filtered, sortKey, sortDir), + [filtered, sortKey, sortDir] + ) + + const { page, setPage, pageItems, totalPages, totalItems } = usePagination( + sorted, + PAGE_SIZE + ) + + useEffect(() => { + setPage(1) + }, [posFilter, sortKey, sortDir, setPage]) + + const loading = status === "loading" || status === "not-requested" + const mapleHref = court && billId ? `/bills/${court}/${billId}` : undefined + + if (!billId) return null + + return ( + <> + + + + + + ← {t("titles.bills")} + +

{billId}

+ {mapleHref && ( +

+ + {t("misc.viewOnMaple")} + +

+ )} + {!loading && filings && ( +

+ {filings.length} {t("fields.filings").toLowerCase()} +

+ )} + + + + {loading && ( +

{t("loading")}

+ )} + {status === "error" && ( +

Error: {error?.message}

+ )} + + {!loading && ( + <> + + +
+ + + {totalItems} {t("fields.filings").toLowerCase()} + +
+ + + + +
+ + + + + + + + + + + + + {pageItems.map(f => ( + + + + + + + + + ))} + +
+ {t("fields.activity")} +
+ + {f.clientName} + + + + {f.entityName} + + + {f.activityTitle || "—"} + + + {f.position || "—"} + + + {f.amount != null + ? f.amount.toLocaleString("en-US", { + style: "currency", + currency: "USD", + maximumFractionDigits: 0 + }) + : "—"} + {f.year}
+
+ + + + + )} + + + + ) +} + +export default createPage({ + titleI18nKey: "titles.lobbying", + Page: BillFilingsPage +}) + +export const getStaticProps = createGetStaticTranslationProps([ + "auth", + "common", + "footer", + "lobbying" +]) + +export async function getStaticPaths() { + return { paths: [], fallback: "blocking" } +} + +// ── Styles ──────────────────────────────────────────────────────────────────── + +const filterRowStyle: React.CSSProperties = { + display: "flex", + gap: "1rem", + alignItems: "center", + marginBottom: "0.5rem" +} + +const selectStyle: React.CSSProperties = { + padding: "0.3rem 0.6rem", + border: `1px solid ${MAPLE_COLORS.borderDefault}`, + borderRadius: 6, + fontSize: 14, + color: MAPLE_COLORS.textBody, + background: MAPLE_COLORS.surfaceBase, + cursor: "pointer" +} + +const tableStyle: React.CSSProperties = { + width: "100%", + borderCollapse: "collapse", + fontSize: 13, + color: MAPLE_COLORS.textBody +} + +const theadRowStyle: React.CSSProperties = { + borderBottom: `2px solid ${MAPLE_COLORS.borderDefault}` +} + +const thStyle: React.CSSProperties = { + padding: "0.4rem 0.75rem", + textAlign: "left", + fontSize: 11, + fontWeight: 700, + textTransform: "uppercase" as const, + letterSpacing: "0.05em", + color: MAPLE_COLORS.textMuted, + whiteSpace: "nowrap" as const +} + +const trStyle: React.CSSProperties = { + borderBottom: `1px solid ${MAPLE_COLORS.borderSubtle}` +} + +const cellStyle: React.CSSProperties = { + padding: "0.45rem 0.75rem", + verticalAlign: "middle" +} + +const chipStyle: React.CSSProperties = { + display: "inline-block", + padding: "0.15rem 0.5rem", + borderRadius: 12, + fontSize: 11, + fontWeight: 600, + border: "1px solid", + whiteSpace: "nowrap" as const +} + +function posChipStyle(pos: string): React.CSSProperties { + if (pos === "support") + return { color: MAPLE_COLORS.green, borderColor: MAPLE_COLORS.green } + if (pos === "oppose") + return { color: MAPLE_COLORS.danger, borderColor: MAPLE_COLORS.danger } + return { + color: MAPLE_COLORS.textMuted, + borderColor: MAPLE_COLORS.borderDefault + } +} diff --git a/pages/lobbying/bills/index.tsx b/pages/lobbying/bills/index.tsx new file mode 100644 index 000000000..51203e95b --- /dev/null +++ b/pages/lobbying/bills/index.tsx @@ -0,0 +1,453 @@ +import React, { useEffect, useMemo, useState } from "react" +import { useTranslation } from "next-i18next" +import { Col, Container, Row } from "components/bootstrap" +import { createPage } from "components/page" +import { createGetStaticTranslationProps } from "components/translations" +import { useLobbyingFilingsForCourt } from "components/db/lobbying" +import { + LobbyingPositionChip, + normalizePosition +} from "components/lobbying/LobbyingPositionChip" +import { MAPLE_COLORS } from "components/lobbying/chartTheme" +import { LobbyingAttribution } from "components/lobbying/LobbyingAttribution" +import { usePagination } from "components/lobbying/usePagination" +import { LobbyingPaginationBar } from "components/lobbying/LobbyingPaginationBar" +import { LobbyingSubnav } from "components/lobbying/LobbyingSubnav" +import styles from "components/lobbying/lobbying.module.css" + +const COURTS = [194, 193, 192, 191, 190, 189, 188, 187, 186, 185, 184] +const PAGE_SIZE = 50 + +type BillSortKey = "id" | "filings" | "support" | "oppose" | "neutral" +type SortDir = "asc" | "desc" + +function SortTh({ + label, + sortKey, + current, + dir, + onSort, + style, + className +}: { + label: string + sortKey: BillSortKey + current: BillSortKey + dir: SortDir + onSort: (k: BillSortKey) => void + style?: React.CSSProperties + className?: string +}) { + const active = sortKey === current + return ( + onSort(sortKey)} + className={className} + style={{ + ...thStyle, + ...style, + cursor: "pointer", + color: active ? MAPLE_COLORS.primary : MAPLE_COLORS.textMuted, + userSelect: "none" + }} + > + {label} + {active && ( + + {dir === "asc" ? "↑" : "↓"} + + )} + + ) +} + +type BillRow = { + billId: string + court: number + total: number + support: number + oppose: number + neutral: number + none: number +} + +function groupByBill( + filings: ReturnType["result"] +): BillRow[] { + if (!filings) return [] + const map = new Map() + for (const f of filings) { + if (!f.billId || f.billId.length <= 2) continue + if (!map.has(f.billId)) { + map.set(f.billId, { + billId: f.billId, + court: f.generalCourt, + total: 0, + support: 0, + oppose: 0, + neutral: 0, + none: 0 + }) + } + const row = map.get(f.billId)! + row[normalizePosition(f.position)]++ + row.total++ + } + return [...map.values()] +} + +function PositionMiniBar({ row }: { row: BillRow }) { + if (row.total === 0) return null + const pct = (n: number) => `${((n / row.total) * 100).toFixed(1)}%` + return ( +
+ {row.support > 0 && ( +
+ )} + {row.oppose > 0 && ( +
+ )} + {row.neutral > 0 && ( +
+ )} +
+ ) +} + +function LobbyingBillsTable() { + const { t } = useTranslation("lobbying") + const [court, setCourt] = useState(194) + const [posFilter, setPosFilter] = useState< + "all" | "support" | "oppose" | "neutral" + >("all") + const [search, setSearch] = useState("") + const [sortKey, setSortKey] = useState("filings") + const [sortDir, setSortDir] = useState("desc") + + function handleSort(key: BillSortKey) { + if (key === sortKey) { + setSortDir(d => (d === "asc" ? "desc" : "asc")) + } else { + setSortKey(key) + setSortDir(key === "id" ? "asc" : "desc") + } + } + + const { result: filings, status, error } = useLobbyingFilingsForCourt(court) + + const bills = useMemo(() => groupByBill(filings), [filings]) + + const filtered = useMemo(() => { + return bills.filter(b => { + if (posFilter !== "all" && b[posFilter] === 0) return false + if (search && !b.billId.toLowerCase().includes(search.toLowerCase())) + return false + return true + }) + }, [bills, posFilter, search]) + + const sorted = useMemo(() => { + const mul = sortDir === "asc" ? 1 : -1 + return [...filtered].sort((a, b) => { + switch (sortKey) { + case "id": + return mul * a.billId.localeCompare(b.billId) + case "support": + return mul * (a.support - b.support) + case "oppose": + return mul * (a.oppose - b.oppose) + case "neutral": + return mul * (a.neutral - b.neutral) + default: + return mul * (a.total - b.total) + } + }) + }, [filtered, sortKey, sortDir]) + + const { page, setPage, pageItems, totalPages, totalItems } = usePagination( + sorted, + PAGE_SIZE + ) + + useEffect(() => { + setPage(1) + }, [court, posFilter, search, sortKey, sortDir, setPage]) + + return ( + <> +
+ + + + + setSearch(e.target.value)} + style={searchStyle} + aria-label={t("filters.search")} + /> +
+ + {status === "loading" || status === "not-requested" ? ( +

+ {t("loading")} +

+ ) : status === "error" ? ( +

+ Error: {error?.message} +

+ ) : sorted.length === 0 ? ( +

+ {t("noData")} +

+ ) : ( + <> +

+ {totalItems} {t("sections.bills").toLowerCase()} +

+
+ + + + + + + + + + + + + + {pageItems.map(b => ( + + + + + + + + + + ))} + +
+ {t("filters.session")} + {t("fields.positions")}
+ + {b.billId} + + + {b.court} + {b.total} + {b.support > 0 && ( + + )}{" "} + {b.support > 0 && ( + {b.support} + )} + + {b.oppose > 0 && ( + + )}{" "} + {b.oppose > 0 && ( + {b.oppose} + )} + + {b.neutral > 0 && ( + + )}{" "} + {b.neutral > 0 && ( + {b.neutral} + )} + + +
+
+ + + )} + + + + ) +} + +function LobbyingBillsPage() { + const { t } = useTranslation("lobbying") + return ( + <> + + + + +

{t("titles.bills")}

+ + + + + + + + + + ) +} + +export default createPage({ + titleI18nKey: "titles.lobbying", + Page: LobbyingBillsPage +}) + +export const getStaticProps = createGetStaticTranslationProps([ + "auth", + "common", + "footer", + "lobbying" +]) + +// ── Styles ──────────────────────────────────────────────────────────────────── + +const filterRowStyle: React.CSSProperties = { + display: "flex", + gap: "0.75rem", + flexWrap: "wrap", + marginBottom: "1rem", + alignItems: "center" +} + +const selectStyle: React.CSSProperties = { + padding: "0.35rem 0.65rem", + border: `1px solid ${MAPLE_COLORS.borderDefault}`, + borderRadius: 6, + fontSize: 14, + color: MAPLE_COLORS.textBody, + background: MAPLE_COLORS.surfaceBase, + cursor: "pointer" +} + +const searchStyle: React.CSSProperties = { + ...selectStyle, + minWidth: 180, + cursor: "text" +} + +const tableStyle: React.CSSProperties = { + width: "100%", + borderCollapse: "collapse", + fontSize: 13, + color: MAPLE_COLORS.textBody +} + +const theadStyle: React.CSSProperties = { + borderBottom: `2px solid ${MAPLE_COLORS.borderDefault}` +} + +const thStyle: React.CSSProperties = { + padding: "0.4rem 0.75rem", + textAlign: "left", + fontSize: 11, + fontWeight: 700, + textTransform: "uppercase", + letterSpacing: "0.05em", + color: MAPLE_COLORS.textMuted, + whiteSpace: "nowrap" +} + +const trStyle: React.CSSProperties = { + borderBottom: `1px solid ${MAPLE_COLORS.borderSubtle}` +} + +const tdStyle: React.CSSProperties = { + padding: "0.45rem 0.75rem", + verticalAlign: "middle" +} diff --git a/pages/lobbying/clients/[clientSlug].tsx b/pages/lobbying/clients/[clientSlug].tsx new file mode 100644 index 000000000..c4fcc6cb9 --- /dev/null +++ b/pages/lobbying/clients/[clientSlug].tsx @@ -0,0 +1,301 @@ +import React, { useEffect, useMemo, useState } from "react" +import { useTranslation } from "next-i18next" +import { useRouter } from "next/router" +import { Col, Container, Row } from "components/bootstrap" +import { createPage } from "components/page" +import { createGetStaticTranslationProps } from "components/translations" +import { + useLobbyingFilingsForClient, + useLobbyingAllRegistrants +} from "components/db/lobbying" +import { LobbyingFilingsTable } from "components/lobbying/LobbyingFilingsTable" +import { + LobbyingPositionChip, + normalizePosition +} from "components/lobbying/LobbyingPositionChip" +import { MAPLE_COLORS } from "components/lobbying/chartTheme" +import { LobbyingAttribution } from "components/lobbying/LobbyingAttribution" +import { LobbyingSubnav } from "components/lobbying/LobbyingSubnav" +import { usePagination } from "components/lobbying/usePagination" +import { LobbyingPaginationBar } from "components/lobbying/LobbyingPaginationBar" +import type { LobbyingRegistrant } from "functions/src/lobbying/types" + +const PAGE_SIZE = 25 + +function findFirmsForClient( + registrants: LobbyingRegistrant[] | undefined, + clientNameNorm: string +): Array<{ + entityName: string + entityNameNorm: string + compensation: number | null +}> { + if (!registrants) return [] + const map = new Map< + string, + { entityName: string; entityNameNorm: string; compensation: number | null } + >() + for (const r of registrants) { + const match = r.clients.find(c => c.clientNameNorm === clientNameNorm) + if (!match) continue + if (!map.has(r.entityNameNorm)) { + map.set(r.entityNameNorm, { + entityName: r.entityName, + entityNameNorm: r.entityNameNorm, + compensation: null + }) + } + const entry = map.get(r.entityNameNorm)! + if (match.compensation != null) { + entry.compensation = (entry.compensation ?? 0) + match.compensation + } + } + return [...map.values()].sort((a, b) => + a.entityNameNorm.localeCompare(b.entityNameNorm) + ) +} + +function ClientDetail() { + const { t } = useTranslation("lobbying") + const { query } = useRouter() + + const clientNameNorm = query.clientSlug + ? decodeURIComponent(query.clientSlug as string) + : "" + + const { + result: filings, + status: filStatus, + error: filError + } = useLobbyingFilingsForClient(clientNameNorm) + const { result: registrants, status: regStatus } = useLobbyingAllRegistrants() + + const firms = useMemo( + () => findFirmsForClient(registrants, clientNameNorm), + [registrants, clientNameNorm] + ) + + const positionCounts = useMemo(() => { + const counts = { support: 0, oppose: 0, neutral: 0, none: 0 } + for (const f of filings ?? []) counts[normalizePosition(f.position)]++ + return counts + }, [filings]) + + const totalCompensation = useMemo( + () => + firms.reduce( + (sum, f) => (f.compensation != null ? sum + f.compensation : sum), + 0 + ), + [firms] + ) + + const years = useMemo( + () => [...new Set((filings ?? []).map(f => f.year))].sort((a, b) => b - a), + [filings] + ) + + const sortedFilings = useMemo( + () => + [...(filings ?? [])].sort( + (a, b) => + b.year - a.year || (a.billId ?? "").localeCompare(b.billId ?? "") + ), + [filings] + ) + + const { page, setPage, pageItems, totalPages, totalItems } = usePagination( + sortedFilings, + PAGE_SIZE + ) + + useEffect(() => { + setPage(1) + }, [clientNameNorm, setPage]) + + const loading = + filStatus === "loading" || + filStatus === "not-requested" || + regStatus === "loading" || + regStatus === "not-requested" + + const displayName = filings?.[0]?.clientName ?? clientNameNorm + + if (!clientNameNorm) return null + + return ( + <> + + + + + + ← {t("titles.clients")} + +

{displayName}

+

+ {years.length > 0 && + (years.length === 1 + ? years[0] + : `${years[years.length - 1]}–${years[0]}`)} + {filings && filings.length > 0 && ( + <> +  ·  {filings.length}{" "} + {t("fields.filings").toLowerCase()} + + )} + {totalCompensation > 0 && ( + <> +  · {" "} + {totalCompensation.toLocaleString("en-US", { + style: "currency", + currency: "USD", + maximumFractionDigits: 0 + })}{" "} + {t("misc.total")} + + )} +

+ + + + {loading && ( +

{t("loading")}

+ )} + {filStatus === "error" && ( +

+ Error: {filError?.message} +

+ )} + + {!loading && ( + + +
{t("sections.bills")}
+ + + + + +
{t("sections.firms")}
+ {firms.length === 0 ? ( +

—

+ ) : ( +
    + {firms.map(f => ( +
  • + + {f.entityName} + + {f.compensation != null && ( + + {" "} + ( + {f.compensation.toLocaleString("en-US", { + style: "currency", + currency: "USD", + maximumFractionDigits: 0 + })} + ) + + )} +
    • + ))} +
+ )} + + {(filings?.length ?? 0) > 0 && ( + <> +
+ {t("filters.position")} +
+
+ {(["support", "oppose", "neutral"] as const).map( + pos => + positionCounts[pos] > 0 && ( +
+ + + {positionCounts[pos]} + +
+ ) + )} +
+ + )} + + + )} + + + + ) +} + +export default createPage({ + titleI18nKey: "titles.lobbying", + Page: ClientDetail +}) + +export const getStaticProps = createGetStaticTranslationProps([ + "auth", + "common", + "footer", + "lobbying" +]) + +export async function getStaticPaths() { + return { paths: [], fallback: "blocking" } +} + +const sectionHeadStyle: React.CSSProperties = { + fontSize: 12, + fontWeight: 700, + textTransform: "uppercase", + letterSpacing: "0.06em", + color: MAPLE_COLORS.textMuted, + marginBottom: "0.6rem" +} diff --git a/pages/lobbying/clients/index.tsx b/pages/lobbying/clients/index.tsx new file mode 100644 index 000000000..29f5712c7 --- /dev/null +++ b/pages/lobbying/clients/index.tsx @@ -0,0 +1,338 @@ +import React, { useEffect, useMemo, useState } from "react" +import { useTranslation } from "next-i18next" +import { Col, Container, Row } from "components/bootstrap" +import { createPage } from "components/page" +import { createGetStaticTranslationProps } from "components/translations" +import { useLobbyingAllRegistrants } from "components/db/lobbying" +import { MAPLE_COLORS } from "components/lobbying/chartTheme" +import { LobbyingAttribution } from "components/lobbying/LobbyingAttribution" +import { usePagination } from "components/lobbying/usePagination" +import { LobbyingPaginationBar } from "components/lobbying/LobbyingPaginationBar" +import type { LobbyingRegistrant } from "functions/src/lobbying/types" +import { LobbyingSubnav } from "components/lobbying/LobbyingSubnav" + +const LEGACY_TOTAL_CLIENT = "_total_salary_" +const PAGE_SIZE = 50 + +type ClientSortKey = "name" | "compensation" | "firms" +type SortDir = "asc" | "desc" + +function SortTh({ + label, + sortKey, + current, + dir, + onSort, + style +}: { + label: string + sortKey: ClientSortKey + current: ClientSortKey + dir: SortDir + onSort: (k: ClientSortKey) => void + style?: React.CSSProperties +}) { + const active = sortKey === current + return ( + onSort(sortKey)} + style={{ + ...thStyle, + ...style, + cursor: "pointer", + color: active ? MAPLE_COLORS.primary : MAPLE_COLORS.textMuted, + userSelect: "none" + }} + > + {label} + {active && ( + + {dir === "asc" ? "↑" : "↓"} + + )} + + ) +} + +type ClientRow = { + clientName: string + clientNameNorm: string + totalCompensation: number | null + registrantCount: number +} + +function deriveClients( + registrants: LobbyingRegistrant[] | undefined +): ClientRow[] { + if (!registrants) return [] + const map = new Map() + for (const r of registrants) { + for (const c of r.clients) { + if ( + !c.clientNameNorm || + c.clientNameNorm === LEGACY_TOTAL_CLIENT || + c.clientName === LEGACY_TOTAL_CLIENT + ) + continue + if (!map.has(c.clientNameNorm)) { + map.set(c.clientNameNorm, { + clientName: c.clientName, + clientNameNorm: c.clientNameNorm, + totalCompensation: null, + registrantCount: 0 + }) + } + const row = map.get(c.clientNameNorm)! + row.registrantCount++ + if (c.compensation != null) { + row.totalCompensation = (row.totalCompensation ?? 0) + c.compensation + } + } + } + return [...map.values()] +} + +function LobbyingClientsTable() { + const { t } = useTranslation("lobbying") + const [search, setSearch] = useState("") + const [sortKey, setSortKey] = useState("name") + const [sortDir, setSortDir] = useState("asc") + + function handleSort(key: ClientSortKey) { + if (key === sortKey) { + setSortDir(d => (d === "asc" ? "desc" : "asc")) + } else { + setSortKey(key) + setSortDir(key === "name" ? "asc" : "desc") + } + } + + const { result: registrants, status, error } = useLobbyingAllRegistrants() + const clients = useMemo(() => deriveClients(registrants), [registrants]) + + const filtered = useMemo( + () => + search + ? clients.filter(c => + c.clientName.toLowerCase().includes(search.toLowerCase()) + ) + : clients, + [clients, search] + ) + + const sorted = useMemo(() => { + const mul = sortDir === "asc" ? 1 : -1 + return [...filtered].sort((a, b) => { + switch (sortKey) { + case "compensation": + return ( + mul * ((a.totalCompensation ?? -1) - (b.totalCompensation ?? -1)) + ) + case "firms": + return mul * (a.registrantCount - b.registrantCount) + default: + return mul * a.clientNameNorm.localeCompare(b.clientNameNorm) + } + }) + }, [filtered, sortKey, sortDir]) + + const { page, setPage, pageItems, totalPages, totalItems } = usePagination( + sorted, + PAGE_SIZE + ) + + useEffect(() => { + setPage(1) + }, [search, sortKey, sortDir, setPage]) + + return ( + <> +
+ setSearch(e.target.value)} + style={searchStyle} + /> +
+ + {status === "loading" || status === "not-requested" ? ( +

+ {t("loading")} +

+ ) : status === "error" ? ( +

{error?.message}

+ ) : ( + <> +

+ {totalItems} {t("sections.clients").toLowerCase()} +

+
+ + + + + + + + + + {pageItems.map(c => ( + + + + + + ))} + +
+ + {c.clientName} + + + {c.registrantCount} + + {c.totalCompensation != null + ? c.totalCompensation.toLocaleString("en-US", { + style: "currency", + currency: "USD", + maximumFractionDigits: 0 + }) + : "—"} +
+
+ + + )} + + ) +} + +function LobbyingClientsPage() { + const { t } = useTranslation("lobbying") + return ( + <> + + + + +

{t("titles.clients")}

+ + + + + + + + + + + + + + + ) +} + +export default createPage({ + titleI18nKey: "titles.lobbying", + Page: LobbyingClientsPage +}) + +export const getStaticProps = createGetStaticTranslationProps([ + "auth", + "common", + "footer", + "lobbying" +]) + +const filterRowStyle: React.CSSProperties = { + display: "flex", + gap: "0.75rem", + flexWrap: "wrap", + marginBottom: "1rem", + alignItems: "center" +} +const selectStyle: React.CSSProperties = { + padding: "0.35rem 0.65rem", + border: `1px solid ${MAPLE_COLORS.borderDefault}`, + borderRadius: 6, + fontSize: 14, + color: MAPLE_COLORS.textBody, + background: MAPLE_COLORS.surfaceBase, + cursor: "pointer" +} +const searchStyle: React.CSSProperties = { + ...selectStyle, + minWidth: 260, + cursor: "text" +} +const tableStyle: React.CSSProperties = { + width: "100%", + borderCollapse: "collapse", + fontSize: 13, + color: MAPLE_COLORS.textBody +} +const theadStyle: React.CSSProperties = { + borderBottom: `2px solid ${MAPLE_COLORS.borderDefault}` +} +const thStyle: React.CSSProperties = { + padding: "0.4rem 0.75rem", + textAlign: "left", + fontSize: 11, + fontWeight: 700, + textTransform: "uppercase", + letterSpacing: "0.05em", + color: MAPLE_COLORS.textMuted, + whiteSpace: "nowrap" +} +const trStyle: React.CSSProperties = { + borderBottom: `1px solid ${MAPLE_COLORS.borderSubtle}` +} +const tdStyle: React.CSSProperties = { + padding: "0.45rem 0.75rem", + verticalAlign: "middle" +} diff --git a/pages/lobbying/firms/[registrantId].tsx b/pages/lobbying/firms/[registrantId].tsx new file mode 100644 index 000000000..55c3c18c6 --- /dev/null +++ b/pages/lobbying/firms/[registrantId].tsx @@ -0,0 +1,276 @@ +import React, { useEffect, useMemo, useState } from "react" +import { useTranslation } from "next-i18next" +import { useRouter } from "next/router" +import { Col, Container, Row } from "components/bootstrap" +import { createPage } from "components/page" +import { createGetStaticTranslationProps } from "components/translations" +import { + useLobbyingRegistrantsByEntityName, + useLobbyingFilingsForEntityName +} from "components/db/lobbying" +import { LobbyingFilingsTable } from "components/lobbying/LobbyingFilingsTable" +import { + LobbyingPositionChip, + normalizePosition +} from "components/lobbying/LobbyingPositionChip" +import { MAPLE_COLORS } from "components/lobbying/chartTheme" +import { LobbyingAttribution } from "components/lobbying/LobbyingAttribution" +import { LobbyingSubnav } from "components/lobbying/LobbyingSubnav" +import { usePagination } from "components/lobbying/usePagination" +import { LobbyingPaginationBar } from "components/lobbying/LobbyingPaginationBar" + +const PAGE_SIZE = 25 + +function FirmDetail() { + const { t } = useTranslation("lobbying") + const { query } = useRouter() + + // URL param is the URL-encoded entityNameNorm + const entityNameNorm = query.registrantId + ? decodeURIComponent(query.registrantId as string) + : "" + + const { result: registrants, status: regStatus } = + useLobbyingRegistrantsByEntityName(entityNameNorm) + const { + result: filings, + status: filStatus, + error: filError + } = useLobbyingFilingsForEntityName(entityNameNorm) + + const loading = + regStatus === "loading" || + regStatus === "not-requested" || + filStatus === "loading" || + filStatus === "not-requested" + + const sortedFilings = useMemo( + () => + [...(filings ?? [])].sort( + (a, b) => + b.year - a.year || (a.billId ?? "").localeCompare(b.billId ?? "") + ), + [filings] + ) + + const { page, setPage, pageItems, totalPages, totalItems } = usePagination( + sortedFilings, + PAGE_SIZE + ) + + useEffect(() => { + setPage(1) + }, [entityNameNorm, setPage]) + + if (!entityNameNorm) return null + + // Aggregate from all registrant docs for this entity + const primary = registrants?.[0] + const years = [...new Set(registrants?.map(r => r.year) ?? [])].sort( + (a, b) => b - a + ) + const allClients = [ + ...new Map( + (registrants ?? []) + .flatMap(r => r.clients) + .map(c => [c.clientNameNorm, c]) + ).values() + ].sort((a, b) => a.clientNameNorm.localeCompare(b.clientNameNorm)) + + const allDisclosureUrls = [ + ...new Set((registrants ?? []).flatMap(r => r.disclosureUrls)) + ] + + const positionCounts = { support: 0, oppose: 0, neutral: 0, none: 0 } + for (const f of filings ?? []) positionCounts[normalizePosition(f.position)]++ + + return ( + <> + + + + + + ← {t("titles.firms")} + +

{primary?.entityName ?? entityNameNorm}

+

+ {primary?.regType} + {years.length > 0 && ( + <> +  · {" "} + {years.length === 1 + ? years[0] + : `${years[years.length - 1]}–${years[0]}`} + + )} +  ·  {filings?.length ?? "—"}{" "} + {t("fields.filings").toLowerCase()} +

+ + + + {loading && ( +

{t("loading")}

+ )} + {filStatus === "error" && ( +

+ Error: {filError?.message} +

+ )} + + {!loading && ( + + {/* Left: filings */} + +
{t("sections.bills")}
+ + + + + {/* Right: clients + disclosure links */} + +
{t("sections.clients")}
+ {allClients.length === 0 ? ( +

—

+ ) : ( +
    + {allClients.map(c => ( +
  • + + {c.clientName} + + {c.compensation != null && ( + + {" "} + ( + {c.compensation.toLocaleString("en-US", { + style: "currency", + currency: "USD", + maximumFractionDigits: 0 + })} + ) + + )} +
    • + ))} +
+ )} + + {allDisclosureUrls.length > 0 && ( + <> +
+ {t("misc.disclosures")} +
+ + + )} + + {(filings?.length ?? 0) > 0 && ( + <> +
+ {t("filters.position")} +
+
+ {(["support", "oppose", "neutral"] as const).map( + pos => + positionCounts[pos] > 0 && ( +
+ + + {positionCounts[pos]} + +
+ ) + )} +
+ + )} + + + )} + + + + ) +} + +export default createPage({ + titleI18nKey: "titles.lobbying", + Page: FirmDetail +}) + +export const getStaticProps = createGetStaticTranslationProps([ + "auth", + "common", + "footer", + "lobbying" +]) + +export async function getStaticPaths() { + return { paths: [], fallback: "blocking" } +} + +const sectionHeadStyle: React.CSSProperties = { + fontSize: 12, + fontWeight: 700, + textTransform: "uppercase", + letterSpacing: "0.06em", + color: MAPLE_COLORS.textMuted, + marginBottom: "0.6rem" +} diff --git a/pages/lobbying/firms/index.tsx b/pages/lobbying/firms/index.tsx new file mode 100644 index 000000000..4be174557 --- /dev/null +++ b/pages/lobbying/firms/index.tsx @@ -0,0 +1,336 @@ +import React, { useEffect, useMemo, useState } from "react" +import { useTranslation } from "next-i18next" +import { Col, Container, Row } from "components/bootstrap" +import { createPage } from "components/page" +import { createGetStaticTranslationProps } from "components/translations" +import { useLobbyingAllRegistrants } from "components/db/lobbying" +import { MAPLE_COLORS } from "components/lobbying/chartTheme" +import { LobbyingAttribution } from "components/lobbying/LobbyingAttribution" +import { usePagination } from "components/lobbying/usePagination" +import { LobbyingPaginationBar } from "components/lobbying/LobbyingPaginationBar" +import { LobbyingSubnav } from "components/lobbying/LobbyingSubnav" +import type { LobbyingRegistrant } from "functions/src/lobbying/types" + +const PAGE_SIZE = 50 + +type FirmSortKey = "name" | "clients" | "sessions" +type SortDir = "asc" | "desc" + +function SortTh({ + label, + sortKey, + current, + dir, + onSort, + style +}: { + label: string + sortKey: FirmSortKey + current: FirmSortKey + dir: SortDir + onSort: (k: FirmSortKey) => void + style?: React.CSSProperties +}) { + const active = sortKey === current + return ( + onSort(sortKey)} + style={{ + ...thStyle, + ...style, + cursor: "pointer", + color: active ? MAPLE_COLORS.primary : MAPLE_COLORS.textMuted, + userSelect: "none" + }} + > + {label} + {active && ( + + {dir === "asc" ? "↑" : "↓"} + + )} + + ) +} + +type FirmRow = { + entityName: string + entityNameNorm: string + registrantId: string + regType: string + years: number[] + clientCount: number +} + +function groupByFirm(registrants: LobbyingRegistrant[] | undefined): FirmRow[] { + if (!registrants) return [] + const map = new Map() + for (const r of registrants) { + if (!map.has(r.entityNameNorm)) { + map.set(r.entityNameNorm, { + entityName: r.entityName, + entityNameNorm: r.entityNameNorm, + registrantId: r.registrantId, + regType: r.regType, + years: [], + clientCount: 0 + }) + } + const row = map.get(r.entityNameNorm)! + if (!row.years.includes(r.year)) row.years.push(r.year) + row.clientCount += r.clients.length + } + for (const row of map.values()) row.years.sort((a, b) => b - a) + return [...map.values()] +} + +function LobbyingFirmsTable() { + const { t } = useTranslation("lobbying") + const [regTypeFilter, setRegTypeFilter] = useState< + "all" | "Lobbyist" | "Employer" + >("all") + const [search, setSearch] = useState("") + const [sortKey, setSortKey] = useState("name") + const [sortDir, setSortDir] = useState("asc") + + function handleSort(key: FirmSortKey) { + if (key === sortKey) { + setSortDir(d => (d === "asc" ? "desc" : "asc")) + } else { + setSortKey(key) + setSortDir(key === "name" ? "asc" : "desc") + } + } + + const { result: registrants, status, error } = useLobbyingAllRegistrants() + const firms = useMemo(() => groupByFirm(registrants), [registrants]) + + const filtered = useMemo(() => { + return firms.filter(f => { + if (regTypeFilter !== "all" && f.regType !== regTypeFilter) return false + if (search && !f.entityName.toLowerCase().includes(search.toLowerCase())) + return false + return true + }) + }, [firms, regTypeFilter, search]) + + const sorted = useMemo(() => { + const mul = sortDir === "asc" ? 1 : -1 + return [...filtered].sort((a, b) => { + switch (sortKey) { + case "clients": + return mul * (a.clientCount - b.clientCount) + case "sessions": + return mul * (a.years.length - b.years.length) + default: + return mul * a.entityNameNorm.localeCompare(b.entityNameNorm) + } + }) + }, [filtered, sortKey, sortDir]) + + const { page, setPage, pageItems, totalPages, totalItems } = usePagination( + sorted, + PAGE_SIZE + ) + + useEffect(() => { + setPage(1) + }, [regTypeFilter, search, sortKey, sortDir, setPage]) + + return ( + <> +
+ + + setSearch(e.target.value)} + style={{ ...selectStyle, minWidth: 220, cursor: "text" }} + aria-label={t("filters.search")} + /> +
+ + {status === "loading" || status === "not-requested" ? ( +

+ {t("loading")} +

+ ) : status === "error" ? ( +

+ Error: {error?.message} +

+ ) : ( + <> +

+ {totalItems} {t("sections.firms").toLowerCase()} +

+
+ + + + + + + + + + + {pageItems.map(f => ( + + + + + + + ))} + +
{t("fields.type")}
+ + {f.entityName} + + {f.regType} + {f.years.length === 1 + ? f.years[0] + : `${f.years[f.years.length - 1]}–${f.years[0]}`} + {f.clientCount}
+
+ + + )} + + ) +} + +function LobbyingFirmsPage() { + const { t } = useTranslation("lobbying") + return ( + <> + + + + +

{t("titles.firms")}

+ + + + + + + + + + + + + + + ) +} + +export default createPage({ + titleI18nKey: "titles.lobbying", + Page: LobbyingFirmsPage +}) + +export const getStaticProps = createGetStaticTranslationProps([ + "auth", + "common", + "footer", + "lobbying" +]) + +const filterRowStyle: React.CSSProperties = { + display: "flex", + gap: "0.75rem", + flexWrap: "wrap", + marginBottom: "1rem", + alignItems: "center" +} +const selectStyle: React.CSSProperties = { + padding: "0.35rem 0.65rem", + border: `1px solid ${MAPLE_COLORS.borderDefault}`, + borderRadius: 6, + fontSize: 14, + color: MAPLE_COLORS.textBody, + background: MAPLE_COLORS.surfaceBase, + cursor: "pointer" +} +const tableStyle: React.CSSProperties = { + width: "100%", + borderCollapse: "collapse", + fontSize: 13, + color: MAPLE_COLORS.textBody +} +const theadStyle: React.CSSProperties = { + borderBottom: `2px solid ${MAPLE_COLORS.borderDefault}` +} +const thStyle: React.CSSProperties = { + padding: "0.4rem 0.75rem", + textAlign: "left", + fontSize: 11, + fontWeight: 700, + textTransform: "uppercase", + letterSpacing: "0.05em", + color: MAPLE_COLORS.textMuted, + whiteSpace: "nowrap" +} +const trStyle: React.CSSProperties = { + borderBottom: `1px solid ${MAPLE_COLORS.borderSubtle}` +} +const tdStyle: React.CSSProperties = { + padding: "0.45rem 0.75rem", + verticalAlign: "middle" +} diff --git a/pages/lobbying/index.tsx b/pages/lobbying/index.tsx new file mode 100644 index 000000000..8e843ccaa --- /dev/null +++ b/pages/lobbying/index.tsx @@ -0,0 +1,377 @@ +import React, { useMemo } from "react" +import { useTranslation } from "next-i18next" +import { Col, Container, Row } from "components/bootstrap" +import { + Bar, + CartesianGrid, + ComposedChart, + Line, + ResponsiveContainer, + Tooltip, + XAxis, + YAxis +} from "recharts" +import { createPage } from "components/page" +import { createGetStaticTranslationProps } from "components/translations" +import { useLobbyingStats } from "components/db/lobbying" +import type { LobbyingStats } from "functions/src/lobbying/types" +import { + BAR_PROPS_COMPACT, + ChartContainer, + formatCount, + formatCurrency, + formatCurrencyCompact, + GRID_PROPS, + LINE_PROPS_EMPHASIS, + MAPLE_COLORS, + MapleTooltip, + X_AXIS_PROPS, + Y_AXIS_PROPS +} from "components/lobbying/chartTheme" +import { LobbyingAttribution } from "components/lobbying/LobbyingAttribution" +import { LobbyingSubnav } from "components/lobbying/LobbyingSubnav" + +// ── Stats bar ───────────────────────────────────────────────────────────────── + +function StatCard({ + label, + value +}: { + label: string + value: string | number | undefined +}) { + return ( +
+
+ {value ?? —} +
+
{label}
+
+ ) +} + +function StatsBar({ stats }: { stats: LobbyingStats | undefined }) { + const { t } = useTranslation("lobbying") + + const totalSpend = useMemo(() => { + if (!stats) return undefined + const total = Object.values(stats.spendByYear).reduce((s, v) => s + v, 0) + return total > 0 ? formatCurrency(total) : undefined + }, [stats]) + + return ( +
+ + + + +
+ ) +} + +// ── Spend + filings chart ───────────────────────────────────────────────────── + +type YearRow = { year: number; spend: number; filings: number } + +function buildYearData(stats: LobbyingStats): YearRow[] { + const years = new Set([ + ...Object.keys(stats.spendByYear), + ...Object.keys(stats.filingsByYear) + ]) + return [...years] + .map(y => ({ + year: Number(y), + spend: stats.spendByYear[y] ?? 0, + filings: stats.filingsByYear[y] ?? 0 + })) + .filter(r => r.year >= 2005) + .sort((a, b) => a.year - b.year) +} + +function SpendFilingsChart({ stats }: { stats: LobbyingStats | undefined }) { + const { t } = useTranslation("lobbying") + const data = useMemo(() => (stats ? buildYearData(stats) : []), [stats]) + + if (!stats || data.length === 0) return null + + return ( + + + + + + + + ( + `${y}`} + valueFormatter={v => { + const n = Number(v) + return n > 10_000 ? formatCurrency(n) : formatCount(n) + }} + /> + )} + /> + + + + + + ) +} + +// ── Entry cards ─────────────────────────────────────────────────────────────── + +function EntryCard({ + title, + description, + href, + count, + countLabel +}: { + title: string + description: string + href: string + count?: number + countLabel?: string +}) { + return ( + +
+
+
{title}
+
{description}
+
+ {count != null && ( +
+ {count.toLocaleString("en-US")} + {countLabel && ( + + {countLabel} + + )} +
+ )} +
+ + ) +} + +// ── Page ────────────────────────────────────────────────────────────────────── + +function LobbyingOverview() { + const { t } = useTranslation("lobbying") + const { result: stats, status } = useLobbyingStats() + const loading = status === "loading" || status === "not-requested" + + return ( + <> + + + + +

{t("titles.overview")}

+

{t("subtitle")}

+ + + + {/* Stats */} + + + {loading ? ( +

+ {t("loading")} +

+ ) : ( + + )} + + + + {/* Chart */} + {!loading && stats && ( + + + + + + )} + + {/* Entry cards */} + + + + + + + + + + + + + + + + + + + + ) +} + +export default createPage({ + titleI18nKey: "titles.lobbying", + Page: LobbyingOverview +}) + +export const getStaticProps = createGetStaticTranslationProps([ + "auth", + "common", + "footer", + "lobbying" +]) + +// ── Styles ──────────────────────────────────────────────────────────────────── + +const statsRowStyle: React.CSSProperties = { + display: "flex", + gap: "1rem", + flexWrap: "wrap" +} + +const statCardStyle: React.CSSProperties = { + flex: "1 1 160px", + background: MAPLE_COLORS.surfaceBase, + border: `1px solid ${MAPLE_COLORS.borderDefault}`, + borderRadius: 8, + padding: "1rem 1.25rem", + boxShadow: "0 0.25rem 1rem rgba(15, 23, 42, 0.04)" +} + +const statValueStyle: React.CSSProperties = { + fontSize: 24, + fontWeight: 700, + color: MAPLE_COLORS.primary, + fontFamily: "Nunito, system-ui, sans-serif", + lineHeight: 1.2 +} + +const statLabelStyle: React.CSSProperties = { + fontSize: 12, + color: MAPLE_COLORS.textMuted, + marginTop: "0.25rem", + textTransform: "uppercase", + letterSpacing: "0.05em", + fontWeight: 600 +} + +const entryCardStyle: React.CSSProperties = { + display: "block", + background: MAPLE_COLORS.surfaceBase, + border: `1px solid ${MAPLE_COLORS.borderDefault}`, + borderRadius: 8, + padding: "1.25rem", + textDecoration: "none", + color: MAPLE_COLORS.textBody, + boxShadow: "0 0.25rem 1rem rgba(15, 23, 42, 0.04)", + transition: "box-shadow 0.15s, border-color 0.15s", + height: "100%" +} + +const entryCardInner: React.CSSProperties = { + display: "flex", + justifyContent: "space-between", + alignItems: "flex-start" +} + +const entryCardTitle: React.CSSProperties = { + fontWeight: 700, + fontSize: 16, + color: MAPLE_COLORS.primary +} + +const entryCardDesc: React.CSSProperties = { + fontSize: 13, + color: MAPLE_COLORS.textMuted, + marginTop: "0.2rem" +} + +const entryCardCount: React.CSSProperties = { + fontSize: 20, + fontWeight: 700, + color: MAPLE_COLORS.textStrong, + fontFamily: "Nunito, system-ui, sans-serif", + whiteSpace: "nowrap" +} diff --git a/public/locales/en/common.json b/public/locales/en/common.json index ea6789c8f..763ff61ec 100644 --- a/public/locales/en/common.json +++ b/public/locales/en/common.json @@ -141,6 +141,7 @@ "viewProfile": "View Profile", "whyUseMaple": "Why Use MAPLE", "inTheNews": "In The News", + "lobbying": "Lobbying", "login": "Login" }, "new_feature": "*NEW*", @@ -199,7 +200,8 @@ "policies": "Policies", "unsubscribe": "Unsubscribe", "why_use_maple": "Why Use Maple?", - "ai_tools": "AI Research Tools" + "ai_tools": "AI Research Tools", + "lobbying": "Lobbying Explorer" }, "user_updates": "User Updates", "view": "View", diff --git a/public/locales/en/lobbying.json b/public/locales/en/lobbying.json new file mode 100644 index 000000000..f40ee32b4 --- /dev/null +++ b/public/locales/en/lobbying.json @@ -0,0 +1,80 @@ +{ + "title": "Lobbying Explorer", + "subtitle": "Browse Massachusetts lobbying disclosures filed with the Secretary of State.", + "subnav": { + "label": "LOBBYING EXPLORER", + "overview": "Overview" + }, + "pagination": { + "prev": "← Prev", + "next": "Next →" + }, + "misc": { + "disclosures": "Disclosures", + "disclosureLink": "Disclosure {{number}}", + "viewOnMaple": "View bill on MAPLE →", + "total": "total" + }, + "registrantType": { + "all": "All types", + "lobbyist": "Lobbyist", + "employer": "Employer" + }, + "stats": { + "totalBills": "Lobbied bills", + "totalClients": "Clients", + "sessionsCovered": "Sessions covered", + "totalSpend": "Total compensation reported" + }, + "sections": { + "bills": "Bills", + "clients": "Clients", + "firms": "Lobbying Firms" + }, + "filters": { + "session": "Session", + "year": "Year", + "position": "Position", + "allSessions": "All sessions", + "allPositions": "All positions", + "allTypes": "All types", + "search": "Search…" + }, + "position": { + "support": "Support", + "oppose": "Oppose", + "neutral": "Neutral", + "none": "No position" + }, + "fields": { + "clientName": "Client", + "firmName": "Lobbying Firm", + "amount": "Compensation", + "year": "Year", + "filings": "Filings", + "bills": "Bills lobbied", + "clients": "Clients represented", + "bill": "Bill", + "positions": "Positions", + "activity": "Activity", + "sessions": "Sessions", + "type": "Type", + "firms": "Firms" + }, + "attribution": "Data from the MA Secretary of State Lobbyist Public Search.", + "noData": "No lobbying filings found.", + "loading": "Loading lobbying data…", + "titles": { + "overview": "Lobbying Explorer", + "bills": "Lobbied Bills", + "clients": "Clients & Sponsors", + "firms": "Lobbying Firms", + "client": "Client Profile", + "firm": "Firm Profile" + }, + "billCard": { + "filingCount_one": "{{count}} lobbying filing", + "filingCount_other": "{{count}} lobbying filings", + "viewAll": "View all lobbying activity →" + } +} diff --git a/scripts/firebase-admin/seedLobbyingStats.ts b/scripts/firebase-admin/seedLobbyingStats.ts new file mode 100644 index 000000000..d5b096fa9 --- /dev/null +++ b/scripts/firebase-admin/seedLobbyingStats.ts @@ -0,0 +1,83 @@ +import { Script } from "./types" + +const FILINGS_COLLECTION = "lobbyingFilings" +const REGISTRANTS_COLLECTION = "lobbyingRegistrants" +const STATS_COLLECTION = "lobbyingMeta" +const STATS_DOC_ID = "stats" + +export const script: Script = async ({ db }) => { + console.log("Reading lobbyingFilings…") + const filingsSnap = await db.collection(FILINGS_COLLECTION).get() + console.log(` ${filingsSnap.size} filings`) + + console.log("Reading lobbyingRegistrants…") + const registrantsSnap = await db.collection(REGISTRANTS_COLLECTION).get() + console.log(` ${registrantsSnap.size} registrants`) + + const entityNames = new Set() + const clientNames = new Set() + const bills = new Set() + const courts = new Set() + const spendByYear: Record = {} + const filingsByYear: Record = {} + + for (const doc of filingsSnap.docs) { + const d = doc.data() + const year: number = d.year + const gc: number = d.generalCourt + + entityNames.add(d.entityNameNorm ?? d.entityName) + + if (d.clientNameNorm) clientNames.add(d.clientNameNorm) + + if (d.billId && d.billId.length > 2) { + bills.add(`${gc}/${d.billId}`) + } + + courts.add(gc) + + const y = String(year) + filingsByYear[y] = (filingsByYear[y] ?? 0) + 1 + if (d.amount != null) { + spendByYear[y] = (spendByYear[y] ?? 0) + d.amount + } + } + + // Also count unique clients from registrant docs (more reliable than filings) + const clientNormsFromRegistrants = new Set() + for (const doc of registrantsSnap.docs) { + const d = doc.data() + for (const c of d.clients ?? []) { + if (c.clientNameNorm) clientNormsFromRegistrants.add(c.clientNameNorm) + } + } + // Use whichever source gives more clients + const totalClients = Math.max( + clientNames.size, + clientNormsFromRegistrants.size + ) + + const stats = { + totalFilings: filingsSnap.size, + totalRegistrants: registrantsSnap.size, + totalClients, + totalBillsWithFilings: bills.size, + courtsWithData: [...courts].sort((a, b) => a - b), + spendByYear, + filingsByYear + } + + console.log("Stats computed:") + console.log(` totalFilings: ${stats.totalFilings}`) + console.log(` totalRegistrants: ${stats.totalRegistrants}`) + console.log(` totalClients: ${stats.totalClients}`) + console.log(` totalBillsWithFilings: ${stats.totalBillsWithFilings}`) + console.log(` courts: ${stats.courtsWithData.join(", ")}`) + + await db + .collection(STATS_COLLECTION) + .doc(STATS_DOC_ID) + .set(stats, { merge: true }) + + console.log(`Written to ${STATS_COLLECTION}/${STATS_DOC_ID}`) +} diff --git a/yarn.lock b/yarn.lock index 2d0c0348b..03bd7ee1c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3406,6 +3406,18 @@ redux-thunk "^2.4.2" reselect "^4.1.8" +"@reduxjs/toolkit@^1.9.0 || 2.x.x": + version "2.12.0" + resolved "https://registry.yarnpkg.com/@reduxjs/toolkit/-/toolkit-2.12.0.tgz#e62787503a38561e04bb8f39e29ca8db689590f9" + integrity sha512-KiT+RzZbp6mQET+Mg+h2c97+9j1sNflUxQkIHI7Yuzf6Peu+OYpmkn6nbHWmLLWj+1ZODUJFwGZ7gx3L9R9EOw== + dependencies: + "@standard-schema/spec" "^1.0.0" + "@standard-schema/utils" "^0.3.0" + immer "^11.0.0" + redux "^5.0.1" + redux-thunk "^3.1.0" + reselect "^5.1.0" + "@remix-run/router@1.13.1": version "1.13.1" resolved "https://registry.npmjs.org/@remix-run/router/-/router-1.13.1.tgz" @@ -3484,6 +3496,16 @@ dependencies: "@sinonjs/commons" "^3.0.0" +"@standard-schema/spec@^1.0.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@standard-schema/spec/-/spec-1.1.0.tgz#a79b55dbaf8604812f52d140b2c9ab41bc150bb8" + integrity sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w== + +"@standard-schema/utils@^0.3.0": + version "0.3.0" + resolved "https://registry.yarnpkg.com/@standard-schema/utils/-/utils-0.3.0.tgz#3d5e608f16c2390c10528e98e59aef6bf73cae7b" + integrity sha512-e7Mew686owMaPJVNNLs55PUvgz371nKgwsc4vxE49zsODpJEnxgxRo2y/OKrqueavXgZNMDVj3DdHFlaSAeU8g== + "@storybook/addon-actions@7.6.4", "@storybook/addon-actions@^7.6.4": version "7.6.4" resolved "https://registry.npmjs.org/@storybook/addon-actions/-/addon-actions-7.6.4.tgz" @@ -4632,6 +4654,57 @@ dependencies: "@types/node" "*" +"@types/d3-array@^3.0.3": + version "3.2.2" + resolved "https://registry.yarnpkg.com/@types/d3-array/-/d3-array-3.2.2.tgz#e02151464d02d4a1b44646d0fcdb93faf88fde8c" + integrity sha512-hOLWVbm7uRza0BYXpIIW5pxfrKe0W+D5lrFiAEYR+pb6w3N2SwSMaJbXdUfSEv+dT4MfHBLtn5js0LAWaO6otw== + +"@types/d3-color@*": + version "3.1.3" + resolved "https://registry.yarnpkg.com/@types/d3-color/-/d3-color-3.1.3.tgz#368c961a18de721da8200e80bf3943fb53136af2" + integrity sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A== + +"@types/d3-ease@^3.0.0": + version "3.0.2" + resolved "https://registry.yarnpkg.com/@types/d3-ease/-/d3-ease-3.0.2.tgz#e28db1bfbfa617076f7770dd1d9a48eaa3b6c51b" + integrity sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA== + +"@types/d3-interpolate@^3.0.1": + version "3.0.4" + resolved "https://registry.yarnpkg.com/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz#412b90e84870285f2ff8a846c6eb60344f12a41c" + integrity sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA== + dependencies: + "@types/d3-color" "*" + +"@types/d3-path@*": + version "3.1.1" + resolved "https://registry.yarnpkg.com/@types/d3-path/-/d3-path-3.1.1.tgz#f632b380c3aca1dba8e34aa049bcd6a4af23df8a" + integrity sha512-VMZBYyQvbGmWyWVea0EHs/BwLgxc+MKi1zLDCONksozI4YJMcTt8ZEuIR4Sb1MMTE8MMW49v0IwI5+b7RmfWlg== + +"@types/d3-scale@^4.0.2": + version "4.0.9" + resolved "https://registry.yarnpkg.com/@types/d3-scale/-/d3-scale-4.0.9.tgz#57a2f707242e6fe1de81ad7bfcccaaf606179afb" + integrity sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw== + dependencies: + "@types/d3-time" "*" + +"@types/d3-shape@^3.1.0": + version "3.1.8" + resolved "https://registry.yarnpkg.com/@types/d3-shape/-/d3-shape-3.1.8.tgz#d1516cc508753be06852cd06758e3bb54a22b0e3" + integrity sha512-lae0iWfcDeR7qt7rA88BNiqdvPS5pFVPpo5OfjElwNaT2yyekbM0C9vK+yqBqEmHr6lDkRnYNoTBYlAgJa7a4w== + dependencies: + "@types/d3-path" "*" + +"@types/d3-time@*", "@types/d3-time@^3.0.0": + version "3.0.4" + resolved "https://registry.yarnpkg.com/@types/d3-time/-/d3-time-3.0.4.tgz#8472feecd639691450dd8000eb33edd444e1323f" + integrity sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g== + +"@types/d3-timer@^3.0.0": + version "3.0.2" + resolved "https://registry.yarnpkg.com/@types/d3-timer/-/d3-timer-3.0.2.tgz#70bbda77dc23aa727413e22e214afa3f0e852f70" + integrity sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw== + "@types/debounce-promise@^3.1.1": version "3.1.9" resolved "https://registry.npmjs.org/@types/debounce-promise/-/debounce-promise-3.1.9.tgz" @@ -5125,6 +5198,11 @@ resolved "https://registry.npmjs.org/@types/use-sync-external-store/-/use-sync-external-store-0.0.3.tgz" integrity sha512-EwmlvuaxPNej9+T4v5AuBPJa2x2UOJVdjCtDHgcDqitUeOtjnJKJ+apYjVcAoBEMjKW1VVFGZLUb5+qqa09XFA== +"@types/use-sync-external-store@^0.0.6": + version "0.0.6" + resolved "https://registry.yarnpkg.com/@types/use-sync-external-store/-/use-sync-external-store-0.0.6.tgz#60be8d21baab8c305132eb9cb912ed497852aadc" + integrity sha512-zFDAD+tlpf2r4asuHEj0XH6pY6i0g5NeAHPn+15wk3BV6JA69eERFXC1gyGThDkVa1zCyKr5jox1+2LbV/AMLg== + "@types/uuid@^9.0.1": version "9.0.7" resolved "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.7.tgz" @@ -7578,6 +7656,77 @@ csv-parse@^5.0.4: resolved "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.3.tgz" integrity sha512-v0KW6C0qlZzoGjk6u5tLmVfyZxNgPGXZsWTXshpAgKVGmGXzaVWGdlCFxNx5iuzcXT/oJN1HHM9DZKwtAtYa+A== +"d3-array@2 - 3", "d3-array@2.10.0 - 3", d3-array@^3.1.6: + version "3.2.4" + resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-3.2.4.tgz#15fec33b237f97ac5d7c986dc77da273a8ed0bb5" + integrity sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg== + dependencies: + internmap "1 - 2" + +"d3-color@1 - 3": + version "3.1.0" + resolved "https://registry.yarnpkg.com/d3-color/-/d3-color-3.1.0.tgz#395b2833dfac71507f12ac2f7af23bf819de24e2" + integrity sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA== + +d3-ease@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/d3-ease/-/d3-ease-3.0.1.tgz#9658ac38a2140d59d346160f1f6c30fda0bd12f4" + integrity sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w== + +"d3-format@1 - 3": + version "3.1.2" + resolved "https://registry.yarnpkg.com/d3-format/-/d3-format-3.1.2.tgz#01fdb46b58beb1f55b10b42ad70b6e344d5eb2ae" + integrity sha512-AJDdYOdnyRDV5b6ArilzCPPwc1ejkHcoyFarqlPqT7zRYjhavcT3uSrqcMvsgh2CgoPbK3RCwyHaVyxYcP2Arg== + +"d3-interpolate@1.2.0 - 3", d3-interpolate@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/d3-interpolate/-/d3-interpolate-3.0.1.tgz#3c47aa5b32c5b3dfb56ef3fd4342078a632b400d" + integrity sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g== + dependencies: + d3-color "1 - 3" + +d3-path@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/d3-path/-/d3-path-3.1.0.tgz#22df939032fb5a71ae8b1800d61ddb7851c42526" + integrity sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ== + +d3-scale@^4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/d3-scale/-/d3-scale-4.0.2.tgz#82b38e8e8ff7080764f8dcec77bd4be393689396" + integrity sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ== + dependencies: + d3-array "2.10.0 - 3" + d3-format "1 - 3" + d3-interpolate "1.2.0 - 3" + d3-time "2.1.1 - 3" + d3-time-format "2 - 4" + +d3-shape@^3.1.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/d3-shape/-/d3-shape-3.2.0.tgz#a1a839cbd9ba45f28674c69d7f855bcf91dfc6a5" + integrity sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA== + dependencies: + d3-path "^3.1.0" + +"d3-time-format@2 - 4": + version "4.1.0" + resolved "https://registry.yarnpkg.com/d3-time-format/-/d3-time-format-4.1.0.tgz#7ab5257a5041d11ecb4fe70a5c7d16a195bb408a" + integrity sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg== + dependencies: + d3-time "1 - 3" + +"d3-time@1 - 3", "d3-time@2.1.1 - 3", d3-time@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/d3-time/-/d3-time-3.1.0.tgz#9310db56e992e3c0175e1ef385e545e48a9bb5c7" + integrity sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q== + dependencies: + d3-array "2 - 3" + +d3-timer@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/d3-timer/-/d3-timer-3.0.1.tgz#6284d2a2708285b1abb7e201eda4380af35e63b0" + integrity sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA== + damerau-levenshtein@^1.0.8: version "1.0.8" resolved "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz" @@ -7691,6 +7840,11 @@ decamelize@^1.1.0, decamelize@^1.2.0: resolved "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz" integrity sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA== +decimal.js-light@^2.5.1: + version "2.5.1" + resolved "https://registry.yarnpkg.com/decimal.js-light/-/decimal.js-light-2.5.1.tgz#134fd32508f19e208f4fb2f8dac0d2626a867934" + integrity sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg== + decimal.js@^10.4.2: version "10.6.0" resolved "https://registry.yarnpkg.com/decimal.js/-/decimal.js-10.6.0.tgz#e649a43e3ab953a72192ff5983865e509f37ed9a" @@ -8551,6 +8705,11 @@ es-to-primitive@^1.2.1: is-date-object "^1.0.1" is-symbol "^1.0.2" +es-toolkit@^1.39.3: + version "1.49.0" + resolved "https://registry.yarnpkg.com/es-toolkit/-/es-toolkit-1.49.0.tgz#93c5b031865792fc03cbf5bd20c132a4f976a52a" + integrity sha512-G5iZ6Pc/FNRY/soKZHC+TxGDD83rHUDXxzaWhGCX44vAv/tMs56WMusnm/KMNK+luUPsgA9U28cGr4RDlSzL2g== + esbuild-plugin-alias@^0.2.1: version "0.2.1" resolved "https://registry.npmjs.org/esbuild-plugin-alias/-/esbuild-plugin-alias-0.2.1.tgz" @@ -8911,6 +9070,11 @@ eventemitter3@^4.0.7: resolved "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz" integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw== +eventemitter3@^5.0.1: + version "5.0.4" + resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-5.0.4.tgz#a86d66170433712dde814707ac52b5271ceb1feb" + integrity sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw== + events-listener@^1.1.0: version "1.1.0" resolved "https://registry.npmjs.org/events-listener/-/events-listener-1.1.0.tgz" @@ -10690,6 +10854,11 @@ immediate@~3.0.5: resolved "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz" integrity sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ== +immer@^11.0.0, immer@^11.1.8: + version "11.1.11" + resolved "https://registry.yarnpkg.com/immer/-/immer-11.1.11.tgz#bbf825a333ae1b16fd450d8da5f61d54de6a553d" + integrity sha512-qzXuyXAkPySAGYkfsAwodDPWT8Zm7/Uo5BNt4BjhMhG5WlWyZZ4wQqnWwdS8kjlQ1Cwu6gjw3A6+0gTQwlyYtw== + immer@^9.0.21: version "9.0.21" resolved "https://registry.npmjs.org/immer/-/immer-9.0.21.tgz" @@ -10931,6 +11100,11 @@ internal-slot@^1.0.7: hasown "^2.0.0" side-channel "^1.0.4" +"internmap@1 - 2": + version "2.0.3" + resolved "https://registry.yarnpkg.com/internmap/-/internmap-2.0.3.tgz#6685f23755e43c524e251d29cbc97248e3061009" + integrity sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg== + invariant@^2.2.4: version "2.2.4" resolved "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz" @@ -15371,6 +15545,14 @@ react-query@^3.32.1, react-query@^3.39.3: broadcast-channel "^3.4.1" match-sorter "^6.0.2" +"react-redux@8.x.x || 9.x.x": + version "9.3.0" + resolved "https://registry.yarnpkg.com/react-redux/-/react-redux-9.3.0.tgz#a30113bb6d95c0a715d54dda4308d450fca6ce09" + integrity sha512-KQopgqFo/p/fgmAs5qz6p5RWaNAzq40WAu7fJIXnQpYxFPbJYtsJPWvGeF2rOBaY/kEuV77AVsX8TsQzKm+A/g== + dependencies: + "@types/use-sync-external-store" "^0.0.6" + use-sync-external-store "^1.4.0" + react-redux@^8.0.2: version "8.1.3" resolved "https://registry.npmjs.org/react-redux/-/react-redux-8.1.3.tgz" @@ -15540,6 +15722,23 @@ recast@^0.23.1, recast@^0.23.3: source-map "~0.6.1" tslib "^2.0.1" +recharts@^3.9.2: + version "3.9.2" + resolved "https://registry.yarnpkg.com/recharts/-/recharts-3.9.2.tgz#ccdadd197335102a91bd64d51fafc6c9a5b69ebb" + integrity sha512-G4fy+Pk46RaXgwWMh+Nzhyo/lbFAVqXo9gtetlyehe6Ehge9CsgDuOTwQDD+i1+llaLktNBiNq4bhnGlDRXFtw== + dependencies: + "@reduxjs/toolkit" "^1.9.0 || 2.x.x" + clsx "^2.1.1" + decimal.js-light "^2.5.1" + es-toolkit "^1.39.3" + eventemitter3 "^5.0.1" + immer "^11.1.8" + react-redux "8.x.x || 9.x.x" + reselect "5.2.0" + tiny-invariant "^1.3.3" + use-sync-external-store "^1.2.2" + victory-vendor "^37.0.2" + redent@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/redent/-/redent-3.0.0.tgz" @@ -15572,6 +15771,11 @@ redux@^4.0.5, redux@^4.2.1: dependencies: "@babel/runtime" "^7.9.2" +redux@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/redux/-/redux-5.0.1.tgz#97fa26881ce5746500125585d5642c77b6e9447b" + integrity sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w== + reflect.getprototypeof@^1.0.4: version "1.0.4" resolved "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.4.tgz" @@ -15767,6 +15971,11 @@ requires-port@^1.0.0: resolved "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz" integrity sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ== +reselect@5.2.0, reselect@^5.1.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/reselect/-/reselect-5.2.0.tgz#f380ef7664332d26ea06c1cba04bdbbdcaa955f1" + integrity sha512-AgZ3UOZm3YndfrJ4OYjgrT7bmCm/1iqkjvEfH/oYjzh6PD2qw4QuT3jjnXIrpdt4MTpMXclMT3lXbmRY+XRakw== + reselect@^4.1.8: version "4.1.8" resolved "https://registry.npmjs.org/reselect/-/reselect-4.1.8.tgz" @@ -17182,6 +17391,11 @@ tiny-invariant@^1.3.1: resolved "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.1.tgz" integrity sha512-AD5ih2NlSssTCwsMznbvwMZpJ1cbhkGd2uueNxzv2jDlEeZdU04JQfRnggJQ8DrcVBGjAsCKwFBbDlVNtEMlzw== +tiny-invariant@^1.3.3: + version "1.3.3" + resolved "https://registry.yarnpkg.com/tiny-invariant/-/tiny-invariant-1.3.3.tgz#46680b7a873a0d5d10005995eb90a70d74d60127" + integrity sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg== + tmp@^0.0.33: version "0.0.33" resolved "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz" @@ -17900,6 +18114,11 @@ use-sync-external-store@^1.0.0: resolved "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz" integrity sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA== +use-sync-external-store@^1.2.2, use-sync-external-store@^1.4.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/use-sync-external-store/-/use-sync-external-store-1.6.0.tgz#b174bfa65cb2b526732d9f2ac0a408027876f32d" + integrity sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w== + usehooks-ts@^2.5.3: version "2.9.1" resolved "https://registry.npmjs.org/usehooks-ts/-/usehooks-ts-2.9.1.tgz" @@ -18006,6 +18225,26 @@ vfile@^5.0.0: unist-util-stringify-position "^3.0.0" vfile-message "^3.0.0" +victory-vendor@^37.0.2: + version "37.3.6" + resolved "https://registry.yarnpkg.com/victory-vendor/-/victory-vendor-37.3.6.tgz#401ac4b029a0b3d33e0cba8e8a1d765c487254da" + integrity sha512-SbPDPdDBYp+5MJHhBCAyI7wKM3d5ivekigc2Dk2s7pgbZ9wIgIBYGVw4zGHBml/qTFbexrofXW6Gu4noGxrOwQ== + dependencies: + "@types/d3-array" "^3.0.3" + "@types/d3-ease" "^3.0.0" + "@types/d3-interpolate" "^3.0.1" + "@types/d3-scale" "^4.0.2" + "@types/d3-shape" "^3.1.0" + "@types/d3-time" "^3.0.0" + "@types/d3-timer" "^3.0.0" + d3-array "^3.1.6" + d3-ease "^3.0.1" + d3-interpolate "^3.0.1" + d3-scale "^4.0.2" + d3-shape "^3.1.0" + d3-time "^3.0.0" + d3-timer "^3.0.1" + vm-browserify@^1.1.2: version "1.1.2" resolved "https://registry.npmjs.org/vm-browserify/-/vm-browserify-1.1.2.tgz"