diff --git a/components/LegislatorProfile/LegislatorProfilePage.tsx b/components/LegislatorProfile/LegislatorProfilePage.tsx index 2f8d127e2..5b6405b2d 100644 --- a/components/LegislatorProfile/LegislatorProfilePage.tsx +++ b/components/LegislatorProfile/LegislatorProfilePage.tsx @@ -20,6 +20,7 @@ import { LegislatorTabs } from "./LegislatorTabs" import { useAuth } from "components/auth" import { Col, Container, Row, Spinner } from "components/bootstrap" import { useDistrict, useMember } from "components/db" +import { useMembersFinance } from "components/db/membersFinance" import { Internal } from "components/links" import { FollowUserButton } from "components/shared/FollowButton" import { CircleImage } from "components/shared/LabeledIcon" @@ -130,6 +131,7 @@ export function LegislatorProfilePage({ member?.Branch, member?.District ) + const { finance } = useMembersFinance(court, memberCode) const { t } = useTranslation("legislators") // const { user } = useAuth() **uncomment when Following Button in enabled** @@ -343,7 +345,11 @@ export function LegislatorProfilePage({ - ? + + {finance?.totalRaised != null + ? "$" + Math.round(finance.totalRaised).toLocaleString() + : "?"} + {t("fundsRaised")} @@ -354,6 +360,7 @@ export function LegislatorProfilePage({ diff --git a/components/LegislatorProfile/LegislatorTabs.tsx b/components/LegislatorProfile/LegislatorTabs.tsx index 40c264453..d6ba55d2b 100644 --- a/components/LegislatorProfile/LegislatorTabs.tsx +++ b/components/LegislatorProfile/LegislatorTabs.tsx @@ -19,6 +19,7 @@ import { TabNavWrapper, TabType } from "components/EditProfilePage/StyledEditProfileComponents" +import { MembersFinance } from "components/db/membersFinance" const tabCategory = [ "priorities", @@ -66,11 +67,13 @@ const TabNavItem = ({ export function LegislatorTabs({ district, districtLoading, - tabCategory + tabCategory, + finance }: { district?: District | undefined districtLoading?: boolean tabCategory?: TabCategories + finance?: MembersFinance }) { const { t } = useTranslation("legislators") @@ -93,7 +96,7 @@ export function LegislatorTabs({ { title: t("tabs.finance"), eventKey: "finance", - content: + content: }, { title: t("tabs.district"), diff --git a/components/LegislatorProfile/TabComponents/FinanceTab.tsx b/components/LegislatorProfile/TabComponents/FinanceTab.tsx index 4c263d26a..3027b1a1f 100644 --- a/components/LegislatorProfile/TabComponents/FinanceTab.tsx +++ b/components/LegislatorProfile/TabComponents/FinanceTab.tsx @@ -1,5 +1,265 @@ -import { TabBlock } from "../LegislatorComponents" +import { useTranslation } from "next-i18next" +import { MembersFinance } from "components/db/membersFinance" -export function FinanceTab() { - return - Finance +function formatCurrency(n?: number | null): string { + if (n == null || isNaN(n)) return "$0" + return "$" + Math.round(n).toLocaleString() +} + +function formatPct(value: number, total: number): string { + if (total === 0) return "0%" + return Math.round((value / total) * 100) + "%" +} + +interface CategoryRow { + name: string + value: number +} + +export function FinanceTab({ finance }: { finance?: MembersFinance }) { + const { t } = useTranslation("legislators") + + if (!finance) { + return

{t("finance.noData")}

+ } + + const inKindTotal = + (finance.inKind?.individual?.amount ?? 0) + + (finance.inKind?.committee?.amount ?? 0) + + (finance.inKind?.union?.amount ?? 0) + + (finance.inKind?.unitemized?.amount ?? 0) + + const candidateFundsTotal = + (finance.candidateFunds?.loans?.amount ?? 0) + + (finance.candidateFunds?.contributions?.amount ?? 0) + + const categories: CategoryRow[] = [ + { + name: t("finance.breakdown.individual"), + value: finance.breakdown?.individual?.amount ?? 0 + }, + { + name: t("finance.breakdown.committee"), + value: finance.breakdown?.committee?.amount ?? 0 + }, + { + name: t("finance.breakdown.union"), + value: finance.breakdown?.union?.amount ?? 0 + }, + { + name: t("finance.breakdown.unitemized"), + value: finance.breakdown?.unitemized?.amount ?? 0 + }, + { name: t("finance.breakdown.candidateFunds"), value: candidateFundsTotal }, + { name: t("finance.breakdown.inKind"), value: inKindTotal } + ] + .filter(c => c.value > 0) + .sort((a, b) => b.value - a.value) + + const total = categories.reduce((sum, c) => sum + c.value, 0) + const nonContribution = finance.otherReceipts?.nonContribution?.amount ?? 0 + + const cycleYears = Object.keys(finance.years ?? {}).map(Number) + const cycleYear = cycleYears.length + ? Math.max(...cycleYears) + : new Date().getFullYear() + + const totalSpentAsOf = finance.lastUpdated + ? finance.lastUpdated + .toDate() + .toLocaleDateString("en-US", { month: "short", year: "numeric" }) + : "" + + const statBoxes = [ + { + label: t("finance.stats.totalRaised"), + value: formatCurrency(finance.totalRaised), + subtitle: t("finance.stats.totalRaisedSubtitle", { + count: finance.contributorCount ?? 0 + }), + footnote: t("finance.stats.totalRaisedFootnote") + }, + { + label: t("finance.stats.totalSpent"), + value: formatCurrency(finance.totalSpent), + subtitle: t("finance.stats.totalSpentSubtitle", { date: totalSpentAsOf }), + footnote: undefined as string | undefined + }, + { + label: t("finance.stats.smallDonors"), + value: "—", + subtitle: t("finance.stats.smallDonorsSubtitle"), + footnote: undefined as string | undefined + }, + { + label: t("finance.stats.cashOnHand"), + value: formatCurrency(finance.cashOnHand), + subtitle: t("finance.stats.cashOnHandSubtitle"), + footnote: undefined as string | undefined + } + ] + + return ( +
+
+ {t("finance.electionCycleHeading", { year: cycleYear })} +
+ +
+ {statBoxes.map(({ label, value, subtitle, footnote }) => ( +
+
+
+ {value} +
+
+ {label} +
+
+ {subtitle} +
+ {footnote && ( +
+ {footnote} +
+ )} +
+
+ ))} +
+ +
+ {t("finance.breakdownHeading")} +
+ + {categories.length === 0 ? ( +

{t("finance.noContributions")}

+ ) : ( +
+
+ {t("finance.table.category")} +
+ + {t("finance.table.amount")} + + + {t("finance.table.share")} + +
+
+ + {categories.map((c, i) => ( +
+ {c.name} +
+ + {formatCurrency(c.value)} + + + + {formatPct(c.value, total)} + + +
+
+ ))} + +

+ {t("finance.source")} +

+
+ )} + + {nonContribution > 0 && ( +

+ {t("finance.otherReceipts", { + amount: formatCurrency(nonContribution) + })} +

+ )} +
+ ) } diff --git a/components/db/membersFinance.ts b/components/db/membersFinance.ts new file mode 100644 index 000000000..e5c4fa0bf --- /dev/null +++ b/components/db/membersFinance.ts @@ -0,0 +1,73 @@ +import { Timestamp } from "firebase/firestore" +import { useMemo } from "react" +import { useAsync } from "react-async-hook" +import { loadDoc } from "./common" + +// Mirror of functions/src/ocpf/types.ts MembersFinance, using client-side Timestamp +export interface FinanceBreakdownEntry { + count: number + amount: number +} + +export interface MembersFinanceBreakdown { + individual: FinanceBreakdownEntry + committee: FinanceBreakdownEntry + union: FinanceBreakdownEntry + unitemized: { amount: number } +} + +export interface MembersFinanceCandidateFunds { + loans: FinanceBreakdownEntry + contributions: FinanceBreakdownEntry +} + +export interface MembersFinanceInKind { + individual: FinanceBreakdownEntry + committee: FinanceBreakdownEntry + union: FinanceBreakdownEntry + unitemized: { amount: number } +} + +export interface MembersFinanceOtherReceipts { + nonContribution: FinanceBreakdownEntry +} + +export interface MembersFinanceYearData { + totalRaised: number + totalSpent: number + breakdown: MembersFinanceBreakdown + finalized: boolean +} + +export interface MembersFinance { + ocpfCpfId: number + totalRaised: number + totalSpent: number + cashOnHand: number + contributorCount: number + lastUpdated: Timestamp + breakdown: MembersFinanceBreakdown + candidateFunds: MembersFinanceCandidateFunds + inKind: MembersFinanceInKind + otherReceipts: MembersFinanceOtherReceipts + years: Record +} + +export function useMembersFinance(court: number, memberId?: string) { + const { loading, result, error } = useAsync(getFinance, [court, memberId]) + return useMemo( + () => ({ finance: result, loading, error }), + [loading, result, error] + ) +} + +async function getFinance( + court: number, + memberId?: string +): Promise { + if (!memberId) return undefined + const data = await loadDoc( + `/generalCourts/${court}/membersFinance/${memberId}` + ) + return data as MembersFinance | undefined +} diff --git a/functions/src/index.ts b/functions/src/index.ts index 796a30871..771b60c88 100644 --- a/functions/src/index.ts +++ b/functions/src/index.ts @@ -61,6 +61,7 @@ export { export { transcription } from "./webhooks" export { matchOcpfMembers } from "./ocpf/matchOcpfMembers" +export { scrapeOcpfFinance } from "./ocpf/scrapeOcpfFinance" export * from "./triggerPubsubFunction" diff --git a/functions/src/ocpf/scrapeOcpfFinance.ts b/functions/src/ocpf/scrapeOcpfFinance.ts new file mode 100644 index 000000000..4d23e27fb --- /dev/null +++ b/functions/src/ocpf/scrapeOcpfFinance.ts @@ -0,0 +1,565 @@ +// TODO: After validating output against the OCPF website, flip to: +// export const scrapeOcpfFinance = functions.pubsub.schedule("every 24 hours").onRun(...) +import * as functions from "firebase-functions" +import { getAuth } from "firebase-admin/auth" +import axios from "axios" +import unzipper from "unzipper" +import * as readline from "readline" +import { db, Timestamp } from "../firebase" +import { currentGeneralCourt } from "../shared" +import { + OcpfMemberMapping, + MembersFinance, + MembersFinanceBreakdown, + MembersFinanceCandidateFunds, + MembersFinanceInKind, + MembersFinanceOtherReceipts, + MembersFinanceYearData +} from "./types" + +// Set to null to run all members. Use individual CPF_ID to validate single-member output. +const TEST_CPF_ID: number | null = null // For testing, can use 16883, Rebecca L. Rausch, RLR0 + +const OCPF_BASE_URL = "https://ocpf2.blob.core.windows.net/downloads/data2" + +const YEARS = ["2025", "2026"] + +// Annual rollup report types to be excluded. +// Including these would double-count both receipts and expeditures. +// Note: 32, 36, 45, and 52 not relevant to individual legislators, but not harmful to exclude +const YEAR_END_REPORT_TYPE_IDS = new Set([ + 11, // Year-End Report (Depository) + 24, // Year-End Report (Non-Depository) + 32, // Year-End Report (PAC) + 36, // IEPAC Year-End Report + 45, // Year-End Report (Ballot Question Committee) + 52, // Year-End Report (Local Party Committee) + 113 // Year-End Report (Municipal) +]) + +// ── Accumulator types ───────────────────────────────────────────────────────── + +interface MutableBreakdownEntry { + count: number + amount: number +} + +interface MemberAccumulator { + cpfId: number + totalRaised: number + totalSpent: number + cashOnHand: number + cashOnHandEndDateMs: number // End_Date (as ms) of the most recent bank report seen + contributorCount: number + breakdown: { + individual: MutableBreakdownEntry + committee: MutableBreakdownEntry + union: MutableBreakdownEntry + unitemized: { amount: number } + } + candidateFunds: { + loans: MutableBreakdownEntry + contributions: MutableBreakdownEntry + } + inKind: { + individual: MutableBreakdownEntry + committee: MutableBreakdownEntry + union: MutableBreakdownEntry + unitemized: { amount: number } + } + otherReceipts: { + nonContribution: MutableBreakdownEntry + } + years: Record< + string, + { + totalRaised: number + totalSpent: number + breakdown: { + individual: MutableBreakdownEntry + committee: MutableBreakdownEntry + union: MutableBreakdownEntry + unitemized: { amount: number } + } + } + > + yearEndCheck: Record< + string, + { receiptsTotal: number; expendituresTotal: number } | null + > +} + +function emptyEntry(): MutableBreakdownEntry { + return { count: 0, amount: 0 } +} + +function newAccumulator(cpfId: number): MemberAccumulator { + const yearInit = () => ({ + totalRaised: 0, + totalSpent: 0, + breakdown: { + individual: emptyEntry(), + committee: emptyEntry(), + union: emptyEntry(), + unitemized: { amount: 0 } + } + }) + return { + cpfId, + totalRaised: 0, + totalSpent: 0, + cashOnHand: 0, + cashOnHandEndDateMs: 0, + contributorCount: 0, + breakdown: { + individual: emptyEntry(), + committee: emptyEntry(), + union: emptyEntry(), + unitemized: { amount: 0 } + }, + candidateFunds: { loans: emptyEntry(), contributions: emptyEntry() }, + inKind: { + individual: emptyEntry(), + committee: emptyEntry(), + union: emptyEntry(), + unitemized: { amount: 0 } + }, + otherReceipts: { nonContribution: emptyEntry() }, + years: Object.fromEntries(YEARS.map(y => [y, yearInit()])), + yearEndCheck: Object.fromEntries(YEARS.map(y => [y, null])) + } +} + +// ── Cloud Function ──────────────────────────────────────────────────────────── + +export const scrapeOcpfFinance = functions + .runWith({ timeoutSeconds: 540, memory: "512MB" }) + .https.onRequest(async (req, res) => { + if (req.method !== "POST") { + res.status(405).send("Method Not Allowed. Use POST.") + return + } + if (process.env.FUNCTIONS_EMULATOR !== "true") { + const authHeader = req.headers.authorization + if (!authHeader?.startsWith("Bearer ")) { + res.status(401).send("Unauthorized") + return + } + try { + const decoded = await getAuth().verifyIdToken(authHeader.slice(7)) + if (decoded["role"] !== "admin") { + res.status(403).send("Forbidden") + return + } + } catch { + res.status(401).send("Unauthorized") + return + } + } + + // ── A. Load member mapping ───────────────────────────────────────────── + const mappingDoc = await db.doc("/config/ocpfMemberMapping").get() + const mapping = (mappingDoc.data() ?? {}) as OcpfMemberMapping + + // Build cpfId → memberCode reverse map + let cpfIdToMemberCode = new Map( + Object.entries(mapping).map(([memberCode, entry]) => [ + entry.cpfId, + memberCode + ]) + ) + + if (TEST_CPF_ID !== null) { + cpfIdToMemberCode = new Map( + [...cpfIdToMemberCode.entries()].filter( + ([cpfId]) => cpfId === TEST_CPF_ID + ) + ) + functions.logger.info("TEST MODE: filtering to single member", { + cpfId: TEST_CPF_ID + }) + } + + functions.logger.info("Loaded member mapping", { + totalMembers: Object.keys(mapping).length, + activeInRun: cpfIdToMemberCode.size + }) + + // ── B. Download each year's ZIP; parse reports.txt then report-items.txt ── + const accumulators = new Map() + + // reportId → memberCode, for joining with report-items + const reportIdToMemberCode = new Map() + + for (const year of YEARS) { + const url = `${OCPF_BASE_URL}/ocpf-${year}-reports.zip` + functions.logger.info(`Downloading ${url}`) + const buf = await downloadBuffer(url) + await parseReports( + buf, + year, + cpfIdToMemberCode, + accumulators, + reportIdToMemberCode + ) + + functions.logger.info(`Streaming report-items for ${year}`) + await streamReportItems(buf, reportIdToMemberCode, accumulators, year) + } + + // ── Reconciliation: year-end report vs. summed periodic totals ──────── + // Year-end reports (type 11, etc.) are excluded from accumulation because + // their Receipts_Total/Expenditures_Total are annual rollups that duplicate + // the periodic (Bank Report) totals. + // This check runs only once a year-end report exists (i.e. + // after the calendar year closes) and compares it against what we've summed. + // + // A mismatch most likely means some other report type is being double-counted + // in the periodic accumulation. Deposit Reports (type 60) are a prime suspect: + // they itemize the same bank deposits the Bank Reports already capture and have + // not been verified yet. Do not ignore a mismatch — investigate the source + // before trusting the displayed totals. + for (const [memberCode, acc] of accumulators) { + for (const year of YEARS) { + const check = acc.yearEndCheck[year] + if (!check) continue + const summedRaised = acc.years[year]?.totalRaised ?? 0 + const summedSpent = acc.years[year]?.totalSpent ?? 0 + const raisedDiff = Math.abs(check.receiptsTotal - summedRaised) + const spentDiff = Math.abs(check.expendituresTotal - summedSpent) + if (raisedDiff > 0.02 || spentDiff > 0.02) { + functions.logger.warn( + "Year-end totals mismatch — investigate for double-counting", + { + memberCode, + year, + yearEnd: { + receiptsTotal: check.receiptsTotal, + expendituresTotal: check.expendituresTotal + }, + summed: { + receiptsTotal: summedRaised, + expendituresTotal: summedSpent + }, + diff: { receipts: raisedDiff, expenditures: spentDiff } + } + ) + } + } + } + + // ── C. Write Firestore docs ─────────────────────────────────────────── + const now = Timestamp.now() + // Firestore batches are limited to 500 operations. MA general courts have ~200 members + // so this is fine, but if we ever exceed 500 members this will need to be chunked. + const batch = db.batch() + + for (const [memberCode, acc] of accumulators) { + const doc = db.doc( + `/generalCourts/${currentGeneralCourt}/membersFinance/${memberCode}` + ) + const data: MembersFinance = { + ocpfCpfId: acc.cpfId, + totalRaised: acc.totalRaised, + totalSpent: acc.totalSpent, + cashOnHand: acc.cashOnHand, + contributorCount: acc.contributorCount, + lastUpdated: now, + breakdown: acc.breakdown as MembersFinanceBreakdown, + candidateFunds: acc.candidateFunds as MembersFinanceCandidateFunds, + inKind: acc.inKind as MembersFinanceInKind, + otherReceipts: acc.otherReceipts as MembersFinanceOtherReceipts, + years: Object.fromEntries( + Object.entries(acc.years).map(([y, yd]) => [ + y, + { + totalRaised: yd.totalRaised, + totalSpent: yd.totalSpent, + breakdown: yd.breakdown as MembersFinanceBreakdown, + finalized: acc.yearEndCheck[y] !== null + } as MembersFinanceYearData + ]) + ) + } + batch.set(doc, data) + } + + await batch.commit() + + functions.logger.info("scrapeOcpfFinance complete", { + processed: accumulators.size, + years: YEARS + }) + + res.status(200).json({ + results: { processed: accumulators.size, years: YEARS } + }) + }) + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +async function downloadBuffer(url: string): Promise { + const response = await axios.get(url, { responseType: "arraybuffer" }) + return Buffer.from(response.data as ArrayBuffer) +} + +const REPORT_COLUMN_ALIASES: Record = { + cpfId: ["cpf_id"], + reportId: ["report_id"], + reportTypeId: ["report_type_id"], + receiptsTotal: ["receipts_total"], + receiptsUnitemizedTotal: ["receipts_unitemized_total"], + expendituresTotal: ["expenditures_total"], + endBalance: ["end_balance"], + endDate: ["end_date"] +} + +const ITEM_COLUMN_ALIASES: Record = { + reportId: ["report_id"], + recordTypeId: ["record_type_id"], + amount: ["amount"] +} + +function buildIndex( + headers: string[], + aliases: Record +): Record { + const normalized = headers.map(h => h.toLowerCase().replace(/\s+/g, "_")) + const index: Record = {} + for (const [field, aliasList] of Object.entries(aliases)) { + for (const alias of aliasList) { + const i = normalized.indexOf(alias) + if (i !== -1) { + index[field] = i + break + } + } + if (!(field in index)) { + throw new Error( + `Required column '${field}' not found. Headers: ${headers.join(", ")}` + ) + } + } + return index +} + +function col(cols: string[], idx: number): string { + return (cols[idx] ?? "").trim().replace(/^"|"$/g, "") +} + +async function parseReports( + buf: Buffer, + year: string, + cpfIdToMemberCode: Map, + accumulators: Map, + reportIdToMemberCode: Map +): Promise { + const directory = await unzipper.Open.buffer(buf) + const entry = directory.files.find( + f => f.type === "File" && f.path.toLowerCase() === "reports.txt" + ) + if (!entry) throw new Error(`reports.txt not found in zip`) + + const text = (await entry.buffer()).toString("utf8") + const lines = text.split(/\r?\n/) + const rawHeaders = lines[0].split("\t").map(h => h.trim()) + const idx = buildIndex(rawHeaders, REPORT_COLUMN_ALIASES) + + let matched = 0 + for (let i = 1; i < lines.length; i++) { + const line = lines[i] + if (!line.trim()) continue + + const cols = line.split("\t") + const cpfId = parseInt(col(cols, idx.cpfId), 10) + const memberCode = cpfIdToMemberCode.get(cpfId) + if (!memberCode) continue + + const reportId = parseInt(col(cols, idx.reportId), 10) + if (isNaN(reportId)) continue + const reportTypeId = parseInt(col(cols, idx.reportTypeId), 10) + const receiptsTotal = parseFloat(col(cols, idx.receiptsTotal)) || 0 + const receiptsUnitemized = + parseFloat(col(cols, idx.receiptsUnitemizedTotal)) || 0 + const expendituresTotal = parseFloat(col(cols, idx.expendituresTotal)) || 0 + const endBalance = parseFloat(col(cols, idx.endBalance)) || 0 + + let acc = accumulators.get(memberCode) + if (!acc) { + acc = newAccumulator(cpfId) + accumulators.set(memberCode, acc) + } + + if (YEAR_END_REPORT_TYPE_IDS.has(reportTypeId)) { + // Annual rollup — skip accumulation to avoid double-counting the periodic + // totals already summed above. Store for the reconciliation check below. + if (acc.yearEndCheck[year] === null) { + acc.yearEndCheck[year] = { receiptsTotal, expendituresTotal } + } + matched++ + continue + } + + // Credit Card (type 80) and Reimbursement (type 90) reports are supplemental: + // they itemize spending that the depository bank already captured as bank + // transactions in the monthly Bank Report's Expenditures_Total. Verified + // empirically — for two members the sum of all Bank Reports matched the + // Year-End Report exactly without including type 80/90 amounts, confirming + // their Expenditures_Total is already counted. Their items (354 Credit Card + // Sub-Items, 351 Reimbursement Sub-Items) are not used by accumulateItem(), + // so there is no reason to register their report IDs. + if (reportTypeId === 80 || reportTypeId === 90) { + matched++ + continue + } + + reportIdToMemberCode.set(reportId, memberCode) + + // Deposit Reports (type 60) carry gross contribution amounts (before fees). + // The same money appears net-of-fees in Bank Report (type 70) Receipts_Total, + // so adding both would double-count. Skip the totals for type 60 but keep + // its report_id registered (for 201/202/203/204 item-level breakdown) and + // its unitemized amount (Bank Reports have blank for that field). + if (reportTypeId !== 60) { + acc.totalRaised += receiptsTotal + acc.totalSpent += expendituresTotal + } + acc.breakdown.unitemized.amount += receiptsUnitemized + + if (acc.years[year]) { + if (reportTypeId !== 60) { + acc.years[year].totalRaised += receiptsTotal + acc.years[year].totalSpent += expendituresTotal + } + acc.years[year].breakdown.unitemized.amount += receiptsUnitemized + } + + // Report_Type_ID 70 = Bank Report — use End_Balance from the report with the latest End_Date + const endDate = col(cols, idx.endDate) + const endDateMs = endDate ? new Date(endDate).getTime() : 0 + if (reportTypeId === 70 && endDateMs > acc.cashOnHandEndDateMs) { + acc.cashOnHand = endBalance + acc.cashOnHandEndDateMs = endDateMs + } + + matched++ + } + + functions.logger.info(`Parsed reports.txt for ${year}`, { matched }) +} + +async function streamReportItems( + buf: Buffer, + reportIdToMemberCode: Map, + accumulators: Map, + year: string +): Promise { + const directory = await unzipper.Open.buffer(buf) + const entry = directory.files.find( + f => f.type === "File" && f.path.toLowerCase() === "report-items.txt" + ) + if (!entry) throw new Error(`report-items.txt not found in zip`) + + const stream = entry.stream() + const rl = readline.createInterface({ input: stream, crlfDelay: Infinity }) + + let isFirst = true + let idx: Record = {} + let processed = 0 + let skipped = 0 + + for await (const line of rl) { + if (!line.trim()) continue + + if (isFirst) { + const rawHeaders = line.split("\t").map(h => h.trim()) + idx = buildIndex(rawHeaders, ITEM_COLUMN_ALIASES) + isFirst = false + continue + } + + const cols = line.split("\t") + const reportId = parseInt(col(cols, idx.reportId), 10) + const memberCode = reportIdToMemberCode.get(reportId) + if (!memberCode) { + skipped++ + continue + } + + const acc = accumulators.get(memberCode) + if (!acc) { + skipped++ + continue + } + + const recordTypeId = parseInt(col(cols, idx.recordTypeId), 10) + const amount = parseFloat(col(cols, idx.amount)) || 0 + + accumulateItem(acc, recordTypeId, amount, year) + processed++ + } + + functions.logger.info(`Streamed report-items.txt for ${year}`, { + processed, + skipped + }) +} + +function accumulateItem( + acc: MemberAccumulator, + recordTypeId: number, + amount: number, + year: string +): void { + const addTo = ( + entry: MutableBreakdownEntry, + alsoYearBreakdown?: MutableBreakdownEntry + ) => { + entry.count++ + entry.amount += amount + if (alsoYearBreakdown) { + alsoYearBreakdown.count++ + alsoYearBreakdown.amount += amount + } + } + + const yb = acc.years[year]?.breakdown + + switch (recordTypeId) { + case 201: // Individual Contribution + addTo(acc.breakdown.individual, yb?.individual) + acc.contributorCount++ + break + case 202: // Committee Contribution + addTo(acc.breakdown.committee, yb?.committee) + break + case 203: // Union/Association Contribution + addTo(acc.breakdown.union, yb?.union) + break + case 204: // Non-contribution receipt + addTo(acc.otherReceipts.nonContribution) + break + case 206: // Candidate Loan + case 331: // Out-of-pocket expense (as loan) + addTo(acc.candidateFunds.loans) + break + case 332: // Out-of-pocket expense (as contribution) + addTo(acc.candidateFunds.contributions) + break + case 401: // Individual In-kind + addTo(acc.inKind.individual) + break + case 402: // Committee In-kind + addTo(acc.inKind.committee) + break + case 403: // Union In-kind + addTo(acc.inKind.union) + break + case 420: // Aggregated un-itemized in-kind + acc.inKind.unitemized.amount += amount + break + // 205 (Bank Interest) and 220 (Aggregated un-itemized) come from reports.txt, not items + default: + break + } +} diff --git a/functions/src/ocpf/types.ts b/functions/src/ocpf/types.ts index ae61e2ce9..46165495d 100644 --- a/functions/src/ocpf/types.ts +++ b/functions/src/ocpf/types.ts @@ -39,3 +39,41 @@ export interface MembersFinanceBreakdown { union: FinanceBreakdownEntry unitemized: { amount: number } } + +export interface MembersFinanceCandidateFunds { + loans: FinanceBreakdownEntry // types 206 + 331 + contributions: FinanceBreakdownEntry // type 332 +} + +export interface MembersFinanceInKind { + individual: FinanceBreakdownEntry // type 401 + committee: FinanceBreakdownEntry // type 402 + union: FinanceBreakdownEntry // type 403 + unitemized: { amount: number } // type 420 +} + +export interface MembersFinanceOtherReceipts { + nonContribution: FinanceBreakdownEntry // type 204 +} + +export interface MembersFinanceYearData { + totalRaised: number + totalSpent: number + breakdown: MembersFinanceBreakdown + finalized: boolean +} + +// Firestore: /generalCourts/{court}/membersFinance/{memberCode} +export interface MembersFinance { + ocpfCpfId: number + totalRaised: number + totalSpent: number + cashOnHand: number + contributorCount: number // count of type-201 rows (row = one itemized contribution) + lastUpdated: FirebaseFirestore.Timestamp + breakdown: MembersFinanceBreakdown + candidateFunds: MembersFinanceCandidateFunds + inKind: MembersFinanceInKind + otherReceipts: MembersFinanceOtherReceipts + years: Record +} diff --git a/public/locales/en/legislators.json b/public/locales/en/legislators.json index 886f6a1ba..4362e82c5 100644 --- a/public/locales/en/legislators.json +++ b/public/locales/en/legislators.json @@ -4,6 +4,38 @@ "contact": "Contact", "cosponsored": "Cosponsored", "districtDetails": "District details are not available yet.", + "finance": { + "breakdown": { + "candidateFunds": "Candidate's own funds", + "committee": "Committees / PACs", + "inKind": "In-kind contributions", + "individual": "Individual donors", + "union": "Unions & associations", + "unitemized": "Small donors (under $50)" + }, + "breakdownHeading": "Contributions Breakdown", + "electionCycleHeading": "{{year}} Election Cycle", + "noContributions": "No itemized contributions recorded yet.", + "noData": "Finance data not yet available for this legislator.", + "otherReceipts": "Other non-contribution receipts: {{amount}} (refunds and miscellaneous receipts — not included above)", + "source": "Source: MA OCPF · ocpf.us", + "stats": { + "cashOnHand": "Cash on Hand", + "cashOnHandSubtitle": "End of reporting period", + "smallDonors": "Small Donors", + "smallDonorsSubtitle": "Under $200 contributions", + "totalRaised": "Total Raised*", + "totalRaisedFootnote": "*after processing fees", + "totalRaisedSubtitle": "From {{count}} contributors", + "totalSpent": "Total Spent", + "totalSpentSubtitle": "As of {{date}}" + }, + "table": { + "amount": "Amount", + "category": "Category", + "share": "Share" + } + }, "fundsRaised": "Funds Raised", "home": "Home", "legislators": "Legislators",