Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
0348b29
initial plan
nesanders Jun 4, 2026
774568e
feat: add lobbying disclosure ingestion pipeline
nesanders Jun 4, 2026
0074294
feat: add Python Cloud Run scraper for lobbying disclosures
nesanders Jun 5, 2026
2bcb783
refactor: remove dead TypeScript scraper code from lobbying module
nesanders Jun 9, 2026
df7c78e
Merge remote-tracking branch 'upstream/main' into lobbying-data-pipeline
nesanders Jun 26, 2026
26150ef
feat: parser fixes for all 4 disclosure eras + GCS archiving infrastr…
nesanders Jun 28, 2026
b66576a
fix: Firestore write path bugs found during dev write test
nesanders Jun 29, 2026
6129130
docs: update lobbying ingestion doc to reflect Python scraper and all…
nesanders Jun 29, 2026
1ef618a
docs: add real Firestore examples and fix registrantId description
nesanders Jun 30, 2026
2e2de7a
feat: lobbying explorer Phase 1 foundation
nesanders Jul 7, 2026
65e87cc
feat: lobbying bills browse page and bill card fixes
nesanders Jul 7, 2026
24d92fb
feat: add lobbying firms and clients browse/detail pages
nesanders Jul 7, 2026
03a12bd
fix: filter sentinel and empty clients from clients browse page
nesanders Jul 7, 2026
96393e5
feat: lobbying overview page with stats, chart, and attribution fix
nesanders Jul 8, 2026
d6e2977
feat: add attribution to all lobbying pages and seed stats script
nesanders Jul 9, 2026
748f507
fix: add lobbyingMeta read rule and move Lobbying after Testimony in nav
nesanders Jul 9, 2026
a918d88
feat: add pagination, subnav, sortable headers, and bill filings deta…
nesanders Jul 9, 2026
efce9f9
fix: i18n all hardcoded strings in lobbying explorer; add mobile resp…
nesanders Jul 9, 2026
6b806da
fix: add archive.py to Dockerfile — was missing from COPY, causing Mo…
nesanders Jul 15, 2026
80901b9
perf: skip already-completed years in backfill to speed up resume
nesanders Jul 16, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -92,5 +92,10 @@ cert.txt
# local MCP server config (contains auth tokens)
.mcp.json
mcp-server/create-agent-key.ts
.gcloudignore

# Claude
CLAUDE.md

#gcloud
.gcloudignore

10 changes: 10 additions & 0 deletions components/Navbar.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
NavbarLinkAiTools,
NavbarLinkBills,
NavbarLinkHearings,
NavbarLinkLobbying,
NavbarLinkEditProfile,
NavbarLinkEffective,
NavbarLinkFAQ,
Expand Down Expand Up @@ -79,6 +80,9 @@ const MobileNav: React.FC<React.PropsWithChildren<unknown>> = () => {
<NavbarLinkHearings handleClick={closeNav} />
) : null}
<NavbarLinkTestimony handleClick={closeNav} />
{flags().lobbyingTable ? (
<NavbarLinkLobbying handleClick={closeNav} />
) : null}
{authenticated ? <NavbarLinkNewsfeed handleClick={closeNav} /> : <></>}
<NavDropdown className={"navLink-primary"} title={t("about")}>
<NavbarLinkGoals handleClick={closeNav} />
Expand Down Expand Up @@ -233,6 +237,12 @@ const DesktopNav: React.FC<React.PropsWithChildren<unknown>> = () => {
<NavbarLinkTestimony />
</div>

{flags().lobbyingTable ? (
<div className={`align-self-center`}>
<NavbarLinkLobbying />
</div>
) : null}

{authenticated ? (
<div className="align-self-center">
<NavbarLinkNewsfeed />
Expand Down
21 changes: 21 additions & 0 deletions components/NavbarComponents.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,27 @@ export const NavbarLinkBills: React.FC<
)
}

export const NavbarLinkLobbying: React.FC<
React.PropsWithChildren<{
handleClick?: any
other?: any
}>
> = ({ handleClick, other }) => {
const isMobile = useMediaQuery("(max-width: 768px)")
const { t } = useTranslation(["common", "auth"])
return (
<Nav.Item onClick={handleClick}>
<NavLink
className={isMobile ? "navLink-primary" : DESKTOP_NAV_ITEM_CLASS}
href="/lobbying"
{...other}
>
{t("navigation.lobbying")}
</NavLink>
</Nav.Item>
)
}

export const NavbarLinkBallotQuestions: React.FC<
React.PropsWithChildren<{
handleClick?: any
Expand Down
8 changes: 6 additions & 2 deletions components/bill/BillDetails.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { Banner } from "../shared/StyledSharedComponents"
import { BillNumber, Styled } from "./BillNumber"
import { BillTestimonies } from "./BillTestimonies"
import BillTrackerConnectedView from "./BillTracker"
import { LobbyingTable } from "./LobbyingTable"
import { LobbyingBillCard } from "components/lobbying/LobbyingBillCard"
import { Committees, Hearing, Sponsors } from "./SponsorsAndCommittees"
import { Status } from "./Status"
import { Summary } from "./Summary"
Expand Down Expand Up @@ -99,7 +99,11 @@ export const BillDetails = ({ bill }: BillProps) => {

<BillTestimonies bill={bill} className="mt-4" />
{flags.lobbyingTable && (
<LobbyingTable bill={bill} className="mt-4 pb-1" />
<LobbyingBillCard
court={bill.court}
billId={bill.id}
className="mt-4 pb-1"
/>
)}
</Col>
<Col md={4}>
Expand Down
228 changes: 228 additions & 0 deletions components/db/lobbying.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,228 @@
import {
collection,
doc,
getDoc,
getDocs,
limit,
orderBy,
query,
where
} from "firebase/firestore"
import { useAsync } from "react-async-hook"
import type {
LobbyingClientSummary,
LobbyingFiling,
LobbyingRegistrant,
LobbyingStats
} from "functions/src/lobbying/types"
import { firestore } from "../firebase"

// Mirror of constants in functions/src/lobbying/types.ts — kept here to avoid
// pulling firebase-admin (a Node-only package) into the browser bundle.
const FILINGS_COLLECTION = "lobbyingFilings"
const REGISTRANTS_COLLECTION = "lobbyingRegistrants"
const CLIENTS_COLLECTION = "lobbyingClients"
const LOBBYING_STATS_COLLECTION = "lobbyingMeta"
const LOBBYING_STATS_DOC_ID = "stats"

// ── Internal fetchers ─────────────────────────────────────────────────────────

async function fetchLobbyingStats(): Promise<LobbyingStats | undefined> {
const snap = await getDoc(
doc(firestore, LOBBYING_STATS_COLLECTION, LOBBYING_STATS_DOC_ID)
)
return snap.exists() ? (snap.data() as LobbyingStats) : undefined
}

async function fetchFilingsForBill(
court: number,
billId: string
): Promise<LobbyingFiling[]> {
const snap = await getDocs(
query(
collection(firestore, FILINGS_COLLECTION),
where("generalCourt", "==", court),
where("billId", "==", billId)
)
)
return snap.docs.map(d => d.data() as LobbyingFiling)
}

async function fetchRegistrants(opts: {
regType?: "Lobbyist" | "Employer"
year?: number
pageSize?: number
}): Promise<LobbyingRegistrant[]> {
const constraints = [
opts.regType ? where("regType", "==", opts.regType) : undefined,
opts.year ? where("year", "==", opts.year) : undefined,
orderBy("entityNameNorm"),
limit(opts.pageSize ?? 50)
].filter(Boolean) as Parameters<typeof query>[1][]

const snap = await getDocs(
query(collection(firestore, REGISTRANTS_COLLECTION), ...constraints)
)
return snap.docs.map(d => d.data() as LobbyingRegistrant)
}

async function fetchRegistrant(
registrantId: string
): Promise<LobbyingRegistrant | undefined> {
const snap = await getDoc(
doc(firestore, REGISTRANTS_COLLECTION, registrantId)
)
return snap.exists() ? (snap.data() as LobbyingRegistrant) : undefined
}

async function fetchFilingsForRegistrant(
registrantId: string
): Promise<LobbyingFiling[]> {
const snap = await getDocs(
query(
collection(firestore, FILINGS_COLLECTION),
where("registrantId", "==", registrantId),
orderBy("year", "desc")
)
)
return snap.docs.map(d => d.data() as LobbyingFiling)
}

async function fetchClients(opts: {
year?: number
pageSize?: number
}): Promise<LobbyingClientSummary[]> {
const constraints = [
opts.year ? where("years", "array-contains", opts.year) : undefined,
orderBy("clientNameNorm"),
limit(opts.pageSize ?? 50)
].filter(Boolean) as Parameters<typeof query>[1][]

const snap = await getDocs(
query(collection(firestore, CLIENTS_COLLECTION), ...constraints)
)
return snap.docs.map(d => d.data() as LobbyingClientSummary)
}

async function fetchClient(
clientSlug: string
): Promise<LobbyingClientSummary | undefined> {
const snap = await getDoc(doc(firestore, CLIENTS_COLLECTION, clientSlug))
return snap.exists() ? (snap.data() as LobbyingClientSummary) : undefined
}

async function fetchAllRegistrants(): Promise<LobbyingRegistrant[]> {
const snap = await getDocs(
query(
collection(firestore, REGISTRANTS_COLLECTION),
orderBy("entityNameNorm"),
limit(2000)
)
)
return snap.docs.map(d => d.data() as LobbyingRegistrant)
}

async function fetchRegistrantsByEntityName(
entityNameNorm: string
): Promise<LobbyingRegistrant[]> {
const snap = await getDocs(
query(
collection(firestore, REGISTRANTS_COLLECTION),
where("entityNameNorm", "==", entityNameNorm),
orderBy("year", "desc")
)
)
return snap.docs.map(d => d.data() as LobbyingRegistrant)
}

async function fetchFilingsForEntityName(
entityNameNorm: string
): Promise<LobbyingFiling[]> {
const snap = await getDocs(
query(
collection(firestore, FILINGS_COLLECTION),
where("entityNameNorm", "==", entityNameNorm),
orderBy("year", "desc")
)
)
return snap.docs.map(d => d.data() as LobbyingFiling)
}

async function fetchFilingsForCourt(court: number): Promise<LobbyingFiling[]> {
const snap = await getDocs(
query(
collection(firestore, FILINGS_COLLECTION),
where("generalCourt", "==", court)
)
)
return snap.docs.map(d => d.data() as LobbyingFiling)
}

async function fetchFilingsForClient(
clientNameNorm: string
): Promise<LobbyingFiling[]> {
const snap = await getDocs(
query(
collection(firestore, FILINGS_COLLECTION),
where("clientNameNorm", "==", clientNameNorm),
orderBy("year", "desc")
)
)
return snap.docs.map(d => d.data() as LobbyingFiling)
}

// ── Public hooks ──────────────────────────────────────────────────────────────

export function useLobbyingFilingsForCourt(court: number) {
return useAsync(fetchFilingsForCourt, [court])
}

export function useLobbyingAllRegistrants() {
return useAsync(fetchAllRegistrants, [])
}

export function useLobbyingRegistrantsByEntityName(entityNameNorm: string) {
return useAsync(fetchRegistrantsByEntityName, [entityNameNorm])
}

export function useLobbyingFilingsForEntityName(entityNameNorm: string) {
return useAsync(fetchFilingsForEntityName, [entityNameNorm])
}

export function useLobbyingStats() {
return useAsync(fetchLobbyingStats, [])
}

export function useLobbyingFilingsForBill(court: number, billId: string) {
return useAsync(fetchFilingsForBill, [court, billId])
}

export function useLobbyingRegistrants(opts: {
regType?: "Lobbyist" | "Employer"
year?: number
pageSize?: number
}) {
return useAsync(fetchRegistrants, [opts])
}

export function useLobbyingRegistrant(registrantId: string) {
return useAsync(fetchRegistrant, [registrantId])
}

export function useLobbyingFilingsForRegistrant(registrantId: string) {
return useAsync(fetchFilingsForRegistrant, [registrantId])
}

export function useLobbyingClients(
opts: { year?: number; pageSize?: number } = {}
) {
return useAsync(fetchClients, [opts])
}

export function useLobbyingClient(clientSlug: string) {
return useAsync(fetchClient, [clientSlug])
}

export function useLobbyingFilingsForClient(clientNameNorm: string) {
return useAsync(fetchFilingsForClient, [clientNameNorm])
}
2 changes: 1 addition & 1 deletion components/featureFlags.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ const defaults: Record<Env, FeatureFlags> = {
notifications: true,
billTracker: true,
followOrg: true,
lobbyingTable: false,
lobbyingTable: true,
hearingsAndTranscriptions: true,
phoneVerificationUI: true,
ballotQuestions: true,
Expand Down
29 changes: 29 additions & 0 deletions components/lobbying/LobbyingAttribution.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
import React from "react"
import { useTranslation } from "next-i18next"
import { MAPLE_COLORS } from "./chartTheme"

const SOS_URL = "https://www.sec.state.ma.us/LobbyistPublicSearch/Default.aspx"

export const LobbyingAttribution: React.FC<{ className?: string }> = ({
className
}) => {
const { t } = useTranslation("lobbying")
return (
<p style={style} className={className}>
<a href={SOS_URL} target="_blank" rel="noreferrer" style={linkStyle}>
{t("attribution")}
</a>
</p>
)
}

const style: React.CSSProperties = {
fontSize: 12,
color: MAPLE_COLORS.textMuted,
marginTop: "1rem"
}

const linkStyle: React.CSSProperties = {
color: MAPLE_COLORS.textMuted,
textDecoration: "underline"
}
Loading