diff --git a/.env.example b/.env.example index fb0f43bd..1d88309d 100644 --- a/.env.example +++ b/.env.example @@ -377,6 +377,26 @@ ARCHIVE_RESYNC_TICK_SECONDS=600 # keeping headroom for the live sync pipeline. ARCHIVE_RESYNC_MIN_RATE_REMAINING=2500 +# Site analytics (pageview ingestion) +# Secret salt for visitor monthly hash (sha256-based; required in production). +SITE_ANALYTICS_HASH_SALT= +# Comma-separated slugs of sites allowed to post events; unknown slugs → 400. +SITE_ANALYTICS_ALLOWED_SITES= +# Raw pageview retention window in days (default 540 ≈ 18 months). +SITE_ANALYTICS_RETENTION_DAYS=540 +# Reverse proxies in front of this app. X-Forwarded-For is client-controlled, so only +# this many entries from the right are trusted (Heroku's router appends exactly one). +# Set to 0 if the app is exposed directly, to ignore X-Forwarded-For entirely. +SITE_ANALYTICS_TRUSTED_PROXY_COUNT=1 +# Beat period for the daily aggregation task (seconds); 0 disables the schedule. +SITE_ANALYTICS_DAILY_AGGREGATE_PERIOD_SECONDS=3600 +# Beat period for the monthly aggregation task (seconds); 0 disables the schedule. +SITE_ANALYTICS_MONTHLY_AGGREGATE_PERIOD_SECONDS=86400 +# Beat period for the retention pruning task (seconds); 0 disables the schedule. +SITE_ANALYTICS_PRUNE_PERIOD_SECONDS=86400 +# Reject requests with an empty User-Agent (stricter bot hardening; default off). +SITE_ANALYTICS_REJECT_EMPTY_UA=0 + # CI filter (opt-in allowlist) # Set SYNCER_CI_FILTER_MODE=allowlist to enable filtering by the allow lists below. # Substrings matched case-insensitively against CheckRun.name and StatusContext.context. diff --git a/AGENTS.md b/AGENTS.md index c0698570..029fd36a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ ## Project Structure & Module Organization - `src/queueboard/` contains the legacy Python data pipeline: GraphQL queries under `queries/`, HTML assets in `static/`, and scripts like `dashboard.py`, `process.py`, and `suggest_reviewer.py`. -- `qb_site/` hosts the Django codebase; apps live in `qb_site/{core,syncer,analyzer,api,zulip_bot}/` and share settings from `qb_site/qb_site/settings/`. +- `qb_site/` hosts the Django codebase; apps live in `qb_site/{core,syncer,analyzer,api,zulip_bot,site_analytics}/` and share settings from `qb_site/qb_site/settings/`. - `scripts/` provides operational helpers; `test/` stores fixture JSON for dashboard regression checks; `docs/` captures architecture plans/decisions. ## Build, Test, and Development Commands @@ -68,7 +68,7 @@ Notes ## Keeping AGENTS.md Files Updated - Every directory with significant logic has its own `AGENTS.md` (mirrored as `CLAUDE.md`). Current locations: root, `qb_site/`, `qb_site/syncer/`, `qb_site/analyzer/`, - `qb_site/zulip_bot/`, `qb_site/console/`, `src/queueboard/`. + `qb_site/zulip_bot/`, `qb_site/console/`, `qb_site/site_analytics/`, `src/queueboard/`. - When you add, rename, or remove management commands, Celery tasks, key services, or directory structure, update the relevant `AGENTS.md` in the same commit/PR. - When you add a new app or significant sub-directory, create a matching `AGENTS.md` diff --git a/docs/design-decisions/031-analytics-ingestion-design.md b/docs/design-decisions/031-analytics-ingestion-design.md index 822eb228..89f43131 100644 --- a/docs/design-decisions/031-analytics-ingestion-design.md +++ b/docs/design-decisions/031-analytics-ingestion-design.md @@ -1,175 +1,208 @@ -# Analytics Ingestion for `qb_site` (Living Plan) +# Site Analytics Ingestion (`site_analytics`) ## Context -- We want lightweight website analytics for funder-facing growth reporting. -- Current `qb_site` structure already has clear boundaries: - - `api/` for HTTP surface area. - - `syncer/` for raw GitHub ingestion. - - `analyzer/` for derived analytics built from stored facts. -- Existing Celery beat and snapshot patterns in `qb_site/qb_site/settings/base.py` and `analyzer/tasks/*` are good templates. -- The previous version of this doc was generic and not mapped to repo-specific modules, rollout, or tests. - -## Problem Statement -- Add privacy-preserving pageview ingestion for static properties without introducing new infrastructure. -- Keep implementation operationally simple and consistent with the current Django/Celery architecture. -- Make implementation trackable as a chunked, testable plan that can be updated during delivery. - -## Goals / Non-Goals -- Goals: - - Collect coarse pageview/referrer signals for multiple sites. - - Produce daily and monthly aggregates suitable for funder reporting. - - Enforce privacy constraints (no raw IP retention, no persistent cross-month identifiers). - - Keep ingestion endpoint lightweight and resilient. -- Non-goals (v1): - - Sessionization, funnels, attribution modeling. - - Cookie-based or long-lived user identity. - - Real-time dashboarding. - - New data infrastructure (Kafka/ClickHouse/etc.). - -## Decision (Current Plan) -- Implement a new Django app: `site_analytics`. -- Expose ingestion under existing API namespace via `qb_site/api/urls.py`. -- Store raw event rows in `site_analytics` (bounded retention), and store reporting reads in aggregated tables. -- Run periodic aggregation with Celery beat, following existing analyzer/syncer task style. -- Keep auth simple in v1: optional per-site shared token + bot/user-agent filtering. - -## Proposed Design - -### App and module placement -- New app: `qb_site/site_analytics/` - - `models/`: raw + aggregate tables. - - `services/`: hashing, bot filtering, aggregation logic. - - `tasks/`: periodic aggregation + retention. - - `tests/`: model/service/task coverage. -- API entrypoint: - - Add endpoint in `qb_site/api/urls.py`. - - View implementation in `qb_site/api/views/analytics_collect.py`. - - This keeps external endpoints discoverable in one API module. - -### HTTP ingestion contract -- Endpoint: `POST /api/v1/analytics/collect` -- Request payload (v1): - - `site` (slug; required) - - `path` (required) - - `referrer` (optional) - - `token` (optional; validated when site is configured as token-required) -- Behavior: - - CSRF-exempt endpoint for third-party/static-site calls. - - Minimal synchronous work: validate, compute hash fields, write one row, return `204`. - - Reject malformed payloads with `400`; unauthorized token with `403`. - -### Data model (planned) -- `AnalyticsPageView` (raw) - - `site`, `path`, `referrer`, `user_agent` - - `occurred_at` (event time, default `timezone.now`) - - `visitor_month_hash` (privacy-preserving monthly hash) - - indexes on `(site, occurred_at)`, `(occurred_at)`, and optionally `(site, path, occurred_at)` -- `AnalyticsDailyMetric` (aggregate) - - `site`, `date` - - `pageviews`, `unique_visitors` - - unique constraint on `(site, date)` -- `AnalyticsMonthlyMetric` (aggregate/reporting convenience) - - `site`, `month` - - `pageviews`, `unique_visitors`, `top_referrers_json`, `top_paths_json` (optional in v1) - - unique constraint on `(site, month)` - -### Privacy and identity strategy -- Do not persist raw IP addresses. -- Compute `visitor_month_hash = sha256(ip + normalized_user_agent + month_key + secret_salt)`. -- `month_key` is UTC `YYYY-MM`; this intentionally prevents cross-month correlation. -- `secret_salt` comes from env/config (e.g., `SITE_ANALYTICS_HASH_SALT`). -- Retain raw rows only for bounded backfill/debug windows (target: 12-18 months). - -### Aggregation strategy -- Add periodic tasks: - - `site_analytics.aggregate_daily_metrics` (hourly or nightly; idempotent upsert). - - `site_analytics.aggregate_monthly_metrics` (daily; recompute current + previous month). - - `site_analytics.prune_old_pageviews` (daily retention cleanup). -- Wire schedules in `qb_site/qb_site/settings/base.py` with env-overridable intervals and retention days. - -### Security and abuse controls -- Lightweight allowlist/denylist for `site` identifiers. -- Basic bot filtering: - - denylist common bot user-agent substrings. - - optional reject when user-agent is empty. -- Optional DRF/Django rate limiting for collection endpoint (can begin with app-level simple limits). - -### Public sanitized backup compatibility -- This data will flow through the public backup pipeline in `.github/workflows/upload_backup.yaml`. -- Keep analytics tables and fields compatible with sanitization/export scripts (`scripts/sanitize_backup.py`, `scripts/export_for_analysis.py`). -- If analytics introduces potentially sensitive columns, update sanitization rules and manifest outputs in the same change. -- Treat this as a release gate for analytics schema changes, consistent with `docs/design-decisions/016-sanitized-backups.md`. - -## Invariants / Subtleties +- Lightweight, privacy-preserving pageview analytics for funder-facing growth reporting on static community sites. +- No new infrastructure: implemented as a Django app inside the existing `qb_site` stack (Postgres, Celery, Redis). +- Privacy constraint: no raw IP retention; visitor identity approximated by a monthly-rotating hash. +- Operational constraint: ingestion endpoint must be fast, resilient, and callable from third-party static sites (CORS required). + +## Decision +- New Django app `site_analytics` with four models, a REST ingestion endpoint, and four Celery periodic tasks. +- Site allowlist (`SITE_ANALYTICS_ALLOWED_SITES`) gates ingestion; no per-site auth tokens in v1. +- Reporting reads from aggregate tables only; raw rows are bounded-retention scratch space. +- No rate limiting in v1 — first rollout is to internally-used sites. See [Deferred: rate limiting](#deferred-rate-limiting-on-the-collection-endpoint) for the rationale, the trigger to revisit, and the intended shape. +- Ingestion fails closed rather than degrading: with no salt available, events are dropped, not stored under a weak hash. + +## Architecture + +### Models +- `AnalyticsPageView` — raw event rows; immutable after insert; pruned after `SITE_ANALYTICS_RETENTION_DAYS` (default 540). + - Fields: `site`, `path`, `referrer`, `user_agent`, `occurred_at`, `visitor_month_hash`. + - Indexes: `(site, occurred_at)`, `(occurred_at)`. +- `AnalyticsDailyMetric` — daily aggregate per site; unique on `(site, date)`. + - Fields: `site`, `date` (UTC), `pageviews`, `unique_visitors`. +- `AnalyticsMonthlyMetric` — monthly aggregate per site; unique on `(site, month)`. + - Fields: `site`, `month` (UTC first-of-month `DateField`, e.g. `2026-03-01`), `pageviews`, `unique_visitors`. +- `SiteAnalyticsSalt` — single live row holding the current month's visitor-hash salt; previous row deleted on rotation. + +### Ingestion endpoint +- `POST /api/v1/analytics/collect` — view in `api/views/analytics_collect.py`. +- Required fields: `site` (must be in `SITE_ANALYTICS_ALLOWED_SITES`), `path`. +- Optional field: `referrer`. +- `User-Agent` read from HTTP header (not payload). +- Returns `204` on success, bot drop, and empty-UA drop; `400` on validation failure. +- CORS headers (`Access-Control-Allow-Origin: *`) on all responses; `OPTIONS` preflight handled. +- No CSRF enforcement: DRF `authentication_classes = []` / `permission_classes = []`. + +### Privacy +- Raw IP not stored. `visitor_month_hash = sha256(ip | normalized_ua | salt)`. +- Fields joined with `|` to prevent cross-field hash collisions. +- UA is lowercased before hashing so casing variation in the same browser does not inflate unique-visitor counts. +- IP extracted from `X-Forwarded-For` taking `SITE_ANALYTICS_TRUSTED_PROXY_COUNT` entries from the **right** (default 1), falling back to `REMOTE_ADDR` for direct connections. Proxies *append* the address they received the connection from, so with one hop (Heroku's router) the rightmost entry is the only trustworthy one; everything to its left is a client-supplied claim. Reading the leftmost entry — as v1 originally did — let any visitor send an arbitrary `X-Forwarded-For` and mint a fresh `visitor_month_hash` per request, inflating unique-visitor counts at zero cost. Set the count to `0` when the app is exposed directly, which ignores the header entirely. +- Cross-month correlation is prevented by monthly salt rotation: the salt is replaced at the start of each month and the old value discarded, so hashes from different months are unlinkable even with knowledge of the current salt (forward secrecy against salt leakage). +- Salt is stored in `SiteAnalyticsSalt` (DB, one live row). `SITE_ANALYTICS_HASH_SALT` env var is the fallback until the first rotation task runs. **Changing the separator invalidates all historical hashes.** +- **Ingestion fails closed with no salt.** `compute_visitor_hash` raises `SaltUnavailable` when neither source yields a salt, and the view drops the event (`204` + error log) rather than persist the hash. An unsalted `sha256(ip | ua)` is brute-forceable across the IPv4 space, so it is a recoverable identifier, not a pseudonymous one — collecting nothing is the correct failure mode for this pipeline. This is the runtime guarantee, and it also covers the salt row disappearing after boot. +- A deploy-time system check (`site_analytics.E001`, `site_analytics/checks.py`) fails `manage.py check` and `migrate` when `SITE_ANALYTICS_ALLOWED_SITES` is non-empty but no salt is configured, so a misconfigured deploy stops in the release phase. It is gated on allowed-sites because analytics is opt-in (CI and dev are unaffected), and reads settings only — never the DB, since `migrate` runs checks before `SiteAnalyticsSalt` exists. Gunicorn does not run system checks when loading the WSGI app, which is why the check alone is not sufficient. + +### Bot filtering +- Substring denylist in `site_analytics/services/bot_filter.py` matched against lowercased UA. +- Bots return `204` (not a distinct error code) to avoid leaking detection heuristics. +- Optional empty-UA rejection via `SITE_ANALYTICS_REJECT_EMPTY_UA=1` (default off); also returns `204`. + +### Aggregation +- `site_analytics.aggregate_daily_metrics` (default: hourly) — upserts `AnalyticsDailyMetric` for a rolling `days_back=2` window so events near UTC midnight are never missed. Fully idempotent. +- `site_analytics.aggregate_monthly_metrics` (default: daily) — upserts `AnalyticsMonthlyMetric` for a rolling `months_back=2` window. +- `site_analytics.prune_old_pageviews` (default: daily) — deletes `AnalyticsPageView` rows older than the retention cutoff. Aggregate tables are never pruned. +- `site_analytics.rotate_salt` (monthly, midnight UTC on the 1st) — generates a new random salt, saves it to `SiteAnalyticsSalt`, and deletes the previous row atomically. +- All three tasks preserve existing aggregate rows when the corresponding raw rows have been pruned, to avoid silently zeroing out reporting data after the retention window. + +### Backup policy +- `site_analytics_analyticspageview` → `TRUNCATE_TABLES` (raw rows contain visitor hashes). +- `site_analytics_analyticsdailymetric`, `site_analytics_analyticsmonthlymetric` → `RETAIN_TABLES` (aggregate-only, safe to share publicly). +- `site_analytics_siteanalyticssalt` → `TRUNCATE_TABLES` (contains the live secret; must never appear in sanitized backups). + +## Invariants - Reporting reads must come from aggregate tables, not raw pageview scans. -- Hashing semantics are part of the privacy contract; any change requires explicit migration/versioning note. -- Aggregation tasks must be idempotent and safe under retries/overlap. -- Time boundary semantics must use UTC consistently for day/month buckets. -- `site` taxonomy must remain stable; renames need explicit backfill/mapping handling. - -## Implementation Plan (Chunks) -1. `A1` App scaffold + settings wiring. - - Create `site_analytics` app and add to `INSTALLED_APPS`. - - Add env settings for hash salt, retention days, and task periods. -2. `A2` Raw ingestion endpoint + validation. - - Add `POST /api/v1/analytics/collect` route and view. - - Implement payload validation, token check, hashing service, and raw insert. -3. `A3` Aggregate models + aggregation services. - - Add daily/monthly metric models and upsert services. - - Add management command entrypoints for manual runs. -4. `A4` Celery task scheduling. - - Add periodic tasks + beat schedule keys in base settings. - - Ensure retry-safe/idempotent task behavior. -5. `A5` Admin + operational visibility. - - Register admin for raw/aggregate models. - - Add concise task result payloads/counters. -6. `A6` Retention and hardening. - - Add pruning task and tests. - - Tune bot filtering and optional throttle policy. - -## Validation Plan -- Unit tests: - - hashing/privacy rules, month rotation behavior. - - bot-filter decisions and payload validation. - - aggregation correctness (`COUNT(*)`, distinct hash counts). -- API tests (`qb_site/api/tests/`): - - `204` success path. - - `400` invalid payload. - - `403` invalid token (when enabled). -- Task tests: - - idempotent reruns. - - retry-safe partial failure behavior. - - retention pruning boundaries. -- Integration checks: - - `uv run ruff check qb_site` - - `uv run ruff format qb_site` - - Compose-backed tests via `bash scripts/repo_check_compose.sh` when available. - -## Rollout Plan -- Phase 0: dark launch ingestion in one low-risk site. -- Phase 1: enable daily/monthly aggregation and validate numbers manually for 1-2 weeks. -- Phase 2: onboard remaining static sites and publish recurring reporting output. -- Phase 3: tighten retention and evaluate need for partitioning at higher volumes. - -## Progress Notes -- 2026-02-26: - - Converted this file from generic guidance to a repo-specific living plan. - - Anchored implementation to existing `qb_site` boundaries and task patterns. - - No code implementation started yet; all chunks currently pending. - -## Open Questions -- Should `site` configuration live in DB (admin-editable) or settings/env (static)? -- Do we need per-path monthly aggregates in v1, or can we defer to v1.1? -- What default retention window is acceptable for privacy/compliance expectations? -- Should endpoint auth be required for all sites from day one, or optional during bootstrap? +- Hashing semantics (field separator `|`, UA normalization, salt source) are part of the privacy contract; any change requires an explicit migration note and bumps all historical hashes. +- A visitor hash is never persisted without a salt. Any future caller of `compute_visitor_hash` must let `SaltUnavailable` propagate or drop the event — never fall back to an unsalted digest. +- Client-supplied headers are never trusted for visitor identity. `X-Forwarded-For` is attacker-controlled to the left of the hops we actually run behind; anything deriving identity (hashing today, rate-limit keys later) must go through `get_client_ip` rather than reading the header directly. +- Aggregation tasks must remain idempotent and safe under retries and overlapping runs. +- All date/time boundaries use UTC. `occurred_at__date=d` with `USE_TZ=True` and `TIME_ZONE=UTC` evaluates at UTC midnight in PostgreSQL. +- `site` slugs must remain stable; renames require an explicit backfill of both raw rows and aggregate tables. + +## Operational Notes + +### Key settings (all env-overridable) +| Setting | Default | Notes | +|---|---|---| +| `SITE_ANALYTICS_HASH_SALT` | `""` | Bootstrap salt until `rotate_salt` first runs. Empty **drops all events** (fail closed), and fails `check`/`migrate` when allowed-sites is set | +| `SITE_ANALYTICS_ALLOWED_SITES` | `""` | Comma-separated slugs; empty list rejects all traffic (and disables the salt check) | +| `SITE_ANALYTICS_RETENTION_DAYS` | `540` | ~18 months of raw row retention | +| `SITE_ANALYTICS_TRUSTED_PROXY_COUNT` | `1` | Proxy hops in front of the app; trusts that many `X-Forwarded-For` entries from the right. `0` ignores the header | +| `SITE_ANALYTICS_DAILY_AGGREGATE_PERIOD_SECONDS` | `3600` | Set to `0` to disable | +| `SITE_ANALYTICS_MONTHLY_AGGREGATE_PERIOD_SECONDS` | `86400` | Set to `0` to disable | +| `SITE_ANALYTICS_PRUNE_PERIOD_SECONDS` | `86400` | Set to `0` to disable | +| `SITE_ANALYTICS_REJECT_EMPTY_UA` | `0` | Set to `1` for stricter bot hardening | + +### Onboarding a new site +1. Add the site slug to `SITE_ANALYTICS_ALLOWED_SITES` (comma-separated, no spaces). +2. Deploy/restart the web dyno so the new slug is live. +3. Add the tracking snippet to the site (see below). +4. Add a visible privacy notice to the site informing visitors that anonymous visit counts are collected (no cookies, no IP addresses stored). See disclosure notes below. +5. Verify events appear in the Django admin under `AnalyticsPageView`. +6. After one aggregation cycle, check `AnalyticsDailyMetric` for counts. + +### Static-site tracking snippet + +Place this snippet at the bottom of each page (or in a shared layout template). +Replace `YOUR_QUEUEBOARD_HOST` and `YOUR_SITE_SLUG` before deploying. + +```html + +``` + +**Notes:** +- The snippet is fire-and-forget; errors are silently swallowed so a tracking failure never affects page load. +- `sendBeacon` is preferred: it survives page unload and does not block navigation. +- No cookies, no persistent identifiers, no third-party scripts. +- The endpoint returns `204` for all non-error outcomes (success, bot drop, unknown UA) so the response body is never read. + +### Disclosure and privacy regulations + +This system is designed to minimise regulatory obligations, but the picture is nuanced enough to warrant documentation. *This section is informational, not legal advice.* + +**ePrivacy Directive (EU, [2002/58/EC](https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX:02002L0058-20091219) Art. 5(3)) and UK PECR (SI 2003/2426 Reg. 6)** — these rules require consent for *storing information in, or gaining access to information already stored in, the user's terminal equipment*. This system performs server-side hash computation on data transmitted in the HTTP request (IP address from the network layer, `User-Agent` header) and writes nothing to the user's device (no cookies, no localStorage, no fingerprinting scripts). The [EDPB Guidelines 2/2023 on the Technical Scope of Art. 5(3)](https://www.edpb.europa.eu/our-work-tools/our-documents/guidelines/guidelines-22023-technical-scope-art-53-eprivacy-directive_en) (adopted October 2024) take a broad view: they state that gaining access to IP addresses triggers Art. 5(3) "in cases where this information originates from the terminal equipment of a subscriber or user." Whether passively transmitted HTTP request metadata (as opposed to data actively read from device storage) falls under this scope is not conclusively settled. The [ICO's guidance on storage and access technologies](https://ico.org.uk/for-organisations/direct-marketing-and-privacy-and-electronic-communications/guidance-on-the-use-of-storage-and-access-technologies/) defines PECR Regulation 6 as covering technologies that "store information on a user's device or gain access to information on a user's device." On balance, a system that neither stores nor reads from the device is likely outside the scope of Art. 5(3) / Reg. 6, but this is an area of evolving regulatory interpretation. + +**GDPR / UK GDPR ([Regulation 2016/679](https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX:32016R0679))** — whether GDPR applies turns on whether the monthly rotating hash constitutes "personal data" under Art. 4(1). Recital 26 excludes "anonymous information" from GDPR's scope, and provides a "means reasonably likely" test: whether identification is feasible given "all objective factors, such as the costs of and the amount of time required for identification, taking into consideration the available technology." The CJEU's ruling in [*Breyer v. Bundesrepublik Deutschland* (C-582/14, 2016)](https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX:62014CJ0582) established that dynamic IP addresses can constitute personal data, but only where the controller has "the legal means which enable it to identify the data subject with additional data which the internet service provider has about that person." In this system the raw IP is never stored and the salt is secret and rotated monthly, which is a strong argument for anonymity under Recital 26. The cautious position treats the hash as pseudonymous personal data; in that case, Art. 6(1)(f) legitimate interests is the appropriate lawful basis for coarse usage analytics — **no consent is required** — but Art. 13 transparency obligations apply, meaning visitors must be able to find information about the processing (e.g., via a linked privacy statement or footer notice). + +**[CNIL guidance](https://www.cnil.fr/fr/cookies-solutions-pour-les-outils-de-mesure-daudience) (France)** — the CNIL's framework for consent-exempt analytics covers tools that use short-lived identifiers with immediate IP anonymisation, provided the data is used solely for audience measurement, does not enable cross-site tracking, and is retained for no more than 25 months. This system satisfies those conditions. The CNIL still expects users to be informed of the tracking (e.g., via the site's privacy policy), even for consent-exempt tools. + +**Practical position:** No consent banner is required. As a matter of good practice — and to satisfy GDPR Art. 13 under the cautious reading that the hash is personal data — sites using this snippet should make a brief privacy notice accessible to visitors. The recommended notice text is: *"This page collects anonymous visit counts for usage reporting (no cookies, no IP addresses stored)."* The queueboard dashboard injects this notice automatically alongside the snippet; other sites should add equivalent wording to their footer or privacy statement. + +### Migrations +- `0001_initial` — `AnalyticsPageView` +- `0002_analyticsdailymetric` — `AnalyticsDailyMetric` +- `0003_analyticsmonthlymetric` — `AnalyticsMonthlyMetric` +- `0004_siteanalyticssalt` — `SiteAnalyticsSalt` + +## Consequences +- Adds ~48 DB tables to backup scope (three new, classified in policy). +- Ingestion is synchronous (one DB write per request); at high volume a write buffer or async insert could be considered. +- Unique-visitor counts are approximate: same visitor across different browsers or after a UA update will be counted separately; this is acceptable for coarse funder reporting. +- Monthly hash rotation means a visitor who spans a month boundary is counted as two unique visitors. This is by design to prevent cross-month correlation. + +## Deferred (v1.1) +- Per-site auth tokens. +- Per-path monthly aggregates (`top_paths_json` field on `AnalyticsMonthlyMetric`). +- Top-referrer aggregates (`top_referrers_json`). +- Partitioning `AnalyticsPageView` by month at higher volumes. + +### Deferred: rate limiting on the collection endpoint + +**Decision.** Ship v1 without a throttle. First rollout targets sites used mainly +internally, where the incentive to inflate counts and the cost exposure from an +anonymous write endpoint are both close to nil. + +**Trigger to revisit — going public / funder-facing, not a date.** That is when both +risks actually appear. Two properties make the deferral safe until then: + +- *It is additive.* Throttling is entirely server-side: same endpoint, same payload, + same tracking snippet. Since the snippet is baked into generated Pages HTML by a + workflow in the `queueboard` repo, anything requiring a client change (tokens, + batching, backoff) would be the expensive kind of retrofit. This is not that. +- *Adding a cache later has no blast radius.* Nothing in the project currently uses + `django.core.cache` (the in-process dicts in `core`/`zulip_bot` are plain + dictionaries), and sessions are DB-backed with no `SESSION_ENGINE` override, so + introducing a `default` cache will not silently change existing behavior. + +**Shape when we do it.** DRF `SimpleRateThrottle` subclass scoped to this view, plus a +Redis-backed `CACHES` entry on a *different* DB index from the Celery broker. Two +things must not be missed: + +- Throttling is backed entirely by `django.core.cache`. On the current implicit + per-process `LocMemCache` the effective limit becomes N × rate across N gunicorn + workers and resets on restart — a throttle without a shared cache is decorative. +- Override `get_cache_key` to use `get_client_ip` rather than DRF's default + `get_ident`, which (with `NUM_PROXIES` unset) keys on the whole `X-Forwarded-For` + chain. That is client-controlled, so a caller could mint a fresh throttle bucket per + request — the same defect this doc records for visitor hashing above. Setting + `NUM_PROXIES = 1` also works, but reusing `get_client_ip` keeps one definition of + caller identity. + +**Scope limits, so this is not oversold.** A per-IP throttle caps a single-source +flood. It does not stop distributed inflation, nor a patient drip staying under the +limit; the endpoint is unauthenticated and CORS-open by design. Set the limit +generously — shared NAT (campus, corporate egress, mobile CGNAT) presents as one IP. + +**What does not rewind.** Raw pageviews prune at `SITE_ANALYTICS_RETENTION_DAYS`, but +daily/monthly aggregate rows persist indefinitely. Repairing polluted counts means +recomputing aggregates from raw rows, which only works while those rows are still +inside the retention window. This is the concrete reason to have a throttle in place +*before* the numbers are reported to anyone, rather than after. ## References -- `docs/design-decisions/README.md` -- `.github/workflows/upload_backup.yaml` +- `qb_site/site_analytics/` — app source +- `qb_site/api/views/analytics_collect.py` — ingestion view +- `qb_site/qb_site/settings/base.py` — beat schedule and settings +- `scripts/backup_policy.py` — table classification - `docs/design-decisions/016-sanitized-backups.md` -- `docs/design-decisions/030-sync-task-dedupe-strategy.md` -- `docs/design-decisions/029-updatedat-discovery-watermark-and-catchup.md` -- `qb_site/qb_site/urls.py` -- `qb_site/api/urls.py` -- `qb_site/qb_site/settings/base.py` -- `qb_site/analyzer/tasks/collect_convergence.py` diff --git a/docs/queueboard_main_workflow.md b/docs/queueboard_main_workflow.md index 45ee622d..14d3bc68 100644 --- a/docs/queueboard_main_workflow.md +++ b/docs/queueboard_main_workflow.md @@ -1,6 +1,32 @@ -Here is the main workflow in the `queueboard` repo, which queries data and generates the dashboard using the code in this repo (`queueboard-core`). For the planned v2 ingestion that replaces these ad‑hoc scripts with a database‑backed syncer, see docs/syncer_ingestion_plan.md. -Note that in the `queueboard` repo, the JSON files in `data/` and `processed-data/` are persisted from run to run by git pushes in this workflow. -This is also true of a few auxiliary text files: `closed_prs_to_backfill.txt`, `missing_prs.txt`, `redownload.txt`, `stubborn_prs.txt`. +This document describes the main GitHub Actions workflow used in the sibling +[`queueboard`](https://github.com/leanprover-community/queueboard) repo. +The workflow runs every 8 minutes, fetches fresh PR metadata from a deployed +instance of `qb_site/` (the Django backend in this repo), generates static +dashboard HTML, and publishes it to GitHub Pages. + +## How it works + +1. **Checkout** — checks out `queueboard-core` (this repo) to get scripts, + GraphQL query templates, and the `queueboard` Python package. +2. **Fetch + generate** — calls `python -m queueboard.dashboard --api` three + times, once per rule set (different queue-classification rules for + experimentation). Each run downloads JSON payloads from the backend API and + renders a set of HTML dashboard pages into `gh-pages//`. +3. **Deploy** — uploads the `gh-pages/` tree as a Pages artifact and deploys + it if the run is on the `master` branch and all three generation steps + succeeded. + +## Required repository secrets + +| Secret | Purpose | +|---|---| +| `QUEUEBOARD_API_BASE_URL` | Base URL of the deployed `qb_site` instance (e.g. `https://queueboard.example.com`). Used both to fetch API payloads and as the analytics endpoint host. | +| `QUEUEBOARD_ANALYTICS_SITE` | Site slug registered in `SITE_ANALYTICS_ALLOWED_SITES` on the server (e.g. `queueboard`). When set, a privacy-preserving analytics snippet is injected into every generated page. Omit to disable analytics. | + +If `QUEUEBOARD_ANALYTICS_SITE` is absent (secret not configured), the snippet +is silently omitted and all other workflow behaviour is unchanged. + +## Workflow YAML ```yaml name: Update PR metadata @@ -55,6 +81,7 @@ jobs: id: generate-dashboard-api-rs1 env: QUEUEBOARD_API_BASE_URL: ${{ secrets.QUEUEBOARD_API_BASE_URL }} + QUEUEBOARD_ANALYTICS_SITE: ${{ secrets.QUEUEBOARD_ANALYTICS_SITE }} run: | uv run python -m queueboard.dashboard \ --api \ @@ -66,6 +93,7 @@ jobs: id: generate-dashboard-api-rs2 env: QUEUEBOARD_API_BASE_URL: ${{ secrets.QUEUEBOARD_API_BASE_URL }} + QUEUEBOARD_ANALYTICS_SITE: ${{ secrets.QUEUEBOARD_ANALYTICS_SITE }} run: | uv run python -m queueboard.dashboard \ --api \ @@ -77,6 +105,7 @@ jobs: id: generate-dashboard-api-rs3 env: QUEUEBOARD_API_BASE_URL: ${{ secrets.QUEUEBOARD_API_BASE_URL }} + QUEUEBOARD_ANALYTICS_SITE: ${{ secrets.QUEUEBOARD_ANALYTICS_SITE }} run: | uv run python -m queueboard.dashboard \ --api \ diff --git a/qb_site/AGENTS.md b/qb_site/AGENTS.md index 00065056..346b011e 100644 --- a/qb_site/AGENTS.md +++ b/qb_site/AGENTS.md @@ -9,12 +9,14 @@ - `api`: DRF views/serializers for queueboard surfaces. - `zulip_bot`: Zulip webhook/command integration and policies. - `console`: GitHub-OAuth reviewer console for accepting/declining assignment proposals (design doc 050). + - `site_analytics`: privacy-preserving pageview ingestion and aggregation for static/funder-facing sites. - Keep new modules inside the owning app (`models/`, `services/`, `tasks/`, `management/commands/`, `tests/`). - App-specific guidance: - `qb_site/api/AGENTS.md` for public API endpoints, common patterns, and authentication notes. - `qb_site/syncer/AGENTS.md` for ingestion, discovery/backfill, and sync admin workflows. - `qb_site/analyzer/AGENTS.md` for revision/queue/dependency sweeps and analytics models. - `qb_site/zulip_bot/AGENTS.md` for webhook/command/policy/registration behavior. + - `qb_site/site_analytics/AGENTS.md` for pageview ingestion, aggregation tasks, and privacy rules. ## Core Commands ```bash diff --git a/qb_site/api/urls.py b/qb_site/api/urls.py index a3d07351..bd6fd42c 100644 --- a/qb_site/api/urls.py +++ b/qb_site/api/urls.py @@ -2,6 +2,7 @@ from django.urls import path from api.views import index +from api.views.analytics_collect import AnalyticsCollectView from api.views.queueboard_dependency_graph import QueueboardDependencyGraphView from api.views.queueboard_snapshot import QueueboardSnapshotView from api.views.reviewer_assignment import AreaStatsView, ReviewerAssignmentsView @@ -9,6 +10,7 @@ urlpatterns: list = [ path("", index, name="index"), + path("v1/analytics/collect", AnalyticsCollectView.as_view(), name="analytics-collect"), path("v1/queueboard/snapshot", QueueboardSnapshotView.as_view(), name="queueboard-snapshot"), path( "v1/queueboard/dependency-graph", diff --git a/qb_site/api/views/analytics_collect.py b/qb_site/api/views/analytics_collect.py new file mode 100644 index 00000000..71aae099 --- /dev/null +++ b/qb_site/api/views/analytics_collect.py @@ -0,0 +1,105 @@ +"""POST /api/v1/analytics/collect — lightweight pageview ingestion endpoint.""" + +from __future__ import annotations + +import logging + +from django.conf import settings +from django.utils import timezone +from rest_framework import status +from rest_framework.request import Request +from rest_framework.response import Response +from rest_framework.views import APIView + +from site_analytics.models import AnalyticsPageView +from site_analytics.services.bot_filter import is_bot +from site_analytics.services.hashing import SaltUnavailable, compute_visitor_hash, get_client_ip + +logger = logging.getLogger(__name__) + +# Hard caps to guard against oversized payloads hitting DB column limits. +_PATH_MAX = 2000 +_REFERRER_MAX = 2000 +_UA_MAX = 1000 + +# CORS headers added to every response so browsers on third-party/static sites +# can call this endpoint without a server-side proxy. +_CORS_HEADERS = { + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "POST, OPTIONS", + "Access-Control-Allow-Headers": "Content-Type", + "Access-Control-Max-Age": "86400", +} + + +def _cors(response: Response) -> Response: + for key, value in _CORS_HEADERS.items(): + response[key] = value + return response + + +class AnalyticsCollectView(APIView): + """Ingest a single pageview event. + + Intentionally minimal: validate, hash, insert, return 204. + All heavier work (aggregation, reporting) happens in periodic tasks. + + CORS headers are always returned so browsers on third-party static sites + can call this endpoint directly. + """ + + authentication_classes: list = [] + permission_classes: list = [] + + def options(self, request: Request, *args: object, **kwargs: object) -> Response: + """Handle CORS preflight requests.""" + return _cors(Response(status=status.HTTP_204_NO_CONTENT)) + + def post(self, request: Request, *args: object, **kwargs: object) -> Response: + site = (request.data.get("site") or "").strip() + path = (request.data.get("path") or "").strip() + referrer = (request.data.get("referrer") or "").strip() + user_agent = request.META.get("HTTP_USER_AGENT", "").strip() + + if not site: + return _cors(Response({"detail": "site is required"}, status=status.HTTP_400_BAD_REQUEST)) + if not path: + return _cors(Response({"detail": "path is required"}, status=status.HTTP_400_BAD_REQUEST)) + + allowed_sites = settings.SITE_ANALYTICS_ALLOWED_SITES + if site not in allowed_sites: + return _cors(Response({"detail": "unknown site"}, status=status.HTTP_400_BAD_REQUEST)) + + # Reject empty UA when the stricter hardening flag is enabled. + if not user_agent and settings.SITE_ANALYTICS_REJECT_EMPTY_UA: + return _cors(Response(status=status.HTTP_204_NO_CONTENT)) + + # Silently drop bot traffic rather than returning an error, to avoid + # leaking information about detection heuristics. + if is_bot(user_agent): + return _cors(Response(status=status.HTTP_204_NO_CONTENT)) + + now = timezone.now() + try: + visitor_month_hash = compute_visitor_hash(get_client_ip(request), user_agent) + except SaltUnavailable: + # Fail closed: dropping the event is strictly better than persisting an + # unsalted (reversible) visitor hash. Logged at error level because this + # means analytics is silently collecting nothing until a salt exists. + logger.error( + "site_analytics: dropping pageview for site %r — no hash salt configured. " + "Set SITE_ANALYTICS_HASH_SALT or run the site_analytics.rotate_salt task.", + site, + ) + return _cors(Response(status=status.HTTP_204_NO_CONTENT)) + + AnalyticsPageView.objects.create( + site=site, + path=path[:_PATH_MAX], + referrer=referrer[:_REFERRER_MAX], + user_agent=user_agent[:_UA_MAX], + occurred_at=now, + visitor_month_hash=visitor_month_hash, + ) + + return _cors(Response(status=status.HTTP_204_NO_CONTENT)) diff --git a/qb_site/qb_site/settings/base.py b/qb_site/qb_site/settings/base.py index 3e24d48b..2f97701c 100644 --- a/qb_site/qb_site/settings/base.py +++ b/qb_site/qb_site/settings/base.py @@ -61,6 +61,7 @@ def env_optional_bounded_int(name: str, *, minimum: int, maximum: int) -> int | "api", "zulip_bot", "console", + "site_analytics", ] MIDDLEWARE = [ @@ -520,6 +521,24 @@ def env_optional_bounded_int(name: str, *, minimum: int, maximum: int) -> int | ARCHIVE_RESYNC_TICK_SECONDS = int(os.getenv("ARCHIVE_RESYNC_TICK_SECONDS", 600)) ARCHIVE_RESYNC_MIN_RATE_REMAINING = int(os.getenv("ARCHIVE_RESYNC_MIN_RATE_REMAINING", 2500)) +# Site analytics settings +# Fallback salt used until the first rotate_salt task runs and writes a DB salt. +# Required in production on first deploy; thereafter the DB salt takes precedence. +SITE_ANALYTICS_HASH_SALT = os.getenv("SITE_ANALYTICS_HASH_SALT", "") +SITE_ANALYTICS_ALLOWED_SITES: list[str] = [ + s.strip() for s in os.getenv("SITE_ANALYTICS_ALLOWED_SITES", "").split(",") if s.strip() +] +SITE_ANALYTICS_RETENTION_DAYS = int(os.getenv("SITE_ANALYTICS_RETENTION_DAYS", 540)) +# Number of reverse proxies in front of this app. X-Forwarded-For is client-controlled, +# so only this many entries from the right of the chain are trustworthy (Heroku's router +# appends one). Set to 0 when the app is exposed directly, to ignore the header entirely. +SITE_ANALYTICS_TRUSTED_PROXY_COUNT = int(os.getenv("SITE_ANALYTICS_TRUSTED_PROXY_COUNT", 1)) +SITE_ANALYTICS_DAILY_AGGREGATE_PERIOD_SECONDS = int(os.getenv("SITE_ANALYTICS_DAILY_AGGREGATE_PERIOD_SECONDS", 3600)) +SITE_ANALYTICS_MONTHLY_AGGREGATE_PERIOD_SECONDS = int(os.getenv("SITE_ANALYTICS_MONTHLY_AGGREGATE_PERIOD_SECONDS", 86400)) +SITE_ANALYTICS_PRUNE_PERIOD_SECONDS = int(os.getenv("SITE_ANALYTICS_PRUNE_PERIOD_SECONDS", 86400)) +# Reject requests with an empty User-Agent header (stricter bot hardening). +SITE_ANALYTICS_REJECT_EMPTY_UA = env_bool(os.getenv("SITE_ANALYTICS_REJECT_EMPTY_UA"), False) + # CI filter (opt-in allowlist mode) # Set mode to 'allowlist' to enable filtering by the following substrings; otherwise all contexts are ingested. SYNCER_CI_FILTER_MODE = os.getenv("SYNCER_CI_FILTER_MODE", "all").lower() @@ -755,3 +774,24 @@ def env_optional_bounded_int(name: str, *, minimum: int, maximum: int) -> int | "fanout": True, }, } +if SITE_ANALYTICS_DAILY_AGGREGATE_PERIOD_SECONDS > 0: + CELERY_BEAT_SCHEDULE["site_analytics_aggregate_daily"] = { + "task": "site_analytics.aggregate_daily_metrics", + "schedule": SITE_ANALYTICS_DAILY_AGGREGATE_PERIOD_SECONDS, + } +if SITE_ANALYTICS_MONTHLY_AGGREGATE_PERIOD_SECONDS > 0: + CELERY_BEAT_SCHEDULE["site_analytics_aggregate_monthly"] = { + "task": "site_analytics.aggregate_monthly_metrics", + "schedule": SITE_ANALYTICS_MONTHLY_AGGREGATE_PERIOD_SECONDS, + } +if SITE_ANALYTICS_PRUNE_PERIOD_SECONDS > 0: + CELERY_BEAT_SCHEDULE["site_analytics_prune_pageviews"] = { + "task": "site_analytics.prune_old_pageviews", + "schedule": SITE_ANALYTICS_PRUNE_PERIOD_SECONDS, + "kwargs": {"retention_days": SITE_ANALYTICS_RETENTION_DAYS}, + } +# Rotate the visitor-hash salt at midnight UTC on the 1st of each month. +CELERY_BEAT_SCHEDULE["site_analytics_rotate_salt"] = { + "task": "site_analytics.rotate_salt", + "schedule": crontab(minute=0, hour=0, day_of_month=1), +} diff --git a/qb_site/site_analytics/AGENTS.md b/qb_site/site_analytics/AGENTS.md new file mode 100644 index 00000000..a83c008e --- /dev/null +++ b/qb_site/site_analytics/AGENTS.md @@ -0,0 +1,54 @@ +# Site Analytics Guidelines + +## Scope +- `qb_site/site_analytics/` implements privacy-preserving pageview ingestion and aggregation for static/funder-facing sites. +- Raw events in `AnalyticsPageView`; aggregate reporting in `AnalyticsDailyMetric` and `AnalyticsMonthlyMetric` (added in A3/A4). +- Design record: `docs/design-decisions/031-analytics-ingestion-design.md`. + +## Module Layout +- `models/pageview.py` — `AnalyticsPageView` raw event rows (immutable after insert). +- `models/daily_metric.py` — `AnalyticsDailyMetric` (added in A3). +- `models/monthly_metric.py` — `AnalyticsMonthlyMetric` (added in A4). +- `models/salt.py` — `SiteAnalyticsSalt` single-row table holding the current month's hash salt. +- `services/` — hashing, bot filtering, aggregation logic. +- `checks.py` — Django system checks (registered in `apps.py:ready()`). +- `tasks/` — periodic Celery tasks for aggregation, pruning, and salt rotation. +- `tests/` — unit and integration tests. +- API ingestion view: `qb_site/api/views/analytics_collect.py` (added in A2). + +## Key Settings (all env-overridable) +- `SITE_ANALYTICS_HASH_SALT` — fallback salt used until the first `rotate_salt` task runs and writes a DB salt. Required on first deploy; thereafter the `SiteAnalyticsSalt` DB row takes precedence. +- `SITE_ANALYTICS_ALLOWED_SITES` — comma-separated site slugs; unknown slugs rejected with `400`. +- `SITE_ANALYTICS_RETENTION_DAYS` — raw pageview retention window (default 540 days / ~18 months). +- `SITE_ANALYTICS_TRUSTED_PROXY_COUNT` — reverse-proxy hops in front of the app (default 1, matching Heroku's router). Controls how many `X-Forwarded-For` entries from the right are trusted; 0 ignores the header entirely. +- `SITE_ANALYTICS_DAILY_AGGREGATE_PERIOD_SECONDS` — beat period for daily aggregation task (default 3600). +- `SITE_ANALYTICS_MONTHLY_AGGREGATE_PERIOD_SECONDS` — beat period for monthly aggregation task (default 86400). +- `SITE_ANALYTICS_PRUNE_PERIOD_SECONDS` — beat period for retention pruning task (default 86400). + +## Task Surface +Celery task names (as registered via `@shared_task(name=…)`): + +- `site_analytics.aggregate_daily_metrics` — idempotent upsert of daily pageview/unique-visitor counts (added in A3). +- `site_analytics.aggregate_monthly_metrics` — idempotent upsert of monthly metrics; recomputes current + previous month (added in A4). +- `site_analytics.prune_old_pageviews` — deletes raw rows older than `SITE_ANALYTICS_RETENTION_DAYS` (added in A4). +- `site_analytics.rotate_salt` — generates a new random visitor-hash salt and discards the previous one; runs at midnight UTC on the 1st of each month. + +## Privacy Invariants +- Raw IP addresses are never stored. +- **Ingestion fails closed without a salt.** `compute_visitor_hash` raises `SaltUnavailable` when neither a `SiteAnalyticsSalt` row nor `SITE_ANALYTICS_HASH_SALT` is set, and the collect view drops the event (204 + error log) rather than persist an unsalted hash — `sha256(ip | ua)` with no secret is brute-forceable over the IPv4 space, so it would be a recoverable identifier, not a pseudonymous one. Collecting nothing is the correct failure mode. +- A deploy-time system check (`site_analytics.E001`, in `checks.py`) fails `manage.py check`/`migrate` when `SITE_ANALYTICS_ALLOWED_SITES` is non-empty but no salt is set. It is gated on allowed-sites because analytics is opt-in, and reads settings only — never the DB, since `migrate` runs checks before `SiteAnalyticsSalt` exists. Gunicorn does not run system checks on boot, so the runtime guarantee is `SaltUnavailable`, not this check. +- `visitor_month_hash = sha256(ip | normalized_user_agent | salt)` where `salt` is the current month's randomly generated value from `SiteAnalyticsSalt`. +- The salt is replaced at month start and the old value deleted, so hashes from different months are unlinkable even with knowledge of the current salt (forward secrecy). +- IP is extracted from `X-Forwarded-For` taking `SITE_ANALYTICS_TRUSTED_PROXY_COUNT` entries from the **right** (proxies append; the leftmost entries are client-supplied and spoofable), falling back to `REMOTE_ADDR`. Set the count to 0 when the app is exposed directly. +- Changing hashing semantics requires an explicit migration/versioning note in the design doc. + +## Backup Policy +- `site_analytics_analyticspageview` → TRUNCATE (raw rows contain visitor hashes; excluded from public backup). +- `site_analytics_analyticsdailymetric`, `site_analytics_analyticsmonthlymetric` → RETAIN (aggregate-only, safe to share). +- Update `scripts/backup_policy.py` whenever adding or removing tables. + +## Testing +```bash +uv run python qb_site/manage.py test site_analytics +bash scripts/repo_check_compose.sh +``` diff --git a/qb_site/site_analytics/CLAUDE.md b/qb_site/site_analytics/CLAUDE.md new file mode 100644 index 00000000..43c994c2 --- /dev/null +++ b/qb_site/site_analytics/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/qb_site/site_analytics/__init__.py b/qb_site/site_analytics/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/qb_site/site_analytics/admin.py b/qb_site/site_analytics/admin.py new file mode 100644 index 00000000..dbfd7e3b --- /dev/null +++ b/qb_site/site_analytics/admin.py @@ -0,0 +1,72 @@ +"""Django admin registrations for site analytics models.""" + +from __future__ import annotations + +from django.contrib import admin + +from site_analytics.models import AnalyticsDailyMetric, AnalyticsMonthlyMetric, AnalyticsPageView + + +@admin.register(AnalyticsPageView) +class AnalyticsPageViewAdmin(admin.ModelAdmin): + list_display = ("site", "path", "occurred_at", "visitor_month_hash_short", "referrer_short") + list_filter = ("site",) + search_fields = ("site", "path", "referrer") + readonly_fields = ("site", "path", "referrer", "user_agent", "occurred_at", "visitor_month_hash") + ordering = ("-occurred_at",) + date_hierarchy = "occurred_at" + + # Raw rows are immutable; disable add/change/delete to enforce that. + def has_add_permission(self, request): + return False + + def has_change_permission(self, request, obj=None): + return False + + def has_delete_permission(self, request, obj=None): + return False + + @admin.display(description="hash (prefix)") + def visitor_month_hash_short(self, obj: AnalyticsPageView) -> str: + return obj.visitor_month_hash[:12] + "…" + + @admin.display(description="referrer") + def referrer_short(self, obj: AnalyticsPageView) -> str: + return (obj.referrer[:60] + "…") if len(obj.referrer) > 60 else obj.referrer + + +@admin.register(AnalyticsDailyMetric) +class AnalyticsDailyMetricAdmin(admin.ModelAdmin): + list_display = ("site", "date", "pageviews", "unique_visitors") + list_filter = ("site",) + search_fields = ("site",) + readonly_fields = ("site", "date", "pageviews", "unique_visitors") + ordering = ("-date", "site") + date_hierarchy = "date" + + def has_add_permission(self, request): + return False + + def has_change_permission(self, request, obj=None): + return False + + def has_delete_permission(self, request, obj=None): + return False + + +@admin.register(AnalyticsMonthlyMetric) +class AnalyticsMonthlyMetricAdmin(admin.ModelAdmin): + list_display = ("site", "month", "pageviews", "unique_visitors") + list_filter = ("site",) + search_fields = ("site",) + readonly_fields = ("site", "month", "pageviews", "unique_visitors") + ordering = ("-month", "site") + + def has_add_permission(self, request): + return False + + def has_change_permission(self, request, obj=None): + return False + + def has_delete_permission(self, request, obj=None): + return False diff --git a/qb_site/site_analytics/apps.py b/qb_site/site_analytics/apps.py new file mode 100644 index 00000000..48cfae21 --- /dev/null +++ b/qb_site/site_analytics/apps.py @@ -0,0 +1,13 @@ +from django.apps import AppConfig +from django.core.checks import register + + +class SiteAnalyticsConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "site_analytics" + verbose_name = "Site Analytics" + + def ready(self) -> None: + from site_analytics.checks import check_hash_salt_configured + + register(check_hash_salt_configured) diff --git a/qb_site/site_analytics/checks.py b/qb_site/site_analytics/checks.py new file mode 100644 index 00000000..15278c94 --- /dev/null +++ b/qb_site/site_analytics/checks.py @@ -0,0 +1,49 @@ +"""Django system checks for site_analytics configuration.""" + +from __future__ import annotations + +from typing import Any + +from django.conf import settings +from django.core.checks import Error + +SALT_MISSING_ID = "site_analytics.E001" + + +def check_hash_salt_configured(app_configs: Any, **kwargs: Any) -> list[Error]: + """Require a bootstrap hash salt whenever analytics ingestion is enabled. + + Gated on ``SITE_ANALYTICS_ALLOWED_SITES`` because analytics is opt-in: a + deployment that never set an allowed site (CI, local dev) accepts no events and + has nothing to hash, so demanding a salt there would be noise. + + Deliberately reads settings only, never the database. System checks run before + migrations have necessarily been applied — ``manage.py migrate`` itself runs them + — so touching ``SiteAnalyticsSalt`` here would make the first deploy unbootable: + the check would need the very table the pending migration creates. + + Raising this as an ``Error`` (not a ``Warning``) makes ``manage.py check`` and + ``migrate`` fail, so a misconfigured deploy stops in the release phase rather + than silently collecting nothing. Note that gunicorn does not run system checks + when loading the WSGI app; the runtime guarantee comes from ``SaltUnavailable`` + in the hashing service, which also covers the salt row vanishing after boot. + """ + if not settings.SITE_ANALYTICS_ALLOWED_SITES: + return [] + if settings.SITE_ANALYTICS_HASH_SALT.strip(): + return [] + return [ + Error( + "SITE_ANALYTICS_HASH_SALT is empty while site analytics ingestion is enabled " + f"(SITE_ANALYTICS_ALLOWED_SITES={settings.SITE_ANALYTICS_ALLOWED_SITES!r}).", + hint=( + "Without a salt, visitor_month_hash would be an unsalted sha256 of IP and " + "user-agent, which is brute-forceable and therefore not pseudonymous. Set " + 'SITE_ANALYTICS_HASH_SALT to a random secret (e.g. `python -c "import secrets; ' + 'print(secrets.token_hex(32))"`). It is only the bootstrap value: once the ' + "site_analytics.rotate_salt task runs, the SiteAnalyticsSalt row takes precedence. " + "Alternatively clear SITE_ANALYTICS_ALLOWED_SITES to disable ingestion." + ), + id=SALT_MISSING_ID, + ) + ] diff --git a/qb_site/site_analytics/migrations/0001_initial.py b/qb_site/site_analytics/migrations/0001_initial.py new file mode 100644 index 00000000..2884c7cd --- /dev/null +++ b/qb_site/site_analytics/migrations/0001_initial.py @@ -0,0 +1,30 @@ +# Generated by Django 5.2.6 on 2026-03-25 19:22 + +import django.utils.timezone +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='AnalyticsPageView', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('site', models.CharField(max_length=100)), + ('path', models.CharField(max_length=2000)), + ('referrer', models.CharField(blank=True, default='', max_length=2000)), + ('user_agent', models.CharField(blank=True, default='', max_length=1000)), + ('occurred_at', models.DateTimeField(default=django.utils.timezone.now)), + ('visitor_month_hash', models.CharField(max_length=64)), + ], + options={ + 'indexes': [models.Index(fields=['site', 'occurred_at'], name='sa_pv_site_occurred_idx'), models.Index(fields=['occurred_at'], name='sa_pv_occurred_idx')], + }, + ), + ] diff --git a/qb_site/site_analytics/migrations/0002_analyticsdailymetric.py b/qb_site/site_analytics/migrations/0002_analyticsdailymetric.py new file mode 100644 index 00000000..5af80ab8 --- /dev/null +++ b/qb_site/site_analytics/migrations/0002_analyticsdailymetric.py @@ -0,0 +1,27 @@ +# Generated by Django 5.2.6 on 2026-03-25 19:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('site_analytics', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='AnalyticsDailyMetric', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('site', models.CharField(max_length=100)), + ('date', models.DateField()), + ('pageviews', models.PositiveIntegerField(default=0)), + ('unique_visitors', models.PositiveIntegerField(default=0)), + ], + options={ + 'indexes': [models.Index(fields=['site', 'date'], name='sa_dailymetric_site_date_idx')], + 'constraints': [models.UniqueConstraint(fields=('site', 'date'), name='sa_dailymetric_site_date_unique')], + }, + ), + ] diff --git a/qb_site/site_analytics/migrations/0003_analyticsmonthlymetric.py b/qb_site/site_analytics/migrations/0003_analyticsmonthlymetric.py new file mode 100644 index 00000000..aaf12554 --- /dev/null +++ b/qb_site/site_analytics/migrations/0003_analyticsmonthlymetric.py @@ -0,0 +1,27 @@ +# Generated by Django 5.2.6 on 2026-03-25 20:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('site_analytics', '0002_analyticsdailymetric'), + ] + + operations = [ + migrations.CreateModel( + name='AnalyticsMonthlyMetric', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('site', models.CharField(max_length=100)), + ('month', models.DateField()), + ('pageviews', models.PositiveIntegerField(default=0)), + ('unique_visitors', models.PositiveIntegerField(default=0)), + ], + options={ + 'indexes': [models.Index(fields=['site', 'month'], name='sa_monthly_site_month_idx')], + 'constraints': [models.UniqueConstraint(fields=('site', 'month'), name='sa_monthly_site_month_uniq')], + }, + ), + ] diff --git a/qb_site/site_analytics/migrations/0004_siteanalyticssalt.py b/qb_site/site_analytics/migrations/0004_siteanalyticssalt.py new file mode 100644 index 00000000..db397b7a --- /dev/null +++ b/qb_site/site_analytics/migrations/0004_siteanalyticssalt.py @@ -0,0 +1,21 @@ +# Generated by Django 5.2.6 on 2026-03-27 03:42 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('site_analytics', '0003_analyticsmonthlymetric'), + ] + + operations = [ + migrations.CreateModel( + name='SiteAnalyticsSalt', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('salt', models.CharField(max_length=64)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ], + ), + ] diff --git a/qb_site/site_analytics/migrations/__init__.py b/qb_site/site_analytics/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/qb_site/site_analytics/models/__init__.py b/qb_site/site_analytics/models/__init__.py new file mode 100644 index 00000000..b2ee1c5b --- /dev/null +++ b/qb_site/site_analytics/models/__init__.py @@ -0,0 +1,6 @@ +"""Site analytics models: raw events and aggregate reporting tables.""" + +from .daily_metric import AnalyticsDailyMetric # noqa: F401 +from .monthly_metric import AnalyticsMonthlyMetric # noqa: F401 +from .pageview import AnalyticsPageView # noqa: F401 +from .salt import SiteAnalyticsSalt # noqa: F401 diff --git a/qb_site/site_analytics/models/daily_metric.py b/qb_site/site_analytics/models/daily_metric.py new file mode 100644 index 00000000..be0bd490 --- /dev/null +++ b/qb_site/site_analytics/models/daily_metric.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from django.db import models + + +class AnalyticsDailyMetric(models.Model): + """Aggregated daily pageview and unique-visitor counts per site. + + Rows are upserted by the ``site_analytics.aggregate_daily_metrics`` task; + never written by the ingestion endpoint. + Reporting queries must use this table, not raw ``AnalyticsPageView`` scans. + """ + + site = models.CharField(max_length=100) + date = models.DateField() # UTC calendar date + pageviews = models.PositiveIntegerField(default=0) + unique_visitors = models.PositiveIntegerField(default=0) + + class Meta: + constraints = [ + models.UniqueConstraint(fields=["site", "date"], name="sa_dailymetric_site_date_unique"), + ] + indexes = [ + models.Index(fields=["site", "date"], name="sa_dailymetric_site_date_idx"), + ] + + def __str__(self) -> str: + return f"{self.site} {self.date}: {self.pageviews} pv / {self.unique_visitors} uv" diff --git a/qb_site/site_analytics/models/monthly_metric.py b/qb_site/site_analytics/models/monthly_metric.py new file mode 100644 index 00000000..5d27e23a --- /dev/null +++ b/qb_site/site_analytics/models/monthly_metric.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from django.db import models + + +class AnalyticsMonthlyMetric(models.Model): + """Aggregated monthly pageview and unique-visitor counts per site. + + Rows are upserted by the ``site_analytics.aggregate_monthly_metrics`` task. + Reporting queries must use this table, not raw ``AnalyticsPageView`` scans. + + ``month`` is stored as the first day of the UTC month (e.g. 2026-03-01) + so it is a plain ``DateField`` with natural ordering and easy filtering. + """ + + site = models.CharField(max_length=100) + month = models.DateField() # UTC first-of-month, e.g. 2026-03-01 + pageviews = models.PositiveIntegerField(default=0) + unique_visitors = models.PositiveIntegerField(default=0) + + class Meta: + constraints = [ + models.UniqueConstraint(fields=["site", "month"], name="sa_monthly_site_month_uniq"), + ] + indexes = [ + models.Index(fields=["site", "month"], name="sa_monthly_site_month_idx"), + ] + + def __str__(self) -> str: + return f"{self.site} {self.month:%Y-%m}: {self.pageviews} pv / {self.unique_visitors} uv" diff --git a/qb_site/site_analytics/models/pageview.py b/qb_site/site_analytics/models/pageview.py new file mode 100644 index 00000000..70e05445 --- /dev/null +++ b/qb_site/site_analytics/models/pageview.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from django.db import models +from django.utils import timezone + + +class AnalyticsPageView(models.Model): + """Raw pageview event row. + + Rows are immutable after insert; never updated. + Raw IP is not stored; privacy-preserving monthly hash is used instead. + Retained for a bounded window (SITE_ANALYTICS_RETENTION_DAYS) then pruned. + """ + + site = models.CharField(max_length=100) + path = models.CharField(max_length=2000) + referrer = models.CharField(max_length=2000, blank=True, default="") + user_agent = models.CharField(max_length=1000, blank=True, default="") + occurred_at = models.DateTimeField(default=timezone.now) + # sha256(ip | normalized_user_agent | salt) — no raw IP stored. The month is not + # part of the payload: unlinkability across months comes from rotating the salt + # itself (see SiteAnalyticsSalt and the site_analytics.rotate_salt task). + visitor_month_hash = models.CharField(max_length=64) + + class Meta: + indexes = [ + models.Index(fields=["site", "occurred_at"], name="sa_pv_site_occurred_idx"), + models.Index(fields=["occurred_at"], name="sa_pv_occurred_idx"), + ] + + def __str__(self) -> str: + return f"{self.site}:{self.path} @ {self.occurred_at}" diff --git a/qb_site/site_analytics/models/salt.py b/qb_site/site_analytics/models/salt.py new file mode 100644 index 00000000..49561fd9 --- /dev/null +++ b/qb_site/site_analytics/models/salt.py @@ -0,0 +1,21 @@ +"""Monthly rotating salt for visitor hashing.""" + +from __future__ import annotations + +from django.db import models + + +class SiteAnalyticsSalt(models.Model): + """Holds the current month's salt used to compute visitor_month_hash. + + Only one row is live at a time. The ``rotate_salt`` Celery task creates a + new row at the start of each month and deletes the previous one. The old + salt is intentionally discarded so past hashes cannot be re-derived even if + the current salt is ever leaked. + """ + + salt = models.CharField(max_length=64) + created_at = models.DateTimeField(auto_now_add=True) + + class Meta: + app_label = "site_analytics" diff --git a/qb_site/site_analytics/services/__init__.py b/qb_site/site_analytics/services/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/qb_site/site_analytics/services/aggregation.py b/qb_site/site_analytics/services/aggregation.py new file mode 100644 index 00000000..24b18283 --- /dev/null +++ b/qb_site/site_analytics/services/aggregation.py @@ -0,0 +1,169 @@ +"""Aggregation and retention services for site analytics.""" + +from __future__ import annotations + +import datetime +from typing import Any + +from django.conf import settings +from django.db.models import Count +from django.utils import timezone + +from site_analytics.models import AnalyticsDailyMetric, AnalyticsMonthlyMetric, AnalyticsPageView + + +def aggregate_daily_metrics( + *, + date: datetime.date | None = None, + days_back: int = 2, +) -> dict[str, Any]: + """Idempotent upsert of AnalyticsDailyMetric for a rolling date window. + + By default recomputes today and yesterday (``days_back=2``) so that events + arriving near midnight or during a prior task run are not missed. The task + is safe to retry: each call overwrites the aggregate with a fresh count from + raw rows, so running it multiple times on the same window is harmless. + + Returns a summary dict suitable for Celery task result storage. + """ + if date is None: + date = timezone.now().date() + + target_dates = [date - datetime.timedelta(days=i) for i in range(days_back)] + upserted = 0 + skipped = 0 + + for d in target_dates: + # COUNT(*) and COUNT(DISTINCT visitor_month_hash) per site for this UTC date. + # Django's __date lookup respects USE_TZ and the configured TIME_ZONE (UTC), + # so the date boundary is always UTC midnight. + rows = ( + AnalyticsPageView.objects.filter(occurred_at__date=d) + .values("site") + .annotate( + pageviews=Count("id"), + unique_visitors=Count("visitor_month_hash", distinct=True), + ) + ) + + for row in rows: + AnalyticsDailyMetric.objects.update_or_create( + site=row["site"], + date=d, + defaults={ + "pageviews": row["pageviews"], + "unique_visitors": row["unique_visitors"], + }, + ) + upserted += 1 + + # If no raw rows exist for this date and site, we leave any existing + # aggregate row in place rather than zeroing it out. This avoids + # accidental data loss if the raw rows were pruned before the aggregate + # was read. + sites_with_existing = set(AnalyticsDailyMetric.objects.filter(date=d).values_list("site", flat=True)) + sites_in_raw = {r["site"] for r in rows} + skipped += len(sites_with_existing - sites_in_raw) + + return { + "dates_processed": [str(d) for d in target_dates], + "upserted": upserted, + "skipped_existing_no_raw": skipped, + } + + +def _month_start(date: datetime.date) -> datetime.date: + """Return the first day of the month containing ``date``.""" + return date.replace(day=1) + + +def aggregate_monthly_metrics( + *, + date: datetime.date | None = None, + months_back: int = 2, +) -> dict[str, Any]: + """Idempotent upsert of AnalyticsMonthlyMetric for a rolling month window. + + By default recomputes the current month and the previous month + (``months_back=2``) so that late-arriving events and month-boundary races + are always captured. Safe to retry. + + ``month`` values are stored as the first day of the UTC month so they sort + and filter naturally as ``DateField`` values. + """ + if date is None: + date = timezone.now().date() + + # Build the list of first-of-month dates to recompute. + target_months: list[datetime.date] = [] + current = _month_start(date) + for _ in range(months_back): + target_months.append(current) + # Step back one month: subtract enough days to land in the previous month + # then take the first of that month. + current = _month_start(current - datetime.timedelta(days=1)) + + upserted = 0 + skipped = 0 + + for month_start in target_months: + # Last day of the month: go to first of next month, subtract one day. + if month_start.month == 12: + month_end = datetime.date(month_start.year + 1, 1, 1) - datetime.timedelta(days=1) + else: + month_end = datetime.date(month_start.year, month_start.month + 1, 1) - datetime.timedelta(days=1) + + rows = ( + AnalyticsPageView.objects.filter(occurred_at__date__gte=month_start, occurred_at__date__lte=month_end) + .values("site") + .annotate( + pageviews=Count("id"), + unique_visitors=Count("visitor_month_hash", distinct=True), + ) + ) + + for row in rows: + AnalyticsMonthlyMetric.objects.update_or_create( + site=row["site"], + month=month_start, + defaults={ + "pageviews": row["pageviews"], + "unique_visitors": row["unique_visitors"], + }, + ) + upserted += 1 + + # Preserve existing rows where raw data has been pruned (same logic as daily). + sites_with_existing = set(AnalyticsMonthlyMetric.objects.filter(month=month_start).values_list("site", flat=True)) + sites_in_raw = {r["site"] for r in rows} + skipped += len(sites_with_existing - sites_in_raw) + + return { + "months_processed": [str(m) for m in target_months], + "upserted": upserted, + "skipped_existing_no_raw": skipped, + } + + +def prune_old_pageviews( + *, + retention_days: int | None = None, +) -> dict[str, Any]: + """Delete AnalyticsPageView rows older than the retention window. + + ``retention_days`` defaults to ``SITE_ANALYTICS_RETENTION_DAYS`` from + settings. Rows whose ``occurred_at`` is strictly before the cutoff are + deleted in a single query; Postgres will handle the index scan efficiently + given the index on ``occurred_at``. + """ + if retention_days is None: + retention_days = settings.SITE_ANALYTICS_RETENTION_DAYS + + cutoff = timezone.now() - datetime.timedelta(days=retention_days) + deleted, _ = AnalyticsPageView.objects.filter(occurred_at__lt=cutoff).delete() + + return { + "deleted": deleted, + "cutoff": cutoff.isoformat(), + "retention_days": retention_days, + } diff --git a/qb_site/site_analytics/services/bot_filter.py b/qb_site/site_analytics/services/bot_filter.py new file mode 100644 index 00000000..e6e01564 --- /dev/null +++ b/qb_site/site_analytics/services/bot_filter.py @@ -0,0 +1,38 @@ +"""Bot/crawler user-agent filtering for analytics ingestion.""" + +from __future__ import annotations + +# Case-insensitive substrings that identify known bots/crawlers/tools. +# Extend conservatively; false positives silently drop legitimate pageviews. +_BOT_UA_SUBSTRINGS: tuple[str, ...] = ( + "bot", + "crawler", + "spider", + "scraper", + "curl/", + "wget/", + "python-requests", + "python-urllib", + "go-http-client", + "java/", + "libwww", + "httpclient", + "okhttp", + "axios/", + "node-fetch", + "got/", + "undici", + "vercel", +) + + +def is_bot(user_agent: str) -> bool: + """Return True if the user-agent matches a known bot/crawler pattern. + + Empty user-agents are allowed (not treated as bots) in v1; that behaviour + can be tightened via a settings flag in a later chunk. + """ + if not user_agent: + return False + ua_lower = user_agent.lower() + return any(sub in ua_lower for sub in _BOT_UA_SUBSTRINGS) diff --git a/qb_site/site_analytics/services/hashing.py b/qb_site/site_analytics/services/hashing.py new file mode 100644 index 00000000..467380cb --- /dev/null +++ b/qb_site/site_analytics/services/hashing.py @@ -0,0 +1,102 @@ +"""Visitor hashing service for privacy-preserving pageview identity.""" + +from __future__ import annotations + +import hashlib +import time + +from django.conf import settings +from django.http import HttpRequest + +from site_analytics.models.salt import SiteAnalyticsSalt + +# Simple in-process cache so we don't hit the DB on every request. +# Each dyno/worker caches independently; a 60-second TTL means the new salt +# is picked up within a minute of rotation, which is acceptable. +_cached_salt: str = "" +_cache_expires: float = 0.0 + + +def _reset_salt_cache() -> None: + """Invalidate the in-process salt cache. Intended for use in tests only.""" + global _cached_salt, _cache_expires + _cached_salt = "" + _cache_expires = 0.0 + + +class SaltUnavailable(RuntimeError): + """Raised when no visitor-hash salt is configured. + + Callers must drop the event rather than hash without one: sha256(ip | ua) with + no secret is brute-forceable across the whole IPv4 space, so an unsalted hash + is a recoverable identifier rather than a pseudonymous one. Collecting nothing + is the correct failure mode for a privacy-preserving pipeline. + """ + + +def _get_current_salt() -> str: + """Return the active hash salt, or "" when none is configured. + + The result is cached for 60s *including* the empty one: a deployment with no + salt would otherwise re-query on every request, and 60s is short enough to pick + up the first ``rotate_salt`` write. + """ + global _cached_salt, _cache_expires + now = time.monotonic() + if now < _cache_expires: + return _cached_salt + try: + _cached_salt = SiteAnalyticsSalt.objects.latest("created_at").salt + except SiteAnalyticsSalt.DoesNotExist: + # Fall back to the static env-var salt until the first rotation task runs. + _cached_salt = settings.SITE_ANALYTICS_HASH_SALT + _cache_expires = now + 60.0 + return _cached_salt + + +def get_client_ip(request: HttpRequest) -> str: + """Return the client IP address, trusting only the proxy hops we run behind. + + X-Forwarded-For is client-controlled: a caller may send any value it likes, and + each proxy *appends* the address it received the connection from. Heroku's router + appends the connecting IP, so with one proxy hop the rightmost entry is the only + one we can trust; the leftmost is whatever the client chose to claim. Reading the + leftmost entry would let a visitor mint a fresh ``visitor_month_hash`` per request + and inflate unique-visitor counts at will. + + ``SITE_ANALYTICS_TRUSTED_PROXY_COUNT`` is the number of proxies in front of this + app; we take that many entries from the right. Set it to 0 when the app is exposed + directly (no proxy), in which case X-Forwarded-For is ignored entirely and only + REMOTE_ADDR is used. + """ + num_proxies = settings.SITE_ANALYTICS_TRUSTED_PROXY_COUNT + if num_proxies > 0: + xff = request.META.get("HTTP_X_FORWARDED_FOR", "").strip() + addrs = [part.strip() for part in xff.split(",") if part.strip()] + if addrs: + # Clamp to the chain length so a shorter-than-expected chain still yields + # the leftmost real entry rather than raising IndexError. + return addrs[-min(num_proxies, len(addrs))] + return request.META.get("REMOTE_ADDR", "") + + +def compute_visitor_hash(ip: str, user_agent: str) -> str: + """Return sha256(ip | normalized_ua | salt) as a hex digest. + + Fields are joined with ``|`` to prevent cross-field collisions. + Cross-month correlation is prevented by the monthly salt rotation: the salt + is replaced at the start of each month and the old value discarded, so + hashes from different months are unlinkable even with knowledge of the + current salt. + + Raises ``SaltUnavailable`` when no salt is configured; callers must drop the + event rather than store an unsalted hash. + """ + salt = _get_current_salt() + if not salt: + raise SaltUnavailable( + "no SiteAnalyticsSalt row exists and SITE_ANALYTICS_HASH_SALT is empty; refusing to compute an unsalted visitor hash" + ) + normalized_ua = user_agent.strip().lower() + payload = f"{ip}|{normalized_ua}|{salt}" + return hashlib.sha256(payload.encode()).hexdigest() diff --git a/qb_site/site_analytics/tasks/__init__.py b/qb_site/site_analytics/tasks/__init__.py new file mode 100644 index 00000000..3966cc92 --- /dev/null +++ b/qb_site/site_analytics/tasks/__init__.py @@ -0,0 +1,17 @@ +"""Celery tasks for site analytics aggregation and retention.""" + +from __future__ import annotations + +from site_analytics.tasks.aggregate_daily import aggregate_daily_metrics_task # noqa: F401 +from site_analytics.tasks.aggregate_monthly import ( # noqa: F401 + aggregate_monthly_metrics_task, + prune_old_pageviews_task, +) +from site_analytics.tasks.rotate_salt import rotate_salt_task # noqa: F401 + +__all__ = [ + "aggregate_daily_metrics_task", + "aggregate_monthly_metrics_task", + "prune_old_pageviews_task", + "rotate_salt_task", +] diff --git a/qb_site/site_analytics/tasks/aggregate_daily.py b/qb_site/site_analytics/tasks/aggregate_daily.py new file mode 100644 index 00000000..676f32a9 --- /dev/null +++ b/qb_site/site_analytics/tasks/aggregate_daily.py @@ -0,0 +1,20 @@ +"""Celery task: aggregate daily pageview metrics.""" + +from __future__ import annotations + +from typing import Any + +from celery import shared_task + +from site_analytics.services.aggregation import aggregate_daily_metrics + + +@shared_task(name="site_analytics.aggregate_daily_metrics") +def aggregate_daily_metrics_task(*, days_back: int = 2) -> dict[str, Any]: + """Idempotent upsert of daily pageview and unique-visitor counts. + + Recomputes the rolling ``days_back`` UTC calendar days so that events + arriving near midnight or during a prior task run are never missed. + Safe to retry: each run overwrites aggregates with a fresh count. + """ + return aggregate_daily_metrics(days_back=days_back) diff --git a/qb_site/site_analytics/tasks/aggregate_monthly.py b/qb_site/site_analytics/tasks/aggregate_monthly.py new file mode 100644 index 00000000..fed96ebb --- /dev/null +++ b/qb_site/site_analytics/tasks/aggregate_monthly.py @@ -0,0 +1,30 @@ +"""Celery tasks: monthly aggregate and raw pageview pruning.""" + +from __future__ import annotations + +from typing import Any + +from celery import shared_task + +from site_analytics.services.aggregation import aggregate_monthly_metrics, prune_old_pageviews + + +@shared_task(name="site_analytics.aggregate_monthly_metrics") +def aggregate_monthly_metrics_task(*, months_back: int = 2) -> dict[str, Any]: + """Idempotent upsert of monthly pageview and unique-visitor counts. + + Recomputes the current month and the previous ``months_back - 1`` months + so late-arriving events and month-boundary races are always captured. + Safe to retry. + """ + return aggregate_monthly_metrics(months_back=months_back) + + +@shared_task(name="site_analytics.prune_old_pageviews") +def prune_old_pageviews_task(*, retention_days: int | None = None) -> dict[str, Any]: + """Delete raw AnalyticsPageView rows older than the retention window. + + Uses ``SITE_ANALYTICS_RETENTION_DAYS`` from settings when ``retention_days`` + is not provided. Aggregate rows (daily/monthly) are never pruned by this task. + """ + return prune_old_pageviews(retention_days=retention_days) diff --git a/qb_site/site_analytics/tasks/rotate_salt.py b/qb_site/site_analytics/tasks/rotate_salt.py new file mode 100644 index 00000000..f5d06281 --- /dev/null +++ b/qb_site/site_analytics/tasks/rotate_salt.py @@ -0,0 +1,26 @@ +"""Celery task: rotate the monthly visitor-hash salt.""" + +from __future__ import annotations + +import secrets +from typing import Any + +from celery import shared_task +from django.db import transaction + +from site_analytics.models.salt import SiteAnalyticsSalt + + +@shared_task(name="site_analytics.rotate_salt") +def rotate_salt_task() -> dict[str, Any]: + """Generate a fresh random salt and discard the previous one. + + Runs at the start of each calendar month. The old salt is deleted so past + visitor hashes cannot be re-derived even if the new salt is ever leaked + (forward secrecy). + """ + new_salt = secrets.token_hex(32) + with transaction.atomic(): + obj = SiteAnalyticsSalt.objects.create(salt=new_salt) + deleted, _ = SiteAnalyticsSalt.objects.exclude(pk=obj.pk).delete() + return {"rotated": True, "old_deleted": deleted} diff --git a/qb_site/site_analytics/tests/__init__.py b/qb_site/site_analytics/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/qb_site/site_analytics/tests/test_aggregation.py b/qb_site/site_analytics/tests/test_aggregation.py new file mode 100644 index 00000000..38ccf90d --- /dev/null +++ b/qb_site/site_analytics/tests/test_aggregation.py @@ -0,0 +1,131 @@ +"""Tests for daily metric aggregation service and task.""" + +from __future__ import annotations + +import datetime +from unittest import mock + +from django.test import TestCase, override_settings + +from site_analytics.models import AnalyticsDailyMetric, AnalyticsPageView +from site_analytics.services.aggregation import aggregate_daily_metrics +from site_analytics.tasks.aggregate_daily import aggregate_daily_metrics_task + +_SALT = override_settings(SITE_ANALYTICS_HASH_SALT="test-salt") + +TODAY = datetime.date(2026, 3, 25) +YESTERDAY = TODAY - datetime.timedelta(days=1) + + +def _pv(site: str, path: str, date: datetime.date, visitor_hash: str = "abc") -> AnalyticsPageView: + return AnalyticsPageView.objects.create( + site=site, + path=path, + occurred_at=datetime.datetime(date.year, date.month, date.day, 12, 0, 0, tzinfo=datetime.timezone.utc), + visitor_month_hash=visitor_hash, + ) + + +@_SALT +class AggregateDailyMetricsServiceTests(TestCase): + def test_basic_count(self): + _pv("s1", "/a", TODAY, "h1") + _pv("s1", "/b", TODAY, "h2") + _pv("s1", "/c", TODAY, "h1") # same hash — same visitor + + result = aggregate_daily_metrics(date=TODAY, days_back=1) + + metric = AnalyticsDailyMetric.objects.get(site="s1", date=TODAY) + self.assertEqual(metric.pageviews, 3) + self.assertEqual(metric.unique_visitors, 2) + self.assertEqual(result["upserted"], 1) + self.assertEqual(result["dates_processed"], [str(TODAY)]) + + def test_idempotent_rerun(self): + _pv("s1", "/a", TODAY, "h1") + aggregate_daily_metrics(date=TODAY, days_back=1) + aggregate_daily_metrics(date=TODAY, days_back=1) + + self.assertEqual(AnalyticsDailyMetric.objects.filter(site="s1", date=TODAY).count(), 1) + self.assertEqual(AnalyticsDailyMetric.objects.get(site="s1", date=TODAY).pageviews, 1) + + def test_rerun_after_new_events_updates_counts(self): + _pv("s1", "/a", TODAY, "h1") + aggregate_daily_metrics(date=TODAY, days_back=1) + _pv("s1", "/b", TODAY, "h2") + aggregate_daily_metrics(date=TODAY, days_back=1) + + metric = AnalyticsDailyMetric.objects.get(site="s1", date=TODAY) + self.assertEqual(metric.pageviews, 2) + self.assertEqual(metric.unique_visitors, 2) + + def test_days_back_covers_yesterday(self): + _pv("s1", "/a", TODAY, "h1") + _pv("s1", "/b", YESTERDAY, "h2") + + aggregate_daily_metrics(date=TODAY, days_back=2) + + self.assertEqual(AnalyticsDailyMetric.objects.get(site="s1", date=TODAY).pageviews, 1) + self.assertEqual(AnalyticsDailyMetric.objects.get(site="s1", date=YESTERDAY).pageviews, 1) + + def test_multiple_sites_aggregated_separately(self): + _pv("site-a", "/", TODAY, "h1") + _pv("site-a", "/", TODAY, "h2") + _pv("site-b", "/", TODAY, "h3") + + aggregate_daily_metrics(date=TODAY, days_back=1) + + self.assertEqual(AnalyticsDailyMetric.objects.get(site="site-a", date=TODAY).pageviews, 2) + self.assertEqual(AnalyticsDailyMetric.objects.get(site="site-b", date=TODAY).pageviews, 1) + + def test_no_raw_rows_does_not_create_metric(self): + aggregate_daily_metrics(date=TODAY, days_back=1) + self.assertEqual(AnalyticsDailyMetric.objects.count(), 0) + + def test_no_raw_rows_preserves_existing_metric(self): + # Existing aggregate from a previous run; no raw rows remain (pruned). + AnalyticsDailyMetric.objects.create(site="s1", date=TODAY, pageviews=5, unique_visitors=3) + aggregate_daily_metrics(date=TODAY, days_back=1) + + metric = AnalyticsDailyMetric.objects.get(site="s1", date=TODAY) + self.assertEqual(metric.pageviews, 5) # preserved, not zeroed + + def test_unique_visitors_uses_distinct_hash(self): + for _ in range(10): + _pv("s1", "/", TODAY, "same-hash") + + aggregate_daily_metrics(date=TODAY, days_back=1) + + metric = AnalyticsDailyMetric.objects.get(site="s1", date=TODAY) + self.assertEqual(metric.pageviews, 10) + self.assertEqual(metric.unique_visitors, 1) + + def test_date_boundary_is_utc(self): + # Event at 23:59 UTC on YESTERDAY must count for YESTERDAY, not TODAY. + AnalyticsPageView.objects.create( + site="s1", + path="/", + occurred_at=datetime.datetime( + YESTERDAY.year, YESTERDAY.month, YESTERDAY.day, 23, 59, 0, tzinfo=datetime.timezone.utc + ), + visitor_month_hash="h1", + ) + + aggregate_daily_metrics(date=TODAY, days_back=2) + + self.assertEqual(AnalyticsDailyMetric.objects.get(site="s1", date=YESTERDAY).pageviews, 1) + self.assertFalse(AnalyticsDailyMetric.objects.filter(site="s1", date=TODAY).exists()) + + +@_SALT +class AggregateDailyMetricsTaskTests(TestCase): + @mock.patch("django.utils.timezone.now") + def test_task_returns_summary_dict(self, mock_now): + # Pin the clock so the task's default date=timezone.now().date() matches + # the pageview date, avoiding midnight-boundary flakiness. + mock_now.return_value = datetime.datetime(TODAY.year, TODAY.month, TODAY.day, 12, 0, 0, tzinfo=datetime.timezone.utc) + _pv("s1", "/", TODAY, "h1") + result = aggregate_daily_metrics_task(days_back=1) + self.assertIn("upserted", result) + self.assertIn("dates_processed", result) + self.assertEqual(result["upserted"], 1) diff --git a/qb_site/site_analytics/tests/test_checks.py b/qb_site/site_analytics/tests/test_checks.py new file mode 100644 index 00000000..0f97490d --- /dev/null +++ b/qb_site/site_analytics/tests/test_checks.py @@ -0,0 +1,42 @@ +"""Tests for the site_analytics deploy-time configuration checks.""" + +from __future__ import annotations + +from django.test import SimpleTestCase, override_settings + +from site_analytics.checks import SALT_MISSING_ID, check_hash_salt_configured + + +class HashSaltCheckTests(SimpleTestCase): + """SimpleTestCase forbids database access, so every test here also proves the + check touches no tables — it must stay runnable during `manage.py migrate`, + before SiteAnalyticsSalt exists.""" + + @override_settings(SITE_ANALYTICS_ALLOWED_SITES=["queueboard"], SITE_ANALYTICS_HASH_SALT="") + def test_error_when_ingestion_enabled_without_salt(self): + errors = check_hash_salt_configured(None) + self.assertEqual(len(errors), 1) + self.assertEqual(errors[0].id, SALT_MISSING_ID) + + @override_settings(SITE_ANALYTICS_ALLOWED_SITES=["queueboard"], SITE_ANALYTICS_HASH_SALT=" ") + def test_whitespace_only_salt_is_treated_as_missing(self): + self.assertEqual(len(check_hash_salt_configured(None)), 1) + + @override_settings(SITE_ANALYTICS_ALLOWED_SITES=["queueboard"], SITE_ANALYTICS_HASH_SALT="s3cret") + def test_no_error_when_salt_present(self): + self.assertEqual(check_hash_salt_configured(None), []) + + @override_settings(SITE_ANALYTICS_ALLOWED_SITES=[], SITE_ANALYTICS_HASH_SALT="") + def test_no_error_when_ingestion_disabled(self): + # Analytics is opt-in: a deployment with no allowed sites accepts no events, + # so it must not be forced to configure a salt (this is the CI/dev case). + self.assertEqual(check_hash_salt_configured(None), []) + + @override_settings(SITE_ANALYTICS_ALLOWED_SITES=["queueboard"], SITE_ANALYTICS_HASH_SALT="") + def test_check_does_not_touch_the_database(self): + # Under SimpleTestCase any query raises DatabaseOperationForbidden, so reaching + # the assertion at all proves the check is settings-only. Guards against someone + # later "improving" it by consulting SiteAnalyticsSalt, which would make the + # first deploy unbootable: migrate runs checks before creating that table. + errors = check_hash_salt_configured(None) + self.assertEqual(errors[0].id, SALT_MISSING_ID) diff --git a/qb_site/site_analytics/tests/test_collect_view.py b/qb_site/site_analytics/tests/test_collect_view.py new file mode 100644 index 00000000..17982d4b --- /dev/null +++ b/qb_site/site_analytics/tests/test_collect_view.py @@ -0,0 +1,214 @@ +"""Endpoint tests for POST /api/v1/analytics/collect.""" + +from __future__ import annotations + +from django.test import TestCase, override_settings +from rest_framework.test import APIClient + +from site_analytics.models import AnalyticsPageView +from site_analytics.services.hashing import _reset_salt_cache + +_ALLOWED = override_settings( + SITE_ANALYTICS_ALLOWED_SITES=["test-site"], + SITE_ANALYTICS_HASH_SALT="test-salt", +) + +URL = "/api/v1/analytics/collect" + + +@_ALLOWED +class AnalyticsCollectViewTests(TestCase): + def setUp(self) -> None: + self.client = APIClient() + _reset_salt_cache() + + def _post(self, data: dict, **kwargs) -> object: + return self.client.post(URL, data, format="json", **kwargs) + + # --- success path --- + + def test_valid_payload_returns_204(self): + resp = self._post({"site": "test-site", "path": "/about"}) + self.assertEqual(resp.status_code, 204) + + def test_valid_payload_creates_pageview_row(self): + self._post({"site": "test-site", "path": "/about", "referrer": "https://example.com"}) + pv = AnalyticsPageView.objects.get() + self.assertEqual(pv.site, "test-site") + self.assertEqual(pv.path, "/about") + self.assertEqual(pv.referrer, "https://example.com") + self.assertEqual(len(pv.visitor_month_hash), 64) + + def test_referrer_optional(self): + resp = self._post({"site": "test-site", "path": "/home"}) + self.assertEqual(resp.status_code, 204) + self.assertEqual(AnalyticsPageView.objects.get().referrer, "") + + def test_user_agent_captured_from_header(self): + self._post( + {"site": "test-site", "path": "/"}, + HTTP_USER_AGENT="Mozilla/5.0 (Test)", + ) + self.assertEqual(AnalyticsPageView.objects.get().user_agent, "Mozilla/5.0 (Test)") + + # --- validation errors --- + + def test_missing_site_returns_400(self): + resp = self._post({"path": "/about"}) + self.assertEqual(resp.status_code, 400) + self.assertIn("site", resp.json()["detail"]) + + def test_missing_path_returns_400(self): + resp = self._post({"site": "test-site"}) + self.assertEqual(resp.status_code, 400) + self.assertIn("path", resp.json()["detail"]) + + def test_unknown_site_returns_400(self): + resp = self._post({"site": "unknown-site", "path": "/about"}) + self.assertEqual(resp.status_code, 400) + self.assertIn("site", resp.json()["detail"]) + + def test_empty_site_returns_400(self): + resp = self._post({"site": "", "path": "/about"}) + self.assertEqual(resp.status_code, 400) + + # --- bot filtering --- + + def test_bot_ua_returns_204_but_no_row(self): + resp = self._post( + {"site": "test-site", "path": "/about"}, + HTTP_USER_AGENT="Googlebot/2.1", + ) + self.assertEqual(resp.status_code, 204) + self.assertEqual(AnalyticsPageView.objects.count(), 0) + + # --- privacy: no raw IP stored --- + + def test_ip_not_stored_in_row(self): + self._post( + {"site": "test-site", "path": "/"}, + REMOTE_ADDR="1.2.3.4", + ) + pv = AnalyticsPageView.objects.get() + row_values = [str(v) for v in [pv.site, pv.path, pv.referrer, pv.user_agent, pv.visitor_month_hash]] + self.assertFalse(any("1.2.3.4" in v for v in row_values)) + + # --- XFF extraction --- + + def test_xff_used_for_hash_differs_from_remote_addr(self): + """Two visitors behind the proxy (distinct appended XFF) → different hashes.""" + self._post( + {"site": "test-site", "path": "/"}, + REMOTE_ADDR="10.0.0.1", + HTTP_X_FORWARDED_FOR="1.1.1.1", + ) + self._post( + {"site": "test-site", "path": "/"}, + REMOTE_ADDR="10.0.0.1", + HTTP_X_FORWARDED_FOR="2.2.2.2", + ) + hashes = list(AnalyticsPageView.objects.values_list("visitor_month_hash", flat=True)) + self.assertEqual(len(hashes), 2) + self.assertNotEqual(hashes[0], hashes[1]) + + def test_client_cannot_inflate_unique_visitors_by_spoofing_xff(self): + """A client prepending its own XFF entries must still hash to one visitor. + + The proxy appends the real address, so only the rightmost entry is trusted; + otherwise a single visitor could mint a fresh hash on every request. + """ + for spoofed in ("1.1.1.1", "2.2.2.2", "3.3.3.3"): + self._post( + {"site": "test-site", "path": "/"}, + REMOTE_ADDR="10.0.0.1", + HTTP_X_FORWARDED_FOR=f"{spoofed}, 203.0.113.9", + HTTP_USER_AGENT="Mozilla/5.0", + ) + hashes = set(AnalyticsPageView.objects.values_list("visitor_month_hash", flat=True)) + self.assertEqual(AnalyticsPageView.objects.count(), 3) + self.assertEqual(len(hashes), 1, "spoofed X-Forwarded-For entries changed the visitor hash") + + # --- field truncation --- + + def test_oversized_path_is_truncated(self): + long_path = "/" + "a" * 3000 + resp = self._post({"site": "test-site", "path": long_path}) + self.assertEqual(resp.status_code, 204) + self.assertEqual(len(AnalyticsPageView.objects.get().path), 2000) + + # --- empty allowed list --- + + @override_settings(SITE_ANALYTICS_ALLOWED_SITES=[]) + def test_empty_allowed_sites_rejects_all(self): + resp = self._post({"site": "test-site", "path": "/about"}) + self.assertEqual(resp.status_code, 400) + + # --- empty UA hardening flag --- + + def test_empty_ua_allowed_by_default(self): + resp = self._post({"site": "test-site", "path": "/"}) # no HTTP_USER_AGENT + self.assertEqual(resp.status_code, 204) + self.assertEqual(AnalyticsPageView.objects.count(), 1) + + @override_settings(SITE_ANALYTICS_REJECT_EMPTY_UA=True) + def test_empty_ua_dropped_when_flag_enabled(self): + resp = self._post({"site": "test-site", "path": "/"}) + self.assertEqual(resp.status_code, 204) + self.assertEqual(AnalyticsPageView.objects.count(), 0) + + @override_settings(SITE_ANALYTICS_REJECT_EMPTY_UA=True) + def test_non_empty_ua_accepted_when_flag_enabled(self): + resp = self._post({"site": "test-site", "path": "/"}, HTTP_USER_AGENT="Mozilla/5.0") + self.assertEqual(resp.status_code, 204) + self.assertEqual(AnalyticsPageView.objects.count(), 1) + + # --- CORS --- + + def test_post_response_includes_cors_header(self): + resp = self._post({"site": "test-site", "path": "/"}, HTTP_USER_AGENT="Mozilla/5.0") + self.assertEqual(resp["Access-Control-Allow-Origin"], "*") + + def test_options_preflight_returns_204_with_cors_headers(self): + resp = self.client.options(URL) + self.assertEqual(resp.status_code, 204) + self.assertEqual(resp["Access-Control-Allow-Origin"], "*") + self.assertIn("POST", resp["Access-Control-Allow-Methods"]) + self.assertIn("Content-Type", resp["Access-Control-Allow-Headers"]) + + +@override_settings(SITE_ANALYTICS_ALLOWED_SITES=["test-site"], SITE_ANALYTICS_HASH_SALT="") +class AnalyticsCollectMissingSaltTests(TestCase): + """With no salt configured the endpoint must drop events, not store weak hashes.""" + + def setUp(self) -> None: + self.client = APIClient() + _reset_salt_cache() + + def tearDown(self) -> None: + _reset_salt_cache() + + def test_event_is_dropped_when_no_salt_configured(self): + with self.assertLogs("api.views.analytics_collect", level="ERROR"): + resp = self.client.post( + URL, + {"site": "test-site", "path": "/"}, + format="json", + HTTP_USER_AGENT="Mozilla/5.0", + ) + # 204 keeps the browser beacon quiet; the row must not exist. + self.assertEqual(resp.status_code, 204) + self.assertEqual(AnalyticsPageView.objects.count(), 0) + + def test_no_unsalted_hash_is_ever_persisted(self): + import hashlib + + with self.assertLogs("api.views.analytics_collect", level="ERROR"): + self.client.post( + URL, + {"site": "test-site", "path": "/"}, + format="json", + HTTP_USER_AGENT="Mozilla/5.0", + REMOTE_ADDR="203.0.113.7", + ) + unsalted = hashlib.sha256(b"203.0.113.7|mozilla/5.0|").hexdigest() + self.assertFalse(AnalyticsPageView.objects.filter(visitor_month_hash=unsalted).exists()) diff --git a/qb_site/site_analytics/tests/test_monthly_aggregation.py b/qb_site/site_analytics/tests/test_monthly_aggregation.py new file mode 100644 index 00000000..bc6a2e4f --- /dev/null +++ b/qb_site/site_analytics/tests/test_monthly_aggregation.py @@ -0,0 +1,146 @@ +"""Tests for monthly metric aggregation service and prune service/task.""" + +from __future__ import annotations + +import datetime +from unittest import mock + +from django.test import TestCase, override_settings + +from site_analytics.models import AnalyticsMonthlyMetric, AnalyticsPageView +from site_analytics.services.aggregation import aggregate_monthly_metrics, prune_old_pageviews +from site_analytics.tasks.aggregate_monthly import aggregate_monthly_metrics_task, prune_old_pageviews_task + +_SALT = override_settings(SITE_ANALYTICS_HASH_SALT="test-salt") + +# Fixed reference dates +MAR_2026 = datetime.date(2026, 3, 25) +MAR_START = datetime.date(2026, 3, 1) +FEB_START = datetime.date(2026, 2, 1) +JAN_START = datetime.date(2026, 1, 1) + + +def _pv( + site: str, + path: str, + date: datetime.date, + visitor_hash: str = "h1", +) -> AnalyticsPageView: + return AnalyticsPageView.objects.create( + site=site, + path=path, + occurred_at=datetime.datetime(date.year, date.month, date.day, 12, 0, 0, tzinfo=datetime.timezone.utc), + visitor_month_hash=visitor_hash, + ) + + +@_SALT +class AggregateMonthlyMetricsServiceTests(TestCase): + def test_basic_monthly_count(self): + _pv("s1", "/a", MAR_2026, "h1") + _pv("s1", "/b", MAR_2026, "h2") + _pv("s1", "/c", MAR_2026, "h1") # same visitor + + result = aggregate_monthly_metrics(date=MAR_2026, months_back=1) + + metric = AnalyticsMonthlyMetric.objects.get(site="s1", month=MAR_START) + self.assertEqual(metric.pageviews, 3) + self.assertEqual(metric.unique_visitors, 2) + self.assertEqual(result["upserted"], 1) + self.assertEqual(result["months_processed"], [str(MAR_START)]) + + def test_idempotent_rerun(self): + _pv("s1", "/a", MAR_2026, "h1") + aggregate_monthly_metrics(date=MAR_2026, months_back=1) + aggregate_monthly_metrics(date=MAR_2026, months_back=1) + + self.assertEqual(AnalyticsMonthlyMetric.objects.filter(site="s1", month=MAR_START).count(), 1) + self.assertEqual(AnalyticsMonthlyMetric.objects.get(site="s1", month=MAR_START).pageviews, 1) + + def test_months_back_covers_previous_month(self): + _pv("s1", "/a", MAR_2026, "h1") + _pv("s1", "/b", datetime.date(2026, 2, 15), "h2") + + aggregate_monthly_metrics(date=MAR_2026, months_back=2) + + self.assertEqual(AnalyticsMonthlyMetric.objects.get(site="s1", month=MAR_START).pageviews, 1) + self.assertEqual(AnalyticsMonthlyMetric.objects.get(site="s1", month=FEB_START).pageviews, 1) + + def test_month_stored_as_first_of_month(self): + _pv("s1", "/", datetime.date(2026, 3, 31), "h1") + aggregate_monthly_metrics(date=MAR_2026, months_back=1) + self.assertTrue(AnalyticsMonthlyMetric.objects.filter(month=MAR_START).exists()) + + def test_events_spanning_month_boundary_separated(self): + _pv("s1", "/", datetime.date(2026, 2, 28), "h1") + _pv("s1", "/", datetime.date(2026, 3, 1), "h2") + + aggregate_monthly_metrics(date=MAR_2026, months_back=2) + + self.assertEqual(AnalyticsMonthlyMetric.objects.get(site="s1", month=FEB_START).pageviews, 1) + self.assertEqual(AnalyticsMonthlyMetric.objects.get(site="s1", month=MAR_START).pageviews, 1) + + def test_no_raw_rows_preserves_existing_metric(self): + AnalyticsMonthlyMetric.objects.create(site="s1", month=MAR_START, pageviews=99, unique_visitors=50) + aggregate_monthly_metrics(date=MAR_2026, months_back=1) + + metric = AnalyticsMonthlyMetric.objects.get(site="s1", month=MAR_START) + self.assertEqual(metric.pageviews, 99) # preserved, not zeroed + + @mock.patch("django.utils.timezone.now") + def test_task_returns_summary_dict(self, mock_now): + # Pin the clock so the task's default date=timezone.now().date() matches + # the pageview date, avoiding midnight-boundary flakiness. + mock_now.return_value = datetime.datetime( + MAR_2026.year, MAR_2026.month, MAR_2026.day, 12, 0, 0, tzinfo=datetime.timezone.utc + ) + _pv("s1", "/", MAR_2026, "h1") + result = aggregate_monthly_metrics_task(months_back=1) + self.assertIn("upserted", result) + self.assertIn("months_processed", result) + self.assertEqual(result["upserted"], 1) + + +@_SALT +class PruneOldPageviewsTests(TestCase): + # Pin the clock so hardcoded dates remain "old" or "recent" as intended. + _NOW = datetime.datetime(2026, 3, 25, 12, 0, 0, tzinfo=datetime.timezone.utc) + + @mock.patch("django.utils.timezone.now") + def test_prunes_rows_older_than_retention(self, mock_now): + mock_now.return_value = self._NOW + old = _pv("s1", "/old", datetime.date(2023, 1, 1)) + recent = _pv("s1", "/new", datetime.date(2026, 3, 1)) + + result = prune_old_pageviews(retention_days=30) + + self.assertFalse(AnalyticsPageView.objects.filter(pk=old.pk).exists()) + self.assertTrue(AnalyticsPageView.objects.filter(pk=recent.pk).exists()) + self.assertEqual(result["deleted"], 1) + + @mock.patch("django.utils.timezone.now") + def test_no_rows_deleted_when_all_recent(self, mock_now): + mock_now.return_value = self._NOW + _pv("s1", "/", datetime.date(2026, 3, 24)) + result = prune_old_pageviews(retention_days=30) + self.assertEqual(result["deleted"], 0) + self.assertEqual(AnalyticsPageView.objects.count(), 1) + + def test_returns_cutoff_and_retention_days(self): + result = prune_old_pageviews(retention_days=90) + self.assertIn("cutoff", result) + self.assertEqual(result["retention_days"], 90) + + @mock.patch("django.utils.timezone.now") + @override_settings(SITE_ANALYTICS_RETENTION_DAYS=7) + def test_uses_settings_default_when_not_specified(self, mock_now): + mock_now.return_value = self._NOW + _pv("s1", "/", datetime.date(2026, 3, 1)) + result = prune_old_pageviews() + self.assertEqual(result["retention_days"], 7) + self.assertEqual(result["deleted"], 1) + + def test_prune_task_returns_summary(self): + result = prune_old_pageviews_task(retention_days=365) + self.assertIn("deleted", result) + self.assertIn("cutoff", result) diff --git a/qb_site/site_analytics/tests/test_salt.py b/qb_site/site_analytics/tests/test_salt.py new file mode 100644 index 00000000..b36ac238 --- /dev/null +++ b/qb_site/site_analytics/tests/test_salt.py @@ -0,0 +1,84 @@ +"""Tests for the monthly salt rotation task.""" + +from __future__ import annotations + +from django.test import TestCase, override_settings + +from site_analytics.models.salt import SiteAnalyticsSalt +from site_analytics.services.hashing import SaltUnavailable, _reset_salt_cache, compute_visitor_hash +from site_analytics.tasks.rotate_salt import rotate_salt_task + + +class RotateSaltTaskTests(TestCase): + def setUp(self): + _reset_salt_cache() + + def test_creates_salt_row_when_none_exists(self): + self.assertEqual(SiteAnalyticsSalt.objects.count(), 0) + rotate_salt_task() + self.assertEqual(SiteAnalyticsSalt.objects.count(), 1) + + def test_created_salt_is_nonempty(self): + rotate_salt_task() + self.assertTrue(SiteAnalyticsSalt.objects.get().salt) + + def test_replaces_existing_salt(self): + SiteAnalyticsSalt.objects.create(salt="old-salt") + rotate_salt_task() + # Only one row should remain after rotation. + self.assertEqual(SiteAnalyticsSalt.objects.count(), 1) + self.assertNotEqual(SiteAnalyticsSalt.objects.get().salt, "old-salt") + + def test_new_salt_differs_from_previous(self): + SiteAnalyticsSalt.objects.create(salt="old-salt") + rotate_salt_task() + self.assertNotEqual(SiteAnalyticsSalt.objects.get().salt, "old-salt") + + def test_returns_summary_dict(self): + result = rotate_salt_task() + self.assertTrue(result["rotated"]) + self.assertIn("old_deleted", result) + + def test_old_deleted_count_is_accurate(self): + SiteAnalyticsSalt.objects.create(salt="first") + SiteAnalyticsSalt.objects.create(salt="second") + result = rotate_salt_task() + # Both pre-existing rows should have been deleted. + self.assertEqual(result["old_deleted"], 2) + self.assertEqual(SiteAnalyticsSalt.objects.count(), 1) + + +@override_settings(SITE_ANALYTICS_HASH_SALT="") +class MissingSaltFailsClosedTests(TestCase): + """No salt anywhere must raise rather than produce an unsalted hash.""" + + def setUp(self): + _reset_salt_cache() + + def tearDown(self): + _reset_salt_cache() + + def test_hashing_raises_when_no_salt_available(self): + self.assertEqual(SiteAnalyticsSalt.objects.count(), 0) + with self.assertRaises(SaltUnavailable): + compute_visitor_hash("1.2.3.4", "Mozilla/5.0") + + def test_hashing_recovers_once_a_salt_row_exists(self): + with self.assertRaises(SaltUnavailable): + compute_visitor_hash("1.2.3.4", "Mozilla/5.0") + SiteAnalyticsSalt.objects.create(salt="fresh-salt") + _reset_salt_cache() + self.assertEqual(len(compute_visitor_hash("1.2.3.4", "Mozilla/5.0")), 64) + + def test_empty_salt_result_is_cached_rather_than_requeried(self): + # A misconfigured deployment must not issue a DB query per request. + with self.assertRaises(SaltUnavailable): + compute_visitor_hash("1.2.3.4", "Mozilla/5.0") + with self.assertNumQueries(0): + for _ in range(5): + with self.assertRaises(SaltUnavailable): + compute_visitor_hash("1.2.3.4", "Mozilla/5.0") + + @override_settings(SITE_ANALYTICS_HASH_SALT="env-fallback-salt") + def test_env_salt_used_as_bootstrap_when_no_row_exists(self): + self.assertEqual(len(compute_visitor_hash("1.2.3.4", "Mozilla/5.0")), 64) diff --git a/qb_site/site_analytics/tests/test_services.py b/qb_site/site_analytics/tests/test_services.py new file mode 100644 index 00000000..a1da280c --- /dev/null +++ b/qb_site/site_analytics/tests/test_services.py @@ -0,0 +1,159 @@ +"""Unit tests for site_analytics hashing and bot-filter services.""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +from django.test import TestCase, override_settings + +from site_analytics.models.salt import SiteAnalyticsSalt +from site_analytics.services.bot_filter import is_bot +from site_analytics.services.hashing import _reset_salt_cache, compute_visitor_hash, get_client_ip + + +class GetClientIpTests(TestCase): + def _make_request(self, remote_addr: str = "", xff: str = "") -> MagicMock: + req = MagicMock() + meta: dict[str, str] = {} + if remote_addr: + meta["REMOTE_ADDR"] = remote_addr + if xff: + meta["HTTP_X_FORWARDED_FOR"] = xff + req.META = meta + return req + + def test_remote_addr_used_when_no_xff(self): + req = self._make_request(remote_addr="1.2.3.4") + self.assertEqual(get_client_ip(req), "1.2.3.4") + + def test_rightmost_xff_entry_is_trusted(self): + # The proxy appends the address it saw, so the rightmost entry is the only + # trustworthy one; "5.6.7.8" here is a client-supplied claim. + req = self._make_request(remote_addr="10.0.0.1", xff="5.6.7.8, 203.0.113.9") + self.assertEqual(get_client_ip(req), "203.0.113.9") + + def test_spoofed_leftmost_entries_are_ignored(self): + # A client prepending junk must not be able to change the derived IP: both + # requests below must resolve to the same address the proxy appended. + req1 = self._make_request(remote_addr="10.0.0.1", xff="1.1.1.1, 203.0.113.9") + req2 = self._make_request(remote_addr="10.0.0.1", xff="2.2.2.2, 203.0.113.9") + self.assertEqual(get_client_ip(req1), get_client_ip(req2)) + self.assertEqual(get_client_ip(req1), "203.0.113.9") + + def test_xff_single_address(self): + req = self._make_request(remote_addr="10.0.0.1", xff="9.9.9.9") + self.assertEqual(get_client_ip(req), "9.9.9.9") + + def test_xff_strips_whitespace(self): + req = self._make_request(xff=" 203.0.113.5 , 10.0.0.2 ") + self.assertEqual(get_client_ip(req), "10.0.0.2") + + def test_empty_xff_falls_back_to_remote_addr(self): + req = self._make_request(remote_addr="1.2.3.4", xff="") + self.assertEqual(get_client_ip(req), "1.2.3.4") + + def test_missing_both_returns_empty_string(self): + req = self._make_request() + self.assertEqual(get_client_ip(req), "") + + @override_settings(SITE_ANALYTICS_TRUSTED_PROXY_COUNT=0) + def test_xff_ignored_entirely_when_no_trusted_proxies(self): + req = self._make_request(remote_addr="10.0.0.1", xff="5.6.7.8, 203.0.113.9") + self.assertEqual(get_client_ip(req), "10.0.0.1") + + @override_settings(SITE_ANALYTICS_TRUSTED_PROXY_COUNT=2) + def test_two_trusted_proxies_take_second_from_right(self): + req = self._make_request(remote_addr="10.0.0.1", xff="1.1.1.1, 203.0.113.9, 10.0.0.5") + self.assertEqual(get_client_ip(req), "203.0.113.9") + + @override_settings(SITE_ANALYTICS_TRUSTED_PROXY_COUNT=3) + def test_short_chain_clamps_to_leftmost_entry(self): + req = self._make_request(remote_addr="10.0.0.1", xff="203.0.113.9, 10.0.0.5") + self.assertEqual(get_client_ip(req), "203.0.113.9") + + +@override_settings(SITE_ANALYTICS_HASH_SALT="test-salt") +class ComputeVisitorHashTests(TestCase): + def setUp(self): + # Ensure each test starts with a cold cache so DB state is respected. + _reset_salt_cache() + + def test_returns_64_char_hex(self): + result = compute_visitor_hash("1.2.3.4", "Mozilla/5.0") + self.assertEqual(len(result), 64) + self.assertTrue(all(c in "0123456789abcdef" for c in result)) + + def test_deterministic(self): + a = compute_visitor_hash("1.2.3.4", "Mozilla/5.0") + b = compute_visitor_hash("1.2.3.4", "Mozilla/5.0") + self.assertEqual(a, b) + + def test_different_ips_produce_different_hashes(self): + a = compute_visitor_hash("1.2.3.4", "Mozilla/5.0") + b = compute_visitor_hash("1.2.3.5", "Mozilla/5.0") + self.assertNotEqual(a, b) + + def test_ua_normalized_case_insensitive(self): + a = compute_visitor_hash("1.2.3.4", "Mozilla/5.0") + b = compute_visitor_hash("1.2.3.4", "MOZILLA/5.0") + self.assertEqual(a, b) + + def test_falls_back_to_settings_salt_when_no_db_row(self): + # No SiteAnalyticsSalt row — should use SITE_ANALYTICS_HASH_SALT. + self.assertEqual(SiteAnalyticsSalt.objects.count(), 0) + a = compute_visitor_hash("1.2.3.4", "Mozilla/5.0") + self.assertEqual(len(a), 64) + + def test_db_salt_takes_precedence_over_settings_salt(self): + # Hash with settings salt only. + hash_settings = compute_visitor_hash("1.2.3.4", "Mozilla/5.0") + + # Insert a DB salt and reset cache so it's picked up. + SiteAnalyticsSalt.objects.create(salt="db-salt-value") + _reset_salt_cache() + + hash_db = compute_visitor_hash("1.2.3.4", "Mozilla/5.0") + self.assertNotEqual(hash_settings, hash_db) + + def test_different_db_salts_produce_different_hashes(self): + SiteAnalyticsSalt.objects.create(salt="salt-one") + _reset_salt_cache() + a = compute_visitor_hash("1.2.3.4", "Mozilla/5.0") + + SiteAnalyticsSalt.objects.all().delete() + SiteAnalyticsSalt.objects.create(salt="salt-two") + _reset_salt_cache() + b = compute_visitor_hash("1.2.3.4", "Mozilla/5.0") + + self.assertNotEqual(a, b) + + +class IsBotTests(TestCase): + def test_known_bot_substrings(self): + bot_uas = [ + "Googlebot/2.1", + "Mozilla/5.0 (compatible; bingbot/2.0)", + "curl/7.68.0", + "wget/1.20", + "python-requests/2.28.0", + "Go-http-client/1.1", + "Java/11.0", + "okhttp/4.9.0", + "axios/1.3.0", + ] + for ua in bot_uas: + with self.subTest(ua=ua): + self.assertTrue(is_bot(ua), f"Expected {ua!r} to be detected as bot") + + def test_legitimate_browser_uas(self): + browser_uas = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15", + "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0", + ] + for ua in browser_uas: + with self.subTest(ua=ua): + self.assertFalse(is_bot(ua), f"Expected {ua!r} not to be detected as bot") + + def test_empty_ua_is_not_bot(self): + self.assertFalse(is_bot("")) diff --git a/scripts/backup_policy.py b/scripts/backup_policy.py index cd9ecf04..098354f1 100644 --- a/scripts/backup_policy.py +++ b/scripts/backup_policy.py @@ -5,6 +5,10 @@ # Tables expected to exist in backups and managed by this policy. # We include implicit Django auth M2M tables and django_migrations explicitly. BACKUP_TABLES: tuple[str, ...] = ( + "site_analytics_analyticspageview", + "site_analytics_analyticsdailymetric", + "site_analytics_analyticsmonthlymetric", + "site_analytics_siteanalyticssalt", "analyzer_analyzerconvergencesnapshot", "analyzer_areastatssnapshot", "analyzer_assignmentproposal", @@ -100,10 +104,16 @@ "syncer_githubwebhookdelivery", # Archive backfill importer worklist (design doc 043) — operational state. "syncer_archiveimportitem", + # Raw analytics pageviews — contain visitor hashes; exclude from public backup + "site_analytics_analyticspageview", + # Live salt is a secret; must never appear in sanitized backups + "site_analytics_siteanalyticssalt", ) # Tables retained in sanitized dump. RETAIN_TABLES: tuple[str, ...] = ( + "site_analytics_analyticsdailymetric", + "site_analytics_analyticsmonthlymetric", "core_repository", "core_user", "syncer_pullrequest", diff --git a/scripts/repo_check_compose.sh b/scripts/repo_check_compose.sh index 9dd8cdb0..a36edab3 100755 --- a/scripts/repo_check_compose.sh +++ b/scripts/repo_check_compose.sh @@ -51,16 +51,16 @@ PY fi if [ "${SKIP_COMPOSE_BUILD:-0}" != "1" ]; then - echo "[0/11] Building compose images (web/migrate/worker/beat) to pick up dependency changes" + echo "[0/12] Building compose images (web/migrate/worker/beat) to pick up dependency changes" docker compose build web migrate worker beat else - echo "[0/11] Skipping compose build (SKIP_COMPOSE_BUILD=1)" + echo "[0/12] Skipping compose build (SKIP_COMPOSE_BUILD=1)" fi -echo "[1/11] Reset compose services/networks to avoid stale startup state" +echo "[1/12] Reset compose services/networks to avoid stale startup state" docker compose down --remove-orphans >/dev/null 2>&1 || true -echo "[2/11] Validate GitHub GraphQL queries (host)" +echo "[2/12] Validate GitHub GraphQL queries (host)" if [ "${SKIP_GRAPHQL_VALIDATE:-0}" = "1" ]; then echo "Skipping GraphQL validation (SKIP_GRAPHQL_VALIDATE=1)" else @@ -71,7 +71,7 @@ else fi fi -echo "[3/11] Starting web (waits on db:healthy via depends_on)" +echo "[3/12] Starting web (waits on db:healthy via depends_on)" if ! docker compose up -d web; then echo "Compose failed to start services. Dumping service status and migrate logs..." >&2 docker compose ps || true @@ -79,33 +79,37 @@ if ! docker compose up -d web; then exit 1 fi -echo "[4/11] Validate backup policy coverage (compose)" +echo "[4/12] Validate backup policy coverage (compose)" docker compose exec -T web python scripts/validate_backup_policy.py -echo "[5/11] Django system checks (compose)" +echo "[5/12] Django system checks (compose)" docker compose exec -T web python qb_site/manage.py check -echo "[6/11] Dry-run makemigrations (compose)" +echo "[6/12] Dry-run makemigrations (compose)" docker compose exec -T web python qb_site/manage.py makemigrations --dry-run --check -echo "[7/11] Run core tests (compose)" +echo "[7/12] Run core tests (compose)" # Use higher verbosity to list skipped tests with reasons. docker compose exec -T web env DJANGO_SETTINGS_MODULE=qb_site.settings.ci python qb_site/manage.py test core # --verbosity 2 -echo "[8/11] Run syncer tests (compose)" +echo "[8/12] Run syncer tests (compose)" # Use higher verbosity to list skipped tests with reasons. docker compose exec -T web env DJANGO_SETTINGS_MODULE=qb_site.settings.ci python qb_site/manage.py test syncer # --verbosity 2 -echo "[9/11] Run analyzer tests (compose)" +echo "[9/12] Run analyzer tests (compose)" # Use higher verbosity to list skipped tests with reasons. docker compose exec -T web env DJANGO_SETTINGS_MODULE=qb_site.settings.ci python qb_site/manage.py test analyzer # --verbosity 2 -echo "[10/11] Run api tests (compose)" +echo "[10/12] Run api tests (compose)" # Use higher verbosity to list skipped tests with reasons. docker compose exec -T web env DJANGO_SETTINGS_MODULE=qb_site.settings.ci python qb_site/manage.py test api # --verbosity 2 -echo "[11/11] Run zulip_bot tests (compose)" +echo "[11/12] Run zulip_bot tests (compose)" # Use higher verbosity to list skipped tests with reasons. docker compose exec -T web env DJANGO_SETTINGS_MODULE=qb_site.settings.ci python qb_site/manage.py test zulip_bot # --verbosity 2 +echo "[12/12] Run site_analytics tests (compose)" +# Use higher verbosity to list skipped tests with reasons. +docker compose exec -T web env DJANGO_SETTINGS_MODULE=qb_site.settings.ci python qb_site/manage.py test site_analytics # --verbosity 2 + echo "Compose checks completed." diff --git a/src/queueboard/dashboard.py b/src/queueboard/dashboard.py index 104f911d..6ac0ac8e 100755 --- a/src/queueboard/dashboard.py +++ b/src/queueboard/dashboard.py @@ -439,17 +439,20 @@ def _inner( if extra_settings is None: extra_settings = ExtraColumnSettings.default() - return _inner(prs[kind], kind, aggregate_info, extra_settings, header) + return _inner(prs.get(kind, []), kind, aggregate_info, extra_settings, header) # Specific code for writing the actual webpage files. -HTML_HEADER = """ - + +def _make_html_header(analytics_host: str = "") -> str: + """Return the HTML header, optionally widening the CSP to allow analytics beacons.""" + connect_src = f" connect-src 'self' {analytics_host};" if analytics_host else "" + return f""" - + Mathlib review and triage dashboard " + cannot terminate the surrounding tag early. + """ + literal = json.dumps(value) + return literal.replace("<", "\\u003c").replace(">", "\\u003e").replace("&", "\\u0026") + + +def _make_analytics_snippet(host: str, site: str) -> str: + """Return the privacy notice paragraph and pageview tracking " + ) + return f"{notice}\n{script}" GH_PAGES_DIR = "gh-pages" API_DIR = "api" +ANALYTICS_HOST: str = "" # set by main() when --analytics-site is provided +ANALYTICS_SNIPPET: str = "" # pre-rendered snippet injected before # Write a webpage with body out a file called 'outfile'. @@ -478,8 +518,9 @@ def write_webpage(body: str, outfile: str, use_tables: bool = True, standard: bo if use_tables else "" ) - footer = f"{script}\n" - print(f"{HTML_HEADER}\n{body}\n{footer}", file=fi) + analytics = f"\n{ANALYTICS_SNIPPET}" if ANALYTICS_SNIPPET else "" + footer = f"{script}{analytics}\n" + print(f"{_make_html_header(ANALYTICS_HOST)}\n{body}\n{footer}", file=fi) def _parse_args(argv: List[str]) -> argparse.Namespace: @@ -529,6 +570,14 @@ def _parse_args(argv: List[str]) -> argparse.Namespace: action="store_true", help="Ask the server to refresh cached payloads before serving (API fetch only).", ) + parser.add_argument( + "--analytics-site", + default=os.environ.get("QUEUEBOARD_ANALYTICS_SITE"), + help=( + "Site slug for pageview analytics (must be in SITE_ANALYTICS_ALLOWED_SITES on the server). " + "Requires --api-base-url. When set, a tracking snippet is injected into every generated page." + ), + ) # Legacy positional arguments remain accepted for compatibility. parser.add_argument("legacy_gh_pages_dir", nargs="?", help="Legacy gh-pages directory positional argument.") parser.add_argument("legacy_api_dir", nargs="?", help="Legacy api directory positional argument.") @@ -1133,8 +1182,15 @@ def main() -> None: args = _parse_args(sys.argv[1:]) global GH_PAGES_DIR global API_DIR + global ANALYTICS_HOST + global ANALYTICS_SNIPPET GH_PAGES_DIR = args.gh_pages_dir or args.legacy_gh_pages_dir or GH_PAGES_DIR # "gh-pages" by default API_DIR = args.api_dir or args.legacy_api_dir or API_DIR # "api" by default + if args.analytics_site and args.api_base_url: + ANALYTICS_HOST = args.api_base_url.rstrip("/") + ANALYTICS_SNIPPET = _make_analytics_snippet(ANALYTICS_HOST, args.analytics_site) + elif args.analytics_site: + print("Warning: --analytics-site requires --api-base-url; analytics snippet will be omitted.", file=sys.stderr) if args.api: _fetch_api_payloads(args, API_DIR) @@ -1188,6 +1244,14 @@ def main() -> None: with as_file(files("queueboard").joinpath("static")) as tmp: shutil.copytree(tmp, GH_PAGES_DIR, dirs_exist_ok=True) + # Inject analytics snippet into the static HTML pages (which bypass write_webpage). + if ANALYTICS_SNIPPET: + for static_html in ("area_stats.html", "dependency_dashboard.html"): + dest = path.join(GH_PAGES_DIR, static_html) + if path.exists(dest): + content = open(dest).read() + open(dest, "w").write(content.replace("", f"{ANALYTICS_SNIPPET}\n", 1)) + if __name__ == "__main__": main() diff --git a/src/queueboard/static/style.css b/src/queueboard/static/style.css index 95568e2c..9516d5be 100644 --- a/src/queueboard/static/style.css +++ b/src/queueboard/static/style.css @@ -174,3 +174,9 @@ span.label { .qb-filter-option span { font-size: 0.95rem; } + +.analytics-notice { + margin-top: 2rem; + font-size: 0.8rem; + color: #888; +}