From 583cc5eb720ab47b61ff46e0fa1e94fb4289ce7a Mon Sep 17 00:00:00 2001 From: Makisuo Date: Thu, 6 Aug 2026 19:16:09 +0200 Subject: [PATCH] fix(billing): bill browser sessions per visit, not per tab MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The billed unit was one session-metadata row with `version == 1`. But `version` exists so `argMax(field, Version)` resolves the newest row of a ReplacingMergeTree — it was never designed to be a meter, and using it as one over-charged in four separate ways. **Per-tab, per-origin scope.** The session record lives in sessionStorage, scoped to one tab *and* one origin, so four open tabs were four charges and maple.dev -> app.maple.dev was two. SessionId stays per-tab (the merge invariant on `(OrgId, SessionId)` depends on it); a new `claimVisit()` claims one billable visit per visitor per 30-minute idle window against the cookie the visitor id already rides, so tabs and subdomains collapse into one charge. The gateway now meters `billable_start == 1 && version == 1`: the flag is sticky across a session's rows so the row surviving the merge still records the charge, and `version` keeps the charge singular. **Sampling didn't reduce billing.** `replay.sampleRate` gated only the rrweb chunk, so an org on 0.1 paid for 100% of its sessions — the one lever named "sample" moved only the part we don't charge for. One draw now feeds `captureSession` (metadata rows + distilled event sink) and `recordReplay`, a strict subset. `replayEnabled: false` deliberately still captures: turning off video is not turning off analytics. **Quota exhaustion billed once a minute.** `readRecord` preferred sessionStorage whenever `getItem` succeeded while `writeRecord` swallowed `setItem` failures, so on quota exhaustion `metaVersion` pinned at 1 and every 60s heartbeat re-billed. A `writesFailed` latch hands control to the in-memory record, and clears when writes recover. **The spend chart wasn't even showing the bill.** `dailySessionCountQuery` used `count()` with no FINAL against a ReplacingMergeTree, counting every unmerged heartbeat row — a 10-minute session rendered as ~12 and moved between refreshes. Now `uniq(SessionId)`. Adds `session_replays.BillableStart` (migration 0013) so the invoice is reproducible from the warehouse for the first time, and a `ingest_billed_browser_sessions_total` counter split by whether the visitor id persisted, to size the one tail this model can't fix: storage-blocked browsers can't hold a claim, so they still re-bill per page load. SCHEMA_VERSION bumps to 13 — BYO-ClickHouse orgs need 0013 applied before they are routed direct ingest again. --- apps/cli/src/server/schema/local-inserts.json | 2 +- apps/cli/src/server/schema/local-schema.sql | 5 +- apps/ingest/src/clickhouse_insert_mappings.rs | 10 +- apps/ingest/src/main.rs | 108 ++++++++++++-- apps/ingest/src/metrics.rs | 25 ++++ packages/browser-session/src/cookie.ts | 124 ++++++++++++++++ packages/browser-session/src/events-sink.ts | 11 +- packages/browser-session/src/meta-row.test.ts | 40 ++++++ packages/browser-session/src/meta-row.ts | 19 +++ .../src/session-lifecycle.test.ts | 60 ++++++++ .../browser-session/src/session-lifecycle.ts | 9 +- packages/browser-session/src/session.test.ts | 30 ++++ packages/browser-session/src/session.ts | 72 +++++++++- packages/browser-session/src/visit.test.ts | 132 ++++++++++++++++++ packages/browser-session/src/visit.ts | 99 +++++++++++++ packages/browser-session/src/visitor.ts | 122 ++-------------- packages/browser/src/config.ts | 15 +- packages/browser/src/init.test.ts | 58 ++++++++ packages/browser/src/init.ts | 42 +++++- .../migrations/0013_session_billable_start.ts | 31 ++++ .../src/clickhouse/migrations/index.test.ts | 20 ++- .../domain/src/clickhouse/migrations/index.ts | 2 + .../domain/src/generated/clickhouse-schema.ts | 4 +- .../generated/tinybird-project-manifest.ts | 4 +- packages/domain/src/tinybird/datasources.ts | 23 +++ .../effect-sdk/src/client/replay-loader.ts | 18 ++- .../src/product/billing-usage.test.ts | 12 +- .../src/product/billing-usage.ts | 36 +++-- packages/query-engine/src/ch/tables.ts | 5 + 29 files changed, 976 insertions(+), 162 deletions(-) create mode 100644 packages/browser-session/src/cookie.ts create mode 100644 packages/browser-session/src/visit.test.ts create mode 100644 packages/browser-session/src/visit.ts create mode 100644 packages/domain/src/clickhouse/migrations/0013_session_billable_start.ts diff --git a/apps/cli/src/server/schema/local-inserts.json b/apps/cli/src/server/schema/local-inserts.json index 1df89201d..cc75af918 100644 --- a/apps/cli/src/server/schema/local-inserts.json +++ b/apps/cli/src/server/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "12a7685236a4ebe4a8a40900e74c0239ab853afec09659d78a9b2f92811561fa", + "projectRevision": "5654f2527545c4b48718669fdf45e05c5a1b8de02c2f6d861a87c2c67bc19368", "orgPlaceholder": "__ORG__", "datasources": { "traces": { diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index d5764ec42..5abdf819e 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,6 +1,6 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: 12a7685236a4ebe4a8a40900e74c0239ab853afec09659d78a9b2f92811561fa +-- projectRevision: 5654f2527545c4b48718669fdf45e05c5a1b8de02c2f6d861a87c2c67bc19368 -- localSchemaVersion: 1 CREATE TABLE IF NOT EXISTS alert_checks ( @@ -642,7 +642,8 @@ CREATE TABLE IF NOT EXISTS session_replays ( EntryPath String DEFAULT '', ExitPath String DEFAULT '', Language LowCardinality(String) DEFAULT '', - LastActivityAt Nullable(DateTime64(9)) + LastActivityAt Nullable(DateTime64(9)), + BillableStart UInt8 DEFAULT 0 ) ENGINE = ReplacingMergeTree PARTITION BY toDate(StartTime) diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index 00020bb5b..da1195dff 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,12 +1,12 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "12a7685236a4ebe4a8a40900e74c0239ab853afec09659d78a9b2f92811561fa"; +pub const PROJECT_REVISION: &str = "5654f2527545c4b48718669fdf45e05c5a1b8de02c2f6d861a87c2c67bc19368"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse // clickHouseSchemaVersion. -pub const SCHEMA_VERSION: &str = "12"; +pub const SCHEMA_VERSION: &str = "13"; pub const ORG_PLACEHOLDER: &str = "__ORG__"; #[derive(Debug)] @@ -64,9 +64,9 @@ pub const DATASOURCES: &[InsertMapping] = &[ InsertMapping { datasource: "session_replays", table: "session_replays", - columns: &["OrgId", "SessionId", "StartTime", "EndTime", "DurationMs", "Status", "UserId", "UrlInitial", "UserAgent", "BrowserName", "OsName", "DeviceType", "Country", "ServiceName", "PageViews", "ClickCount", "ErrorCount", "TraceIds", "ResourceAttributes", "Version", "VisitorId", "VisitorIsNew", "UserEmail", "UserName", "GroupId", "GroupName", "UserTraits", "Referrer", "ReferrerHost", "UtmSource", "UtmMedium", "UtmCampaign", "UtmTerm", "UtmContent", "Host", "EntryPath", "ExitPath", "Language", "LastActivityAt"], - selects: &["__ORG__", "session_id", "start_time", "end_time", "duration_ms", "status", "user_id", "url_initial", "user_agent", "browser_name", "os_name", "device_type", "country", "service_name", "page_views", "click_count", "error_count", "trace_ids", "resource_attributes", "version", "visitor_id", "visitor_is_new", "user_email", "user_name", "group_id", "group_name", "user_traits", "referrer", "referrer_host", "utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content", "host", "entry_path", "exit_path", "language", "last_activity_at"], - input_schema: "session_id String, start_time DateTime64(9), end_time Nullable(DateTime64(9)), duration_ms Nullable(UInt32), status LowCardinality(String), user_id String, url_initial String, user_agent String, browser_name LowCardinality(String), os_name LowCardinality(String), device_type LowCardinality(String), country LowCardinality(String), service_name LowCardinality(String), page_views UInt32, click_count UInt32, error_count UInt32, trace_ids Array(String), resource_attributes Map(LowCardinality(String), String), version UInt32, visitor_id String, visitor_is_new UInt8, user_email String, user_name String, group_id String, group_name String, user_traits Map(String, String), referrer String, referrer_host LowCardinality(String), utm_source LowCardinality(String), utm_medium LowCardinality(String), utm_campaign LowCardinality(String), utm_term String, utm_content String, host LowCardinality(String), entry_path String, exit_path String, language LowCardinality(String), last_activity_at Nullable(DateTime64(9))", + columns: &["OrgId", "SessionId", "StartTime", "EndTime", "DurationMs", "Status", "UserId", "UrlInitial", "UserAgent", "BrowserName", "OsName", "DeviceType", "Country", "ServiceName", "PageViews", "ClickCount", "ErrorCount", "TraceIds", "ResourceAttributes", "Version", "VisitorId", "VisitorIsNew", "UserEmail", "UserName", "GroupId", "GroupName", "UserTraits", "Referrer", "ReferrerHost", "UtmSource", "UtmMedium", "UtmCampaign", "UtmTerm", "UtmContent", "Host", "EntryPath", "ExitPath", "Language", "LastActivityAt", "BillableStart"], + selects: &["__ORG__", "session_id", "start_time", "end_time", "duration_ms", "status", "user_id", "url_initial", "user_agent", "browser_name", "os_name", "device_type", "country", "service_name", "page_views", "click_count", "error_count", "trace_ids", "resource_attributes", "version", "visitor_id", "visitor_is_new", "user_email", "user_name", "group_id", "group_name", "user_traits", "referrer", "referrer_host", "utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content", "host", "entry_path", "exit_path", "language", "last_activity_at", "billable_start"], + input_schema: "session_id String, start_time DateTime64(9), end_time Nullable(DateTime64(9)), duration_ms Nullable(UInt32), status LowCardinality(String), user_id String, url_initial String, user_agent String, browser_name LowCardinality(String), os_name LowCardinality(String), device_type LowCardinality(String), country LowCardinality(String), service_name LowCardinality(String), page_views UInt32, click_count UInt32, error_count UInt32, trace_ids Array(String), resource_attributes Map(LowCardinality(String), String), version UInt32, visitor_id String, visitor_is_new UInt8, user_email String, user_name String, group_id String, group_name String, user_traits Map(String, String), referrer String, referrer_host LowCardinality(String), utm_source LowCardinality(String), utm_medium LowCardinality(String), utm_campaign LowCardinality(String), utm_term String, utm_content String, host LowCardinality(String), entry_path String, exit_path String, language LowCardinality(String), last_activity_at Nullable(DateTime64(9)), billable_start UInt8", }, InsertMapping { datasource: "session_replay_events", diff --git a/apps/ingest/src/main.rs b/apps/ingest/src/main.rs index ae38a4718..93f4d4f46 100644 --- a/apps/ingest/src/main.rs +++ b/apps/ingest/src/main.rs @@ -2323,6 +2323,38 @@ async fn handle_replay_meta( } } +/// Whether one session-metadata row starts a visit Autumn should be charged for. +/// +/// See the call site for why it takes both conditions. The `billable_start` +/// fallback is not defensive: customers pin SDK versions, so bundles that predate +/// the field keep posting for months, and treating their absence as "not +/// billable" would silently stop billing every one of those orgs. Absent means +/// "an older SDK that could not claim a visit", and the old `version == 1` rule +/// is the best answer available for it. Remove the fallback only once the +/// oldest SDK in the wild emits the field. +/// Whether the SDK could persist this visitor's id across the page load. +/// +/// The SDK emits `maple.visitor.persisted: "false"` only when both the cookie and +/// localStorage were blocked, so an absent key means persisted — including for +/// older bundles that never emitted it. +fn visitor_id_is_persisted(obj: &serde_json::Map) -> bool { + obj.get("resource_attributes") + .and_then(|v| v.as_object()) + .and_then(|attrs| attrs.get("maple.visitor.persisted")) + .and_then(|v| v.as_str()) + != Some("false") +} + +fn is_billable_visit_start(obj: &serde_json::Map) -> bool { + if obj.get("version").and_then(|v| v.as_u64()) != Some(1) { + return false; + } + match obj.get("billable_start") { + Some(value) => value.as_u64() == Some(1), + None => true, + } +} + async fn handle_replay_meta_inner( state: &AppState, headers: &HeaderMap, @@ -2371,16 +2403,27 @@ async fn handle_replay_meta_inner( // NDJSON: one session-metadata object per line. The org_id is always taken // from the authenticated key, never from the client-supplied body. // - // Count session-start rows so we can meter one browser session per session to - // Autumn. The browser SDK posts a start row (`version: 1` / `status: "active"`) - // at session start and an end row (`version: 2`) at unload; counting only starts - // avoids double-counting. Caveat: an in-tab reload recreates the SDK session sink - // and re-posts a start row for the same SessionId, so reloads can slightly - // over-count — consistent with the at-least-once metering used for the - // logs/traces/metrics signals. + // Count billable visit starts so we meter one browser session per *visit* to + // Autumn. Two conditions, and both are load-bearing: + // + // `billable_start == 1` — the SDK claimed this visit against a store shared + // across tabs and subdomains (`visit.ts`). It is sticky: every row of a + // billable session carries it, so the row that survives the + // ReplacingMergeTree merge still records that the session was charged. + // `version == 1` — the first row of that session record. Since the + // flag is sticky, this is what keeps the ~1/minute heartbeats and the + // unload row from re-billing a visit already paid for. + // + // Billing used to be `version == 1` alone. That charged once per tab and once + // per origin, because the session record lives in sessionStorage, which is + // scoped to both — one person with four tabs open paid four times. let country = derive_country(headers, state.config.trust_proxy_geo); let mut rows: Vec> = Vec::new(); let mut session_starts: u64 = 0; + // Of those, the ones from a visitor whose id does not survive the page load — + // the tail that re-claims and re-bills on every navigation. Counted, not + // corrected: there is nowhere on such a browser to keep the claim. + let mut unpersisted_starts: u64 = 0; for line in body.split(|&b| b == b'\n') { if line.iter().all(u8::is_ascii_whitespace) { continue; @@ -2422,8 +2465,11 @@ async fn handle_replay_meta_inner( // LowCardinality columns. Clamp before it reaches the warehouse — the // SDK's own trimming ships in customer JavaScript. sanitize_session_meta(obj); - if obj.get("version").and_then(|v| v.as_u64()) == Some(1) { + if is_billable_visit_start(obj) { session_starts += 1; + if !visitor_id_is_persisted(obj) { + unpersisted_starts += 1; + } } rows.push( serde_json::to_vec(&value).map_err(|e| { @@ -2456,6 +2502,8 @@ async fn handle_replay_meta_inner( if let Some(tracker) = &state.autumn_tracker { if org_id != SENTINEL_ORG_ID && session_starts > 0 { tracker.track(&org_id, "browser_sessions", session_starts as f64); + metrics::billed_browser_sessions(session_starts - unpersisted_starts, true); + metrics::billed_browser_sessions(unpersisted_starts, false); } } @@ -6433,6 +6481,50 @@ mod tests { )); } + fn meta_row(json: &str) -> serde_json::Map { + match serde_json::from_str(json).expect("valid JSON") { + serde_json::Value::Object(map) => map, + other => panic!("expected an object, got {other}"), + } + } + + #[test] + fn only_the_first_row_of_a_claimed_visit_is_billed() { + // The claiming session's first row: the one charge for this visit. + assert!(is_billable_visit_start(&meta_row( + r#"{"version":1,"billable_start":1,"status":"active"}"# + ))); + + // `billable_start` is sticky across the session's rows so the merged row + // still records the charge — which is exactly why `version` has to gate it. + // Heartbeats and the unload row must not re-bill a visit already paid for. + assert!(!is_billable_visit_start(&meta_row( + r#"{"version":2,"billable_start":1,"status":"active"}"# + ))); + assert!(!is_billable_visit_start(&meta_row( + r#"{"version":9,"billable_start":1,"status":"ended"}"# + ))); + + // A second tab or a second subdomain: a real session, its own record, its + // own version 1 — and not a second charge, because the visit was claimed. + assert!(!is_billable_visit_start(&meta_row( + r#"{"version":1,"billable_start":0,"status":"active"}"# + ))); + } + + #[test] + fn a_row_without_the_field_falls_back_to_the_old_version_rule() { + // Customers pin SDK versions, so bundles predating `billable_start` keep + // posting for months. Reading their silence as "not billable" would stop + // billing those orgs entirely. + assert!(is_billable_visit_start(&meta_row( + r#"{"version":1,"status":"active"}"# + ))); + assert!(!is_billable_visit_start(&meta_row( + r#"{"version":2,"status":"ended"}"# + ))); + } + #[test] fn a_browser_sessions_cap_blocks_replay_without_touching_other_signals() { // Replay is metered as `browser_sessions`, so the cap has to reach it — diff --git a/apps/ingest/src/metrics.rs b/apps/ingest/src/metrics.rs index 602c6a634..ceb1b3c75 100644 --- a/apps/ingest/src/metrics.rs +++ b/apps/ingest/src/metrics.rs @@ -204,6 +204,13 @@ static AUTUMN_FLUSHES_TOTAL: LazyLock> = LazyLock::new(|| { .build() }); +static BILLED_BROWSER_SESSIONS_TOTAL: LazyLock> = LazyLock::new(|| { + METER + .u64_counter("ingest_billed_browser_sessions_total") + .with_description("Browser sessions metered to Autumn, by how the SDK identified the visitor") + .build() +}); + // --- Up/down counter ------------------------------------------------------ static REQUESTS_IN_FLIGHT: LazyLock> = LazyLock::new(|| { @@ -664,6 +671,24 @@ pub fn metrics_summary_dropped() { METRICS_SUMMARY_DROPPED_TOTAL.add(1, &[]); } +/// Browser sessions charged to an org, split by whether the visitor id survives +/// the page load. +/// +/// `visitor_persisted=false` means both the cookie and localStorage were blocked +/// (Safari ITP, incognito, a cookie-blocking extension). Those visitors cannot +/// hold a visit claim, so every page load they make re-claims and is charged +/// again — the one over-count the per-visit model does not fix. This exists so +/// the size of that tail is a number before anyone tries to price around it. +pub fn billed_browser_sessions(count: u64, visitor_persisted: bool) { + if count == 0 { + return; + } + BILLED_BROWSER_SESSIONS_TOTAL.add( + count, + &[KeyValue::new("visitor_persisted", visitor_persisted)], + ); +} + /// An Autumn usage-tracking flush cycle completed (`status` is `ok` or `error`). pub fn autumn_flush(status: &'static str, duration_secs: f64) { AUTUMN_FLUSH_DURATION_SECONDS.record(duration_secs, &[]); diff --git a/packages/browser-session/src/cookie.ts b/packages/browser-session/src/cookie.ts new file mode 100644 index 000000000..c56abfdbf --- /dev/null +++ b/packages/browser-session/src/cookie.ts @@ -0,0 +1,124 @@ +/** + * First-party cookie plumbing, and the one decision every cookie this SDK writes + * has to agree on: which `Domain=` they are scoped to. + * + * Extracted from `visitor.ts` when a second consumer appeared. `visit.ts` needs + * the *same* domain the visitor id uses — a visit claim written host-only while + * the visitor id spans subdomains would silently stop deduplicating exactly + * where it matters most, on the marketing-site → app hop. Sharing the probe is + * what keeps the two from drifting. + */ + +/** Scope cookies to the registered domain so subdomains share them. */ +let crossSubdomainCookie = true +/** Explicit `Domain=` override; `""` forces a host-only cookie. */ +let cookieDomainOverride: string | undefined +/** Memoized probe result. `undefined` = not resolved yet. */ +let probedCookieDomain: string | undefined + +/** + * Apply the host app's cookie configuration. Called from `configurePrivacy`, so + * both SDKs get it from the single call they already make. + * + * Like the consent gates, this only ever *tightens*: an app that initializes two + * SDKs, only one of which passes a `privacy` block, must not have the other's + * absent option widen the cookie back out to every subdomain. + * + * "Tighter" for `cookieDomain` means *narrower scope*, which is why this is not + * first-write-wins: `""` (host-only) is the tightest value there is, and a + * second SDK asking for it has to win over an earlier `"example.com"`. Between + * two non-empty domains the shorter one is the broader — `example.com` covers + * `app.example.com` and not the reverse — so the longer string wins. + */ +export function configureCookieScope(options: { + readonly crossSubdomainCookie?: boolean | undefined + readonly cookieDomain?: string | undefined +}): void { + if (options.crossSubdomainCookie === false) crossSubdomainCookie = false + if (options.cookieDomain !== undefined) { + cookieDomainOverride = tighterCookieDomain(cookieDomainOverride, options.cookieDomain) + } + probedCookieDomain = undefined +} + +/** The narrower of two `Domain=` values, treating `undefined` as "unset". */ +function tighterCookieDomain(current: string | undefined, next: string): string { + if (current === undefined) return next + // Host-only beats any domain-scoped cookie, whichever side asked for it. + if (current === "" || next === "") return "" + return next.length > current.length ? next : current +} + +export function readRawCookie(name: string): string | undefined { + if (typeof document === "undefined") return undefined + try { + for (const part of document.cookie.split(";")) { + const raw = part.trim() + if (!raw.startsWith(`${name}=`)) continue + return decodeURIComponent(raw.slice(name.length + 1)) + } + } catch { + // Cookies disabled entirely — indistinguishable from "not set". + } + return undefined +} + +export function setRawCookie(name: string, value: string, domain: string, maxAgeSeconds: number): boolean { + if (typeof document === "undefined") return false + const attributes = [ + `${name}=${encodeURIComponent(value)}`, + "path=/", + `max-age=${Math.max(0, Math.floor(maxAgeSeconds))}`, + "SameSite=Lax", + ] + if (domain) attributes.push(`domain=.${domain}`) + // A `Secure` cookie is rejected over http, which is exactly the local-dev case. + if (typeof location !== "undefined" && location.protocol === "https:") attributes.push("Secure") + try { + document.cookie = attributes.join("; ") + return true + } catch { + return false + } +} + +/** + * The broadest domain this browser will actually accept a cookie for, found by + * probing rather than by carrying a public-suffix list — the same trick + * posthog-js uses. Candidates start at the broadest (the last two labels) and + * narrow a label at a time, with the first that sticks winning — so + * `app.example.co.uk` tries the rejected `co.uk`, then lands on `example.co.uk`. + * + * Returns `""` (host-only cookie) for single-label hosts like `localhost` and + * for bare IPs, neither of which can carry a `Domain=` attribute. + */ +function probeCookieDomain(): string { + if (typeof document === "undefined" || typeof location === "undefined") return "" + const hostname = location.hostname + if (!hostname || /^[\d.]+$/.test(hostname) || hostname.includes(":")) return "" + const parts = hostname.split(".") + if (parts.length < 2) return "" + for (let i = parts.length - 2; i >= 0; i--) { + const candidate = parts.slice(i).join(".") + const probe = "__maple_probe" + if (setRawCookie(probe, "1", candidate, 60) && readRawCookie(probe) === "1") { + setRawCookie(probe, "", candidate, 0) + return candidate + } + } + return "" +} + +export function cookieDomain(): string { + if (cookieDomainOverride !== undefined) return cookieDomainOverride + if (!crossSubdomainCookie) return "" + if (probedCookieDomain === undefined) probedCookieDomain = probeCookieDomain() + return probedCookieDomain +} + +/** Test seam — drops the memoized domain and any configured scope. */ +export function resetCookieScopeForTests(): void { + crossSubdomainCookie = true + cookieDomainOverride = undefined + probedCookieDomain = undefined +} diff --git a/packages/browser-session/src/events-sink.ts b/packages/browser-session/src/events-sink.ts index f62316a66..22ab189c7 100644 --- a/packages/browser-session/src/events-sink.ts +++ b/packages/browser-session/src/events-sink.ts @@ -101,10 +101,13 @@ export function getActiveSink(): SessionEventSink | undefined { /** * Start (or reuse) the distilled-event sink for a session. * - * Runs on **every** page load, not just sampled-for-replay ones: page views and - * `track()` calls are the analytics substrate, and gating them behind replay - * sampling would make unique visitors and top pages a sample rather than a - * count. The rrweb recorder stays sampled — it is the expensive part. + * Callers start this only for sessions the sample rate kept. It used to run on + * every page load — the argument being that page views and `track()` calls are + * the analytics substrate, so sampling them would turn unique visitors and top + * pages into estimates rather than counts. That argument lost to billing: the + * metadata row is the billed unit, sampling now suppresses it, and events + * without their parent session row are orphans no session view can render. A + * sampled-out visitor is absent, not partially present. */ export function startEventSink(config: ReplayEngineConfig, sessionId: string): SessionEventSink { const existing = holder()[SINK_KEY] diff --git a/packages/browser-session/src/meta-row.test.ts b/packages/browser-session/src/meta-row.test.ts index c0a042251..8ee9c8859 100644 --- a/packages/browser-session/src/meta-row.test.ts +++ b/packages/browser-session/src/meta-row.test.ts @@ -206,3 +206,43 @@ describe("buildSessionMetaRow analytics fields", () => { expect(row.group_id).toBe("") }) }) + +describe("buildSessionMetaRow billing marker", () => { + // This key is the wire contract between three layers that cannot see each + // other's types: the SDK writes it, the Rust gateway meters on it + // (`is_billable_visit_start`), and the warehouse maps it via + // `$.billable_start`. Renaming it here silently stops billing. + it("emits billable_start as 0/1, never a boolean or an absent key", () => { + const claimed = buildSessionMetaRow({ + ...base, + status: "active", + recorded: false, + billableStart: true, + }) + expect(claimed.billable_start).toBe(1) + + const unclaimed = buildSessionMetaRow({ + ...base, + status: "active", + recorded: false, + billableStart: false, + }) + expect(unclaimed.billable_start).toBe(0) + + // Absent input must still emit the key — the column is UInt8, and an + // omitted key would read as the DEFAULT rather than an explicit "no". + const unset = buildSessionMetaRow({ ...base, status: "active", recorded: false }) + expect(unset.billable_start).toBe(0) + }) + + it("keeps the marker on the ended row, which is what survives the merge", () => { + const row = buildSessionMetaRow({ + ...base, + version: 7, + status: "ended", + recorded: false, + billableStart: true, + }) + expect(row.billable_start).toBe(1) + }) +}) diff --git a/packages/browser-session/src/meta-row.ts b/packages/browser-session/src/meta-row.ts index 68dc3788a..e0805c882 100644 --- a/packages/browser-session/src/meta-row.ts +++ b/packages/browser-session/src/meta-row.ts @@ -57,6 +57,19 @@ export interface SessionMetaRowInput { * rendering a player with nothing to play. */ readonly recorded: boolean + /** + * Whether this session owns its visit claim — see `visit.ts`. Sticky: set on + * every row of a billable session, so the row that survives the merge still + * says whether it was charged. + * + * The gateway meters `billable_start == 1 && version == 1`. Billing used to key + * off `version === 1` alone, which conflated two unrelated jobs: `version` + * exists so `argMax(field, Version)` resolves the newest row, and it happened to + * be 1 on the first row of each session record — one per tab, per origin. The + * claim narrows that to one per visitor per idle window; `version` keeps it to + * one row. + */ + readonly billableStart?: boolean | undefined } /** @@ -119,6 +132,12 @@ export function buildSessionMetaRow(input: SessionMetaRowInput): Record { documentListeners = new Listeners() windowListeners = new Listeners() resetVisitorCacheForTests() + // The claim's in-memory fallback is module state; without this a later test + // inherits the previous one's paid visit and reports itself unbillable. + resetVisitClaimForTests() doc = { visibilityState: "visible", addEventListener: documentListeners.add, @@ -209,6 +213,62 @@ describe("idle rotation", () => { }) }) +describe("billable_start", () => { + it("stays on every row of the session, so the merged row reports the charge", () => { + const handle = start() + expect(last().billable_start).toBe(1) + expect(last().version).toBe(1) + + // Heartbeats and the ended row repeat the flag but not version 1, which is + // the conjunction the gateway meters on. Dropping it here would let the + // ended row replace the flagged one in the ReplacingMergeTree and leave the + // warehouse unable to reproduce the invoice. + vi.advanceTimersByTime(HEARTBEAT) + expect(last().billable_start).toBe(1) + expect(last().version).toBe(2) + + windowListeners.dispatch("pagehide") + expect(last().status).toBe("ended") + expect(last().billable_start).toBe(1) + expect(Number(last().version)).toBeGreaterThan(1) + expect(handle).toBeDefined() + }) + + it("is 0 for a session whose visit was already claimed elsewhere", () => { + const first = start() + expect(last().billable_start).toBe(1) + + // A second tab: same visitor, same 30-minute window, but its own + // sessionStorage and so its own session record. It is a real session the + // product must show — and not a second charge. Only the shared claim store + // (localStorage here, the cookie in a real browser) can know that. + posted = [] + const globals = globalThis as Record + const sharedLocalStorage = (globals.window as { localStorage: FakeStorage }).localStorage + globals.window = { sessionStorage: new FakeStorage(), localStorage: sharedLocalStorage } + + start() + expect(last().billable_start).toBe(0) + expect(last().version).toBe(1) + expect(first).toBeDefined() + }) + + it("charges again once the visit window has elapsed", async () => { + start() + expect(last().billable_start).toBe(1) + + // Past the idle window the heartbeat has already ended the run; the rotation + // is what re-arms it, and the new session is a new visit. + vi.advanceTimersByTime(31 * MINUTE) + posted = [] + rotateSession() + await flushMicrotasks() + + expect(last().status).toBe("active") + expect(last().billable_start).toBe(1) + }) +}) + describe("visibility", () => { it("ends on hide with keepalive and resumes with a fresh active row", () => { const suspends: SessionSuspendOptions[] = [] diff --git a/packages/browser-session/src/session-lifecycle.ts b/packages/browser-session/src/session-lifecycle.ts index 16a27fa4d..c057d1344 100644 --- a/packages/browser-session/src/session-lifecycle.ts +++ b/packages/browser-session/src/session-lifecycle.ts @@ -16,8 +16,10 @@ import { noteCounts, onSessionRotate, peekSession, + resolveBillable, type SessionRecord, } from "./session" +import { claimVisit } from "./visit" import { getVisitorId, isVisitorIdPersisted } from "./visitor" /** @@ -28,7 +30,7 @@ import { getVisitorId, isVisitorIdPersisted } from "./visitor" * posted at session start, whose counters are all zero and therefore read as a * bounce. 60s is the floor worth using: the table is a ReplacingMergeTree, so * every heartbeat is an unmerged part until the next merge. Billing meters only - * `version == 1` rows, so heartbeats do not double-bill. + * `billable_start == 1 && version == 1` rows, so heartbeats do not double-bill. */ const HEARTBEAT_INTERVAL_MS = 60_000 @@ -180,6 +182,11 @@ export function startSessionLifecycle( // record already carries it, and the two can only disagree. visitorIsNew: record.visitorIsNew === true, visitorIdPersisted: isVisitorIdPersisted(), + // Asked once per session and then read back off the record, so every + // row of a session agrees about whether it was charged. A second tab + // and a second subdomain reach this too and get `false` — they are + // real sessions, just not separate charges. + billableStart: resolveBillable(record.id, claimVisit), entry: entryContextOf(record), lastUrl: record.lastUrl, clickCount: counts.clickCount, diff --git a/packages/browser-session/src/session.test.ts b/packages/browser-session/src/session.test.ts index 33197179a..5b955aa79 100644 --- a/packages/browser-session/src/session.test.ts +++ b/packages/browser-session/src/session.test.ts @@ -8,6 +8,7 @@ import { nextMetaVersion, onSessionRotate, peekSession, + resetSessionStorageStateForTests, } from "./session" import { resetVisitorCacheForTests } from "./visitor" @@ -15,10 +16,13 @@ import { resetVisitorCacheForTests } from "./visitor" // rotation logic can be exercised under Node with a controllable clock. class FakeStorage { private store = new Map() + /** Reads keep working while writes throw — the quota-exhaustion shape. */ + writesThrow = false getItem(key: string): string | null { return this.store.has(key) ? this.store.get(key)! : null } setItem(key: string, value: string): void { + if (this.writesThrow) throw new DOMException("QuotaExceededError") this.store.set(key, value) } clear(): void { @@ -36,6 +40,7 @@ beforeEach(() => { vi.setSystemTime(new Date("2026-05-22T12:00:00Z")) storage = new FakeStorage() resetVisitorCacheForTests() + resetSessionStorageStateForTests() ;(globalThis as { window?: unknown }).window = { sessionStorage: storage, localStorage: new FakeStorage(), @@ -237,6 +242,31 @@ describe("nextMetaVersion", () => { getSession() // rotation resets it expect(nextMetaVersion()).toBe(1) }) + + it("keeps incrementing when reads succeed but writes throw", () => { + // Quota exhaustion: getItem keeps returning the last durable snapshot while + // setItem throws. Before the ephemeral fallback outranked storage this + // pinned the counter at 1, and since the gateway meters `version == 1` + // rows, every 60s heartbeat was billed as a whole new session. + getSession() + storage.writesThrow = true + + expect(nextMetaVersion()).toBe(1) + expect(nextMetaVersion()).toBe(2) + expect(nextMetaVersion()).toBe(3) + }) + + it("hands control back to storage once writes recover", () => { + getSession() + storage.writesThrow = true + nextMetaVersion() // 1 + nextMetaVersion() // 2 + storage.writesThrow = false + + expect(nextMetaVersion()).toBe(3) + // The durable copy is authoritative again, so the counter survives a reload. + expect(JSON.parse(storage.getItem(STORAGE_KEY)!).metaVersion).toBe(3) + }) }) describe("markActivity", () => { diff --git a/packages/browser-session/src/session.ts b/packages/browser-session/src/session.ts index 90d0a08bb..38146c349 100644 --- a/packages/browser-session/src/session.ts +++ b/packages/browser-session/src/session.ts @@ -2,8 +2,15 @@ import { claimNewVisitor } from "./visitor" const STORAGE_KEY = "maple.session" -/** Rotate the session after this much inactivity (PostHog's default). */ -const IDLE_TIMEOUT_MS = 30 * 60_000 +/** + * Rotate the session after this much inactivity (PostHog's default). + * + * Exported because the billable-visit claim in `visit.ts` has to use the exact + * same window: if the visit window were longer, an idle rotation would produce a + * session nobody is charged for; if shorter, one continuous session would be + * charged twice. + */ +export const IDLE_TIMEOUT_MS = 30 * 60_000 /** Hard cap on a single session's lifetime regardless of activity. */ const MAX_SESSION_MS = 24 * 60 * 60_000 /** @@ -44,6 +51,16 @@ export interface SessionRecord { * used versions 1 (active) and 2 (ended), so the absent case resumes at 2. */ metaVersion?: number + /** + * Whether this session won its billing claim (see `visit.ts`). Persisted so + * every row of the session reports the same answer — a reload must not + * re-claim, and the `ended` row must agree with the `active` one or the merged + * row would misreport whether the session was charged. + * + * `undefined` means "not yet resolved"; `resolveBillable` in + * `session-lifecycle.ts` settles it once, on the run's first post. + */ + billable?: boolean // --- Analytics context ------------------------------------------------- // All optional: `readRecord`'s validator deliberately still accepts records @@ -88,6 +105,19 @@ const UTM_KEYS = ["utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_c /** In-memory fallback when sessionStorage is unavailable (private mode). */ let ephemeral: SessionRecord | undefined +/** + * Set once a write has failed, after which `ephemeral` outranks whatever is in + * sessionStorage. + * + * The failure mode this exists for is quota exhaustion, where `getItem` keeps + * succeeding while `setItem` throws — so a read-side check for "is storage + * usable" sees a healthy store and returns a record that can never advance. + * `metaVersion` then stays pinned at 1 forever, and since the gateway meters + * `version == 1` rows, every 60s heartbeat is billed as a new session. Do not + * collapse this back into the `getItem` try/catch: a throwing read is a + * different, rarer fault than a throwing write. + */ +let writesFailed = false export type SessionRotationListener = (previous: SessionRecord, next: SessionRecord) => void const rotationListeners = new Set() @@ -139,6 +169,9 @@ function freshRecord(now: number): SessionRecord { } function readRecord(): SessionRecord | undefined { + // Once writes are failing the stored copy is a stale snapshot that can only + // get staler; the in-memory one is the only record still advancing. + if (writesFailed && ephemeral) return ephemeral try { const raw = window.sessionStorage.getItem(STORAGE_KEY) if (!raw) return undefined @@ -161,8 +194,14 @@ function writeRecord(record: SessionRecord): void { ephemeral = record try { window.sessionStorage.setItem(STORAGE_KEY, JSON.stringify(record)) + // Cleared on success rather than latched: a quota freed by another tab + // closing should put the durable store back in charge, so the record + // survives the next reload. + writesFailed = false } catch { - // Private mode / storage disabled — the ephemeral copy is the source of truth. + // Private mode / storage disabled / quota exhausted — the ephemeral copy is + // the source of truth from here on, and `readRecord` has to be told so. + writesFailed = true } } @@ -297,6 +336,27 @@ export function noteCounts(counts: { clickCount?: number; errorCount?: number }) }) } +/** + * Settle whether `sessionId` is the one charged for its visit, asking `claim` + * only the first time and persisting the answer. + * + * The persistence is the point: a reload, a hide/resume cycle and every 60s + * heartbeat all rebuild the metadata row, and each must report the same verdict + * the first row did. Re-asking would re-claim on the reload that follows the + * claim expiring, and disagree with the row already in the warehouse. + * + * The `claim` callback is injected rather than imported so `session.ts` stays + * free of a dependency on `visit.ts`, which reads this module's idle window. + */ +export function resolveBillable(sessionId: string, claim: () => boolean): boolean { + const record = readRecord() + if (!record || record.id !== sessionId) return false + if (record.billable !== undefined) return record.billable + const billable = claim() + writeRecord({ ...record, billable }) + return billable +} + /** * The persisted session record as-is — no activity touch, no rotation. Use * this to read counters when posting a metadata row; `getSession()` would @@ -358,3 +418,9 @@ export function nextMetaVersion(): number { writeRecord({ ...record, metaVersion: version }) return version } + +/** Test seam — drops the in-memory fallback and its write-failure latch. */ +export function resetSessionStorageStateForTests(): void { + ephemeral = undefined + writesFailed = false +} diff --git a/packages/browser-session/src/visit.test.ts b/packages/browser-session/src/visit.test.ts new file mode 100644 index 000000000..ce26e18e0 --- /dev/null +++ b/packages/browser-session/src/visit.test.ts @@ -0,0 +1,132 @@ +import { beforeEach, describe, expect, it, vi } from "vitest" +import { resetCookieScopeForTests } from "./cookie" +import { claimVisit, resetVisitClaimForTests } from "./visit" + +const MINUTE = 60_000 +const IDLE_WINDOW = 30 * MINUTE + +interface CookieEntry { + value: string + /** `""` = host-only. */ + domain: string +} + +/** + * A `document.cookie` stand-in shared by every origin in a test, which is the + * whole point: a cookie set on `example.com` has to be visible to + * `app.example.com`, because that hop is what the visit claim exists to + * deduplicate. + */ +function installCookies(jar: Map, hostname = "app.example.com"): void { + vi.stubGlobal("location", { hostname, protocol: "https:" }) + vi.stubGlobal("document", { + get cookie(): string { + return [...jar].map(([name, entry]) => `${name}=${entry.value}`).join("; ") + }, + set cookie(raw: string) { + const [pair, ...attrs] = raw.split(";").map((part) => part.trim()) + const eq = pair?.indexOf("=") ?? -1 + if (!pair || eq < 0) return + const name = pair.slice(0, eq) + const value = pair.slice(eq + 1) + const domain = (attrs.find((a) => a.toLowerCase().startsWith("domain="))?.slice(7) ?? "").replace( + /^\./, + "", + ) + const maxAge = attrs.find((a) => a.toLowerCase().startsWith("max-age="))?.slice(8) + if (maxAge === "0") { + jar.delete(name) + return + } + jar.set(name, { value, domain }) + }, + }) +} + +/** A fresh, per-origin localStorage — the store that must *not* be what dedupes. */ +function installStorage(options: { throwOnWrite?: boolean } = {}): void { + const store = new Map() + vi.stubGlobal("window", { + localStorage: { + getItem: (key: string) => store.get(key) ?? null, + setItem: (key: string, value: string) => { + if (options.throwOnWrite) throw new Error("QuotaExceededError") + store.set(key, value) + }, + removeItem: (key: string) => store.delete(key), + }, + }) +} + +describe("claimVisit", () => { + let jar: Map + + beforeEach(() => { + vi.unstubAllGlobals() + resetVisitClaimForTests() + resetCookieScopeForTests() + jar = new Map() + installStorage() + installCookies(jar) + }) + + it("grants the first claim and refuses the rest of the window", () => { + const start = Date.UTC(2026, 4, 22, 12) + + expect(claimVisit(start)).toBe(true) + expect(claimVisit(start + MINUTE)).toBe(false) + expect(claimVisit(start + 29 * MINUTE)).toBe(false) + }) + + it("grants again once the idle window has passed", () => { + const start = Date.UTC(2026, 4, 22, 12) + expect(claimVisit(start)).toBe(true) + + // The window matches the session idle timeout exactly: the rotation that + // mints a new session id is the same moment a new visit begins, so this + // boundary is what keeps rotated sessions from going uncharged. + expect(claimVisit(start + IDLE_WINDOW)).toBe(true) + expect(claimVisit(start + IDLE_WINDOW + MINUTE)).toBe(false) + }) + + it("refuses a second tab — a fresh page-load state, the same cookie jar", () => { + const start = Date.UTC(2026, 4, 22, 12) + expect(claimVisit(start)).toBe(true) + + // A second tab runs its own module instance against its own sessionStorage, + // so nothing but the shared cookie can tell it the visit is already paid for. + resetVisitClaimForTests() + installStorage() + installCookies(jar) + + expect(claimVisit(start + MINUTE)).toBe(false) + }) + + it("refuses a second subdomain", () => { + const start = Date.UTC(2026, 4, 22, 12) + installCookies(jar, "example.com") + expect(claimVisit(start)).toBe(true) + + // localStorage is origin-scoped, so the app subdomain gets a brand new one — + // exactly the case that used to bill a marketing-site visitor twice. + resetVisitClaimForTests() + resetCookieScopeForTests() + installStorage() + installCookies(jar, "app.example.com") + + expect(claimVisit(start + MINUTE)).toBe(false) + }) + + it("still deduplicates within a page load when localStorage writes throw", () => { + const start = Date.UTC(2026, 4, 22, 12) + installStorage({ throwOnWrite: true }) + + expect(claimVisit(start)).toBe(true) + expect(claimVisit(start + MINUTE)).toBe(false) + }) + + it("returns false outside a browser rather than billing a server render", () => { + vi.unstubAllGlobals() + expect(claimVisit(Date.UTC(2026, 4, 22, 12))).toBe(false) + }) +}) diff --git a/packages/browser-session/src/visit.ts b/packages/browser-session/src/visit.ts new file mode 100644 index 000000000..dc0008b48 --- /dev/null +++ b/packages/browser-session/src/visit.ts @@ -0,0 +1,99 @@ +/** + * The billable unit: one *visit*, not one session record. + * + * ## Why this is not just the session + * + * `SessionId` lives in sessionStorage, which is scoped to one tab **and** one + * origin. That is deliberate and load-bearing — `session_replays` is a + * ReplacingMergeTree keyed `(OrgId, SessionId)` whose fields resolve by + * `argMax(field, Version)` over the whole row, so two tabs or two origins + * writing under one id would overwrite each other rather than merge (see + * `visitor.ts`). It is also a terrible thing to charge for: a customer with four + * tabs open was billed four times, and a visitor crossing from `example.com` to + * `app.example.com` was billed twice, for one person doing one thing. + * + * So the storage identity stays per-tab and the *billing* identity is claimed + * here, from the same cookie + localStorage pair the visitor id already uses — + * the one store in this SDK that spans tabs and subdomains. + * + * ## Why the claim is a timestamp and not a flag + * + * The window has to match the session idle window exactly, or the two disagree + * about how many things happened: a 30-minute idle rotation genuinely starts a + * new visit and must re-bill, while a second tab opened one minute in must not. + * Storing "when did the last claim happen" answers both from one value, and + * needs no cleanup — a stale marker simply fails the window check. + * + * ## What this deliberately does not fix + * + * A browser with both stores blocked can't hold a claim, so every page load + * re-claims and re-bills. That is not fixable from here; the metadata row + * carries `maple.visitor.persisted: "false"` so the size of that tail is + * measurable rather than assumed. + */ + +import { cookieDomain, readRawCookie, setRawCookie } from "./cookie" +import { IDLE_TIMEOUT_MS } from "./session" + +const STORAGE_KEY = "maple.visit" +/** Cookie names cannot contain `.` per RFC 6265's token grammar. */ +const COOKIE_NAME = "maple_visit" + +/** + * In-memory fallback, and the reason a single page load never double-claims even + * with both stores blocked. + */ +let ephemeral: number | undefined + +function parseClaim(raw: string | undefined): number | undefined { + if (!raw) return undefined + const parsed = Number.parseInt(raw, 10) + return Number.isFinite(parsed) ? parsed : undefined +} + +function readClaim(): number | undefined { + // Cookie first, for the same reason the visitor id prefers it: it is the copy + // shared across subdomains, so the app sees the claim the marketing site made. + const fromCookie = parseClaim(readRawCookie(COOKIE_NAME)) + if (fromCookie !== undefined) return fromCookie + try { + const fromStorage = parseClaim(window.localStorage.getItem(STORAGE_KEY) ?? undefined) + if (fromStorage !== undefined) return fromStorage + } catch { + // Storage blocked — fall through to the in-memory copy. + } + return ephemeral +} + +function writeClaim(now: number): void { + ephemeral = now + const value = String(now) + try { + window.localStorage.setItem(STORAGE_KEY, value) + } catch { + // Private mode / quota — the cookie and the in-memory copy still stand. + } + // Max-age is the window itself: an expired claim is indistinguishable from no + // claim, so letting the browser drop it saves us the staleness check. + setRawCookie(COOKIE_NAME, value, cookieDomain(), IDLE_TIMEOUT_MS / 1000) +} + +/** + * Claim this visit for billing, returning whether the claim was ours to make. + * + * `true` at most once per `IDLE_TIMEOUT_MS` across every tab and subdomain the + * cookie reaches. Every caller that gets `false` is a session that is real, + * stored, and queryable — just not separately charged. + */ +export function claimVisit(now = Date.now()): boolean { + if (typeof window === "undefined") return false + const last = readClaim() + if (last !== undefined && now - last < IDLE_TIMEOUT_MS && now >= last) return false + writeClaim(now) + return true +} + +/** Test seam — drops the in-memory claim without touching storage. */ +export function resetVisitClaimForTests(): void { + ephemeral = undefined +} diff --git a/packages/browser-session/src/visitor.ts b/packages/browser-session/src/visitor.ts index e1a356933..c18c0a2f7 100644 --- a/packages/browser-session/src/visitor.ts +++ b/packages/browser-session/src/visitor.ts @@ -38,6 +38,14 @@ * per-surface; `VisitorId` is the join key between them. */ +import { + configureCookieScope, + cookieDomain, + readRawCookie, + resetCookieScopeForTests, + setRawCookie, +} from "./cookie" + const STORAGE_KEY = "maple.visitor" /** * Cookie names cannot contain `.` per RFC 6265's token grammar, so this is not @@ -64,114 +72,12 @@ let cached: VisitorRecord | undefined let persisted = false let mintedThisLoad = false -/** Scope the cookie to the registered domain so subdomains share the id. */ -let crossSubdomainCookie = true -/** Explicit `Domain=` override; `""` forces a host-only cookie. */ -let cookieDomainOverride: string | undefined -/** Memoized probe result. `undefined` = not resolved yet. */ -let probedCookieDomain: string | undefined - /** - * Apply the host app's cookie configuration. Called from `configurePrivacy`, so - * both SDKs get it from the single call they already make. - * - * Like the consent gates, this only ever *tightens*: an app that initializes two - * SDKs, only one of which passes a `privacy` block, must not have the other's - * absent option widen the cookie back out to every subdomain. - * - * "Tighter" for `cookieDomain` means *narrower scope*, which is why this is not - * first-write-wins: `""` (host-only) is the tightest value there is, and a - * second SDK asking for it has to win over an earlier `"example.com"`. Between - * two non-empty domains the shorter one is the broader — `example.com` covers - * `app.example.com` and not the reverse — so the longer string wins. + * Configure the cookie scope. Kept exported from here because `configurePrivacy` + * and the SDK tests have always called it by this name; the implementation and + * the domain probe now live in `cookie.ts`, shared with the visit claim. */ -export function configureVisitorCookie(options: { - readonly crossSubdomainCookie?: boolean | undefined - readonly cookieDomain?: string | undefined -}): void { - if (options.crossSubdomainCookie === false) crossSubdomainCookie = false - if (options.cookieDomain !== undefined) { - cookieDomainOverride = tighterCookieDomain(cookieDomainOverride, options.cookieDomain) - } - probedCookieDomain = undefined -} - -/** The narrower of two `Domain=` values, treating `undefined` as "unset". */ -function tighterCookieDomain(current: string | undefined, next: string): string { - if (current === undefined) return next - // Host-only beats any domain-scoped cookie, whichever side asked for it. - if (current === "" || next === "") return "" - return next.length > current.length ? next : current -} - -// --- Cookie plumbing ------------------------------------------------------- - -function readRawCookie(name: string): string | undefined { - if (typeof document === "undefined") return undefined - try { - for (const part of document.cookie.split(";")) { - const raw = part.trim() - if (!raw.startsWith(`${name}=`)) continue - return decodeURIComponent(raw.slice(name.length + 1)) - } - } catch { - // Cookies disabled entirely — indistinguishable from "not set". - } - return undefined -} - -function setRawCookie(name: string, value: string, domain: string, maxAgeSeconds: number): boolean { - if (typeof document === "undefined") return false - const attributes = [ - `${name}=${encodeURIComponent(value)}`, - "path=/", - `max-age=${Math.max(0, Math.floor(maxAgeSeconds))}`, - "SameSite=Lax", - ] - if (domain) attributes.push(`domain=.${domain}`) - // A `Secure` cookie is rejected over http, which is exactly the local-dev case. - if (typeof location !== "undefined" && location.protocol === "https:") attributes.push("Secure") - try { - document.cookie = attributes.join("; ") - return true - } catch { - return false - } -} - -/** - * The broadest domain this browser will actually accept a cookie for, found by - * probing rather than by carrying a public-suffix list — the same trick - * posthog-js uses. Candidates start at the broadest (the last two labels) and - * narrow a label at a time, with the first that sticks winning — so - * `app.example.co.uk` tries the rejected `co.uk`, then lands on `example.co.uk`. - * - * Returns `""` (host-only cookie) for single-label hosts like `localhost` and - * for bare IPs, neither of which can carry a `Domain=` attribute. - */ -function probeCookieDomain(): string { - if (typeof document === "undefined" || typeof location === "undefined") return "" - const hostname = location.hostname - if (!hostname || /^[\d.]+$/.test(hostname) || hostname.includes(":")) return "" - const parts = hostname.split(".") - if (parts.length < 2) return "" - for (let i = parts.length - 2; i >= 0; i--) { - const candidate = parts.slice(i).join(".") - const probe = "__maple_probe" - if (setRawCookie(probe, "1", candidate, 60) && readRawCookie(probe) === "1") { - setRawCookie(probe, "", candidate, 0) - return candidate - } - } - return "" -} - -function cookieDomain(): string { - if (cookieDomainOverride !== undefined) return cookieDomainOverride - if (!crossSubdomainCookie) return "" - if (probedCookieDomain === undefined) probedCookieDomain = probeCookieDomain() - return probedCookieDomain -} +export const configureVisitorCookie = configureCookieScope // --- Record read/write ----------------------------------------------------- @@ -320,7 +226,5 @@ export function resetVisitorCacheForTests(): void { persisted = false mintedThisLoad = false enabled = true - crossSubdomainCookie = true - cookieDomainOverride = undefined - probedCookieDomain = undefined + resetCookieScopeForTests() } diff --git a/packages/browser/src/config.ts b/packages/browser/src/config.ts index 2d858a84e..df5a34828 100644 --- a/packages/browser/src/config.ts +++ b/packages/browser/src/config.ts @@ -43,9 +43,20 @@ export interface MapleBrowserConfig { readonly instrumentFetch?: boolean } readonly replay?: { - /** Default true. */ + /** + * Whether to record video of captured sessions. Default true. Turning this + * off keeps session analytics — page views, `track()` calls, durations — + * and drops only the rrweb recording. + */ readonly enabled?: boolean - /** Fraction of sessions to record, 0–1. Default 1. */ + /** + * Fraction of sessions to capture, 0–1. Default 1. + * + * This is the billing lever: an unsampled visitor produces no session rows + * at all, so they cost nothing and appear nowhere in session analytics. It + * is not a recording-only sample — dropping it to 0.1 means one session in + * ten exists, not ten in ten of which one has video. + */ readonly sampleRate?: number } readonly privacy?: { diff --git a/packages/browser/src/init.test.ts b/packages/browser/src/init.test.ts index 5c8fc697c..e39d8249c 100644 --- a/packages/browser/src/init.test.ts +++ b/packages/browser/src/init.test.ts @@ -101,6 +101,64 @@ describe("lazy replay chunk", () => { }) }) +describe("sampling", () => { + const capturePosts = (): Array<{ url: string; body: string }> => { + const posts: Array<{ url: string; body: string }> = [] + vi.stubGlobal("fetch", async (input: RequestInfo | URL, requestInit?: RequestInit) => { + posts.push({ + url: typeof input === "string" ? input : input instanceof URL ? input.href : input.url, + body: typeof requestInit?.body === "string" ? requestInit.body : "", + }) + return new Response(null, { status: 200 }) + }) + vi.stubGlobal("window", { + sessionStorage: new MemoryStorage(), + localStorage: new MemoryStorage(), + location: { href: "https://app.example.com/", host: "app.example.com" }, + }) + return posts + } + + it("posts nothing at all at sampleRate 0 — no metadata row, so no charge", async () => { + const posts = capturePosts() + const handle = init({ + ingestKey: "public-key", + serviceName: "test-web", + endpoint: "https://collector.test", + tracing: { enabled: false }, + replay: { enabled: true, sampleRate: 0 }, + }) + + track("should-not-ship") + await handle.shutdown() + + expect(replay.start).not.toHaveBeenCalled() + // The metadata row is the billed unit; the event rows would be orphans + // without it. Neither may leave the page. + expect(posts.filter((post) => post.url.includes("/v1/sessionReplays/meta"))).toHaveLength(0) + expect(posts.filter((post) => post.url.endsWith("/v1/sessionEvents"))).toHaveLength(0) + }) + + it("still captures the session at sampleRate 1 with replay disabled", async () => { + const posts = capturePosts() + const handle = init({ + ingestKey: "public-key", + serviceName: "test-web", + endpoint: "https://collector.test", + tracing: { enabled: false }, + replay: { enabled: false, sampleRate: 1 }, + }) + + track("keep-me") + await handle.shutdown() + + // Turning off video is not a request to turn off analytics. + expect(replay.start).not.toHaveBeenCalled() + expect(posts.some((post) => post.url.includes("/v1/sessionReplays/meta"))).toBe(true) + expect(posts.some((post) => post.body.includes("keep-me"))).toBe(true) + }) +}) + describe("browser consent lifecycle", () => { it("starts on grant, discards revoked buffers, and restarts with a new session", async () => { const posts: Array<{ url: string; body: string }> = [] diff --git a/packages/browser/src/init.ts b/packages/browser/src/init.ts index 965d5ec28..9b576c82a 100644 --- a/packages/browser/src/init.ts +++ b/packages/browser/src/init.ts @@ -35,7 +35,8 @@ export interface MapleBrowserHandle { interface BrowserRuntime { readonly initialSessionId: string - readonly sink: SessionEventSink + /** Absent when the sample rate excluded this visitor — see `captureSession`. */ + readonly sink: SessionEventSink | undefined replay?: ReplaySessionHandle | undefined metadata?: MetadataSessionHandle | undefined /** Settles when the lazy replay chunk resolved; absent on the metadata path. */ @@ -64,7 +65,23 @@ export function init(rawConfig: MapleBrowserConfig): MapleBrowserHandle { if (!hasConsent()) clearPendingEvents() setActiveTraceIdProvider(() => trace.getActiveSpan()?.spanContext().traceId) - const recordReplay = config.replayEnabled && Math.random() < config.replaySampleRate + // One draw, two decisions. `captureSession` governs everything that produces a + // session — the metadata rows, which are the billed unit, and the distilled + // event sink that gives them their contents. `recordReplay` is a strict subset: + // there is no such thing as recording a session you are not capturing. + // + // Sampling deliberately reaches the metadata rows. It used to gate only the + // rrweb chunk, which meant an org on `sampleRate: 0.1` was billed for 100% of + // its sessions and had no way to buy less — the one lever named "sample" moved + // only the part we don't charge for. The cost is that unsampled traffic is + // absent from session analytics rather than present-but-unrecorded; that is the + // trade a sample rate is supposed to make. + // + // `replayEnabled: false` must not suppress capture: turning off video is not + // the same request as turning off analytics. + const sampledIn = Math.random() < config.replaySampleRate + const captureSession = sampledIn + const recordReplay = config.replayEnabled && sampledIn let runtime: BrowserRuntime | undefined let stopped = false let rotateOnNextStart = false @@ -78,6 +95,17 @@ export function init(rawConfig: MapleBrowserConfig): MapleBrowserHandle { setVisitorTracking(config.persistVisitorId && mayPersistIdentifier()) const session = (rotateOnNextStart ? rotateSession() : undefined) ?? getSession() rotateOnNextStart = false + if (config.tracingEnabled && !shutdownTracing) shutdownTracing = setupTracing(config) + + // Unsampled: no sink published, so `TraceIdCollector` finds none and spans + // carry no `session.id`. A link to a session row that was never written is + // worse than no link — it dead-ends in the trace UI. Tracing itself is + // untouched; spans are billed as traces and sampled by their own tracer. + if (!captureSession) { + runtime = { initialSessionId: session.id, sink: undefined } + return + } + publishSessionSink(session.id) const sink = startEventSink( { @@ -88,7 +116,6 @@ export function init(rawConfig: MapleBrowserConfig): MapleBrowserHandle { }, session.id, ) - if (config.tracingEnabled && !shutdownTracing) shutdownTracing = setupTracing(config) const shared = { endpoint: config.endpoint, ingestKey: config.ingestKey, @@ -150,9 +177,12 @@ export function init(rawConfig: MapleBrowserConfig): MapleBrowserHandle { const liveSink = getActiveSink() const sink = liveSink && currentSessionId && liveSink.sessionId === currentSessionId ? liveSink : previous.sink - if (flush) await sink.flush(true) - sink.stop() - if (sink !== previous.sink) previous.sink.stop() + // Both are absent for an unsampled runtime, which never started a sink. + if (sink) { + if (flush) await sink.flush(true) + sink.stop() + } + if (sink !== previous.sink) previous.sink?.stop() clearSessionSink(currentSessionId ?? previous.initialSessionId) // Awaiting the import too keeps `shutdown()` a real quiescence point: it // resolves with no replay work still scheduled behind it. diff --git a/packages/domain/src/clickhouse/migrations/0013_session_billable_start.ts b/packages/domain/src/clickhouse/migrations/0013_session_billable_start.ts new file mode 100644 index 000000000..8932256fb --- /dev/null +++ b/packages/domain/src/clickhouse/migrations/0013_session_billable_start.ts @@ -0,0 +1,31 @@ +/** + * Migration 0013 — `session_replays.BillableStart`. + * + * Makes the browser-session invoice reproducible from the warehouse. Until now + * nothing here recorded which sessions were charged: the ingest gateway metered + * rows whose `Version` was 1, `Version` exists only so `argMax(field, Version)` + * resolves the newest row of a ReplacingMergeTree, and the two facts never met + * in the same table. "Why is this month's bill what it is" had no query. + * + * The billed unit is a *visit* — one visitor per 30-minute idle window, spanning + * tabs and subdomains — not a `SessionId`, which is scoped to one tab and one + * origin by sessionStorage. So `countIf(BillableStart = 1)` and + * `uniq(SessionId)` are both correct and deliberately different numbers, and + * only the first matches Autumn. + * + * Appended at the end of the table with a constant DEFAULT, so this is a + * metadata-only `ALTER` on ClickHouse and Tinybird alike and existing parts read + * the default for free — no `MATERIALIZE COLUMN`. That default also means rows + * already in the table read as unbilled even where the gateway did bill them + * under the old rule; the column is only truthful going forward, and the 30-day + * TTL rolls the ambiguous window out on its own. + * + * `requiredForIngest` is left at its default (true): the native ClickHouse + * INSERT names every column explicitly, so a BYO cluster that has not applied + * this migration must not be routed direct ingest. + */ +export const migration_0013_session_billable_start = { + version: 13, + description: "Add BillableStart to session_replays so the browser-session bill is auditable", + statements: ["ALTER TABLE session_replays ADD COLUMN IF NOT EXISTS BillableStart UInt8 DEFAULT 0"], +} as const diff --git a/packages/domain/src/clickhouse/migrations/index.test.ts b/packages/domain/src/clickhouse/migrations/index.test.ts index 5c0c28267..6493f499f 100644 --- a/packages/domain/src/clickhouse/migrations/index.test.ts +++ b/packages/domain/src/clickhouse/migrations/index.test.ts @@ -15,6 +15,7 @@ import { import { migration_0010_search_indexes } from "./0010_search_indexes" import { migration_0011_session_analytics_columns } from "./0011_session_analytics_columns" import { migration_0012_session_event_attribute_keys } from "./0012_session_event_attribute_keys" +import { migration_0013_session_billable_start } from "./0013_session_billable_start" import { clickHouseSchemaVersion, latestMigrationVersion, migrations } from "./index" const backfills = migration_0004_service_namespace_projections.statements.filter( @@ -29,14 +30,23 @@ const renderedSql = migration_0004_service_namespace_projections.statements describe("ClickHouse migrations", () => { it("keeps migrations ordered by version", () => { - expect(migrations.map((m) => m.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]) - expect(migrations.at(-1)).toBe(migration_0012_session_event_attribute_keys) - expect(latestMigrationVersion).toBe(12) - // 0010 is performance-only, so the ingest-gating version skips it: 9 → 12. - expect(clickHouseSchemaVersion).toBe("12") + expect(migrations.map((m) => m.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]) + expect(migrations.at(-1)).toBe(migration_0013_session_billable_start) + expect(latestMigrationVersion).toBe(13) + // 0010 is performance-only, so the ingest-gating version skips it: 9 → 13. + expect(clickHouseSchemaVersion).toBe("13") expect(migration_0010_search_indexes.requiredForIngest).toBe(false) }) + it("adds BillableStart with a constant default, so the ALTER stays metadata-only", () => { + const sql = migration_0013_session_billable_start.statements.join("\n") + + // Trailing column + constant DEFAULT is what keeps this off the mutation + // path; without the default, rows from SDKs predating the field quarantine. + expect(sql).toContain("ADD COLUMN IF NOT EXISTS BillableStart UInt8 DEFAULT 0") + expect(sql).not.toContain("MATERIALIZE COLUMN") + }) + it("adds session analytics columns with defaults so older SDK rows never quarantine", () => { const sql = migration_0011_session_analytics_columns.statements.join("\n") diff --git a/packages/domain/src/clickhouse/migrations/index.ts b/packages/domain/src/clickhouse/migrations/index.ts index 75da48a09..b7df35ab0 100644 --- a/packages/domain/src/clickhouse/migrations/index.ts +++ b/packages/domain/src/clickhouse/migrations/index.ts @@ -11,6 +11,7 @@ import { migration_0009_one_year_service_history } from "./0009_one_year_service import { migration_0010_search_indexes } from "./0010_search_indexes" import { migration_0011_session_analytics_columns } from "./0011_session_analytics_columns" import { migration_0012_session_event_attribute_keys } from "./0012_session_event_attribute_keys" +import { migration_0013_session_billable_start } from "./0013_session_billable_start" /** * A migration statement is either a raw SQL string (structural DDL) or a @@ -54,6 +55,7 @@ export const migrations: ReadonlyArray = [ migration_0010_search_indexes, migration_0011_session_analytics_columns, migration_0012_session_event_attribute_keys, + migration_0013_session_billable_start, ] as const /** Highest migration `version` bundled — i.e. the schema level a fully-applied diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index 32885dd85..76161ef6f 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "12a7685236a4ebe4a8a40900e74c0239ab853afec09659d78a9b2f92811561fa" as const +export const projectRevision = "5654f2527545c4b48718669fdf45e05c5a1b8de02c2f6d861a87c2c67bc19368" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS alert_checks (\n OrgId LowCardinality(String),\n RuleId String,\n GroupKey String,\n Timestamp DateTime64(3),\n Status LowCardinality(String),\n SignalType LowCardinality(String),\n Comparator LowCardinality(String),\n Threshold Float64,\n ObservedValue Nullable(Float64),\n SampleCount UInt32,\n WindowMinutes UInt16,\n WindowStart DateTime64(3),\n WindowEnd DateTime64(3),\n ConsecutiveBreaches UInt16,\n ConsecutiveHealthy UInt16,\n IncidentId Nullable(String),\n IncidentTransition LowCardinality(String),\n EvaluationDurationMs UInt32,\n ErrorMessage Nullable(String),\n ErrorCategory LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, RuleId, GroupKey, Timestamp)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", @@ -32,7 +32,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS service_usage (\n OrgId LowCardinality(String),\n ServiceName LowCardinality(String),\n Hour DateTime,\n LogCount UInt64,\n LogSizeBytes UInt64,\n TraceCount UInt64,\n TraceSizeBytes UInt64,\n SumMetricCount UInt64,\n SumMetricSizeBytes UInt64,\n GaugeMetricCount UInt64,\n GaugeMetricSizeBytes UInt64,\n HistogramMetricCount UInt64,\n HistogramMetricSizeBytes UInt64,\n ExpHistogramMetricCount UInt64,\n ExpHistogramMetricSizeBytes UInt64\n)\nENGINE = SummingMergeTree\nORDER BY (OrgId, ServiceName, Hour)\nTTL Hour + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS session_events (\n OrgId LowCardinality(String),\n SessionId String,\n Timestamp DateTime64(9),\n Seq UInt32 DEFAULT 0,\n Type LowCardinality(String),\n Url String DEFAULT '',\n TraceId String DEFAULT '',\n Level LowCardinality(String) DEFAULT '',\n Message String DEFAULT '',\n TargetSelector String DEFAULT '',\n TargetText String DEFAULT '',\n NetMethod LowCardinality(String) DEFAULT '',\n NetUrl String DEFAULT '',\n NetStatus UInt16 DEFAULT 0,\n NetDurationMs UInt32 DEFAULT 0,\n ErrorStack String DEFAULT '',\n Attributes Map(String, String),\n INDEX idx_type Type TYPE set(16) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, SessionId, Timestamp, Seq)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS session_replay_events (\n OrgId LowCardinality(String),\n SessionId String,\n ChunkSeq UInt32,\n Timestamp DateTime64(9),\n DurationMs UInt32 DEFAULT 0,\n EventCount UInt32 DEFAULT 0,\n ByteSize UInt32 DEFAULT 0,\n Events String,\n IsCheckpoint UInt8 DEFAULT 0\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, SessionId, ChunkSeq)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", - "CREATE TABLE IF NOT EXISTS session_replays (\n OrgId LowCardinality(String),\n SessionId String,\n StartTime DateTime64(9),\n EndTime Nullable(DateTime64(9)),\n DurationMs Nullable(UInt32),\n Status LowCardinality(String),\n UserId String,\n UrlInitial String,\n UserAgent String,\n BrowserName LowCardinality(String),\n OsName LowCardinality(String),\n DeviceType LowCardinality(String),\n Country LowCardinality(String) DEFAULT '',\n ServiceName LowCardinality(String),\n PageViews UInt32 DEFAULT 0,\n ClickCount UInt32 DEFAULT 0,\n ErrorCount UInt32 DEFAULT 0,\n TraceIds Array(String) DEFAULT [],\n ResourceAttributes Map(LowCardinality(String), String),\n Version UInt32,\n VisitorId String DEFAULT '',\n VisitorIsNew UInt8 DEFAULT 0,\n UserEmail String DEFAULT '',\n UserName String DEFAULT '',\n GroupId String DEFAULT '',\n GroupName String DEFAULT '',\n UserTraits Map(String, String) DEFAULT map(),\n Referrer String DEFAULT '',\n ReferrerHost LowCardinality(String) DEFAULT '',\n UtmSource LowCardinality(String) DEFAULT '',\n UtmMedium LowCardinality(String) DEFAULT '',\n UtmCampaign LowCardinality(String) DEFAULT '',\n UtmTerm String DEFAULT '',\n UtmContent String DEFAULT '',\n Host LowCardinality(String) DEFAULT '',\n EntryPath String DEFAULT '',\n ExitPath String DEFAULT '',\n Language LowCardinality(String) DEFAULT '',\n LastActivityAt Nullable(DateTime64(9))\n)\nENGINE = ReplacingMergeTree\nPARTITION BY toDate(StartTime)\nORDER BY (OrgId, SessionId)\nTTL toDate(StartTime) + INTERVAL 30 DAY", + "CREATE TABLE IF NOT EXISTS session_replays (\n OrgId LowCardinality(String),\n SessionId String,\n StartTime DateTime64(9),\n EndTime Nullable(DateTime64(9)),\n DurationMs Nullable(UInt32),\n Status LowCardinality(String),\n UserId String,\n UrlInitial String,\n UserAgent String,\n BrowserName LowCardinality(String),\n OsName LowCardinality(String),\n DeviceType LowCardinality(String),\n Country LowCardinality(String) DEFAULT '',\n ServiceName LowCardinality(String),\n PageViews UInt32 DEFAULT 0,\n ClickCount UInt32 DEFAULT 0,\n ErrorCount UInt32 DEFAULT 0,\n TraceIds Array(String) DEFAULT [],\n ResourceAttributes Map(LowCardinality(String), String),\n Version UInt32,\n VisitorId String DEFAULT '',\n VisitorIsNew UInt8 DEFAULT 0,\n UserEmail String DEFAULT '',\n UserName String DEFAULT '',\n GroupId String DEFAULT '',\n GroupName String DEFAULT '',\n UserTraits Map(String, String) DEFAULT map(),\n Referrer String DEFAULT '',\n ReferrerHost LowCardinality(String) DEFAULT '',\n UtmSource LowCardinality(String) DEFAULT '',\n UtmMedium LowCardinality(String) DEFAULT '',\n UtmCampaign LowCardinality(String) DEFAULT '',\n UtmTerm String DEFAULT '',\n UtmContent String DEFAULT '',\n Host LowCardinality(String) DEFAULT '',\n EntryPath String DEFAULT '',\n ExitPath String DEFAULT '',\n Language LowCardinality(String) DEFAULT '',\n LastActivityAt Nullable(DateTime64(9)),\n BillableStart UInt8 DEFAULT 0\n)\nENGINE = ReplacingMergeTree\nPARTITION BY toDate(StartTime)\nORDER BY (OrgId, SessionId)\nTTL toDate(StartTime) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n SpanKind LowCardinality(String),\n AttrFingerprint UInt64,\n ResourceFingerprint UInt64,\n StartTimeUnix DateTime64(9),\n LastValue AggregateFunction(argMax, Float64, DateTime64(9))\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix)\nTTL toDate(Hour) + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS trace_detail_spans (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n ResourceAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String))\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, TraceId, SpanId)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS trace_list_mv (\n OrgId LowCardinality(String),\n TraceId String,\n Timestamp DateTime,\n ServiceName LowCardinality(String),\n SpanName String,\n SpanKind LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n HttpMethod LowCardinality(String),\n HttpRoute String,\n HttpStatusCode LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n HasError UInt8,\n TraceState String,\n ServiceNamespace LowCardinality(String),\n INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, TraceId)\nTTL Timestamp + INTERVAL 30 DAY", diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index 0c02f6147..1e2f48be9 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "12a7685236a4ebe4a8a40900e74c0239ab853afec09659d78a9b2f92811561fa" as const +export const projectRevision = "5654f2527545c4b48718669fdf45e05c5a1b8de02c2f6d861a87c2c67bc19368" as const export const datasources = [ { @@ -147,7 +147,7 @@ export const datasources = [ { name: "session_replays", content: - "DESCRIPTION >\n Per-session browser replay metadata (one row per session). Ingested directly from the @maple-dev/browser SDK via POST /v1/sessionReplays/meta. Event payloads live inline in session_replay_events; this holds only queryable metadata. ReplacingMergeTree(Version) for start/end upsert.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.org_id`,\n SessionId String `json:$.session_id`,\n StartTime DateTime64(9) `json:$.start_time`,\n EndTime Nullable(DateTime64(9)) `json:$.end_time`,\n DurationMs Nullable(UInt32) `json:$.duration_ms`,\n Status LowCardinality(String) `json:$.status`,\n UserId String `json:$.user_id`,\n UrlInitial String `json:$.url_initial`,\n UserAgent String `json:$.user_agent`,\n BrowserName LowCardinality(String) `json:$.browser_name`,\n OsName LowCardinality(String) `json:$.os_name`,\n DeviceType LowCardinality(String) `json:$.device_type`,\n Country LowCardinality(String) `json:$.country` DEFAULT '',\n ServiceName LowCardinality(String) `json:$.service_name`,\n PageViews UInt32 `json:$.page_views` DEFAULT 0,\n ClickCount UInt32 `json:$.click_count` DEFAULT 0,\n ErrorCount UInt32 `json:$.error_count` DEFAULT 0,\n TraceIds Array(String) `json:$.trace_ids[:]` DEFAULT [],\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n Version UInt32 `json:$.version`,\n VisitorId String `json:$.visitor_id` DEFAULT '',\n VisitorIsNew UInt8 `json:$.visitor_is_new` DEFAULT 0,\n UserEmail String `json:$.user_email` DEFAULT '',\n UserName String `json:$.user_name` DEFAULT '',\n GroupId String `json:$.group_id` DEFAULT '',\n GroupName String `json:$.group_name` DEFAULT '',\n UserTraits Map(String, String) `json:$.user_traits` DEFAULT map(),\n Referrer String `json:$.referrer` DEFAULT '',\n ReferrerHost LowCardinality(String) `json:$.referrer_host` DEFAULT '',\n UtmSource LowCardinality(String) `json:$.utm_source` DEFAULT '',\n UtmMedium LowCardinality(String) `json:$.utm_medium` DEFAULT '',\n UtmCampaign LowCardinality(String) `json:$.utm_campaign` DEFAULT '',\n UtmTerm String `json:$.utm_term` DEFAULT '',\n UtmContent String `json:$.utm_content` DEFAULT '',\n Host LowCardinality(String) `json:$.host` DEFAULT '',\n EntryPath String `json:$.entry_path` DEFAULT '',\n ExitPath String `json:$.exit_path` DEFAULT '',\n Language LowCardinality(String) `json:$.language` DEFAULT '',\n LastActivityAt Nullable(DateTime64(9)) `json:$.last_activity_at`\n\nENGINE \"ReplacingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(StartTime)\"\nENGINE_SORTING_KEY \"OrgId, SessionId\"\nENGINE_TTL \"toDate(StartTime) + INTERVAL 30 DAY\"\nENGINE_VER \"Version\"", + "DESCRIPTION >\n Per-session browser replay metadata (one row per session). Ingested directly from the @maple-dev/browser SDK via POST /v1/sessionReplays/meta. Event payloads live inline in session_replay_events; this holds only queryable metadata. ReplacingMergeTree(Version) for start/end upsert.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.org_id`,\n SessionId String `json:$.session_id`,\n StartTime DateTime64(9) `json:$.start_time`,\n EndTime Nullable(DateTime64(9)) `json:$.end_time`,\n DurationMs Nullable(UInt32) `json:$.duration_ms`,\n Status LowCardinality(String) `json:$.status`,\n UserId String `json:$.user_id`,\n UrlInitial String `json:$.url_initial`,\n UserAgent String `json:$.user_agent`,\n BrowserName LowCardinality(String) `json:$.browser_name`,\n OsName LowCardinality(String) `json:$.os_name`,\n DeviceType LowCardinality(String) `json:$.device_type`,\n Country LowCardinality(String) `json:$.country` DEFAULT '',\n ServiceName LowCardinality(String) `json:$.service_name`,\n PageViews UInt32 `json:$.page_views` DEFAULT 0,\n ClickCount UInt32 `json:$.click_count` DEFAULT 0,\n ErrorCount UInt32 `json:$.error_count` DEFAULT 0,\n TraceIds Array(String) `json:$.trace_ids[:]` DEFAULT [],\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n Version UInt32 `json:$.version`,\n VisitorId String `json:$.visitor_id` DEFAULT '',\n VisitorIsNew UInt8 `json:$.visitor_is_new` DEFAULT 0,\n UserEmail String `json:$.user_email` DEFAULT '',\n UserName String `json:$.user_name` DEFAULT '',\n GroupId String `json:$.group_id` DEFAULT '',\n GroupName String `json:$.group_name` DEFAULT '',\n UserTraits Map(String, String) `json:$.user_traits` DEFAULT map(),\n Referrer String `json:$.referrer` DEFAULT '',\n ReferrerHost LowCardinality(String) `json:$.referrer_host` DEFAULT '',\n UtmSource LowCardinality(String) `json:$.utm_source` DEFAULT '',\n UtmMedium LowCardinality(String) `json:$.utm_medium` DEFAULT '',\n UtmCampaign LowCardinality(String) `json:$.utm_campaign` DEFAULT '',\n UtmTerm String `json:$.utm_term` DEFAULT '',\n UtmContent String `json:$.utm_content` DEFAULT '',\n Host LowCardinality(String) `json:$.host` DEFAULT '',\n EntryPath String `json:$.entry_path` DEFAULT '',\n ExitPath String `json:$.exit_path` DEFAULT '',\n Language LowCardinality(String) `json:$.language` DEFAULT '',\n LastActivityAt Nullable(DateTime64(9)) `json:$.last_activity_at`,\n BillableStart UInt8 `json:$.billable_start` DEFAULT 0\n\nENGINE \"ReplacingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(StartTime)\"\nENGINE_SORTING_KEY \"OrgId, SessionId\"\nENGINE_TTL \"toDate(StartTime) + INTERVAL 30 DAY\"\nENGINE_VER \"Version\"", }, { name: "span_metrics_calls_hourly", diff --git a/packages/domain/src/tinybird/datasources.ts b/packages/domain/src/tinybird/datasources.ts index a935162a3..d27d84ae1 100644 --- a/packages/domain/src/tinybird/datasources.ts +++ b/packages/domain/src/tinybird/datasources.ts @@ -1799,6 +1799,29 @@ export const sessionReplays = defineDatasource("session_replays", { LastActivityAt: column(t.dateTime64(9).nullable(), { jsonPath: "$.last_activity_at", }), + + /** + * Whether this session was billed to Autumn as a browser session. + * + * The billed unit is a *visit*, not a session row: `SessionId` lives in + * sessionStorage and is therefore scoped per tab and per origin, so a + * visitor with four tabs open owns four sessions. The SDK claims a visit + * once per visitor per 30-minute idle window against a cookie shared across + * subdomains, and only the claiming session carries a 1 here. + * + * Set on every row of a billable session, not just the first, because this + * is a ReplacingMergeTree that replaces the whole row at the latest + * `Version` — a flag present only on the first row would be erased by the + * unload row. `countIf(BillableStart = 1)` is what reproduces the invoice; + * `uniq(SessionId)` counts sessions, which is the larger number. + * + * Defaults to 0, so rows from SDK bundles predating the field read as + * unbilled here even though the gateway did bill them under the older + * `Version = 1` rule. + */ + BillableStart: column(t.uint8().default(0), { + jsonPath: "$.billable_start", + }), }, engine: engine.replacingMergeTree({ partitionKey: "toDate(StartTime)", diff --git a/packages/effect-sdk/src/client/replay-loader.ts b/packages/effect-sdk/src/client/replay-loader.ts index 002c742e2..c18f62ea9 100644 --- a/packages/effect-sdk/src/client/replay-loader.ts +++ b/packages/effect-sdk/src/client/replay-loader.ts @@ -21,9 +21,12 @@ import { setupStandaloneSession } from "./standalone-session.js" import { getCurrentIdentity } from "./user.js" export interface ClientReplayConfig { - /** Record rrweb session replays. Default `true`. */ + /** Record rrweb session replays. Default `true`. Session analytics are kept. */ readonly enabled?: boolean | undefined - /** Fraction of sessions to record, 0–1. Default `1`. */ + /** + * Fraction of sessions to capture, 0–1. Default `1`. An unsampled visitor + * produces no session rows and is not billed. + */ readonly sampleRate?: number | undefined /** Mask all `` values in the recording. Default `true`. */ readonly maskAllInputs?: boolean | undefined @@ -78,7 +81,12 @@ export const startClientSession = (config: ClientSessionConfig): ClientSessionHa maskAllText: config.replay?.maskAllText ?? false, } const replayEnabled = (config.replay?.enabled ?? true) && typeof document !== "undefined" - const sampled = replayEnabled && Math.random() < (config.replay?.sampleRate ?? 1) + // One draw, two decisions — see the same split in `@maple-dev/browser`'s + // `init.ts`. The sample rate reaches the metadata rows because those are what + // Autumn bills; `enabled: false` drops only the recording, never the session. + const sampledIn = Math.random() < (config.replay?.sampleRate ?? 1) + const captureSession = sampledIn + const sampled = replayEnabled && sampledIn let runtime: Runtime | undefined let stopped = false let generation = 0 @@ -109,6 +117,10 @@ export const startClientSession = (config: ClientSessionConfig): ClientSessionHa setVisitorTracking((config.privacy?.persistVisitorId ?? true) && mayPersistIdentifier()) const session = (rotateOnNextStart ? rotateSession() : undefined) ?? getSession() rotateOnNextStart = false + // Sampled out: no sink, no metadata row, nothing billed. Leaving `runtime` + // unset keeps `stopRuntime` a no-op, which is exactly right — there is + // nothing to flush. + if (!captureSession) return const next: Runtime = { sink: startEventSink(engineConfig, session.id) } runtime = next const ownGeneration = ++generation diff --git a/packages/query-engine-integrations/src/product/billing-usage.test.ts b/packages/query-engine-integrations/src/product/billing-usage.test.ts index 59c146374..67659ed9a 100644 --- a/packages/query-engine-integrations/src/product/billing-usage.test.ts +++ b/packages/query-engine-integrations/src/product/billing-usage.test.ts @@ -78,10 +78,20 @@ describe("dailySessionCountQuery", () => { expect(sql).toContain("FROM session_replays") expect(sql).toContain("OrgId = 'org_123'") expect(sql).toContain("toStartOfInterval(StartTime, INTERVAL 86400 SECOND) AS day") - expect(sql).toContain("count() AS sessions") expect(sql).toContain("GROUP BY day") }) + it("counts distinct session ids, not rows", () => { + const { sql } = compileCH(dailySessionCountQuery(), params) + + // The SDK posts a row at session start, one per 60s heartbeat, and one at + // unload. This is a ReplacingMergeTree read without FINAL, so count() bills + // the chart for every row a merge has not collapsed yet — a 10-minute + // session rendered as ~12. + expect(sql).toContain("uniq(SessionId) AS sessions") + expect(sql).not.toContain("count()") + }) + it("filters on StartTime so the partition key prunes", () => { const { sql } = compileCH(dailySessionCountQuery(), params) diff --git a/packages/query-engine-integrations/src/product/billing-usage.ts b/packages/query-engine-integrations/src/product/billing-usage.ts index 16b412968..2a600ea3a 100644 --- a/packages/query-engine-integrations/src/product/billing-usage.ts +++ b/packages/query-engine-integrations/src/product/billing-usage.ts @@ -10,10 +10,11 @@ // chart's totals reconcile with them rather than telling a second story. // // `dailySessionCountQuery` — browser sessions per UTC day, from -// `session_replays` (one row per session). Kept as a separate query rather -// than a UNION branch: the two tables disagree on column types (byte sums -// are UInt64, the session count is UInt64 but the bucket column comes from -// DateTime64) and unifying them has bitten us with 502s before. +// `session_replays` (many rows per session — see the query). Kept as a +// separate query rather than a UNION branch: the two tables disagree on +// column types (byte sums are UInt64, the session count is UInt64 but the +// bucket column comes from DateTime64) and unifying them has bitten us with +// 502s before. // // Day buckets are UTC, matching how the warehouse stores every timestamp. Byte // sums are UInt64 and arrive as JSON strings on BYO-ClickHouse, so both row @@ -90,15 +91,34 @@ export const dailySessionCountRowSchema: CompiledQueryRowSchema ({ day: CH.toStartOfInterval($.StartTime, DAY_SECONDS), - sessions: CH.count(), + sessions: CH.uniq($.SessionId), })) .where(($) => [ $.OrgId.eq(param.string("orgId")), diff --git a/packages/query-engine/src/ch/tables.ts b/packages/query-engine/src/ch/tables.ts index 3bcb94e01..38111cd02 100644 --- a/packages/query-engine/src/ch/tables.ts +++ b/packages/query-engine/src/ch/tables.ts @@ -564,6 +564,11 @@ export const SessionReplays = table("session_replays", { // Heartbeat-refreshed. Recovers duration for sessions killed without an // unload beacon, where EndTime is null. LastActivityAt: T.nullable(T.dateTime64), + // 1 on every row of a session that was billed to Autumn. The billed unit is a + // visit (one visitor per 30-minute window, spanning tabs and subdomains), not + // a SessionId — so countIf(BillableStart = 1) reproduces the invoice and + // uniq(SessionId) is the larger, session-level count. + BillableStart: T.uint8, }) export const SessionReplayEvents = table("session_replay_events", {