From 81bae0d6e32c3afb97f9e7718640be83090d9f10 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Fri, 7 Aug 2026 15:18:10 -0400 Subject: [PATCH 1/2] refactor(alerting): extract host-neutral core --- apps/api/package.json | 1 + apps/api/src/services/alerts/AlertsService.ts | 398 ++++++------------ bun.lock | 12 + packages/alerting-core/README.md | 28 ++ packages/alerting-core/package.json | 18 + packages/alerting-core/src/index.test.ts | 193 +++++++++ packages/alerting-core/src/index.ts | 321 ++++++++++++++ packages/alerting-core/tsconfig.json | 17 + 8 files changed, 718 insertions(+), 270 deletions(-) create mode 100644 packages/alerting-core/README.md create mode 100644 packages/alerting-core/package.json create mode 100644 packages/alerting-core/src/index.test.ts create mode 100644 packages/alerting-core/src/index.ts create mode 100644 packages/alerting-core/tsconfig.json diff --git a/apps/api/package.json b/apps/api/package.json index 423de550b..dc3f6283d 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -32,6 +32,7 @@ "@distilled.cloud/core": "1.0.0-rc.2", "@effect/platform-bun": "catalog:effect", "@maple-dev/effect-sdk": "workspace:*", + "@maple/alerting-core": "workspace:*", "@maple/auth": "workspace:*", "@maple/cache": "workspace:*", "@maple/db": "workspace:*", diff --git a/apps/api/src/services/alerts/AlertsService.ts b/apps/api/src/services/alerts/AlertsService.ts index 382ccac76..19097030a 100644 --- a/apps/api/src/services/alerts/AlertsService.ts +++ b/apps/api/src/services/alerts/AlertsService.ts @@ -1,4 +1,14 @@ import { randomUUID } from "node:crypto" +import { + alertDeliveryRetryDelayMs, + canRetryAlertDelivery, + evaluateAlertObservation, + interleaveAlertRulesByTenant, + makeAlertDeliveryKey, + planAlertLifecycle, + type AlertEvaluation as EvaluatedRule, + type AlertLifecycleInput, +} from "@maple/alerting-core" import { CompiledAlertQueryPlan, QueryEngineAlertReducer, @@ -103,9 +113,7 @@ import { } from "effect" import * as AlertingMetrics from "@/observability/AlertingMetrics" import { warehouseHandlers } from "@/services/warehouse/warehouse-error-handlers" -import { - INVESTIGATION_FANOUT_BINDING, -} from "@/services/errors/ai-triage-enqueue" +import { INVESTIGATION_FANOUT_BINDING } from "@/services/errors/ai-triage-enqueue" import { upsertAlertIssue } from "@/services/errors/issue-hub" import { probeLiveness } from "@/services/alerts/telemetry-liveness" import { WorkerEnvironment } from "@maple/effect-cloudflare/worker-environment" @@ -193,24 +201,6 @@ interface NormalizedRule { readonly updatedAt: number } -interface EvaluatedRule { - readonly status: Schema.Schema.Type - readonly value: number | null - readonly sampleCount: number - readonly threshold: number - readonly thresholdUpper: number | null - readonly comparator: AlertComparator - readonly reason: string - /** - * The window returned nothing and `noDataBehavior: "zero"` synthesized the - * value. Such a status is a statement about the absence of data, not about - * the health of the system — a `gt` rule reads a total ingest outage as - * `healthy` this way. Anything that acts on "healthy" destructively (i.e. - * resolving an open incident) must prove telemetry is still flowing first. - */ - readonly derivedFromNoData: boolean -} - type DispatchContext = Omit< DeliveryDispatchContext, "ruleId" | "incidentId" | "incidentStatus" | "sentAtMs" @@ -233,7 +223,6 @@ interface DeliveryAttemptFailure { readonly retryable: boolean } -const MAX_DELIVERY_ATTEMPTS = 5 const ALERT_TEST_DELIVERY_CONCURRENCY = 5 const ALERT_CHECK_INGEST_CONCURRENCY = 4 // Storm fuse: cap issue-hub upserts per scheduler tick so a pathological @@ -410,25 +399,7 @@ const MAX_ACTIVE_ALERT_RULES_PER_ORG = 100 /** Preserve each org's oldest-first order while preventing one org from monopolizing a tick. */ export const interleaveAlertRulesByOrg = ( rows: ReadonlyArray, -): ReadonlyArray => { - const queues = new Map() - for (const row of rows) { - const queue = queues.get(row.orgId) - if (queue) queue.push(row) - else queues.set(row.orgId, [row]) - } - - const fair: T[] = [] - let index = 0 - while (fair.length < rows.length) { - for (const queue of queues.values()) { - const row = queue[index] - if (row !== undefined) fair.push(row) - } - index += 1 - } - return fair -} +): ReadonlyArray => interleaveAlertRulesByTenant(rows, (row) => row.orgId) // Tinybird DateTime64(3) wire format for alert_checks ingest: // "YYYY-MM-DD HH:MM:SS.SSS" (UTC, no timezone). @@ -438,27 +409,6 @@ const toIngestDateTime64 = (epochMs: number) => { return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}.${pad(d.getUTCMilliseconds(), 3)}` } -const compareThreshold = ( - value: number, - comparator: AlertComparator, - threshold: number, - thresholdUpper: number | null = null, -): boolean => - Match.value(comparator).pipe( - Match.when("gt", () => value > threshold), - Match.when("gte", () => value >= threshold), - Match.when("lt", () => value < threshold), - Match.when("lte", () => value <= threshold), - Match.when("eq", () => value === threshold), - Match.when("neq", () => value !== threshold), - Match.when("between", () => thresholdUpper != null && value >= threshold && value <= thresholdUpper), - Match.when( - "not_between", - () => thresholdUpper != null && (value < threshold || value > thresholdUpper), - ), - Match.exhaustive, - ) - const normalizeOptionalString = (value: string | null | undefined) => { const trimmed = value?.trim() return trimmed && trimmed.length > 0 ? trimmed : null @@ -1626,79 +1576,26 @@ export class AlertsService extends Context.Service, reasonOverride?: string, - ): EvaluatedRule => { - const noDataBehavior = rule.compiledPlan.noDataBehavior - // Sample-weighted counts arrive fractional from the warehouse - // (`sum(SampleRate)`), and this flows into `last_sample_count`, an - // `integer` column — an unrounded value fails the insert outright. - const sampleCount = Math.round(obs.sampleCount) - const value = obs.hasData ? obs.value : noDataBehavior === "zero" ? 0 : null - - if (!obs.hasData && noDataBehavior === "skip") { - return { - status: "skipped", - value: null, - sampleCount, - threshold: rule.threshold, - thresholdUpper: rule.thresholdUpper, - comparator: rule.comparator, - reason: "No data in the selected window", - // Inert: `skipped` never resolves an incident, so this branch - // short-circuits before any status is derived from a synthesized value. - derivedFromNoData: false, - } - } - - if (sampleCount < rule.minimumSampleCount) { - return { - status: "skipped", - value, - sampleCount, - threshold: rule.threshold, - thresholdUpper: rule.thresholdUpper, + ): EvaluatedRule => + evaluateAlertObservation( + { comparator: rule.comparator, - reason: `Sample count ${sampleCount} is below minimum ${rule.minimumSampleCount}`, - derivedFromNoData: false, - } - } - - if (value == null) { - return { - status: "skipped", - value: null, - sampleCount, threshold: rule.threshold, thresholdUpper: rule.thresholdUpper, - comparator: rule.comparator, - reason: "Alert evaluation did not return a scalar value", - derivedFromNoData: false, - } - } - - return { - status: compareThreshold(value, rule.comparator, rule.threshold, rule.thresholdUpper) - ? "breached" - : "healthy", - value, - sampleCount, - threshold: rule.threshold, - thresholdUpper: rule.thresholdUpper, - comparator: rule.comparator, - reason: - reasonOverride ?? + minimumSampleCount: rule.minimumSampleCount, + noDataBehavior: rule.compiledPlan.noDataBehavior, + }, + obs, + reasonOverride ?? `${rule.signalType} ${formatComparator(rule.comparator, rule.threshold, rule.thresholdUpper)}`, - // Only reachable with `noDataBehavior: "zero"` — the "skip" branch - // returned above. The comparison ran against a fabricated 0. - derivedFromNoData: !obs.hasData, - } - } + ) const buildDeliveryKey = ( incidentId: string, destinationId: string, eventType: AlertEventTypeValue, scheduledAt: number, - ) => [incidentId, destinationId, eventType, scheduledAt].join(":") + ) => makeAlertDeliveryKey(incidentId, destinationId, eventType, scheduledAt) const insertDeliveryEventRecord = ( db: DatabaseExecutor, @@ -1969,9 +1866,8 @@ export class AlertsService extends Context.Service= against *Required, so saturating keeps open/resolve behavior - // identical while letting steady-state ticks skip the state upsert above. - const consecutiveBreaches = - evaluation.status === "breached" - ? Math.min( - (state?.consecutiveBreaches ?? 0) + 1, - normalized.consecutiveBreachesRequired, - ) - : 0 - const consecutiveHealthy = - evaluation.status === "healthy" - ? Math.min( - (state?.consecutiveHealthy ?? 0) + 1, - normalized.consecutiveHealthyRequired, - ) - : 0 - + let lifecycle = planAlertLifecycle(lifecycleInput) + // Persist the counter/state decision before follow-up adapter work, as + // before extraction. A failed flap-history or liveness query must not + // discard an evaluation that already completed successfully. yield* upsertState({ - consecutiveBreaches, - consecutiveHealthy, + consecutiveBreaches: lifecycle.state.consecutiveBreaches, + consecutiveHealthy: lifecycle.state.consecutiveHealthy, lastStatus: evaluation.status, lastValue: evaluation.value, lastSampleCount: evaluation.sampleCount, }) - if ( - evaluation.status === "breached" && - openIncident == null && - consecutiveBreaches >= normalized.consecutiveBreachesRequired - ) { - // Flap suppression: a metric oscillating around the threshold opens - // a fresh incident per flap, which would email an identical trigger - // notification every few minutes. If the previous incident for this - // (rule, group) was notified within the renotify interval, open the - // incident but skip the trigger notification and carry the prior - // lastNotifiedAt forward — the renotify gate then enforces one - // email per interval while the flapping persists. + // Ask the persistence adapter for flap history only when the pure core + // has decided that a new incident is otherwise ready to open. + if (lifecycle.transition === "opened") { const priorNotified = (yield* dbExecute((db) => db @@ -3942,9 +3827,46 @@ export class AlertsService extends Context.Service db.insert(alertIncidents).values(incident)) - if (flapSuppressedAt != null) { + if (lifecycle.notificationSuppression === "flapping") { yield* Effect.logInfo("Skipping trigger notification for flapping incident").pipe( Effect.annotateLogs({ ruleId: row.id, incidentId, groupKey, - priorNotifiedAt: flapSuppressedAt.toISOString(), + priorNotifiedAt: inheritedNotificationAt?.toISOString(), }), ) - } else { + } else if (lifecycle.eventType === "trigger") { yield* queueIncidentNotifications( row.orgId, normalized, incident, evaluation, - "trigger", + lifecycle.eventType, timestamp, ) } return { - transition: "opened" as const, + transition: lifecycle.transition, incidentId, openedIncidentId: incidentId, - consecutiveBreaches, - consecutiveHealthy, + consecutiveBreaches: lifecycle.state.consecutiveBreaches, + consecutiveHealthy: lifecycle.state.consecutiveHealthy, } } - if (evaluation.status === "breached" && openIncident != null) { + if (lifecycle.transition === "continued" && openIncident != null) { const refreshedIncident = { ...openIncident, lastTriggeredAt: new Date(timestamp), @@ -4013,19 +3932,6 @@ export class AlertsService extends Context.Service db .update(alertIncidents) @@ -4035,72 +3941,32 @@ export class AlertsService extends Context.Service= normalized.consecutiveHealthyRequired - ) { - // A "healthy" synthesized from an empty window is a statement - // about missing data, not about a recovered system: with - // `noDataBehavior: "zero"` a total ingest outage compares as 0 < - // threshold and would resolve every incident it touches, paging - // out a wave of false all-clears. Believe it only once telemetry - // is provably still arriving. - if (evaluation.derivedFromNoData) { - const liveness = yield* telemetryStillFlowing( - row.orgId, - normalized, - openIncident.firstTriggeredAt.getTime(), - timestamp, - ) - if (!liveness.dataFlowing) { - yield* Effect.logWarning( - "Holding incident open: healthy evaluation came from missing telemetry", - ).pipe( - Effect.annotateLogs({ - orgId: row.orgId, - ruleId: row.id, - incidentId: openIncident.id, - groupKey, - livenessReason: liveness.reason, - observedCount: liveness.observedCount, - baselineCount: liveness.baselineCount, - }), - ) - return { - transition: "none" as const, - incidentId: carriedIncidentId, - openedIncidentId: null, - consecutiveBreaches, - consecutiveHealthy, - } - } - } - + if (lifecycle.transition === "resolved" && openIncident != null) { const resolvedIncident = { ...openIncident, status: "resolved" as const, @@ -4110,7 +3976,6 @@ export class AlertsService extends Context.Service db .update(alertIncidents) @@ -4124,14 +3989,7 @@ export class AlertsService extends Context.Service=4.0.0-beta.100 || >=4.0.0", }, }, + "packages/alerting-core": { + "name": "@maple/alerting-core", + "version": "0.0.0", + "devDependencies": { + "@types/node": "catalog:tooling", + "typescript": "catalog:tooling", + "vitest": "catalog:", + }, + }, "packages/auth": { "name": "@maple/auth", "dependencies": { @@ -1515,6 +1525,8 @@ "@maple/alerting": ["@maple/alerting@workspace:apps/alerting"], + "@maple/alerting-core": ["@maple/alerting-core@workspace:packages/alerting-core"], + "@maple/api": ["@maple/api@workspace:apps/api"], "@maple/auth": ["@maple/auth@workspace:packages/auth"], diff --git a/packages/alerting-core/README.md b/packages/alerting-core/README.md new file mode 100644 index 000000000..a464cce3a --- /dev/null +++ b/packages/alerting-core/README.md @@ -0,0 +1,28 @@ +# `@maple/alerting-core` + +Host-neutral alert evaluation and incident-lifecycle semantics shared by Maple +deployment targets. + +The core is deliberately free of database, telemetry warehouse, scheduler, +network, and wall-clock dependencies. A host supplies observations and durable +state, calls the pure decision functions, then applies the returned transition +and delivery intent through its own adapters. + +Current hosted adapters live in `apps/api` and are scheduled by +`apps/alerting`. A Maple Local adapter can use the same core with chDB-backed +queries, Local durable state, an in-process scheduler, and its own outbound URL +policy without importing either hosted application. + +The boundary is: + +- query adapter -> `AlertObservation`; +- evaluation policy + observation -> `AlertEvaluation`; +- persistence snapshot + evaluation -> `AlertLifecyclePlan`; +- host persists the plan and sends its optional `eventType` through a delivery + adapter; +- delivery adapters share idempotency-key and bounded retry policy helpers; +- host clock supplies `nowMs`; the core never reads global time. + +Rule CRUD, storage schemas, scheduler claims, destination configuration, and +delivery transports remain host concerns. This keeps Local UI work optional: +the alert runtime can evaluate and deliver while no browser is open. diff --git a/packages/alerting-core/package.json b/packages/alerting-core/package.json new file mode 100644 index 000000000..0ce27ea84 --- /dev/null +++ b/packages/alerting-core/package.json @@ -0,0 +1,18 @@ +{ + "name": "@maple/alerting-core", + "version": "0.0.0", + "private": true, + "type": "module", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "test": "vitest run", + "typecheck": "tsc --noEmit" + }, + "devDependencies": { + "@types/node": "catalog:tooling", + "typescript": "catalog:tooling", + "vitest": "catalog:" + } +} diff --git a/packages/alerting-core/src/index.test.ts b/packages/alerting-core/src/index.test.ts new file mode 100644 index 000000000..9ad7c26dc --- /dev/null +++ b/packages/alerting-core/src/index.test.ts @@ -0,0 +1,193 @@ +import { describe, expect, it } from "vitest" +import { + alertDeliveryRetryDelayMs, + canRetryAlertDelivery, + evaluateAlertObservation, + interleaveAlertRulesByTenant, + makeAlertDeliveryKey, + planAlertLifecycle, + type AlertEvaluation, +} from "./index" + +const breached: AlertEvaluation = { + status: "breached", + value: 11, + sampleCount: 5, + threshold: 10, + thresholdUpper: null, + comparator: "gt", + reason: "above threshold", + derivedFromNoData: false, +} + +const healthy: AlertEvaluation = { ...breached, status: "healthy", value: 9 } + +const policy = { + consecutiveBreachesRequired: 2, + consecutiveHealthyRequired: 2, + renotifyIntervalMinutes: 10, +} + +describe("evaluateAlertObservation", () => { + it("applies thresholds and rounds weighted sample counts", () => { + expect( + evaluateAlertObservation( + { + comparator: "between", + threshold: 10, + thresholdUpper: 20, + minimumSampleCount: 2, + noDataBehavior: "skip", + }, + { value: 15, sampleCount: 2.4, hasData: true }, + "inside range", + ), + ).toMatchObject({ status: "breached", sampleCount: 2, reason: "inside range" }) + }) + + it("marks a zero synthesized from no data so lifecycle resolution can fail closed", () => { + expect( + evaluateAlertObservation( + { + comparator: "gt", + threshold: 10, + thresholdUpper: null, + minimumSampleCount: 0, + noDataBehavior: "zero", + }, + { value: null, sampleCount: 0, hasData: false }, + "above threshold", + ), + ).toMatchObject({ status: "healthy", value: 0, derivedFromNoData: true }) + }) +}) + +describe("planAlertLifecycle", () => { + it("opens only after the configured breach count", () => { + const first = planAlertLifecycle({ + policy, + evaluation: breached, + state: null, + openIncident: null, + nowMs: 1_000, + }) + expect(first).toMatchObject({ transition: "none", state: { consecutiveBreaches: 1 } }) + + const second = planAlertLifecycle({ + policy, + evaluation: breached, + state: first.state, + openIncident: null, + nowMs: 2_000, + }) + expect(second).toMatchObject({ transition: "opened", eventType: "trigger" }) + }) + + it("suppresses a flapping trigger and its matching resolve", () => { + const opened = planAlertLifecycle({ + policy, + evaluation: breached, + state: { consecutiveBreaches: 1, consecutiveHealthy: 0 }, + openIncident: null, + nowMs: 600_000, + previousNotificationAtMs: 300_000, + }) + expect(opened).toMatchObject({ + transition: "opened", + eventType: null, + notificationSuppression: "flapping", + inheritedNotificationAtMs: 300_000, + }) + + const resolved = planAlertLifecycle({ + policy, + evaluation: healthy, + state: { consecutiveBreaches: 0, consecutiveHealthy: 1 }, + openIncident: { + firstTriggeredAtMs: 600_000, + lastNotifiedAtMs: opened.inheritedNotificationAtMs, + lastDeliveredEventType: null, + }, + nowMs: 700_000, + }) + expect(resolved).toMatchObject({ + transition: "resolved", + eventType: null, + notificationSuppression: "flap_resolution", + }) + }) + + it("advances the notification anchor when renotify becomes due", () => { + const plan = planAlertLifecycle({ + policy, + evaluation: breached, + state: { consecutiveBreaches: 2, consecutiveHealthy: 0 }, + openIncident: { + firstTriggeredAtMs: 0, + lastNotifiedAtMs: 1_000, + lastDeliveredEventType: "trigger", + }, + nowMs: 601_000, + }) + expect(plan).toMatchObject({ + transition: "continued", + eventType: "renotify", + advanceNotificationAnchor: true, + }) + }) + + it("holds a no-data recovery until the host proves telemetry liveness", () => { + const noDataHealthy = { ...healthy, derivedFromNoData: true } + const input = { + policy, + evaluation: noDataHealthy, + state: { consecutiveBreaches: 0, consecutiveHealthy: 1 }, + openIncident: { + firstTriggeredAtMs: 0, + lastNotifiedAtMs: 0, + lastDeliveredEventType: "trigger" as const, + }, + nowMs: 1_000, + } + expect(planAlertLifecycle(input)).toMatchObject({ transition: "none", hold: "missing_telemetry" }) + expect(planAlertLifecycle({ ...input, allowNoDataResolution: true })).toMatchObject({ + transition: "resolved", + eventType: "resolve", + }) + }) +}) + +describe("interleaveAlertRulesByTenant", () => { + it("preserves each tenant's order while round-robining tenants", () => { + const rows = [ + { tenantId: "a", id: "a1" }, + { tenantId: "a", id: "a2" }, + { tenantId: "b", id: "b1" }, + { tenantId: "a", id: "a3" }, + { tenantId: "b", id: "b2" }, + ] + expect(interleaveAlertRulesByTenant(rows, (row) => row.tenantId).map(({ id }) => id)).toEqual([ + "a1", + "b1", + "a2", + "b2", + "a3", + ]) + }) +}) + +describe("delivery policy", () => { + it("builds stable idempotency keys", () => { + expect(makeAlertDeliveryKey("incident", "destination", "trigger", 42)).toBe( + "incident:destination:trigger:42", + ) + }) + + it("caps exponential retry delay and attempts", () => { + expect(alertDeliveryRetryDelayMs(1, 123)).toBe(60_123) + expect(alertDeliveryRetryDelayMs(5, 999)).toBe(900_999) + expect(canRetryAlertDelivery(4, true)).toBe(true) + expect(canRetryAlertDelivery(5, true)).toBe(false) + expect(canRetryAlertDelivery(1, false)).toBe(false) + }) +}) diff --git a/packages/alerting-core/src/index.ts b/packages/alerting-core/src/index.ts new file mode 100644 index 000000000..4130d25b6 --- /dev/null +++ b/packages/alerting-core/src/index.ts @@ -0,0 +1,321 @@ +export type AlertComparator = "gt" | "gte" | "lt" | "lte" | "eq" | "neq" | "between" | "not_between" + +export type AlertEvaluationStatus = "breached" | "healthy" | "skipped" + +export interface AlertObservation { + readonly value: number | null + readonly sampleCount: number + readonly hasData: boolean +} + +export interface AlertEvaluationPolicy { + readonly comparator: AlertComparator + readonly threshold: number + readonly thresholdUpper: number | null + readonly minimumSampleCount: number + readonly noDataBehavior: "skip" | "zero" +} + +export interface AlertEvaluation { + readonly status: AlertEvaluationStatus + readonly value: number | null + readonly sampleCount: number + readonly threshold: number + readonly thresholdUpper: number | null + readonly comparator: AlertComparator + readonly reason: string + /** A healthy result derived from an empty window synthesized as zero. */ + readonly derivedFromNoData: boolean +} + +export const compareAlertThreshold = ( + value: number, + comparator: AlertComparator, + threshold: number, + thresholdUpper: number | null = null, +): boolean => { + switch (comparator) { + case "gt": + return value > threshold + case "gte": + return value >= threshold + case "lt": + return value < threshold + case "lte": + return value <= threshold + case "eq": + return value === threshold + case "neq": + return value !== threshold + case "between": + return thresholdUpper != null && value >= threshold && value <= thresholdUpper + case "not_between": + return thresholdUpper != null && (value < threshold || value > thresholdUpper) + } +} + +export const evaluateAlertObservation = ( + policy: AlertEvaluationPolicy, + observation: AlertObservation, + reason: string, +): AlertEvaluation => { + // Sample-weighted counts can be fractional while durable alert state commonly + // stores an integer. Normalize at the host-neutral boundary. + const sampleCount = Math.round(observation.sampleCount) + const value = observation.hasData ? observation.value : policy.noDataBehavior === "zero" ? 0 : null + + if (!observation.hasData && policy.noDataBehavior === "skip") { + return { + status: "skipped", + value: null, + sampleCount, + threshold: policy.threshold, + thresholdUpper: policy.thresholdUpper, + comparator: policy.comparator, + reason: "No data in the selected window", + derivedFromNoData: false, + } + } + + if (sampleCount < policy.minimumSampleCount) { + return { + status: "skipped", + value, + sampleCount, + threshold: policy.threshold, + thresholdUpper: policy.thresholdUpper, + comparator: policy.comparator, + reason: `Sample count ${sampleCount} is below minimum ${policy.minimumSampleCount}`, + derivedFromNoData: false, + } + } + + if (value == null) { + return { + status: "skipped", + value: null, + sampleCount, + threshold: policy.threshold, + thresholdUpper: policy.thresholdUpper, + comparator: policy.comparator, + reason: "Alert evaluation did not return a scalar value", + derivedFromNoData: false, + } + } + + return { + status: compareAlertThreshold(value, policy.comparator, policy.threshold, policy.thresholdUpper) + ? "breached" + : "healthy", + value, + sampleCount, + threshold: policy.threshold, + thresholdUpper: policy.thresholdUpper, + comparator: policy.comparator, + reason, + derivedFromNoData: !observation.hasData, + } +} + +export interface AlertLifecyclePolicy { + readonly consecutiveBreachesRequired: number + readonly consecutiveHealthyRequired: number + readonly renotifyIntervalMinutes: number +} + +export interface AlertLifecycleState { + readonly consecutiveBreaches: number + readonly consecutiveHealthy: number +} + +export interface AlertLifecycleIncident { + readonly firstTriggeredAtMs: number + readonly lastNotifiedAtMs: number | null + readonly lastDeliveredEventType: AlertEventType | null +} + +export type AlertEventType = "trigger" | "resolve" | "renotify" | "test" +export type AlertIncidentTransition = "none" | "opened" | "continued" | "resolved" +export type AlertNotificationSuppression = "flapping" | "flap_resolution" | null +export type AlertLifecycleHold = "missing_telemetry" | null + +export interface AlertLifecycleInput { + readonly policy: AlertLifecyclePolicy + readonly evaluation: AlertEvaluation + readonly state: AlertLifecycleState | null + readonly openIncident: AlertLifecycleIncident | null + readonly nowMs: number + /** Most recent notification for a resolved incident with the same rule and group. */ + readonly previousNotificationAtMs?: number | null + /** Set only after the host's telemetry-query adapter proves data is still arriving. */ + readonly allowNoDataResolution?: boolean +} + +export interface AlertLifecyclePlan { + readonly state: AlertLifecycleState + readonly transition: AlertIncidentTransition + readonly eventType: AlertEventType | null + readonly notificationSuppression: AlertNotificationSuppression + readonly hold: AlertLifecycleHold + /** Notification anchor to copy to a newly opened, flap-suppressed incident. */ + readonly inheritedNotificationAtMs: number | null + /** Whether the host must advance lastNotifiedAt before queueing the event. */ + readonly advanceNotificationAnchor: boolean +} + +export interface AlertDeliveryRetryPolicy { + readonly maxAttempts: number + readonly baseDelayMs: number + readonly maxDelayMs: number +} + +export const DEFAULT_ALERT_DELIVERY_RETRY_POLICY: AlertDeliveryRetryPolicy = { + maxAttempts: 5, + baseDelayMs: 60_000, + maxDelayMs: 15 * 60_000, +} + +/** Stable idempotency key shared by every alert delivery adapter. */ +export const makeAlertDeliveryKey = ( + incidentId: string, + destinationId: string, + eventType: AlertEventType, + scheduledAtMs: number, +): string => [incidentId, destinationId, eventType, scheduledAtMs].join(":") + +export const canRetryAlertDelivery = ( + attemptNumber: number, + retryable: boolean, + policy: AlertDeliveryRetryPolicy = DEFAULT_ALERT_DELIVERY_RETRY_POLICY, +): boolean => retryable && attemptNumber < policy.maxAttempts + +/** Exponential retry delay; the host supplies jitter from its own random source. */ +export const alertDeliveryRetryDelayMs = ( + attemptNumber: number, + jitterMs: number, + policy: AlertDeliveryRetryPolicy = DEFAULT_ALERT_DELIVERY_RETRY_POLICY, +): number => { + const exponent = Math.max(0, attemptNumber - 1) + const base = Math.min(policy.baseDelayMs * Math.pow(2, exponent), policy.maxDelayMs) + return base + Math.max(0, Math.floor(jitterMs)) +} + +const noTransition = (state: AlertLifecycleState, hold: AlertLifecycleHold = null): AlertLifecyclePlan => ({ + state, + transition: "none", + eventType: null, + notificationSuppression: null, + hold, + inheritedNotificationAtMs: null, + advanceNotificationAnchor: false, +}) + +/** + * Decide the next alert state and lifecycle intent without performing I/O. + * + * The caller owns persistence, incident identifiers, delivery, telemetry + * liveness checks, and time. This makes the same lifecycle semantics usable by + * the hosted PostgreSQL/Tinybird adapter and a future Maple Local adapter. + */ +export const planAlertLifecycle = (input: AlertLifecycleInput): AlertLifecyclePlan => { + const { evaluation, policy, openIncident, nowMs } = input + const previous = input.state ?? { consecutiveBreaches: 0, consecutiveHealthy: 0 } + + if (evaluation.status === "skipped") return noTransition(previous) + + const state: AlertLifecycleState = { + consecutiveBreaches: + evaluation.status === "breached" + ? Math.min(previous.consecutiveBreaches + 1, policy.consecutiveBreachesRequired) + : 0, + consecutiveHealthy: + evaluation.status === "healthy" + ? Math.min(previous.consecutiveHealthy + 1, policy.consecutiveHealthyRequired) + : 0, + } + + if ( + evaluation.status === "breached" && + openIncident == null && + state.consecutiveBreaches >= policy.consecutiveBreachesRequired + ) { + const previousNotificationAtMs = input.previousNotificationAtMs ?? null + const flapSuppressed = + previousNotificationAtMs != null && + previousNotificationAtMs >= nowMs - policy.renotifyIntervalMinutes * 60_000 + return { + state, + transition: "opened", + eventType: flapSuppressed ? null : "trigger", + notificationSuppression: flapSuppressed ? "flapping" : null, + hold: null, + inheritedNotificationAtMs: flapSuppressed ? previousNotificationAtMs : null, + advanceNotificationAnchor: false, + } + } + + if (evaluation.status === "breached" && openIncident != null) { + const renotifyDueAt = + (openIncident.lastNotifiedAtMs ?? openIncident.firstTriggeredAtMs) + + policy.renotifyIntervalMinutes * 60_000 + const renotifyDue = renotifyDueAt <= nowMs + return { + state, + transition: "continued", + eventType: renotifyDue ? "renotify" : null, + notificationSuppression: null, + hold: null, + inheritedNotificationAtMs: null, + advanceNotificationAnchor: renotifyDue, + } + } + + if ( + evaluation.status === "healthy" && + openIncident != null && + state.consecutiveHealthy >= policy.consecutiveHealthyRequired + ) { + if (evaluation.derivedFromNoData && input.allowNoDataResolution !== true) { + return noTransition(state, "missing_telemetry") + } + + const flapResolutionSuppressed = + openIncident.lastDeliveredEventType == null && openIncident.lastNotifiedAtMs != null + return { + state, + transition: "resolved", + eventType: flapResolutionSuppressed ? null : "resolve", + notificationSuppression: flapResolutionSuppressed ? "flap_resolution" : null, + hold: null, + inheritedNotificationAtMs: null, + advanceNotificationAnchor: false, + } + } + + return noTransition(state) +} + +/** Preserve per-tenant order while preventing one tenant from monopolizing a tick. */ +export const interleaveAlertRulesByTenant = ( + rows: ReadonlyArray, + tenantIdOf: (row: T) => string, +): ReadonlyArray => { + const queues = new Map() + for (const row of rows) { + const tenantId = tenantIdOf(row) + const queue = queues.get(tenantId) + if (queue) queue.push(row) + else queues.set(tenantId, [row]) + } + + const fair: T[] = [] + let index = 0 + while (fair.length < rows.length) { + for (const queue of queues.values()) { + const row = queue[index] + if (row !== undefined) fair.push(row) + } + index += 1 + } + return fair +} diff --git a/packages/alerting-core/tsconfig.json b/packages/alerting-core/tsconfig.json new file mode 100644 index 000000000..3d83a7d0c --- /dev/null +++ b/packages/alerting-core/tsconfig.json @@ -0,0 +1,17 @@ +{ + "include": ["**/*.ts"], + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "lib": ["ES2022"], + "types": ["node"], + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + "skipLibCheck": true, + "strict": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedSideEffectImports": true + } +} From 180a6efafe9fa2e0df5c5371f86bfd2391910698 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Fri, 7 Aug 2026 20:45:29 -0400 Subject: [PATCH 2/2] feat(eventing): add typed signal projection architecture --- apps/api/package.json | 1 + .../src/planetscale-webhook-runtime.test.ts | 42 +- apps/api/src/planetscale-webhook-runtime.ts | 13 +- .../v1/planetscale-webhook.http.test.ts | 11 +- .../src/routes/v1/planetscale-webhook.http.ts | 29 +- apps/api/src/services/alerts/AlertsService.ts | 66 +- .../PlanetScaleWebhookQueue.test.ts | 26 +- .../planetscale/PlanetScaleWebhookQueue.ts | 2 + .../planetscale/webhook-events.test.ts | 58 ++ .../planetscale/webhook-events.ts | 156 +++- apps/cli/package.json | 3 +- apps/cli/src/server/checkpoints.ts | 102 +- apps/cli/src/server/eventing/control-store.ts | 437 +++++++++ apps/cli/src/server/eventing/otlp.ts | 411 ++++++++ apps/cli/src/server/eventing/runtime.ts | 217 +++++ apps/cli/src/server/serve.ts | 166 +++- apps/cli/test/checkpoints.test.ts | 44 +- .../test/local-eventing-control-store.test.ts | 159 ++++ apps/cli/test/local-eventing-ingest.test.ts | 138 +++ apps/cli/test/local-eventing-runtime.test.ts | 203 ++++ apps/cli/test/server-network.test.ts | 3 +- bun.lock | 20 + docs/signal-to-event-projection.md | 881 ++++++++++++++++++ packages/alerting-core/README.md | 16 +- packages/alerting-core/package.json | 3 + packages/alerting-core/src/index.test.ts | 34 + packages/alerting-core/src/index.ts | 83 ++ packages/eventing-core/README.md | 22 + packages/eventing-core/fixtures/v1.json | 177 ++++ packages/eventing-core/package.json | 24 + .../schemas/cloud-event.v1.schema.json | 162 ++++ .../schemas/signal-projection.v1.schema.json | 349 +++++++ .../schemas/signal-scalar.v1.schema.json | 110 +++ .../eventing-core/scripts/generate-schemas.ts | 61 ++ packages/eventing-core/src/event.ts | 117 +++ packages/eventing-core/src/index.ts | 5 + packages/eventing-core/src/model.ts | 219 +++++ packages/eventing-core/src/predicate.test.ts | 148 +++ packages/eventing-core/src/predicate.ts | 341 +++++++ packages/eventing-core/src/registry.test.ts | 261 ++++++ packages/eventing-core/src/registry.ts | 184 ++++ packages/eventing-core/src/source.ts | 141 +++ packages/eventing-core/tsconfig.json | 23 + 43 files changed, 5578 insertions(+), 90 deletions(-) create mode 100644 apps/cli/src/server/eventing/control-store.ts create mode 100644 apps/cli/src/server/eventing/otlp.ts create mode 100644 apps/cli/src/server/eventing/runtime.ts create mode 100644 apps/cli/test/local-eventing-control-store.test.ts create mode 100644 apps/cli/test/local-eventing-ingest.test.ts create mode 100644 apps/cli/test/local-eventing-runtime.test.ts create mode 100644 docs/signal-to-event-projection.md create mode 100644 packages/eventing-core/README.md create mode 100644 packages/eventing-core/fixtures/v1.json create mode 100644 packages/eventing-core/package.json create mode 100644 packages/eventing-core/schemas/cloud-event.v1.schema.json create mode 100644 packages/eventing-core/schemas/signal-projection.v1.schema.json create mode 100644 packages/eventing-core/schemas/signal-scalar.v1.schema.json create mode 100644 packages/eventing-core/scripts/generate-schemas.ts create mode 100644 packages/eventing-core/src/event.ts create mode 100644 packages/eventing-core/src/index.ts create mode 100644 packages/eventing-core/src/model.ts create mode 100644 packages/eventing-core/src/predicate.test.ts create mode 100644 packages/eventing-core/src/predicate.ts create mode 100644 packages/eventing-core/src/registry.test.ts create mode 100644 packages/eventing-core/src/registry.ts create mode 100644 packages/eventing-core/src/source.ts create mode 100644 packages/eventing-core/tsconfig.json diff --git a/apps/api/package.json b/apps/api/package.json index dc3f6283d..807b3cbf6 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -39,6 +39,7 @@ "@maple/domain": "workspace:*", "@maple/effect-cloudflare": "workspace:*", "@maple/email": "workspace:*", + "@maple/eventing-core": "workspace:*", "@maple/infra": "workspace:*", "@maple/llm": "workspace:*", "@maple/query-engine": "workspace:*", diff --git a/apps/api/src/planetscale-webhook-runtime.test.ts b/apps/api/src/planetscale-webhook-runtime.test.ts index 6c877a323..858b76567 100644 --- a/apps/api/src/planetscale-webhook-runtime.test.ts +++ b/apps/api/src/planetscale-webhook-runtime.test.ts @@ -1,27 +1,41 @@ import type { MessageBatch } from "@cloudflare/workers-types" import { afterEach, assert, describe, it } from "@effect/vitest" -import { Effect, Layer } from "effect" +import { OrgId } from "@maple/domain/http" +import { Effect, Layer, Schema } from "effect" import { Database, DatabaseError } from "@/platform/DatabaseLive" import { cleanupTestDbs, createTestDb, queryFirstRow, type TestDb } from "@/platform/test-pglite" import { processPlanetScaleWebhookBatch } from "./planetscale-webhook-runtime" +import { projectPlanetScaleWebhookEvent } from "./services/integrations/planetscale/webhook-events" import type { PlanetScaleWebhookJob } from "./services/integrations/planetscale/PlanetScaleWebhookQueue" const trackedDbs: TestDb[] = [] afterEach(() => cleanupTestDbs(trackedDbs)) -const job: PlanetScaleWebhookJob = { - kind: "planetscale-webhook", - orgId: "org_1", - connectionId: "connection_1", - payload: { +const orgId = Schema.decodeUnknownSync(OrgId)("org_1") + +const makeJob = ( + payload: PlanetScaleWebhookJob["payload"] = { event: "branch.out_of_memory", organization: "acme", database: "shop", resource: { name: "main" }, }, +): PlanetScaleWebhookJob => ({ + kind: "planetscale-webhook", + orgId, + connectionId: "connection_1", + payload, receivedAt: 1_000, -} + event: projectPlanetScaleWebhookEvent({ + orgId, + connectionId: "connection_1", + payload, + receivedAt: 1_000, + }), +}) + +const job = makeJob() const makeBatch = (body: unknown) => { let acknowledged = false @@ -86,10 +100,7 @@ describe("PlanetScale webhook queue consumer", () => { it.effect("writes a lifecycle event to the timeline but not to the issue hub", () => { const testDb = createTestDb(trackedDbs) - const delivery = makeBatch({ - ...job, - payload: { ...job.payload, event: "branch.ready" }, - }) + const delivery = makeBatch(makeJob({ ...job.payload, event: "branch.ready" })) return Effect.gen(function* () { yield* processPlanetScaleWebhookBatch(delivery.batch) assert.isTrue(delivery.acknowledged()) @@ -138,14 +149,13 @@ describe("PlanetScale webhook queue consumer", () => { it.effect("carries the deploy-request number so redelivery dedupes", () => { const testDb = createTestDb(trackedDbs) - const delivery = makeBatch({ - ...job, - payload: { + const delivery = makeBatch( + makeJob({ ...job.payload, event: "deploy_request.schema_applied", resource: { number: 42 }, - }, - }) + }), + ) return Effect.gen(function* () { yield* processPlanetScaleWebhookBatch(delivery.batch) const event = yield* Effect.promise(() => diff --git a/apps/api/src/planetscale-webhook-runtime.ts b/apps/api/src/planetscale-webhook-runtime.ts index 71bc6c596..5a9e80fd2 100644 --- a/apps/api/src/planetscale-webhook-runtime.ts +++ b/apps/api/src/planetscale-webhook-runtime.ts @@ -54,9 +54,20 @@ export const processPlanetScaleWebhookBatch = (batch: MessageBatch) => ), ), onSuccess: (job) => { - const classified = classifyPlanetScaleEvent(job.payload.event) + const event = job.event + const eventData = + typeof event.data === "object" && + event.data !== null && + !Array.isArray(event.data) + ? (event.data as { readonly [key: string]: unknown }) + : null + const eventName = + typeof eventData?.event === "string" ? eventData.event : job.payload.event + const classified = classifyPlanetScaleEvent(eventName) const annotateJob = Effect.annotateCurrentSpan({ orgId: job.orgId, + "maple.event.id": event.id, + "maple.event.type": event.type, "maple.planetscale.connection_id": job.connectionId, "maple.planetscale.webhook.event": job.payload.event, }) diff --git a/apps/api/src/routes/v1/planetscale-webhook.http.test.ts b/apps/api/src/routes/v1/planetscale-webhook.http.test.ts index 3b22e91f4..3d5bd92ba 100644 --- a/apps/api/src/routes/v1/planetscale-webhook.http.test.ts +++ b/apps/api/src/routes/v1/planetscale-webhook.http.test.ts @@ -187,11 +187,13 @@ describe("PlanetScaleWebhookRouter", () => { assert.strictEqual(jobs[0]?.orgId, "org_1") assert.strictEqual(jobs[0]?.connectionId, CONNECTION_ID) assert.strictEqual(jobs[0]?.payload.event, "branch.out_of_memory") + assert.strictEqual(jobs[0]?.event.type, "dev.maple.planetscale.webhook.received.v1") + assert.strictEqual(jobs[0]?.event.tenantid, "org_1") }).pipe(Effect.ensuring(Effect.promise(dispose))) }).pipe(Effect.provide(testDb.layer)) }) - it.effect("enqueues lifecycle events too, and still drops genuinely unknown ones", () => { + it.effect("enqueues every verified factual event before downstream classification", () => { const testDb = createTestDb(trackedDbs) const jobs: PlanetScaleWebhookJob[] = [] return Effect.gen(function* () { @@ -260,15 +262,16 @@ describe("PlanetScaleWebhookRouter", () => { assert.strictEqual(branchReady.status, 202) assert.strictEqual(jobs.length, 2) - // Forward-compatibility must not become "enqueue everything": an - // event neither side knows is acknowledged and dropped. + // Unknown provider facts also enter the typed event layer. The current + // issue/timeline consumer may ignore them, but other consumers can opt in. const unknown = yield* post({ event: "branch.some_future_event", organization: "acme", database: "shop", }) assert.strictEqual(unknown.status, 202) - assert.strictEqual(jobs.length, 2) + assert.strictEqual(jobs.length, 3) + assert.strictEqual(jobs[2]?.payload.event, "branch.some_future_event") }).pipe(Effect.ensuring(Effect.promise(dispose))) }).pipe(Effect.provide(testDb.layer)) }) diff --git a/apps/api/src/routes/v1/planetscale-webhook.http.ts b/apps/api/src/routes/v1/planetscale-webhook.http.ts index d792a7505..9153c7898 100644 --- a/apps/api/src/routes/v1/planetscale-webhook.http.ts +++ b/apps/api/src/routes/v1/planetscale-webhook.http.ts @@ -9,6 +9,7 @@ import { Env } from "@/platform/Env" import { classifyPlanetScaleEvent, decodePlanetScaleWebhookPayload, + projectPlanetScaleWebhookEvent, verifyPlanetScaleSignature, } from "@/services/integrations/planetscale/webhook-events" import { PlanetScaleWebhookQueue } from "@/services/integrations/planetscale/PlanetScaleWebhookQueue" @@ -165,17 +166,33 @@ export const PlanetScaleWebhookRouter = HttpRouter.use((router) => return textResponse("ok", 200) } - // Both issue-worthy and timeline-only events go through the queue: the - // durable retry is what makes a missed deploy marker recoverable. - if (classified.action === "issue" || classified.action === "timeline") { + // Every verified factual event is normalized and projected before the + // durable queue boundary. The queued CloudEvent is the stable contract; + // payload remains temporarily for parity with the existing consumers. + { const now = yield* Clock.currentTimeMillis + const orgId = decodeOrgIdSync(connection.orgId) + const event = yield* Effect.try({ + try: () => + projectPlanetScaleWebhookEvent({ + orgId, + connectionId, + payload, + receivedAt: now, + }), + catch: () => + new PlanetScaleWebhookUnavailable({ + body: "Webhook event projection unavailable", + }), + }) const enqueued = yield* webhookQueue .send({ kind: "planetscale-webhook", - orgId: decodeOrgIdSync(connection.orgId), + orgId, connectionId, payload, receivedAt: now, + event, }) .pipe( Effect.tapError((error) => @@ -200,10 +217,6 @@ export const PlanetScaleWebhookRouter = HttpRouter.use((router) => event: payload.event, }), ) - } else { - yield* Effect.logInfo("PlanetScale webhook lifecycle event acknowledged").pipe( - Effect.annotateLogs({ orgId: connection.orgId, event: payload.event }), - ) } yield* Effect.annotateCurrentSpan({ diff --git a/apps/api/src/services/alerts/AlertsService.ts b/apps/api/src/services/alerts/AlertsService.ts index 19097030a..1e5be0656 100644 --- a/apps/api/src/services/alerts/AlertsService.ts +++ b/apps/api/src/services/alerts/AlertsService.ts @@ -6,6 +6,7 @@ import { interleaveAlertRulesByTenant, makeAlertDeliveryKey, planAlertLifecycle, + projectAlertLifecycleEvent, type AlertEvaluation as EvaluatedRule, type AlertLifecycleInput, } from "@maple/alerting-core" @@ -1715,7 +1716,25 @@ export class AlertsService extends Context.Service ({ + const buildPayload = (context: DeliveryPayloadContext, tenantId: string) => ({ + event: projectAlertLifecycleEvent({ + tenantId, + ruleId: context.ruleId, + ruleName: context.ruleName, + incidentId: context.incidentId, + eventType: context.eventType, + incidentStatus: context.incidentStatus, + groupKey: context.groupKey, + signalType: context.signalType, + severity: context.severity, + comparator: context.comparator, + threshold: context.threshold, + thresholdUpper: context.thresholdUpper, + windowMinutes: context.windowMinutes, + value: context.value, + sampleCount: context.sampleCount, + occurredAtMs: context.sentAtMs, + }), eventType: context.eventType, incidentId: context.incidentId, incidentStatus: context.incidentStatus, @@ -1783,7 +1802,7 @@ export class AlertsService extends Context.Service [row.id, row])) - const payload = buildPayload({ - eventType, - incidentId: incident.id, - incidentStatus: decodeAlertIncidentStatusSync(incident.status), - dedupeKey: incident.dedupeKey, - ruleId: rule.id, - ruleName: rule.name, - groupKey: incident.groupKey, - signalType: rule.signalType, - severity: rule.severity, - comparator: rule.comparator, - threshold: rule.threshold, - thresholdUpper: rule.thresholdUpper, - windowMinutes: rule.windowMinutes, - value: evaluation.value, - sampleCount: evaluation.sampleCount, - template: rule.notificationTemplate, - linkUrl: resolveNotificationLinkUrl(rule, incident.groupKey), - sentAtMs: scheduledAt, - }) + const payload = buildPayload( + { + eventType, + incidentId: incident.id, + incidentStatus: decodeAlertIncidentStatusSync(incident.status), + dedupeKey: incident.dedupeKey, + ruleId: rule.id, + ruleName: rule.name, + groupKey: incident.groupKey, + signalType: rule.signalType, + severity: rule.severity, + comparator: rule.comparator, + threshold: rule.threshold, + thresholdUpper: rule.thresholdUpper, + windowMinutes: rule.windowMinutes, + value: evaluation.value, + sampleCount: evaluation.sampleCount, + template: rule.notificationTemplate, + linkUrl: resolveNotificationLinkUrl(rule, incident.groupKey), + sentAtMs: scheduledAt, + }, + orgId, + ) yield* Effect.forEach( rule.destinationIds, diff --git a/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.test.ts b/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.test.ts index 0bd3f5c2f..a32063655 100644 --- a/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.test.ts +++ b/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.test.ts @@ -1,19 +1,29 @@ import { assert, describe, it } from "@effect/vitest" +import { OrgId } from "@maple/domain/http" import { WorkerEnvironment } from "@maple/effect-cloudflare" -import { Effect, Layer } from "effect" +import { Effect, Layer, Schema } from "effect" +import { projectPlanetScaleWebhookEvent } from "./webhook-events" import { PlanetScaleWebhookQueue, type PlanetScaleWebhookJob } from "./PlanetScaleWebhookQueue" +const orgId = Schema.decodeUnknownSync(OrgId)("org_1") +const payload = { + event: "branch.anomaly", + organization: "acme", + database: "shop", + resource: { name: "main" }, +} const job: PlanetScaleWebhookJob = { kind: "planetscale-webhook", - orgId: "org_1", + orgId, connectionId: "connection_1", - payload: { - event: "branch.anomaly", - organization: "acme", - database: "shop", - resource: { name: "main" }, - }, + payload, receivedAt: 1_000, + event: projectPlanetScaleWebhookEvent({ + orgId, + connectionId: "connection_1", + payload, + receivedAt: 1_000, + }), } const provideQueue = (environment: Record) => diff --git a/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.ts b/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.ts index bdf053340..5f6674ab1 100644 --- a/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.ts +++ b/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.ts @@ -1,6 +1,7 @@ import type { Queue } from "@cloudflare/workers-types" import { OrgId } from "@maple/domain/http" import { WorkerEnvironment } from "@maple/effect-cloudflare/worker-environment" +import { MapleCloudEventSchema } from "@maple/eventing-core" import { Context, Data, Effect, Layer, Schema } from "effect" import { PlanetScaleWebhookPayload } from "./webhook-events" @@ -12,6 +13,7 @@ export const PlanetScaleWebhookJob = Schema.Struct({ connectionId: Schema.String, payload: PlanetScaleWebhookPayload, receivedAt: Schema.Number, + event: MapleCloudEventSchema, }) export type PlanetScaleWebhookJob = Schema.Schema.Type diff --git a/apps/api/src/services/integrations/planetscale/webhook-events.test.ts b/apps/api/src/services/integrations/planetscale/webhook-events.test.ts index eb4a0f1d7..1a1383a1d 100644 --- a/apps/api/src/services/integrations/planetscale/webhook-events.test.ts +++ b/apps/api/src/services/integrations/planetscale/webhook-events.test.ts @@ -10,6 +10,7 @@ import { deployRequestNumber, insertPlanetScaleEvent, planetScaleIssueFingerprint, + projectPlanetScaleWebhookEvent, truncateToSecond, upsertPlanetScaleIssue, verifyPlanetScaleSignature, @@ -47,6 +48,63 @@ describe("verifyPlanetScaleSignature", () => { }) describe("classifyPlanetScaleEvent", () => { + it("normalizes queued webhooks into deterministic common CloudEvents", () => { + const payload = Schema.decodeUnknownSync(PlanetScaleWebhookPayload)(JSON.parse(OOM_PAYLOAD)) + const input = { + orgId: "org_events", + connectionId: "connection-1", + payload, + receivedAt: 1_698_252_880_000, + } + const event = projectPlanetScaleWebhookEvent(input) + assert.deepStrictEqual(event, projectPlanetScaleWebhookEvent(input)) + assert.strictEqual(event.type, "dev.maple.planetscale.webhook.received.v1") + assert.strictEqual(event.tenantid, "org_events") + assert.strictEqual(event.subject, "planetscale-databases/main-db") + assert.strictEqual((event.data as { readonly event: string }).event, "branch.out_of_memory") + assert.throws( + () => projectPlanetScaleWebhookEvent({ ...input, receivedAt: Number.MAX_SAFE_INTEGER }), + /outside the supported date range/, + ) + }) + + it("keeps source-timestamp identities stable and receipt-time fallbacks payload-consistent", () => { + const timestamped = Schema.decodeUnknownSync(PlanetScaleWebhookPayload)(JSON.parse(OOM_PAYLOAD)) + const first = projectPlanetScaleWebhookEvent({ + orgId: "org_events", + connectionId: "connection-1", + payload: timestamped, + receivedAt: 1_698_252_880_000, + }) + const redelivery = projectPlanetScaleWebhookEvent({ + orgId: "org_events", + connectionId: "connection-1", + payload: timestamped, + receivedAt: 1_698_252_990_000, + }) + assert.strictEqual(first.id, redelivery.id) + assert.strictEqual(first.time, redelivery.time) + + const withoutTimestamp = Schema.decodeUnknownSync(PlanetScaleWebhookPayload)({ + event: "branch.ready", + database: "main-db", + }) + const receivedFirst = projectPlanetScaleWebhookEvent({ + orgId: "org_events", + connectionId: "connection-1", + payload: withoutTimestamp, + receivedAt: 1_698_252_880_000, + }) + const receivedAgain = projectPlanetScaleWebhookEvent({ + orgId: "org_events", + connectionId: "connection-1", + payload: withoutTimestamp, + receivedAt: 1_698_252_990_000, + }) + assert.notStrictEqual(receivedFirst.id, receivedAgain.id) + assert.notStrictEqual(receivedFirst.time, receivedAgain.time) + }) + it("maps health events to issues and lifecycle events to timeline rows", () => { assert.strictEqual(classifyPlanetScaleEvent("branch.out_of_memory").action, "issue") assert.strictEqual(classifyPlanetScaleEvent("branch.anomaly").action, "issue") diff --git a/apps/api/src/services/integrations/planetscale/webhook-events.ts b/apps/api/src/services/integrations/planetscale/webhook-events.ts index d010ef600..f445d6da8 100644 --- a/apps/api/src/services/integrations/planetscale/webhook-events.ts +++ b/apps/api/src/services/integrations/planetscale/webhook-events.ts @@ -1,4 +1,15 @@ -import { createHmac, randomUUID, timingSafeEqual } from "node:crypto" +import { createHash, createHmac, randomUUID, timingSafeEqual } from "node:crypto" +import { + canonicalJson, + CompiledProjectionRegistry, + defineSignalFields, + ProjectorRegistry, + SignalSourceRegistry, + type JsonValue, + type MapleCloudEvent, + type SignalProjector, + type SignalSourceAdapter, +} from "@maple/eventing-core" import type { IssueSeverity, OrgId, WorkflowState } from "@maple/domain/http" import { ActorId, ErrorIssueEventId, ErrorIssueId } from "@maple/domain/primitives" import { @@ -57,6 +68,149 @@ export const decodePlanetScaleWebhookPayload = Schema.decodeUnknownEffect( Schema.fromJsonString(PlanetScaleWebhookPayload), ) +export interface PlanetScaleWebhookEventInput { + readonly orgId: string + readonly connectionId: string + readonly payload: PlanetScaleWebhookPayload + readonly receivedAt: number +} + +interface PlanetScaleWebhookAdapterInput { + readonly connectionId: string + readonly payload: PlanetScaleWebhookPayload +} + +interface PlanetScaleWebhookAdapterContext { + readonly tenantId: string + readonly acceptedAt: string +} + +const validDate = (epochMs: number, label: string): Date => { + if (!Number.isSafeInteger(epochMs) || epochMs < 0) + throw new Error(`${label} must be a non-negative epoch millisecond`) + const date = new Date(epochMs) + if (Number.isNaN(date.getTime())) throw new Error(`${label} is outside the supported date range`) + return date +} + +export const PLANETSCALE_WEBHOOK_ADAPTER: SignalSourceAdapter< + PlanetScaleWebhookAdapterInput, + PlanetScaleWebhookAdapterContext +> = { + definition: { + sourceKind: "planetscale.webhook", + fields: [ + { + field: { namespace: "signal", key: "event.name", type: "string" }, + operators: ["exists", "eq", "neq", "contains", "in"], + sensitivity: "public", + replay: "unavailable", + }, + ], + }, + normalize: ({ connectionId, payload }, context) => { + const observedAtDate = new Date(context.acceptedAt) + if (Number.isNaN(observedAtDate.getTime())) + throw new Error("PlanetScale receipt time is outside the supported date range") + const payloadJson = payload as unknown as JsonValue + const occurredAtMs = + payload.timestamp != null && payload.timestamp > 0 + ? Math.trunc(payload.timestamp * 1_000) + : observedAtDate.getTime() + const occurredAt = validDate(occurredAtMs, "PlanetScale event timestamp").toISOString() + const occurrenceId = `derived:sha256:${createHash("sha256") + .update(connectionId) + .update("\0") + .update(canonicalJson(payloadJson)) + .update("\0") + .update(occurredAt) + .digest("hex")}` + return [ + { + sourceKind: "planetscale.webhook", + source: `urn:maple:planetscale:${connectionId}`, + tenantId: context.tenantId, + occurrenceId, + identityQuality: "derived", + occurredAt, + observedAt: observedAtDate.toISOString(), + subject: + payload.database == null + ? `planetscale-connections/${connectionId}` + : `planetscale-databases/${payload.database}`, + fields: defineSignalFields([ + { + field: { namespace: "signal", key: "event.name", type: "string" }, + value: { type: "string", value: payload.event }, + }, + ]), + data: { + connectionId, + event: payload.event, + organization: payload.organization ?? null, + database: payload.database ?? null, + resource: (payload.resource ?? null) as JsonValue, + }, + }, + ] + }, +} + +const PLANETSCALE_WEBHOOK_PROJECTOR: SignalProjector> = { + id: "planetscale.webhook", + version: 1, + sourceKinds: ["planetscale.webhook"], + outputType: "dev.maple.planetscale.webhook.received.v1", + dataSchema: "urn:maple:event-schema:planetscale-webhook:v1", + decodeConfig: (value) => { + if ( + typeof value !== "object" || + value === null || + Array.isArray(value) || + Object.keys(value).length > 0 + ) + throw new Error("PlanetScale webhook projector config must be empty") + return {} + }, + project: (signal) => ({ data: signal.data as JsonValue }), +} + +const PLANETSCALE_SOURCES = new SignalSourceRegistry().register(PLANETSCALE_WEBHOOK_ADAPTER.definition) +const PLANETSCALE_PROJECTORS = new ProjectorRegistry().register(PLANETSCALE_WEBHOOK_PROJECTOR) + +/** Normalize and project one verified, durably queued PlanetScale webhook through the common layer. */ +export const projectPlanetScaleWebhookEvent = (input: PlanetScaleWebhookEventInput): MapleCloudEvent => { + const observedAt = validDate(input.receivedAt, "PlanetScale receipt time").toISOString() + const [signal] = PLANETSCALE_WEBHOOK_ADAPTER.normalize( + { connectionId: input.connectionId, payload: input.payload }, + { tenantId: input.orgId, acceptedAt: observedAt }, + ) + if (!signal) throw new Error("PlanetScale webhook adapter produced no signal") + const registry = CompiledProjectionRegistry.compile( + [ + { + id: "planetscale-webhook", + revision: 1, + enabled: true, + tenantId: input.orgId, + sourceKind: "planetscale.webhook", + selector: { + op: "exists", + field: { namespace: "signal", key: "event.name", type: "string" }, + }, + projector: { id: "planetscale.webhook", version: 1, config: {} }, + activeFrom: observedAt, + }, + ], + PLANETSCALE_SOURCES, + PLANETSCALE_PROJECTORS, + ) + const result = registry.evaluate(signal) + if (result.failures.length > 0) throw new Error(result.failures[0]!.message) + if (result.events.length !== 1) throw new Error("PlanetScale webhook projection produced no event") + return result.events[0]! +} + // --------------------------------------------------------------------------- // Classification // --------------------------------------------------------------------------- diff --git a/apps/cli/package.json b/apps/cli/package.json index 03c8ee02b..0cc45719c 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -14,13 +14,14 @@ "@effect/platform-bun": "catalog:effect", "@maple-dev/effect-sdk": "workspace:*", "@maple/domain": "workspace:*", + "@maple/eventing-core": "workspace:*", "@maple/query-engine": "workspace:*", "effect": "catalog:effect", "protobufjs": "^8.6.1" }, "devDependencies": { - "@effect/vitest": "4.0.0-beta.105", "@effect/language-service": "catalog:effect", + "@effect/vitest": "4.0.0-beta.105", "@types/bun": "^1.3.11", "@types/node": "catalog:tooling", "typescript": "catalog:tooling", diff --git a/apps/cli/src/server/checkpoints.ts b/apps/cli/src/server/checkpoints.ts index 32544c9dd..26fefd8e9 100644 --- a/apps/cli/src/server/checkpoints.ts +++ b/apps/cli/src/server/checkpoints.ts @@ -1,4 +1,4 @@ -import { randomUUID } from "node:crypto" +import { createHash, randomUUID } from "node:crypto" import { spawnSync } from "node:child_process" import { existsSync, lstatSync, readFileSync, rmSync, writeFileSync } from "node:fs" import { cp, lstat, mkdir, readFile, readdir, rm, stat } from "node:fs/promises" @@ -18,6 +18,11 @@ import { syncDirectory, syncTree, } from "./durable-files" +import { + eventingControlSnapshotPath, + LocalEventingControlStore, + type EventingControlSnapshotValidation, +} from "./eventing/control-store" import { SCHEMA_FINGERPRINT } from "./serve" import { CURRENT_LOCAL_SCHEMA } from "./schema-identity" import schemaSql from "./schema/local-schema.sql" with { type: "text" } @@ -29,12 +34,12 @@ import { } from "./store-version" const STATE_FORMAT_VERSION = 1 -const MANIFEST_FORMAT_VERSION = 1 +const MANIFEST_FORMAT_VERSION = 2 const OPERATION_FORMAT_VERSION = 1 const RESTORE_TRANSACTION_FORMAT_VERSION = 1 const RESET_TRANSACTION_FORMAT_VERSION = 1 const CHECKPOINT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i -const RESETTABLE_CHDB_ENTRIES = new Set(["data", "metadata", "status", "store", "tmp"]) +const RESETTABLE_LIVE_ENTRIES = new Set(["control", "data", "metadata", "status", "store", "tmp"]) export const CHECKPOINT_REOPEN_PROBE_ENV = "MAPLE_INTERNAL_CHECKPOINT_REOPEN_DATA_DIR" const CheckpointUuid = Schema.String.check(Schema.isPattern(CHECKPOINT_ID)) @@ -84,8 +89,7 @@ const CheckpointValidationSchema = Schema.Struct({ export type CheckpointValidation = Schema.Schema.Type -const CheckpointManifestSchema = Schema.Struct({ - formatVersion: Schema.Literal(MANIFEST_FORMAT_VERSION), +const CheckpointManifestFields = { checkpointId: CheckpointId, operationId: CheckpointOperationId, mapleVersion: Schema.String, @@ -96,8 +100,28 @@ const CheckpointManifestSchema = Schema.Struct({ backupRelativePath: Schema.String, backupBytes: NonNegativeInt, validation: CheckpointValidationSchema, +} as const + +const EventingControlSnapshotValidationSchema = Schema.Struct({ + schemaVersion: NonNegativeInt, + projectionRevisions: NonNegativeInt, + projectionFailures: NonNegativeInt, + stagedEvents: NonNegativeInt, + readyEvents: NonNegativeInt, }) +const CheckpointManifestSchema = Schema.Union([ + Schema.Struct({ formatVersion: Schema.Literal(1), ...CheckpointManifestFields }), + Schema.Struct({ + formatVersion: Schema.Literal(MANIFEST_FORMAT_VERSION), + ...CheckpointManifestFields, + controlRelativePath: Schema.String, + controlBytes: NonNegativeInt, + controlSha256: Schema.String.check(Schema.isPattern(/^[0-9a-f]{64}$/)), + controlValidation: EventingControlSnapshotValidationSchema, + }), +]) + export type CheckpointManifest = Schema.Schema.Type const CheckpointStateSchema = Schema.Struct({ @@ -125,7 +149,7 @@ const RestoreTransactionPhase = Schema.Literals([ "markers-committed", ]) const ResetTransactionPhase = Schema.Literals(["intent", "live-cleared", "markers-cleared"]) -const ResetTarget = Schema.Literals(["data", "metadata", "status", "store", "tmp"]) +const ResetTarget = Schema.Literals(["control", "data", "metadata", "status", "store", "tmp"]) const CheckpointOperationSchema = Schema.Struct({ formatVersion: Schema.Literal(OPERATION_FORMAT_VERSION), @@ -314,9 +338,23 @@ const snapshotManifestPath = (dataDir: string, checkpointId: CheckpointId): stri const snapshotBackupDir = (dataDir: string, checkpointId: CheckpointId): string => join(checkpointSnapshotDir(dataDir, checkpointId), "backup") const snapshotBackupRelativePath = (checkpointId: CheckpointId): string => `snapshots/${checkpointId}/backup` +const snapshotControlRelativePath = (checkpointId: CheckpointId): string => + `snapshots/${checkpointId}/control.sqlite` const snapshotBackupSqlPath = (checkpointId: CheckpointId): string => `backups/${snapshotBackupRelativePath(checkpointId)}` +const sha256File = (path: string): string => createHash("sha256").update(readFileSync(path)).digest("hex") + +const controlValidationMatches = ( + left: EventingControlSnapshotValidation, + right: EventingControlSnapshotValidation, +): boolean => + left.schemaVersion === right.schemaVersion && + left.projectionRevisions === right.projectionRevisions && + left.projectionFailures === right.projectionFailures && + left.stagedEvents === right.stagedEvents && + left.readyEvents === right.readyEvents + const assertContained = (root: string, candidate: string, label: string): string => { const absoluteRoot = resolve(root) const absoluteCandidate = resolve(candidate) @@ -667,6 +705,12 @@ export const parseCheckpointManifest = ( if (manifest.backupRelativePath !== snapshotBackupRelativePath(manifest.checkpointId)) { throw new Error("checkpoint backup path does not match its immutable ID") } + if ( + manifest.formatVersion === MANIFEST_FORMAT_VERSION && + manifest.controlRelativePath !== snapshotControlRelativePath(manifest.checkpointId) + ) { + throw new Error("checkpoint control-store path does not match its immutable ID") + } if (manifest.chdbVersion !== CHDB_VERSION) { throw new Error( `checkpoint chDB version mismatch (checkpoint: ${manifest.chdbVersion}; build: ${CHDB_VERSION})`, @@ -777,6 +821,24 @@ const resolveCheckpointById = async ( `checkpoint backup size mismatch (manifest: ${manifest.backupBytes}; actual: ${actualBackupBytes})`, ) } + const controlPath = eventingControlSnapshotPath(dataDir, checkpointId) + if (manifest.formatVersion === MANIFEST_FORMAT_VERSION) { + await assertNoSymlink(snapshotsRoot, controlPath) + await assertRealFile(controlPath, "checkpoint eventing control snapshot") + const controlBytes = (await stat(controlPath)).size + if (controlBytes !== manifest.controlBytes) + throw new Error( + `checkpoint control-store size mismatch (manifest: ${manifest.controlBytes}; actual: ${controlBytes})`, + ) + const controlSha256 = sha256File(controlPath) + if (controlSha256 !== manifest.controlSha256) + throw new Error("checkpoint control-store digest mismatch") + const controlValidation = LocalEventingControlStore.validateSnapshot(controlPath) + if (!controlValidationMatches(manifest.controlValidation, controlValidation)) + throw new Error("checkpoint control-store validation does not match its manifest") + } else if (existsSync(controlPath)) { + throw new Error("legacy checkpoint contains an unsigned eventing control snapshot") + } return { checkpointId, snapshotDir, @@ -823,6 +885,15 @@ const restoreResolvedInto = async ( `RESTORE DATABASE default FROM Disk('src', '${resolvedCheckpoint.backupSqlPath}') ` + "SETTINGS allow_different_database_def=1", ) + if (resolvedCheckpoint.manifest.formatVersion === MANIFEST_FORMAT_VERSION) { + await LocalEventingControlStore.restoreSnapshot( + join(resolvedCheckpoint.snapshotDir, "control.sqlite"), + targetDataDir, + ) + } else { + const controlStore = await LocalEventingControlStore.open(targetDataDir) + controlStore.close() + } return { db, validation: validateRestoredDatabase(db) } } catch (error) { db?.close() @@ -1524,10 +1595,14 @@ export const createCheckpoint = Effect.fn("CheckpointService.create")(function* const { oldState, snapshot, startedAt } = prepared let { operation } = prepared await syncTree(snapshotBackupDir(options.dataDir, checkpointId)) + const controlPath = eventingControlSnapshotPath(options.dataDir, checkpointId) + await assertNoSymlink(checkpointSnapshotsRoot(options.dataDir), controlPath) + await assertRealFile(controlPath, "checkpoint eventing control snapshot") + const controlValidation = LocalEventingControlStore.validateSnapshot(controlPath) operation = { ...operation, phase: "backup-complete" } await writeOperation(options.dataDir, operation, options.faults) const provisionalManifest: CheckpointManifest = { - formatVersion: 1, + formatVersion: MANIFEST_FORMAT_VERSION, checkpointId, operationId, mapleVersion: MAPLE_VERSION, @@ -1537,6 +1612,10 @@ export const createCheckpoint = Effect.fn("CheckpointService.create")(function* sourceDataDir: resolve(options.dataDir), backupRelativePath: snapshotBackupRelativePath(checkpointId), backupBytes: await dirSize(snapshotBackupDir(options.dataDir, checkpointId)), + controlRelativePath: snapshotControlRelativePath(checkpointId), + controlBytes: (await stat(controlPath)).size, + controlSha256: sha256File(controlPath), + controlValidation, validation: { validatedAt: startedAt, traces: 0, @@ -1707,7 +1786,7 @@ const beginResetTransactionUnlocked = async ( const entries = await readdir(live, { withFileTypes: true }) for (const entry of entries) { if (entry.name === "backups") continue - if (!RESETTABLE_CHDB_ENTRIES.has(entry.name)) { + if (!RESETTABLE_LIVE_ENTRIES.has(entry.name)) { unknown.push(join(live, entry.name)) continue } @@ -1955,9 +2034,10 @@ export const reconcileCheckpointRecovery = Effect.fn("CheckpointService.reconcil }) /** - * Explicitly remove the live chDB store while preserving the checkpoint - * registry below `/backups`. The maintenance lock serializes this - * destructive operation with checkpoint, restore, and archive work. + * Explicitly remove the live chDB and eventing control stores while preserving + * the checkpoint registry below `/backups`. The maintenance lock + * serializes this destructive operation with checkpoint, restore, and archive + * work. */ export const resetLiveStorePreservingCheckpoints = Effect.fn("CheckpointService.reset")(function* ( dataDir: string, diff --git a/apps/cli/src/server/eventing/control-store.ts b/apps/cli/src/server/eventing/control-store.ts new file mode 100644 index 000000000..df8780ef7 --- /dev/null +++ b/apps/cli/src/server/eventing/control-store.ts @@ -0,0 +1,437 @@ +import { constants as sqliteConstants, Database } from "bun:sqlite" +import { chmodSync, existsSync, lstatSync, readFileSync } from "node:fs" +import { join, resolve } from "node:path" +import { pathToFileURL } from "node:url" +import { + canonicalJson, + isJsonValue, + MapleCloudEventSchema, + SignalProjectionSpecSchema, + type MapleCloudEvent, + type JsonValue, + type ProjectionFailure, + type SignalProjectionSpec, +} from "@maple/eventing-core" +import { Schema } from "effect" +import { durableWrite, ensurePrivateDirectory } from "../durable-files" + +const CONTROL_SCHEMA_VERSION = 1 +const CONTROL_DIRECTORY = "control" +const CONTROL_DATABASE = "eventing.sqlite" +const MAX_EVENT_BYTES = 256 * 1024 +const MAX_FAILURES_PER_TENANT = 10_000 + +export const eventingControlDirectory = (dataDir: string): string => join(resolve(dataDir), CONTROL_DIRECTORY) +export const eventingControlPath = (dataDir: string): string => + join(eventingControlDirectory(dataDir), CONTROL_DATABASE) +export const eventingControlSnapshotPath = (dataDir: string, checkpointId: string): string => + join(resolve(dataDir), "backups", "snapshots", checkpointId, "control.sqlite") + +const CREATE_SCHEMA = ` +CREATE TABLE projection_revisions ( + tenant_id TEXT NOT NULL, + projection_id TEXT NOT NULL, + revision INTEGER NOT NULL CHECK (revision > 0), + enabled INTEGER NOT NULL CHECK (enabled IN (0, 1)), + spec_json TEXT NOT NULL, + created_at TEXT NOT NULL, + PRIMARY KEY (tenant_id, projection_id, revision) +) STRICT; + +CREATE TABLE active_projections ( + tenant_id TEXT NOT NULL, + projection_id TEXT NOT NULL, + revision INTEGER NOT NULL, + PRIMARY KEY (tenant_id, projection_id), + FOREIGN KEY (tenant_id, projection_id, revision) + REFERENCES projection_revisions (tenant_id, projection_id, revision) + ON DELETE RESTRICT +) STRICT; + +CREATE TABLE outbox_events ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + event_id TEXT NOT NULL UNIQUE, + tenant_id TEXT NOT NULL, + projection_id TEXT NOT NULL, + projection_revision INTEGER NOT NULL CHECK (projection_revision > 0), + state TEXT NOT NULL CHECK (state IN ('staged', 'ready')), + event_json TEXT NOT NULL, + staged_at TEXT NOT NULL, + ready_at TEXT +) STRICT; + +CREATE INDEX outbox_events_ready_sequence + ON outbox_events (state, sequence); + +CREATE TABLE projection_failures ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + tenant_id TEXT NOT NULL, + projection_id TEXT NOT NULL, + projection_revision INTEGER NOT NULL CHECK (projection_revision > 0), + occurrence_id TEXT, + message TEXT NOT NULL, + created_at TEXT NOT NULL +) STRICT; + +CREATE UNIQUE INDEX projection_failures_occurrence + ON projection_failures (tenant_id, projection_id, projection_revision, occurrence_id) + WHERE occurrence_id IS NOT NULL; + +PRAGMA user_version = 1; +` + +interface UserVersionRow { + readonly user_version: number | bigint +} + +interface RevisionRow { + readonly revision: number | bigint | null +} + +interface ProjectionJsonRow { + readonly spec_json: string +} + +interface EventRow { + readonly event_id: string + readonly event_json: string + readonly state: "staged" | "ready" +} + +interface EventJsonRow { + readonly event_json: string +} + +interface CountRow { + readonly count: number | bigint +} + +interface QuickCheckRow { + readonly quick_check: string +} + +export interface StageEventsResult { + readonly inserted: number + readonly deduplicated: number + readonly eventIds: readonly string[] +} + +export interface EventingControlSnapshotValidation { + readonly schemaVersion: number + readonly projectionRevisions: number + readonly projectionFailures: number + readonly stagedEvents: number + readonly readyEvents: number +} + +const asNumber = (value: number | bigint): number => { + const number = Number(value) + if (!Number.isSafeInteger(number) || number < 0) throw new Error(`invalid SQLite integer: ${value}`) + return number +} + +const decodeProjection = (json: string): SignalProjectionSpec => + Schema.decodeUnknownSync(SignalProjectionSpecSchema)(JSON.parse(json) as unknown) + +const decodeEvent = (json: string): MapleCloudEvent => { + const value = Schema.decodeUnknownSync(MapleCloudEventSchema)(JSON.parse(json) as unknown) + if (!isJsonValue(value.data)) throw new Error("stored CloudEvent data is not finite JSON") + return value as MapleCloudEvent +} + +const assertRealDatabaseFile = (path: string): void => { + let info + try { + info = lstatSync(path) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return + throw error + } + if (info.isSymbolicLink() || !info.isFile()) + throw new Error(`eventing control database is not a real file: ${path}`) +} + +const configure = (db: Database): void => { + db.exec("PRAGMA foreign_keys = ON") + db.exec("PRAGMA trusted_schema = OFF") + db.exec("PRAGMA busy_timeout = 5000") +} + +const validateOpenDatabase = (db: Database): EventingControlSnapshotValidation => { + const quick = db.query("PRAGMA quick_check").get() + if (quick?.quick_check !== "ok") throw new Error(`eventing control database quick_check failed`) + const version = db.query("PRAGMA user_version").get() + if (!version) throw new Error("eventing control database has no schema version") + const schemaVersion = asNumber(version.user_version) + if (schemaVersion !== CONTROL_SCHEMA_VERSION) + throw new Error( + `unsupported eventing control schema ${schemaVersion}; expected ${CONTROL_SCHEMA_VERSION}`, + ) + const count = (where: string): number => { + const row = db.query(`SELECT count(*) AS count FROM outbox_events ${where}`).get() + if (!row) throw new Error("eventing control count query returned no row") + return asNumber(row.count) + } + const revisions = db.query("SELECT count(*) AS count FROM projection_revisions").get() + if (!revisions) throw new Error("eventing projection count query returned no row") + const failures = db.query("SELECT count(*) AS count FROM projection_failures").get() + if (!failures) throw new Error("eventing projection-failure count query returned no row") + return { + schemaVersion, + projectionRevisions: asNumber(revisions.count), + projectionFailures: asNumber(failures.count), + stagedEvents: count("WHERE state = 'staged'"), + readyEvents: count("WHERE state = 'ready'"), + } +} + +export class LocalEventingControlStore { + readonly #db: Database + readonly path: string + + private constructor(path: string, db: Database) { + this.path = path + this.#db = db + } + + static async open(dataDir: string): Promise { + const directory = eventingControlDirectory(dataDir) + await ensurePrivateDirectory(directory) + const path = eventingControlPath(dataDir) + assertRealDatabaseFile(path) + const db = new Database(path, { create: true, readwrite: true, strict: true, safeIntegers: true }) + try { + configure(db) + db.exec("PRAGMA journal_mode = WAL") + db.exec("PRAGMA synchronous = FULL") + const version = db.query("PRAGMA user_version").get() + if (!version) throw new Error("eventing control database has no schema version") + const schemaVersion = asNumber(version.user_version) + if (schemaVersion === 0) db.transaction(() => db.exec(CREATE_SCHEMA)).exclusive() + else if (schemaVersion !== CONTROL_SCHEMA_VERSION) + throw new Error( + `unsupported eventing control schema ${schemaVersion}; expected ${CONTROL_SCHEMA_VERSION}`, + ) + chmodSync(path, 0o600) + validateOpenDatabase(db) + return new LocalEventingControlStore(path, db) + } catch (error) { + db.close() + throw error + } + } + + close(): void { + this.#db.exec("PRAGMA wal_checkpoint(TRUNCATE)") + this.#db.close(true) + } + + saveProjection(spec: SignalProjectionSpec, createdAt = new Date().toISOString()): void { + const decoded = Schema.decodeUnknownSync(SignalProjectionSpecSchema)(spec) + if (!isJsonValue(decoded as unknown)) throw new Error("projection spec must be finite JSON") + const specJson = canonicalJson(decoded as unknown as JsonValue) + this.#db + .transaction(() => { + const existing = this.#db + .query( + "SELECT spec_json FROM projection_revisions WHERE tenant_id = ? AND projection_id = ? AND revision = ?", + ) + .get(decoded.tenantId, decoded.id, decoded.revision) + if (existing) { + if (existing.spec_json !== specJson) + throw new Error( + `projection revision is immutable: ${decoded.tenantId}:${decoded.id}@${decoded.revision}`, + ) + } else { + const latest = this.#db + .query( + "SELECT max(revision) AS revision FROM projection_revisions WHERE tenant_id = ? AND projection_id = ?", + ) + .get(decoded.tenantId, decoded.id) + const expected = latest?.revision == null ? 1 : asNumber(latest.revision) + 1 + if (decoded.revision !== expected) + throw new Error( + `projection revision must be ${expected}: ${decoded.tenantId}:${decoded.id}@${decoded.revision}`, + ) + this.#db.run( + "INSERT INTO projection_revisions (tenant_id, projection_id, revision, enabled, spec_json, created_at) VALUES (?, ?, ?, ?, ?, ?)", + [ + decoded.tenantId, + decoded.id, + decoded.revision, + decoded.enabled ? 1 : 0, + specJson, + createdAt, + ], + ) + } + + if (decoded.enabled) + this.#db.run( + "INSERT INTO active_projections (tenant_id, projection_id, revision) VALUES (?, ?, ?) ON CONFLICT (tenant_id, projection_id) DO UPDATE SET revision = excluded.revision", + [decoded.tenantId, decoded.id, decoded.revision], + ) + else + this.#db.run("DELETE FROM active_projections WHERE tenant_id = ? AND projection_id = ?", [ + decoded.tenantId, + decoded.id, + ]) + }) + .immediate() + } + + loadEnabledProjections(tenantId: string): readonly SignalProjectionSpec[] { + return this.#db + .query( + `SELECT r.spec_json + FROM active_projections a + JOIN projection_revisions r + ON r.tenant_id = a.tenant_id + AND r.projection_id = a.projection_id + AND r.revision = a.revision + WHERE a.tenant_id = ? + ORDER BY a.projection_id`, + ) + .all(tenantId) + .map(({ spec_json }) => decodeProjection(spec_json)) + } + + stageEvents(events: readonly MapleCloudEvent[], stagedAt = new Date().toISOString()): StageEventsResult { + let inserted = 0 + let deduplicated = 0 + const eventIds: string[] = [] + this.#db + .transaction(() => { + for (const candidate of events) { + const event = Schema.decodeUnknownSync(MapleCloudEventSchema)(candidate) + if (!isJsonValue(event as unknown)) throw new Error("CloudEvent must be finite JSON") + const eventJson = canonicalJson(event as unknown as JsonValue) + if (Buffer.byteLength(eventJson, "utf8") > MAX_EVENT_BYTES) + throw new Error(`CloudEvent exceeds ${MAX_EVENT_BYTES} UTF-8 bytes`) + const existing = this.#db + .query( + "SELECT event_id, event_json, state FROM outbox_events WHERE event_id = ?", + ) + .get(event.id) + if (existing) { + if (existing.event_json !== eventJson) + throw new Error(`event ID collision with different payload: ${event.id}`) + deduplicated += 1 + } else { + this.#db.run( + "INSERT INTO outbox_events (event_id, tenant_id, projection_id, projection_revision, state, event_json, staged_at) VALUES (?, ?, ?, ?, 'staged', ?, ?)", + [ + event.id, + event.tenantid, + event.projectionid, + event.projectionrevision, + eventJson, + stagedAt, + ], + ) + inserted += 1 + } + eventIds.push(event.id) + } + }) + .immediate() + return { inserted, deduplicated, eventIds } + } + + markReady(eventIds: readonly string[], readyAt = new Date().toISOString()): void { + this.#db + .transaction(() => { + for (const eventId of eventIds) { + const row = this.#db + .query, [string]>( + "SELECT state FROM outbox_events WHERE event_id = ?", + ) + .get(eventId) + if (!row) throw new Error(`cannot mark unknown event ready: ${eventId}`) + if (row.state === "ready") continue + this.#db.run( + "UPDATE outbox_events SET state = 'ready', ready_at = ? WHERE event_id = ? AND state = 'staged'", + [readyAt, eventId], + ) + } + }) + .immediate() + } + + listReady(limit = 100): readonly MapleCloudEvent[] { + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) + throw new Error("ready-event limit must be between 1 and 1000") + return this.#db + .query( + "SELECT event_json FROM outbox_events WHERE state = 'ready' ORDER BY sequence LIMIT ?", + ) + .all(limit) + .map(({ event_json }) => decodeEvent(event_json)) + } + + listStaged(limit = 100): readonly MapleCloudEvent[] { + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) + throw new Error("staged-event limit must be between 1 and 1000") + return this.#db + .query( + "SELECT event_json FROM outbox_events WHERE state = 'staged' ORDER BY sequence LIMIT ?", + ) + .all(limit) + .map(({ event_json }) => decodeEvent(event_json)) + } + + recordProjectionFailures( + tenantId: string, + failures: readonly ProjectionFailure[], + createdAt = new Date().toISOString(), + ): void { + this.#db + .transaction(() => { + for (const failure of failures) + this.#db.run( + "INSERT OR IGNORE INTO projection_failures (tenant_id, projection_id, projection_revision, occurrence_id, message, created_at) VALUES (?, ?, ?, ?, ?, ?)", + [ + tenantId, + failure.projectionId, + failure.projectionRevision, + failure.occurrenceId, + failure.message.slice(0, 4_096), + createdAt, + ], + ) + this.#db.run( + "DELETE FROM projection_failures WHERE tenant_id = ? AND sequence NOT IN (SELECT sequence FROM projection_failures WHERE tenant_id = ? ORDER BY sequence DESC LIMIT ?)", + [tenantId, tenantId, MAX_FAILURES_PER_TENANT], + ) + }) + .immediate() + } + + validate(): EventingControlSnapshotValidation { + return validateOpenDatabase(this.#db) + } + + async backupTo(path: string): Promise { + const bytes = this.#db.serialize() + await durableWrite(path, bytes) + return LocalEventingControlStore.validateSnapshot(path) + } + + static validateSnapshot(path: string): EventingControlSnapshotValidation { + assertRealDatabaseFile(path) + if (!existsSync(path)) throw new Error(`eventing control snapshot is missing: ${path}`) + const uri = `${pathToFileURL(path).href}?immutable=1` + const db = new Database(uri, sqliteConstants.SQLITE_OPEN_READONLY | sqliteConstants.SQLITE_OPEN_URI) + try { + configure(db) + return validateOpenDatabase(db) + } finally { + db.close(true) + } + } + + static async restoreSnapshot(snapshotPath: string, dataDir: string): Promise { + LocalEventingControlStore.validateSnapshot(snapshotPath) + await durableWrite(eventingControlPath(dataDir), readFileSync(snapshotPath)) + } +} diff --git a/apps/cli/src/server/eventing/otlp.ts b/apps/cli/src/server/eventing/otlp.ts new file mode 100644 index 000000000..b93b4659c --- /dev/null +++ b/apps/cli/src/server/eventing/otlp.ts @@ -0,0 +1,411 @@ +import { createHash } from "node:crypto" +import { + canonicalJson, + defineSignalFields, + type JsonValue, + type NormalizedSignal, + type SignalFieldCatalogEntry, + type SignalScalar, + type SignalSourceAdapter, + type SignalSourceDefinition, +} from "@maple/eventing-core" +import { OtlpFieldError, spanIdHex, traceIdHex } from "../otlp/encode" + +interface AnyValue { + readonly stringValue?: string + readonly boolValue?: boolean + readonly intValue?: string | number + readonly doubleValue?: number + readonly bytesValue?: string + readonly arrayValue?: { readonly values?: readonly AnyValue[] } + readonly kvlistValue?: { readonly values?: readonly KeyValue[] } +} + +interface KeyValue { + readonly key?: string + readonly value?: AnyValue +} + +interface OtlpLogsRequest { + readonly resourceLogs?: readonly { + readonly resource?: { readonly attributes?: readonly KeyValue[] } + readonly scopeLogs?: readonly { + readonly scope?: { + readonly name?: string + readonly version?: string + readonly attributes?: readonly KeyValue[] + } + readonly logRecords?: readonly { + readonly timeUnixNano?: string | number + readonly observedTimeUnixNano?: string | number + readonly severityNumber?: number + readonly severityText?: string + readonly eventName?: string + readonly body?: AnyValue + readonly attributes?: readonly KeyValue[] + readonly traceId?: string + readonly spanId?: string + }[] + }[] + }[] +} + +const MAX_ATTRIBUTES = 256 +const MAX_STRING_BYTES = 16 * 1024 +const MAX_DATA_BYTES = 256 * 1024 +const MAX_VALUE_DEPTH = 8 +const MAX_VALUE_NODES = 1_024 +const SENSITIVE_KEY = + /(?:^|[._-])(authorization|cookie|password|passwd|secret|token|api[._-]?key)(?:$|[._-])/i + +const allOperators = ["exists", "eq", "neq", "gt", "gte", "lt", "lte", "contains", "in"] as const +const equalityOperators = ["exists", "eq", "neq", "contains", "in"] as const + +const catalog = ( + key: string, + type: SignalScalar["type"], + operators: SignalFieldCatalogEntry["operators"] = allOperators, +): SignalFieldCatalogEntry => ({ + field: { namespace: "signal", key, type }, + operators, + sensitivity: "public", + replay: "exact", +}) + +export const OTLP_LOG_SOURCE: SignalSourceDefinition = { + sourceKind: "otel.log", + fields: [ + catalog("event.name", "string", equalityOperators), + catalog("severity.number", "int64"), + catalog("severity.text", "string", equalityOperators), + catalog("trace.id", "string", equalityOperators), + catalog("span.id", "string", equalityOperators), + catalog("time", "timestamp"), + catalog("observed_time", "timestamp"), + ], + openFields: [ + { + namespace: "resource", + types: ["string", "boolean", "int64", "float64"], + operators: allOperators, + sensitivity: "public", + replay: "coerced", + }, + { + namespace: "scope", + types: ["string", "boolean", "int64", "float64"], + operators: allOperators, + sensitivity: "public", + replay: "coerced", + }, + { + namespace: "attribute", + types: ["string", "boolean", "int64", "float64"], + operators: allOperators, + sensitivity: "public", + replay: "coerced", + }, + { + namespace: "body", + types: ["string", "boolean", "int64", "float64"], + operators: allOperators, + sensitivity: "public", + replay: "coerced", + }, + ], +} + +interface ValueBudget { + nodes: number +} + +const assertStringBound = (value: string, label: string): string => { + if (Buffer.byteLength(value, "utf8") > MAX_STRING_BYTES) + throw new OtlpFieldError(`${label} exceeds ${MAX_STRING_BYTES} UTF-8 bytes`) + return value +} + +const int64 = (value: string | number, label: string): string => { + if (typeof value === "number" && !Number.isSafeInteger(value)) + throw new OtlpFieldError( + `${label} must encode int64 as a decimal string when outside safe integer range`, + ) + const decimal = String(value) + if (!/^-?(?:0|[1-9][0-9]*)$/.test(decimal)) throw new OtlpFieldError(`${label} is not an int64`) + const parsed = BigInt(decimal) + if (parsed < -(1n << 63n) || parsed > (1n << 63n) - 1n) + throw new OtlpFieldError(`${label} is outside the int64 range`) + return decimal +} + +const anyValueScalar = (value: AnyValue | undefined, label: string): SignalScalar | null => { + if (!value) return null + if (value.stringValue !== undefined) + return { type: "string", value: assertStringBound(value.stringValue, label) } + if (value.boolValue !== undefined) return { type: "boolean", value: value.boolValue } + if (value.intValue !== undefined) return { type: "int64", value: int64(value.intValue, label) } + if (value.doubleValue !== undefined) { + if (!Number.isFinite(value.doubleValue)) throw new OtlpFieldError(`${label} must be finite`) + return { type: "float64", value: value.doubleValue } + } + return null +} + +const anyValueJson = ( + value: AnyValue | undefined, + label: string, + depth = 0, + budget: ValueBudget = { nodes: 0 }, +): JsonValue | null => { + budget.nodes += 1 + if (budget.nodes > MAX_VALUE_NODES) throw new OtlpFieldError(`${label} exceeds value node limit`) + if (depth > MAX_VALUE_DEPTH) throw new OtlpFieldError(`${label} exceeds value depth limit`) + const scalar = anyValueScalar(value, label) + if (scalar) return scalar.value + if (!value) return null + if (value.bytesValue !== undefined) return assertStringBound(value.bytesValue, `${label}.bytesValue`) + if (value.arrayValue !== undefined) + return (value.arrayValue.values ?? []).map((item, index) => + anyValueJson(item, `${label}[${index}]`, depth + 1, budget), + ) + if (value.kvlistValue !== undefined) { + const output: Record = {} + for (const [index, entry] of (value.kvlistValue.values ?? []).entries()) { + const key = assertStringBound(entry.key ?? "", `${label}.key[${index}]`) + if (key.length === 0 || SENSITIVE_KEY.test(key)) continue + output[key] = anyValueJson(entry.value, `${label}.${key}`, depth + 1, budget) + } + return output + } + return null +} + +interface NormalizedAttributes { + readonly scalars: ReadonlyArray<{ readonly key: string; readonly value: SignalScalar }> + readonly data: Readonly> +} + +const attributes = (values: readonly KeyValue[] | undefined, label: string): NormalizedAttributes => { + if ((values?.length ?? 0) > MAX_ATTRIBUTES) + throw new OtlpFieldError(`${label} exceeds ${MAX_ATTRIBUTES} attributes`) + const scalars = new Map() + const data: Record = {} + for (const [index, entry] of (values ?? []).entries()) { + const key = assertStringBound(entry.key ?? "", `${label}[${index}].key`) + if (key.length === 0 || SENSITIVE_KEY.test(key)) continue + const scalar = anyValueScalar(entry.value, `${label}.${key}`) + if (scalar) scalars.set(key, scalar) + data[key] = anyValueJson(entry.value, `${label}.${key}`) + } + return { scalars: [...scalars].map(([key, value]) => ({ key, value })), data } +} + +const epochNanos = (value: string | number | undefined): bigint | null => { + if (value === undefined || value === "" || value === 0 || value === "0") return null + try { + const parsed = BigInt(value) + return parsed >= 0 ? parsed : null + } catch { + return null + } +} + +const nanosToTimestamp = (nanos: bigint): string => { + const seconds = nanos / 1_000_000_000n + const fraction = nanos % 1_000_000_000n + const milliseconds = Number(seconds) * 1_000 + const date = new Date(milliseconds) + if (!Number.isFinite(milliseconds) || Number.isNaN(date.getTime())) + throw new OtlpFieldError("OTLP timestamp is outside the supported date range") + return `${date.toISOString().slice(0, 19)}.${fraction.toString().padStart(9, "0")}Z` +} + +const stringAttribute = (attrs: NormalizedAttributes, key: string): string | null => { + const scalar = attrs.scalars.find((entry) => entry.key === key)?.value + return scalar?.type === "string" ? scalar.value : null +} + +const boundedIdentity = (value: string, prefix: string): string => + value.length <= 256 + ? value + : `${prefix}:sha256:${createHash("sha256").update(value, "utf8").digest("hex")}` + +const sourceUri = (resource: NormalizedAttributes, record: NormalizedAttributes): string => { + const explicit = stringAttribute(record, "event.source") ?? stringAttribute(record, "cloudevents.source") + if (explicit) return boundedIdentity(assertStringBound(explicit, "event source"), "urn:maple:source") + const service = stringAttribute(resource, "service.name") + const source = service + ? `urn:maple:source:otel:${encodeURIComponent(service)}` + : "urn:maple:source:otel:local" + return boundedIdentity(source, "urn:maple:source") +} + +const sourceOccurrenceId = (record: NormalizedAttributes): string | null => { + const value = + stringAttribute(record, "event.id") ?? + stringAttribute(record, "cloudevents.id") ?? + stringAttribute(record, "gitlab.event.id") + return value === null ? null : boundedIdentity(value, "source") +} + +const derivedOccurrenceId = (input: JsonValue): string => + `derived:sha256:${createHash("sha256").update(canonicalJson(input)).digest("hex")}` + +export const normalizeOtlpLogs = ( + request: unknown, + acceptedAt = new Date().toISOString(), + tenantId = "local", +): readonly NormalizedSignal[] => { + const input = (request ?? {}) as OtlpLogsRequest + const signals: NormalizedSignal[] = [] + for (const resourceLogs of input.resourceLogs ?? []) { + const resource = attributes(resourceLogs.resource?.attributes, "resource.attributes") + for (const scopeLogs of resourceLogs.scopeLogs ?? []) { + const scope = attributes(scopeLogs.scope?.attributes, "scope.attributes") + for (const log of scopeLogs.logRecords ?? []) { + const record = attributes(log.attributes, "log.attributes") + const occurredNanos = epochNanos(log.timeUnixNano) ?? epochNanos(log.observedTimeUnixNano) + const observedNanos = epochNanos(log.observedTimeUnixNano) + const occurredAt = occurredNanos ? nanosToTimestamp(occurredNanos) : acceptedAt + const sourceObservedAt = observedNanos ? nanosToTimestamp(observedNanos) : acceptedAt + const bodyScalar = anyValueScalar(log.body, "log.body") + const traceId = traceIdHex(log.traceId, "logRecord.traceId") + const spanId = spanIdHex(log.spanId, "logRecord.spanId") + const data: JsonValue = { + resource: resource.data, + scope: { + name: assertStringBound(scopeLogs.scope?.name ?? "", "scope.name"), + version: assertStringBound(scopeLogs.scope?.version ?? "", "scope.version"), + attributes: scope.data, + }, + record: { + eventName: assertStringBound(log.eventName ?? "", "log.eventName"), + severityNumber: log.severityNumber ?? 0, + severityText: assertStringBound(log.severityText ?? "", "log.severityText"), + traceId, + spanId, + body: anyValueJson(log.body, "log.body"), + attributes: record.data, + }, + } + if (Buffer.byteLength(canonicalJson(data), "utf8") > MAX_DATA_BYTES) + throw new OtlpFieldError(`normalized log event exceeds ${MAX_DATA_BYTES} UTF-8 bytes`) + const source = sourceUri(resource, record) + const occurrenceId = sourceOccurrenceId(record) + const subject = + stringAttribute(record, "event.subject") ?? stringAttribute(record, "cloudevents.subject") + signals.push({ + sourceKind: "otel.log", + source, + tenantId, + occurrenceId: + occurrenceId ?? + derivedOccurrenceId({ source, occurredAt, signalKind: "otel.log", data }), + identityQuality: occurrenceId ? "source" : "derived", + occurredAt, + observedAt: acceptedAt, + subject, + fields: defineSignalFields([ + ...(log.eventName + ? [ + { + field: { + namespace: "signal" as const, + key: "event.name", + type: "string" as const, + }, + value: { type: "string" as const, value: log.eventName }, + }, + ] + : []), + { + field: { namespace: "signal", key: "severity.number", type: "int64" }, + value: { + type: "int64", + value: int64(log.severityNumber ?? 0, "severity.number"), + }, + }, + ...(log.severityText + ? [ + { + field: { + namespace: "signal" as const, + key: "severity.text", + type: "string" as const, + }, + value: { type: "string" as const, value: log.severityText }, + }, + ] + : []), + ...(traceId + ? [ + { + field: { + namespace: "signal" as const, + key: "trace.id", + type: "string" as const, + }, + value: { type: "string" as const, value: traceId }, + }, + ] + : []), + ...(spanId + ? [ + { + field: { + namespace: "signal" as const, + key: "span.id", + type: "string" as const, + }, + value: { type: "string" as const, value: spanId }, + }, + ] + : []), + { + field: { namespace: "signal", key: "time", type: "timestamp" }, + value: { type: "timestamp", value: occurredAt }, + }, + { + field: { namespace: "signal", key: "observed_time", type: "timestamp" }, + value: { type: "timestamp", value: sourceObservedAt }, + }, + ...resource.scalars.map(({ key, value }) => ({ + field: { namespace: "resource" as const, key, type: value.type }, + value, + })), + ...scope.scalars.map(({ key, value }) => ({ + field: { namespace: "scope" as const, key, type: value.type }, + value, + })), + ...record.scalars.map(({ key, value }) => ({ + field: { namespace: "attribute" as const, key, type: value.type }, + value, + })), + ...(bodyScalar + ? [ + { + field: { + namespace: "body" as const, + key: "value", + type: bodyScalar.type, + }, + value: bodyScalar, + }, + ] + : []), + ]), + data, + }) + } + } + } + return signals +} + +export const OTLP_LOG_ADAPTER: SignalSourceAdapter< + unknown, + { readonly acceptedAt: string; readonly tenantId: string } +> = { + definition: OTLP_LOG_SOURCE, + normalize: (raw, context) => normalizeOtlpLogs(raw, context.acceptedAt, context.tenantId), +} diff --git a/apps/cli/src/server/eventing/runtime.ts b/apps/cli/src/server/eventing/runtime.ts new file mode 100644 index 000000000..2655c5865 --- /dev/null +++ b/apps/cli/src/server/eventing/runtime.ts @@ -0,0 +1,217 @@ +import { + CompiledProjectionRegistry, + ProjectorRegistry, + SignalSourceRegistry, + fieldKey, + isJsonValue, + SignalProjectionSpecSchema, + type MapleCloudEvent, + type NormalizedSignal, + type ProjectionFailure, + type SignalProjectionSpec, + type SignalScalar, +} from "@maple/eventing-core" +import { Schema } from "effect" +import { LocalEventingControlStore } from "./control-store" +import { OTLP_LOG_ADAPTER } from "./otlp" + +const TENANT_ID = "local" + +interface GitLabIssueProjectorConfig { + readonly includeBody: boolean +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const gitlabProjectorConfig = (value: unknown): GitLabIssueProjectorConfig => { + if (!isRecord(value)) throw new Error("gitlab.issue.created projector config must be an object") + const keys = Object.keys(value) + if (keys.some((key) => key !== "includeBody")) + throw new Error("gitlab.issue.created projector config contains an unknown field") + if (value.includeBody !== undefined && typeof value.includeBody !== "boolean") + throw new Error("gitlab.issue.created includeBody must be boolean") + return { includeBody: value.includeBody === true } +} + +const field = (signal: NormalizedSignal, namespace: "resource" | "attribute", key: string) => + signal.fields.get(fieldKey({ namespace, key })) + +const scalarString = (value: SignalScalar | undefined, label: string, required = false) => { + if (value === undefined) { + if (required) throw new Error(`GitLab issue event is missing ${label}`) + return undefined + } + if (value.type !== "string") throw new Error(`GitLab issue event ${label} must be a string`) + return value.value +} + +const scalarInt64 = (value: SignalScalar | undefined, label: string, required = false) => { + if (value === undefined) { + if (required) throw new Error(`GitLab issue event is missing ${label}`) + return undefined + } + if (value.type !== "int64") throw new Error(`GitLab issue event ${label} must be an int64`) + return value.value +} + +const gitlabIssueProjector = { + id: "gitlab.issue.created", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.issue.created.v1", + dataSchema: "urn:maple:event-schema:gitlab-issue-created:v1", + decodeConfig: gitlabProjectorConfig, + project: (signal: NormalizedSignal, config: GitLabIssueProjectorConfig) => { + const projectId = scalarInt64(field(signal, "attribute", "gitlab.project.id"), "gitlab.project.id") + const projectPath = scalarString( + field(signal, "attribute", "gitlab.project.path"), + "gitlab.project.path", + true, + )! + const issueId = scalarInt64(field(signal, "attribute", "gitlab.issue.id"), "gitlab.issue.id") + const issueIid = scalarInt64( + field(signal, "attribute", "gitlab.issue.iid"), + "gitlab.issue.iid", + true, + )! + const title = scalarString(field(signal, "attribute", "gitlab.issue.title"), "gitlab.issue.title") + const url = scalarString(field(signal, "attribute", "gitlab.issue.url"), "gitlab.issue.url") + const actorId = scalarInt64(field(signal, "attribute", "gitlab.user.id"), "gitlab.user.id") + const actorUsername = scalarString( + field(signal, "attribute", "gitlab.user.username"), + "gitlab.user.username", + ) + const serviceName = scalarString(field(signal, "resource", "service.name"), "service.name") + const candidateBody = + isRecord(signal.data) && isRecord(signal.data.record) ? signal.data.record.body : undefined + const body = isJsonValue(candidateBody) ? candidateBody : undefined + return { + subject: `${projectPath}/issues/${issueIid}`, + data: { + project: { + ...(projectId === undefined ? {} : { id: projectId }), + path: projectPath, + }, + issue: { + ...(issueId === undefined ? {} : { id: issueId }), + iid: issueIid, + ...(title === undefined ? {} : { title }), + ...(url === undefined ? {} : { url }), + }, + ...(actorId === undefined && actorUsername === undefined + ? {} + : { + actor: { + ...(actorId === undefined ? {} : { id: actorId }), + ...(actorUsername === undefined ? {} : { username: actorUsername }), + }, + }), + ...(serviceName === undefined ? {} : { serviceName }), + ...(config.includeBody && body !== undefined ? { body } : {}), + }, + } + }, +} as const + +export interface LocalProjectionEvaluation { + readonly events: readonly MapleCloudEvent[] + readonly failures: readonly ProjectionFailure[] + readonly typeMismatchFields: readonly string[] +} + +const emptyEvaluation = (): LocalProjectionEvaluation => ({ + events: [], + failures: [], + typeMismatchFields: [], +}) + +export class LocalEventingRuntime { + readonly #store: LocalEventingControlStore + readonly #sources: SignalSourceRegistry + readonly #projectors: ProjectorRegistry + #compiled: CompiledProjectionRegistry + #activeSourceKinds = new Set() + + constructor(store: LocalEventingControlStore) { + this.#store = store + this.#sources = new SignalSourceRegistry().register(OTLP_LOG_ADAPTER.definition) + this.#projectors = new ProjectorRegistry().register(gitlabIssueProjector) + const specs = store.loadEnabledProjections(TENANT_ID) + this.#compiled = CompiledProjectionRegistry.compile(specs, this.#sources, this.#projectors) + this.#activeSourceKinds = new Set(specs.map(({ sourceKind }) => sourceKind)) + } + + hasActiveSource(sourceKind: string): boolean { + return this.#activeSourceKinds.has(sourceKind) + } + + activate(candidate: unknown): void { + const spec = Schema.decodeUnknownSync(SignalProjectionSpecSchema)(candidate) + if (spec.tenantId !== TENANT_ID) + throw new Error(`Maple Local only accepts projections for tenant ${TENANT_ID}`) + const active = this.#store + .loadEnabledProjections(TENANT_ID) + .filter((candidate) => candidate.id !== spec.id) + const next = spec.enabled ? [...active, spec] : active + const compiled = CompiledProjectionRegistry.compile(next, this.#sources, this.#projectors) + this.#store.saveProjection(spec) + this.#compiled = compiled + this.#activeSourceKinds = new Set(next.map(({ sourceKind }) => sourceKind)) + } + + listActive(): readonly SignalProjectionSpec[] { + return this.#store.loadEnabledProjections(TENANT_ID) + } + + evaluateOtlp( + signal: "traces" | "logs" | "metrics", + decoded: unknown, + isRetiredUtcDay: (rangeDate: string) => boolean = () => false, + ): LocalProjectionEvaluation { + const sourceKind = signal === "logs" ? "otel.log" : signal === "traces" ? "otel.span" : "otel.metric" + if (!this.hasActiveSource(sourceKind)) return emptyEvaluation() + const acceptedAt = new Date().toISOString() + const normalized = ( + signal === "logs" ? OTLP_LOG_ADAPTER.normalize(decoded, { acceptedAt, tenantId: TENANT_ID }) : [] + ).filter((occurrence) => !isRetiredUtcDay(occurrence.occurredAt.slice(0, 10))) + const snapshot = this.#compiled + const events: MapleCloudEvent[] = [] + const failures: ProjectionFailure[] = [] + const typeMismatchFields = new Set() + for (const occurrence of normalized) { + const result = snapshot.evaluate(occurrence) + events.push(...result.events) + failures.push(...result.failures) + for (const mismatch of result.typeMismatchFields) typeMismatchFields.add(mismatch) + } + return { events, failures, typeMismatchFields: [...typeMismatchFields] } + } + + persistFailures(failures: readonly ProjectionFailure[]): void { + if (failures.length > 0) this.#store.recordProjectionFailures(TENANT_ID, failures) + } + + stage(events: readonly MapleCloudEvent[]) { + return this.#store.stageEvents(events) + } + + markReady(eventIds: readonly string[]): void { + this.#store.markReady(eventIds) + } + + listReady(limit?: number): readonly MapleCloudEvent[] { + return this.#store.listReady(limit) + } + + listStaged(limit?: number): readonly MapleCloudEvent[] { + return this.#store.listStaged(limit) + } + + health() { + return { + activeProjections: this.listActive().length, + ...this.#store.validate(), + } + } +} diff --git a/apps/cli/src/server/serve.ts b/apps/cli/src/server/serve.ts index 8f98df7d7..9c7f7b0bd 100644 --- a/apps/cli/src/server/serve.ts +++ b/apps/cli/src/server/serve.ts @@ -17,6 +17,8 @@ import { rawTelemetryTtlStatements, } from "./chdb" import { buildInsertStatements } from "./inserts" +import { eventingControlSnapshotPath, LocalEventingControlStore } from "./eventing/control-store" +import { LocalEventingRuntime } from "./eventing/runtime" import { encodeLogs, encodeMetrics, encodeTraces, type EncodedBatch, OtlpFieldError } from "./otlp/encode" import { decodeLogsRequest, @@ -102,7 +104,7 @@ export const corsHeadersForAllowedOrigin = ( ? { "access-control-allow-origin": origin, "access-control-allow-methods": "GET, POST, OPTIONS", - "access-control-allow-headers": "content-type, content-encoding", + "access-control-allow-headers": "content-type, content-encoding, x-maple-maintenance-token", "access-control-allow-private-network": "true", vary: "Origin", } @@ -170,6 +172,7 @@ interface IngestResult { async function ingest( db: Chdb, authority: RetiredDayAuthority, + eventing: LocalEventingRuntime, signal: Signal, req: Request, ): Promise { @@ -199,6 +202,17 @@ async function ingest( requestBytes, } } + let evaluation: ReturnType + try { + evaluation = eventing.evaluateOtlp(signal, decoded, (rangeDate) => authority.isRetired(rangeDate)) + } catch (error) { + const status = error instanceof OtlpFieldError ? 400 : 503 + return { + response: text(`event projection ${signal}: ${(error as Error).message}`, status), + accepted: 0, + requestBytes, + } + } let batches: EncodedBatch[] try { batches = encodeFor(signal, decoded) @@ -213,6 +227,18 @@ async function ingest( requestBytes, } } + let stagedEventIds: readonly string[] = [] + try { + eventing.persistFailures(evaluation.failures) + if (evaluation.events.length > 0) stagedEventIds = eventing.stage(evaluation.events).eventIds + } catch (error) { + const status = error instanceof OtlpFieldError ? 400 : 503 + return { + response: text(`event projection ${signal}: ${(error as Error).message}`, status), + accepted: 0, + requestBytes, + } + } let rejected = 0 batches = batches.map((batch) => { const filtered = authority.filterBatch(batch.datasource, batch.ndjson) @@ -235,6 +261,15 @@ async function ingest( accepted += statement.rowCount } } + try { + if (stagedEventIds.length > 0) eventing.markReady(stagedEventIds) + } catch (error) { + return { + response: text(`event outbox readiness ${signal}: ${(error as Error).message}`, 503), + accepted, + requestBytes, + } + } const errorMessage = rejected > 0 ? "telemetry from permanently retired UTC days was rejected" : "" if (contentType.includes("json")) { const rejectedField = @@ -394,6 +429,7 @@ const ingestSpan = ( runSpan: SpanRunner, db: Chdb, authority: RetiredDayAuthority, + eventing: LocalEventingRuntime, signal: Signal, req: Request, ): Promise => @@ -401,7 +437,7 @@ const ingestSpan = ( recoverResponse( Effect.gen(function* () { const { response, accepted, requestBytes } = yield* Effect.promise(() => - ingest(db, authority, signal, req), + ingest(db, authority, eventing, signal, req), ) yield* Effect.annotateCurrentSpan({ "http.request.body.size": requestBytes, @@ -539,7 +575,13 @@ const handleRetirement = async ( const CHECKPOINT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i /** Typed, authenticated replacement for sending BACKUP through /local/query. */ -const handleCheckpointBackup = async (db: Chdb, token: string, req: Request): Promise => { +const handleCheckpointBackup = async ( + db: Chdb, + controlStore: LocalEventingControlStore, + dataDir: string, + token: string, + req: Request, +): Promise => { if (!maintenanceTokenMatches(token, req.headers.get("x-maple-maintenance-token"))) return text("maintenance authorization required", 403) let body: unknown @@ -554,10 +596,10 @@ const handleCheckpointBackup = async (db: Chdb, token: string, req: Request): Pr return text("invalid checkpoint fields", 400) if (!CHECKPOINT_ID.test(record.checkpointId)) return text("invalid checkpoint ID", 400) try { - db.exec( - `BACKUP DATABASE default TO Disk('default', 'backups/snapshots/${record.checkpointId.toLowerCase()}/backup')`, - ) - return json({ checkpointId: record.checkpointId.toLowerCase() }) + const checkpointId = record.checkpointId.toLowerCase() + const control = await controlStore.backupTo(eventingControlSnapshotPath(dataDir, checkpointId)) + db.exec(`BACKUP DATABASE default TO Disk('default', 'backups/snapshots/${checkpointId}/backup')`) + return json({ checkpointId, control }) } catch (error) { return text( `checkpoint backup failed: ${error instanceof Error ? error.message : String(error)}`, @@ -566,6 +608,60 @@ const handleCheckpointBackup = async (db: Chdb, token: string, req: Request): Pr } } +const eventingAuthorized = (token: string, req: Request): Response | null => + maintenanceTokenMatches(token, req.headers.get("x-maple-maintenance-token")) + ? null + : text("maintenance authorization required", 403) + +const handleProjectionActivation = async ( + eventing: LocalEventingRuntime, + token: string, + req: Request, +): Promise => { + const unauthorized = eventingAuthorized(token, req) + if (unauthorized) return unauthorized + let body: unknown + try { + body = await req.json() + } catch { + return text("invalid JSON body", 400) + } + try { + eventing.activate(body) + return json({ active: eventing.listActive() }) + } catch (error) { + return text( + `invalid event projection: ${error instanceof Error ? error.message : String(error)}`, + 400, + ) + } +} + +const handleEventingRead = ( + eventing: LocalEventingRuntime, + token: string, + req: Request, + url: URL, +): Response => { + const unauthorized = eventingAuthorized(token, req) + if (unauthorized) return unauthorized + if (url.pathname === "/local/eventing/health") return json(eventing.health()) + if (url.pathname === "/local/eventing/projections") return json(eventing.listActive()) + if (url.pathname === "/local/eventing/outbox") { + const rawLimit = url.searchParams.get("limit") + const limit = rawLimit === null ? 100 : Number(rawLimit) + const state = url.searchParams.get("state") ?? "ready" + try { + if (state === "ready") return json(eventing.listReady(limit)) + if (state === "staged") return json(eventing.listStaged(limit)) + return text("outbox state must be ready or staged", 400) + } catch (error) { + return text(error instanceof Error ? error.message : String(error), 400) + } + } + return text("not found", 404) +} + /** The `Bun.serve` fetch handler, closed over the chDB connection. Each ingest * and query request is run through `runSpan` so it leaves a trace; `/health` * and `OPTIONS` are skipped (loop-prevention convention — no health-check noise). */ @@ -577,6 +673,8 @@ const makeFetch = authority: RetiredDayAuthority, gate: RequestQuiescenceGate, maintenanceToken: string, + controlStore: LocalEventingControlStore, + eventing: LocalEventingRuntime, ) => async (req: Request): Promise => { const url = new URL(req.url) @@ -590,18 +688,34 @@ const makeFetch = if (url.pathname === "/health") return respond(text("OK")) if (req.method === "POST") { if (url.pathname === "/v1/traces") - return respond(await admitted(gate, () => ingestSpan(runSpan, db, authority, "traces", req))) + return respond( + await admitted(gate, () => ingestSpan(runSpan, db, authority, eventing, "traces", req)), + ) if (url.pathname === "/v1/logs") - return respond(await admitted(gate, () => ingestSpan(runSpan, db, authority, "logs", req))) + return respond( + await admitted(gate, () => ingestSpan(runSpan, db, authority, eventing, "logs", req)), + ) if (url.pathname === "/v1/metrics") - return respond(await admitted(gate, () => ingestSpan(runSpan, db, authority, "metrics", req))) + return respond( + await admitted(gate, () => ingestSpan(runSpan, db, authority, eventing, "metrics", req)), + ) if (url.pathname === "/local/query") return respond(await admitted(gate, () => querySpan(runSpan, db, authority, req))) if (url.pathname === "/local/checkpoint/backup") - return respond(await admitted(gate, () => handleCheckpointBackup(db, maintenanceToken, req))) + return respond( + await gate.exclusive(() => + handleCheckpointBackup(db, controlStore, options.dataDir, maintenanceToken, req), + ), + ) + if (url.pathname === "/local/eventing/projections") + return respond( + await gate.exclusive(() => handleProjectionActivation(eventing, maintenanceToken, req)), + ) if (url.pathname === "/local/retention/retire") return respond(await handleRetirement(db, authority, gate, maintenanceToken, req)) } + if (req.method === "GET" && url.pathname.startsWith("/local/eventing/")) + return respond(handleEventingRead(eventing, maintenanceToken, req, url)) if (req.method === "GET" && options.assets) return respond(serveAsset(options.assets, url.pathname)) return respond(text("not found", 404)) } @@ -637,6 +751,23 @@ export const startServer = ( configFile: options.configFile, rawTelemetryRetentionDays: retention.effective, }) + const controlStore = yield* Effect.acquireRelease( + Effect.tryPromise({ + try: () => LocalEventingControlStore.open(options.dataDir), + catch: (error) => + new ChdbError({ + message: `failed to open local eventing control store: ${error instanceof Error ? error.message : String(error)}`, + }), + }), + (store) => Effect.sync(() => store.close()), + ) + const eventing = yield* Effect.try({ + try: () => new LocalEventingRuntime(controlStore), + catch: (error) => + new ChdbError({ + message: `failed to compile local event projections: ${error instanceof Error ? error.message : String(error)}`, + }), + }) // `CREATE ... IF NOT EXISTS` does not repair a table whose physical // definition was altered out of band. Inspect the opened store before the // listener is bound; a mismatch fails startup rather than allowing new @@ -711,7 +842,16 @@ export const startServer = ( Bun.serve({ port: options.port, hostname: options.hostname, - fetch: makeFetch(db, options, runSpan, authority, gate, maintenanceToken), + fetch: makeFetch( + db, + options, + runSpan, + authority, + gate, + maintenanceToken, + controlStore, + eventing, + ), }), catch: (error) => new ServerBindError({ @@ -725,4 +865,4 @@ export const startServer = ( return { port: server.port ?? options.port } }) -export const __testables = { recordServerResponse } +export const __testables = { handleEventingRead, ingest, recordServerResponse } diff --git a/apps/cli/test/checkpoints.test.ts b/apps/cli/test/checkpoints.test.ts index d06451b9c..da51a468a 100644 --- a/apps/cli/test/checkpoints.test.ts +++ b/apps/cli/test/checkpoints.test.ts @@ -1,4 +1,5 @@ import { describe, it } from "@effect/vitest" +import { createHash } from "node:crypto" import { Effect, Exit, Option } from "effect" import { deepStrictEqual, match, ok, rejects, strictEqual, throws } from "node:assert" import { @@ -54,6 +55,7 @@ import { import { SCHEMA_FINGERPRINT } from "../src/server/serve" import { storeMarkerPath, storeOpenMarkerPath } from "../src/server/store-version" import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" +import { eventingControlSnapshotPath, LocalEventingControlStore } from "../src/server/eventing/control-store" const withDataDir = async (run: (dataDir: string) => Promise | void): Promise => { const parent = mkdtempSync(join(tmpdir(), "maple-checkpoint-test-")) @@ -306,6 +308,41 @@ describe("checkpoint IDs and strict parsers", () => { }) describe("checkpoint state resolution", () => { + it("binds a version-2 checkpoint to its eventing control snapshot", async () => { + await withDataDir(async (dataDir) => { + const checkpointId = newCheckpointId() + const operationId = newCheckpointOperationId() + const snapshot = checkpointSnapshotDir(dataDir, checkpointId) + mkdirSync(join(snapshot, "backup"), { recursive: true }) + writeFileSync(join(snapshot, "backup", "data.bin"), "backup") + + const store = await LocalEventingControlStore.open(dataDir) + const controlPath = eventingControlSnapshotPath(dataDir, checkpointId) + const controlValidation = await store.backupTo(controlPath) + store.close() + const controlBytes = readFileSync(controlPath) + writeFileSync( + join(snapshot, "manifest.json"), + `${JSON.stringify({ + ...manifest(checkpointId, operationId, dataDir), + formatVersion: 2, + backupBytes: 6, + controlRelativePath: `snapshots/${checkpointId}/control.sqlite`, + controlBytes: controlBytes.byteLength, + controlSha256: createHash("sha256").update(controlBytes).digest("hex"), + controlValidation, + })}\n`, + ) + writeState(dataDir, checkpointId) + strictEqual((await resolveCheckpoint(dataDir)).manifest.formatVersion, 2) + + const corrupted = Buffer.from(controlBytes) + corrupted[corrupted.length - 1] ^= 1 + writeFileSync(controlPath, corrupted) + await rejects(resolveCheckpoint(dataDir), /digest mismatch|quick_check failed/) + }) + }) + it("resolves immutable current, previous, and explicit IDs", async () => { await withDataDir(async (dataDir) => { const current = newCheckpointId() @@ -666,9 +703,11 @@ describe("live-store reset safety", () => { writeSnapshot(dataDir, checkpointId) writeState(dataDir, checkpointId) mkdirSync(join(dataDir, "store"), { recursive: true }) + mkdirSync(join(dataDir, "control"), { recursive: true }) mkdirSync(join(dataDir, "metadata"), { recursive: true }) mkdirSync(join(dataDir, "tmp"), { recursive: true }) writeFileSync(join(dataDir, "store", "part.bin"), "live") + writeFileSync(join(dataDir, "control", "eventing.sqlite"), "live") writeFileSync(join(dataDir, "metadata", "table.sql"), "live") writeFileSync(join(dataDir, "status"), "live") writeFileSync(join(dataDir, "tmp", "scratch.bin"), "live") @@ -680,6 +719,7 @@ describe("live-store reset safety", () => { strictEqual((await readCheckpointState(dataDir)).current, checkpointId) ok(existsSync(checkpointSnapshotDir(dataDir, checkpointId))) ok(!existsSync(join(dataDir, "store"))) + ok(!existsSync(join(dataDir, "control"))) ok(!existsSync(join(dataDir, "metadata"))) ok(!existsSync(join(dataDir, "status"))) ok(!existsSync(join(dataDir, "tmp"))) @@ -735,7 +775,7 @@ describe("live-store reset safety", () => { const checkpointId = newCheckpointId() writeSnapshot(dataDir, checkpointId) writeState(dataDir, checkpointId) - for (const entry of ["data", "metadata", "store", "tmp"]) { + for (const entry of ["control", "data", "metadata", "store", "tmp"]) { mkdirSync(join(dataDir, entry), { recursive: true }) writeFileSync(join(dataDir, entry, "live.bin"), "live") } @@ -757,7 +797,7 @@ describe("live-store reset safety", () => { ) await Effect.runPromise(reconcileCheckpointRecovery(dataDir)) - for (const entry of ["data", "metadata", "status", "store", "tmp"]) { + for (const entry of ["control", "data", "metadata", "status", "store", "tmp"]) { ok(!existsSync(join(dataDir, entry)), `${boundary}: ${entry}`) } strictEqual((await readCheckpointState(dataDir)).current, checkpointId) diff --git a/apps/cli/test/local-eventing-control-store.test.ts b/apps/cli/test/local-eventing-control-store.test.ts new file mode 100644 index 000000000..2f592e866 --- /dev/null +++ b/apps/cli/test/local-eventing-control-store.test.ts @@ -0,0 +1,159 @@ +import { deepStrictEqual, rejects, strictEqual, throws } from "node:assert" +import { mkdirSync, mkdtempSync, rmSync, symlinkSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, it } from "vitest" +import type { MapleCloudEvent, SignalProjectionSpec } from "@maple/eventing-core" +import { eventingControlPath, LocalEventingControlStore } from "../src/server/eventing/control-store" + +const withDataDir = async (run: (dataDir: string) => Promise): Promise => { + const parent = mkdtempSync(join(tmpdir(), "maple-eventing-control-")) + const dataDir = join(parent, "data") + mkdirSync(dataDir, { recursive: true }) + try { + await run(dataDir) + } finally { + rmSync(parent, { recursive: true, force: true }) + } +} + +const projection = (overrides: Partial = {}): SignalProjectionSpec => ({ + id: "gitlab-issue-created", + revision: 1, + enabled: true, + tenantId: "tenant-a", + sourceKind: "otel.log", + selector: { + op: "eq", + field: { namespace: "attribute", key: "event.name", type: "string" }, + value: { type: "string", value: "gitlab.issue.created" }, + }, + projector: { id: "gitlab.issue", version: 1, config: { includeTitle: true } }, + activeFrom: "2026-08-07T00:00:00Z", + ...overrides, +}) + +const event = (overrides: Partial = {}): MapleCloudEvent => ({ + specversion: "1.0", + id: "sha256:061c0b5d99b92ef65ab8813c6d84988e4b1582e705e0077c952e62a0e84b6b08", + source: "urn:maple:source:otel:local", + type: "dev.maple.gitlab.issue.created.v1", + subject: "project/example/issues/42", + time: "2026-08-07T19:42:00.123456789Z", + datacontenttype: "application/json", + dataschema: "urn:maple:event-schema:gitlab-issue:v1", + tenantid: "tenant-a", + projectionid: "gitlab-issue-created", + projectionrevision: 1, + projectorid: "gitlab.issue", + projectorversion: 1, + data: { iid: 42, title: "Example" }, + ...overrides, +}) + +describe("LocalEventingControlStore", () => { + it("stores immutable sequential revisions and only loads the active revision", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + store.saveProjection(projection()) + deepStrictEqual(store.loadEnabledProjections("tenant-a"), [projection()]) + throws( + () => + store.saveProjection( + projection({ projector: { id: "changed", version: 1, config: {} } }), + ), + /immutable/, + ) + throws(() => store.saveProjection(projection({ revision: 3 })), /must be 2/) + + store.saveProjection(projection({ revision: 2, enabled: false })) + deepStrictEqual(store.loadEnabledProjections("tenant-a"), []) + store.saveProjection(projection({ revision: 3 })) + deepStrictEqual(store.loadEnabledProjections("tenant-a"), [projection({ revision: 3 })]) + deepStrictEqual(store.validate(), { + schemaVersion: 1, + projectionRevisions: 3, + projectionFailures: 0, + stagedEvents: 0, + readyEvents: 0, + }) + } finally { + store.close() + } + })) + + it("deduplicates staged events, rejects collisions, and preserves ready order", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + deepStrictEqual(store.stageEvents([event(), event()]), { + inserted: 1, + deduplicated: 1, + eventIds: [event().id, event().id], + }) + throws(() => store.stageEvents([event({ data: { iid: 43 } })]), /collision/) + throws(() => store.markReady(["unknown"]), /unknown event/) + store.markReady([event().id]) + store.markReady([event().id]) + deepStrictEqual(store.listStaged(), []) + deepStrictEqual(store.listReady(), [event()]) + } finally { + store.close() + } + })) + + it("survives restart and round-trips through a validated standalone snapshot", async () => + withDataDir(async (dataDir) => { + let store = await LocalEventingControlStore.open(dataDir) + store.saveProjection(projection()) + store.stageEvents([event()]) + store.markReady([event().id]) + store.recordProjectionFailures("tenant-a", [ + { + projectionId: "gitlab-issue-created", + projectionRevision: 1, + occurrenceId: "issue-42", + message: "test failure", + }, + ]) + store.close() + + store = await LocalEventingControlStore.open(dataDir) + deepStrictEqual(store.loadEnabledProjections("tenant-a"), [projection()]) + deepStrictEqual(store.listReady(), [event()]) + const snapshot = join(dataDir, "backups", "snapshot", "control.sqlite") + const validation = await store.backupTo(snapshot) + deepStrictEqual(validation, { + schemaVersion: 1, + projectionRevisions: 1, + projectionFailures: 1, + stagedEvents: 0, + readyEvents: 1, + }) + store.close() + + const restored = join(dataDir, "restored") + await LocalEventingControlStore.restoreSnapshot(snapshot, restored) + deepStrictEqual( + LocalEventingControlStore.validateSnapshot(eventingControlPath(restored)), + validation, + ) + const restoredStore = await LocalEventingControlStore.open(restored) + try { + deepStrictEqual(restoredStore.loadEnabledProjections("tenant-a"), [projection()]) + deepStrictEqual(restoredStore.listReady(), [event()]) + } finally { + restoredStore.close() + } + })) + + it("refuses a symlink in place of the database", async () => + withDataDir(async (dataDir) => { + const controlPath = eventingControlPath(dataDir) + mkdirSync(join(dataDir, "control"), { recursive: true }) + symlinkSync(join(dataDir, "target.sqlite"), controlPath) + await rejects(() => LocalEventingControlStore.open(dataDir), /not a real file/) + strictEqual(controlPath.endsWith("control/eventing.sqlite"), true) + })) +}) diff --git a/apps/cli/test/local-eventing-ingest.test.ts b/apps/cli/test/local-eventing-ingest.test.ts new file mode 100644 index 000000000..a0e4b347e --- /dev/null +++ b/apps/cli/test/local-eventing-ingest.test.ts @@ -0,0 +1,138 @@ +import { deepStrictEqual, strictEqual } from "node:assert" +import { describe, it } from "vitest" +import { __testables } from "../src/server/serve" + +describe("Local eventing ingest seam", () => { + it("requires maintenance authorization and exposes staged records only when requested", async () => { + const eventing = { + health: () => ({ activeProjections: 1 }), + listActive: () => [], + listReady: () => [{ id: "ready" }], + listStaged: () => [{ id: "staged" }], + } + const unauthorized = __testables.handleEventingRead( + eventing as never, + "maintenance-secret", + new Request("http://127.0.0.1/local/eventing/outbox?state=staged"), + new URL("http://127.0.0.1/local/eventing/outbox?state=staged"), + ) + strictEqual(unauthorized.status, 403) + + const request = new Request("http://127.0.0.1/local/eventing/outbox?state=staged", { + headers: { "x-maple-maintenance-token": "maintenance-secret" }, + }) + const authorized = __testables.handleEventingRead( + eventing as never, + "maintenance-secret", + request, + new URL(request.url), + ) + strictEqual(authorized.status, 200) + deepStrictEqual(await authorized.json(), [{ id: "staged" }]) + }) + + it("evaluates and stages before chDB write, then marks ready before acknowledging", async () => { + const order: string[] = [] + const event = { id: "event-1" } + const db = { + exec: () => { + order.push("chdb-insert") + }, + } + const authority = { + isRetired: () => false, + filterBatch: (_datasource: string, ndjson: string) => { + order.push("retention-filter") + return { ndjson, accepted: 1, rejected: 0 } + }, + } + const eventing = { + evaluateOtlp: () => { + order.push("evaluate") + return { events: [event], failures: [], typeMismatchFields: [] } + }, + persistFailures: () => order.push("persist-failures"), + stage: () => { + order.push("stage") + return { inserted: 1, deduplicated: 0, eventIds: [event.id] } + }, + markReady: () => order.push("ready"), + } + const request = new Request("http://127.0.0.1/v1/logs", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + resourceLogs: [ + { + scopeLogs: [ + { + logRecords: [ + { + timeUnixNano: "1786131720123456789", + body: { stringValue: "one" }, + }, + ], + }, + ], + }, + ], + }), + }) + + const result = await __testables.ingest( + db as never, + authority as never, + eventing as never, + "logs", + request, + ) + strictEqual(result.response.status, 200) + strictEqual(result.accepted, 1) + deepStrictEqual(order, [ + "evaluate", + "persist-failures", + "stage", + "retention-filter", + "chdb-insert", + "ready", + ]) + }) + + it("leaves a staged event non-ready when the warehouse write fails", async () => { + let markedReady = false + const request = new Request("http://127.0.0.1/v1/logs", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + resourceLogs: [{ scopeLogs: [{ logRecords: [{ body: { stringValue: "one" } }] }] }], + }), + }) + const result = await __testables.ingest( + { + exec: () => { + throw new Error("write failed") + }, + } as never, + { + isRetired: () => false, + filterBatch: (_datasource: string, ndjson: string) => ({ + ndjson, + accepted: 1, + rejected: 0, + }), + } as never, + { + evaluateOtlp: () => ({ events: [{ id: "event-1" }], failures: [], typeMismatchFields: [] }), + persistFailures: () => undefined, + stage: () => ({ inserted: 1, deduplicated: 0, eventIds: ["event-1"] }), + markReady: () => { + markedReady = true + }, + } as never, + "logs", + request, + ) + strictEqual(result.response.status, 500) + strictEqual(markedReady, false) + }) +}) diff --git a/apps/cli/test/local-eventing-runtime.test.ts b/apps/cli/test/local-eventing-runtime.test.ts new file mode 100644 index 000000000..30c1a7915 --- /dev/null +++ b/apps/cli/test/local-eventing-runtime.test.ts @@ -0,0 +1,203 @@ +import { deepStrictEqual, strictEqual } from "node:assert" +import { mkdirSync, mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, it } from "vitest" +import type { SignalProjectionSpec } from "@maple/eventing-core" +import { LocalEventingControlStore } from "../src/server/eventing/control-store" +import { normalizeOtlpLogs } from "../src/server/eventing/otlp" +import { LocalEventingRuntime } from "../src/server/eventing/runtime" +import { encodeLogs } from "../src/server/otlp/encode" + +const withDataDir = async (run: (dataDir: string) => Promise): Promise => { + const parent = mkdtempSync(join(tmpdir(), "maple-eventing-runtime-")) + const dataDir = join(parent, "data") + mkdirSync(dataDir, { recursive: true }) + try { + await run(dataDir) + } finally { + rmSync(parent, { recursive: true, force: true }) + } +} + +const attr = (key: string, value: Record) => ({ key, value }) + +const gitlabIssueCreated = { + resourceLogs: [ + { + resource: { + attributes: [ + attr("service.name", { stringValue: "gitlab-rails" }), + attr("service.version", { stringValue: "19.1.0" }), + ], + }, + scopeLogs: [ + { + scope: { name: "gitlab.event_store", version: "1.0.0" }, + logRecords: [ + { + timeUnixNano: "1786131720123456789", + observedTimeUnixNano: "1786131721123456789", + eventName: "gitlab.issue.created", + severityNumber: 9, + severityText: "INFO", + body: { stringValue: "Issue 42 created" }, + attributes: [ + attr("event.id", { stringValue: "01K20GITLABISSUE42" }), + attr("event.source", { stringValue: "https://gitlab.internal" }), + attr("gitlab.project.id", { intValue: "7" }), + attr("gitlab.project.path", { stringValue: "platform/maple" }), + attr("gitlab.issue.id", { intValue: "4200" }), + attr("gitlab.issue.iid", { intValue: "42" }), + attr("gitlab.issue.title", { stringValue: "Wire GitLab events" }), + attr("gitlab.issue.url", { + stringValue: "https://gitlab.internal/platform/maple/-/issues/42", + }), + attr("gitlab.user.id", { intValue: "9" }), + attr("gitlab.user.username", { stringValue: "operator" }), + ], + }, + ], + }, + ], + }, + ], +} + +const projection = (overrides: Partial = {}): SignalProjectionSpec => ({ + id: "gitlab-issue-created", + revision: 1, + enabled: true, + tenantId: "local", + sourceKind: "otel.log", + selector: { + op: "all", + clauses: [ + { + op: "eq", + field: { namespace: "signal", key: "event.name", type: "string" }, + value: { type: "string", value: "gitlab.issue.created" }, + }, + { + op: "gte", + field: { namespace: "attribute", key: "gitlab.issue.iid", type: "int64" }, + value: { type: "int64", value: "1" }, + }, + ], + }, + projector: { id: "gitlab.issue.created", version: 1, config: {} }, + activeFrom: "2000-01-01T00:00:00Z", + ...overrides, +}) + +describe("LocalEventingRuntime", () => { + it("normalizes typed GitLab OTLP fields while preserving the existing warehouse encoding", () => { + const [signal] = normalizeOtlpLogs(gitlabIssueCreated, "2026-08-07T20:00:00Z") + strictEqual(signal?.occurrenceId, "01K20GITLABISSUE42") + strictEqual(signal?.identityQuality, "source") + strictEqual(signal?.source, "https://gitlab.internal") + deepStrictEqual(signal?.fields.get("attribute:gitlab.issue.iid"), { + type: "int64", + value: "42", + }) + const batches = encodeLogs(gitlabIssueCreated) + strictEqual(batches.length, 1) + strictEqual(batches[0]?.rowCount, 1) + strictEqual(JSON.parse(batches[0]!.ndjson).log_attributes["gitlab.issue.iid"], "42") + }) + + it("projects before storage, deduplicates retry delivery, and makes the event ready after commit", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + const runtime = new LocalEventingRuntime(store) + strictEqual(runtime.hasActiveSource("otel.log"), false) + runtime.activate(projection()) + const first = runtime.evaluateOtlp("logs", gitlabIssueCreated) + strictEqual(first.failures.length, 0) + strictEqual(first.events.length, 1) + deepStrictEqual(first.events[0], { + specversion: "1.0", + id: first.events[0]!.id, + source: "https://gitlab.internal", + type: "dev.maple.gitlab.issue.created.v1", + subject: "platform/maple/issues/42", + time: "2026-08-07T19:42:00.123456789Z", + datacontenttype: "application/json", + dataschema: "urn:maple:event-schema:gitlab-issue-created:v1", + tenantid: "local", + projectionid: "gitlab-issue-created", + projectionrevision: 1, + projectorid: "gitlab.issue.created", + projectorversion: 1, + data: { + project: { id: "7", path: "platform/maple" }, + issue: { + id: "4200", + iid: "42", + title: "Wire GitLab events", + url: "https://gitlab.internal/platform/maple/-/issues/42", + }, + actor: { id: "9", username: "operator" }, + serviceName: "gitlab-rails", + }, + }) + const staged = runtime.stage(first.events) + strictEqual(staged.inserted, 1) + strictEqual(runtime.listReady().length, 0) + deepStrictEqual(runtime.listStaged(), first.events) + const retry = runtime.evaluateOtlp("logs", gitlabIssueCreated) + strictEqual(retry.events[0]?.id, first.events[0]?.id) + strictEqual(runtime.stage(retry.events).deduplicated, 1) + runtime.markReady(staged.eventIds) + deepStrictEqual(runtime.listReady(), first.events) + deepStrictEqual(runtime.listStaged(), []) + } finally { + store.close() + } + })) + + it("activates a validated revision without restart and reloads it after restart", async () => + withDataDir(async (dataDir) => { + let store = await LocalEventingControlStore.open(dataDir) + let runtime = new LocalEventingRuntime(store) + runtime.activate(projection()) + strictEqual(runtime.evaluateOtlp("logs", gitlabIssueCreated).events.length, 1) + runtime.activate( + projection({ + revision: 2, + selector: { + op: "eq", + field: { namespace: "signal", key: "event.name", type: "string" }, + value: { type: "string", value: "gitlab.issue.closed" }, + }, + }), + ) + strictEqual(runtime.evaluateOtlp("logs", gitlabIssueCreated).events.length, 0) + store.close() + + store = await LocalEventingControlStore.open(dataDir) + try { + runtime = new LocalEventingRuntime(store) + strictEqual(runtime.listActive()[0]?.revision, 2) + strictEqual(runtime.evaluateOtlp("logs", gitlabIssueCreated).events.length, 0) + } finally { + store.close() + } + })) + + it("does no normalization or event work for a source with no active projection", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + const runtime = new LocalEventingRuntime(store) + deepStrictEqual(runtime.evaluateOtlp("logs", { malformed: Symbol("not decoded") }), { + events: [], + failures: [], + typeMismatchFields: [], + }) + } finally { + store.close() + } + })) +}) diff --git a/apps/cli/test/server-network.test.ts b/apps/cli/test/server-network.test.ts index 53c413abb..46f389d8a 100644 --- a/apps/cli/test/server-network.test.ts +++ b/apps/cli/test/server-network.test.ts @@ -171,7 +171,8 @@ describe("browser origin policy", () => { deepStrictEqual(corsHeadersForAllowedOrigin(hostedOrigin), { "access-control-allow-origin": hostedOrigin, "access-control-allow-methods": "GET, POST, OPTIONS", - "access-control-allow-headers": "content-type, content-encoding", + "access-control-allow-headers": + "content-type, content-encoding, x-maple-maintenance-token", "access-control-allow-private-network": "true", vary: "Origin", }) diff --git a/bun.lock b/bun.lock index 6d6b2bf3e..439db99dc 100644 --- a/bun.lock +++ b/bun.lock @@ -54,6 +54,7 @@ "@maple/domain": "workspace:*", "@maple/effect-cloudflare": "workspace:*", "@maple/email": "workspace:*", + "@maple/eventing-core": "workspace:*", "@maple/infra": "workspace:*", "@maple/llm": "workspace:*", "@maple/query-engine": "workspace:*", @@ -88,6 +89,7 @@ "@effect/platform-bun": "catalog:effect", "@maple-dev/effect-sdk": "workspace:*", "@maple/domain": "workspace:*", + "@maple/eventing-core": "workspace:*", "@maple/query-engine": "workspace:*", "effect": "catalog:effect", "protobufjs": "^8.6.1", @@ -500,6 +502,9 @@ "packages/alerting-core": { "name": "@maple/alerting-core", "version": "0.0.0", + "dependencies": { + "@maple/eventing-core": "workspace:*", + }, "devDependencies": { "@types/node": "catalog:tooling", "typescript": "catalog:tooling", @@ -635,6 +640,19 @@ "react": "^19.0.0", }, }, + "packages/eventing-core": { + "name": "@maple/eventing-core", + "version": "0.0.0", + "dependencies": { + "effect": "catalog:effect", + }, + "devDependencies": { + "@effect/language-service": "catalog:effect", + "@types/node": "catalog:tooling", + "typescript": "catalog:tooling", + "vitest": "catalog:", + }, + }, "packages/infra": { "name": "@maple/infra", "devDependencies": { @@ -1551,6 +1569,8 @@ "@maple/email": ["@maple/email@workspace:packages/email"], + "@maple/eventing-core": ["@maple/eventing-core@workspace:packages/eventing-core"], + "@maple/infra": ["@maple/infra@workspace:packages/infra"], "@maple/ingest": ["@maple/ingest@workspace:apps/ingest"], diff --git a/docs/signal-to-event-projection.md b/docs/signal-to-event-projection.md new file mode 100644 index 000000000..b1e1f91ce --- /dev/null +++ b/docs/signal-to-event-projection.md @@ -0,0 +1,881 @@ +# Signal-to-event projection architecture + +Status: implemented on `codex/issue-222-alerting-core`; downstream delivery remains out of scope + +Related work: [issue #222](https://github.com/MapleTechLabs/maple/issues/222), +`@maple/alerting-core` + +Audience: Maple maintainers and implementers of hosted or Maple Local runtimes + +## Decision summary + +Maple will treat immediate, per-occurrence event generation as an ingest concern, +not as a scheduled warehouse-query concern. + +- Each accepted OTLP record or provider webhook is decoded and normalized into a + typed signal once. +- An immutable snapshot of enabled signal projections is evaluated against that + signal before its scalar types are flattened for warehouse storage. +- Every matching projection invokes a registered, pure projector that produces a + factual [CloudEvents 1.0](https://github.com/cloudevents/spec/blob/main/cloudevents/spec.md) + event. +- Produced events enter a durable, idempotent outbox. Consumers and delivery + transports are downstream of that boundary. +- The original telemetry continues through the existing warehouse write path. +- chDB is not polled to discover newly arrived records. It remains the analytics + store and an optional, explicitly invoked replay source. +- Scheduled aggregate alerts remain query-driven. Alert lifecycle transitions + become another producer of typed events and use the same outbox as ingest-time + projections. + +The configurable matching model is a small, structured, typed predicate tree. It +is not arbitrary SQL and it is not a new textual expression language. The live +runtime evaluates the tree in memory. A warehouse adapter may lower the supported +subset to parameterized ClickHouse expressions for explicit historical replay, +but SQL behavior does not define the predicate semantics. + +## Problem + +Maple currently contains several mechanisms that are related but not expressed +through one event boundary: + +- hosted alert rules periodically query telemetry, update incident lifecycle + state, and request deliveries; +- PlanetScale receives signed webhooks and performs provider-specific work; +- Maple Local accepts OTLP records and writes them directly to chDB; +- future automation needs individual facts, such as a GitLab issue-created + signal, to become events that agents or other consumers can act on. + +Using the alert scheduler for the last case would give it the wrong semantics. +A windowed query answers a question about a set of stored records and normally +produces one aggregate observation. It cannot faithfully represent every +individual occurrence without cursors, overlap windows, late-arrival handling, +and deduplication. + +The current Local `logs` table has no ingestion sequence or native event ID. Its +sort key is designed for observability queries, and arbitrary OTLP attributes are +stored as strings. Repeatedly querying that table once per rule would therefore: + +- compete with ingest, UI queries, checkpoints, retention, and archive work; +- miss late records or repeatedly rediscover records unless a second deduplication + system is added; +- require casts that cannot always recover the source value's original type; +- turn an embedded analytical database into an inefficient message queue. + +The event layer is still useful. It belongs in front of chDB for live signals, +with chDB retained behind it for analytics and aggregate alert evaluation. + +## Goals + +1. Allow operators and integrations to configure which incoming signals become + typed events without writing SQL or changing core runtime code. +2. Evaluate each delivered signal in one ingest pass against all applicable + projections; do not issue one warehouse query per projection. +3. Preserve scalar types for string, boolean, integer, floating-point, + timestamp, and duration comparisons. +4. Make source adapters, projectors, event persistence, and consumers replaceable + behind explicit interfaces. +5. Give emitted events stable identities so retries do not create duplicate + logical events when the source provides stable occurrence identity. +6. Reuse the same event envelope and outbox for query-alert lifecycle events. +7. Keep Maple Local headless: matching and event persistence must work while no + browser is open. +8. Keep the core deterministic, bounded, tenant-scoped, and independent of a + database, network, scheduler, wall clock, or particular deployment host. + +## Non-goals + +- Adding NATS, JetStream, Kafka, or another general-purpose broker as a required + Maple component. +- Loading arbitrary third-party code into a running Maple process. A "plugin" in + this document is a compile-time registered module behind a stable interface. +- Defining sink delivery, Matrix behavior, agent authorization, or action policy. +- Replacing the Collector's routing, filtering, queueing, or authentication. +- Replacing scheduled queries for rates, percentiles, absence, threshold state, + or other aggregate alerts. +- Guaranteeing exactly-once external side effects across an uncooperative source, + Maple, and an arbitrary consumer. +- Automatically replaying old telemetry whenever a projection is created or + changed. +- Providing a general scripting language, joins, aggregation, arithmetic, + regular expressions, or user-provided SQL in the first version. + +## Terminology + +**Signal** +: One factual input occurrence after authentication, decoding, and normalization. +It may originate as an OTLP log/span/metric point or a provider webhook. + +**Source adapter** +: A module that verifies or accepts a source payload, normalizes occurrences into +typed signals, declares known fields, and supplies source identity when +available. + +**Signal projection** +: Durable configuration pairing a source kind, typed selector, and registered +projector. It says which source occurrences should be promoted into which +event representation. It is distinct from a downstream event subscription. + +**Selector** +: A bounded structured predicate over typed signal fields. + +**Projector** +: A pure, versioned function that maps one matching signal to a declared event +type and data schema. Provider-specific meaning belongs here rather than in the +eventing core. + +**Event** +: An immutable CloudEvents 1.0 envelope containing a typed factual payload. + +**Event outbox** +: Durable host storage that makes event creation idempotent and separates event +production from downstream delivery. + +**Event consumer** +: A downstream component interested in one or more event types. Webhooks, Matrix, +agents, issue creation, and existing provider responses are consumer concerns, +not selector or projector concerns. + +## Architecture + +There are two intentionally different event-production paths. They converge only +after a factual event has been produced. + +```mermaid +flowchart LR + Source["OTLP or provider source"] --> Gate["Authenticate / verify"] + Gate --> Decode["Decode once"] + Decode --> Signal["Typed normalized signal"] + + Signal --> Match["Ingest-time selector evaluation"] + Match --> Project["Registered signal projector"] + Project --> Outbox["Durable event outbox"] + + Signal --> Encode["Warehouse encoder"] + Encode --> Warehouse["chDB / hosted warehouse"] + + Warehouse --> Scheduled["Scheduled aggregate query"] + Scheduled --> Lifecycle["Alert evaluation and lifecycle"] + Lifecycle --> AlertProjector["Alert lifecycle projector"] + AlertProjector --> Outbox +``` + +The upper path handles occurrences such as "this GitLab issue was created". The +lower path handles conclusions such as "the error rate has remained above five +percent for ten minutes". Both can ultimately notify the same consumers without +pretending they have the same input or timing semantics. + +### Required module boundaries + +The architecture has four replaceable boundaries: + +1. **Source adapters** turn authenticated source payloads into typed signals. +2. **Selectors** determine whether a normalized signal qualifies. +3. **Projectors** map a qualifying signal to a typed factual event. +4. **Consumers** subscribe to event types downstream of the durable outbox. + +The eventing core owns the contracts and deterministic behavior. It does not know +about PlanetScale, GitLab, Matrix, chDB, PostgreSQL, Cloudflare Queues, or HTTP. + +PlanetScale is therefore one installed composition, not the model itself. Its +module can register a webhook source adapter and PlanetScale-specific projectors. +Those projectors can be replaced or supplemented without changing the selector +evaluator or downstream event contract. Existing PlanetScale behavior can later +be moved behind consumers of those typed events without putting provider actions +inside the projector. + +## Core data contracts + +The TypeScript below is illustrative. Canonical persisted encodings must be +defined with runtime schemas and shared conformance fixtures. + +### Typed values + +```ts +type SignalScalar = + | { readonly type: "string"; readonly value: string } + | { readonly type: "boolean"; readonly value: boolean } + | { readonly type: "int64"; readonly value: string } + | { readonly type: "float64"; readonly value: number } + | { readonly type: "timestamp"; readonly value: string } + | { readonly type: "duration"; readonly value: string } +``` + +`int64` and `duration` use decimal strings in serialized form so JavaScript does +not lose precision. Runtime evaluators may compile them to native `bigint` or the +equivalent host type. Timestamp values use canonical RFC 3339 with an explicit +offset in serialized form and compare as UTC instants. Duration values represent +integer nanoseconds. `float64` values must be finite; `NaN` and infinities are +rejected during normalization. + +Arrays and objects may be preserved for projector payloads, but selectors operate +only on declared scalar fields in version 1. + +### Normalized signal + +```ts +interface NormalizedSignal { + readonly sourceKind: string + readonly source: string + readonly tenantId: string + readonly occurrenceId: string | null + readonly identityQuality: "source" | "derived" | "none" + readonly occurredAt: string + readonly observedAt: string + readonly subject: string | null + readonly fields: ReadonlyMap + readonly data: unknown +} +``` + +- `sourceKind` chooses the compatible field catalog and projector registry. +- `source` is a stable URI identifying the logical producer or integration. +- `occurrenceId` is a source-issued stable identifier when one exists. +- `identityQuality: "source"` means the adapter expects the ID to survive source + retries and rebatching. `"derived"` identifies a canonical content fingerprint + with documented collision/collapse limitations. `"none"` cannot support a + durable once-only automation guarantee. +- `occurredAt` is source time; `observedAt` is Maple acceptance time. +- `fields` contains canonical built-ins and namespaced source attributes. It must + not contain secrets merely because they were present in the incoming payload. +- `data` is a bounded, schema-validated, source-specific representation available + to compatible projectors. It may contain arrays and objects that are not + selector-addressable, but it follows the adapter's redaction policy and is not + an unvalidated raw request body. + +The source adapter must not expose an unbounded raw payload as the selector field +space or projector input. + +### Field references and catalogs + +A selector uses logical field references, never physical column names: + +```ts +interface FieldRef { + readonly namespace: "signal" | "resource" | "scope" | "attribute" | "body" + readonly key: string + readonly type: SignalScalar["type"] +} +``` + +Each source adapter exposes a field catalog for known fields. A catalog entry +declares: + +- logical name and scalar type; +- allowed selector operators; +- sensitivity and whether a projector may expose it by default; +- whether historical replay is `exact`, `coerced`, or `unavailable`; +- an optional backend-owned replay binding. This binding is not user SQL. + +OTLP resource, scope, and record attributes are open-ended. A projection may +reference an uncatalogued attribute by explicitly declaring its expected scalar +type. At runtime a differently typed value does not get coerced; it does not +match, and a bounded type-mismatch metric is recorded. Source-specific modules +should publish catalogs for common attributes so users do not need to repeat +those declarations. + +### Selector AST + +```ts +type SignalPredicate = + | { readonly op: "all"; readonly clauses: readonly SignalPredicate[] } + | { readonly op: "any"; readonly clauses: readonly SignalPredicate[] } + | { readonly op: "not"; readonly clause: SignalPredicate } + | { readonly op: "exists"; readonly field: FieldRef } + | { + readonly op: "eq" | "neq" | "gt" | "gte" | "lt" | "lte" | "contains" + readonly field: FieldRef + readonly value: SignalScalar + } + | { + readonly op: "in" + readonly field: FieldRef + readonly values: readonly SignalScalar[] + } +``` + +Version 1 has the following semantics: + +| Operation | Supported types | Semantics | +| ------------------------ | ------------------------------------------- | ------------------------------------------------------------------------- | +| `exists` | all | True only when the field is present with a valid typed scalar. | +| `eq`, `neq` | all | Exact same-type comparison. A missing or mistyped field makes both false. | +| `gt`, `gte`, `lt`, `lte` | `int64`, `float64`, `timestamp`, `duration` | Ordered same-type comparison. | +| `contains` | `string` | Case-sensitive Unicode substring comparison. | +| `in` | all | Exact same-type membership; all literals must share the field type. | +| `all`, `any`, `not` | predicates | Total boolean composition with short-circuit evaluation. | + +There are no implicit casts. The string `"3"` is not the integer `3`; an integer +is not silently promoted to a float; and a string that resembles a date is not a +timestamp. Adapters may deliberately normalize a provider value into a declared +type, but that conversion is part of the source contract and is tested there. + +Missing values are not equivalent to null. Null source values are treated as +missing in version 1. Consequently `neq` requires a present field, whereas +`not(eq(...))` also matches a missing field. Configuration tooling should prefer +the explicit form that expresses the intended behavior. + +Validation happens before a projection can become active. Version 1 limits a +selector to: + +- nesting depth of 8; +- 64 total predicate nodes; +- 100 members in one `in` predicate; +- 4 KiB per string literal; +- no regular expressions, functions, arithmetic, joins, or user code. + +These bounds keep evaluation predictable and leave room for indexing active +projections by source kind and simple discriminating fields. + +### Signal projection + +```ts +interface SignalProjectionSpec { + readonly id: string + readonly revision: number + readonly enabled: boolean + readonly tenantId: string + readonly sourceKind: string + readonly selector: SignalPredicate + readonly projector: { + readonly id: string + readonly version: number + readonly config: unknown + } + readonly activeFrom: string +} +``` + +Every semantic edit creates a new immutable revision. Activation is not +retroactive: the new revision sees signals accepted after the runtime atomically +installs its compiled registry snapshot. Historical processing requires an +explicit replay operation. + +The configuration record is data. Source adapters and projector implementations +are registered code. This is how matching remains configurable without making +authentication, provider semantics, or executable code user-supplied. + +For example, the first GitLab projection uses the following concrete contract: + +```json +{ + "id": "gitlab-issue-created", + "revision": 1, + "enabled": true, + "tenantId": "local", + "sourceKind": "otel.log", + "selector": { + "op": "all", + "clauses": [ + { + "op": "eq", + "field": { "namespace": "signal", "key": "event.name", "type": "string" }, + "value": { "type": "string", "value": "gitlab.issue.created" } + }, + { + "op": "gte", + "field": { "namespace": "attribute", "key": "gitlab.issue.iid", "type": "int64" }, + "value": { "type": "int64", "value": "1" } + } + ] + }, + "projector": { "id": "gitlab.issue.created", "version": 1, "config": {} }, + "activeFrom": "2026-08-07T00:00:00Z" +} +``` + +The `gte` comparison above is an integer comparison, not lexicographic string +ordering. A timestamp predicate would similarly carry a `timestamp` literal and +compare normalized instants rather than formatted text. No query is generated +for either comparison on the live path. + +### Projector contract + +```ts +interface SignalProjector { + readonly id: string + readonly version: number + readonly sourceKinds: readonly string[] + readonly outputType: string + readonly dataSchema: string + readonly validateConfig: (value: unknown) => ProjectorConfig + readonly project: (signal: NormalizedSignal, config: ProjectorConfig) => ProjectedEventData +} +``` + +A projector must be pure, deterministic, bounded, versioned, and free of I/O. It +does not create issues, call Matrix, send notifications, or mutate provider +state. It produces a factual event payload conforming to its declared schema. + +The registry may include a bounded generic field-mapping projector for +operator-defined factual events. Provider modules register semantic projectors +when field copying is insufficient. No runtime module loading is required. + +### Event envelope + +Produced events use CloudEvents 1.0 structured representation: + +```json +{ + "specversion": "1.0", + "id": "sha256:...", + "source": "urn:maple:source:otel:local", + "type": "dev.maple.gitlab.issue.created.v1", + "subject": "project/example/issues/42", + "time": "2026-08-07T19:42:00.000000000Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:gitlab-issue:v1", + "tenantid": "...", + "projectionid": "...", + "projectionrevision": 3, + "data": {} +} +``` + +Names above are illustrative until the repository reserves its canonical event +type and schema namespace. + +The event ID is deterministic when stable occurrence identity exists: + +```text +SHA-256(tenant ID, source kind, source URI, occurrence ID, projection ID, projection revision) +``` + +The hash input uses a canonical length-delimited encoding, not string +concatenation. Projector version and output schema version are already fixed by +the immutable projection revision and must be recorded with the event. + +Sensitive source details belong in `data`, under the projector's explicit schema +and redaction policy. They must not be copied into CloudEvents context attributes, +logs, metrics labels, or idempotency keys. + +## Runtime behavior + +### Projection compilation and activation + +The host loads enabled projections for a tenant, validates them against the +source and projector registries, and compiles them into immutable predicate +functions. The active registry is swapped atomically. Every decoded ingest batch +uses exactly one registry snapshot, even if configuration changes while the batch +is being processed. + +The initial implementation may evaluate all projections in the applicable +`sourceKind` bucket. The registry may later index projections by exact-match +discriminators such as event name or service name. This is an optimization and +must not alter selector semantics or ordering. + +Projection evaluation is deterministic and side-effect free. All matching +projections run; this is not first-match routing. A signal may therefore produce +zero, one, or several different factual events. + +### Maple Local OTLP ingest + +Maple Local already decodes an OTLP request and then passes the decoded payload +to the warehouse encoder. The event seam belongs between those operations. + +The implementation should refactor decoding/normalization so that: + +1. the OTLP request is parsed once; +2. typed record values remain available to the matcher; +3. the existing warehouse rows are produced without changing their stored shape; +4. matched events are staged idempotently before ingest acknowledges success; +5. the telemetry insert completes; +6. staged events are marked ready for downstream consumption; +7. only then is the OTLP request acknowledged. + +When no projection matches, the path adds only bounded predicate work before the +existing chDB insert. + +If the event store cannot stage a required event, ingest returns a retryable +failure rather than silently losing automation. A source retry reuses the same +event ID when stable occurrence identity is available, so staging is idempotent. + +Staging and chDB insertion are not one transaction. A process crash after the +chDB insert but before the OTLP acknowledgement can still cause a duplicate raw +telemetry row on retry; that is already possible with at-least-once OTLP +delivery. The staged/ready outbox protocol prevents an event from becoming +dispatchable before the ingest attempt reaches its warehouse commit point. + +If atomic exactly-once storage across both systems later becomes a requirement, +the correct addition is a durable ingress journal before both writes. chDB +polling does not solve that problem. + +### Provider webhooks + +Provider authentication and replay protection run before normalization. The +host must establish a durable event boundary before acknowledging the provider. +The hosted PlanetScale route therefore projects a verified payload first and +enqueues the resulting CloudEvent together with its temporary parity payload; +the queue is its durable event boundary. + +The provider source adapter supplies the strongest available delivery or event +identity. It then uses the same selector, projector, event ID, and outbox +contracts as OTLP. Provider-specific response behavior does not live in the core; +it can be migrated behind consumers of the emitted event types. + +### Query-driven alerts + +Scheduled alert rules retain their existing execution model: + +1. the host schedules and claims a rule; +2. a warehouse query produces an aggregate `AlertObservation`; +3. `@maple/alerting-core` evaluates threshold and lifecycle state; +4. an alert lifecycle projector converts `trigger`, `resolve`, `renotify`, or + `test` intent into a CloudEvent; +5. the host persists it through the common event outbox. + +This path queries chDB or the hosted warehouse because its input is an aggregate +over time. It does not reuse the ingest-time signal selector, and the ingest-time +path does not impersonate an alert incident. + +### Historical replay + +Replay is an operator-invoked batch operation, never the live event mechanism. +It evaluates one projection revision over a bounded time range and must support a +dry-run count/sample mode before it can persist events. + +Every field catalog entry declares replay capability: + +- `exact`: stored data retains enough type and identity information to reproduce + live semantics; +- `coerced`: the adapter can apply an explicit cast, but the source type was lost + or identity is derived; +- `unavailable`: the backend cannot implement the live predicate faithfully. + +A replay request using a `coerced` field requires explicit operator +acknowledgement. A request using an unavailable field is rejected. The warehouse +compiler emits parameterized expressions through existing query-building +facilities; it never interpolates field names or literals supplied directly by a +user. + +Current Local OTLP attribute maps store strings, so arbitrary typed attributes +will generally be `coerced`, not `exact`. Replay event IDs are guaranteed to +deduplicate against live events only when the warehouse retained the same stable +source occurrence ID. + +## Processing and delivery guarantees + +The architecture uses precise, layered guarantees rather than the blanket phrase +"exactly once". + +| Boundary | Guarantee | +| -------------------------------------- | -------------------------------------------------------------------------------------------------------------- | +| Source to Maple | At least once when the source/Collector retries; source-specific otherwise. | +| One accepted batch | One evaluation against one immutable projection-registry snapshot. | +| Projection with source-stable identity | Effectively-once event creation through deterministic ID plus unique outbox insertion. | +| Projection with derived identity | Best-effort deduplication; identical real occurrences may collapse and re-encoded retries may diverge. | +| Projection with no identity | At-least-once event creation only; durable automation should reject this configuration by default. | +| Outbox to consumer | At least once with an event ID/idempotency key; consumer-side external effects are outside this specification. | +| chDB telemetry row | Existing OTLP semantics; duplicate storage remains possible after ambiguous failures. | + +A projection intended to trigger external automation must require +`identityQuality: "source"` unless an operator explicitly accepts weaker +semantics. GitLab event instrumentation should therefore furnish a stable event +or delivery identifier as part of its source contract. + +## chDB responsibilities + +chDB is responsible for: + +- storing telemetry for interactive and analytical queries; +- serving scheduled aggregate-alert queries; +- serving bounded explicit replay where field capabilities allow it; +- participating in existing checkpoint, retention, and archive workflows. + +chDB is not responsible for: + +- acting as a live queue; +- maintaining one cursor per signal projection; +- deduplicating event delivery; +- storing mutable projection configuration or delivery attempts merely because + it stores the source telemetry; +- defining selector type semantics through ClickHouse casts. + +Version 1 requires no new column or sort-key change to the existing telemetry +tables. A future narrow event journal or ingress-identity column may improve +replay, but it must be justified separately and must not turn wide raw-telemetry +tables into queue state. + +## Alternatives considered + +| Alternative | Decision | +| ------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| One periodic chDB query per projection | Rejected. It repeats wide scans, introduces cursor/late-arrival problems, and competes with the analytical workload. | +| One shared query that tails all recent chDB rows | Rejected as the live path. It reduces query count but still lacks a reliable ingestion cursor and evaluates after scalar type loss. It may inform an explicit replay implementation. | +| ClickHouse materialized views per projection | Rejected. Mutable user configuration would become DDL, current attribute storage has already flattened types, and lifecycle/deduplication state still needs another store. | +| Collector OTTL as Maple's rule language | Kept as an optional deployment optimization. It is valuable for OTel-only routing but does not define provider-webhook behavior or Maple-managed dynamic configuration. | +| CEL as the first expression language | Deferred. CEL is safe and capable, but embedding compatible runtimes and defining warehouse lowering is more surface than the initial predicates require. Reconsider it if the bounded AST is demonstrably insufficient. | +| CloudEvents SQL as the signal selector | Rejected for raw signals. [CESQL 1.0](https://github.com/cloudevents/spec/blob/main/cesql/spec.md) filters CloudEvent context attributes but does not address arbitrary event `data`; it may be useful for downstream CloudEvent subscriptions. | +| NATS or another broker as the event abstraction | Rejected as a requirement. A broker can later implement an event transport port, but it does not replace source normalization, selector semantics, projectors, identity, or host persistence. | +| A custom textual DSL | Rejected. The structured predicate tree is the persisted intermediate representation; configuration UIs and APIs do not need a parser. | + +## Durable host ports + +The core needs interfaces rather than a prescribed database: + +```ts +interface SignalProjectionStore { + loadEnabled(tenantId: string): Promise +} + +interface EventOutboxStore { + stage(events: readonly CloudEvent[]): Promise + markReady(eventIds: readonly string[]): Promise +} +``` + +The real contracts also need revision/change notification, unique event IDs, +bounded batch operations, health inspection, and recovery of staged records. + +Hosted Maple may implement these ports with its relational state and queue +infrastructure. Maple Local needs a small transactional control-state store whose +rules, outbox, and migration identity survive restart. That state is not covered +by chDB checkpoints automatically; backup, restore, and schema migration are part +of the Local host adapter's acceptance criteria. + +The physical Local store is an implementation decision, but it must provide: + +- uniqueness on event ID; +- atomic projection revision writes; +- atomic event staging and readiness transitions; +- bounded recovery of stranded staged events; +- crash-safe migrations and explicit backup/restore behavior; +- no dependency on a browser process. + +## Package and host ownership + +The intended ownership is: + +- `packages/eventing-core` (new): language-neutral schemas, selector validation, + the reference TypeScript evaluator, projector registry contracts, canonical + event identity, and conformance fixtures. No database, network, scheduler, or + global clock dependencies. +- `packages/alerting-core` (existing): aggregate alert evaluation and incident + lifecycle. It remains distinct and later emits through an eventing-core port. +- `packages/domain`: public/API schemas when projection CRUD becomes public. +- `apps/cli`: Maple Local OTLP source adapter, compiled-registry lifecycle, + durable Local ports, ingest staging, and optional replay adapter. +- `apps/api`: provider webhook adapters and hosted persistence wiring. +- `apps/ingest`: a future Rust OTLP adapter only when hosted per-signal projection + is required. + +The canonical JSON schemas and fixture corpus, rather than TypeScript source +types, define cross-language behavior. A Rust implementation must pass the same +valid/invalid selector cases, typed comparison cases, canonical event-ID vectors, +and projection fixtures before it can claim compatibility. + +[OpenTelemetry Transformation Language](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/pkg/ottl) +can remain a Collector-side optimization or adapter. It is not the universal +Maple contract because it is coupled to OTel Collector contexts and does not +cover provider webhooks. [CEL](https://cel.dev/overview/cel-overview) is the +preferred language to reconsider if real requirements outgrow the bounded AST; +version 1 does not embed CEL runtimes or define a CEL-to-ClickHouse compiler. + +## Security and tenancy + +- Authentication or provider signature verification occurs before a source + adapter may produce a signal. +- Every signal, projection, event, and outbox operation carries an explicit + tenant ID. Cross-tenant registry lookup or event fanout is forbidden. +- User configuration cannot name SQL columns, inject SQL fragments, load code, + call functions, or select secrets outside the source field catalog. +- Source adapters mark sensitive fields. Generic projectors exclude them by + default; provider projectors must opt in deliberately and document why. +- Projected event size and source-field size are bounded before outbox insertion. +- Runtime errors and telemetry must not record full sensitive payloads. +- Sink URL validation, private-network policy, signing, and agent authorization + remain downstream policies. General eventing must not weaken hosted SSRF + protections. + +## Failure handling and observability + +Malformed projection configuration is rejected before activation. The reference +evaluator is total: missing fields and runtime type mismatches produce defined +non-matches rather than exceptions. + +A projector must return either schema-valid event data or a bounded typed +projection failure. A bad occurrence must not create an infinite source retry +loop. The host records the failure against projection ID/revision and occurrence +identity, exposes degraded health, and quarantines or dead-letters according to a +bounded policy. Exact quarantine policy belongs to the host adapter, but silently +dismissing a durable projection failure is not allowed. + +Required low-cardinality telemetry includes: + +- received signals by source kind; +- selector evaluations and matches by projection ID; +- selector type mismatches by source kind and field catalog key; +- projection failures; +- outbox staged, deduplicated, ready, and stranded counts; +- evaluation and staging latency; +- active projection count and registry revision; +- replay scanned, matched, emitted, and deduplicated counts. + +Raw field values, subjects, event IDs, and arbitrary event types must not become +unbounded metric labels. + +## Compatibility and migration + +This design extends rather than replaces the host-neutral alert-core extraction +already on the issue-222 branch. + +1. Existing hosted aggregate alerts continue using their scheduler, query, + lifecycle, and delivery behavior while the event contract is introduced. +2. The new eventing core lands without runtime activation and with conformance + fixtures. +3. Maple Local adds ingest-time projection behind an explicit feature/config + gate. With no active projections, observable ingest and chDB behavior remain + unchanged. +4. A GitLab issue-created OTLP fixture proves the end-to-end source identity, + typed selector, projector, retry deduplication, and durable outbox path. +5. PlanetScale is adapted behind the same source/projector interfaces while its + existing externally visible behavior remains intact. A compare/dual-observe + period should precede removal of direct hard-coded handling. +6. Alert lifecycle intents are projected into the same CloudEvents/outbox model + after parity tests show no change to trigger, resolve, renotify, test, + suppression, or retry semantics. +7. Warehouse replay is added only after the live path is proven and replay + capability metadata is implemented. + +No migration step requires NATS, a per-rule chDB cursor, or a new raw-telemetry +sort key. + +## Implementation slices for the next goal + +### Slice 1 — Contract and evaluator + +- Add `packages/eventing-core`. +- Define runtime schemas for typed values, fields, predicates, projection specs, + projector registrations, and CloudEvent output. +- Implement validation, compilation, and the pure reference evaluator. +- Add canonical JSON and event-ID test vectors. +- Add complexity-limit and hostile-input tests. + +### Slice 2 — Local durable control state + +- Select and document the Local transactional store. +- Implement projection revision and outbox ports, migrations, recovery, and + backup/restore hooks. +- Expose headless health inspection before UI work. + +### Slice 3 — Local ingest seam + +- Refactor OTLP normalization to preserve typed values without decoding twice. +- Load and atomically swap compiled projection snapshots. +- Stage matching events, insert telemetry, mark events ready, and acknowledge. +- Prove that the live path executes no chDB `SELECT` and adds no scheduler. + +### Slice 4 — First vertical: GitLab event to durable Maple event + +- Capture the real GitLab OTLP field contract and stable occurrence identity. +- Register its field catalog and issue-event projector. +- Configure an issue-created projection without hard-coded selector values. +- Verify duplicate source deliveries create one logical outbox event. + +This slice stops at the outbox. Matrix and agent-action behavior are a downstream +goal using the produced typed event. + +### Slice 5 — Existing producer convergence + +- Adapt PlanetScale webhook inputs to the source/projector contracts. +- Project alert lifecycle intents into CloudEvents. +- Preserve existing provider and alert behavior with parity fixtures before + switching consumers. + +### Slice 6 — Optional replay + +- Add per-field replay capability declarations. +- Implement bounded dry-run and explicit emission modes. +- Add evaluator-versus-ClickHouse conformance tests for every `exact` binding. + +## Acceptance criteria + +The first usable implementation is complete when all of the following are true: + +1. A configured GitLab issue-created OTLP signal is matched before chDB encoding + and produces a schema-valid CloudEvent while the telemetry record is still + stored normally. +2. Re-delivery of a source-stable occurrence produces the same event ID and one + logical outbox record. +3. A nonmatching signal performs no warehouse read and creates no event. +4. Several active projections are evaluated from one registry snapshot, and all + matches run. +5. Integer, float, timestamp, duration, boolean, and string truth-table fixtures + pass with no implicit coercion. +6. Projection changes are validated, revisioned, persisted, and activated + atomically without restarting Maple Local. +7. Rules and ready/staged outbox records survive process restart and participate + in documented backup and recovery. +8. chDB query alerts retain their existing aggregate and lifecycle behavior. +9. No implementation requires a browser, a new broker, arbitrary runtime code, + raw SQL configuration, or a per-projection chDB poller. +10. The event envelope and selector fixture corpus are sufficient for a second + language implementation to demonstrate semantic parity. + +## Settled implementation choices + +The TypeScript reference implementation settles the remaining host choices as +follows: + +- Maple Local stores projection revisions, failures, and the staged/ready outbox + in SQLite at `/control/eventing.sqlite`, using WAL and `synchronous = +FULL`. A version-2 Maple checkpoint contains `control.sqlite` beside the chDB + backup and binds its byte count, SHA-256 digest, schema version, and row counts + in the checkpoint manifest. Version-1 checkpoints remain readable and restore + an empty control store. +- The first GitLab instrumentation contract is an OTLP log whose LogRecord + `eventName` is `gitlab.issue.created`. `event.id` is the preferred stable + source occurrence identifier; `cloudevents.id` and `gitlab.event.id` are + accepted aliases. The semantic projector requires `gitlab.project.path` and + integer `gitlab.issue.iid`; it also recognizes project/issue IDs, title, URL, + and actor attributes. GitLab itself does not synthesize this contract merely + because Maple is running: the emitting instrumentation or adapter must attach + those fields. +- Maple-owned event types use `dev.maple.*.v1`; schemas use + `urn:maple:event-schema:*:v1`. The current vertical emits + `dev.maple.gitlab.issue.created.v1` with + `urn:maple:event-schema:gitlab-issue-created:v1`. +- Attribute strings are limited to 16 KiB, source/event identities to 256 + characters (long stable inputs are represented by a SHA-256 URN), each + attribute namespace to 256 entries, + nested values to depth 8 and 1,024 nodes, normalized source data to 256 KiB, + and a canonical outbox CloudEvent to 256 KiB. Secret-like attribute names are + excluded from the projection field and data views. +- The Local TypeScript path is the reference live implementation. Hosted Rust + ingest remains a later adapter and must pass the shared schemas and fixture + corpus before claiming parity. +- Verified non-test PlanetScale webhooks run through a registered + `planetscale.webhook` source adapter, selector, and projector before the route + acknowledges them. The dedicated Cloudflare Queue durably carries + `dev.maple.planetscale.webhook.received.v1`; its temporary provider payload + keeps the existing issue and timeline consumers behaviorally unchanged while + they migrate to the event contract. +- Hosted query-alert delivery rows remain that producer's durable outbox. Their + payload now includes an additive deterministic + `dev.maple.alert.lifecycle.{trigger,resolve,renotify,test}.v1` CloudEvent while + retaining every legacy top-level delivery field. +- Historical replay execution remains deliberately unimplemented in this + change. Field catalogs already declare `exact`, `coerced`, or `unavailable`, + but Local's current arbitrary attribute maps have lost source scalar type and + its warehouse rows do not furnish a native occurrence ID. A later bounded, + operator-invoked replay adapter must require explicit coercion acknowledgement + and pass live-evaluator conformance tests; the live path never falls back to a + chDB poller in the meantime. +- Projector failures with a source occurrence ID are idempotent per projection + revision. Local retains a bounded newest 10,000 failure rows per tenant and + exposes the count through the authenticated headless health endpoint. A + projector failure does not retry a valid telemetry occurrence forever; + infrastructure failure to persist required state remains retryable. + +Maple Local activates immutable revisions with authenticated +`POST /local/eventing/projections`. The same maintenance credential protects +`GET /local/eventing/projections`, `/local/eventing/health`, and +`/local/eventing/outbox`. The outbox endpoint returns ready events by default; +`?state=staged` exposes bounded inspection of records stranded before the chDB +commit point. Re-delivery is the safe recovery operation: it deduplicates the +same staged event ID and promotes it only after the warehouse write succeeds. +Maple never blindly promotes an old staged record because, after a crash, the +control store alone cannot prove whether the corresponding chDB write committed. +Activation compiles the entire candidate registry +before the SQLite commit and swaps the immutable runtime snapshot while ingest +is quiesced, so a request observes exactly one registry version. diff --git a/packages/alerting-core/README.md b/packages/alerting-core/README.md index a464cce3a..0571b49e0 100644 --- a/packages/alerting-core/README.md +++ b/packages/alerting-core/README.md @@ -13,16 +13,28 @@ Current hosted adapters live in `apps/api` and are scheduled by queries, Local durable state, an in-process scheduler, and its own outbound URL policy without importing either hosted application. +This package covers scheduled aggregate alerts. Immediate per-occurrence events +use the separate ingest-time architecture described in +[`docs/signal-to-event-projection.md`](../../docs/signal-to-event-projection.md). +Both paths may ultimately publish through the same typed event outbox, but raw +signal matching does not poll chDB or impersonate an alert lifecycle. + The boundary is: - query adapter -> `AlertObservation`; - evaluation policy + observation -> `AlertEvaluation`; - persistence snapshot + evaluation -> `AlertLifecyclePlan`; -- host persists the plan and sends its optional `eventType` through a delivery - adapter; +- host persists the plan, projects its optional `eventType` into the common + CloudEvents envelope, and sends that event through a delivery adapter; - delivery adapters share idempotency-key and bounded retry policy helpers; - host clock supplies `nowMs`; the core never reads global time. Rule CRUD, storage schemas, scheduler claims, destination configuration, and delivery transports remain host concerns. This keeps Local UI work optional: the alert runtime can evaluate and deliver while no browser is open. + +Hosted alert delivery rows are the existing durable outbox for this producer. +Their additive `event` payload contains the deterministic +`dev.maple.alert.lifecycle.{trigger,resolve,renotify,test}.v1` envelope; current +destinations continue to receive the legacy top-level payload fields during the +migration. diff --git a/packages/alerting-core/package.json b/packages/alerting-core/package.json index 0ce27ea84..9c7034969 100644 --- a/packages/alerting-core/package.json +++ b/packages/alerting-core/package.json @@ -10,6 +10,9 @@ "test": "vitest run", "typecheck": "tsc --noEmit" }, + "dependencies": { + "@maple/eventing-core": "workspace:*" + }, "devDependencies": { "@types/node": "catalog:tooling", "typescript": "catalog:tooling", diff --git a/packages/alerting-core/src/index.test.ts b/packages/alerting-core/src/index.test.ts index 9ad7c26dc..f0786e57a 100644 --- a/packages/alerting-core/src/index.test.ts +++ b/packages/alerting-core/src/index.test.ts @@ -6,6 +6,7 @@ import { interleaveAlertRulesByTenant, makeAlertDeliveryKey, planAlertLifecycle, + projectAlertLifecycleEvent, type AlertEvaluation, } from "./index" @@ -177,6 +178,39 @@ describe("interleaveAlertRulesByTenant", () => { }) describe("delivery policy", () => { + it("projects lifecycle intents into deterministic common CloudEvents", () => { + const input = { + tenantId: "org-1", + ruleId: "rule-1", + ruleName: "High errors", + incidentId: "incident-1", + eventType: "trigger" as const, + incidentStatus: "open", + groupKey: "checkout", + signalType: "error_rate", + severity: "critical", + comparator: "gt" as const, + threshold: 5, + thresholdUpper: null, + windowMinutes: 5, + value: 7.2, + sampleCount: 12, + occurredAtMs: 1_786_131_720_123, + } + const event = projectAlertLifecycleEvent(input) + expect(event).toEqual(projectAlertLifecycleEvent(input)) + expect(event).toMatchObject({ + type: "dev.maple.alert.lifecycle.trigger.v1", + subject: "alert-incidents/incident-1", + tenantid: "org-1", + projectionid: "alert-lifecycle", + data: { eventType: "trigger", incidentId: "incident-1" }, + }) + expect(() => projectAlertLifecycleEvent({ ...input, occurredAtMs: Number.MAX_SAFE_INTEGER })).toThrow( + "outside the supported date range", + ) + }) + it("builds stable idempotency keys", () => { expect(makeAlertDeliveryKey("incident", "destination", "trigger", 42)).toBe( "incident:destination:trigger:42", diff --git a/packages/alerting-core/src/index.ts b/packages/alerting-core/src/index.ts index 4130d25b6..6d5b523d9 100644 --- a/packages/alerting-core/src/index.ts +++ b/packages/alerting-core/src/index.ts @@ -1,3 +1,5 @@ +import { makeCloudEvent, type MapleCloudEvent } from "@maple/eventing-core" + export type AlertComparator = "gt" | "gte" | "lt" | "lte" | "eq" | "neq" | "between" | "not_between" export type AlertEvaluationStatus = "breached" | "healthy" | "skipped" @@ -139,6 +141,87 @@ export type AlertIncidentTransition = "none" | "opened" | "continued" | "resolve export type AlertNotificationSuppression = "flapping" | "flap_resolution" | null export type AlertLifecycleHold = "missing_telemetry" | null +export interface AlertLifecycleEventInput { + readonly tenantId: string + readonly ruleId: string + readonly ruleName: string + readonly incidentId: string | null + readonly eventType: AlertEventType + readonly incidentStatus: string + readonly groupKey: string | null + readonly signalType: string + readonly severity: string + readonly comparator: AlertComparator + readonly threshold: number + readonly thresholdUpper: number | null + readonly windowMinutes: number + readonly value: number | null + readonly sampleCount: number | null + readonly occurredAtMs: number +} + +/** Project a query-alert lifecycle intent into the common factual event envelope. */ +export const projectAlertLifecycleEvent = (input: AlertLifecycleEventInput): MapleCloudEvent => { + if (!Number.isSafeInteger(input.occurredAtMs) || input.occurredAtMs < 0) + throw new Error("alert lifecycle event time must be a non-negative epoch millisecond") + const occurredAtDate = new Date(input.occurredAtMs) + if (Number.isNaN(occurredAtDate.getTime())) + throw new Error("alert lifecycle event time is outside the supported date range") + const occurredAt = occurredAtDate.toISOString() + const occurrenceId = `${input.incidentId ?? input.ruleId}:${input.eventType}:${input.occurredAtMs}` + return makeCloudEvent({ + signal: { + sourceKind: "alert.lifecycle", + source: `urn:maple:alert-rule:${input.ruleId}`, + tenantId: input.tenantId, + occurrenceId, + identityQuality: "source", + occurredAt, + observedAt: occurredAt, + subject: + input.incidentId === null + ? `alert-rules/${input.ruleId}` + : `alert-incidents/${input.incidentId}`, + fields: new Map(), + data: {}, + }, + projection: { + id: "alert-lifecycle", + revision: 1, + enabled: true, + tenantId: input.tenantId, + sourceKind: "alert.lifecycle", + selector: { + op: "exists", + field: { namespace: "signal", key: "event_type", type: "string" }, + }, + projector: { id: "alert.lifecycle", version: 1, config: {} }, + activeFrom: occurredAt, + }, + projectorId: "alert.lifecycle", + projectorVersion: 1, + outputType: `dev.maple.alert.lifecycle.${input.eventType}.v1`, + dataSchema: "urn:maple:event-schema:alert-lifecycle:v1", + data: { + eventType: input.eventType, + incidentId: input.incidentId, + incidentStatus: input.incidentStatus, + rule: { + id: input.ruleId, + name: input.ruleName, + signalType: input.signalType, + severity: input.severity, + groupKey: input.groupKey, + comparator: input.comparator, + threshold: input.threshold, + thresholdUpper: input.thresholdUpper, + windowMinutes: input.windowMinutes, + }, + observed: { value: input.value, sampleCount: input.sampleCount }, + }, + }) +} + export interface AlertLifecycleInput { readonly policy: AlertLifecyclePolicy readonly evaluation: AlertEvaluation diff --git a/packages/eventing-core/README.md b/packages/eventing-core/README.md new file mode 100644 index 000000000..9e5a883c3 --- /dev/null +++ b/packages/eventing-core/README.md @@ -0,0 +1,22 @@ +# `@maple/eventing-core` + +Host-neutral signal-to-event contracts and deterministic runtime semantics. + +The package owns typed signal values, bounded selectors, pure projector +registration, canonical event identity, and an immutable compiled projection +registry. It has no database, network, scheduler, or wall-clock dependency. A +host authenticates and normalizes source input, supplies durable projection and +outbox adapters, and decides when compiled registries become active. + +See [`docs/signal-to-event-projection.md`](../../docs/signal-to-event-projection.md) +for the architecture and acceptance contract. + +The versioned interoperability artifacts are generated under `schemas/`, with +valid comparison and identity vectors in `fixtures/v1.json`. Run `bun test` to +verify generated-schema drift, hostile selector bounds, typed comparison +semantics, deterministic event IDs, and projector isolation. + +The first host adapter is Maple Local in `apps/cli/src/server/eventing`. It uses +an authenticated configuration endpoint, a SQLite projection/outbox store, and +the pre-chDB OTLP seam. The package itself deliberately contains none of those +host decisions. diff --git a/packages/eventing-core/fixtures/v1.json b/packages/eventing-core/fixtures/v1.json new file mode 100644 index 000000000..dfc1c66da --- /dev/null +++ b/packages/eventing-core/fixtures/v1.json @@ -0,0 +1,177 @@ +{ + "version": 1, + "eventIdVectors": [ + { + "name": "tenant-scoped projected occurrence", + "input": { + "tenantId": "tenant-a", + "sourceKind": "otel.log", + "source": "urn:maple:source:otel:local", + "occurrenceId": "event-123", + "projectionId": "gitlab-issue-created", + "projectionRevision": 3 + }, + "output": "sha256:061c0b5d99b92ef65ab8813c6d84988e4b1582e705e0077c952e62a0e84b6b08" + } + ], + "predicateVectors": [ + { + "name": "int64 remains exact above JavaScript safe integer range", + "predicate": { + "op": "gt", + "field": { "namespace": "attribute", "key": "counter", "type": "int64" }, + "value": { "type": "int64", "value": "9007199254740992" } + }, + "fields": [ + { + "namespace": "attribute", + "key": "counter", + "value": { "type": "int64", "value": "9007199254740993" } + } + ], + "matches": true + }, + { + "name": "timestamps compare as UTC instants", + "predicate": { + "op": "eq", + "field": { "namespace": "signal", "key": "occurred_at", "type": "timestamp" }, + "value": { "type": "timestamp", "value": "2026-08-07T19:42:00.123456789Z" } + }, + "fields": [ + { + "namespace": "signal", + "key": "occurred_at", + "value": { "type": "timestamp", "value": "2026-08-07T15:42:00.123456789-04:00" } + } + ], + "matches": true + }, + { + "name": "numeric strings do not coerce", + "predicate": { + "op": "gte", + "field": { "namespace": "attribute", "key": "attempt", "type": "int64" }, + "value": { "type": "int64", "value": "3" } + }, + "fields": [ + { + "namespace": "attribute", + "key": "attempt", + "value": { "type": "string", "value": "12" } + } + ], + "matches": false, + "typeMismatches": ["attribute:attempt"] + }, + { + "name": "neq does not match a missing field", + "predicate": { + "op": "neq", + "field": { "namespace": "attribute", "key": "state", "type": "string" }, + "value": { "type": "string", "value": "closed" } + }, + "fields": [], + "matches": false + }, + { + "name": "boolean composition and string containment", + "predicate": { + "op": "all", + "clauses": [ + { + "op": "eq", + "field": { "namespace": "attribute", "key": "active", "type": "boolean" }, + "value": { "type": "boolean", "value": true } + }, + { + "op": "contains", + "field": { "namespace": "body", "key": "text", "type": "string" }, + "value": { "type": "string", "value": "issue created" } + } + ] + }, + "fields": [ + { + "namespace": "attribute", + "key": "active", + "value": { "type": "boolean", "value": true } + }, + { + "namespace": "body", + "key": "text", + "value": { "type": "string", "value": "gitlab issue created successfully" } + } + ], + "matches": true + }, + { + "name": "float64 ordering is numeric", + "predicate": { + "op": "lt", + "field": { "namespace": "attribute", "key": "ratio", "type": "float64" }, + "value": { "type": "float64", "value": 10.25 } + }, + "fields": [ + { + "namespace": "attribute", + "key": "ratio", + "value": { "type": "float64", "value": 9.5 } + } + ], + "matches": true + }, + { + "name": "durations compare as exact nanoseconds", + "predicate": { + "op": "gte", + "field": { "namespace": "signal", "key": "duration", "type": "duration" }, + "value": { "type": "duration", "value": "1000000000" } + }, + "fields": [ + { + "namespace": "signal", + "key": "duration", + "value": { "type": "duration", "value": "1000000001" } + } + ], + "matches": true + }, + { + "name": "boolean equality has no string coercion", + "predicate": { + "op": "eq", + "field": { "namespace": "attribute", "key": "enabled", "type": "boolean" }, + "value": { "type": "boolean", "value": true } + }, + "fields": [ + { + "namespace": "attribute", + "key": "enabled", + "value": { "type": "string", "value": "true" } + } + ], + "matches": false, + "typeMismatches": ["attribute:enabled"] + }, + { + "name": "string membership is exact and case-sensitive", + "predicate": { + "op": "in", + "field": { "namespace": "attribute", "key": "state", "type": "string" }, + "values": [ + { "type": "string", "value": "opened" }, + { "type": "string", "value": "closed" } + ] + }, + "fields": [ + { + "namespace": "attribute", + "key": "state", + "value": { "type": "string", "value": "Closed" } + } + ], + "matches": false + } + ] +} diff --git a/packages/eventing-core/package.json b/packages/eventing-core/package.json new file mode 100644 index 000000000..3dd387ac9 --- /dev/null +++ b/packages/eventing-core/package.json @@ -0,0 +1,24 @@ +{ + "name": "@maple/eventing-core", + "version": "0.0.0", + "private": true, + "type": "module", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "schemas": "bun run scripts/generate-schemas.ts", + "schemas:check": "bun run scripts/generate-schemas.ts --check", + "test": "bun run schemas:check && vitest run", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "effect": "catalog:effect" + }, + "devDependencies": { + "@effect/language-service": "catalog:effect", + "@types/node": "catalog:tooling", + "typescript": "catalog:tooling", + "vitest": "catalog:" + } +} diff --git a/packages/eventing-core/schemas/cloud-event.v1.schema.json b/packages/eventing-core/schemas/cloud-event.v1.schema.json new file mode 100644 index 000000000..adae790a5 --- /dev/null +++ b/packages/eventing-core/schemas/cloud-event.v1.schema.json @@ -0,0 +1,162 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:eventing:schema:cloud-event:v1", + "$ref": "#/$defs/MapleCloudEvent", + "$defs": { + "MapleCloudEvent": { + "type": "object", + "properties": { + "specversion": { + "type": "string", + "enum": ["1.0"] + }, + "id": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "source": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "type": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "subject": { + "type": "string" + }, + "time": { + "type": "string", + "allOf": [ + { + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{1,9})?(?:Z|[+-]\\d{2}:\\d{2})$" + } + ] + }, + "datacontenttype": { + "type": "string", + "enum": ["application/json"] + }, + "dataschema": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "tenantid": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "projectionid": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "projectionrevision": { + "type": "integer", + "allOf": [ + { + "exclusiveMinimum": 0 + } + ] + }, + "projectorid": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "projectorversion": { + "type": "integer", + "allOf": [ + { + "exclusiveMinimum": 0 + } + ] + }, + "data": {} + }, + "required": [ + "specversion", + "id", + "source", + "type", + "time", + "datacontenttype", + "dataschema", + "tenantid", + "projectionid", + "projectionrevision", + "projectorid", + "projectorversion", + "data" + ], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/signal-projection.v1.schema.json b/packages/eventing-core/schemas/signal-projection.v1.schema.json new file mode 100644 index 000000000..44f2adbae --- /dev/null +++ b/packages/eventing-core/schemas/signal-projection.v1.schema.json @@ -0,0 +1,349 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:eventing:schema:signal-projection:v1", + "$ref": "#/$defs/SignalProjectionSpec", + "$defs": { + "SignalFieldRef": { + "type": "object", + "properties": { + "namespace": { + "type": "string", + "enum": ["signal", "resource", "scope", "attribute", "body"] + }, + "key": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 512 + } + ] + }, + "type": { + "type": "string", + "enum": ["string", "boolean", "int64", "float64", "timestamp", "duration"] + } + }, + "required": ["namespace", "key", "type"], + "additionalProperties": false + }, + "SignalScalar": { + "anyOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["string"] + }, + "value": { + "type": "string" + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["boolean"] + }, + "value": { + "type": "boolean" + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["int64"] + }, + "value": { + "type": "string", + "allOf": [ + { + "pattern": "^-?(?:0|[1-9][0-9]*)$" + } + ] + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["float64"] + }, + "value": { + "type": "number" + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["timestamp"] + }, + "value": { + "type": "string", + "allOf": [ + { + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{1,9})?(?:Z|[+-]\\d{2}:\\d{2})$" + } + ] + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["duration"] + }, + "value": { + "type": "string", + "allOf": [ + { + "pattern": "^-?(?:0|[1-9][0-9]*)$" + } + ] + } + }, + "required": ["type", "value"], + "additionalProperties": false + } + ] + }, + "SignalPredicate": { + "anyOf": [ + { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": ["all"] + }, + "clauses": { + "type": "array", + "items": { + "$ref": "#/$defs/SignalPredicate" + } + } + }, + "required": ["op", "clauses"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": ["any"] + }, + "clauses": { + "type": "array", + "items": { + "$ref": "#/$defs/SignalPredicate" + } + } + }, + "required": ["op", "clauses"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": ["not"] + }, + "clause": { + "$ref": "#/$defs/SignalPredicate" + } + }, + "required": ["op", "clause"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": ["exists"] + }, + "field": { + "$ref": "#/$defs/SignalFieldRef" + } + }, + "required": ["op", "field"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": ["eq", "neq", "gt", "gte", "lt", "lte", "contains"] + }, + "field": { + "$ref": "#/$defs/SignalFieldRef" + }, + "value": { + "$ref": "#/$defs/SignalScalar" + } + }, + "required": ["op", "field", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": ["in"] + }, + "field": { + "$ref": "#/$defs/SignalFieldRef" + }, + "values": { + "type": "array", + "items": { + "$ref": "#/$defs/SignalScalar" + } + } + }, + "required": ["op", "field", "values"], + "additionalProperties": false + } + ] + }, + "SignalProjectionSpec": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "revision": { + "type": "integer", + "allOf": [ + { + "exclusiveMinimum": 0 + } + ] + }, + "enabled": { + "type": "boolean" + }, + "tenantId": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "sourceKind": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "selector": { + "$ref": "#/$defs/SignalPredicate" + }, + "projector": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "version": { + "type": "integer", + "allOf": [ + { + "exclusiveMinimum": 0 + } + ] + }, + "config": {} + }, + "required": ["id", "version", "config"], + "additionalProperties": false + }, + "activeFrom": { + "type": "string", + "allOf": [ + { + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{1,9})?(?:Z|[+-]\\d{2}:\\d{2})$" + } + ] + } + }, + "required": [ + "id", + "revision", + "enabled", + "tenantId", + "sourceKind", + "selector", + "projector", + "activeFrom" + ], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/signal-scalar.v1.schema.json b/packages/eventing-core/schemas/signal-scalar.v1.schema.json new file mode 100644 index 000000000..bbf5af1c1 --- /dev/null +++ b/packages/eventing-core/schemas/signal-scalar.v1.schema.json @@ -0,0 +1,110 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:eventing:schema:signal-scalar:v1", + "$ref": "#/$defs/SignalScalar", + "$defs": { + "SignalScalar": { + "anyOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["string"] + }, + "value": { + "type": "string" + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["boolean"] + }, + "value": { + "type": "boolean" + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["int64"] + }, + "value": { + "type": "string", + "allOf": [ + { + "pattern": "^-?(?:0|[1-9][0-9]*)$" + } + ] + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["float64"] + }, + "value": { + "type": "number" + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["timestamp"] + }, + "value": { + "type": "string", + "allOf": [ + { + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{1,9})?(?:Z|[+-]\\d{2}:\\d{2})$" + } + ] + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["duration"] + }, + "value": { + "type": "string", + "allOf": [ + { + "pattern": "^-?(?:0|[1-9][0-9]*)$" + } + ] + } + }, + "required": ["type", "value"], + "additionalProperties": false + } + ] + } + } +} diff --git a/packages/eventing-core/scripts/generate-schemas.ts b/packages/eventing-core/scripts/generate-schemas.ts new file mode 100644 index 000000000..2ea54a516 --- /dev/null +++ b/packages/eventing-core/scripts/generate-schemas.ts @@ -0,0 +1,61 @@ +import { execFileSync } from "node:child_process" +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs" +import { dirname, resolve } from "node:path" +import { Schema } from "effect" +import { MapleCloudEventSchema, SignalProjectionSpecSchema, SignalScalarSchema } from "../src/model" + +const root = resolve(import.meta.dirname, "..") +const check = process.argv.includes("--check") + +const documents = [ + { + path: "schemas/signal-scalar.v1.schema.json", + id: "urn:maple:eventing:schema:signal-scalar:v1", + schema: SignalScalarSchema, + }, + { + path: "schemas/signal-projection.v1.schema.json", + id: "urn:maple:eventing:schema:signal-projection:v1", + schema: SignalProjectionSpecSchema, + }, + { + path: "schemas/cloud-event.v1.schema.json", + id: "urn:maple:eventing:schema:cloud-event:v1", + schema: MapleCloudEventSchema, + }, +] as const + +let stale = false +for (const entry of documents) { + const document = Schema.toJsonSchemaDocument(entry.schema) + const unformatted = `${JSON.stringify( + { + $schema: "https://json-schema.org/draft/2020-12/schema", + $id: entry.id, + ...document.schema, + ...(Object.keys(document.definitions).length === 0 ? {} : { $defs: document.definitions }), + }, + null, + "\t", + )}\n` + const serialized = execFileSync( + resolve(root, "../../node_modules/.bin/oxfmt"), + ["--stdin-filepath", entry.path], + { + input: unformatted, + encoding: "utf8", + }, + ) + const path = resolve(root, entry.path) + if (check) { + if (!existsSync(path) || readFileSync(path, "utf8") !== serialized) { + console.error(`${entry.path} is stale; run bun run schemas`) + stale = true + } + } else { + mkdirSync(dirname(path), { recursive: true }) + writeFileSync(path, serialized) + } +} + +if (stale) process.exitCode = 1 diff --git a/packages/eventing-core/src/event.ts b/packages/eventing-core/src/event.ts new file mode 100644 index 000000000..e70308e78 --- /dev/null +++ b/packages/eventing-core/src/event.ts @@ -0,0 +1,117 @@ +import { createHash } from "node:crypto" +import type { JsonValue, MapleCloudEvent, NormalizedSignal, SignalProjectionSpec } from "./model" +import { timestampToEpochNanos } from "./predicate" + +export interface EventIdentityInput { + readonly tenantId: string + readonly sourceKind: string + readonly source: string + readonly occurrenceId: string + readonly projectionId: string + readonly projectionRevision: number +} + +const updateLengthDelimited = (hash: ReturnType, value: string): void => { + const encoded = Buffer.from(value, "utf8") + const length = Buffer.allocUnsafe(4) + length.writeUInt32BE(encoded.byteLength) + hash.update(length) + hash.update(encoded) +} + +/** Canonical v1 identity shared by every host implementation. */ +export const makeEventId = (input: EventIdentityInput): string => { + const hash = createHash("sha256") + for (const field of [ + "maple-event-v1", + input.tenantId, + input.sourceKind, + input.source, + input.occurrenceId, + input.projectionId, + String(input.projectionRevision), + ]) + updateLengthDelimited(hash, field) + return `sha256:${hash.digest("hex")}` +} + +export const isJsonValue = (value: unknown, seen: Set = new Set()): value is JsonValue => { + if (value === null || typeof value === "string" || typeof value === "boolean") return true + if (typeof value === "number") return Number.isFinite(value) + if (typeof value !== "object") return false + if (seen.has(value)) return false + seen.add(value) + try { + if (Array.isArray(value)) return value.every((item) => isJsonValue(item, seen)) + const prototype = Object.getPrototypeOf(value) + if (prototype !== Object.prototype && prototype !== null) return false + return Object.values(value).every((item) => isJsonValue(item, seen)) + } finally { + // Track the active recursion path. Repeated references serialize as a + // JSON tree and are not themselves cycles. + seen.delete(value) + } +} + +const canonicalizeJson = (value: JsonValue): JsonValue => { + if (value === null || typeof value !== "object") return value + if (Array.isArray(value)) return value.map(canonicalizeJson) + const record = value as { readonly [key: string]: JsonValue } + return Object.fromEntries( + Object.keys(record) + .sort() + .map((key) => [key, canonicalizeJson(record[key]!)]), + ) +} + +/** Stable JSON encoding for outbox collision checks and cross-host fixtures. */ +export const canonicalJson = (value: JsonValue): string => { + if (!isJsonValue(value)) throw new Error("value must be finite acyclic JSON") + return JSON.stringify(canonicalizeJson(value)) +} + +export const makeCloudEvent = (input: { + readonly signal: NormalizedSignal + readonly projection: SignalProjectionSpec + readonly projectorId: string + readonly projectorVersion: number + readonly outputType: string + readonly dataSchema: string + readonly subject?: string | null + readonly time?: string + readonly data: JsonValue +}): MapleCloudEvent => { + if (input.signal.occurrenceId === null || input.signal.identityQuality === "none") + throw new Error("durable event projection requires stable or derived occurrence identity") + if (!isJsonValue(input.data)) throw new Error("projected event data must be finite JSON") + if (input.outputType.length === 0) throw new Error("projected event type must not be empty") + if (input.dataSchema.length === 0) throw new Error("projected event data schema must not be empty") + if (input.signal.source.length === 0) throw new Error("signal source must not be empty") + + const subject = input.subject ?? input.signal.subject + const time = input.time ?? input.signal.occurredAt + if (timestampToEpochNanos(time) === null) throw new Error("projected event time must be a valid instant") + return { + specversion: "1.0", + id: makeEventId({ + tenantId: input.signal.tenantId, + sourceKind: input.signal.sourceKind, + source: input.signal.source, + occurrenceId: input.signal.occurrenceId, + projectionId: input.projection.id, + projectionRevision: input.projection.revision, + }), + source: input.signal.source, + type: input.outputType, + ...(subject == null ? {} : { subject }), + time, + datacontenttype: "application/json", + dataschema: input.dataSchema, + tenantid: input.signal.tenantId, + projectionid: input.projection.id, + projectionrevision: input.projection.revision, + projectorid: input.projectorId, + projectorversion: input.projectorVersion, + data: input.data, + } +} diff --git a/packages/eventing-core/src/index.ts b/packages/eventing-core/src/index.ts new file mode 100644 index 000000000..87253218d --- /dev/null +++ b/packages/eventing-core/src/index.ts @@ -0,0 +1,5 @@ +export * from "./event" +export * from "./model" +export * from "./predicate" +export * from "./registry" +export * from "./source" diff --git a/packages/eventing-core/src/model.ts b/packages/eventing-core/src/model.ts new file mode 100644 index 000000000..9b291b5e4 --- /dev/null +++ b/packages/eventing-core/src/model.ts @@ -0,0 +1,219 @@ +import { Schema } from "effect" + +const NonEmptyIdentifier = Schema.String.check( + Schema.isMinLength(1), + Schema.isMaxLength(256), + Schema.isTrimmed(), +) + +const DecimalInt64 = Schema.String.check(Schema.isPattern(/^-?(?:0|[1-9][0-9]*)$/)) + +const Rfc3339Timestamp = Schema.String.check( + Schema.isPattern(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?(?:Z|[+-]\d{2}:\d{2})$/), +) + +export const StringSignalScalar = Schema.Struct({ + type: Schema.Literal("string"), + value: Schema.String, +}) + +export const BooleanSignalScalar = Schema.Struct({ + type: Schema.Literal("boolean"), + value: Schema.Boolean, +}) + +export const Int64SignalScalar = Schema.Struct({ + type: Schema.Literal("int64"), + value: DecimalInt64, +}) + +export const Float64SignalScalar = Schema.Struct({ + type: Schema.Literal("float64"), + value: Schema.Finite, +}) + +export const TimestampSignalScalar = Schema.Struct({ + type: Schema.Literal("timestamp"), + value: Rfc3339Timestamp, +}) + +export const DurationSignalScalar = Schema.Struct({ + type: Schema.Literal("duration"), + value: DecimalInt64, +}) + +export const SignalScalarSchema = Schema.Union([ + StringSignalScalar, + BooleanSignalScalar, + Int64SignalScalar, + Float64SignalScalar, + TimestampSignalScalar, + DurationSignalScalar, +]).annotate({ identifier: "SignalScalar" }) +export type SignalScalar = Schema.Schema.Type +export type SignalScalarType = SignalScalar["type"] + +export const FieldNamespaceSchema = Schema.Literals(["signal", "resource", "scope", "attribute", "body"]) +export type FieldNamespace = Schema.Schema.Type + +export const FieldRefSchema = Schema.Struct({ + namespace: FieldNamespaceSchema, + key: Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(512)), + type: Schema.Literals(["string", "boolean", "int64", "float64", "timestamp", "duration"]), +}).annotate({ identifier: "SignalFieldRef" }) +export type FieldRef = Schema.Schema.Type + +export interface AllPredicate { + readonly op: "all" + readonly clauses: readonly SignalPredicate[] +} + +export interface AnyPredicate { + readonly op: "any" + readonly clauses: readonly SignalPredicate[] +} + +export interface NotPredicate { + readonly op: "not" + readonly clause: SignalPredicate +} + +export interface ExistsPredicate { + readonly op: "exists" + readonly field: FieldRef +} + +export interface ComparisonPredicate { + readonly op: "eq" | "neq" | "gt" | "gte" | "lt" | "lte" | "contains" + readonly field: FieldRef + readonly value: SignalScalar +} + +export interface InPredicate { + readonly op: "in" + readonly field: FieldRef + readonly values: readonly SignalScalar[] +} + +export type SignalPredicate = + | AllPredicate + | AnyPredicate + | NotPredicate + | ExistsPredicate + | ComparisonPredicate + | InPredicate + +export const SignalPredicateSchema: Schema.Codec = Schema.suspend( + (): Schema.Codec => + Schema.Union([ + Schema.Struct({ + op: Schema.Literal("all"), + clauses: Schema.Array(SignalPredicateSchema), + }), + Schema.Struct({ + op: Schema.Literal("any"), + clauses: Schema.Array(SignalPredicateSchema), + }), + Schema.Struct({ + op: Schema.Literal("not"), + clause: SignalPredicateSchema, + }), + Schema.Struct({ + op: Schema.Literal("exists"), + field: FieldRefSchema, + }), + Schema.Struct({ + op: Schema.Literals(["eq", "neq", "gt", "gte", "lt", "lte", "contains"]), + field: FieldRefSchema, + value: SignalScalarSchema, + }), + Schema.Struct({ + op: Schema.Literal("in"), + field: FieldRefSchema, + values: Schema.Array(SignalScalarSchema), + }), + ]) as Schema.Codec, +).annotate({ identifier: "SignalPredicate" }) + +export const ProjectorRefSchema = Schema.Struct({ + id: NonEmptyIdentifier, + version: Schema.Int.check(Schema.isGreaterThan(0)), + config: Schema.Unknown, +}) +export type ProjectorRef = Schema.Schema.Type + +export const SignalProjectionSpecSchema = Schema.Struct({ + id: NonEmptyIdentifier, + revision: Schema.Int.check(Schema.isGreaterThan(0)), + enabled: Schema.Boolean, + tenantId: NonEmptyIdentifier, + sourceKind: NonEmptyIdentifier, + selector: SignalPredicateSchema, + projector: ProjectorRefSchema, + activeFrom: Rfc3339Timestamp, +}).annotate({ identifier: "SignalProjectionSpec" }) +export type SignalProjectionSpec = Schema.Schema.Type + +export interface NormalizedSignal { + readonly sourceKind: string + readonly source: string + readonly tenantId: string + readonly occurrenceId: string | null + readonly identityQuality: "source" | "derived" | "none" + readonly occurredAt: string + readonly observedAt: string + readonly subject: string | null + readonly fields: ReadonlyMap + readonly data: TData +} + +export type JsonPrimitive = string | number | boolean | null +export type JsonValue = JsonPrimitive | { readonly [key: string]: JsonValue } | readonly JsonValue[] + +export interface ProjectedEventData { + readonly subject?: string | null + readonly time?: string + readonly data: JsonValue +} + +export interface MapleCloudEvent { + readonly specversion: "1.0" + readonly id: string + readonly source: string + readonly type: string + readonly subject?: string + readonly time: string + readonly datacontenttype: "application/json" + readonly dataschema: string + readonly tenantid: string + readonly projectionid: string + readonly projectionrevision: number + readonly projectorid: string + readonly projectorversion: number + readonly data: JsonValue +} + +export const MapleCloudEventSchema = Schema.Struct({ + specversion: Schema.Literal("1.0"), + id: NonEmptyIdentifier, + source: NonEmptyIdentifier, + type: NonEmptyIdentifier, + subject: Schema.optionalKey(Schema.String), + time: Rfc3339Timestamp, + datacontenttype: Schema.Literal("application/json"), + dataschema: NonEmptyIdentifier, + tenantid: NonEmptyIdentifier, + projectionid: NonEmptyIdentifier, + projectionrevision: Schema.Int.check(Schema.isGreaterThan(0)), + projectorid: NonEmptyIdentifier, + projectorversion: Schema.Int.check(Schema.isGreaterThan(0)), + data: Schema.Unknown, +}).annotate({ identifier: "MapleCloudEvent" }) + +export const fieldKey = (field: Pick): string => + `${field.namespace}:${field.key}` + +export const defineSignalFields = ( + fields: ReadonlyArray<{ readonly field: FieldRef; readonly value: SignalScalar }>, +): ReadonlyMap => + new Map(fields.map(({ field, value }) => [fieldKey(field), value] as const)) diff --git a/packages/eventing-core/src/predicate.test.ts b/packages/eventing-core/src/predicate.test.ts new file mode 100644 index 000000000..c4cf98362 --- /dev/null +++ b/packages/eventing-core/src/predicate.test.ts @@ -0,0 +1,148 @@ +import { readFileSync } from "node:fs" +import { Schema } from "effect" +import { describe, expect, it } from "vitest" +import { + compileSignalPredicate, + defineSignalFields, + fieldKey, + makeEventId, + MAX_PREDICATE_DEPTH, + SignalPredicateSchema, + SignalScalarSchema, + timestampToEpochNanos, + validateSignalPredicate, + type EventIdentityInput, + type FieldNamespace, + type FieldRef, + type NormalizedSignal, +} from "./index" + +interface ConformanceFixture { + readonly eventIdVectors: ReadonlyArray<{ + readonly name: string + readonly input: EventIdentityInput + readonly output: string + }> + readonly predicateVectors: ReadonlyArray<{ + readonly name: string + readonly predicate: unknown + readonly fields: ReadonlyArray<{ + readonly namespace: FieldNamespace + readonly key: string + readonly value: unknown + }> + readonly matches: boolean + readonly typeMismatches?: readonly string[] + }> +} + +const fixture = JSON.parse( + readFileSync(new URL("../fixtures/v1.json", import.meta.url), "utf8"), +) as ConformanceFixture + +const signalFor = (fields: ConformanceFixture["predicateVectors"][number]["fields"]): NormalizedSignal => ({ + sourceKind: "otel.log", + source: "urn:maple:source:otel:local", + tenantId: "tenant-a", + occurrenceId: "occurrence-1", + identityQuality: "source", + occurredAt: "2026-08-07T19:42:00Z", + observedAt: "2026-08-07T19:42:01Z", + subject: null, + fields: defineSignalFields( + fields.map(({ namespace, key, value }) => ({ + field: { + namespace, + key, + type: Schema.decodeUnknownSync(SignalScalarSchema)(value).type, + }, + value: Schema.decodeUnknownSync(SignalScalarSchema)(value), + })), + ), + data: {}, +}) + +describe("cross-language conformance vectors", () => { + for (const vector of fixture.eventIdVectors) { + it(`event ID: ${vector.name}`, () => { + expect(makeEventId(vector.input)).toBe(vector.output) + }) + } + + for (const vector of fixture.predicateVectors) { + it(`predicate: ${vector.name}`, () => { + const predicate = Schema.decodeUnknownSync(SignalPredicateSchema)(vector.predicate) + const result = compileSignalPredicate(predicate)(signalFor(vector.fields)) + expect(result.matches).toBe(vector.matches) + expect(result.typeMismatches.map(fieldKey)).toEqual(vector.typeMismatches ?? []) + }) + } +}) + +describe("selector validation", () => { + it("rejects wrong literal types and unsupported ordering", () => { + const field: FieldRef = { namespace: "attribute", key: "enabled", type: "boolean" } + expect( + validateSignalPredicate({ op: "gt", field, value: { type: "string", value: "true" } }), + ).toEqual( + expect.arrayContaining([ + expect.objectContaining({ message: "gt is not supported for boolean" }), + expect.objectContaining({ message: "field and literal types must match" }), + ]), + ) + }) + + it("rejects empty combinators and excessive nesting", () => { + expect(validateSignalPredicate({ op: "all", clauses: [] })).toContainEqual({ + path: "selector.clauses", + message: "all requires at least one clause", + }) + + let nested = { + op: "exists" as const, + field: { namespace: "attribute" as const, key: "x", type: "string" as const }, + } + for (let i = 0; i < MAX_PREDICATE_DEPTH; i++) nested = { op: "not", clause: nested } as never + expect(validateSignalPredicate(nested)).toEqual( + expect.arrayContaining([expect.objectContaining({ message: `predicate depth exceeds 8` })]), + ) + }) + + it("rejects invalid calendar dates and int64 overflow", () => { + expect(timestampToEpochNanos("2026-02-31T00:00:00Z")).toBeNull() + expect( + validateSignalPredicate({ + op: "eq", + field: { namespace: "attribute", key: "n", type: "int64" }, + value: { type: "int64", value: "9223372036854775808" }, + }), + ).toContainEqual( + expect.objectContaining({ message: "int64 must be a signed 64-bit decimal integer" }), + ) + }) +}) + +describe("total runtime behavior", () => { + it("treats malformed source scalars as mismatches rather than throwing", () => { + const field: FieldRef = { namespace: "attribute", key: "n", type: "int64" } + const evaluate = compileSignalPredicate({ + op: "gte", + field, + value: { type: "int64", value: "1" }, + }) + const signal = signalFor([]) + const fields = new Map(signal.fields) + fields.set(fieldKey(field), { type: "int64", value: "not-an-integer" }) + expect(evaluate({ ...signal, fields })).toMatchObject({ + matches: false, + typeMismatches: [field], + }) + }) + + it("distinguishes neq from not(eq) for a missing field", () => { + const field: FieldRef = { namespace: "attribute", key: "state", type: "string" } + const eq = { op: "eq" as const, field, value: { type: "string" as const, value: "closed" } } + expect(compileSignalPredicate({ ...eq, op: "neq" })(signalFor([])).matches).toBe(false) + expect(compileSignalPredicate({ op: "not", clause: eq })(signalFor([])).matches).toBe(true) + }) +}) diff --git a/packages/eventing-core/src/predicate.ts b/packages/eventing-core/src/predicate.ts new file mode 100644 index 000000000..ede5adf13 --- /dev/null +++ b/packages/eventing-core/src/predicate.ts @@ -0,0 +1,341 @@ +import type { + FieldRef, + NormalizedSignal, + SignalPredicate, + SignalProjectionSpec, + SignalScalar, + SignalScalarType, +} from "./model" +import { fieldKey } from "./model" + +export const MAX_PREDICATE_DEPTH = 8 +export const MAX_PREDICATE_NODES = 64 +export const MAX_IN_VALUES = 100 +export const MAX_STRING_LITERAL_BYTES = 4 * 1024 + +const INT64_MIN = -(1n << 63n) +const INT64_MAX = (1n << 63n) - 1n +const ORDERED_TYPES = new Set(["int64", "float64", "timestamp", "duration"]) + +export interface ValidationIssue { + readonly path: string + readonly message: string +} + +export class SignalPredicateValidationError extends Error { + readonly issues: readonly ValidationIssue[] + + constructor(issues: readonly ValidationIssue[]) { + super(issues.map(({ path, message }) => `${path}: ${message}`).join("; ")) + this.name = "SignalPredicateValidationError" + this.issues = issues + } +} + +const stringBytes = (value: string): number => new TextEncoder().encode(value).byteLength + +const parseInt64 = (value: string): bigint | null => { + try { + const parsed = BigInt(value) + return parsed >= INT64_MIN && parsed <= INT64_MAX ? parsed : null + } catch { + return null + } +} + +const isLeapYear = (year: number): boolean => year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0) + +const daysInMonth = (year: number, month: number): number => { + switch (month) { + case 2: + return isLeapYear(year) ? 29 : 28 + case 4: + case 6: + case 9: + case 11: + return 30 + default: + return 31 + } +} + +const TIMESTAMP = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,9}))?(Z|([+-])(\d{2}):(\d{2}))$/ + +/** Parse the v1 RFC 3339 subset into exact UTC nanoseconds. */ +export const timestampToEpochNanos = (value: string): bigint | null => { + const match = TIMESTAMP.exec(value) + if (!match) return null + const year = Number(match[1]) + const month = Number(match[2]) + const day = Number(match[3]) + const hour = Number(match[4]) + const minute = Number(match[5]) + const second = Number(match[6]) + const fraction = match[7] ?? "" + if ( + month < 1 || + month > 12 || + day < 1 || + day > daysInMonth(year, month) || + hour > 23 || + minute > 59 || + second > 59 + ) + return null + + let offsetMinutes = 0 + if (match[8] !== "Z") { + const offsetHours = Number(match[10]) + const offsetMinutePart = Number(match[11]) + if (offsetHours > 23 || offsetMinutePart > 59) return null + offsetMinutes = offsetHours * 60 + offsetMinutePart + if (match[9] === "-") offsetMinutes = -offsetMinutes + } + + const date = new Date(0) + date.setUTCFullYear(year, month - 1, day) + date.setUTCHours(hour, minute, second, 0) + const milliseconds = date.getTime() - offsetMinutes * 60_000 + if (!Number.isFinite(milliseconds)) return null + const nanos = BigInt(fraction.padEnd(9, "0")) + return BigInt(milliseconds) * 1_000_000n + nanos +} + +export const validateSignalScalar = (scalar: SignalScalar, path = "value"): readonly ValidationIssue[] => { + const issues: ValidationIssue[] = [] + switch (scalar.type) { + case "string": + if (stringBytes(scalar.value) > MAX_STRING_LITERAL_BYTES) + issues.push({ path, message: `string exceeds ${MAX_STRING_LITERAL_BYTES} UTF-8 bytes` }) + break + case "boolean": + break + case "int64": + case "duration": + if (parseInt64(scalar.value) === null) + issues.push({ path, message: `${scalar.type} must be a signed 64-bit decimal integer` }) + break + case "float64": + if (!Number.isFinite(scalar.value)) issues.push({ path, message: "float64 must be finite" }) + break + case "timestamp": + if (timestampToEpochNanos(scalar.value) === null) + issues.push({ + path, + message: "timestamp must be a valid RFC 3339 instant with an explicit offset", + }) + break + } + return issues +} + +export const validateSignalPredicate = (predicate: SignalPredicate): readonly ValidationIssue[] => { + const issues: ValidationIssue[] = [] + let nodes = 0 + + const visit = (node: SignalPredicate, path: string, depth: number): void => { + nodes += 1 + if (nodes > MAX_PREDICATE_NODES) return + if (depth > MAX_PREDICATE_DEPTH) { + issues.push({ path, message: `predicate depth exceeds ${MAX_PREDICATE_DEPTH}` }) + return + } + + switch (node.op) { + case "all": + case "any": + if (node.clauses.length === 0) + issues.push({ + path: `${path}.clauses`, + message: `${node.op} requires at least one clause`, + }) + for (let i = 0; i < node.clauses.length; i++) + visit(node.clauses[i]!, `${path}.clauses[${i}]`, depth + 1) + break + case "not": + visit(node.clause, `${path}.clause`, depth + 1) + break + case "exists": + break + case "contains": + if (node.field.type !== "string" || node.value.type !== "string") + issues.push({ path, message: "contains requires a string field and string literal" }) + issues.push(...validateSignalScalar(node.value, `${path}.value`)) + break + case "gt": + case "gte": + case "lt": + case "lte": + if (!ORDERED_TYPES.has(node.field.type)) + issues.push({ path, message: `${node.op} is not supported for ${node.field.type}` }) + if (node.field.type !== node.value.type) + issues.push({ path, message: "field and literal types must match" }) + issues.push(...validateSignalScalar(node.value, `${path}.value`)) + break + case "eq": + case "neq": + if (node.field.type !== node.value.type) + issues.push({ path, message: "field and literal types must match" }) + issues.push(...validateSignalScalar(node.value, `${path}.value`)) + break + case "in": + if (node.values.length === 0) + issues.push({ path: `${path}.values`, message: "in requires at least one value" }) + if (node.values.length > MAX_IN_VALUES) + issues.push({ path: `${path}.values`, message: `in exceeds ${MAX_IN_VALUES} values` }) + for (let i = 0; i < node.values.length; i++) { + const value = node.values[i]! + if (value.type !== node.field.type) + issues.push({ + path: `${path}.values[${i}]`, + message: "field and literal types must match", + }) + issues.push(...validateSignalScalar(value, `${path}.values[${i}]`)) + } + break + } + } + + visit(predicate, "selector", 1) + if (nodes > MAX_PREDICATE_NODES) + issues.push({ path: "selector", message: `predicate exceeds ${MAX_PREDICATE_NODES} nodes` }) + return issues +} + +export const assertValidSignalPredicate = (predicate: SignalPredicate): void => { + const issues = validateSignalPredicate(predicate) + if (issues.length > 0) throw new SignalPredicateValidationError(issues) +} + +export const validateSignalProjectionSpec = ( + projection: SignalProjectionSpec, +): readonly ValidationIssue[] => [ + ...(timestampToEpochNanos(projection.activeFrom) === null + ? [{ path: "activeFrom", message: "must be a valid RFC 3339 instant with an explicit offset" }] + : []), + ...validateSignalPredicate(projection.selector), +] + +export interface PredicateEvaluation { + readonly matches: boolean + readonly typeMismatches: readonly FieldRef[] +} + +const scalarEquals = (left: SignalScalar, right: SignalScalar): boolean => { + if (left.type !== right.type) return false + switch (left.type) { + case "string": + return right.type === "string" && left.value === right.value + case "boolean": + return right.type === "boolean" && left.value === right.value + case "float64": + return right.type === "float64" && left.value === right.value + case "int64": + return right.type === "int64" && BigInt(left.value) === BigInt(right.value) + case "duration": + return right.type === "duration" && BigInt(left.value) === BigInt(right.value) + case "timestamp": + return ( + right.type === "timestamp" && + timestampToEpochNanos(left.value) === timestampToEpochNanos(right.value) + ) + } +} + +const scalarOrder = (left: SignalScalar, right: SignalScalar): number | null => { + if (left.type !== right.type || !ORDERED_TYPES.has(left.type)) return null + switch (left.type) { + case "int64": { + if (right.type !== "int64") return null + const a = BigInt(left.value) + const b = BigInt(right.value) + return a < b ? -1 : a > b ? 1 : 0 + } + case "duration": { + if (right.type !== "duration") return null + const a = BigInt(left.value) + const b = BigInt(right.value) + return a < b ? -1 : a > b ? 1 : 0 + } + case "float64": + return right.type !== "float64" + ? null + : left.value < right.value + ? -1 + : left.value > right.value + ? 1 + : 0 + case "timestamp": { + if (right.type !== "timestamp") return null + const a = timestampToEpochNanos(left.value)! + const b = timestampToEpochNanos(right.value)! + return a < b ? -1 : a > b ? 1 : 0 + } + default: + return null + } +} + +export type CompiledSignalPredicate = (signal: NormalizedSignal) => PredicateEvaluation + +export const compileSignalPredicate = (predicate: SignalPredicate): CompiledSignalPredicate => { + assertValidSignalPredicate(predicate) + + return (signal) => { + const typeMismatches: FieldRef[] = [] + const readField = (field: FieldRef): SignalScalar | undefined => { + const value = signal.fields.get(fieldKey(field)) + if (value === undefined) return undefined + if (value.type !== field.type || validateSignalScalar(value).length > 0) { + typeMismatches.push(field) + return undefined + } + return value + } + + const evaluate = (node: SignalPredicate): boolean => { + switch (node.op) { + case "all": + return node.clauses.every(evaluate) + case "any": + return node.clauses.some(evaluate) + case "not": + return !evaluate(node.clause) + case "exists": { + return readField(node.field) !== undefined + } + case "eq": + case "neq": + case "gt": + case "gte": + case "lt": + case "lte": + case "contains": { + const value = readField(node.field) + if (value === undefined) return false + if (node.op === "eq") return scalarEquals(value, node.value) + if (node.op === "neq") return !scalarEquals(value, node.value) + if (node.op === "contains") + return ( + value.type === "string" && + node.value.type === "string" && + value.value.includes(node.value.value) + ) + const order = scalarOrder(value, node.value) + if (order === null) return false + if (node.op === "gt") return order > 0 + if (node.op === "gte") return order >= 0 + if (node.op === "lt") return order < 0 + return order <= 0 + } + case "in": { + const value = readField(node.field) + if (value === undefined) return false + return node.values.some((candidate) => scalarEquals(value, candidate)) + } + } + } + + return { matches: evaluate(predicate), typeMismatches } + } +} diff --git a/packages/eventing-core/src/registry.test.ts b/packages/eventing-core/src/registry.test.ts new file mode 100644 index 000000000..e7bfd3a9a --- /dev/null +++ b/packages/eventing-core/src/registry.test.ts @@ -0,0 +1,261 @@ +import { describe, expect, it } from "vitest" +import { + CompiledProjectionRegistry, + canonicalJson, + defineSignalFields, + makeEventId, + ProjectorRegistry, + SignalSourceRegistry, + type NormalizedSignal, + type SignalProjectionSpec, +} from "./index" + +const signal = (overrides: Partial = {}): NormalizedSignal => ({ + sourceKind: "otel.log", + source: "urn:maple:source:otel:local", + tenantId: "tenant-a", + occurrenceId: "event-123", + identityQuality: "source", + occurredAt: "2026-08-07T19:42:00.123456789Z", + observedAt: "2026-08-07T19:42:01Z", + subject: "project/example/issues/42", + fields: defineSignalFields([ + { + field: { namespace: "attribute", key: "event.name", type: "string" }, + value: { type: "string", value: "gitlab.issue.created" }, + }, + ]), + data: { issue: { iid: 42, title: "Example" } }, + ...overrides, +}) + +const projection = (overrides: Partial = {}): SignalProjectionSpec => ({ + id: "gitlab-issue-created", + revision: 3, + enabled: true, + tenantId: "tenant-a", + sourceKind: "otel.log", + selector: { + op: "eq", + field: { namespace: "attribute", key: "event.name", type: "string" }, + value: { type: "string", value: "gitlab.issue.created" }, + }, + projector: { id: "gitlab.issue", version: 1, config: { includeTitle: true } }, + activeFrom: "2026-08-07T00:00:00Z", + ...overrides, +}) + +const projectors = (): ProjectorRegistry => + new ProjectorRegistry().register({ + id: "gitlab.issue", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.gitlab.issue.created.v1", + dataSchema: "urn:maple:event-schema:gitlab-issue:v1", + decodeConfig: (value) => { + if (typeof value !== "object" || value === null) throw new Error("invalid projector config") + return value + }, + project: (input) => ({ data: input.data as { issue: { iid: number; title: string } } }), + }) + +const sources = (): SignalSourceRegistry => + new SignalSourceRegistry().register({ + sourceKind: "otel.log", + fields: [ + { + field: { namespace: "attribute", key: "event.name", type: "string" }, + operators: ["exists", "eq", "neq", "contains", "in"], + sensitivity: "public", + replay: "coerced", + }, + ], + openFields: [ + { + namespace: "attribute", + types: ["string", "boolean", "int64", "float64", "timestamp", "duration"], + operators: ["exists", "eq", "neq", "gt", "gte", "lt", "lte", "contains", "in"], + sensitivity: "public", + replay: "coerced", + }, + ], + }) + +describe("CompiledProjectionRegistry", () => { + it("canonicalizes JSON independently of object insertion order", () => { + expect(canonicalJson({ z: 1, nested: { b: true, a: [2, 1] }, a: "first" })).toBe( + '{"a":"first","nested":{"a":[2,1],"b":true},"z":1}', + ) + const shared = { value: 1 } + expect(canonicalJson({ left: shared, right: shared })).toBe( + '{"left":{"value":1},"right":{"value":1}}', + ) + const cyclic: { self?: unknown } = {} + cyclic.self = cyclic + expect(() => canonicalJson(cyclic as never)).toThrow("finite acyclic JSON") + expect(() => canonicalJson({ invalid: Number.NaN })).toThrow("finite acyclic JSON") + }) + + it("projects every match into a deterministic CloudEvent", () => { + const registry = CompiledProjectionRegistry.compile([projection()], sources(), projectors()) + const first = registry.evaluate(signal()) + const second = registry.evaluate(signal()) + expect(first.failures).toEqual([]) + expect(first.events).toEqual(second.events) + expect(first.events).toHaveLength(1) + expect(first.events[0]).toMatchObject({ + specversion: "1.0", + id: makeEventId({ + tenantId: "tenant-a", + sourceKind: "otel.log", + source: "urn:maple:source:otel:local", + occurrenceId: "event-123", + projectionId: "gitlab-issue-created", + projectionRevision: 3, + }), + type: "dev.maple.gitlab.issue.created.v1", + subject: "project/example/issues/42", + projectionrevision: 3, + data: signal().data, + }) + }) + + it("runs every matching projection from one immutable registry snapshot", () => { + const registry = CompiledProjectionRegistry.compile( + [projection(), projection({ id: "gitlab-issue-created-audit" })], + sources(), + projectors(), + ) + const result = registry.evaluate(signal()) + expect(result.failures).toEqual([]) + expect(result.events.map(({ projectionid }) => projectionid)).toEqual([ + "gitlab-issue-created", + "gitlab-issue-created-audit", + ]) + }) + + it("runs all matching projections and isolates projector failures", () => { + const registryDefinitions = projectors().register({ + id: "broken", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.broken.v1", + dataSchema: "urn:maple:event-schema:broken:v1", + decodeConfig: () => ({}), + project: () => { + throw new Error("projector invariant failed") + }, + }) + const registry = CompiledProjectionRegistry.compile( + [ + projection(), + projection({ id: "broken-projection", projector: { id: "broken", version: 1, config: {} } }), + ], + sources(), + registryDefinitions, + ) + const result = registry.evaluate(signal()) + expect(result.events).toHaveLength(1) + expect(result.failures).toEqual([ + expect.objectContaining({ + projectionId: "broken-projection", + message: "projector invariant failed", + }), + ]) + }) + + it("isolates tenants, source kinds, activation time, and disabled revisions", () => { + const registry = CompiledProjectionRegistry.compile( + [ + projection(), + projection({ id: "future", revision: 1, activeFrom: "2026-08-08T00:00:00Z" }), + projection({ id: "disabled", revision: 1, enabled: false }), + projection({ id: "other-tenant", revision: 1, tenantId: "tenant-b" }), + ], + sources(), + projectors(), + ) + expect(registry.evaluate(signal()).events.map(({ projectionid }) => projectionid)).toEqual([ + "gitlab-issue-created", + ]) + expect(registry.evaluate(signal({ sourceKind: "otel.span" })).events).toEqual([]) + }) + + it("requires occurrence identity for durable projection", () => { + const registry = CompiledProjectionRegistry.compile([projection()], sources(), projectors()) + const result = registry.evaluate(signal({ occurrenceId: null, identityQuality: "none" })) + expect(result.events).toEqual([]) + expect(result.failures[0]?.message).toBe( + "durable event projection requires stable or derived occurrence identity", + ) + }) + + it("rejects duplicate registrations, projection revisions, and invalid projector bindings", () => { + const definitions = projectors() + expect(() => + definitions.register({ + id: "gitlab.issue", + version: 1, + sourceKinds: ["otel.log"], + outputType: "duplicate", + dataSchema: "duplicate", + decodeConfig: (value) => value, + project: () => ({ data: {} }), + }), + ).toThrow("duplicate projector registration") + expect(() => + CompiledProjectionRegistry.compile([projection(), projection()], sources(), projectors()), + ).toThrow("duplicate projection revision") + expect(() => + CompiledProjectionRegistry.compile( + [projection({ projector: { id: "missing", version: 1, config: {} } })], + sources(), + projectors(), + ), + ).toThrow("unregistered projector") + }) + + it("validates selector fields and operators against the source catalog", () => { + const closed = new SignalSourceRegistry().register({ + sourceKind: "otel.log", + fields: [ + { + field: { namespace: "attribute", key: "event.name", type: "string" }, + operators: ["eq"], + sensitivity: "public", + replay: "coerced", + }, + ], + }) + expect(() => + CompiledProjectionRegistry.compile( + [ + projection({ + selector: { + op: "contains", + field: { namespace: "attribute", key: "event.name", type: "string" }, + value: { type: "string", value: "gitlab" }, + }, + }), + ], + closed, + projectors(), + ), + ).toThrow("contains is not allowed for catalog field") + expect(() => + CompiledProjectionRegistry.compile( + [ + projection({ + selector: { + op: "eq", + field: { namespace: "attribute", key: "unknown", type: "string" }, + value: { type: "string", value: "x" }, + }, + }), + ], + closed, + projectors(), + ), + ).toThrow("unknown field attribute:unknown") + }) +}) diff --git a/packages/eventing-core/src/registry.ts b/packages/eventing-core/src/registry.ts new file mode 100644 index 000000000..b5eb66adb --- /dev/null +++ b/packages/eventing-core/src/registry.ts @@ -0,0 +1,184 @@ +import { makeCloudEvent } from "./event" +import { Schema } from "effect" +import type { + JsonValue, + MapleCloudEvent, + NormalizedSignal, + ProjectedEventData, + SignalProjectionSpec, +} from "./model" +import { SignalProjectionSpecSchema } from "./model" +import { timestampToEpochNanos, compileSignalPredicate, validateSignalProjectionSpec } from "./predicate" +import { SignalSourceRegistry, validatePredicateAgainstSource } from "./source" + +export interface SignalProjector { + readonly id: string + readonly version: number + readonly sourceKinds: readonly string[] + readonly outputType: string + readonly dataSchema: string + readonly decodeConfig: (value: unknown) => TConfig + readonly project: (signal: NormalizedSignal, config: TConfig) => ProjectedEventData +} + +type ErasedSignalProjector = SignalProjector + +export class ProjectorRegistry { + readonly #projectors = new Map() + + register(projector: SignalProjector): this { + if (projector.id.trim().length === 0) throw new Error("projector ID must not be empty") + if (!Number.isSafeInteger(projector.version) || projector.version < 1) + throw new Error("projector version must be a positive safe integer") + if (projector.sourceKinds.length === 0) throw new Error("projector must accept a source kind") + if (projector.outputType.length === 0) throw new Error("projector output type must not be empty") + if (projector.dataSchema.length === 0) throw new Error("projector data schema must not be empty") + const key = ProjectorRegistry.key(projector.id, projector.version) + if (this.#projectors.has(key)) throw new Error(`duplicate projector registration: ${key}`) + this.#projectors.set(key, projector as ErasedSignalProjector) + return this + } + + get(id: string, version: number): ErasedSignalProjector | undefined { + return this.#projectors.get(ProjectorRegistry.key(id, version)) + } + + static key(id: string, version: number): string { + return `${id}@${version}` + } +} + +interface CompiledProjection { + readonly spec: SignalProjectionSpec + readonly evaluate: ReturnType + readonly projector: ErasedSignalProjector + readonly config: unknown + readonly activeFromNanos: bigint +} + +export interface ProjectionFailure { + readonly projectionId: string + readonly projectionRevision: number + readonly occurrenceId: string | null + readonly message: string +} + +export interface ProjectionBatchResult { + readonly events: readonly MapleCloudEvent[] + readonly failures: readonly ProjectionFailure[] + readonly typeMismatchFields: readonly string[] +} + +const validateProjectedData = (value: ProjectedEventData): ProjectedEventData => { + if (value.time !== undefined && timestampToEpochNanos(value.time) === null) + throw new Error("projector returned an invalid event timestamp") + return value +} + +/** Immutable compiled snapshot. Hosts atomically replace the whole instance. */ +export class CompiledProjectionRegistry { + readonly #bySourceKind: ReadonlyMap + + private constructor(bySourceKind: ReadonlyMap) { + this.#bySourceKind = bySourceKind + } + + static compile( + specs: readonly SignalProjectionSpec[], + sources: SignalSourceRegistry, + projectors: ProjectorRegistry, + ): CompiledProjectionRegistry { + const bySourceKind = new Map() + const revisions = new Set() + + for (const candidate of specs) { + const spec = Schema.decodeUnknownSync(SignalProjectionSpecSchema)(candidate) + const source = sources.get(spec.sourceKind) + if (!source) + throw new Error( + `projection ${spec.id}@${spec.revision} references an unregistered source ${spec.sourceKind}`, + ) + const issues = [ + ...validateSignalProjectionSpec(spec), + ...validatePredicateAgainstSource(spec.selector, source), + ] + if (issues.length > 0) + throw new Error( + `invalid projection ${spec.id}@${spec.revision}: ${issues + .map(({ path, message }) => `${path}: ${message}`) + .join("; ")}`, + ) + const revisionKey = `${spec.tenantId}:${spec.id}@${spec.revision}` + if (revisions.has(revisionKey)) throw new Error(`duplicate projection revision: ${revisionKey}`) + revisions.add(revisionKey) + if (!spec.enabled) continue + + const projector = projectors.get(spec.projector.id, spec.projector.version) + if (!projector) + throw new Error( + `projection ${spec.id}@${spec.revision} references an unregistered projector ${spec.projector.id}@${spec.projector.version}`, + ) + if (!projector.sourceKinds.includes(spec.sourceKind)) + throw new Error( + `projector ${projector.id}@${projector.version} does not accept ${spec.sourceKind}`, + ) + + const compiled: CompiledProjection = { + spec, + evaluate: compileSignalPredicate(spec.selector), + projector, + config: projector.decodeConfig(spec.projector.config), + activeFromNanos: timestampToEpochNanos(spec.activeFrom)!, + } + const bucket = bySourceKind.get(spec.sourceKind) + if (bucket) bucket.push(compiled) + else bySourceKind.set(spec.sourceKind, [compiled]) + } + + return new CompiledProjectionRegistry(bySourceKind) + } + + evaluate(signal: NormalizedSignal): ProjectionBatchResult { + const events: MapleCloudEvent[] = [] + const failures: ProjectionFailure[] = [] + const typeMismatchFields = new Set() + const observedAtNanos = timestampToEpochNanos(signal.observedAt) + + for (const projection of this.#bySourceKind.get(signal.sourceKind) ?? []) { + if (projection.spec.tenantId !== signal.tenantId) continue + if (observedAtNanos === null || observedAtNanos < projection.activeFromNanos) continue + const evaluation = projection.evaluate(signal) + for (const field of evaluation.typeMismatches) + typeMismatchFields.add(`${field.namespace}:${field.key}`) + if (!evaluation.matches) continue + + try { + const projected = validateProjectedData( + projection.projector.project(signal, projection.config), + ) + events.push( + makeCloudEvent({ + signal, + projection: projection.spec, + projectorId: projection.projector.id, + projectorVersion: projection.projector.version, + outputType: projection.projector.outputType, + dataSchema: projection.projector.dataSchema, + subject: projected.subject, + time: projected.time, + data: projected.data as JsonValue, + }), + ) + } catch (error) { + failures.push({ + projectionId: projection.spec.id, + projectionRevision: projection.spec.revision, + occurrenceId: signal.occurrenceId, + message: error instanceof Error ? error.message : String(error), + }) + } + } + + return { events, failures, typeMismatchFields: [...typeMismatchFields] } + } +} diff --git a/packages/eventing-core/src/source.ts b/packages/eventing-core/src/source.ts new file mode 100644 index 000000000..a9a6a7ee2 --- /dev/null +++ b/packages/eventing-core/src/source.ts @@ -0,0 +1,141 @@ +import type { FieldNamespace, FieldRef, NormalizedSignal, SignalPredicate, SignalScalarType } from "./model" +import { fieldKey } from "./model" +import type { ValidationIssue } from "./predicate" + +export type SignalLeafOperator = "exists" | "eq" | "neq" | "gt" | "gte" | "lt" | "lte" | "contains" | "in" +export type ReplayCapability = "exact" | "coerced" | "unavailable" + +export interface SignalFieldCatalogEntry { + readonly field: FieldRef + readonly operators: readonly SignalLeafOperator[] + readonly sensitivity: "public" | "sensitive" + readonly replay: ReplayCapability +} + +export interface OpenFieldNamespacePolicy { + readonly namespace: FieldNamespace + readonly types: readonly SignalScalarType[] + readonly operators: readonly SignalLeafOperator[] + readonly sensitivity: "public" | "sensitive" + readonly replay: ReplayCapability +} + +export interface SignalSourceDefinition { + readonly sourceKind: string + readonly fields: readonly SignalFieldCatalogEntry[] + readonly openFields?: readonly OpenFieldNamespacePolicy[] +} + +export interface SignalSourceAdapter { + readonly definition: SignalSourceDefinition + readonly normalize: (raw: TRaw, context: TContext) => readonly NormalizedSignal[] +} + +interface RegisteredSignalSource { + readonly definition: SignalSourceDefinition + readonly fields: ReadonlyMap + readonly openFields: ReadonlyMap +} + +export class SignalSourceRegistry { + readonly #sources = new Map() + + register(definition: SignalSourceDefinition): this { + if (definition.sourceKind.trim().length === 0) throw new Error("source kind must not be empty") + if (this.#sources.has(definition.sourceKind)) + throw new Error(`duplicate source registration: ${definition.sourceKind}`) + + const fields = new Map() + for (const entry of definition.fields) { + const key = fieldKey(entry.field) + if (fields.has(key)) + throw new Error(`duplicate field catalog entry: ${definition.sourceKind}:${key}`) + if (entry.operators.length === 0) throw new Error(`field catalog entry has no operators: ${key}`) + fields.set(key, entry) + } + + const openFields = new Map() + for (const policy of definition.openFields ?? []) { + if (openFields.has(policy.namespace)) + throw new Error(`duplicate open field policy: ${definition.sourceKind}:${policy.namespace}`) + if (policy.types.length === 0 || policy.operators.length === 0) + throw new Error(`open field policy must declare types and operators: ${policy.namespace}`) + openFields.set(policy.namespace, policy) + } + + this.#sources.set(definition.sourceKind, { definition, fields, openFields }) + return this + } + + get(sourceKind: string): RegisteredSignalSource | undefined { + return this.#sources.get(sourceKind) + } +} + +const leafFields = ( + predicate: SignalPredicate, +): ReadonlyArray<{ + readonly field: FieldRef + readonly operator: SignalLeafOperator + readonly path: string +}> => { + const fields: Array<{ field: FieldRef; operator: SignalLeafOperator; path: string }> = [] + const visit = (node: SignalPredicate, path: string): void => { + switch (node.op) { + case "all": + case "any": + for (let i = 0; i < node.clauses.length; i++) visit(node.clauses[i]!, `${path}.clauses[${i}]`) + break + case "not": + visit(node.clause, `${path}.clause`) + break + default: + fields.push({ field: node.field, operator: node.op, path }) + } + } + visit(predicate, "selector") + return fields +} + +export const validatePredicateAgainstSource = ( + predicate: SignalPredicate, + source: RegisteredSignalSource, +): readonly ValidationIssue[] => { + const issues: ValidationIssue[] = [] + for (const leaf of leafFields(predicate)) { + const catalogEntry = source.fields.get(fieldKey(leaf.field)) + if (catalogEntry) { + if (catalogEntry.field.type !== leaf.field.type) + issues.push({ + path: `${leaf.path}.field.type`, + message: `catalog field ${fieldKey(leaf.field)} has type ${catalogEntry.field.type}`, + }) + if (!catalogEntry.operators.includes(leaf.operator)) + issues.push({ + path: `${leaf.path}.op`, + message: `${leaf.operator} is not allowed for catalog field ${fieldKey(leaf.field)}`, + }) + continue + } + + const open = source.openFields.get(leaf.field.namespace) + if (!open) { + issues.push({ + path: `${leaf.path}.field`, + message: `unknown field ${fieldKey(leaf.field)} for source ${source.definition.sourceKind}`, + }) + continue + } + if (!open.types.includes(leaf.field.type)) + issues.push({ + path: `${leaf.path}.field.type`, + message: `${leaf.field.type} is not allowed for open ${leaf.field.namespace} fields`, + }) + if (!open.operators.includes(leaf.operator)) + issues.push({ + path: `${leaf.path}.op`, + message: `${leaf.operator} is not allowed for open ${leaf.field.namespace} fields`, + }) + } + return issues +} diff --git a/packages/eventing-core/tsconfig.json b/packages/eventing-core/tsconfig.json new file mode 100644 index 000000000..12d9920b4 --- /dev/null +++ b/packages/eventing-core/tsconfig.json @@ -0,0 +1,23 @@ +{ + "include": ["**/*.ts"], + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "lib": ["ES2022", "DOM"], + "types": ["node"], + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + "skipLibCheck": true, + "strict": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedSideEffectImports": true, + "plugins": [ + { + "name": "@effect/language-service", + "reportSuggestionsAsWarningsInTsc": true + } + ] + } +}