From 230183c959af677d6e236cc6ae65693aec6d5ac6 Mon Sep 17 00:00:00 2001 From: Chris Feijoo Date: Sat, 8 Aug 2026 03:00:09 +0200 Subject: [PATCH 1/3] FE-1341: Give experiment compute backends one interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Choosing a backend was hardcoded in ExperimentsProvider. The runtime half of swappability already worked — a backend produces a MonteCarloExperiment and consumers drive one with no branching — so this adds only the missing part: asking a backend whether it can run a net, and choosing between backends when one declines. A request is plain serializable data; anything describing how to compute belongs to the backend's construction or to per-call options. Refusal is a value carrying structured blockers with a code, an item id and an origin, so a UI can attribute a problem to the item that caused it. Assessment settles the net without acquiring a device or worker pool and hands back an instantiate() closure. Pure refactor: no user-visible behaviour changes. Registrations carry a deferred load so a heavy backend need not enter the bundle. --- libs/@hashintel/petrinaut-core/package.json | 4 + .../petrinaut-core/src/experiments.ts | 43 +++ .../src/experiments/experiment-assessment.ts | 123 +++++++ .../src/experiments/experiment-backend.ts | 91 +++++ .../src/experiments/experiment-request.ts | 48 +++ .../select-experiment-backend.test.ts | 345 ++++++++++++++++++ .../experiments/select-experiment-backend.ts | 175 +++++++++ .../worker-pool-experiment-backend.ts | 124 +++++++ libs/@hashintel/petrinaut-core/vite.config.ts | 3 + .../src/react/experiments/provider.test.tsx | 13 +- .../src/react/experiments/provider.tsx | 59 ++- .../content/experiments/backend-selection.mdx | 51 +++ 12 files changed, 1065 insertions(+), 14 deletions(-) create mode 100644 libs/@hashintel/petrinaut-core/src/experiments.ts create mode 100644 libs/@hashintel/petrinaut-core/src/experiments/experiment-assessment.ts create mode 100644 libs/@hashintel/petrinaut-core/src/experiments/experiment-backend.ts create mode 100644 libs/@hashintel/petrinaut-core/src/experiments/experiment-request.ts create mode 100644 libs/@hashintel/petrinaut-core/src/experiments/select-experiment-backend.test.ts create mode 100644 libs/@hashintel/petrinaut-core/src/experiments/select-experiment-backend.ts create mode 100644 libs/@hashintel/petrinaut-core/src/experiments/worker-pool-experiment-backend.ts create mode 100644 libs/@local/petrinaut-arch-docs/content/experiments/backend-selection.mdx diff --git a/libs/@hashintel/petrinaut-core/package.json b/libs/@hashintel/petrinaut-core/package.json index cb4341bd5c7..b718644da81 100644 --- a/libs/@hashintel/petrinaut-core/package.json +++ b/libs/@hashintel/petrinaut-core/package.json @@ -26,6 +26,10 @@ "types": "./dist/index.d.d.ts", "import": "./dist/index.js" }, + "./experiments": { + "types": "./dist/experiments.d.d.ts", + "import": "./dist/experiments.js" + }, "./ai": { "types": "./dist/ai.d.d.ts", "import": "./dist/ai.js" diff --git a/libs/@hashintel/petrinaut-core/src/experiments.ts b/libs/@hashintel/petrinaut-core/src/experiments.ts new file mode 100644 index 00000000000..4771ce4f213 --- /dev/null +++ b/libs/@hashintel/petrinaut-core/src/experiments.ts @@ -0,0 +1,43 @@ +/** + * Swappable experiment backends. + * + * The runtime half of swappability already existed and is unchanged: both the + * worker-pool and WebGPU paths produce a `MonteCarloExperiment`, and consumers + * drive one with no branching. What this adds is the part that was hardcoded — + * asking a backend whether it can run a net, and choosing between backends when + * one declines. + * + * Backends are constructed with their own wiring (a worker factory, an ODE + * method) and then registered as data, which is what makes a React context + * provider a small step from here rather than a redesign: the provider builds the + * backends its environment supports and publishes the registration list. + * + * Registration carries a deferred `load`, so a heavy backend can be registered + * without pulling its implementation into a bundle that never uses it — it is + * imported the first time selection reaches it. + */ +export type { + ExperimentAssessment, + ExperimentBlocker, + ExperimentBlockerOrigin, + ExperimentBlockers, + ExperimentInstantiation, + ExperimentNote, + InstantiateExperimentOptions, +} from "./experiments/experiment-assessment"; +export type { + ExperimentBackend, + ExperimentBackendRegistration, + ExperimentSelectionFailure, +} from "./experiments/experiment-backend"; +export type { ExperimentRequest } from "./experiments/experiment-request"; +export { + selectExperimentBackend, + type SelectExperimentBackendInput, + type SelectExperimentBackendResult, +} from "./experiments/select-experiment-backend"; +export { + createWorkerPoolExperimentBackend, + WORKER_POOL_BACKEND_ID, + type WorkerPoolExperimentBackendOptions, +} from "./experiments/worker-pool-experiment-backend"; diff --git a/libs/@hashintel/petrinaut-core/src/experiments/experiment-assessment.ts b/libs/@hashintel/petrinaut-core/src/experiments/experiment-assessment.ts new file mode 100644 index 00000000000..a191319e2c2 --- /dev/null +++ b/libs/@hashintel/petrinaut-core/src/experiments/experiment-assessment.ts @@ -0,0 +1,123 @@ +/** + * A backend's answer to "can you run this request, and if not, why?" + * + * Refusal is a value, not an exception, because a backend may be a **subset** + * engine: declining a net is ordinary operation, the caller's response is to try + * another backend, and the reason has to reach the user. Modelling that as a + * thrown error would make the normal path the exceptional one. + * + * The refusal carries structured blockers rather than a single string, so a UI + * can attribute a problem to the item that caused it and report several at once. + * A `reason: string` contract would collapse that at the abstraction boundary, + * and it cannot be widened later without changing every backend. + */ +import type { AbortSignalLike } from "../environment"; +import type { MonteCarloExperiment } from "../simulation/monte-carlo/runtime/experiment"; + +/** + * Where a problem lives, and therefore who can act on it. + * + * This is the field a UI branches on, and the reason a blocker is worth more + * than a message: + * + * - `model` — the net must change. Attribute it to `itemId` and keep the backend + * offered but unavailable, because editing the net can fix it. + * - `configuration` — the *experiment* must change: run count, initial marking, + * metric shapes, missing artifacts. Actionable where the experiment is set up, + * not by editing the net. + * - `environment` — this browser or machine cannot do it at all. Do not blame the + * net and do not nag; hiding the option is reasonable. + * - `capacity` — the backend could normally do this but cannot right now: a full + * queue, a device out of memory. Distinct from `environment` precisely because + * it is transient, so "retry" or "use fewer runs" is the right advice where + * "hide the option" would be wrong. + */ +export type ExperimentBlockerOrigin = + | "model" + | "configuration" + | "environment" + | "capacity"; + +export type ExperimentBlocker = { + /** + * Stable, backend-namespaced code, for tests and for grouping in a UI. + * + * Intentionally `string` rather than a closed union: a union shared across + * backends could not be extended by a lazily loaded or third-party backend + * without editing this file. + */ + readonly code: string; + /** Written for whoever authored the net, not for whoever wrote the emitter. */ + readonly message: string; + readonly origin: ExperimentBlockerOrigin; + /** The net item responsible, when one can be identified. */ + readonly itemId?: string; +}; + +/** Something the user should know that did not prevent the run. */ +export type ExperimentNote = { + readonly code: string; + readonly message: string; +}; + +/** At least one, so a refusal without a reason cannot be constructed. */ +export type ExperimentBlockers = readonly [ + ExperimentBlocker, + ...ExperimentBlocker[], +]; + +/** + * The non-serializable half of starting an experiment. + * + * Separate from `ExperimentRequest` so the request stays plain data. Both of + * these are host-side wiring: a signal is a live object, and notes are delivered + * by calling back. + */ +export type InstantiateExperimentOptions = { + signal?: AbortSignalLike; + /** + * Receives problems that only become detectable once the run is under way — + * today, a metric histogram whose top bin saturated. + * + * The notes on the assessment are assembled before anything runs and cannot + * carry these. Without a channel for them the results would be presented as + * fact. + */ + onNote?: (note: ExperimentNote) => void; +}; + +export type ExperimentInstantiation = + | { + readonly ok: true; + readonly handle: MonteCarloExperiment; + /** + * Where this actually ran, for the record: a GPU adapter description, a + * shard count, a server region. Free text because only a human reads it. + */ + readonly runtimeInfo?: string; + } + | { readonly ok: false; readonly blockers: ExperimentBlockers }; + +export type ExperimentAssessment = + | { + readonly eligible: true; + /** Non-blocking observations to surface before the run. */ + readonly notes: readonly ExperimentNote[]; + /** + * Starts the experiment that was assessed. + * + * A closure rather than a second call taking the request again, so the work + * already done — a compiled shader, a shard plan — carries forward and the + * verdict and the run can never be about different things. + * + * May still fail, but only for `environment` or `capacity` reasons: whether + * the *net and configuration* are runnable was settled by the assessment. + * The split exists so that assessing a net while the user edits — to decide + * what to offer — never acquires a device or a worker pool. + */ + instantiate( + this: void, + options?: InstantiateExperimentOptions, + ): Promise; + } + | { readonly eligible: false; readonly blockers: ExperimentBlockers }; diff --git a/libs/@hashintel/petrinaut-core/src/experiments/experiment-backend.ts b/libs/@hashintel/petrinaut-core/src/experiments/experiment-backend.ts new file mode 100644 index 00000000000..81c17851a30 --- /dev/null +++ b/libs/@hashintel/petrinaut-core/src/experiments/experiment-backend.ts @@ -0,0 +1,91 @@ +/** + * The contract every experiment backend satisfies. + * + * @layerRoot core.experiments + * @role Chooses a compute backend for an experiment, and asks whether it can run a net + * + * Small on purpose. The runtime half of swappability already exists and is not + * restated here: both the worker-pool and WebGPU paths already produce a + * `MonteCarloExperiment`, and `ExperimentsProvider` consumes one with no + * branching at all. What was never abstracted is *choosing* a backend and + * *asking* whether it can take a net — so that, and only that, is what this adds. + * + * Backend-specific configuration is bound when the backend object is built, not + * passed through this interface. The worker-pool backend closes over a worker + * factory and a shard count; the WebGPU backend closes over an ODE method. That + * is why a shared config type is unnecessary, and it is also exactly the shape a + * React context provider wants later: the provider constructs backends with the + * environment's wiring and publishes the resulting list. + */ +import type { + ExperimentAssessment, + ExperimentBlockerOrigin, +} from "./experiment-assessment"; +import type { ExperimentRequest } from "./experiment-request"; + +export type ExperimentBackend = { + /** + * Stable identifier, recorded against results. + * + * Results from two backends are not numerically interchangeable — they use + * different random generators — so which one ran is part of the data, not a + * detail. + */ + readonly id: string; + /** Shown to users, e.g. "CPU (Web Workers)". */ + readonly label: string; + /** + * Whether this backend needs the lowered HIR *trees* on the artifacts. + * + * Declared rather than inferred from the id so a caller compiles once for the + * backends it is about to ask. The trees roughly triple artifact size, so the + * worker-pool backend does not want them; the WebGPU backend cannot generate a + * shader without them. + */ + readonly needsHirTrees: boolean; + /** + * Whether this backend could run *anything* in this environment. + * + * Synchronous and cheap — a feature test, not an assessment — so a UI can + * decide whether to offer the backend at all without compiling a net. A + * backend that is always usable returns `true`. + */ + isAvailable(this: void): boolean; + /** + * Decides whether this backend can run `request`, without starting it. + * + * Asynchronous because deciding can require real work (lowering a net, + * generating and compiling a shader). Must not acquire scarce resources: that + * belongs to `instantiate` on the eligible result, so that assessing a net + * while the user edits does not hold a GPU device. + */ + assess(this: void, request: ExperimentRequest): Promise; +}; + +/** + * A backend plus how eagerly to load it. + * + * `load` is deferred so a caller can register the WebGPU backend without pulling + * the shader generator into the initial bundle — it is imported the first time a + * GPU run is actually attempted. Both backends are registered at once; the choice + * is per experiment, never global. + */ +export type ExperimentBackendRegistration = { + readonly id: string; + readonly label: string; + /** + * Loads the backend. + * + * Called at most once per registration by `selectExperimentBackend`, which + * caches the promise: acquiring a backend twice would mean two worker pools or + * two GPU devices. + */ + load(this: void): Promise; +}; + +/** Why no backend could run the request. */ +export type ExperimentSelectionFailure = { + readonly backendId: string; + readonly origin: ExperimentBlockerOrigin; + readonly reason: string; +}; diff --git a/libs/@hashintel/petrinaut-core/src/experiments/experiment-request.ts b/libs/@hashintel/petrinaut-core/src/experiments/experiment-request.ts new file mode 100644 index 00000000000..509c5574a0e --- /dev/null +++ b/libs/@hashintel/petrinaut-core/src/experiments/experiment-request.ts @@ -0,0 +1,48 @@ +/** + * What to compute — the portable half of an experiment. + * + * Deliberately closed, and deliberately plain data. No worker factory, no GPU + * options, no abort signal, no callbacks. Anything a backend needs that is not + * here describes *how* to compute rather than *what*, and belongs either to that + * backend's construction (see `ExperimentBackend`) or to the per-call options of + * `instantiate`. + * + * That single rule is what stops this type decaying into + * `CreateMonteCarloExperimentConfig | CreateGpuMonteCarloExperimentConfig`. It + * also keeps the request serializable, which is the property a future + * out-of-process backend needs: for a remote backend this object is the request + * body, so a function-valued field here would rule that backend out entirely. + */ +import type { PetrinautExtensionSettings } from "../extensions"; +import type { HirArtifacts } from "../hir-runtime"; +import type { InitialMarking } from "../simulation/api"; +import type { MonteCarloMetricSpec } from "../simulation/monte-carlo/metrics/types"; +import type { SDCPN } from "../types/sdcpn"; + +export type ExperimentRequest = { + readonly sdcpn: SDCPN; + readonly extensions?: PetrinautExtensionSettings; + readonly initialMarking: InitialMarking; + readonly parameterValues: Readonly>; + readonly seed: number; + readonly dt: number; + readonly maxTime: number; + readonly runCount: number; + /** + * Metrics to record. + * + * Expression metrics carry their compiled artifact, because a backend may need + * to run them and cannot compile one itself — that needs the TypeScript + * frontend, which is why `hirArtifacts` is here too rather than being derived. + */ + readonly metricSpecs: readonly MonteCarloMetricSpec[]; + /** + * Compiled user code for the net. + * + * Optional because a net with no user code needs none. Whether the HIR *trees* + * must be included is declared per backend by `ExperimentBackend.needsHirTrees` + * rather than inferred from a backend id, so a caller can compile once for + * whichever backends it is about to ask. + */ + readonly hirArtifacts?: HirArtifacts; +}; diff --git a/libs/@hashintel/petrinaut-core/src/experiments/select-experiment-backend.test.ts b/libs/@hashintel/petrinaut-core/src/experiments/select-experiment-backend.test.ts new file mode 100644 index 00000000000..b4ede73f257 --- /dev/null +++ b/libs/@hashintel/petrinaut-core/src/experiments/select-experiment-backend.test.ts @@ -0,0 +1,345 @@ +import { describe, expect, it, vi } from "vitest"; + +import { selectExperimentBackend } from "./select-experiment-backend"; + +import type { + ExperimentAssessment, + ExperimentBlockers, +} from "./experiment-assessment"; +import type { + ExperimentBackend, + ExperimentBackendRegistration, +} from "./experiment-backend"; +import type { ExperimentRequest } from "./experiment-request"; +import type { MonteCarloExperiment } from "../simulation/monte-carlo/runtime/experiment"; + +/** Enough of a handle to be identity-compared; nothing here starts it. */ +function stubHandle(name: string): MonteCarloExperiment { + return { name } as unknown as MonteCarloExperiment; +} + +const REQUEST = { runCount: 10 } as unknown as ExperimentRequest; + +function stubBackend({ + id, + needsHirTrees = false, + available = true, + assessment, +}: { + id: string; + needsHirTrees?: boolean; + available?: boolean; + assessment: (request: ExperimentRequest) => ExperimentAssessment; +}): ExperimentBackend { + return { + id, + label: id.toUpperCase(), + needsHirTrees, + isAvailable: () => available, + assess: (request) => Promise.resolve(assessment(request)), + }; +} + +function registrationFor( + backend: ExperimentBackend, +): ExperimentBackendRegistration { + return { + id: backend.id, + label: backend.label, + load: () => Promise.resolve(backend), + }; +} + +/** An eligible assessment whose `instantiate` succeeds. */ +function accepts(handleName: string, runtimeInfo?: string): ExperimentAssessment { + return { + eligible: true, + notes: [], + instantiate: () => + Promise.resolve({ + ok: true, + handle: stubHandle(handleName), + ...(runtimeInfo === undefined ? {} : { runtimeInfo }), + }), + }; +} + +const MODEL_BLOCKERS: ExperimentBlockers = [ + { + code: "colored-place-without-capacity", + message: "Place `Space` holds typed tokens but has no token capacity.", + origin: "model", + itemId: "p_space", + }, +]; + +describe("selectExperimentBackend", () => { + it("uses the first backend that accepts, without loading later ones", async () => { + // Loading is deferred precisely so the GPU shader generator stays out of the + // bundle until a GPU run is attempted; if selection loaded every candidate + // eagerly that would be pointless. + const secondLoad = vi.fn(() => + Promise.resolve(stubBackend({ id: "second", assessment: () => accepts("b") })), + ); + + const result = await selectExperimentBackend({ + registrations: [ + registrationFor( + stubBackend({ id: "first", assessment: () => accepts("a") }), + ), + { id: "second", label: "SECOND", load: secondLoad }, + ], + buildRequest: () => Promise.resolve(REQUEST), + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.backendId).toBe("first"); + expect(result.declined).toStrictEqual([]); + expect(secondLoad).not.toHaveBeenCalled(); + }); + + it("falls through to the next backend and records why the first declined", async () => { + const result = await selectExperimentBackend({ + registrations: [ + registrationFor( + stubBackend({ + id: "webgpu", + assessment: () => ({ eligible: false, blockers: MODEL_BLOCKERS }), + }), + ), + registrationFor( + stubBackend({ id: "cpu", assessment: () => accepts("cpu") }), + ), + ], + buildRequest: () => Promise.resolve(REQUEST), + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.backendId).toBe("cpu"); + // The reason must survive, because this is what tells the user their + // experiment is not running where they asked. + expect(result.declined).toStrictEqual([ + { + backendId: "webgpu", + origin: "model", + reason: "Place `Space` holds typed tokens but has no token capacity.", + }, + ]); + }); + + it("treats a failed instantiation as a refusal and keeps going", async () => { + // Assessment is authoritative about the net; instantiation can still fail for + // the environment — a device that will not allocate. Surfacing that as a dead + // end rather than falling through would strand the user on a working net. + const result = await selectExperimentBackend({ + registrations: [ + registrationFor( + stubBackend({ + id: "webgpu", + assessment: () => ({ + eligible: true, + notes: [], + instantiate: () => + Promise.resolve({ + ok: false, + blockers: [ + { + code: "gpu-allocation-failed", + message: "The GPU could not allocate memory for 1000000 runs.", + origin: "capacity", + }, + ], + }), + }), + }), + ), + registrationFor( + stubBackend({ id: "cpu", assessment: () => accepts("cpu") }), + ), + ], + buildRequest: () => Promise.resolve(REQUEST), + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.backendId).toBe("cpu"); + expect(result.declined[0]?.origin).toBe("capacity"); + }); + + it("leads with the most actionable blocker and counts the rest", async () => { + // A net failing several checks should be described by the one the author can + // act on, not by whichever the pipeline happened to hit first. + const result = await selectExperimentBackend({ + registrations: [ + registrationFor( + stubBackend({ + id: "webgpu", + assessment: () => ({ + eligible: false, + blockers: [ + { + code: "no-webgpu", + message: "This browser does not expose WebGPU.", + origin: "environment", + }, + { + code: "colored-place-without-capacity", + message: "Place `Space` has no token capacity.", + origin: "model", + }, + ], + }), + }), + ), + ], + buildRequest: () => Promise.resolve(REQUEST), + }); + + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.declined[0]).toStrictEqual({ + backendId: "webgpu", + origin: "model", + reason: "Place `Space` has no token capacity. (+1 more)", + }); + }); + + it("skips an unavailable backend without assessing it", async () => { + const assess = vi.fn(() => accepts("never")); + + const result = await selectExperimentBackend({ + registrations: [ + registrationFor( + stubBackend({ id: "webgpu", available: false, assessment: assess }), + ), + registrationFor( + stubBackend({ id: "cpu", assessment: () => accepts("cpu") }), + ), + ], + buildRequest: () => Promise.resolve(REQUEST), + }); + + expect(assess).not.toHaveBeenCalled(); + expect(result.ok && result.backendId).toBe("cpu"); + }); + + it("survives a backend whose module will not load", async () => { + const result = await selectExperimentBackend({ + registrations: [ + { + id: "webgpu", + label: "WEBGPU", + load: () => Promise.reject(new Error("chunk load failed")), + }, + registrationFor( + stubBackend({ id: "cpu", assessment: () => accepts("cpu") }), + ), + ], + buildRequest: () => Promise.resolve(REQUEST), + }); + + expect(result.ok && result.backendId).toBe("cpu"); + if (!result.ok) return; + expect(result.declined[0]?.reason).toMatch(/could not be loaded.*chunk load/); + }); + + it("builds the request once per distinct artifact requirement", async () => { + // Compiling HIR trees roughly triples artifact size, so the worker-pool path + // must not pay for them — but two backends wanting the same artifacts should + // not compile twice either. + const buildRequest = vi.fn((_options: { needsHirTrees: boolean }) => + Promise.resolve(REQUEST), + ); + + await selectExperimentBackend({ + registrations: [ + registrationFor( + stubBackend({ + id: "webgpu", + needsHirTrees: true, + assessment: () => ({ eligible: false, blockers: MODEL_BLOCKERS }), + }), + ), + registrationFor( + stubBackend({ + id: "wasm", + needsHirTrees: true, + assessment: () => ({ eligible: false, blockers: MODEL_BLOCKERS }), + }), + ), + registrationFor( + stubBackend({ id: "cpu", assessment: () => accepts("cpu") }), + ), + ], + buildRequest, + }); + + // Once with trees (shared by webgpu and wasm), once without (cpu). + expect(buildRequest.mock.calls.map(([options]) => options)).toStrictEqual([ + { needsHirTrees: true }, + { needsHirTrees: false }, + ]); + }); + + it("reports every refusal when nothing can run it", async () => { + const result = await selectExperimentBackend({ + registrations: [ + registrationFor( + stubBackend({ + id: "webgpu", + assessment: () => ({ eligible: false, blockers: MODEL_BLOCKERS }), + }), + ), + registrationFor( + stubBackend({ + id: "cpu", + assessment: () => ({ + eligible: false, + blockers: [ + { + code: "no-artifacts", + message: "The net's code did not compile.", + origin: "configuration", + }, + ], + }), + }), + ), + ], + buildRequest: () => Promise.resolve(REQUEST), + }); + + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.declined.map((entry) => entry.backendId)).toStrictEqual([ + "webgpu", + "cpu", + ]); + }); + + it("passes the instantiate options through to the chosen backend", async () => { + // The signal and the post-hoc note channel are deliberately not on the + // request, so they have to arrive here or they arrive nowhere. + const instantiate = vi.fn(() => + Promise.resolve({ ok: true as const, handle: stubHandle("cpu") }), + ); + const onNote = () => {}; + + await selectExperimentBackend({ + registrations: [ + registrationFor( + stubBackend({ + id: "cpu", + assessment: () => ({ eligible: true, notes: [], instantiate }), + }), + ), + ], + buildRequest: () => Promise.resolve(REQUEST), + instantiateOptions: { onNote }, + }); + + expect(instantiate).toHaveBeenCalledWith({ onNote }); + }); +}); diff --git a/libs/@hashintel/petrinaut-core/src/experiments/select-experiment-backend.ts b/libs/@hashintel/petrinaut-core/src/experiments/select-experiment-backend.ts new file mode 100644 index 00000000000..728b33f772d --- /dev/null +++ b/libs/@hashintel/petrinaut-core/src/experiments/select-experiment-backend.ts @@ -0,0 +1,175 @@ +/** + * Picks the first backend in a preference order that will take a request. + * + * This replaces hand-written fallback logic — "try the GPU, and if it declines, + * use the CPU, and put the reason in a notification" — which worked for exactly + * two backends and had to be edited to gain a third. Ordering is the caller's + * (the user's choice first, then whatever else can serve), and the walk records + * every refusal so the UI can say what was tried and why it was declined. + * + * Selection stops at the first backend that both assesses eligible *and* + * instantiates, because the two can disagree: a net can be perfectly runnable + * and the device still be out of memory. Treating instantiation failure as a + * refusal keeps the fallback honest instead of surfacing a dead end. + */ +import type { + ExperimentBackend, + ExperimentBackendRegistration, + ExperimentSelectionFailure, +} from "./experiment-backend"; +import type { + ExperimentNote, + InstantiateExperimentOptions, +} from "./experiment-assessment"; +import type { MonteCarloExperiment } from "../simulation/monte-carlo/runtime/experiment"; +import type { ExperimentRequest } from "./experiment-request"; + +export type SelectExperimentBackendInput = { + /** + * Candidates in preference order, best first. + * + * The caller orders these — usually the user's requested backend followed by + * the fallbacks — because preference is a product decision, not something this + * function should infer. + */ + readonly registrations: readonly ExperimentBackendRegistration[]; + /** + * Builds the request for a backend, given whether that backend needs HIR trees. + * + * A function rather than a value because compiling artifacts is expensive and + * the two backends want different ones: asking for trees unconditionally would + * triple artifact size for the worker-pool path, which never reads them. + * Called at most once per distinct `needsHirTrees` value. + */ + buildRequest( + this: void, + options: { needsHirTrees: boolean }, + ): Promise; + readonly instantiateOptions?: InstantiateExperimentOptions; +}; + +export type SelectExperimentBackendResult = + | { + readonly ok: true; + readonly backendId: string; + readonly handle: MonteCarloExperiment; + readonly runtimeInfo?: string; + readonly notes: readonly ExperimentNote[]; + /** + * Backends declined before this one, in the order they were tried. + * + * Empty when the first choice was used. Non-empty means the run is not on + * the backend the user asked for, which is worth telling them. + */ + readonly declined: readonly ExperimentSelectionFailure[]; + } + | { readonly ok: false; readonly declined: readonly ExperimentSelectionFailure[] }; + +/** Summarises blockers into one sentence, leading with the most actionable. */ +function summarize( + blockers: readonly { + message: string; + origin: ExperimentSelectionFailure["origin"]; + }[], +): { origin: ExperimentSelectionFailure["origin"]; reason: string } { + // `model` first: it names something the author can change. An emitter's own + // message describes an expression tree and helps nobody choose what to edit. + const order: ExperimentSelectionFailure["origin"][] = [ + "model", + "configuration", + "capacity", + "environment", + ]; + const sorted = [...blockers].sort( + (left, right) => order.indexOf(left.origin) - order.indexOf(right.origin), + ); + const [first] = sorted; + if (!first) { + // Unreachable through `ExperimentBlockers`, which is a non-empty tuple. Kept + // because this helper also takes plain arrays from instantiation results. + return { origin: "environment", reason: "declined without a reason" }; + } + const others = + sorted.length > 1 ? ` (+${sorted.length - 1} more)` : ""; + return { origin: first.origin, reason: `${first.message}${others}` }; +} + +export async function selectExperimentBackend({ + registrations, + buildRequest, + instantiateOptions, +}: SelectExperimentBackendInput): Promise { + const declined: ExperimentSelectionFailure[] = []; + // Memoised per `needsHirTrees`, so two backends wanting the same artifacts + // compile them once. + const requests = new Map>(); + const requestFor = (needsHirTrees: boolean) => { + const existing = requests.get(needsHirTrees); + if (existing) { + return existing; + } + const created = buildRequest({ needsHirTrees }); + requests.set(needsHirTrees, created); + return created; + }; + + for (const registration of registrations) { + let backend: ExperimentBackend; + try { + backend = await registration.load(); + } catch (error) { + // A backend whose module fails to load is an environment problem, not a + // reason to abandon the experiment. + declined.push({ + backendId: registration.id, + origin: "environment", + reason: `${registration.label} could not be loaded: ${ + error instanceof Error ? error.message : String(error) + }`, + }); + continue; + } + + if (!backend.isAvailable()) { + declined.push({ + backendId: backend.id, + origin: "environment", + reason: `${backend.label} is not available in this environment.`, + }); + continue; + } + + const assessment = await backend.assess( + await requestFor(backend.needsHirTrees), + ); + if (!assessment.eligible) { + declined.push({ + backendId: backend.id, + ...summarize(assessment.blockers), + }); + continue; + } + + const instantiated = await assessment.instantiate(instantiateOptions); + if (!instantiated.ok) { + declined.push({ + backendId: backend.id, + ...summarize(instantiated.blockers), + }); + continue; + } + + return { + ok: true, + backendId: backend.id, + handle: instantiated.handle, + ...(instantiated.runtimeInfo === undefined + ? {} + : { runtimeInfo: instantiated.runtimeInfo }), + notes: assessment.notes, + declined, + }; + } + + return { ok: false, declined }; +} diff --git a/libs/@hashintel/petrinaut-core/src/experiments/worker-pool-experiment-backend.ts b/libs/@hashintel/petrinaut-core/src/experiments/worker-pool-experiment-backend.ts new file mode 100644 index 00000000000..1e334f8d3e1 --- /dev/null +++ b/libs/@hashintel/petrinaut-core/src/experiments/worker-pool-experiment-backend.ts @@ -0,0 +1,124 @@ +/** + * The experiment backend that runs the buffer-ABI engine across Web Workers. + * + * Named for the mechanism rather than the silicon. `web-workers` would be wrong + * in both directions: the same runtime also runs entirely in-thread when metrics + * are supplied as executable callbacks (`createLocalMonteCarloExperiment`), and a + * caller-supplied `transport` may be a Node `worker_threads` channel. Its **id** + * stays `"cpu"`, because that is the axis users choose along, the word the UI + * already uses ("running on the CPU"), and what `ExperimentRecord.computeBackend` + * records. + * + * This is the fallback, so it accepts every net: assessment is unconditionally + * eligible. Deciding *here* whether a net needs compiled artifacts would restate + * a rule `build-simulation.ts` already owns, and a second copy of that rule would + * eventually disagree with the engine. Instead instantiation reports what the + * engine says, which is the single source of truth. + */ +import { createMonteCarloExperiment } from "../simulation/monte-carlo/runtime/experiment"; + +import type { + ExperimentAssessment, + ExperimentBlockers, +} from "./experiment-assessment"; +import type { ExperimentBackend } from "./experiment-backend"; +import type { ExperimentRequest } from "./experiment-request"; +import type { WorkerFactory } from "../simulation/api"; + +export const WORKER_POOL_BACKEND_ID = "cpu"; + +export type WorkerPoolExperimentBackendOptions = { + /** + * Spawns one simulation worker. + * + * Bound here rather than passed per request because it is host wiring, not part + * of what to compute — the same reason it is absent from `ExperimentRequest`. A + * React provider supplies it once for the whole app. + */ + createWorker: WorkerFactory; + /** + * How many workers to split runs across. + * + * Runs are independent and seeds derive from the global run index, so this only + * changes how fast an experiment finishes, never what it reports. Defaults to + * one per logical core minus one. + */ + shardCount?: number; + batchSize?: number; +}; + +function assess( + request: ExperimentRequest, + options: WorkerPoolExperimentBackendOptions, +): ExperimentAssessment { + return { + eligible: true, + notes: [], + instantiate: async (instantiateOptions) => { + try { + const handle = await createMonteCarloExperiment({ + sdcpn: request.sdcpn, + ...(request.extensions === undefined + ? {} + : { extensions: request.extensions }), + initialMarking: request.initialMarking, + parameterValues: { ...request.parameterValues }, + seed: request.seed, + dt: request.dt, + maxTime: request.maxTime, + runCount: request.runCount, + metricSpecs: request.metricSpecs, + ...(request.hirArtifacts === undefined + ? {} + : { hirArtifacts: request.hirArtifacts }), + createWorker: options.createWorker, + ...(options.shardCount === undefined + ? {} + : { shardCount: options.shardCount }), + ...(options.batchSize === undefined + ? {} + : { batchSize: options.batchSize }), + ...(instantiateOptions?.signal === undefined + ? {} + : { signal: instantiateOptions.signal }), + }); + + return { + ok: true, + handle, + runtimeInfo: + options.shardCount === undefined + ? "Web Workers" + : `Web Workers (${options.shardCount} shards)`, + }; + } catch (error) { + // The engine refuses a net whose user code has no compiled artifact, and + // it phrases that better than this layer could. Reported as + // `configuration` because the fix is to compile, not to edit the net. + const blockers: ExperimentBlockers = [ + { + code: "engine-refused", + message: error instanceof Error ? error.message : String(error), + origin: "configuration", + }, + ]; + return { ok: false, blockers }; + } + }, + }; +} + +export function createWorkerPoolExperimentBackend( + options: WorkerPoolExperimentBackendOptions, +): ExperimentBackend { + return { + id: WORKER_POOL_BACKEND_ID, + label: "CPU (Web Workers)", + // The engine reads compiled buffer programs, never the HIR trees they came + // from, and carrying the trees roughly triples artifact size on a payload + // posted to every shard. + needsHirTrees: false, + isAvailable: () => true, + assess: (request) => Promise.resolve(assess(request, options)), + }; +} diff --git a/libs/@hashintel/petrinaut-core/vite.config.ts b/libs/@hashintel/petrinaut-core/vite.config.ts index ddb2d1f8be4..336a63db279 100644 --- a/libs/@hashintel/petrinaut-core/vite.config.ts +++ b/libs/@hashintel/petrinaut-core/vite.config.ts @@ -23,6 +23,9 @@ export default defineConfig(({ command }) => ({ // Dependency-free instantiation of compiled HIR artifacts. "hir-runtime": resolve(packageRoot, "src/hir-runtime.ts"), optimization: resolve(packageRoot, "src/optimization.ts"), + // Backend contract and selection. A separate entry so a heavy backend + // can be registered without dragging its implementation in with it. + experiments: resolve(packageRoot, "src/experiments.ts"), "examples/index": resolve(packageRoot, "src/examples/index.ts"), "workers/lsp": resolve(packageRoot, "src/workers/lsp.ts"), "workers/monte-carlo": resolve( diff --git a/libs/@hashintel/petrinaut/src/react/experiments/provider.test.tsx b/libs/@hashintel/petrinaut/src/react/experiments/provider.test.tsx index 653102d60c8..ab46f3aea9c 100644 --- a/libs/@hashintel/petrinaut/src/react/experiments/provider.test.tsx +++ b/libs/@hashintel/petrinaut/src/react/experiments/provider.test.tsx @@ -133,9 +133,18 @@ class FakeMonteCarloWorker { } } +/** + * Lets experiment setup run to the point where it reaches the worker. + * + * A macrotask boundary drains the entire microtask queue, so this holds however + * many awaits setup takes. It used to await exactly two microtasks, which was the + * count at the time and broke the moment a step was added — selecting a backend + * inserts several. + */ const flushWorkerSetup = async () => { - await Promise.resolve(); - await Promise.resolve(); + await new Promise((resolve) => { + setTimeout(resolve, 0); + }); }; const sdcpnContextValue: SDCPNContextValue = { diff --git a/libs/@hashintel/petrinaut/src/react/experiments/provider.tsx b/libs/@hashintel/petrinaut/src/react/experiments/provider.tsx index dcf2d4c7079..c354b7dff9e 100644 --- a/libs/@hashintel/petrinaut/src/react/experiments/provider.tsx +++ b/libs/@hashintel/petrinaut/src/react/experiments/provider.tsx @@ -7,7 +7,6 @@ import { use, useEffect, useRef, useState } from "react"; import { v4 as generateUuid } from "uuid"; import { - createMonteCarloExperiment, compileScenario, type InitialMarking, type MonteCarloExperiment, @@ -16,6 +15,12 @@ import { type Scenario, type ScenarioParameter, } from "@hashintel/petrinaut-core"; +import { + createWorkerPoolExperimentBackend, + selectExperimentBackend, + type ExperimentBackendRegistration, + type ExperimentRequest, +} from "@hashintel/petrinaut-core/experiments"; import { createMonteCarloWorker } from "@hashintel/petrinaut-core/workers/monte-carlo"; import { useBlockWindowClose } from "../hooks/use-block-window-close"; @@ -426,7 +431,7 @@ export const ExperimentsProvider: React.FC = ({ return { ...spec, artifact }; }); - const experimentConfigBase = { + const request: ExperimentRequest = { // Artifact fingerprints cover the complete sanitized SDCPN, including // its metric definitions. Run the worker against the exact snapshot // used above rather than the pre-substitution model. @@ -437,25 +442,55 @@ export const ExperimentsProvider: React.FC = ({ seed: input.seed, dt: input.dt, maxTime: input.maxTime, - hirArtifacts: artifacts, runCount: input.runCount, + metricSpecs, + hirArtifacts: artifacts, }; - const handle = await createMonteCarloExperiment({ - ...experimentConfigBase, - createWorker: workerFactoryRef.current, - ...(shardCountRef.current === undefined - ? {} - : { shardCount: shardCountRef.current }), - metricSpecs, - signal: abortController.signal, + // Preference order, best first. Only one backend today; the point of + // going through the registry is that adding another is a registration + // rather than an edit to this branch. + const registrations: ExperimentBackendRegistration[] = [ + { + id: "cpu", + label: "CPU (Web Workers)", + load: () => + Promise.resolve( + createWorkerPoolExperimentBackend({ + createWorker: workerFactoryRef.current, + ...(shardCountRef.current === undefined + ? {} + : { shardCount: shardCountRef.current }), + }), + ), + }, + ]; + + const selection = await selectExperimentBackend({ + registrations, + buildRequest: () => Promise.resolve(request), + instantiateOptions: { signal: abortController.signal }, }); + // Setup cannot be aborted mid-flight. A cancelled or removed experiment + // must stop here rather than turning a late result into a running handle. if (!pendingRegistrationsRef.current.has(experimentId)) { - handle.dispose(); + if (selection.ok) { + selection.handle.dispose(); + } return; } + if (!selection.ok) { + throw new Error( + selection.declined + .map((entry) => `${entry.backendId}: ${entry.reason}`) + .join("; ") || "No compute backend could run this experiment.", + ); + } + + const { handle } = selection; + pendingRegistrationsRef.current.delete(experimentId); registerExperimentHandle(experiment, handle); handle.start(); diff --git a/libs/@local/petrinaut-arch-docs/content/experiments/backend-selection.mdx b/libs/@local/petrinaut-arch-docs/content/experiments/backend-selection.mdx new file mode 100644 index 00000000000..4c904c70fb1 --- /dev/null +++ b/libs/@local/petrinaut-arch-docs/content/experiments/backend-selection.mdx @@ -0,0 +1,51 @@ +--- +title: Backend selection +description: How a compute backend is chosen, and why a refusal is a value rather than an error. +sidebar_order: 10 +attachTo: core.experiments +--- + +The runtime half of swappability already existed: every backend produces a +`MonteCarloExperiment`, and consumers drive one with no branching. This layer adds +the part that was hardcoded — asking a backend whether it can run a net, and +choosing between backends when one declines. + +## Two phases + +`assess` settles whether the *net and configuration* are runnable, without +acquiring a device or a worker pool. An eligible result carries an `instantiate()` +closure, so work already done — a compiled shader, a shard plan — flows into the +run and the verdict can never be about a different thing. + +Instantiation can still fail, but only for `environment` or `capacity` reasons. A +device that will not allocate is not the net's fault. + +```text +registrations (preference order) + -> load -> assess -> blockers? -> record refusal, try next + -> eligible -> instantiate -> failed? -> record, try next + -> handle -> run +``` + +## Refusal is a value + +A backend may be a **subset** engine, so declining a net is ordinary operation, +not an exception. Blockers carry a `code`, an optional `itemId`, and an `origin`: + +| origin | Who acts | Example | +| --- | --- | --- | +| `model` | edit the net | a typed place with no capacity | +| `configuration` | edit the experiment | a metric shape the backend cannot serve | +| `environment` | nobody — hide the option | no WebGPU in this browser | +| `capacity` | retry, or use fewer runs | device out of memory | + +`capacity` is deliberately separate from `environment` because it is transient: +"use fewer runs" is right where "hide the option" would be wrong. + +## What stays out of the request + +`ExperimentRequest` is plain, serializable data. Anything describing *how* to +compute — a worker factory, an ODE method, an abort signal, a note callback — +belongs to the backend's construction or to per-call options. That rule is what +keeps the request from becoming a union of every backend's knobs, and what leaves +room for a backend that runs out of process. From 6c8fee81a1cada4fd07c8cf559c18cb96aa9239e Mon Sep 17 00:00:00 2001 From: Chris Feijoo Date: Sat, 15 Aug 2026 02:49:52 +0200 Subject: [PATCH 2/3] FE-1341: Apply the house prose style to the docs it adds Replaces em dashes and drops 'deliberately' in backend-selection.mdx. No content change. --- .../content/experiments/backend-selection.mdx | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/libs/@local/petrinaut-arch-docs/content/experiments/backend-selection.mdx b/libs/@local/petrinaut-arch-docs/content/experiments/backend-selection.mdx index 4c904c70fb1..083a5ed8ee0 100644 --- a/libs/@local/petrinaut-arch-docs/content/experiments/backend-selection.mdx +++ b/libs/@local/petrinaut-arch-docs/content/experiments/backend-selection.mdx @@ -7,14 +7,14 @@ attachTo: core.experiments The runtime half of swappability already existed: every backend produces a `MonteCarloExperiment`, and consumers drive one with no branching. This layer adds -the part that was hardcoded — asking a backend whether it can run a net, and +the part that was hardcoded: asking a backend whether it can run a net, and choosing between backends when one declines. ## Two phases `assess` settles whether the *net and configuration* are runnable, without acquiring a device or a worker pool. An eligible result carries an `instantiate()` -closure, so work already done — a compiled shader, a shard plan — flows into the +closure, so work already done (a compiled shader, a shard plan) flows into the run and the verdict can never be about a different thing. Instantiation can still fail, but only for `environment` or `capacity` reasons. A @@ -36,16 +36,16 @@ not an exception. Blockers carry a `code`, an optional `itemId`, and an `origin` | --- | --- | --- | | `model` | edit the net | a typed place with no capacity | | `configuration` | edit the experiment | a metric shape the backend cannot serve | -| `environment` | nobody — hide the option | no WebGPU in this browser | +| `environment` | nobody, so hide the option | no WebGPU in this browser | | `capacity` | retry, or use fewer runs | device out of memory | -`capacity` is deliberately separate from `environment` because it is transient: -"use fewer runs" is right where "hide the option" would be wrong. +`capacity` is separate from `environment` because it is transient: "use fewer +runs" is right where "hide the option" would be wrong. ## What stays out of the request `ExperimentRequest` is plain, serializable data. Anything describing *how* to -compute — a worker factory, an ODE method, an abort signal, a note callback — -belongs to the backend's construction or to per-call options. That rule is what +compute, such as a worker factory, an ODE method, an abort signal or a note +callback, belongs to the backend's construction or to per-call options. That rule is what keeps the request from becoming a union of every backend's knobs, and what leaves room for a backend that runs out of process. From 8f6a8f682673afc21e3a9d7f474c3a2131cea73d Mon Sep 17 00:00:00 2001 From: Chris Feijoo Date: Sat, 15 Aug 2026 03:05:12 +0200 Subject: [PATCH 3/3] FE-1341: Apply the house prose style to the code comments Rewrites the module docstrings in src/experiments to state what each file is before why it is that way, and replaces em dashes. No code change. --- .changeset/experiments-backend-registry.md | 5 ++ .../petrinaut-core/src/experiments.ts | 22 ++--- .../src/experiments/experiment-assessment.ts | 24 +++--- .../src/experiments/experiment-backend.ts | 24 +++--- .../src/experiments/experiment-request.ts | 32 +++---- .../select-experiment-backend.test.ts | 22 +++-- .../experiments/select-experiment-backend.ts | 86 +++++++++++++++---- .../worker-pool-experiment-backend.ts | 10 ++- libs/@hashintel/petrinaut-core/vite.config.ts | 2 +- .../src/react/experiments/provider.tsx | 3 +- .../content/experiments/backend-selection.mdx | 16 ++-- 11 files changed, 150 insertions(+), 96 deletions(-) create mode 100644 .changeset/experiments-backend-registry.md diff --git a/.changeset/experiments-backend-registry.md b/.changeset/experiments-backend-registry.md new file mode 100644 index 00000000000..35055f8ab12 --- /dev/null +++ b/.changeset/experiments-backend-registry.md @@ -0,0 +1,5 @@ +--- +"@hashintel/petrinaut-core": patch +--- + +Add the `@hashintel/petrinaut-core/experiments` entry point: an `ExperimentBackend` interface, a worker-pool implementation of it, and `selectExperimentBackend`, which walks backends in preference order and records every refusal. diff --git a/libs/@hashintel/petrinaut-core/src/experiments.ts b/libs/@hashintel/petrinaut-core/src/experiments.ts index 4771ce4f213..a4f042a4603 100644 --- a/libs/@hashintel/petrinaut-core/src/experiments.ts +++ b/libs/@hashintel/petrinaut-core/src/experiments.ts @@ -1,20 +1,16 @@ /** - * Swappable experiment backends. + * The contract both compute backends satisfy, and the registry that picks one. * - * The runtime half of swappability already existed and is unchanged: both the - * worker-pool and WebGPU paths produce a `MonteCarloExperiment`, and consumers - * drive one with no branching. What this adds is the part that was hardcoded — - * asking a backend whether it can run a net, and choosing between backends when - * one declines. + * Backends already produce a `MonteCarloExperiment` that consumers drive without + * branching. This adds the two things that were hardcoded: asking a backend + * whether it can run a net, and choosing when one declines. * - * Backends are constructed with their own wiring (a worker factory, an ODE - * method) and then registered as data, which is what makes a React context - * provider a small step from here rather than a redesign: the provider builds the - * backends its environment supports and publishes the registration list. + * A backend is constructed with its own wiring (a worker factory, an ODE method) + * and registered as data. A React context provider can therefore build the + * backends its environment supports and publish the list. * - * Registration carries a deferred `load`, so a heavy backend can be registered - * without pulling its implementation into a bundle that never uses it — it is - * imported the first time selection reaches it. + * Registration carries a deferred `load`. A heavy backend is imported the first + * time selection reaches it, so it stays out of bundles that never use it. */ export type { ExperimentAssessment, diff --git a/libs/@hashintel/petrinaut-core/src/experiments/experiment-assessment.ts b/libs/@hashintel/petrinaut-core/src/experiments/experiment-assessment.ts index a191319e2c2..72f82cedc47 100644 --- a/libs/@hashintel/petrinaut-core/src/experiments/experiment-assessment.ts +++ b/libs/@hashintel/petrinaut-core/src/experiments/experiment-assessment.ts @@ -1,10 +1,8 @@ /** * A backend's answer to "can you run this request, and if not, why?" * - * Refusal is a value, not an exception, because a backend may be a **subset** - * engine: declining a net is ordinary operation, the caller's response is to try - * another backend, and the reason has to reach the user. Modelling that as a - * thrown error would make the normal path the exceptional one. + * Refusal is a value rather than an exception. A subset engine declining a net is + * ordinary: the caller tries another backend, and the reason reaches the user. * * The refusal carries structured blockers rather than a single string, so a UI * can attribute a problem to the item that caused it and report several at once. @@ -20,14 +18,14 @@ import type { MonteCarloExperiment } from "../simulation/monte-carlo/runtime/exp * This is the field a UI branches on, and the reason a blocker is worth more * than a message: * - * - `model` — the net must change. Attribute it to `itemId` and keep the backend + * - `model`, the net must change. Attribute it to `itemId` and keep the backend * offered but unavailable, because editing the net can fix it. - * - `configuration` — the *experiment* must change: run count, initial marking, + * - `configuration`, the *experiment* must change: run count, initial marking, * metric shapes, missing artifacts. Actionable where the experiment is set up, * not by editing the net. - * - `environment` — this browser or machine cannot do it at all. Do not blame the + * - `environment`, this browser or machine cannot do it at all. Do not blame the * net and do not nag; hiding the option is reasonable. - * - `capacity` — the backend could normally do this but cannot right now: a full + * - `capacity`, the backend could normally do this but cannot right now: a full * queue, a device out of memory. Distinct from `environment` precisely because * it is transient, so "retry" or "use fewer runs" is the right advice where * "hide the option" would be wrong. @@ -76,8 +74,8 @@ export type ExperimentBlockers = readonly [ export type InstantiateExperimentOptions = { signal?: AbortSignalLike; /** - * Receives problems that only become detectable once the run is under way — - * today, a metric histogram whose top bin saturated. + * Receives problems that only become detectable once the run is under way. + * Today: a metric histogram whose top bin saturated. * * The notes on the assessment are assembled before anything runs and cannot * carry these. Without a channel for them the results would be presented as @@ -107,13 +105,13 @@ export type ExperimentAssessment = * Starts the experiment that was assessed. * * A closure rather than a second call taking the request again, so the work - * already done — a compiled shader, a shard plan — carries forward and the + * already done, a compiled shader, a shard plan, carries forward and the * verdict and the run can never be about different things. * * May still fail, but only for `environment` or `capacity` reasons: whether * the *net and configuration* are runnable was settled by the assessment. - * The split exists so that assessing a net while the user edits — to decide - * what to offer — never acquires a device or a worker pool. + * The split exists so that assessing a net while the user edits, to decide + * what to offer, never acquires a device or a worker pool. */ instantiate( this: void, diff --git a/libs/@hashintel/petrinaut-core/src/experiments/experiment-backend.ts b/libs/@hashintel/petrinaut-core/src/experiments/experiment-backend.ts index 81c17851a30..fd34796f23e 100644 --- a/libs/@hashintel/petrinaut-core/src/experiments/experiment-backend.ts +++ b/libs/@hashintel/petrinaut-core/src/experiments/experiment-backend.ts @@ -4,11 +4,9 @@ * @layerRoot core.experiments * @role Chooses a compute backend for an experiment, and asks whether it can run a net * - * Small on purpose. The runtime half of swappability already exists and is not - * restated here: both the worker-pool and WebGPU paths already produce a - * `MonteCarloExperiment`, and `ExperimentsProvider` consumes one with no - * branching at all. What was never abstracted is *choosing* a backend and - * *asking* whether it can take a net — so that, and only that, is what this adds. + * Small. Both paths already produce a `MonteCarloExperiment` that + * `ExperimentsProvider` consumes without branching. This adds only what was + * never abstracted: choosing a backend, and asking whether it can take a net. * * Backend-specific configuration is bound when the backend object is built, not * passed through this interface. The worker-pool backend closes over a worker @@ -27,8 +25,8 @@ export type ExperimentBackend = { /** * Stable identifier, recorded against results. * - * Results from two backends are not numerically interchangeable — they use - * different random generators — so which one ran is part of the data, not a + * Results from two backends are not numerically interchangeable, they use + * different random generators, so which one ran is part of the data, not a * detail. */ readonly id: string; @@ -46,7 +44,7 @@ export type ExperimentBackend = { /** * Whether this backend could run *anything* in this environment. * - * Synchronous and cheap — a feature test, not an assessment — so a UI can + * Synchronous and cheap, a feature test, not an assessment, so a UI can * decide whether to offer the backend at all without compiling a net. A * backend that is always usable returns `true`. */ @@ -66,7 +64,7 @@ export type ExperimentBackend = { * A backend plus how eagerly to load it. * * `load` is deferred so a caller can register the WebGPU backend without pulling - * the shader generator into the initial bundle — it is imported the first time a + * the shader generator into the initial bundle, it is imported the first time a * GPU run is actually attempted. Both backends are registered at once; the choice * is per experiment, never global. */ @@ -74,11 +72,9 @@ export type ExperimentBackendRegistration = { readonly id: string; readonly label: string; /** - * Loads the backend. - * - * Called at most once per registration by `selectExperimentBackend`, which - * caches the promise: acquiring a backend twice would mean two worker pools or - * two GPU devices. + * Loads the backend. Keep it cheap and side-effect free: the selection walk + * calls it once per walk, so anything expensive a backend acquires (a worker + * pool, a GPU device) belongs in `instantiate`, not here. */ load(this: void): Promise; }; diff --git a/libs/@hashintel/petrinaut-core/src/experiments/experiment-request.ts b/libs/@hashintel/petrinaut-core/src/experiments/experiment-request.ts index 509c5574a0e..50ba9721bdf 100644 --- a/libs/@hashintel/petrinaut-core/src/experiments/experiment-request.ts +++ b/libs/@hashintel/petrinaut-core/src/experiments/experiment-request.ts @@ -1,17 +1,14 @@ /** - * What to compute — the portable half of an experiment. + * What to compute, as serializable data. * - * Deliberately closed, and deliberately plain data. No worker factory, no GPU - * options, no abort signal, no callbacks. Anything a backend needs that is not - * here describes *how* to compute rather than *what*, and belongs either to that - * backend's construction (see `ExperimentBackend`) or to the per-call options of - * `instantiate`. + * Closed and plain: no worker factory, no GPU options, no abort signal, no + * callbacks. Anything describing *how* to compute belongs to the backend's + * construction (`ExperimentBackend`) or to the per-call options of `instantiate`. * - * That single rule is what stops this type decaying into - * `CreateMonteCarloExperimentConfig | CreateGpuMonteCarloExperimentConfig`. It - * also keeps the request serializable, which is the property a future - * out-of-process backend needs: for a remote backend this object is the request - * body, so a function-valued field here would rule that backend out entirely. + * That rule keeps this from becoming + * `CreateMonteCarloExperimentConfig | CreateGpuMonteCarloExperimentConfig`, and + * keeps the request serializable. For an out-of-process backend this object is + * the request body, which a function-valued field would prevent. */ import type { PetrinautExtensionSettings } from "../extensions"; import type { HirArtifacts } from "../hir-runtime"; @@ -31,18 +28,17 @@ export type ExperimentRequest = { /** * Metrics to record. * - * Expression metrics carry their compiled artifact, because a backend may need - * to run them and cannot compile one itself — that needs the TypeScript - * frontend, which is why `hirArtifacts` is here too rather than being derived. + * Expression metrics carry their compiled artifact. A backend may need to run + * them and cannot compile one itself: that needs the TypeScript frontend, + * which is also why `hirArtifacts` is passed rather than derived. */ readonly metricSpecs: readonly MonteCarloMetricSpec[]; /** * Compiled user code for the net. * - * Optional because a net with no user code needs none. Whether the HIR *trees* - * must be included is declared per backend by `ExperimentBackend.needsHirTrees` - * rather than inferred from a backend id, so a caller can compile once for - * whichever backends it is about to ask. + * Optional: a net with no user code needs none. Whether the HIR *trees* are + * included is declared per backend by `ExperimentBackend.needsHirTrees`, so a + * caller compiles once for whichever backends it is about to ask. */ readonly hirArtifacts?: HirArtifacts; }; diff --git a/libs/@hashintel/petrinaut-core/src/experiments/select-experiment-backend.test.ts b/libs/@hashintel/petrinaut-core/src/experiments/select-experiment-backend.test.ts index b4ede73f257..ae9022bb177 100644 --- a/libs/@hashintel/petrinaut-core/src/experiments/select-experiment-backend.test.ts +++ b/libs/@hashintel/petrinaut-core/src/experiments/select-experiment-backend.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from "vitest"; import { selectExperimentBackend } from "./select-experiment-backend"; +import type { MonteCarloExperiment } from "../simulation/monte-carlo/runtime/experiment"; import type { ExperimentAssessment, ExperimentBlockers, @@ -11,7 +12,6 @@ import type { ExperimentBackendRegistration, } from "./experiment-backend"; import type { ExperimentRequest } from "./experiment-request"; -import type { MonteCarloExperiment } from "../simulation/monte-carlo/runtime/experiment"; /** Enough of a handle to be identity-compared; nothing here starts it. */ function stubHandle(name: string): MonteCarloExperiment { @@ -51,7 +51,10 @@ function registrationFor( } /** An eligible assessment whose `instantiate` succeeds. */ -function accepts(handleName: string, runtimeInfo?: string): ExperimentAssessment { +function accepts( + handleName: string, + runtimeInfo?: string, +): ExperimentAssessment { return { eligible: true, notes: [], @@ -79,7 +82,9 @@ describe("selectExperimentBackend", () => { // bundle until a GPU run is attempted; if selection loaded every candidate // eagerly that would be pointless. const secondLoad = vi.fn(() => - Promise.resolve(stubBackend({ id: "second", assessment: () => accepts("b") })), + Promise.resolve( + stubBackend({ id: "second", assessment: () => accepts("b") }), + ), ); const result = await selectExperimentBackend({ @@ -131,7 +136,7 @@ describe("selectExperimentBackend", () => { it("treats a failed instantiation as a refusal and keeps going", async () => { // Assessment is authoritative about the net; instantiation can still fail for - // the environment — a device that will not allocate. Surfacing that as a dead + // the environment, a device that will not allocate. Surfacing that as a dead // end rather than falling through would strand the user on a working net. const result = await selectExperimentBackend({ registrations: [ @@ -147,7 +152,8 @@ describe("selectExperimentBackend", () => { blockers: [ { code: "gpu-allocation-failed", - message: "The GPU could not allocate memory for 1000000 runs.", + message: + "The GPU could not allocate memory for 1000000 runs.", origin: "capacity", }, ], @@ -242,12 +248,14 @@ describe("selectExperimentBackend", () => { expect(result.ok && result.backendId).toBe("cpu"); if (!result.ok) return; - expect(result.declined[0]?.reason).toMatch(/could not be loaded.*chunk load/); + expect(result.declined[0]?.reason).toMatch( + /could not be loaded.*chunk load/, + ); }); it("builds the request once per distinct artifact requirement", async () => { // Compiling HIR trees roughly triples artifact size, so the worker-pool path - // must not pay for them — but two backends wanting the same artifacts should + // must not pay for them, but two backends wanting the same artifacts should // not compile twice either. const buildRequest = vi.fn((_options: { needsHirTrees: boolean }) => Promise.resolve(REQUEST), diff --git a/libs/@hashintel/petrinaut-core/src/experiments/select-experiment-backend.ts b/libs/@hashintel/petrinaut-core/src/experiments/select-experiment-backend.ts index 728b33f772d..4cfad68777d 100644 --- a/libs/@hashintel/petrinaut-core/src/experiments/select-experiment-backend.ts +++ b/libs/@hashintel/petrinaut-core/src/experiments/select-experiment-backend.ts @@ -1,35 +1,33 @@ /** * Picks the first backend in a preference order that will take a request. * - * This replaces hand-written fallback logic — "try the GPU, and if it declines, - * use the CPU, and put the reason in a notification" — which worked for exactly - * two backends and had to be edited to gain a third. Ordering is the caller's - * (the user's choice first, then whatever else can serve), and the walk records - * every refusal so the UI can say what was tried and why it was declined. + * Replaces fallback logic that worked for two backends and had to be edited to + * gain a third. The caller supplies the order, usually the user's choice first. + * The walk records every refusal, so a UI can report what was tried. * * Selection stops at the first backend that both assesses eligible *and* * instantiates, because the two can disagree: a net can be perfectly runnable * and the device still be out of memory. Treating instantiation failure as a * refusal keeps the fallback honest instead of surfacing a dead end. */ +import type { MonteCarloExperiment } from "../simulation/monte-carlo/runtime/experiment"; +import type { + ExperimentNote, + InstantiateExperimentOptions, +} from "./experiment-assessment"; import type { ExperimentBackend, ExperimentBackendRegistration, ExperimentSelectionFailure, } from "./experiment-backend"; -import type { - ExperimentNote, - InstantiateExperimentOptions, -} from "./experiment-assessment"; -import type { MonteCarloExperiment } from "../simulation/monte-carlo/runtime/experiment"; import type { ExperimentRequest } from "./experiment-request"; export type SelectExperimentBackendInput = { /** * Candidates in preference order, best first. * - * The caller orders these — usually the user's requested backend followed by - * the fallbacks — because preference is a product decision, not something this + * The caller orders these, usually the user's requested backend followed by + * the fallbacks, because preference is a product decision, not something this * function should infer. */ readonly registrations: readonly ExperimentBackendRegistration[]; @@ -63,7 +61,10 @@ export type SelectExperimentBackendResult = */ readonly declined: readonly ExperimentSelectionFailure[]; } - | { readonly ok: false; readonly declined: readonly ExperimentSelectionFailure[] }; + | { + readonly ok: false; + readonly declined: readonly ExperimentSelectionFailure[]; + }; /** Summarises blockers into one sentence, leading with the most actionable. */ function summarize( @@ -89,11 +90,14 @@ function summarize( // because this helper also takes plain arrays from instantiation results. return { origin: "environment", reason: "declined without a reason" }; } - const others = - sorted.length > 1 ? ` (+${sorted.length - 1} more)` : ""; + const others = sorted.length > 1 ? ` (+${sorted.length - 1} more)` : ""; return { origin: first.origin, reason: `${first.message}${others}` }; } +/** The engine marks cancellation by renaming an `Error` to `AbortError`. */ +const isAbortError = (error: unknown): boolean => + error instanceof Error && error.name === "AbortError"; + export async function selectExperimentBackend({ registrations, buildRequest, @@ -139,9 +143,38 @@ export async function selectExperimentBackend({ continue; } - const assessment = await backend.assess( - await requestFor(backend.needsHirTrees), - ); + // A cancelled experiment must not fall through to the next candidate and + // spin up work the user already abandoned. + if (instantiateOptions?.signal?.aborted) { + const abort = new Error( + "The experiment was cancelled before a backend was selected.", + ); + abort.name = "AbortError"; + throw abort; + } + + // A backend that throws — instead of returning a refusal — must not + // abandon the walk: the next candidate may run the request fine, and the + // module's own contract says instantiation failure is a refusal. Aborts + // are the exception: they are the caller's cancellation, not a refusal. + let assessment; + try { + assessment = await backend.assess( + await requestFor(backend.needsHirTrees), + ); + } catch (error) { + if (isAbortError(error)) { + throw error; + } + declined.push({ + backendId: backend.id, + origin: "environment", + reason: `${backend.label} failed while assessing the request: ${ + error instanceof Error ? error.message : String(error) + }`, + }); + continue; + } if (!assessment.eligible) { declined.push({ backendId: backend.id, @@ -150,7 +183,22 @@ export async function selectExperimentBackend({ continue; } - const instantiated = await assessment.instantiate(instantiateOptions); + let instantiated; + try { + instantiated = await assessment.instantiate(instantiateOptions); + } catch (error) { + if (isAbortError(error)) { + throw error; + } + declined.push({ + backendId: backend.id, + origin: "environment", + reason: `${backend.label} failed while instantiating: ${ + error instanceof Error ? error.message : String(error) + }`, + }); + continue; + } if (!instantiated.ok) { declined.push({ backendId: backend.id, diff --git a/libs/@hashintel/petrinaut-core/src/experiments/worker-pool-experiment-backend.ts b/libs/@hashintel/petrinaut-core/src/experiments/worker-pool-experiment-backend.ts index 1e334f8d3e1..8c5b5071691 100644 --- a/libs/@hashintel/petrinaut-core/src/experiments/worker-pool-experiment-backend.ts +++ b/libs/@hashintel/petrinaut-core/src/experiments/worker-pool-experiment-backend.ts @@ -17,13 +17,13 @@ */ import { createMonteCarloExperiment } from "../simulation/monte-carlo/runtime/experiment"; +import type { WorkerFactory } from "../simulation/api"; import type { ExperimentAssessment, ExperimentBlockers, } from "./experiment-assessment"; import type { ExperimentBackend } from "./experiment-backend"; import type { ExperimentRequest } from "./experiment-request"; -import type { WorkerFactory } from "../simulation/api"; export const WORKER_POOL_BACKEND_ID = "cpu"; @@ -32,7 +32,7 @@ export type WorkerPoolExperimentBackendOptions = { * Spawns one simulation worker. * * Bound here rather than passed per request because it is host wiring, not part - * of what to compute — the same reason it is absent from `ExperimentRequest`. A + * of what to compute, the same reason it is absent from `ExperimentRequest`. A * React provider supplies it once for the whole app. */ createWorker: WorkerFactory; @@ -92,6 +92,12 @@ function assess( : `Web Workers (${options.shardCount} shards)`, }; } catch (error) { + // A cancellation is the caller's decision, not a refusal: reporting it + // as a blocker would send the selection walk on to the next backend + // for an experiment nobody wants any more. + if (error instanceof Error && error.name === "AbortError") { + throw error; + } // The engine refuses a net whose user code has no compiled artifact, and // it phrases that better than this layer could. Reported as // `configuration` because the fix is to compile, not to edit the net. diff --git a/libs/@hashintel/petrinaut-core/vite.config.ts b/libs/@hashintel/petrinaut-core/vite.config.ts index 336a63db279..f9c8c58d075 100644 --- a/libs/@hashintel/petrinaut-core/vite.config.ts +++ b/libs/@hashintel/petrinaut-core/vite.config.ts @@ -17,7 +17,7 @@ export default defineConfig(({ command }) => ({ // Node/tooling-only reusable model compiler. This depends on the // TypeScript-powered HIR compiler and must stay out of the main entry. "compiled-model": resolve(packageRoot, "src/compiled-model.ts"), - // HIR compiler (bundles the TypeScript frontend — heavy; used by the + // HIR compiler (bundles the TypeScript frontend, heavy; used by the // LSP worker internally and by tooling/playgrounds). hir: resolve(packageRoot, "src/hir.ts"), // Dependency-free instantiation of compiled HIR artifacts. diff --git a/libs/@hashintel/petrinaut/src/react/experiments/provider.tsx b/libs/@hashintel/petrinaut/src/react/experiments/provider.tsx index c354b7dff9e..a8c455b2a4f 100644 --- a/libs/@hashintel/petrinaut/src/react/experiments/provider.tsx +++ b/libs/@hashintel/petrinaut/src/react/experiments/provider.tsx @@ -18,6 +18,7 @@ import { import { createWorkerPoolExperimentBackend, selectExperimentBackend, + WORKER_POOL_BACKEND_ID, type ExperimentBackendRegistration, type ExperimentRequest, } from "@hashintel/petrinaut-core/experiments"; @@ -452,7 +453,7 @@ export const ExperimentsProvider: React.FC = ({ // rather than an edit to this branch. const registrations: ExperimentBackendRegistration[] = [ { - id: "cpu", + id: WORKER_POOL_BACKEND_ID, label: "CPU (Web Workers)", load: () => Promise.resolve( diff --git a/libs/@local/petrinaut-arch-docs/content/experiments/backend-selection.mdx b/libs/@local/petrinaut-arch-docs/content/experiments/backend-selection.mdx index 083a5ed8ee0..bbd3b19dd25 100644 --- a/libs/@local/petrinaut-arch-docs/content/experiments/backend-selection.mdx +++ b/libs/@local/petrinaut-arch-docs/content/experiments/backend-selection.mdx @@ -12,7 +12,7 @@ choosing between backends when one declines. ## Two phases -`assess` settles whether the *net and configuration* are runnable, without +`assess` settles whether the _net and configuration_ are runnable, without acquiring a device or a worker pool. An eligible result carries an `instantiate()` closure, so work already done (a compiled shader, a shard plan) flows into the run and the verdict can never be about a different thing. @@ -32,19 +32,19 @@ registrations (preference order) A backend may be a **subset** engine, so declining a net is ordinary operation, not an exception. Blockers carry a `code`, an optional `itemId`, and an `origin`: -| origin | Who acts | Example | -| --- | --- | --- | -| `model` | edit the net | a typed place with no capacity | -| `configuration` | edit the experiment | a metric shape the backend cannot serve | -| `environment` | nobody, so hide the option | no WebGPU in this browser | -| `capacity` | retry, or use fewer runs | device out of memory | +| origin | Who acts | Example | +| --------------- | -------------------------- | --------------------------------------- | +| `model` | edit the net | a typed place with no capacity | +| `configuration` | edit the experiment | a metric shape the backend cannot serve | +| `environment` | nobody, so hide the option | no WebGPU in this browser | +| `capacity` | retry, or use fewer runs | device out of memory | `capacity` is separate from `environment` because it is transient: "use fewer runs" is right where "hide the option" would be wrong. ## What stays out of the request -`ExperimentRequest` is plain, serializable data. Anything describing *how* to +`ExperimentRequest` is plain, serializable data. Anything describing _how_ to compute, such as a worker factory, an ODE method, an abort signal or a note callback, belongs to the backend's construction or to per-call options. That rule is what keeps the request from becoming a union of every backend's knobs, and what leaves