From 95d4fd185cbad3c090b3ebba544bc8272cb7326c Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Mon, 27 Jul 2026 22:35:43 -0700 Subject: [PATCH 1/8] fix trace span secret sanitization --- .../executor/execution/block-executor.test.ts | 361 +++++++++++++++++- apps/sim/executor/execution/block-executor.ts | 86 ++++- .../environment-secret-sanitizer.test.ts | 144 +++++++ .../utils/environment-secret-sanitizer.ts | 223 +++++++++++ apps/sim/executor/variables/resolver.test.ts | 41 ++ apps/sim/executor/variables/resolver.ts | 10 +- .../human-in-the-loop-manager.test.ts | 38 ++ .../executor/human-in-the-loop-manager.ts | 147 ++++--- .../lib/workflows/streaming/streaming.test.ts | 59 +++ apps/sim/lib/workflows/streaming/streaming.ts | 24 +- 10 files changed, 1025 insertions(+), 108 deletions(-) create mode 100644 apps/sim/executor/utils/environment-secret-sanitizer.test.ts create mode 100644 apps/sim/executor/utils/environment-secret-sanitizer.ts diff --git a/apps/sim/executor/execution/block-executor.test.ts b/apps/sim/executor/execution/block-executor.test.ts index fe4bb0e624d..bc7cc437c66 100644 --- a/apps/sim/executor/execution/block-executor.test.ts +++ b/apps/sim/executor/execution/block-executor.test.ts @@ -4,11 +4,14 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { clearLargeValueCacheForTests } from '@/lib/execution/payloads/cache' import { isLargeArrayManifest } from '@/lib/execution/payloads/large-array-manifest-metadata' +import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans' import { BlockType } from '@/executor/constants' import type { DAGNode } from '@/executor/dag/builder' import { BlockExecutor } from '@/executor/execution/block-executor' +import { serializePauseSnapshot } from '@/executor/execution/snapshot-serializer' import { ExecutionState } from '@/executor/execution/state' -import type { BlockHandler, ExecutionContext } from '@/executor/types' +import type { ContextExtensions } from '@/executor/execution/types' +import type { BlockHandler, ExecutionContext, ExecutionResult } from '@/executor/types' import { VariableResolver } from '@/executor/variables/resolver' import type { SerializedBlock, SerializedWorkflow } from '@/serializer/types' @@ -75,6 +78,44 @@ function createNode(block: SerializedBlock): DAGNode { } } +function createExecutorForTest( + block: SerializedBlock, + state: ExecutionState, + handler: BlockHandler, + extensions: Partial = {} +): BlockExecutor { + const workflow: SerializedWorkflow = { + version: '1', + blocks: [block], + connections: [], + loops: {}, + parallels: {}, + } + const resolver = new VariableResolver(workflow, {}, state) + + return new BlockExecutor( + [handler], + resolver, + { + workspaceId: 'workspace-1', + executionId: 'execution-1', + userId: 'user-1', + metadata: { + requestId: 'request-1', + executionId: 'execution-1', + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + userId: 'user-1', + triggerType: 'manual', + useDraftState: false, + startTime: new Date().toISOString(), + }, + ...extensions, + }, + state + ) +} + describe('BlockExecutor', () => { beforeEach(() => { vi.clearAllMocks() @@ -82,6 +123,304 @@ describe('BlockExecutor', () => { mockUploadFile.mockImplementation(async ({ customKey }) => ({ key: customKey })) }) + it.each([ + { label: 'provider time segments', includeTimeSegments: true }, + { label: 'fallback tool calls', includeTimeSegments: false }, + ])( + 'sanitizes agent trace I/O while preserving runtime values with $label', + async ({ includeTimeSegments }) => { + const secret = 'trace-secret/value' + const unreferencedValue = 'us-east-1' + const block: SerializedBlock = { + ...createBlock(), + id: 'agent-block-1', + metadata: { id: BlockType.AGENT, name: 'Agent' }, + config: { + tool: BlockType.AGENT, + params: { + prompt: 'Use {{TRACE_SECRET}}', + }, + }, + } + const state = new ExecutionState() + const execute = vi.fn(async (_ctx, _block, inputs) => { + expect(inputs.prompt).toBe(`Use ${secret}`) + return { + content: `Echoed ${secret}`, + region: unreferencedValue, + providerTiming: { + startTime: '2024-01-01T10:00:00.000Z', + endTime: '2024-01-01T10:00:02.000Z', + duration: 2000, + ...(includeTimeSegments && { + timeSegments: [ + { + type: 'model' as const, + name: 'Model', + startTime: 1704103200000, + endTime: 1704103201000, + duration: 1000, + assistantContent: `Calling with ${secret}`, + toolCalls: [ + { + id: 'call-1', + name: 'lookup', + arguments: { query: secret }, + }, + ], + }, + { + type: 'tool' as const, + name: 'lookup', + startTime: 1704103201000, + endTime: 1704103202000, + duration: 1000, + }, + ], + }), + }, + toolCalls: { + list: [ + { + name: 'lookup', + arguments: { query: secret }, + result: { echoed: secret }, + duration: 1000, + }, + ], + count: 1, + }, + childTraceSpans: [ + { + id: 'child-1', + name: 'Child', + type: 'function', + duration: 1, + startTime: '2024-01-01T10:00:00.000Z', + endTime: '2024-01-01T10:00:00.001Z', + input: { query: secret }, + output: { echoed: secret }, + }, + ], + } + }) + const handler: BlockHandler = { + canHandle: () => true, + execute, + } + const onBlockComplete = vi.fn(async () => {}) + const executor = createExecutorForTest(block, state, handler, { onBlockComplete }) + const ctx = createContext(state) + ctx.environmentVariables = { + TRACE_SECRET: secret, + UNREFERENCED_REGION: unreferencedValue, + } + + const output = await executor.execute(ctx, createNode(block), block) + + expect(output.content).toBe(`Echoed ${secret}`) + expect(output.toolCalls.list[0].arguments.query).toBe(secret) + expect(state.getBlockOutput(block.id)?.content).toBe(`Echoed ${secret}`) + + await vi.waitFor(() => { + expect(onBlockComplete).toHaveBeenCalled() + }) + + const serializedLog = JSON.stringify(ctx.blockLogs[0]) + const serializedCallback = JSON.stringify(onBlockComplete.mock.calls[0]) + expect(serializedLog).not.toContain(secret) + expect(serializedLog).toContain('{{TRACE_SECRET}}') + expect(serializedLog).toContain(unreferencedValue) + expect(serializedCallback).not.toContain(secret) + expect(serializedCallback).toContain('{{TRACE_SECRET}}') + + const { traceSpans } = buildTraceSpans({ + success: true, + output: {}, + logs: ctx.blockLogs, + } as ExecutionResult) + const serializedSpans = JSON.stringify(traceSpans) + expect(serializedSpans).not.toContain(secret) + expect(serializedSpans).toContain('{{TRACE_SECRET}}') + } + ) + + it('sanitizes failed logs and callbacks while preserving runtime errors', async () => { + const secret = 'failure-secret' + const block: SerializedBlock = { + ...createBlock(), + config: { + tool: BlockType.FUNCTION, + params: { + code: 'throw new Error("{{TRACE_SECRET}}")', + }, + }, + } + const state = new ExecutionState() + const handler: BlockHandler = { + canHandle: () => true, + execute: async (_ctx, _block, inputs) => { + expect(inputs.code).toContain(secret) + throw new Error(`Execution failed with ${secret}`) + }, + } + const onBlockComplete = vi.fn(async () => {}) + const executor = createExecutorForTest(block, state, handler, { onBlockComplete }) + const ctx = createContext(state) + ctx.environmentVariables = { TRACE_SECRET: secret } + + await expect(executor.execute(ctx, createNode(block), block)).rejects.toThrow(secret) + + expect(state.getBlockOutput(block.id)?.error).toContain(secret) + expect(ctx.blockLogs[0].error).toBe('Execution failed with {{TRACE_SECRET}}') + expect(JSON.stringify(ctx.blockLogs[0])).not.toContain(secret) + await vi.waitFor(() => { + expect(onBlockComplete).toHaveBeenCalled() + }) + expect(JSON.stringify(onBlockComplete.mock.calls[0])).not.toContain(secret) + }) + + it('sanitizes error-handler logs without changing the handled runtime output', async () => { + const secret = 'handled-secret' + const block: SerializedBlock = { + ...createBlock(), + config: { + tool: BlockType.FUNCTION, + params: { + code: 'throw new Error("{{TRACE_SECRET}}")', + }, + }, + } + const state = new ExecutionState() + const handler: BlockHandler = { + canHandle: () => true, + execute: async () => { + throw new Error(`Handled ${secret}`) + }, + } + const executor = createExecutorForTest(block, state, handler) + const infoSpy = vi.spyOn( + ( + executor as unknown as { + execLogger: { info: (message: string, metadata?: Record) => void } + } + ).execLogger, + 'info' + ) + const ctx = createContext(state) + ctx.environmentVariables = { TRACE_SECRET: secret } + const node = createNode(block) + node.outgoingEdges.set('error-edge', { + id: 'error-edge', + source: block.id, + target: 'error-handler', + sourceHandle: 'error', + targetHandle: 'target', + }) + + const output = await executor.execute(ctx, node, block) + + expect(output.error).toBe(`Handled ${secret}`) + expect(state.getBlockOutput(block.id)?.error).toBe(`Handled ${secret}`) + expect(ctx.blockLogs[0].errorHandled).toBe(true) + expect(JSON.stringify(ctx.blockLogs[0])).not.toContain(secret) + expect(JSON.stringify(ctx.blockLogs[0])).toContain('{{TRACE_SECRET}}') + expect(infoSpy).toHaveBeenCalledWith( + 'Block has error port - returning error output instead of throwing', + expect.objectContaining({ error: 'Handled {{TRACE_SECRET}}' }) + ) + }) + + it('sanitizes soft-abort agent inputs while leaving execution resolution unchanged', async () => { + const secret = 'abort-secret' + const block: SerializedBlock = { + ...createBlock(), + id: 'agent-block-1', + metadata: { id: BlockType.AGENT, name: 'Agent' }, + config: { + tool: BlockType.AGENT, + params: { + prompt: 'Use {{TRACE_SECRET}}', + }, + }, + } + const state = new ExecutionState() + const abortController = new AbortController() + const handler: BlockHandler = { + canHandle: () => true, + execute: async (_ctx, _block, inputs) => { + expect(inputs.prompt).toBe(`Use ${secret}`) + abortController.abort('user') + throw new DOMException(`Stopped ${secret}`, 'AbortError') + }, + } + const onBlockComplete = vi.fn(async () => {}) + const executor = createExecutorForTest(block, state, handler, { onBlockComplete }) + const ctx = createContext(state) + ctx.environmentVariables = { TRACE_SECRET: secret } + ctx.abortSignal = abortController.signal + + const output = await executor.execute(ctx, createNode(block), block) + + expect(output).toEqual({ content: '' }) + expect(ctx.blockLogs[0].success).toBe(true) + expect(JSON.stringify(ctx.blockLogs[0])).not.toContain(secret) + expect(JSON.stringify(ctx.blockLogs[0])).toContain('{{TRACE_SECRET}}') + await vi.waitFor(() => { + expect(onBlockComplete).toHaveBeenCalled() + }) + expect(JSON.stringify(onBlockComplete.mock.calls[0])).not.toContain(secret) + }) + + it('keeps snapshot state executable and re-resolves current values on retry', async () => { + const firstSecret = 'first-runtime-secret' + const secondSecret = 'second-runtime-secret' + const block: SerializedBlock = { + ...createBlock(), + config: { + tool: BlockType.FUNCTION, + params: { + code: 'return "{{TRACE_SECRET}}"', + }, + }, + } + const state = new ExecutionState() + const receivedRuntimeCode: string[] = [] + const handler: BlockHandler = { + canHandle: () => true, + execute: async (_ctx, _block, inputs) => { + receivedRuntimeCode.push(inputs.code) + return { result: inputs.code } + }, + } + const executor = createExecutorForTest(block, state, handler) + const firstContext = createContext(state) + firstContext.environmentVariables = { TRACE_SECRET: firstSecret } + + await executor.execute(firstContext, createNode(block), block) + const snapshot = JSON.parse(serializePauseSnapshot(firstContext, ['next-block']).snapshot) as { + state: { + blockStates: Record + blockLogs: ExecutionContext['blockLogs'] + } + } + + expect(snapshot.state.blockStates[block.id].output.result).toContain(firstSecret) + expect(JSON.stringify(snapshot.state.blockLogs)).not.toContain(firstSecret) + expect(JSON.stringify(snapshot.state.blockLogs)).toContain('{{TRACE_SECRET}}') + + const resumedContext = createContext(state) + resumedContext.blockLogs.push(...snapshot.state.blockLogs) + resumedContext.environmentVariables = { TRACE_SECRET: secondSecret } + + await executor.execute(resumedContext, createNode(block), block) + + expect(receivedRuntimeCode).toEqual([`return "${firstSecret}"`, `return "${secondSecret}"`]) + expect(state.getBlockOutput(block.id)?.result).toContain(secondSecret) + expect(JSON.stringify(resumedContext.blockLogs)).not.toContain(firstSecret) + expect(JSON.stringify(resumedContext.blockLogs)).not.toContain(secondSecret) + }) + it('persists function output arrays as manifests in execution state', async () => { const block = createBlock() const workflow: SerializedWorkflow = { @@ -439,20 +778,20 @@ describe('BlockExecutor', () => { }) describe('BlockExecutor streaming pump', () => { - function createAgentBlock(): SerializedBlock { + function createAgentBlock(params: Record = {}): SerializedBlock { return { id: 'agent-block-1', metadata: { id: BlockType.AGENT, name: 'Agent' }, position: { x: 0, y: 0 }, - config: { tool: BlockType.AGENT, params: {} }, + config: { tool: BlockType.AGENT, params }, inputs: {}, outputs: {}, enabled: true, } } - function createExecutor(handler: BlockHandler) { - const block = createAgentBlock() + function createExecutor(handler: BlockHandler, params: Record = {}) { + const block = createAgentBlock(params) const workflow: SerializedWorkflow = { version: '1', blocks: [block], @@ -676,16 +1015,20 @@ describe('BlockExecutor streaming pump', () => { }) it('fails on timeout but keeps drained answer text in block output', async () => { + const secret = 'partial-stream-secret' const abortController = new AbortController() const handler = createAgentEventsStreamingHandler({ events: [ - { type: 'text_delta', text: 'partial before timeout', turn: 'final' }, + { type: 'text_delta', text: `partial ${secret} before timeout`, turn: 'final' }, { type: 'thinking_delta', text: 'more' }, ], }) - const { executor, block, state } = createExecutor(handler) + const { executor, block, state } = createExecutor(handler, { + prompt: '{{TRACE_SECRET}}', + }) const ctx = createContext(state) + ctx.environmentVariables = { TRACE_SECRET: secret } ctx.abortSignal = abortController.signal ctx.onStream = async (streamingExec) => { streamingExec.subscribe?.({ onEvent: async () => {} }) @@ -707,7 +1050,9 @@ describe('BlockExecutor streaming pump', () => { const output = state.getBlockOutput(block.id) expect(output?.error).toBeTruthy() - expect(output?.content).toBe('partial before timeout') + expect(output?.content).toBe(`partial ${secret} before timeout`) + expect(JSON.stringify(ctx.blockLogs[0])).not.toContain(secret) + expect(ctx.blockLogs[0].output?.content).toBe('partial {{TRACE_SECRET}} before timeout') }) it('with PII redaction: no live forward and strips thinking from traces', async () => { diff --git a/apps/sim/executor/execution/block-executor.ts b/apps/sim/executor/execution/block-executor.ts index f4b12617509..84ad85291f9 100644 --- a/apps/sim/executor/execution/block-executor.ts +++ b/apps/sim/executor/execution/block-executor.ts @@ -43,6 +43,10 @@ import { type StreamingExecution, } from '@/executor/types' import { streamingResponseFormatProcessor } from '@/executor/utils' +import { + createEnvironmentSecretSanitizer, + type EnvironmentSecretSanitizer, +} from '@/executor/utils/environment-secret-sanitizer' import { buildBlockExecutionError, normalizeError } from '@/executor/utils/errors' import { buildUnifiedParentIterations, @@ -100,6 +104,10 @@ export class BlockExecutor { const blockType = block.metadata?.id ?? '' const isSentinel = isSentinelBlockType(blockType) + const sanitizeEnvironmentSecrets = createEnvironmentSecretSanitizer( + block.config, + ctx.environmentVariables + ) // Capture startedAt and startTime at the same synchronous instant so // blockLog.startedAt and performance.now()-derived durationMs share a @@ -159,7 +167,11 @@ export class BlockExecutor { } if (blockLog) { - blockLog.input = this.sanitizeInputsForLog(inputsForLog, block.metadata?.id) + blockLog.input = this.sanitizeInputsForLog( + inputsForLog, + sanitizeEnvironmentSecrets, + block.metadata?.id + ) } } catch (error) { cleanupSelfReference?.() @@ -173,6 +185,7 @@ export class BlockExecutor { blockLog, inputsForLog, isSentinel, + sanitizeEnvironmentSecrets, 'input_resolution' ) } @@ -277,9 +290,13 @@ export class BlockExecutor { blockLog.endedAt = endedAt blockLog.durationMs = duration blockLog.success = true - blockLog.output = filterOutputForLog(block.metadata?.id || '', normalizedOutput, { block }) + blockLog.output = this.sanitizeOutputForLog( + block, + normalizedOutput, + sanitizeEnvironmentSecrets + ) if (normalizedOutput.childTraceSpans && Array.isArray(normalizedOutput.childTraceSpans)) { - blockLog.childTraceSpans = normalizedOutput.childTraceSpans + blockLog.childTraceSpans = sanitizeEnvironmentSecrets(normalizedOutput.childTraceSpans) } } @@ -291,15 +308,17 @@ export class BlockExecutor { typeof normalizedOutput._childWorkflowInstanceId === 'string' ? normalizedOutput._childWorkflowInstanceId : undefined - const displayOutput = filterOutputForLog(block.metadata?.id || '', normalizedOutput, { + const displayOutput = this.sanitizeOutputForLog( block, - }) + normalizedOutput, + sanitizeEnvironmentSecrets + ) this.fireBlockCompleteCallback( blockStartPromise, ctx, node, block, - this.sanitizeInputsForLog(inputsForLog, block.metadata?.id), + this.sanitizeInputsForLog(inputsForLog, sanitizeEnvironmentSecrets, block.metadata?.id), displayOutput, duration, blockLog.startedAt, @@ -321,6 +340,7 @@ export class BlockExecutor { blockLog, inputsForLog, isSentinel, + sanitizeEnvironmentSecrets, 'execution', streamingPartialOutput ) @@ -379,12 +399,14 @@ export class BlockExecutor { blockLog: BlockLog | undefined, inputsForLog: Record, isSentinel: boolean, + sanitizeEnvironmentSecrets: EnvironmentSecretSanitizer, phase: 'input_resolution' | 'execution', streamingPartialOutput?: Record ): Promise { const endedAt = new Date().toISOString() const duration = performance.now() - startTime const errorMessage = normalizeError(error) + const sanitizedErrorMessage = sanitizeEnvironmentSecrets(errorMessage) const hasLogInputs = inputsForLog && typeof inputsForLog === 'object' && Object.keys(inputsForLog).length > 0 const input = hasLogInputs @@ -412,8 +434,12 @@ export class BlockExecutor { blockLog.durationMs = duration blockLog.success = true blockLog.error = undefined - blockLog.input = this.sanitizeInputsForLog(input, block.metadata?.id) - blockLog.output = filterOutputForLog(block.metadata?.id || '', softOutput, { block }) + blockLog.input = this.sanitizeInputsForLog( + input, + sanitizeEnvironmentSecrets, + block.metadata?.id + ) + blockLog.output = this.sanitizeOutputForLog(block, softOutput, sanitizeEnvironmentSecrets) } this.execLogger.info('Block stream aborted by client; soft-completing', { @@ -427,8 +453,8 @@ export class BlockExecutor { ctx, node, block, - this.sanitizeInputsForLog(input, block.metadata?.id), - filterOutputForLog(block.metadata?.id || '', softOutput, { block }), + this.sanitizeInputsForLog(input, sanitizeEnvironmentSecrets, block.metadata?.id), + this.sanitizeOutputForLog(block, softOutput, sanitizeEnvironmentSecrets), duration, blockLog.startedAt, blockLog.executionOrder, @@ -463,12 +489,16 @@ export class BlockExecutor { blockLog.endedAt = endedAt blockLog.durationMs = duration blockLog.success = false - blockLog.error = errorMessage - blockLog.input = this.sanitizeInputsForLog(input, block.metadata?.id) - blockLog.output = filterOutputForLog(block.metadata?.id || '', errorOutput, { block }) + blockLog.error = sanitizedErrorMessage + blockLog.input = this.sanitizeInputsForLog( + input, + sanitizeEnvironmentSecrets, + block.metadata?.id + ) + blockLog.output = this.sanitizeOutputForLog(block, errorOutput, sanitizeEnvironmentSecrets) if (ChildWorkflowError.isChildWorkflowError(error) && error.childTraceSpans.length > 0) { - blockLog.childTraceSpans = error.childTraceSpans + blockLog.childTraceSpans = sanitizeEnvironmentSecrets(error.childTraceSpans) } } @@ -477,7 +507,7 @@ export class BlockExecutor { { blockId: node.id, blockType: block.metadata?.id, - error: errorMessage, + error: sanitizedErrorMessage, } ) @@ -485,13 +515,17 @@ export class BlockExecutor { const childWorkflowInstanceId = ChildWorkflowError.isChildWorkflowError(error) ? error.childWorkflowInstanceId : undefined - const displayOutput = filterOutputForLog(block.metadata?.id || '', errorOutput, { block }) + const displayOutput = this.sanitizeOutputForLog( + block, + errorOutput, + sanitizeEnvironmentSecrets + ) this.fireBlockCompleteCallback( blockStartPromise, ctx, node, block, - this.sanitizeInputsForLog(input, block.metadata?.id), + this.sanitizeInputsForLog(input, sanitizeEnvironmentSecrets, block.metadata?.id), displayOutput, duration, blockLog.startedAt, @@ -508,7 +542,7 @@ export class BlockExecutor { } this.execLogger.info('Block has error port - returning error output instead of throwing', { blockId: node.id, - error: errorMessage, + error: sanitizedErrorMessage, }) return errorOutput } @@ -615,6 +649,7 @@ export class BlockExecutor { */ private sanitizeInputsForLog( inputs: Record, + sanitizeEnvironmentSecrets: EnvironmentSecretSanitizer, blockType?: string ): Record { // Custom (deploy-as-block) blocks run via an internal `workflow_executor`; the @@ -671,7 +706,20 @@ export class BlockExecutor { } } - return redactApiKeys(result) + return sanitizeEnvironmentSecrets(redactApiKeys(result)) + } + + /** + * Builds the display-only output shared by persisted logs and live callbacks. + * Runtime output remains untouched for state, handlers, retries, and resume. + */ + private sanitizeOutputForLog( + block: SerializedBlock, + output: NormalizedBlockOutput, + sanitizeEnvironmentSecrets: EnvironmentSecretSanitizer + ): NormalizedBlockOutput { + const filteredOutput = filterOutputForLog(block.metadata?.id ?? '', output, { block }) + return sanitizeEnvironmentSecrets(redactApiKeys(filteredOutput)) as NormalizedBlockOutput } /** diff --git a/apps/sim/executor/utils/environment-secret-sanitizer.test.ts b/apps/sim/executor/utils/environment-secret-sanitizer.test.ts new file mode 100644 index 00000000000..6f74eade253 --- /dev/null +++ b/apps/sim/executor/utils/environment-secret-sanitizer.test.ts @@ -0,0 +1,144 @@ +import { describe, expect, it } from 'vitest' +import { createEnvironmentSecretSanitizer } from '@/executor/utils/environment-secret-sanitizer' + +describe('createEnvironmentSecretSanitizer', () => { + it('sanitizes referenced values recursively without mutating the source', () => { + const secret = 'top-secret/value' + const source = { + message: `Bearer ${secret}`, + nested: [{ [secret]: secret }], + } + const sanitize = createEnvironmentSecretSanitizer( + { credential: '{{API_SECRET}}' }, + { API_SECRET: secret } + ) + + const sanitized = sanitize(source) + + expect(sanitized).toEqual({ + message: 'Bearer {{API_SECRET}}', + nested: [{ '{{API_SECRET}}': '{{API_SECRET}}' }], + }) + expect(source).toEqual({ + message: `Bearer ${secret}`, + nested: [{ [secret]: secret }], + }) + expect(sanitized).not.toBe(source) + expect(sanitized.nested).not.toBe(source.nested) + }) + + it('sanitizes URL-encoded secret values', () => { + const secret = 'secret/value with spaces' + const sanitize = createEnvironmentSecretSanitizer('{{API_SECRET}}', { API_SECRET: secret }) + + expect(sanitize(`token=${encodeURIComponent(secret)}`)).toBe('token={{API_SECRET}}') + }) + + it('sanitizes mixed-case percent escapes without folding ordinary characters', () => { + const secret = 'CaseSensitive/value:next' + const sanitize = createEnvironmentSecretSanitizer('{{API_SECRET}}', { API_SECRET: secret }) + + expect(sanitize('token=CaseSensitive%2fvalue%3Anext')).toBe('token={{API_SECRET}}') + expect(sanitize('token=casesensitive%2fvalue%3Anext')).toBe( + 'token=casesensitive%2fvalue%3Anext' + ) + }) + + it('sanitizes form-encoded secrets that use plus for spaces', () => { + const secret = 'secret value/next' + const sanitize = createEnvironmentSecretSanitizer('{{API_SECRET}}', { API_SECRET: secret }) + + expect(sanitize('token=secret+value%2fnext')).toBe('token={{API_SECRET}}') + }) + + it('still sanitizes literal values that cannot be URI encoded', () => { + const secret = `secret-${String.fromCharCode(0xd800)}` + const sanitize = createEnvironmentSecretSanitizer('{{API_SECRET}}', { API_SECRET: secret }) + + expect(sanitize(secret)).toBe('{{API_SECRET}}') + }) + + it('only uses environment variables referenced by the block configuration', () => { + const sanitize = createEnvironmentSecretSanitizer( + { credential: '{{REFERENCED}}' }, + { + REFERENCED: 'replace-me', + UNREFERENCED: 'ordinary-value', + } + ) + + expect(sanitize('replace-me ordinary-value')).toBe('{{REFERENCED}} ordinary-value') + }) + + it('ignores empty and missing referenced values', () => { + const sanitize = createEnvironmentSecretSanitizer(['{{EMPTY}}', '{{MISSING}}'], { EMPTY: '' }) + + expect(sanitize('unchanged')).toBe('unchanged') + }) + + it('replaces overlapping values longest first', () => { + const sanitize = createEnvironmentSecretSanitizer(['{{SHORT}}', '{{LONG}}'], { + SHORT: 'secret', + LONG: 'secret-suffix', + }) + + expect(sanitize('secret-suffix secret')).toBe('{{LONG}} {{SHORT}}') + }) + + it('uses the lexicographically first name for duplicate values', () => { + const sanitize = createEnvironmentSecretSanitizer(['{{Z_SECRET}}', '{{A_SECRET}}'], { + Z_SECRET: 'same-value', + A_SECRET: 'same-value', + }) + + expect(sanitize('same-value')).toBe('{{A_SECRET}}') + }) + + it('does not re-sanitize placeholders inserted earlier in the same string', () => { + const sanitize = createEnvironmentSecretSanitizer(['{{A_SECRET}}', '{{B_SECRET}}'], { + A_SECRET: 'secret-value', + B_SECRET: 'A_SECRET', + }) + + expect(sanitize('secret-value A_SECRET')).toBe('{{A_SECRET}} {{B_SECRET}}') + }) + + it('handles special object keys without changing the object prototype', () => { + const source = Object.create(null) as Record + Object.defineProperty(source, '__proto__', { + value: 'secret', + enumerable: true, + configurable: true, + writable: true, + }) + source.constructor = 'secret' + const sanitize = createEnvironmentSecretSanitizer('{{SECRET}}', { SECRET: 'secret' }) + + const sanitized = sanitize(source) + + expect(Object.getPrototypeOf(sanitized)).toBeNull() + expect(Object.keys(sanitized)).toEqual(['__proto__', 'constructor']) + expect(sanitized.__proto__).toBe('{{SECRET}}') + expect(sanitized.constructor).toBe('{{SECRET}}') + }) + + it('leaves non-plain runtime objects unchanged', () => { + const date = new Date() + const sanitize = createEnvironmentSecretSanitizer('{{SECRET}}', { SECRET: 'secret' }) + + expect(sanitize(date)).toBe(date) + }) + + it('finds references in nested configuration keys and tolerates cycles', () => { + const configured: Record = {} + configured['prefix-{{SECRET}}'] = configured + const source: Record = { value: 'secret' } + source.self = source + const sanitize = createEnvironmentSecretSanitizer(configured, { SECRET: 'secret' }) + + const sanitized = sanitize(source) + + expect(sanitized.value).toBe('{{SECRET}}') + expect(sanitized.self).toBe(sanitized) + }) +}) diff --git a/apps/sim/executor/utils/environment-secret-sanitizer.ts b/apps/sim/executor/utils/environment-secret-sanitizer.ts new file mode 100644 index 00000000000..5daff377108 --- /dev/null +++ b/apps/sim/executor/utils/environment-secret-sanitizer.ts @@ -0,0 +1,223 @@ +import { createEnvVarPattern } from '@/executor/utils/reference-validation' + +interface SecretReplacement { + value: string + comparisonValue: string + normalizePercentEscapes: boolean + placeholder: string +} + +export type EnvironmentSecretSanitizer = (value: T) => T + +function isPlainObject(value: object): value is Record { + const prototype = Object.getPrototypeOf(value) + return prototype === Object.prototype || prototype === null +} + +function collectReferencedEnvironmentVariables( + value: unknown, + referencedNames: Set, + visited: WeakSet +): void { + if (typeof value === 'string') { + for (const match of value.matchAll(createEnvVarPattern())) { + const name = match[1]?.trim() + if (name) { + referencedNames.add(name) + } + } + return + } + + if (value === null || typeof value !== 'object' || visited.has(value)) { + return + } + visited.add(value) + + if (Array.isArray(value)) { + for (const item of value) { + collectReferencedEnvironmentVariables(item, referencedNames, visited) + } + return + } + + if (!isPlainObject(value)) { + return + } + + for (const [key, item] of Object.entries(value)) { + collectReferencedEnvironmentVariables(key, referencedNames, visited) + collectReferencedEnvironmentVariables(item, referencedNames, visited) + } +} + +function compareStrings(left: string, right: string): number { + if (left < right) return -1 + if (left > right) return 1 + return 0 +} + +function normalizePercentEscapeCase(value: string): string { + return value.replace(/%[0-9a-f]{2}/gi, (percentEscape) => percentEscape.toUpperCase()) +} + +function buildSecretReplacements( + configuredValue: unknown, + environmentVariables: Record +): SecretReplacement[] { + const referencedNames = new Set() + collectReferencedEnvironmentVariables(configuredValue, referencedNames, new WeakSet()) + + const replacementsByValue = new Map() + for (const name of [...referencedNames].sort()) { + const value = environmentVariables[name] + if (!value) { + continue + } + + const placeholder = `{{${name}}}` + const exactKey = `exact:${value}` + if (!replacementsByValue.has(exactKey)) { + replacementsByValue.set(exactKey, { + value, + comparisonValue: value, + normalizePercentEscapes: false, + placeholder, + }) + } + + let encodedValue: string + try { + encodedValue = encodeURIComponent(value) + } catch { + continue + } + + const encodedVariants = new Set([encodedValue, encodedValue.replaceAll('%20', '+')]) + for (const encodedVariant of encodedVariants) { + if (encodedVariant === value) { + continue + } + + const comparisonValue = normalizePercentEscapeCase(encodedVariant) + const encodedKey = `encoded:${comparisonValue}` + if (!replacementsByValue.has(encodedKey)) { + replacementsByValue.set(encodedKey, { + value: encodedVariant, + comparisonValue, + normalizePercentEscapes: true, + placeholder, + }) + } + } + } + + return [...replacementsByValue.values()].sort( + (left, right) => + right.value.length - left.value.length || + compareStrings(left.placeholder, right.placeholder) || + compareStrings(left.comparisonValue, right.comparisonValue) + ) +} + +function sanitizeString(value: string, replacements: SecretReplacement[]): string { + let cursor = 0 + let sanitized = '' + const percentNormalizedValue = replacements.some( + (replacement) => replacement.normalizePercentEscapes + ) + ? normalizePercentEscapeCase(value) + : value + + while (cursor < value.length) { + let nextIndex = -1 + let nextReplacement: SecretReplacement | undefined + + for (const replacement of replacements) { + const source = replacement.normalizePercentEscapes ? percentNormalizedValue : value + const index = source.indexOf(replacement.comparisonValue, cursor) + if (index !== -1 && (nextIndex === -1 || index < nextIndex)) { + nextIndex = index + nextReplacement = replacement + } + } + + if (!nextReplacement) { + sanitized += value.slice(cursor) + break + } + + sanitized += value.slice(cursor, nextIndex) + sanitized += nextReplacement.placeholder + cursor = nextIndex + nextReplacement.value.length + } + + return sanitized +} + +function sanitizeValue( + value: unknown, + replacements: SecretReplacement[], + visited: WeakMap +): unknown { + if (typeof value === 'string') { + return sanitizeString(value, replacements) + } + + if (value === null || typeof value !== 'object') { + return value + } + + const existing = visited.get(value) + if (existing !== undefined) { + return existing + } + + if (Array.isArray(value)) { + const sanitized: unknown[] = [] + visited.set(value, sanitized) + for (const item of value) { + sanitized.push(sanitizeValue(item, replacements, visited)) + } + return sanitized + } + + if (!isPlainObject(value)) { + return value + } + + const sanitized = Object.create(Object.getPrototypeOf(value)) as Record + visited.set(value, sanitized) + + for (const [key, item] of Object.entries(value)) { + const sanitizedKey = sanitizeString(key, replacements) + Object.defineProperty(sanitized, sanitizedKey, { + value: sanitizeValue(item, replacements, visited), + enumerable: true, + configurable: true, + writable: true, + }) + } + + return sanitized +} + +/** + * Creates a sanitizer for observability values using environment variables + * explicitly referenced by the unresolved block configuration. + * + * Runtime values are never changed. Sanitized arrays and plain objects are + * copied, while unsupported object types are returned unchanged. + */ +export function createEnvironmentSecretSanitizer( + configuredValue: unknown, + environmentVariables: Record +): EnvironmentSecretSanitizer { + const replacements = buildSecretReplacements(configuredValue, environmentVariables) + + if (replacements.length === 0) { + return (value: T): T => value + } + + return (value: T): T => sanitizeValue(value, replacements, new WeakMap()) as T +} diff --git a/apps/sim/executor/variables/resolver.test.ts b/apps/sim/executor/variables/resolver.test.ts index e7672297f4f..92771e5a156 100644 --- a/apps/sim/executor/variables/resolver.test.ts +++ b/apps/sim/executor/variables/resolver.test.ts @@ -137,6 +137,47 @@ describe('VariableResolver function block inputs', () => { expect(result.contextVariables).toEqual({ __blockRef_0: 'hello world' }) }) + it('resolves environment variables only in runtime function code', async () => { + const { block, ctx, resolver } = createResolver('javascript') + ctx.environmentVariables = { API_SECRET: 'resolved-secret' } + + const result = await resolver.resolveInputsForFunctionBlock( + ctx, + 'function', + { code: 'return "{{API_SECRET}}"' }, + block + ) + + expect(result.resolvedInputs.code).toBe('return "resolved-secret"') + expect(result.displayInputs.code).toBe('return "{{API_SECRET}}"') + }) + + it('preserves environment references in multi-part function display code', async () => { + const { block, ctx, resolver } = createResolver('javascript') + ctx.environmentVariables = { API_SECRET: 'resolved-secret' } + + const result = await resolver.resolveInputsForFunctionBlock( + ctx, + 'function', + { + code: [ + { language: 'javascript', content: 'const key = "{{API_SECRET}}"' }, + { language: 'javascript', content: 'return "{{API_SECRET}}"' }, + ], + }, + block + ) + + expect(result.resolvedInputs.code).toEqual([ + { language: 'javascript', content: 'const key = "resolved-secret"' }, + { language: 'javascript', content: 'return "resolved-secret"' }, + ]) + expect(result.displayInputs.code).toEqual([ + { language: 'javascript', content: 'const key = "{{API_SECRET}}"' }, + { language: 'javascript', content: 'return "{{API_SECRET}}"' }, + ]) + }) + it('allows Variables block assignments to receive whole large refs', async () => { const producer = createBlock('producer', 'Producer', BlockType.API) const variablesBlock = createBlock('variables', 'Variables', BlockType.VARIABLES, { diff --git a/apps/sim/executor/variables/resolver.ts b/apps/sim/executor/variables/resolver.ts index 851358cd2c2..c4029c39384 100644 --- a/apps/sim/executor/variables/resolver.ts +++ b/apps/sim/executor/variables/resolver.ts @@ -367,9 +367,9 @@ export class VariableResolver { /** * Resolves a code template for a function block. Block output references are stored * in `contextVarAccumulator` as named variables (e.g. `__blockRef_0`) and replaced - * with those variable names in the returned code string. Non-block references (loop - * items, workflow variables, env vars) are still inlined as literals so they remain - * available without any extra passing mechanism. + * with those variable names in the returned code string. Environment variables are + * inlined only in runtime code; display code keeps their unresolved references so + * logs and function errors do not expose their values. */ private async resolveCodeWithContextVars( ctx: ExecutionContext, @@ -598,10 +598,6 @@ export class VariableResolver { const resolved = await this.resolveReference(match, resolutionContext) return typeof resolved === 'string' ? resolved : match }) - displayResult = await replaceEnvVarsAsync(displayResult, async (match) => { - const resolved = await this.resolveReference(match, resolutionContext) - return typeof resolved === 'string' ? resolved : match - }) return { resolvedCode: result, displayCode: displayResult } } diff --git a/apps/sim/lib/workflows/executor/human-in-the-loop-manager.test.ts b/apps/sim/lib/workflows/executor/human-in-the-loop-manager.test.ts index 43403742eb0..6e03a9bcec0 100644 --- a/apps/sim/lib/workflows/executor/human-in-the-loop-manager.test.ts +++ b/apps/sim/lib/workflows/executor/human-in-the-loop-manager.test.ts @@ -21,6 +21,7 @@ vi.mock('@/lib/execution/payloads/large-value-metadata', () => ({ })) import { + buildResumedOutput, PauseResumeManager, updateResumeOutputInAggregationBuffers, } from '@/lib/workflows/executor/human-in-the-loop-manager' @@ -92,6 +93,43 @@ describe('automatic resume waiting metadata compatibility', () => { }) }) +describe('buildResumedOutput', () => { + it('keeps raw state and sanitized log sources separate during resume', () => { + const secret = 'resolved-pause-secret' + const resumeParams = { + submissionPayload: { approved: true }, + resumeInput: { submission: { approved: true } }, + submittedAt: '2026-07-27T12:00:00.000Z', + pauseKind: 'human' as const, + parentExecutionId: 'execution-1', + pauseDurationMs: 1000, + } + + const runtimeOutput = buildResumedOutput({ + ...resumeParams, + existingOutput: { + prompt: secret, + response: { data: { prompt: secret } }, + }, + }) + const logOutput = buildResumedOutput({ + ...resumeParams, + existingOutput: { + prompt: '{{TRACE_SECRET}}', + response: { data: { prompt: '{{TRACE_SECRET}}' } }, + }, + }) + + expect(JSON.stringify(runtimeOutput)).toContain(secret) + expect(JSON.stringify(logOutput)).not.toContain(secret) + expect(JSON.stringify(logOutput)).toContain('{{TRACE_SECRET}}') + expect(logOutput).toMatchObject({ + submission: { approved: true }, + response: { data: { submission: { approved: true } } }, + }) + }) +}) + describe('updateResumeOutputInAggregationBuffers', () => { it('replaces a paused parallel branch placeholder with the resumed HITL output', () => { const pausedOutput = { diff --git a/apps/sim/lib/workflows/executor/human-in-the-loop-manager.ts b/apps/sim/lib/workflows/executor/human-in-the-loop-manager.ts index 69adefd2174..2e66d3b3abf 100644 --- a/apps/sim/lib/workflows/executor/human-in-the-loop-manager.ts +++ b/apps/sim/lib/workflows/executor/human-in-the-loop-manager.ts @@ -9,6 +9,7 @@ import type { Edge } from 'reactflow' import { releaseExecutionSlot } from '@/lib/billing/calculations/usage-reservation' import { assertBillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' import { createTimeoutAbortController, getTimeoutErrorMessage } from '@/lib/core/execution-limits' +import { redactApiKeys } from '@/lib/core/security/redaction' import { createExecutionEventWriter, flushExecutionStreamReplayBuffer, @@ -92,6 +93,74 @@ function isPausedOutputForContext(output: unknown, contextId: string): boolean { return isRecordLike(metadata) && metadata.contextId === contextId } +/** + * Rebuilds a resumed pause output from the caller's chosen source. + * Runtime state passes raw output, while trace logs pass their sanitized copy. + */ +export function buildResumedOutput(params: { + existingOutput: Record + submissionPayload: Record + resumeInput: Record + submittedAt: string + pauseKind: PauseKind + parentExecutionId: string + pauseDurationMs: number +}): Record { + const { + existingOutput, + submissionPayload, + resumeInput, + submittedAt, + pauseKind, + parentExecutionId, + pauseDurationMs, + } = params + const existingResponse = isRecordLike(existingOutput.response) ? existingOutput.response : {} + const existingResponseData = isRecordLike(existingResponse.data) ? existingResponse.data : {} + const response = { + ...existingResponse, + data: { + ...existingResponseData, + submission: submissionPayload, + submittedAt, + }, + status: existingResponse.status ?? 200, + headers: existingResponse.headers ?? { 'Content-Type': 'application/json' }, + resume: existingResponse.resume ?? existingOutput.resume, + } + const output: Record = { + ...existingOutput, + response, + submission: submissionPayload, + resumeInput, + submittedAt, + _resumed: true, + _resumedFrom: parentExecutionId, + _pauseDurationMs: pauseDurationMs, + } + + if (pauseKind === 'time') { + output.status = 'completed' + } + + output.resume = output.resume ?? response.resume + const resumeLinks = output.resume ?? response.resume + if (isRecordLike(resumeLinks)) { + if (resumeLinks.uiUrl) { + output.url = resumeLinks.uiUrl + } + if (resumeLinks.apiUrl) { + output.resumeEndpoint = resumeLinks.apiUrl + } + } + + for (const [key, value] of Object.entries(submissionPayload)) { + output[key] = value + } + + return output +} + export function updateResumeOutputInAggregationBuffers( state: SerializableExecutionState, stateBlockKey: string, @@ -969,62 +1038,16 @@ export class PauseResumeManager { ? (normalizedResumeInputRaw.submission as Record) : (normalizedResumeInputRaw as Record) - const existingOutput = pauseBlockState.output || {} - const existingResponse = existingOutput.response || {} - const existingResponseData = - existingResponse && - typeof existingResponse.data === 'object' && - !Array.isArray(existingResponse.data) - ? existingResponse.data - : {} - const submittedAt = new Date().toISOString() - - const mergedResponseData = { - ...existingResponseData, - submission: submissionPayload, - submittedAt, - } - - const mergedResponse = { - ...existingResponse, - data: mergedResponseData, - status: existingResponse.status ?? 200, - headers: existingResponse.headers ?? { 'Content-Type': 'application/json' }, - resume: existingResponse.resume ?? existingOutput.resume, - } - - const mergedOutput: Record = { - ...existingOutput, - response: mergedResponse, - submission: submissionPayload, + const mergedOutput = buildResumedOutput({ + existingOutput: pauseBlockState.output || {}, + submissionPayload, resumeInput: normalizedResumeInputRaw, submittedAt, - _resumed: true, - _resumedFrom: pausedExecution.executionId, - _pauseDurationMs: pauseDurationMs, - } - - if (pausePoint.pauseKind === 'time') { - mergedOutput.status = 'completed' - } - - mergedOutput.resume = mergedOutput.resume ?? mergedResponse.resume - - // Preserve url and resumeEndpoint from resume links - const resumeLinks = mergedOutput.resume ?? mergedResponse.resume - if (resumeLinks && typeof resumeLinks === 'object') { - if (resumeLinks.uiUrl) { - mergedOutput.url = resumeLinks.uiUrl - } - if (resumeLinks.apiUrl) { - mergedOutput.resumeEndpoint = resumeLinks.apiUrl - } - } - - for (const [key, value] of Object.entries(submissionPayload)) { - mergedOutput[key] = value - } + pauseKind: pausePoint.pauseKind ?? 'human', + parentExecutionId: pausedExecution.executionId, + pauseDurationMs, + }) pauseBlockState.output = mergedOutput terminalResumeOutput = mergedOutput @@ -1056,11 +1079,21 @@ export class PauseResumeManager { log.blockId === contextId ) if (blockLogIndex !== -1) { - // Filter output for logging using shared utility - // 'resume' is redundant with url/resumeEndpoint so we filter it out - const filteredOutput = filterOutputForLog('human_in_the_loop', mergedOutput, { - additionalHiddenKeys: ['resume'], + const existingLogOutput = stateCopy.blockLogs[blockLogIndex].output ?? {} + const resumedLogOutput = buildResumedOutput({ + existingOutput: existingLogOutput, + submissionPayload, + resumeInput: normalizedResumeInputRaw, + submittedAt, + pauseKind: pausePoint.pauseKind ?? 'human', + parentExecutionId: pausedExecution.executionId, + pauseDurationMs, }) + const filteredOutput = redactApiKeys( + filterOutputForLog('human_in_the_loop', resumedLogOutput, { + additionalHiddenKeys: ['resume'], + }) + ) stateCopy.blockLogs[blockLogIndex] = { ...stateCopy.blockLogs[blockLogIndex], blockId: stateBlockKey, diff --git a/apps/sim/lib/workflows/streaming/streaming.test.ts b/apps/sim/lib/workflows/streaming/streaming.test.ts index c3c46efff1c..698a25553dd 100644 --- a/apps/sim/lib/workflows/streaming/streaming.test.ts +++ b/apps/sim/lib/workflows/streaming/streaming.test.ts @@ -1309,6 +1309,65 @@ describe('createStreamingResponse agent-events-v1', () => { expect(events).toContainEqual({ event: 'error', error: 'Client cancelled request' }) }) + it('does not replace sanitized trace content with raw streamed text', async () => { + const secret = 'raw-stream-secret' + const safeComplete = vi.fn(async () => {}) + const stream = await createStreamingResponse({ + requestId: 'request-1', + streamConfig: {}, + executeFn: async ({ onStream }) => { + const textStream = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(`Echo ${secret}`)) + controller.close() + }, + }) + + await onStream({ + stream: textStream, + streamFormat: 'text', + execution: { + blockId: 'agent-1', + success: true, + output: { content: `Echo ${secret}` }, + logs: [], + metadata: {}, + }, + } as any) + + return { + success: true, + output: { content: `Echo ${secret}` }, + logs: [ + { + blockId: 'agent-1', + blockName: 'Agent', + blockType: 'agent', + input: { prompt: 'Use {{TRACE_SECRET}}' }, + output: { content: 'Echo {{TRACE_SECRET}}' }, + startedAt: '2026-01-01T00:00:00.000Z', + endedAt: '2026-01-01T00:00:00.001Z', + durationMs: 1, + executionOrder: 1, + success: true, + }, + ], + _streamingMetadata: { + loggingSession: { safeComplete }, + processedInput: {}, + }, + } as any + }, + }) + + await collectSSEEvents(stream) + + expect(safeComplete).toHaveBeenCalledOnce() + const serializedSpans = JSON.stringify(safeComplete.mock.calls[0][0].traceSpans) + expect(serializedSpans).not.toContain(secret) + expect(serializedSpans).toContain('{{TRACE_SECRET}}') + }) + it('thinking never enters streamedChunks / log content rewrite', async () => { const headers = new Headers({ 'x-sim-stream-protocol': 'agent-events-v1', diff --git a/apps/sim/lib/workflows/streaming/streaming.ts b/apps/sim/lib/workflows/streaming/streaming.ts index 56da480803c..5d4b0fe61da 100644 --- a/apps/sim/lib/workflows/streaming/streaming.ts +++ b/apps/sim/lib/workflows/streaming/streaming.ts @@ -447,29 +447,20 @@ async function buildMinimalResult( return minimalResult } -function updateLogsWithStreamedContent( +function updateLogsWithStreamCompletionTimes( logs: BlockLog[], - streamedContent: Map, streamCompletionTimes: Map ): BlockLog[] { return logs.map((log: BlockLog) => { - if (!streamedContent.has(log.blockId)) { + if (!streamCompletionTimes.has(log.blockId)) { return log } - const content = streamedContent.get(log.blockId) const updatedLog = { ...log } - - if (streamCompletionTimes.has(log.blockId)) { - const completionTime = streamCompletionTimes.get(log.blockId)! - const startTime = new Date(log.startedAt).getTime() - updatedLog.endedAt = new Date(completionTime).toISOString() - updatedLog.durationMs = completionTime - startTime - } - - if (log.output && content) { - updatedLog.output = { ...log.output, content } - } + const completionTime = streamCompletionTimes.get(log.blockId)! + const startTime = new Date(log.startedAt).getTime() + updatedLog.endedAt = new Date(completionTime).toISOString() + updatedLog.durationMs = completionTime - startTime return updatedLog }) @@ -822,9 +813,8 @@ export async function createStreamingResponse( state.streamedChunks.size > 0 ? resolveStreamedContent(state) : new Map() if (result.logs && streamedContent.size > 0) { - result.logs = updateLogsWithStreamedContent( + result.logs = updateLogsWithStreamCompletionTimes( result.logs, - streamedContent, state.streamCompletionTimes ) processStreamingBlockLogs(result.logs, streamedContent) From 55814db883fb8e79b2036061a0f896c875f114d5 Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Mon, 27 Jul 2026 23:23:01 -0700 Subject: [PATCH 2/8] sanitize workflow output logs --- .../logs/execution/logging-session.test.ts | 118 ++++++++++++++ .../sim/lib/logs/execution/logging-session.ts | 81 +++++++--- .../workflows/executor/execution-core.test.ts | 145 ++++++++++++++++++ .../lib/workflows/executor/execution-core.ts | 7 + .../lib/workflows/streaming/streaming.test.ts | 26 +++- 5 files changed, 348 insertions(+), 29 deletions(-) diff --git a/apps/sim/lib/logs/execution/logging-session.test.ts b/apps/sim/lib/logs/execution/logging-session.test.ts index 4e9bf1f1646..fa9ee2ec6b0 100644 --- a/apps/sim/lib/logs/execution/logging-session.test.ts +++ b/apps/sim/lib/logs/execution/logging-session.test.ts @@ -73,6 +73,7 @@ vi.mock('@/lib/logs/execution/logging-factory', () => ({ })) import { calculateCostSummary } from '@/lib/logs/execution/logging-factory' +import { createEnvironmentSecretSanitizer } from '@/executor/utils/environment-secret-sanitizer' import { LoggingSession } from './logging-session' afterAll(resetDbChainMock) @@ -268,6 +269,123 @@ describe('LoggingSession completion retries', () => { ) }) + it('sanitizes workflow output and final trace spans without mutating runtime values', async () => { + const session = new LoggingSession('workflow-1', 'execution-safe', 'api', 'req-1') + const secret = 'sk-demo / trace?token=7f3a91' + const rawFinalOutput = { + result: { + resolvedAtRuntime: true, + echoed: `prefix:${secret}:suffix`, + encoded: encodeURIComponent(secret), + ordinary: 'us-east-1', + }, + } + const rawTraceSpans = [ + { + id: 'span-safe', + name: 'Function', + type: 'function', + duration: 1, + startTime: '2026-07-01T00:00:00.000Z', + endTime: '2026-07-01T00:00:00.001Z', + status: 'success', + output: { echoed: secret }, + }, + ] + + session.setEnvironmentSecretSanitizer( + createEnvironmentSecretSanitizer( + { code: 'return "{{OPENAI_API_KEY}}"' }, + { + OPENAI_API_KEY: secret, + UNREFERENCED_REGION: 'us-east-1', + } + ) + ) + completeWorkflowExecutionMock.mockResolvedValue({}) + + await session.safeComplete({ + finalOutput: rawFinalOutput, + traceSpans: rawTraceSpans as any, + }) + + expect(completeWorkflowExecutionMock).toHaveBeenCalledWith( + expect.objectContaining({ + finalOutput: { + result: { + resolvedAtRuntime: true, + echoed: 'prefix:{{OPENAI_API_KEY}}:suffix', + encoded: '{{OPENAI_API_KEY}}', + ordinary: 'us-east-1', + }, + }, + traceSpans: [ + expect.objectContaining({ + output: { echoed: '{{OPENAI_API_KEY}}' }, + }), + ], + }) + ) + expect(rawFinalOutput.result.echoed).toBe(`prefix:${secret}:suffix`) + expect(rawTraceSpans[0].output.echoed).toBe(secret) + expect(calculateCostSummary).toHaveBeenCalledWith(rawTraceSpans) + }) + + it('sanitizes synthetic workflow errors and completion failure metadata', async () => { + const session = new LoggingSession('workflow-1', 'execution-error-safe', 'api', 'req-1') + const secret = 'sk-demo-error-7f3a91' + + session.setEnvironmentSecretSanitizer( + createEnvironmentSecretSanitizer( + { code: 'throw new Error("{{OPENAI_API_KEY}}")' }, + { OPENAI_API_KEY: secret } + ) + ) + completeWorkflowExecutionMock.mockResolvedValue({}) + + await session.safeCompleteWithError({ + error: { message: `Function failed with ${secret}` }, + }) + + expect(completeWorkflowExecutionMock).toHaveBeenCalledWith( + expect.objectContaining({ + finalOutput: { error: 'Function failed with {{OPENAI_API_KEY}}' }, + traceSpans: [ + expect.objectContaining({ + output: { error: 'Function failed with {{OPENAI_API_KEY}}' }, + }), + ], + completionFailure: 'Function failed with {{OPENAI_API_KEY}}', + }) + ) + }) + + it('keeps workflow output sanitized when completion falls back to cost-only persistence', async () => { + const session = new LoggingSession('workflow-1', 'execution-fallback-safe', 'api', 'req-1') + const secret = 'sk-demo-fallback-7f3a91' + + session.setEnvironmentSecretSanitizer( + createEnvironmentSecretSanitizer( + { code: 'return "{{OPENAI_API_KEY}}"' }, + { OPENAI_API_KEY: secret } + ) + ) + completeWorkflowExecutionMock + .mockRejectedValueOnce(new Error('primary persistence failed')) + .mockResolvedValueOnce({}) + + await session.safeComplete({ + finalOutput: { echoed: secret }, + }) + + expect(completeWorkflowExecutionMock).toHaveBeenLastCalledWith( + expect.objectContaining({ + finalOutput: { echoed: '{{OPENAI_API_KEY}}' }, + finalizationPath: 'fallback_completed', + }) + ) + }) + it('derives fallback cost from trace spans when the primary completion fails', async () => { const session = new LoggingSession('workflow-1', 'execution-6', 'api', 'req-1') as any diff --git a/apps/sim/lib/logs/execution/logging-session.ts b/apps/sim/lib/logs/execution/logging-session.ts index d8ba08e5ad5..c6ba028a4c7 100644 --- a/apps/sim/lib/logs/execution/logging-session.ts +++ b/apps/sim/lib/logs/execution/logging-session.ts @@ -30,6 +30,7 @@ import type { WorkflowState, } from '@/lib/logs/types' import type { SerializableExecutionState } from '@/executor/execution/types' +import type { EnvironmentSecretSanitizer } from '@/executor/utils/environment-secret-sanitizer' type TriggerData = Record & { correlation?: NonNullable['correlation'] @@ -86,6 +87,8 @@ const logger = createLogger('LoggingSession') type CompletionAttempt = 'complete' | 'error' | 'cancelled' | 'paused' +const identityEnvironmentSecretSanitizer: EnvironmentSecretSanitizer = (value: T): T => value + export interface SessionStartParams { userId?: string /** Explicit initiating actor for callers that do not populate `userId`. */ @@ -155,6 +158,8 @@ export class LoggingSession { private completionAttemptFailed = false private pendingProgressWrites = new Set>() private postExecutionPromise: Promise | null = null + private environmentSecretSanitizer: EnvironmentSecretSanitizer = + identityEnvironmentSecretSanitizer constructor( workflowId: string, @@ -170,6 +175,19 @@ export class LoggingSession { this.requestId = requestId } + /** + * Installs the workflow-scoped sanitizer used only for persisted and emitted + * observability values. The closure is retained in memory for this session + * and is never added to execution data. + */ + setEnvironmentSecretSanitizer(sanitizer: EnvironmentSecretSanitizer): void { + this.environmentSecretSanitizer = sanitizer + } + + private sanitizeForLog(value: T): T { + return this.environmentSecretSanitizer(value) + } + async onBlockStart( blockId: string, blockName: string, @@ -287,17 +305,24 @@ export class LoggingSession { level?: 'info' | 'error' status?: 'completed' | 'failed' | 'cancelled' | 'pending' }): Promise { + const finalOutput = this.sanitizeForLog(params.finalOutput) + const traceSpans = this.sanitizeForLog(params.traceSpans) + const completionFailure = + params.completionFailure === undefined + ? undefined + : this.sanitizeForLog(params.completionFailure) + await executionLogger.completeWorkflowExecution({ executionId: this.executionId, endedAt: params.endedAt, totalDurationMs: params.totalDurationMs, costSummary: params.costSummary, - finalOutput: params.finalOutput, - traceSpans: params.traceSpans, + finalOutput, + traceSpans, workflowInput: params.workflowInput, executionState: params.executionState, finalizationPath: params.finalizationPath, - completionFailure: params.completionFailure, + completionFailure, isResume: this.isResume, level: params.level, status: params.status, @@ -403,11 +428,13 @@ export class LoggingSession { } this.completing = true - const { endedAt, totalDurationMs, finalOutput, traceSpans, workflowInput, executionState } = - params + const { endedAt, totalDurationMs, workflowInput, executionState } = params + const finalOutput = this.sanitizeForLog(params.finalOutput || {}) + const rawTraceSpans = params.traceSpans || [] + const traceSpans = this.sanitizeForLog(rawTraceSpans) try { - const costSummary = calculateCostSummary(traceSpans || []) + const costSummary = calculateCostSummary(rawTraceSpans) const endTime = endedAt || new Date().toISOString() const duration = totalDurationMs || 0 @@ -415,8 +442,8 @@ export class LoggingSession { endedAt: endTime, totalDurationMs: duration, costSummary, - finalOutput: finalOutput || {}, - traceSpans: traceSpans || [], + finalOutput, + traceSpans, workflowInput, executionState, finalizationPath: 'completed', @@ -424,13 +451,13 @@ export class LoggingSession { this.completed = true - if (traceSpans && traceSpans.length > 0) { + if (traceSpans.length > 0) { try { const { PlatformEvents, createOTelSpansForWorkflowExecution } = await import( '@/lib/core/telemetry' ) - const hasErrors = traceSpans.some((span: any) => { + const hasErrors = rawTraceSpans.some((span: any) => { const checkForErrors = (s: any): boolean => { if (s.status === 'error' && !s.errorHandled) return true if (s.children && Array.isArray(s.children)) { @@ -506,13 +533,15 @@ export class LoggingSession { return } - const { endedAt, totalDurationMs, error, traceSpans, skipCost } = params + const { endedAt, totalDurationMs, error, skipCost } = params + const rawTraceSpans = params.traceSpans || [] + const traceSpans = this.sanitizeForLog(rawTraceSpans) const endTime = endedAt ? new Date(endedAt) : new Date() const durationMs = typeof totalDurationMs === 'number' ? totalDurationMs : 0 const startTime = new Date(endTime.getTime() - Math.max(1, durationMs)) - const hasProvidedSpans = Array.isArray(traceSpans) && traceSpans.length > 0 + const hasProvidedSpans = traceSpans.length > 0 // calculateCostSummary([]) / (undefined) already returns the base-charge // summary, so the no-spans branch needs no separate literal. @@ -528,9 +557,9 @@ export class LoggingSession { models: {}, charges: {}, } - : calculateCostSummary(traceSpans) + : calculateCostSummary(rawTraceSpans) - const message = error?.message || 'Run failed before starting blocks' + const message = this.sanitizeForLog(error?.message || 'Run failed before starting blocks') const errorSpan: TraceSpan = { id: 'workflow-error-root', @@ -615,7 +644,9 @@ export class LoggingSession { this.completing = true try { - const { endedAt, totalDurationMs, traceSpans } = params + const { endedAt, totalDurationMs } = params + const rawTraceSpans = params.traceSpans || [] + const traceSpans = this.sanitizeForLog(rawTraceSpans) const endTime = endedAt ? new Date(endedAt) : new Date() const durationMs = typeof totalDurationMs === 'number' ? totalDurationMs : 0 @@ -639,14 +670,14 @@ export class LoggingSession { // calculateCostSummary handles empty/undefined spans by returning the // base-charge summary, so no separate no-spans literal is needed. - const costSummary = calculateCostSummary(traceSpans) + const costSummary = calculateCostSummary(rawTraceSpans) await this.completeExecutionWithFinalization({ endedAt: endTime.toISOString(), totalDurationMs: Math.max(1, durationMs), costSummary, finalOutput: { cancelled: true }, - traceSpans: traceSpans || [], + traceSpans, finalizationPath: 'cancelled', status: 'cancelled', }) @@ -662,11 +693,11 @@ export class LoggingSession { durationMs: Math.max(1, durationMs), status: 'cancelled', trigger: this.triggerType, - blocksExecuted: traceSpans?.length || 0, + blocksExecuted: traceSpans.length, hasErrors: false, }) - if (traceSpans && traceSpans.length > 0) { + if (traceSpans.length > 0) { const startTime = new Date(endTime.getTime() - Math.max(1, durationMs)) createOTelSpansForWorkflowExecution({ workflowId: this.workflowId, @@ -709,7 +740,9 @@ export class LoggingSession { this.completing = true try { - const { endedAt, totalDurationMs, traceSpans, workflowInput } = params + const { endedAt, totalDurationMs, workflowInput } = params + const rawTraceSpans = params.traceSpans || [] + const traceSpans = this.sanitizeForLog(rawTraceSpans) const endTime = endedAt ? new Date(endedAt) : new Date() const durationMs = typeof totalDurationMs === 'number' ? totalDurationMs : 0 @@ -733,14 +766,14 @@ export class LoggingSession { // calculateCostSummary handles empty/undefined spans by returning the // base-charge summary, so no separate no-spans literal is needed. - const costSummary = calculateCostSummary(traceSpans) + const costSummary = calculateCostSummary(rawTraceSpans) await this.completeExecutionWithFinalization({ endedAt: endTime.toISOString(), totalDurationMs: Math.max(1, durationMs), costSummary, finalOutput: { paused: true }, - traceSpans: traceSpans || [], + traceSpans, workflowInput, finalizationPath: 'paused', status: 'pending', @@ -757,12 +790,12 @@ export class LoggingSession { durationMs: Math.max(1, durationMs), status: 'paused', trigger: this.triggerType, - blocksExecuted: traceSpans?.length || 0, + blocksExecuted: traceSpans.length, hasErrors: false, totalCost: costSummary.totalCost || 0, }) - if (traceSpans && traceSpans.length > 0) { + if (traceSpans.length > 0) { const startTime = new Date(endTime.getTime() - Math.max(1, durationMs)) createOTelSpansForWorkflowExecution({ workflowId: this.workflowId, diff --git a/apps/sim/lib/workflows/executor/execution-core.test.ts b/apps/sim/lib/workflows/executor/execution-core.test.ts index a5f255bcf33..af94cbbfc6e 100644 --- a/apps/sim/lib/workflows/executor/execution-core.test.ts +++ b/apps/sim/lib/workflows/executor/execution-core.test.ts @@ -23,6 +23,7 @@ const { onBlockStartPersistenceMock, executorConstructorMock, findStartBlockMock, + setEnvironmentSecretSanitizerMock, } = vi.hoisted(() => ({ mergeSubblockStateWithValuesMock: vi.fn(), safeStartMock: vi.fn(), @@ -38,6 +39,7 @@ const { onBlockStartPersistenceMock: vi.fn(), executorConstructorMock: vi.fn(), findStartBlockMock: vi.fn(), + setEnvironmentSecretSanitizerMock: vi.fn(), })) const getPersonalAndWorkspaceEnvMock = environmentUtilsMockFns.mockGetPersonalAndWorkspaceEnv @@ -110,6 +112,7 @@ describe('executeWorkflowCore terminal finalization sequencing', () => { hasCompleted: hasCompletedMock, onBlockStart: onBlockStartPersistenceMock, onBlockComplete: vi.fn(), + setEnvironmentSecretSanitizer: setEnvironmentSecretSanitizerMock, setPostExecutionPromise: vi.fn(), waitForPostExecution: vi.fn().mockResolvedValue(undefined), } @@ -500,6 +503,148 @@ describe('executeWorkflowCore terminal finalization sequencing', () => { ]) }) + it('registers a workflow-scoped log sanitizer while preserving raw runtime output', async () => { + const secret = 'sk-demo-core-7f3a91' + const runtimeOutput = { + keyWasResolved: true, + echoedKey: secret, + ordinary: 'us-east-1', + } + + getPersonalAndWorkspaceEnvMock.mockResolvedValue({ + personalEncrypted: {}, + workspaceEncrypted: { OPENAI_API_KEY: 'encrypted' }, + personalDecrypted: {}, + workspaceDecrypted: { + OPENAI_API_KEY: secret, + UNREFERENCED_REGION: 'us-east-1', + }, + }) + serializeWorkflowMock.mockReturnValue({ + blocks: [ + { + id: 'function-1', + config: { + tool: 'function', + params: { code: 'return "{{OPENAI_API_KEY}}"' }, + }, + }, + ], + connections: [], + loops: {}, + parallels: {}, + }) + executorExecuteMock.mockResolvedValue({ + success: true, + status: 'completed', + output: runtimeOutput, + logs: [], + metadata: { duration: 123, startTime: 'start', endTime: 'end' }, + }) + + const result = await executeWorkflowCore({ + snapshot: createSnapshot() as any, + callbacks: {}, + loggingSession: loggingSession as any, + }) + await loggingSession.setPostExecutionPromise.mock.calls[0][0] + + const sanitizer = setEnvironmentSecretSanitizerMock.mock.calls[0]?.[0] + expect(sanitizer).toBeTypeOf('function') + expect(sanitizer(runtimeOutput)).toEqual({ + keyWasResolved: true, + echoedKey: '{{OPENAI_API_KEY}}', + ordinary: 'us-east-1', + }) + expect(result.output).toEqual(runtimeOutput) + expect(result.output.echoedKey).toBe(secret) + expect(safeCompleteMock).toHaveBeenCalledWith( + expect.objectContaining({ finalOutput: runtimeOutput }) + ) + }) + + it('rebuilds the log sanitizer with current environment values for resumed runs', async () => { + const firstSecret = 'sk-demo-before-resume' + const resumedSecret = 'sk-demo-after-resume' + const workflowWithSecret = { + blocks: [ + { + id: 'function-1', + config: { + tool: 'function', + params: { code: 'return "{{OPENAI_API_KEY}}"' }, + }, + }, + ], + connections: [], + loops: {}, + parallels: {}, + } + + serializeWorkflowMock.mockReturnValue(workflowWithSecret) + getPersonalAndWorkspaceEnvMock + .mockResolvedValueOnce({ + personalEncrypted: {}, + workspaceEncrypted: { OPENAI_API_KEY: 'encrypted-before' }, + personalDecrypted: {}, + workspaceDecrypted: { OPENAI_API_KEY: firstSecret }, + }) + .mockResolvedValueOnce({ + personalEncrypted: {}, + workspaceEncrypted: { OPENAI_API_KEY: 'encrypted-after' }, + personalDecrypted: {}, + workspaceDecrypted: { OPENAI_API_KEY: resumedSecret }, + }) + executorExecuteMock + .mockResolvedValueOnce({ + success: true, + status: 'completed', + output: { echoedKey: firstSecret }, + logs: [], + metadata: { duration: 1, startTime: 'start', endTime: 'end' }, + }) + .mockResolvedValueOnce({ + success: true, + status: 'completed', + output: { echoedKey: resumedSecret }, + logs: [], + metadata: { duration: 1, startTime: 'start', endTime: 'end' }, + }) + + await executeWorkflowCore({ + snapshot: createSnapshot() as any, + callbacks: {}, + loggingSession: loggingSession as any, + }) + await loggingSession.setPostExecutionPromise.mock.calls[0][0] + + const resumedSnapshot = createSnapshot() + resumedSnapshot.metadata = { + ...resumedSnapshot.metadata, + executionId: 'execution-resumed', + resumeFromSnapshot: true, + resumeTerminalNoop: true, + } as any + + const resumedResult = await executeWorkflowCore({ + snapshot: resumedSnapshot as any, + callbacks: {}, + loggingSession: loggingSession as any, + skipLogCreation: true, + }) + await loggingSession.setPostExecutionPromise.mock.calls[1][0] + + const initialSanitizer = setEnvironmentSecretSanitizerMock.mock.calls[0]?.[0] + const resumedSanitizer = setEnvironmentSecretSanitizerMock.mock.calls[1]?.[0] + expect(initialSanitizer({ echoedKey: firstSecret })).toEqual({ + echoedKey: '{{OPENAI_API_KEY}}', + }) + expect(resumedSanitizer({ echoedKey: resumedSecret })).toEqual({ + echoedKey: '{{OPENAI_API_KEY}}', + }) + expect(resumedResult.output).toEqual({ echoedKey: resumedSecret }) + }) + it('awaits wrapped lifecycle persistence before terminal finalization returns', async () => { let releaseBlockStart: (() => void) | undefined const blockStartPromise = new Promise((resolve) => { diff --git a/apps/sim/lib/workflows/executor/execution-core.ts b/apps/sim/lib/workflows/executor/execution-core.ts index 5733adef130..aabb80ba2d7 100644 --- a/apps/sim/lib/workflows/executor/execution-core.ts +++ b/apps/sim/lib/workflows/executor/execution-core.ts @@ -44,6 +44,7 @@ import type { NormalizedBlockOutput, StartBlockRunMetadata, } from '@/executor/types' +import { createEnvironmentSecretSanitizer } from '@/executor/utils/environment-secret-sanitizer' import { hasExecutionResult } from '@/executor/utils/errors' import { isRunMetadataEnabled } from '@/executor/utils/start-block' import { buildParallelSentinelEndId, buildSentinelEndId } from '@/executor/utils/subflow-utils' @@ -523,6 +524,12 @@ async function executeWorkflowCoreImpl( parallels, true ) + loggingSession.setEnvironmentSecretSanitizer( + createEnvironmentSecretSanitizer( + serializedWorkflow.blocks.map((block) => block.config), + decryptedEnvVars + ) + ) processedInput = input || {} diff --git a/apps/sim/lib/workflows/streaming/streaming.test.ts b/apps/sim/lib/workflows/streaming/streaming.test.ts index 698a25553dd..002d0882fd4 100644 --- a/apps/sim/lib/workflows/streaming/streaming.test.ts +++ b/apps/sim/lib/workflows/streaming/streaming.test.ts @@ -8,6 +8,7 @@ import { agentStreamProtocolResponseHeaders, createStreamingResponse, } from '@/lib/workflows/streaming/streaming' +import { createEnvironmentSecretSanitizer } from '@/executor/utils/environment-secret-sanitizer' const { mockDownloadFile } = vi.hoisted(() => ({ mockDownloadFile: vi.fn(), @@ -1311,7 +1312,18 @@ describe('createStreamingResponse agent-events-v1', () => { it('does not replace sanitized trace content with raw streamed text', async () => { const secret = 'raw-stream-secret' - const safeComplete = vi.fn(async () => {}) + const sanitizer = createEnvironmentSecretSanitizer( + { prompt: 'Use {{TRACE_SECRET}}' }, + { TRACE_SECRET: secret } + ) + const persistedCompletion = vi.fn() + const safeComplete = vi.fn(async (params: Record) => { + persistedCompletion({ + ...params, + finalOutput: sanitizer(params.finalOutput), + traceSpans: sanitizer(params.traceSpans), + }) + }) const stream = await createStreamingResponse({ requestId: 'request-1', streamConfig: {}, @@ -1360,12 +1372,16 @@ describe('createStreamingResponse agent-events-v1', () => { }, }) - await collectSSEEvents(stream) + const events = await collectSSEEvents(stream) expect(safeComplete).toHaveBeenCalledOnce() - const serializedSpans = JSON.stringify(safeComplete.mock.calls[0][0].traceSpans) - expect(serializedSpans).not.toContain(secret) - expect(serializedSpans).toContain('{{TRACE_SECRET}}') + expect(persistedCompletion).toHaveBeenCalledOnce() + const serializedCompletion = JSON.stringify(persistedCompletion.mock.calls[0][0]) + expect(serializedCompletion).not.toContain(secret) + expect(serializedCompletion).toContain('{{TRACE_SECRET}}') + + const finalEvent = events.find((event) => event.event === 'final') + expect(JSON.stringify(finalEvent)).toContain(secret) }) it('thinking never enters streamedChunks / log content rewrite', async () => { From e73afbefcedd222b867aae60cc1303e4aad0a16e Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Mon, 27 Jul 2026 23:33:51 -0700 Subject: [PATCH 3/8] preserve streaming usage estimates --- .../executor/execution/block-executor.test.ts | 46 +++++++++++++++++++ apps/sim/executor/execution/block-executor.ts | 14 ++++++ apps/sim/lib/tokenization/streaming.test.ts | 25 ++++++++++ apps/sim/lib/tokenization/streaming.ts | 16 ++++++- 4 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 apps/sim/lib/tokenization/streaming.test.ts diff --git a/apps/sim/executor/execution/block-executor.test.ts b/apps/sim/executor/execution/block-executor.test.ts index bc7cc437c66..b6fcb7a2f97 100644 --- a/apps/sim/executor/execution/block-executor.test.ts +++ b/apps/sim/executor/execution/block-executor.test.ts @@ -5,6 +5,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { clearLargeValueCacheForTests } from '@/lib/execution/payloads/cache' import { isLargeArrayManifest } from '@/lib/execution/payloads/large-array-manifest-metadata' import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans' +import { calculateStreamingCost } from '@/lib/tokenization' import { BlockType } from '@/executor/constants' import type { DAGNode } from '@/executor/dag/builder' import { BlockExecutor } from '@/executor/execution/block-executor' @@ -954,6 +955,51 @@ describe('BlockExecutor streaming pump', () => { expect(state.getBlockOutput(block.id)?.content).toBe('offline answer') }) + it('estimates missing streaming usage from resolved input before sanitizing logs', async () => { + const secret = `sk-${'resolved-secret-'.repeat(20)}` + const params = { prompt: '{{OPENAI_API_KEY}}', model: 'gpt-4o' } + const handler: BlockHandler = { + canHandle: () => true, + execute: async (_ctx, _block, resolvedInputs) => ({ + stream: new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('streamed answer')) + controller.close() + }, + }), + execution: { + success: true, + output: { content: '', model: 'gpt-4o' }, + logs: [], + metadata: { + startTime: new Date().toISOString(), + endTime: new Date().toISOString(), + duration: 1, + }, + }, + }), + } + const { executor, block, state } = createExecutor(handler, params) + const ctx = createContext(state) + ctx.environmentVariables = { OPENAI_API_KEY: secret } + + await executor.execute(ctx, createNode(block), block) + + const expected = calculateStreamingCost( + 'gpt-4o', + JSON.stringify({ prompt: secret, model: 'gpt-4o' }), + 'streamed answer' + ) + expect(state.getBlockOutput(block.id)?.tokens).toEqual(expected.tokens) + expect(state.getBlockOutput(block.id)?.cost).toEqual(expected.cost) + expect(ctx.blockLogs[0].input).toEqual({ + prompt: '{{OPENAI_API_KEY}}', + model: 'gpt-4o', + }) + expect(ctx.blockLogs[0].output?.tokens).toEqual(expected.tokens) + expect(JSON.stringify(ctx.blockLogs[0])).not.toContain(secret) + }) + it('throws on mid-stream provider error (no truncated success)', async () => { const handler = createAgentEventsStreamingHandler({ failAfterText: 'partial', diff --git a/apps/sim/executor/execution/block-executor.ts b/apps/sim/executor/execution/block-executor.ts index 84ad85291f9..a80692a06e9 100644 --- a/apps/sim/executor/execution/block-executor.ts +++ b/apps/sim/executor/execution/block-executor.ts @@ -7,6 +7,7 @@ import { getBaseUrl } from '@/lib/core/utils/urls' import { compactExecutionPayload } from '@/lib/execution/payloads/serializer' import { redactLargeValueRefsInValue } from '@/lib/logs/execution/pii-large-values' import { redactObjectStrings } from '@/lib/logs/execution/pii-redaction' +import { processStreamingBlockLog } from '@/lib/tokenization' import { containsUserFileWithMetadata, hydrateUserFilesWithBase64, @@ -1037,6 +1038,19 @@ export class BlockExecutor { if (!parsedForFormat) { executionOutput.content = fullContent } + + // Fallback usage estimation must happen while the resolved input is + // still available. The log copy is sanitized later, and estimating from + // `{{ENV_VAR}}` placeholders would skew token counts and cost. + processStreamingBlockLog( + { + blockId, + blockType: block.metadata?.id, + input: resolvedInputs, + output: streamingExec.execution.output, + }, + fullContent + ) } if (streamingExec.onFullContent) { diff --git a/apps/sim/lib/tokenization/streaming.test.ts b/apps/sim/lib/tokenization/streaming.test.ts new file mode 100644 index 00000000000..bceb9e1923d --- /dev/null +++ b/apps/sim/lib/tokenization/streaming.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from 'vitest' +import { processStreamingBlockLog } from '@/lib/tokenization/streaming' + +describe('processStreamingBlockLog', () => { + it('does not estimate usage from sanitized environment references', () => { + const log = { + blockId: 'agent-1', + blockType: 'agent', + input: { + prompt: 'Use {{OPENAI_API_KEY}} to answer', + model: 'gpt-4o', + }, + output: { + content: 'streamed answer', + model: 'gpt-4o', + }, + } + + expect(processStreamingBlockLog(log, 'streamed answer')).toBe(false) + expect(log.output).toEqual({ + content: 'streamed answer', + model: 'gpt-4o', + }) + }) +}) diff --git a/apps/sim/lib/tokenization/streaming.ts b/apps/sim/lib/tokenization/streaming.ts index ca552fa8292..03448f0120b 100644 --- a/apps/sim/lib/tokenization/streaming.ts +++ b/apps/sim/lib/tokenization/streaming.ts @@ -16,11 +16,17 @@ import { import type { BlockLog } from '@/executor/types' const logger = createLogger('StreamingTokenization') +const ENVIRONMENT_REFERENCE_PATTERN = /\{\{[^}]+\}\}/ + +type StreamingTokenizationLog = Pick /** * Processes a block log and adds tokenization data if needed */ -export function processStreamingBlockLog(log: BlockLog, streamedContent: string): boolean { +export function processStreamingBlockLog( + log: StreamingTokenizationLog, + streamedContent: string +): boolean { // Check if this block should be tokenized if (!isTokenizableBlockType(log.blockType)) { return false @@ -47,6 +53,12 @@ export function processStreamingBlockLog(log: BlockLog, streamedContent: string) // Prepare input text from log const inputText = extractTextContent(log.input) + // Environment values are restored to references before logs leave the + // executor. Never use those shorter placeholders as a billing estimate: + // the executor performs this fallback once with the raw resolved input. + if (ENVIRONMENT_REFERENCE_PATTERN.test(inputText)) { + return false + } // Calculate streaming cost const systemPrompt = @@ -101,7 +113,7 @@ export function processStreamingBlockLog(log: BlockLog, streamedContent: string) /** * Determines the appropriate model for a block */ -function getModelForBlock(log: BlockLog): string { +function getModelForBlock(log: StreamingTokenizationLog): string { // Try to get model from output first if (log.output?.model?.trim()) { return log.output.model From 9754d5f105cfcc6d5122108b30910b688e67737b Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Fri, 31 Jul 2026 13:20:58 -0700 Subject: [PATCH 4/8] Fix logging session test after staging merge --- apps/sim/lib/logs/execution/logging-session.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/sim/lib/logs/execution/logging-session.test.ts b/apps/sim/lib/logs/execution/logging-session.test.ts index 47a007c9f45..9881c628304 100644 --- a/apps/sim/lib/logs/execution/logging-session.test.ts +++ b/apps/sim/lib/logs/execution/logging-session.test.ts @@ -328,7 +328,7 @@ describe('LoggingSession completion retries', () => { ) expect(rawFinalOutput.result.echoed).toBe(`prefix:${secret}:suffix`) expect(rawTraceSpans[0].output.echoed).toBe(secret) - expect(calculateCostSummary).toHaveBeenCalledWith(rawTraceSpans) + expect(calculateCostSummary).toHaveBeenCalledWith(rawTraceSpans, undefined) }) it('sanitizes synthetic workflow errors and completion failure metadata', async () => { From 0acc9054d75ef93bd60b26d33a02e18a7f4785b5 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Fri, 31 Jul 2026 20:08:35 -0700 Subject: [PATCH 5/8] secrets sanitization correctness --- .../app/api/function/execute/route.test.ts | 109 +- apps/sim/app/api/function/execute/route.ts | 160 +- apps/sim/app/api/logs/export/route.ts | 16 +- .../app/api/mcp/tools/execute/route.test.ts | 185 +++ apps/sim/app/api/mcp/tools/execute/route.ts | 472 +++--- .../app/api/mothership/execute/route.test.ts | 380 ++++- apps/sim/app/api/mothership/execute/route.ts | 202 ++- apps/sim/app/api/v1/logs/[id]/route.ts | 5 +- apps/sim/app/api/v1/logs/route.ts | 5 +- .../[id]/execute/route.async.test.ts | 195 +++ .../app/api/workflows/[id]/execute/route.ts | 201 ++- .../[id]/executions/[executionId]/route.ts | 51 +- .../app/api/workflows/[id]/log/route.test.ts | 58 + apps/sim/app/api/workflows/[id]/log/route.ts | 25 + .../app/workspace/[workspaceId]/logs/logs.tsx | 15 +- .../app/workspace/[workspaceId]/logs/utils.ts | 35 - .../hooks/use-workflow-execution.test.tsx | 152 +- .../hooks/use-workflow-execution.ts | 89 +- ...rkflow-execution-utils.integration.test.ts | 43 + .../utils/workflow-execution-utils.test.ts | 275 +++- .../utils/workflow-execution-utils.ts | 132 +- .../async-preprocessing-correlation.test.ts | 107 +- apps/sim/background/resume-execution.test.ts | 94 ++ .../sim/background/schedule-execution.test.ts | 76 +- apps/sim/background/schedule-execution.ts | 135 +- apps/sim/background/webhook-execution.test.ts | 138 +- apps/sim/background/webhook-execution.ts | 41 +- apps/sim/background/workflow-execution.ts | 1 + .../executor/execution/block-executor.test.ts | 510 ++---- apps/sim/executor/execution/block-executor.ts | 100 +- apps/sim/executor/execution/engine.ts | 1 + apps/sim/executor/execution/executor.ts | 1 + .../execution/snapshot-serializer.test.ts | 40 + .../executor/execution/snapshot-serializer.ts | 13 + apps/sim/executor/execution/types.ts | 39 +- .../executor/handlers/agent/agent-handler.ts | 91 +- .../mothership/mothership-handler.test.ts | 160 +- .../handlers/mothership/mothership-handler.ts | 77 +- .../workflow/custom-block-tool-runner.test.ts | 14 + .../workflow/custom-block-tool-runner.ts | 13 +- .../workflow/workflow-handler.test.ts | 68 +- .../handlers/workflow/workflow-handler.ts | 42 +- apps/sim/executor/types.ts | 2 + .../environment-secret-sanitizer.test.ts | 144 -- .../utils/environment-secret-sanitizer.ts | 223 --- .../utils/reference-validation.test.ts | 44 + .../executor/utils/reference-validation.ts | 15 +- .../resolved-secret-trace-registry.test.ts | 551 +++++++ .../utils/resolved-secret-trace-registry.ts | 620 ++++++++ apps/sim/executor/variables/resolver.test.ts | 63 +- apps/sim/executor/variables/resolver.ts | 10 +- .../executor/variables/resolvers/env.test.ts | 18 + apps/sim/executor/variables/resolvers/env.ts | 3 + apps/sim/hooks/queries/logs.ts | 14 +- apps/sim/hooks/use-execution-stream.ts | 10 +- apps/sim/lib/api/contracts/workflows.test.ts | 6 + apps/sim/lib/api/contracts/workflows.ts | 2 + apps/sim/lib/copilot/mcp-tools.test.ts | 127 +- apps/sim/lib/copilot/mcp-tools.ts | 47 +- .../lib/copilot/request/lifecycle/run.test.ts | 39 + apps/sim/lib/copilot/request/lifecycle/run.ts | 11 + .../copilot/tool-executor/executor.test.ts | 30 + .../sim/lib/copilot/tool-executor/executor.ts | 6 +- apps/sim/lib/copilot/tool-executor/types.ts | 2 + .../tools/handlers/function-execute.test.ts | 26 + .../tools/handlers/function-execute.ts | 6 +- .../tools/handlers/workflow/mutations.test.ts | 185 +++ .../tools/handlers/workflow/mutations.ts | 28 +- .../tools/server/jobs/get-job-logs.test.ts | 131 ++ .../copilot/tools/server/jobs/get-job-logs.ts | 64 +- apps/sim/lib/core/telemetry.ts | 55 +- apps/sim/lib/data-drains/sources/job-logs.ts | 12 + .../lib/data-drains/sources/workflow-logs.ts | 4 +- apps/sim/lib/execution/payloads/serializer.ts | 22 +- .../execution/private-tool-metadata.test.ts | 62 + .../lib/execution/private-tool-metadata.ts | 44 + apps/sim/lib/logs/execution/display-types.ts | 6 + .../logs/execution/functional-outputs.test.ts | 66 + .../lib/logs/execution/functional-outputs.ts | 62 + apps/sim/lib/logs/execution/logger.test.ts | 121 +- apps/sim/lib/logs/execution/logger.ts | 225 ++- .../logs/execution/logging-session.test.ts | 338 +++- .../sim/lib/logs/execution/logging-session.ts | 274 +++- .../execution/trace-secret-projection.test.ts | 797 ++++++++++ .../logs/execution/trace-secret-projection.ts | 1391 +++++++++++++++++ .../lib/logs/execution/trace-store.test.ts | 140 ++ apps/sim/lib/logs/execution/trace-store.ts | 120 +- apps/sim/lib/logs/fetch-log-detail.ts | 21 +- apps/sim/lib/logs/types.ts | 9 + apps/sim/lib/mcp/client.ts | 8 + apps/sim/lib/mcp/resolve-config.test.ts | 158 ++ apps/sim/lib/mcp/resolve-config.ts | 55 +- apps/sim/lib/mcp/service-pool.test.ts | 218 ++- apps/sim/lib/mcp/service.ts | 162 +- apps/sim/lib/mcp/types.ts | 3 + apps/sim/lib/table/backfill-runner.ts | 42 +- apps/sim/lib/tokenization/streaming.test.ts | 25 - apps/sim/lib/tokenization/streaming.ts | 16 +- apps/sim/lib/webhooks/env-resolver.test.ts | 34 + apps/sim/lib/webhooks/env-resolver.ts | 31 +- .../executor/execute-workflow.test.ts | 19 + .../workflows/executor/execute-workflow.ts | 5 + .../workflows/executor/execution-core.test.ts | 390 ++++- .../lib/workflows/executor/execution-core.ts | 99 +- .../workflows/executor/execution-events.ts | 42 +- .../human-in-the-loop-manager.test.ts | 38 - .../executor/human-in-the-loop-manager.ts | 212 +-- .../forward-agent-stream-events.test.ts | 69 +- .../streaming/forward-agent-stream-events.ts | 43 +- .../lib/workflows/streaming/streaming.test.ts | 75 - apps/sim/lib/workflows/streaming/streaming.ts | 24 +- apps/sim/providers/anthropic/core.ts | 4 +- .../anthropic/streaming-tool-loop.ts | 4 +- apps/sim/providers/azure-openai/index.ts | 4 +- apps/sim/providers/baseten/index.ts | 4 +- apps/sim/providers/bedrock/index.ts | 4 +- .../providers/bedrock/streaming-tool-loop.ts | 4 +- apps/sim/providers/cerebras/index.ts | 4 +- apps/sim/providers/deepseek/index.ts | 4 +- apps/sim/providers/fireworks/index.ts | 4 +- apps/sim/providers/gemini/core.ts | 4 +- .../providers/gemini/streaming-tool-loop.ts | 4 +- apps/sim/providers/groq/index.ts | 4 +- apps/sim/providers/index.ts | 11 +- apps/sim/providers/kimi/index.ts | 4 +- apps/sim/providers/litellm/index.ts | 4 +- apps/sim/providers/meta/index.ts | 4 +- apps/sim/providers/mistral/index.ts | 4 +- apps/sim/providers/nvidia/index.ts | 4 +- apps/sim/providers/ollama/core.ts | 4 +- .../openai-compat/streaming-tool-loop.ts | 4 +- apps/sim/providers/openai/core.ts | 4 +- .../providers/openai/streaming-tool-loop.ts | 4 +- apps/sim/providers/openrouter/index.ts | 4 +- apps/sim/providers/runtime-context.test.ts | 76 + apps/sim/providers/runtime-context.ts | 30 + apps/sim/providers/sakana/index.ts | 4 +- apps/sim/providers/together/index.ts | 4 +- apps/sim/providers/vllm/index.ts | 4 +- apps/sim/providers/xai/index.ts | 4 +- apps/sim/providers/zai/index.ts | 4 +- .../stores/terminal/console/storage.test.ts | 87 ++ apps/sim/stores/terminal/console/storage.ts | 93 +- .../sim/stores/terminal/console/store.test.ts | 25 + apps/sim/stores/terminal/console/store.ts | 11 +- apps/sim/stores/terminal/console/types.ts | 1 + apps/sim/tools/index.test.ts | 395 +++++ apps/sim/tools/index.ts | 304 +++- 148 files changed, 11461 insertions(+), 2210 deletions(-) create mode 100644 apps/sim/app/api/mcp/tools/execute/route.test.ts create mode 100644 apps/sim/background/resume-execution.test.ts delete mode 100644 apps/sim/executor/utils/environment-secret-sanitizer.test.ts delete mode 100644 apps/sim/executor/utils/environment-secret-sanitizer.ts create mode 100644 apps/sim/executor/utils/reference-validation.test.ts create mode 100644 apps/sim/executor/utils/resolved-secret-trace-registry.test.ts create mode 100644 apps/sim/executor/utils/resolved-secret-trace-registry.ts create mode 100644 apps/sim/lib/copilot/tools/server/jobs/get-job-logs.test.ts create mode 100644 apps/sim/lib/execution/private-tool-metadata.test.ts create mode 100644 apps/sim/lib/execution/private-tool-metadata.ts create mode 100644 apps/sim/lib/logs/execution/display-types.ts create mode 100644 apps/sim/lib/logs/execution/functional-outputs.test.ts create mode 100644 apps/sim/lib/logs/execution/functional-outputs.ts create mode 100644 apps/sim/lib/logs/execution/trace-secret-projection.test.ts create mode 100644 apps/sim/lib/logs/execution/trace-secret-projection.ts create mode 100644 apps/sim/lib/logs/execution/trace-store.test.ts create mode 100644 apps/sim/lib/mcp/resolve-config.test.ts delete mode 100644 apps/sim/lib/tokenization/streaming.test.ts create mode 100644 apps/sim/providers/runtime-context.test.ts create mode 100644 apps/sim/providers/runtime-context.ts create mode 100644 apps/sim/stores/terminal/console/storage.test.ts diff --git a/apps/sim/app/api/function/execute/route.test.ts b/apps/sim/app/api/function/execute/route.test.ts index b240a68a099..f78dd26b928 100644 --- a/apps/sim/app/api/function/execute/route.test.ts +++ b/apps/sim/app/api/function/execute/route.test.ts @@ -796,16 +796,115 @@ describe('Function Execute API Route', () => { describe('Template Variable Resolution', () => { it.concurrent('should resolve environment variables with {{var_name}} syntax', async () => { - const req = createMockRequest('POST', { - code: 'return {{API_KEY}}', - envVars: { - API_KEY: 'secret-key-123', + const req = createMockRequest( + 'POST', + { + code: 'return {{API_KEY}}', + envVars: { + API_KEY: 'secret-key-123', + }, }, - }) + { + 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', + } + ) const response = await POST(req) + const data = await response.json() expect(response.status).toBe(200) + expect(data.__resolvedSecretNames).toEqual(['API_KEY']) + }) + + it('reports only successful references sourced from scoped environment variables', async () => { + const envResponse = await POST( + createMockRequest( + 'POST', + { + code: 'return {{SHARED}} + {{ENV_ONLY}} + {{MISSING}}', + params: { SHARED: 'param-value', MISSING: 'ordinary-param' }, + envVars: { SHARED: 'secret-value', ENV_ONLY: 'other-secret' }, + }, + { + 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', + } + ) + ) + const envData = await envResponse.json() + + const directResponse = await POST( + createMockRequest( + 'POST', + { + code: 'return environmentVariables.API_KEY + params.API_KEY', + params: { API_KEY: 'ordinary-param' }, + envVars: { API_KEY: 'secret-value' }, + }, + { + 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', + } + ) + ) + const directData = await directResponse.json() + + expect(envData.__resolvedSecretNames).toEqual(['ENV_ONLY', 'SHARED']) + expect(directData.__resolvedSecretNames).toEqual([]) + }) + + it('reports shell {{NAME}} substitutions but not direct shell environment access', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + + const referencedResponse = await POST( + createMockRequest( + 'POST', + { + code: 'printf "%s" "{{API_KEY}}"', + language: 'shell', + envVars: { API_KEY: 'secret-value' }, + }, + { + 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', + } + ) + ) + const referencedData = await referencedResponse.json() + + const directResponse = await POST( + createMockRequest( + 'POST', + { + code: 'printf "%s" "$API_KEY"', + language: 'shell', + envVars: { API_KEY: 'secret-value' }, + }, + { + 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', + } + ) + ) + const directData = await directResponse.json() + + expect(referencedData.__resolvedSecretNames).toEqual(['API_KEY']) + expect(directData.__resolvedSecretNames).toEqual([]) + }) + + it('reports only substitutions allowed by the Function secret scope', async () => { + const response = await POST( + createMockRequest( + 'POST', + { + code: 'return {{ALLOWED}} + {{BLOCKED}}', + envVars: { ALLOWED: 'allowed-secret', BLOCKED: 'blocked-secret' }, + secretScope: 'selected', + mountedSecrets: ['ALLOWED'], + }, + { + 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', + } + ) + ) + + expect((await response.json()).__resolvedSecretNames).toEqual(['ALLOWED']) }) it.concurrent('should resolve tag variables with syntax', async () => { diff --git a/apps/sim/app/api/function/execute/route.ts b/apps/sim/app/api/function/execute/route.ts index 59a295bc924..4f72d777a38 100644 --- a/apps/sim/app/api/function/execute/route.ts +++ b/apps/sim/app/api/function/execute/route.ts @@ -35,6 +35,12 @@ import { } from '@/lib/execution/payloads/materialization.server' import { compactExecutionPayload } from '@/lib/execution/payloads/serializer' import { materializeLargeValueRef } from '@/lib/execution/payloads/store' +import { + PRIVATE_TOOL_METADATA_RESPONSE_HEADER, + RESOLVED_SECRET_NAMES_FIELD, + RESOLVED_SECRET_NAMES_METADATA_V1, + requestsPrivateToolMetadata, +} from '@/lib/execution/private-tool-metadata' import { executeInSandbox, executeShellInSandbox, @@ -65,6 +71,8 @@ const E2B_JS_WRAPPER_LINES = 3 const E2B_PYTHON_WRAPPER_LINES = 1 const MAX_SANDBOX_OUTPUT_FILES = 20 const MAX_SANDBOX_OUTPUT_BYTES = 50 * 1024 * 1024 +const MAX_PRIVATE_RESOLVED_SECRET_NAMES = 10_000 +const MAX_PRIVATE_RESOLVED_SECRET_NAMES_BYTES = 1024 * 1024 /** Matches valid JS identifier names (letters, digits, underscore; no leading digit). */ const SAFE_IDENTIFIER = /^[a-zA-Z_][a-zA-Z0-9_]*$/ @@ -586,7 +594,8 @@ function resolveEnvironmentVariables( code: string, params: Record, envVars: Record, - contextVariables: Record + contextVariables: Record, + onResolvedSecret?: (name: string) => void ): string { let resolvedCode = code @@ -627,6 +636,9 @@ function resolveEnvironmentVariables( const safeVarName = `__var_${varName.replace(/[^a-zA-Z0-9_]/g, '_')}` contextVariables[safeVarName] = varValue + if (Object.hasOwn(envVars, varName) && envVars[varName] === varValue) { + onResolvedSecret?.(varName) + } resolvedCode = resolvedCode.slice(0, index) + safeVarName + resolvedCode.slice(index + matchStr.length) } @@ -704,13 +716,20 @@ function resolveCodeVariables( blockNameMapping: Record = {}, blockOutputSchemas: Record = {}, workflowVariables: Record = {}, - language = 'javascript' + language = 'javascript', + onResolvedSecret?: (name: string) => void ): { resolvedCode: string; contextVariables: Record } { let resolvedCode = code const contextVariables: Record = {} resolvedCode = resolveWorkflowVariables(resolvedCode, workflowVariables, contextVariables) - resolvedCode = resolveEnvironmentVariables(resolvedCode, params, envVars, contextVariables) + resolvedCode = resolveEnvironmentVariables( + resolvedCode, + params, + envVars, + contextVariables, + onResolvedSecret + ) resolvedCode = resolveTagVariables( resolvedCode, blockData, @@ -796,6 +815,8 @@ interface FunctionRouteExecutionContext { allowLargeValueWorkflowScope?: boolean userId?: string requestId: string + resolvedSecretNames: Set + includePrivateResolvedSecretNames: boolean } function asRecord(value: unknown): Record { @@ -931,7 +952,7 @@ async function functionJsonResponse( context: FunctionRouteExecutionContext, init?: ResponseInit ) { - return NextResponse.json( + const response = NextResponse.json( await compactFunctionRouteBody( { ...body, @@ -942,6 +963,52 @@ async function functionJsonResponse( ), init ) + return appendResolvedSecretNames(response, context) +} + +function getPrivateResolvedSecretNames(context: FunctionRouteExecutionContext): string[] | null { + if (context.resolvedSecretNames.size > MAX_PRIVATE_RESOLVED_SECRET_NAMES) return null + + const names = Array.from(context.resolvedSecretNames).sort() + let bytes = 0 + for (const name of names) { + bytes += Buffer.byteLength(name, 'utf8') + if (bytes > MAX_PRIVATE_RESOLVED_SECRET_NAMES_BYTES) return null + } + return names +} + +async function appendResolvedSecretNames( + response: NextResponse, + context: FunctionRouteExecutionContext +): Promise { + const names = context.includePrivateResolvedSecretNames + ? getPrivateResolvedSecretNames(context) + : null + return appendPrivateResolvedSecretNames(response, names) +} + +async function appendPrivateResolvedSecretNames( + response: NextResponse, + names: string[] | null +): Promise { + if (!names) return response + + try { + const body = (await response.clone().json()) as Record + const headers = new Headers(response.headers) + headers.delete('content-length') + headers.set(PRIVATE_TOOL_METADATA_RESPONSE_HEADER, RESOLVED_SECRET_NAMES_METADATA_V1) + return NextResponse.json( + { + ...body, + [RESOLVED_SECRET_NAMES_FIELD]: names, + }, + { status: response.status, statusText: response.statusText, headers } + ) + } catch { + return response + } } /** @@ -1411,6 +1478,7 @@ export const POST = withRouteHandler(async (req: NextRequest) => { let resolvedCode = '' // Store resolved code for error reporting let sourceCodeForErrors: string | undefined let routeContext: FunctionRouteExecutionContext | undefined + let includePrivateResolvedSecretNames = false try { const auth = await checkInternalAuth(req) @@ -1419,8 +1487,18 @@ export const POST = withRouteHandler(async (req: NextRequest) => { return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) } + includePrivateResolvedSecretNames = requestsPrivateToolMetadata( + req.headers, + RESOLVED_SECRET_NAMES_METADATA_V1 + ) + const parsed = await parseRequest(functionExecuteContract, req, {}) - if (!parsed.success) return parsed.response + if (!parsed.success) { + return appendPrivateResolvedSecretNames( + parsed.response, + includePrivateResolvedSecretNames ? [] : null + ) + } const { body } = parsed.data const { DEFAULT_EXECUTION_TIMEOUT_MS } = await import('@/lib/execution/constants') @@ -1473,12 +1551,15 @@ export const POST = withRouteHandler(async (req: NextRequest) => { .map((file) => file.sandboxPath) .filter((path): path is string => Boolean(path)) if (outputSandboxPaths.length > MAX_SANDBOX_OUTPUT_FILES) { - return NextResponse.json( - { - success: false, - error: `Too many sandbox output files requested (${outputSandboxPaths.length}). Maximum is ${MAX_SANDBOX_OUTPUT_FILES}.`, - }, - { status: 400 } + return appendPrivateResolvedSecretNames( + NextResponse.json( + { + success: false, + error: `Too many sandbox output files requested (${outputSandboxPaths.length}). Maximum is ${MAX_SANDBOX_OUTPUT_FILES}.`, + }, + { status: 400 } + ), + includePrivateResolvedSecretNames ? [] : null ) } @@ -1504,6 +1585,8 @@ export const POST = withRouteHandler(async (req: NextRequest) => { allowLargeValueWorkflowScope, userId: auth.userId, requestId, + resolvedSecretNames: new Set(), + includePrivateResolvedSecretNames, } const lang = isValidCodeLanguage(language) ? language : DEFAULT_CODE_LANGUAGE @@ -1512,7 +1595,12 @@ export const POST = withRouteHandler(async (req: NextRequest) => { if (lang === CodeLanguage.Shell) { // For shell, env vars are injected as OS env vars via shellEnvs. // Replace {{VAR}} placeholders with $VAR so the shell can access them natively. - resolvedCode = code.replace(/\{\{([A-Za-z_][A-Za-z0-9_]*)\}\}/g, '$$$1') + resolvedCode = code.replace(/\{\{([A-Za-z_][A-Za-z0-9_]*)\}\}/g, (_match, name) => { + if (Object.hasOwn(envVars, name)) { + routeContext?.resolvedSecretNames.add(name) + } + return `$${name}` + }) // Carry pre-resolved block output variables (e.g. __blockRef_N) so they can be // injected as shell env vars below. The executor replaces block references in the // code with these names, so the values must be present at runtime. @@ -1526,7 +1614,8 @@ export const POST = withRouteHandler(async (req: NextRequest) => { blockNameMapping, blockOutputSchemas, workflowVariables, - lang + lang, + (name) => routeContext?.resolvedSecretNames.add(name) ) resolvedCode = codeResolution.resolvedCode // Merge pre-resolved block output variables from the executor. These take precedence @@ -1625,7 +1714,9 @@ export const POST = withRouteHandler(async (req: NextRequest) => { stdout: shellStdout, executionTime, }) - if (fileExportResponse) return fileExportResponse + if (fileExportResponse) { + return appendResolvedSecretNames(fileExportResponse, routeContext) + } } return functionJsonResponse( @@ -1794,7 +1885,9 @@ export const POST = withRouteHandler(async (req: NextRequest) => { stdout, executionTime, }) - if (fileExportResponse) return fileExportResponse + if (fileExportResponse) { + return appendResolvedSecretNames(fileExportResponse, routeContext) + } } return functionJsonResponse( @@ -1884,7 +1977,9 @@ export const POST = withRouteHandler(async (req: NextRequest) => { stdout, executionTime, }) - if (fileExportResponse) return fileExportResponse + if (fileExportResponse) { + return appendResolvedSecretNames(fileExportResponse, routeContext) + } } return functionJsonResponse( @@ -2043,17 +2138,20 @@ export const POST = withRouteHandler(async (req: NextRequest) => { { status: error.statusCode } ) } - return NextResponse.json( - { - success: false, - error: error.message, - output: { - result: null, - stdout: cleanStdout(stdout), - executionTime, + return appendPrivateResolvedSecretNames( + NextResponse.json( + { + success: false, + error: error.message, + output: { + result: null, + stdout: cleanStdout(stdout), + executionTime, + }, }, - }, - { status: error.statusCode } + { status: error.statusCode } + ), + includePrivateResolvedSecretNames ? [] : null ) } @@ -2072,7 +2170,10 @@ export const POST = withRouteHandler(async (req: NextRequest) => { } return routeContext ? functionJsonResponse(killResponse, routeContext, { status: 500 }) - : NextResponse.json(killResponse, { status: 500 }) + : appendPrivateResolvedSecretNames( + NextResponse.json(killResponse, { status: 500 }), + includePrivateResolvedSecretNames ? [] : null + ) } logger.error(`[${requestId}] Function execution failed`, { @@ -2120,6 +2221,9 @@ export const POST = withRouteHandler(async (req: NextRequest) => { return functionJsonResponse(errorResponse, routeContext, { status: 500 }) } - return NextResponse.json(errorResponse, { status: 500 }) + return appendPrivateResolvedSecretNames( + NextResponse.json(errorResponse, { status: 500 }), + includePrivateResolvedSecretNames ? [] : null + ) } }) diff --git a/apps/sim/app/api/logs/export/route.ts b/apps/sim/app/api/logs/export/route.ts index bca5e283796..0d8ebc032a5 100644 --- a/apps/sim/app/api/logs/export/route.ts +++ b/apps/sim/app/api/logs/export/route.ts @@ -8,7 +8,7 @@ import { getSession } from '@/lib/auth' import { MATERIALIZE_CONCURRENCY, mapWithConcurrency } from '@/lib/core/utils/concurrency' import { neutralizeCsvFormula } from '@/lib/core/utils/csv' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { materializeExecutionData } from '@/lib/logs/execution/trace-store' +import { materializeExecutionDataForDisplay } from '@/lib/logs/execution/trace-store' import { buildFilterConditions, LogFilterParamsSchema } from '@/lib/logs/filters' import { expandFolderIdsWithDescendants } from '@/lib/logs/folder-expansion' import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils' @@ -112,11 +112,15 @@ export const GET = withRouteHandler(async (request: NextRequest) => { rows as any[], MATERIALIZE_CONCURRENCY, (r) => - materializeExecutionData(r.executionData as Record | null, { - workspaceId: params.workspaceId, - workflowId: r.workflowId, - executionId: r.executionId, - }) + materializeExecutionDataForDisplay( + r.executionData as Record | null, + { + workspaceId: params.workspaceId, + workflowId: r.workflowId, + executionId: r.executionId, + userId: session.user.id, + } + ) ) for (let j = 0; j < rows.length; j++) { diff --git a/apps/sim/app/api/mcp/tools/execute/route.test.ts b/apps/sim/app/api/mcp/tools/execute/route.test.ts new file mode 100644 index 00000000000..29d78f43e9e --- /dev/null +++ b/apps/sim/app/api/mcp/tools/execute/route.test.ts @@ -0,0 +1,185 @@ +/** + * @vitest-environment node + */ +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockDiscoverServerTools, mockExecuteTool } = vi.hoisted(() => ({ + mockDiscoverServerTools: vi.fn(), + mockExecuteTool: vi.fn(), +})) + +vi.mock('@/lib/mcp/middleware', () => ({ + withMcpAuth: + () => + ( + handler: ( + request: NextRequest, + context: { + userId: string + workspaceId: string + requestId: string + authType: 'internal_jwt' | 'session' + }, + routeContext: { params: Promise> } + ) => Promise + ) => + (request: NextRequest) => + handler( + request, + { + userId: 'user-1', + workspaceId: 'workspace-1', + requestId: 'request-1', + authType: request.headers.has('x-test-session') ? 'session' : 'internal_jwt', + }, + { params: Promise.resolve({}) } + ), + readMcpJsonBodyWithLimit: (request: NextRequest) => request.json(), + mcpBodyReadErrorResponse: () => null, +})) + +vi.mock('@/lib/mcp/service', () => ({ + mcpService: { + discoverServerTools: mockDiscoverServerTools, + executeTool: mockExecuteTool, + }, +})) + +vi.mock('@/lib/billing/core/billing-attribution', () => ({ + requireBillingAttributionHeader: () => ({ payerSubscription: { plan: 'pro' } }), + resolveBillingAttribution: async () => ({ payerSubscription: { plan: 'pro' } }), +})) + +vi.mock('@/lib/core/execution-limits', () => ({ + DEFAULT_EXECUTION_TIMEOUT_MS: 30_000, + getExecutionTimeout: () => 0, +})) + +vi.mock('@/ee/access-control/utils/permission-check', () => ({ + assertPermissionsAllowed: async () => {}, + McpToolsNotAllowedError: class McpToolsNotAllowedError extends Error {}, +})) + +vi.mock('@/lib/core/telemetry', () => ({ + PlatformEvents: { mcpToolExecuted: vi.fn() }, +})) + +import { POST } from '@/app/api/mcp/tools/execute/route' + +const URL = 'http://localhost/api/mcp/tools/execute' +const REQUEST_BODY = { + workspaceId: 'workspace-1', + serverId: 'server-1', + toolName: 'example_tool', + arguments: {}, +} + +function createRequest(headers: Record = {}): NextRequest { + return new NextRequest(URL, { + method: 'POST', + headers: { 'content-type': 'application/json', ...headers }, + body: JSON.stringify(REQUEST_BODY), + }) +} + +describe('MCP tool execution private secret provenance', () => { + beforeEach(() => { + vi.clearAllMocks() + mockDiscoverServerTools.mockResolvedValue([{ name: 'example_tool', inputSchema: {} }]) + mockExecuteTool.mockResolvedValue({ content: [{ type: 'text', text: 'ok' }] }) + }) + + it('returns scoped encrypted provenance only to an authenticated internal caller', async () => { + mockDiscoverServerTools.mockImplementationOnce( + async ( + _userId: string, + _serverId: string, + _workspaceId: string, + _forceRefresh: boolean, + report: (value: unknown) => void + ) => { + report({ + version: 1, + complete: false, + entries: [{ name: 'OLD_TOKEN', encryptedValue: 'encrypted-v1' }], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + }) + return [{ name: 'example_tool', inputSchema: {} }] + } + ) + mockExecuteTool.mockImplementationOnce( + async ( + _userId: string, + _serverId: string, + _toolCall: unknown, + _workspaceId: string, + _headers: unknown, + report: (value: unknown) => void + ) => { + report({ + version: 1, + complete: true, + entries: [{ name: 'NEW_TOKEN', encryptedValue: 'encrypted-v2' }], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + }) + return { content: [{ type: 'text', text: 'ok' }] } + } + ) + const request = createRequest({ + 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1', + }) + + const response = await POST(request, {}) + const body = (await response.json()) as Record + + expect(response.headers.get('x-sim-private-tool-metadata')).toBe( + 'resolved-secret-provenance-v1' + ) + expect(body.__resolvedSecretTraceProvenance).toEqual({ + version: 1, + complete: false, + entries: [ + { name: 'OLD_TOKEN', encryptedValue: 'encrypted-v1' }, + { name: 'NEW_TOKEN', encryptedValue: 'encrypted-v2' }, + ], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + }) + }) + + it('does not expose private provenance metadata to a session caller', async () => { + const request = createRequest({ + 'x-test-session': 'true', + 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1', + }) + + const response = await POST(request, {}) + const body = (await response.json()) as Record + + expect(response.headers.has('x-sim-private-tool-metadata')).toBe(false) + expect(body).not.toHaveProperty('__resolvedSecretTraceProvenance') + }) + + it('fails closed when attaching provenance would exceed the response budget', async () => { + mockExecuteTool.mockResolvedValueOnce({ + content: [{ type: 'text', text: 'x'.repeat(10 * 1024 * 1024) }], + }) + const request = createRequest({ + 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1', + }) + + const response = await POST(request, {}) + const body = (await response.json()) as Record + + expect(body).toEqual({ + success: false, + error: 'Internal MCP response could not be verified', + __resolvedSecretTraceProvenance: { + version: 1, + complete: false, + entries: [], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + }, + }) + }) +}) diff --git a/apps/sim/app/api/mcp/tools/execute/route.ts b/apps/sim/app/api/mcp/tools/execute/route.ts index a60c32443f6..5986ea7bf30 100644 --- a/apps/sim/app/api/mcp/tools/execute/route.ts +++ b/apps/sim/app/api/mcp/tools/execute/route.ts @@ -11,8 +11,15 @@ import { } from '@/lib/billing/core/billing-attribution' import { getExecutionTimeout } from '@/lib/core/execution-limits' import type { SubscriptionPlan } from '@/lib/core/rate-limiter/types' +import { readResponseToBufferWithLimit } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { SIM_VIA_HEADER } from '@/lib/execution/call-chain' +import { + PRIVATE_TOOL_METADATA_RESPONSE_HEADER, + RESOLVED_SECRET_PROVENANCE_FIELD, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, + requestsPrivateToolMetadata, +} from '@/lib/execution/private-tool-metadata' import { mcpBodyReadErrorResponse, readMcpJsonBodyWithLimit, @@ -31,8 +38,13 @@ import { assertPermissionsAllowed, McpToolsNotAllowedError, } from '@/ee/access-control/utils/permission-check' +import { + ResolvedSecretTraceProvenanceAccumulator, + type ResolvedSecretTraceProvenanceV1, +} from '@/executor/utils/resolved-secret-trace-registry' const logger = createLogger('McpToolExecutionAPI') +const MAX_PRIVATE_MCP_RESPONSE_BYTES = 10 * 1024 * 1024 export const dynamic = 'force-dynamic' @@ -55,6 +67,36 @@ function hasType(prop: unknown): prop is SchemaProperty { return typeof prop === 'object' && prop !== null && 'type' in prop } +async function attachPrivateProvenance( + response: NextResponse, + provenance: ResolvedSecretTraceProvenanceAccumulator +): Promise { + let payload: Record + try { + const body = await readResponseToBufferWithLimit(response, { + maxBytes: MAX_PRIVATE_MCP_RESPONSE_BYTES, + label: 'MCP private metadata response', + allowNoBodyFallback: true, + }) + const parsed: unknown = JSON.parse(body.toString('utf8')) + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error('MCP response is not a JSON object') + } + payload = parsed as Record + } catch { + payload = { success: false, error: 'Internal MCP response could not be verified' } + provenance.markIncomplete({ discardEntries: true }) + } + + const headers = new Headers(response.headers) + headers.delete('content-length') + headers.set(PRIVATE_TOOL_METADATA_RESPONSE_HEADER, RESOLVED_SECRET_PROVENANCE_METADATA_V1) + return NextResponse.json( + { ...payload, [RESOLVED_SECRET_PROVENANCE_FIELD]: provenance.exportProvenance() }, + { status: response.status, headers } + ) +} + /** * POST - Execute a tool on an MCP server */ @@ -62,239 +104,267 @@ export const POST = withRouteHandler( withMcpAuth('read')( async (request: NextRequest, { userId, workspaceId, requestId, authType }) => { let serverId: string | undefined - try { - const rawBody = await readMcpJsonBodyWithLimit(request) - const parsedBody = mcpToolExecutionBodySchema.safeParse(rawBody) + const resolvedSecretTraceProvenance = new ResolvedSecretTraceProvenanceAccumulator({ + userId, + workspaceId, + }) + const recordProvenance = (provenance: ResolvedSecretTraceProvenanceV1): void => { + resolvedSecretTraceProvenance.record(provenance) + } + const includePrivateProvenance = + authType === AuthType.INTERNAL_JWT && + requestsPrivateToolMetadata(request.headers, RESOLVED_SECRET_PROVENANCE_METADATA_V1) + const response = await (async (): Promise => { + try { + const rawBody = await readMcpJsonBodyWithLimit(request) + const parsedBody = mcpToolExecutionBodySchema.safeParse(rawBody) - if (!parsedBody.success) { - return createMcpErrorResponse(parsedBody.error, 'Invalid request format', 400) - } + if (!parsedBody.success) { + return createMcpErrorResponse(parsedBody.error, 'Invalid request format', 400) + } - const body = parsedBody.data + const body = parsedBody.data - logger.info(`[${requestId}] MCP tool execution request received`, { - hasAuthHeader: !!request.headers.get('authorization'), - bodyKeys: Object.keys(body), - serverId: body.serverId, - toolName: body.toolName, - hasWorkflowId: !!body.workflowId, - workflowId: body.workflowId, - userId: userId, - }) + logger.info(`[${requestId}] MCP tool execution request received`, { + hasAuthHeader: !!request.headers.get('authorization'), + bodyKeys: Object.keys(body), + serverId: body.serverId, + toolName: body.toolName, + hasWorkflowId: !!body.workflowId, + workflowId: body.workflowId, + userId: userId, + }) - const { toolName, arguments: rawArgs } = body - serverId = body.serverId - const args = rawArgs || {} + const { toolName, arguments: rawArgs } = body + serverId = body.serverId + const args = rawArgs || {} - try { - await assertPermissionsAllowed({ - userId, - workspaceId, - toolKind: 'mcp', - }) - } catch (err) { - if (err instanceof McpToolsNotAllowedError) { - return createMcpErrorResponse(err, err.message, 403) + try { + await assertPermissionsAllowed({ + userId, + workspaceId, + toolKind: 'mcp', + }) + } catch (err) { + if (err instanceof McpToolsNotAllowedError) { + return createMcpErrorResponse(err, err.message, 403) + } + throw err } - throw err - } - - logger.info( - `[${requestId}] Executing tool ${toolName} on server ${serverId} for user ${userId} in workspace ${workspaceId}` - ) - let tool: McpTool | null = null - try { - const tools = await mcpService.discoverServerTools(userId, serverId, workspaceId) - tool = tools.find((t) => t.name === toolName) ?? null + logger.info( + `[${requestId}] Executing tool ${toolName} on server ${serverId} for user ${userId} in workspace ${workspaceId}` + ) - if (!tool) { - logger.warn(`[${requestId}] Tool ${toolName} not found on server ${serverId}`, { - availableTools: tools.map((t) => t.name), - }) - return createMcpErrorResponse( - new Error('Tool not found'), - 'Tool not found on the specified server', - 404 + let tool: McpTool | null = null + try { + const tools = await mcpService.discoverServerTools( + userId, + serverId, + workspaceId, + false, + recordProvenance ) - } + tool = tools.find((t) => t.name === toolName) ?? null - if (tool.inputSchema?.properties) { - for (const [paramName, paramSchema] of Object.entries(tool.inputSchema.properties)) { - const schema = hasType(paramSchema) ? paramSchema : null - if (!schema) continue - const value = args[paramName] + if (!tool) { + logger.warn(`[${requestId}] Tool ${toolName} not found on server ${serverId}`, { + availableTools: tools.map((t) => t.name), + }) + return createMcpErrorResponse( + new Error('Tool not found'), + 'Tool not found on the specified server', + 404 + ) + } - if (value === undefined || value === null) { - continue - } + if (tool.inputSchema?.properties) { + for (const [paramName, paramSchema] of Object.entries(tool.inputSchema.properties)) { + const schema = hasType(paramSchema) ? paramSchema : null + if (!schema) continue + const value = args[paramName] - if ( - (schema.type === 'number' || schema.type === 'integer') && - typeof value === 'string' - ) { - const numValue = - schema.type === 'integer' ? Number.parseInt(value) : Number.parseFloat(value) - if (!Number.isNaN(numValue)) { - args[paramName] = numValue + if (value === undefined || value === null) { + continue } - } else if (schema.type === 'boolean' && typeof value === 'string') { - if (value.toLowerCase() === 'true') { - args[paramName] = true - } else if (value.toLowerCase() === 'false') { - args[paramName] = false - } - } else if (schema.type === 'array' && typeof value === 'string') { - const stringValue = value.trim() - if (stringValue) { - try { - const parsed = JSON.parse(stringValue) - if (Array.isArray(parsed)) { - args[paramName] = parsed - } else { - args[paramName] = [parsed] - } - } catch { - if (stringValue.includes(',')) { - args[paramName] = stringValue - .split(',') - .map((item) => item.trim()) - .filter((item) => item) - } else { - args[paramName] = [stringValue] + + if ( + (schema.type === 'number' || schema.type === 'integer') && + typeof value === 'string' + ) { + const numValue = + schema.type === 'integer' ? Number.parseInt(value) : Number.parseFloat(value) + if (!Number.isNaN(numValue)) { + args[paramName] = numValue + } + } else if (schema.type === 'boolean' && typeof value === 'string') { + if (value.toLowerCase() === 'true') { + args[paramName] = true + } else if (value.toLowerCase() === 'false') { + args[paramName] = false + } + } else if (schema.type === 'array' && typeof value === 'string') { + const stringValue = value.trim() + if (stringValue) { + try { + const parsed = JSON.parse(stringValue) + if (Array.isArray(parsed)) { + args[paramName] = parsed + } else { + args[paramName] = [parsed] + } + } catch { + if (stringValue.includes(',')) { + args[paramName] = stringValue + .split(',') + .map((item) => item.trim()) + .filter((item) => item) + } else { + args[paramName] = [stringValue] + } } + } else { + args[paramName] = [] } - } else { - args[paramName] = [] } } } - } - } catch (error) { - logger.warn( - `[${requestId}] Failed to discover tools for validation, proceeding without schema`, - error - ) - } - - if (tool) { - const validationError = validateToolArguments(tool, args) - if (validationError) { - logger.warn(`[${requestId}] Tool validation failed: ${validationError}`) - return createMcpErrorResponse( - new Error(`Invalid arguments for tool ${toolName}: ${validationError}`), - 'Invalid tool arguments', - 400 + } catch (error) { + logger.warn( + `[${requestId}] Failed to discover tools for validation, proceeding without schema`, + error ) } - } - const toolCall: McpToolCall = { - name: toolName, - arguments: args, - } + if (tool) { + const validationError = validateToolArguments(tool, args) + if (validationError) { + logger.warn(`[${requestId}] Tool validation failed: ${validationError}`) + return createMcpErrorResponse( + new Error(`Invalid arguments for tool ${toolName}: ${validationError}`), + 'Invalid tool arguments', + 400 + ) + } + } - const billingAttribution = - authType === AuthType.INTERNAL_JWT - ? requireBillingAttributionHeader(request.headers, { - actorUserId: userId, - workspaceId, - }) - : await resolveBillingAttribution({ actorUserId: userId, workspaceId }) - const executionTimeout = getExecutionTimeout( - billingAttribution.payerSubscription?.plan as SubscriptionPlan | undefined, - 'sync' - ) - - const simViaHeader = request.headers.get(SIM_VIA_HEADER) - const extraHeaders: Record = {} - if (simViaHeader) { - extraHeaders[SIM_VIA_HEADER] = simViaHeader - } + const toolCall: McpToolCall = { + name: toolName, + arguments: args, + } - let timeoutHandle: ReturnType | undefined - const executePromise = mcpService.executeTool( - userId, - serverId, - toolCall, - workspaceId, - extraHeaders - ) - // A zero timeout means "no timeout" (billing-disabled deployments). - const result = await (executionTimeout > 0 - ? Promise.race([ - executePromise, - new Promise((_, reject) => { - timeoutHandle = setTimeout( - () => reject(new Error('Tool execution timeout')), - executionTimeout - ) - }), - ]) - : executePromise - ).finally(() => { - if (timeoutHandle !== undefined) clearTimeout(timeoutHandle) - }) - - const transformedResult = transformToolResult(result) - - if (result.isError) { - logger.warn(`[${requestId}] Tool execution returned error for ${toolName} on ${serverId}`) - return createMcpErrorResponse( - transformedResult, - transformedResult.error || 'Tool execution failed', - 400 + const billingAttribution = + authType === AuthType.INTERNAL_JWT + ? requireBillingAttributionHeader(request.headers, { + actorUserId: userId, + workspaceId, + }) + : await resolveBillingAttribution({ actorUserId: userId, workspaceId }) + const executionTimeout = getExecutionTimeout( + billingAttribution.payerSubscription?.plan as SubscriptionPlan | undefined, + 'sync' ) - } - logger.info(`[${requestId}] Successfully executed tool ${toolName} on server ${serverId}`) - try { - const { PlatformEvents } = await import('@/lib/core/telemetry') - PlatformEvents.mcpToolExecuted({ - serverId, - toolName, - status: 'success', - workspaceId, - }) - } catch (error) { - logger.warn('Failed to record MCP tool execution telemetry', { - error: getErrorMessage(error), + const simViaHeader = request.headers.get(SIM_VIA_HEADER) + const extraHeaders: Record = {} + if (simViaHeader) { + extraHeaders[SIM_VIA_HEADER] = simViaHeader + } + + let timeoutHandle: ReturnType | undefined + const executePromise = mcpService.executeTool( + userId, serverId, - toolName, + toolCall, workspaceId, + extraHeaders, + recordProvenance + ) + // A zero timeout means "no timeout" (billing-disabled deployments). + const result = await (executionTimeout > 0 + ? Promise.race([ + executePromise, + new Promise((_, reject) => { + timeoutHandle = setTimeout( + () => reject(new Error('Tool execution timeout')), + executionTimeout + ) + }), + ]) + : executePromise + ).finally(() => { + if (timeoutHandle !== undefined) clearTimeout(timeoutHandle) }) - } - return createMcpSuccessResponse(transformedResult) - } catch (error) { - const bodyErrorResponse = mcpBodyReadErrorResponse(error, request) - if (bodyErrorResponse) return bodyErrorResponse - if ( - error instanceof McpOauthAuthorizationRequiredError || - error instanceof McpOauthRedirectRequired || - error instanceof UnauthorizedError - ) { - const errorServerId = - error instanceof McpOauthAuthorizationRequiredError ? error.serverId : serverId - logger.warn(`[${requestId}] OAuth re-authorization required for MCP tool execution`, { - serverId: errorServerId, - }) - return NextResponse.json( - { - success: false, - error: 'OAuth re-authorization required', - code: 'reauth_required', + const transformedResult = transformToolResult(result) + + if (result.isError) { + logger.warn( + `[${requestId}] Tool execution returned error for ${toolName} on ${serverId}` + ) + return createMcpErrorResponse( + transformedResult, + transformedResult.error || 'Tool execution failed', + 400 + ) + } + logger.info(`[${requestId}] Successfully executed tool ${toolName} on server ${serverId}`) + + try { + const { PlatformEvents } = await import('@/lib/core/telemetry') + PlatformEvents.mcpToolExecuted({ + serverId, + toolName, + status: 'success', + workspaceId, + }) + } catch (error) { + logger.warn('Failed to record MCP tool execution telemetry', { + error: getErrorMessage(error), + serverId, + toolName, + workspaceId, + }) + } + + return createMcpSuccessResponse(transformedResult) + } catch (error) { + if (getErrorMessage(error) === 'Tool execution timeout') { + resolvedSecretTraceProvenance.markIncomplete() + } + const bodyErrorResponse = mcpBodyReadErrorResponse(error, request) + if (bodyErrorResponse) return bodyErrorResponse + if ( + error instanceof McpOauthAuthorizationRequiredError || + error instanceof McpOauthRedirectRequired || + error instanceof UnauthorizedError + ) { + const errorServerId = + error instanceof McpOauthAuthorizationRequiredError ? error.serverId : serverId + logger.warn(`[${requestId}] OAuth re-authorization required for MCP tool execution`, { serverId: errorServerId, - }, - { status: 401 } - ) - } + }) + return NextResponse.json( + { + success: false, + error: 'OAuth re-authorization required', + code: 'reauth_required', + serverId: errorServerId, + }, + { status: 401 } + ) + } - logger.error(`[${requestId}] Error executing MCP tool:`, error) + logger.error(`[${requestId}] Error executing MCP tool:`, error) - const { message, status } = categorizeError(error) - return createMcpErrorResponse(new Error(message), message, status) - } + const { message, status } = categorizeError(error) + return createMcpErrorResponse(new Error(message), message, status) + } + })() + + return includePrivateProvenance + ? attachPrivateProvenance(response, resolvedSecretTraceProvenance) + : response } ) ) diff --git a/apps/sim/app/api/mothership/execute/route.test.ts b/apps/sim/app/api/mothership/execute/route.test.ts index 014a86134b7..4490dca0ed2 100644 --- a/apps/sim/app/api/mothership/execute/route.test.ts +++ b/apps/sim/app/api/mothership/execute/route.test.ts @@ -1,7 +1,98 @@ -import { describe, expect, it } from 'vitest' +/** + * @vitest-environment node + */ +import { createMockRequest } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockAssertActiveWorkspaceAccess, + mockBuildIntegrationToolSchemas, + mockBuildSelectedMcpToolSchemas, + mockBuildTaggedMcpToolSchemas, + mockCheckInternalAuth, + mockComputeWorkspaceEntitlements, + mockDecryptSecret, + mockGenerateWorkspaceContext, + mockGetPersonalAndWorkspaceEnv, + mockProcessContextsServer, + mockRequestExplicitStreamAbort, + mockRequireBillingAttributionHeader, + mockRunHeadlessCopilotLifecycle, +} = vi.hoisted(() => ({ + mockAssertActiveWorkspaceAccess: vi.fn(), + mockBuildIntegrationToolSchemas: vi.fn(), + mockBuildSelectedMcpToolSchemas: vi.fn(), + mockBuildTaggedMcpToolSchemas: vi.fn(), + mockCheckInternalAuth: vi.fn(), + mockComputeWorkspaceEntitlements: vi.fn(), + mockDecryptSecret: vi.fn(), + mockGenerateWorkspaceContext: vi.fn(), + mockGetPersonalAndWorkspaceEnv: vi.fn(), + mockProcessContextsServer: vi.fn(), + mockRequestExplicitStreamAbort: vi.fn(), + mockRequireBillingAttributionHeader: vi.fn(), + mockRunHeadlessCopilotLifecycle: vi.fn(), +})) + +vi.mock('@/lib/core/security/encryption', () => ({ + decryptSecret: mockDecryptSecret, +})) + +vi.mock('@/lib/auth/hybrid', () => ({ + checkInternalAuth: mockCheckInternalAuth, +})) + +vi.mock('@/lib/billing/core/billing-attribution', () => ({ + requireBillingAttributionHeader: mockRequireBillingAttributionHeader, +})) + +vi.mock('@/lib/copilot/chat/payload', () => ({ + buildIntegrationToolSchemas: mockBuildIntegrationToolSchemas, +})) + +vi.mock('@/lib/copilot/chat/process-contents', () => ({ + processContextsServer: mockProcessContextsServer, +})) + +vi.mock('@/lib/copilot/chat/workspace-context', () => ({ + generateWorkspaceContext: mockGenerateWorkspaceContext, +})) + +vi.mock('@/lib/copilot/entitlements', () => ({ + computeWorkspaceEntitlements: mockComputeWorkspaceEntitlements, +})) + +vi.mock('@/lib/copilot/mcp-tools', () => ({ + buildSelectedMcpToolSchemas: mockBuildSelectedMcpToolSchemas, + buildTaggedMcpToolSchemas: mockBuildTaggedMcpToolSchemas, +})) + +vi.mock('@/lib/copilot/request/lifecycle/headless', () => ({ + runHeadlessCopilotLifecycle: mockRunHeadlessCopilotLifecycle, +})) + +vi.mock('@/lib/copilot/request/session/explicit-abort', () => ({ + requestExplicitStreamAbort: mockRequestExplicitStreamAbort, +})) + +vi.mock('@/lib/core/config/env-flags', () => ({ + isDocSandboxEnabled: false, +})) + +vi.mock('@/lib/environment/utils', () => ({ + getPersonalAndWorkspaceEnv: mockGetPersonalAndWorkspaceEnv, +})) + +vi.mock('@/lib/workspaces/permissions/utils', () => ({ + assertActiveWorkspaceAccess: mockAssertActiveWorkspaceAccess, + isWorkspaceAccessDeniedError: vi.fn(() => false), +})) + +import type { CopilotLifecycleOptions } from '@/lib/copilot/request/lifecycle/run' import { buildExecuteResponsePayload, CALLER_VISIBLE_SERVER_TOOLS, + POST, } from '@/app/api/mothership/execute/route' type Payload = Parameters[0] @@ -48,3 +139,290 @@ describe('buildExecuteResponsePayload', () => { expect(CALLER_VISIBLE_SERVER_TOOLS.has('complete_scheduled_task')).toBe(true) }) }) + +describe('mothership private trace provenance transport', () => { + const requestBody = { + messages: [{ role: 'user', content: 'hello' }], + workspaceId: 'workspace-1', + userId: 'user-1', + chatId: 'chat-1', + messageId: 'message-1', + requestId: 'request-1', + } + + beforeEach(() => { + vi.clearAllMocks() + mockCheckInternalAuth.mockResolvedValue({ + success: true, + userId: 'user-1', + authType: 'internal_jwt', + }) + mockAssertActiveWorkspaceAccess.mockResolvedValue({ permission: 'write' }) + mockRequireBillingAttributionHeader.mockReturnValue({ + actorUserId: 'user-1', + workspaceId: 'workspace-1', + }) + mockGetPersonalAndWorkspaceEnv.mockResolvedValue({ + personalEncrypted: { API_KEY: 'encrypted-secret' }, + workspaceEncrypted: {}, + personalDecrypted: { API_KEY: 'secret-value' }, + workspaceDecrypted: {}, + decryptionFailures: [], + }) + mockGenerateWorkspaceContext.mockResolvedValue({}) + mockBuildIntegrationToolSchemas.mockResolvedValue([]) + mockBuildSelectedMcpToolSchemas.mockResolvedValue([]) + mockBuildTaggedMcpToolSchemas.mockResolvedValue([]) + mockComputeWorkspaceEntitlements.mockResolvedValue([]) + mockDecryptSecret.mockResolvedValue({ decrypted: 'secret-value' }) + mockProcessContextsServer.mockResolvedValue([]) + mockRequestExplicitStreamAbort.mockResolvedValue(undefined) + }) + + function successResult() { + return { + success: true, + content: 'secret-value', + contentBlocks: [], + toolCalls: [], + chatId: 'chat-1', + } + } + + function activateSecret(options: CopilotLifecycleOptions): void { + options.resolvedSecretTraceRegistry?.recordResolved('API_KEY', 'secret-value') + } + + it('does not expose private provenance unless the internal caller requests it', async () => { + mockRunHeadlessCopilotLifecycle.mockImplementation( + async (_payload: Record, options: CopilotLifecycleOptions) => { + activateSecret(options) + return successResult() + } + ) + + const response = await POST( + createMockRequest( + 'POST', + requestBody, + { Authorization: 'Bearer internal', 'x-sim-billing-attribution': 'billing' }, + 'http://localhost:3000/api/mothership/execute' + ) + ) + const body = await response.json() + + expect(response.status).toBe(200) + expect(response.headers.get('x-sim-private-tool-metadata')).toBeNull() + expect(body.content).toBe('secret-value') + expect(body).not.toHaveProperty('__resolvedSecretTraceProvenance') + }) + + it('returns encrypted provenance on a marker-gated successful request', async () => { + mockRunHeadlessCopilotLifecycle.mockImplementation( + async (_payload: Record, options: CopilotLifecycleOptions) => { + activateSecret(options) + return successResult() + } + ) + + const response = await POST( + createMockRequest( + 'POST', + requestBody, + { + Authorization: 'Bearer internal', + 'x-sim-billing-attribution': 'billing', + 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1', + }, + 'http://localhost:3000/api/mothership/execute' + ) + ) + const body = await response.json() + + expect(response.headers.get('x-sim-private-tool-metadata')).toBe( + 'resolved-secret-provenance-v1' + ) + expect(body.content).toBe('secret-value') + expect(body.__resolvedSecretTraceProvenance).toEqual({ + version: 1, + complete: true, + entries: [{ name: 'API_KEY', encryptedValue: 'encrypted-secret' }], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + }) + expect(JSON.stringify(body.__resolvedSecretTraceProvenance)).not.toContain('secret-value') + }) + + it('imports MCP schema-discovery provenance before starting the lifecycle', async () => { + const provenance = { + version: 1, + complete: true, + entries: [{ name: 'API_KEY', encryptedValue: 'encrypted-secret' }], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + } + mockBuildTaggedMcpToolSchemas.mockImplementationOnce( + async ( + _userId: string, + _workspaceId: string, + _serverIds: string[], + report: (value: unknown) => void + ) => { + report(provenance) + return [] + } + ) + mockRunHeadlessCopilotLifecycle.mockImplementation( + async (payload: Record, options: CopilotLifecycleOptions) => { + expect(options.resolvedSecretTraceRegistry?.exportProvenance()).toEqual(provenance) + expect(JSON.stringify(payload)).not.toContain('encrypted-secret') + expect(JSON.stringify(payload)).not.toContain('__resolvedSecretTraceProvenance') + return successResult() + } + ) + + const response = await POST( + createMockRequest( + 'POST', + { + ...requestBody, + contexts: [{ kind: 'mcp', label: 'Docs', serverId: 'server-1' }], + }, + { + Authorization: 'Bearer internal', + 'x-sim-billing-attribution': 'billing', + 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1', + }, + 'http://localhost:3000/api/mothership/execute' + ) + ) + const body = await response.json() + + expect({ status: response.status, provenance: body.__resolvedSecretTraceProvenance }).toEqual({ + status: 200, + provenance, + }) + }) + + it('marks the lifecycle registry incomplete for malformed MCP discovery provenance', async () => { + mockBuildTaggedMcpToolSchemas.mockImplementationOnce( + async ( + _userId: string, + _workspaceId: string, + _serverIds: string[], + report: (value: unknown) => void + ) => { + report({ version: 1, complete: true, entries: 'invalid' }) + return [] + } + ) + mockRunHeadlessCopilotLifecycle.mockImplementation( + async (_payload: Record, options: CopilotLifecycleOptions) => { + expect(options.resolvedSecretTraceRegistry?.isComplete()).toBe(false) + return successResult() + } + ) + + const response = await POST( + createMockRequest( + 'POST', + { + ...requestBody, + contexts: [{ kind: 'mcp', label: 'Docs', serverId: 'server-1' }], + }, + { + Authorization: 'Bearer internal', + 'x-sim-billing-attribution': 'billing', + 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1', + }, + 'http://localhost:3000/api/mothership/execute' + ) + ) + const body = await response.json() + + expect(response.status).toBe(200) + expect(body.__resolvedSecretTraceProvenance).toEqual({ + version: 1, + complete: false, + entries: [], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + }) + }) + + it('returns encrypted provenance with marker-gated failures', async () => { + mockRunHeadlessCopilotLifecycle.mockImplementation( + async (_payload: Record, options: CopilotLifecycleOptions) => { + activateSecret(options) + return { + ...successResult(), + success: false, + error: 'failed with secret-value', + content: 'secret-value', + } + } + ) + + const response = await POST( + createMockRequest( + 'POST', + requestBody, + { + Authorization: 'Bearer internal', + 'x-sim-billing-attribution': 'billing', + 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1', + }, + 'http://localhost:3000/api/mothership/execute' + ) + ) + const body = await response.json() + + expect(response.status).toBe(500) + expect(response.headers.get('x-sim-private-tool-metadata')).toBe( + 'resolved-secret-provenance-v1' + ) + expect(body.content).toBe('secret-value') + expect(body.__resolvedSecretTraceProvenance.entries).toEqual([ + { name: 'API_KEY', encryptedValue: 'encrypted-secret' }, + ]) + }) + + it('places encrypted provenance only on the terminal streamed event', async () => { + mockRunHeadlessCopilotLifecycle.mockImplementation( + async (_payload: Record, options: CopilotLifecycleOptions) => { + activateSecret(options) + return successResult() + } + ) + + const response = await POST( + createMockRequest( + 'POST', + requestBody, + { + Authorization: 'Bearer internal', + 'x-sim-billing-attribution': 'billing', + 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1', + 'x-mothership-execute-stream': 'ndjson', + }, + 'http://localhost:3000/api/mothership/execute' + ) + ) + const events = (await response.text()) + .trim() + .split('\n') + .map((line) => JSON.parse(line) as Record) + + expect(response.headers.get('x-sim-private-tool-metadata')).toBe( + 'resolved-secret-provenance-v1' + ) + expect(events[0]).toMatchObject({ type: 'heartbeat' }) + expect(events[0]).not.toHaveProperty('__resolvedSecretTraceProvenance') + expect(events.at(-1)).toMatchObject({ + type: 'final', + data: { + content: 'secret-value', + __resolvedSecretTraceProvenance: { + entries: [{ name: 'API_KEY', encryptedValue: 'encrypted-secret' }], + }, + }, + }) + }) +}) diff --git a/apps/sim/app/api/mothership/execute/route.ts b/apps/sim/app/api/mothership/execute/route.ts index 3e46be58217..768d4a5ccf0 100644 --- a/apps/sim/app/api/mothership/execute/route.ts +++ b/apps/sim/app/api/mothership/execute/route.ts @@ -20,10 +20,22 @@ import { requestExplicitStreamAbort } from '@/lib/copilot/request/session/explic import type { StreamEvent } from '@/lib/copilot/request/types' import { isDocSandboxEnabled } from '@/lib/core/config/env-flags' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { getPersonalAndWorkspaceEnv } from '@/lib/environment/utils' +import { + PRIVATE_TOOL_METADATA_RESPONSE_HEADER, + RESOLVED_SECRET_PROVENANCE_FIELD, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, + requestsPrivateToolMetadata, +} from '@/lib/execution/private-tool-metadata' import { assertActiveWorkspaceAccess, isWorkspaceAccessDeniedError, } from '@/lib/workspaces/permissions/utils' +import { + createResolvedSecretTraceRegistry, + ResolvedSecretTraceProvenanceAccumulator, + type ResolvedSecretTraceRegistry, +} from '@/executor/utils/resolved-secret-trace-registry' import type { ChatContext } from '@/stores/panel' export const maxDuration = 3600 @@ -35,6 +47,28 @@ const MOTHERSHIP_EXECUTE_STREAM_CONTENT_TYPE = 'application/x-ndjson' const MOTHERSHIP_EXECUTE_HEARTBEAT_INTERVAL_MS = 15_000 const ndjsonEncoder = new TextEncoder() +function withPrivateProvenance>( + payload: T, + registry: ResolvedSecretTraceRegistry | undefined, + include: boolean +): T & Partial> { + return { + ...payload, + ...(include && registry + ? { [RESOLVED_SECRET_PROVENANCE_FIELD]: registry.exportProvenance() } + : {}), + } +} + +function privateResponseHeaders( + registry: ResolvedSecretTraceRegistry | undefined, + include: boolean +): Record { + return include && registry + ? { [PRIVATE_TOOL_METADATA_RESPONSE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1 } + : {} +} + function isAbortError(error: unknown): boolean { return error instanceof Error && error.name === 'AbortError' } @@ -99,6 +133,11 @@ export function buildExecuteResponsePayload( export const POST = withRouteHandler(async (req: NextRequest) => { let messageId: string | undefined let requestId: string | undefined + let resolvedSecretTraceRegistry: ResolvedSecretTraceRegistry | undefined + const includePrivateProvenance = requestsPrivateToolMetadata( + req.headers, + RESOLVED_SECRET_PROVENANCE_METADATA_V1 + ) try { const auth = await checkInternalAuth(req, { requireWorkflowId: false }) @@ -146,6 +185,23 @@ export const POST = withRouteHandler(async (req: NextRequest) => { actorUserId: userId, workspaceId, }) + const environment = await getPersonalAndWorkspaceEnv(userId, workspaceId) + resolvedSecretTraceRegistry = await createResolvedSecretTraceRegistry({ + personalEncrypted: environment.personalEncrypted, + workspaceEncrypted: environment.workspaceEncrypted, + personalDecrypted: environment.personalDecrypted, + workspaceDecrypted: environment.workspaceDecrypted, + decryptionFailures: environment.decryptionFailures, + scope: { userId, workspaceId }, + }) + const activeResolvedSecretTraceRegistry = resolvedSecretTraceRegistry + const mcpDiscoveryProvenance = new ResolvedSecretTraceProvenanceAccumulator({ + userId, + workspaceId, + }) + const recordMcpDiscoveryProvenance = (provenance: unknown): void => { + mcpDiscoveryProvenance.record(provenance) + } const effectiveChatId = chatId || generateId() messageId = providedMessageId || generateId() @@ -164,17 +220,36 @@ export const POST = withRouteHandler(async (req: NextRequest) => { ) const nonMcpAgentMentions = agentMentions?.filter((context) => context.kind !== 'mcp') const userPermission = workspaceAccess.permission + const mothershipToolsPromise = Promise.allSettled([ + buildSelectedMcpToolSchemas( + userId, + workspaceId, + mcpTools ?? [], + recordMcpDiscoveryProvenance + ), + buildTaggedMcpToolSchemas( + userId, + workspaceId, + taggedMcpServerIds, + recordMcpDiscoveryProvenance + ), + ]).then(async (results) => { + await activeResolvedSecretTraceRegistry.importProvenance( + mcpDiscoveryProvenance.exportProvenance(), + { trusted: true } + ) + const groups = results.map((result) => { + if (result.status === 'rejected') throw result.reason + return result.value + }) + const byName = new Map(groups.flat().map((tool) => [tool.name, tool])) + return [...byName.values()] + }) const [workspaceContext, integrationTools, mothershipTools, entitlements, agentContexts] = await Promise.all([ generateWorkspaceContext(workspaceId, userId, { workspaceAccess }), buildIntegrationToolSchemas(userId, messageId, undefined, workspaceId), - Promise.all([ - buildSelectedMcpToolSchemas(userId, workspaceId, mcpTools ?? []), - buildTaggedMcpToolSchemas(userId, workspaceId, taggedMcpServerIds), - ]).then((groups) => { - const byName = new Map(groups.flat().map((tool) => [tool.name, tool])) - return [...byName.values()] - }), + mothershipToolsPromise, computeWorkspaceEntitlements(workspaceId, userId), processContextsServer( nonMcpAgentMentions, @@ -282,6 +357,7 @@ export const POST = withRouteHandler(async (req: NextRequest) => { interactive: false, abortSignal: lifecycleAbortController.signal, billingAttribution, + resolvedSecretTraceRegistry, onEvent, }) @@ -326,7 +402,13 @@ export const POST = withRouteHandler(async (req: NextRequest) => { allowExplicitAbort = false if (lifecycleAbortController.signal.aborted) { - send({ type: 'error', error: 'Sim execution aborted' }) + send( + withPrivateProvenance( + { type: 'error', error: 'Sim execution aborted' }, + resolvedSecretTraceRegistry, + includePrivateProvenance + ) + ) return } @@ -343,17 +425,27 @@ export const POST = withRouteHandler(async (req: NextRequest) => { errors: result.errors, } ) - send({ - type: 'error', - error: result.error || 'Sim execution failed', - content: result.content || '', - }) + send( + withPrivateProvenance( + { + type: 'error', + error: result.error || 'Sim execution failed', + content: result.content || '', + }, + resolvedSecretTraceRegistry, + includePrivateProvenance + ) + ) return } send({ type: 'final', - data: buildExecuteResponsePayload(result, effectiveChatId, integrationTools), + data: withPrivateProvenance( + buildExecuteResponsePayload(result, effectiveChatId, integrationTools), + resolvedSecretTraceRegistry, + includePrivateProvenance + ), }) } catch (error) { if ( @@ -367,7 +459,13 @@ export const POST = withRouteHandler(async (req: NextRequest) => { : 'Mothership execute aborted', { requestId } ) - send({ type: 'error', error: 'Sim execution aborted' }) + send( + withPrivateProvenance( + { type: 'error', error: 'Sim execution aborted' }, + resolvedSecretTraceRegistry, + includePrivateProvenance + ) + ) return } @@ -380,10 +478,16 @@ export const POST = withRouteHandler(async (req: NextRequest) => { error: getErrorMessage(error, 'Unknown error'), } ) - send({ - type: 'error', - error: getErrorMessage(error, 'Internal server error'), - }) + send( + withPrivateProvenance( + { + type: 'error', + error: getErrorMessage(error, 'Internal server error'), + }, + resolvedSecretTraceRegistry, + includePrivateProvenance + ) + ) } finally { allowExplicitAbort = false if (heartbeatId) { @@ -410,6 +514,7 @@ export const POST = withRouteHandler(async (req: NextRequest) => { headers: { 'Content-Type': `${MOTHERSHIP_EXECUTE_STREAM_CONTENT_TYPE}; charset=utf-8`, 'Cache-Control': 'no-cache, no-transform', + ...privateResponseHeaders(resolvedSecretTraceRegistry, includePrivateProvenance), }, }) } @@ -421,7 +526,17 @@ export const POST = withRouteHandler(async (req: NextRequest) => { if (lifecycleAbortController.signal.aborted || req.signal.aborted) { reqLogger.info('Mothership execute aborted after lifecycle completion') - return NextResponse.json({ error: 'Sim execution aborted' }, { status: 499 }) + return NextResponse.json( + withPrivateProvenance( + { error: 'Sim execution aborted' }, + resolvedSecretTraceRegistry, + includePrivateProvenance + ), + { + status: 499, + headers: privateResponseHeaders(resolvedSecretTraceRegistry, includePrivateProvenance), + } + ) } if (!result.success) { @@ -438,16 +553,30 @@ export const POST = withRouteHandler(async (req: NextRequest) => { } ) return NextResponse.json( + withPrivateProvenance( + { + error: result.error || 'Sim execution failed', + content: result.content || '', + }, + resolvedSecretTraceRegistry, + includePrivateProvenance + ), { - error: result.error || 'Sim execution failed', - content: result.content || '', - }, - { status: 500 } + status: 500, + headers: privateResponseHeaders(resolvedSecretTraceRegistry, includePrivateProvenance), + } ) } return NextResponse.json( - buildExecuteResponsePayload(result, effectiveChatId, integrationTools) + withPrivateProvenance( + buildExecuteResponsePayload(result, effectiveChatId, integrationTools), + resolvedSecretTraceRegistry, + includePrivateProvenance + ), + { + headers: privateResponseHeaders(resolvedSecretTraceRegistry, includePrivateProvenance), + } ) } finally { allowExplicitAbort = false @@ -465,7 +594,17 @@ export const POST = withRouteHandler(async (req: NextRequest) => { } ) - return NextResponse.json({ error: 'Sim execution aborted' }, { status: 499 }) + return NextResponse.json( + withPrivateProvenance( + { error: 'Sim execution aborted' }, + resolvedSecretTraceRegistry, + includePrivateProvenance + ), + { + status: 499, + headers: privateResponseHeaders(resolvedSecretTraceRegistry, includePrivateProvenance), + } + ) } if (isWorkspaceAccessDeniedError(error)) { @@ -481,8 +620,15 @@ export const POST = withRouteHandler(async (req: NextRequest) => { ) return NextResponse.json( - { error: getErrorMessage(error, 'Internal server error') }, - { status: 500 } + withPrivateProvenance( + { error: getErrorMessage(error, 'Internal server error') }, + resolvedSecretTraceRegistry, + includePrivateProvenance + ), + { + status: 500, + headers: privateResponseHeaders(resolvedSecretTraceRegistry, includePrivateProvenance), + } ) } }) diff --git a/apps/sim/app/api/v1/logs/[id]/route.ts b/apps/sim/app/api/v1/logs/[id]/route.ts index 148d3811895..108e9fc534e 100644 --- a/apps/sim/app/api/v1/logs/[id]/route.ts +++ b/apps/sim/app/api/v1/logs/[id]/route.ts @@ -7,7 +7,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { v1GetLogContract } from '@/lib/api/contracts/v1/logs' import { parseRequest } from '@/lib/api/server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { materializeExecutionData } from '@/lib/logs/execution/trace-store' +import { materializeExecutionDataForDisplay } from '@/lib/logs/execution/trace-store' import { createApiResponse, getUserLimits } from '@/app/api/v1/logs/meta' import { checkRateLimit, @@ -100,12 +100,13 @@ export const GET = withRouteHandler( totalDurationMs: log.totalDurationMs, files: log.files || undefined, workflow: workflowSummary, - executionData: (await materializeExecutionData( + executionData: (await materializeExecutionDataForDisplay( log.executionData as Record | null, { workspaceId: log.workspaceId, workflowId: log.workflowId, executionId: log.executionId, + userId, } )) as any, cost: log.costTotal != null ? { total: Number(log.costTotal) } : null, diff --git a/apps/sim/app/api/v1/logs/route.ts b/apps/sim/app/api/v1/logs/route.ts index be40f9ae2dd..1da2529a84f 100644 --- a/apps/sim/app/api/v1/logs/route.ts +++ b/apps/sim/app/api/v1/logs/route.ts @@ -8,7 +8,7 @@ import { v1ListLogsContract } from '@/lib/api/contracts/v1/logs' import { parseRequest } from '@/lib/api/server' import { MATERIALIZE_CONCURRENCY, mapWithConcurrency } from '@/lib/core/utils/concurrency' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { materializeExecutionData } from '@/lib/logs/execution/trace-store' +import { materializeExecutionDataForDisplay } from '@/lib/logs/execution/trace-store' import { buildLogFilters, getOrderBy } from '@/app/api/v1/logs/filters' import { createApiResponse, getUserLimits } from '@/app/api/v1/logs/meta' import { @@ -167,12 +167,13 @@ export const GET = withRouteHandler(async (request: NextRequest) => { ? await mapWithConcurrency(data, MATERIALIZE_CONCURRENCY, async (log) => { const result = buildBase(log) if (log.executionData) { - const execData = (await materializeExecutionData( + const execData = (await materializeExecutionDataForDisplay( log.executionData as Record | null, { workspaceId: log.workspaceId, workflowId: log.workflowId, executionId: log.executionId, + userId, } )) as any if (params.includeFinalOutput && execData.finalOutput) { diff --git a/apps/sim/app/api/workflows/[id]/execute/route.async.test.ts b/apps/sim/app/api/workflows/[id]/execute/route.async.test.ts index 771e7cda706..0ccf344ab5a 100644 --- a/apps/sim/app/api/workflows/[id]/execute/route.async.test.ts +++ b/apps/sim/app/api/workflows/[id]/execute/route.async.test.ts @@ -9,9 +9,11 @@ import { executionPreprocessingMockFns, hybridAuthMockFns, loggingSessionMock, + queueTableRows, requestUtilsMockFns, resetDbChainMock, resetEnvMock, + schemaMock, setEnv, workflowAuthzMockFns, workflowsPersistenceUtilsMock, @@ -339,6 +341,82 @@ describe('workflow execute async route', () => { mockHandlePostExecutionPauseState.mockResolvedValue(undefined) }) + it('reuses raw workflow input by execution ID without returning it to the client', async () => { + const sourceInput = { token: 'raw-secret-1234', nested: { value: 42 } } + queueTableRows(schemaMock.workflowExecutionLogs, [ + { + executionId: 'source-execution', + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + executionData: { workflowInput: sourceInput }, + }, + ]) + const request = createMockRequest( + 'POST', + { inputFromExecutionId: 'source-execution' }, + { + 'Content-Type': 'application/json', + 'X-Execution-Mode': 'async', + Cookie: 'session=value', + } + ) + + const response = await POST(request, { params: Promise.resolve({ id: 'workflow-1' }) }) + const responseBody = await response.json() + + expect(response.status).toBe(202) + expect(responseBody).not.toHaveProperty('input') + expect(JSON.stringify(responseBody)).not.toContain('raw-secret-1234') + expect(mockEnqueue).toHaveBeenCalledWith( + 'workflow-execution', + expect.objectContaining({ input: sourceInput }), + expect.any(Object) + ) + }) + + it('rejects client input alongside a stored execution input reference', async () => { + const response = await POST( + createMockRequest( + 'POST', + { + input: { replacement: true }, + inputFromExecutionId: 'source-execution', + }, + { 'Content-Type': 'application/json', Cookie: 'session=value' } + ), + { params: Promise.resolve({ id: 'workflow-1' }) } + ) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ + error: 'Provide either input or inputFromExecutionId, not both', + }) + expect(mockEnqueue).not.toHaveBeenCalled() + }) + + it('rejects stored execution input references from external callers', async () => { + mockCheckHybridAuth.mockResolvedValue({ + success: true, + userId: 'personal-key-user-1', + authType: 'api_key', + apiKeyType: 'personal', + }) + const response = await POST( + createMockRequest( + 'POST', + { inputFromExecutionId: 'source-execution' }, + { 'Content-Type': 'application/json', 'X-API-Key': 'personal-key' } + ), + { params: Promise.resolve({ id: 'workflow-1' }) } + ) + + expect(response.status).toBe(403) + await expect(response.json()).resolves.toEqual({ + error: 'Stored execution input can only be reused by an authenticated session', + }) + expect(mockEnqueue).not.toHaveBeenCalled() + }) + it('queues async execution with matching correlation metadata', async () => { const req = createMockRequest( 'POST', @@ -750,6 +828,123 @@ describe('workflow execute async route', () => { expect(mockReleaseExecutionIdClaim).not.toHaveBeenCalled() }) + it('loads trusted run-from-block state by execution ID and preserves its source identity', async () => { + const sourceState = { + blockStates: { previous: { output: { value: 'cached' } } }, + executedBlocks: ['previous'], + blockLogs: [], + decisions: { router: {}, condition: {} }, + completedLoops: [], + activeExecutionPath: [], + resolvedSecretTraceProvenance: { + version: 1, + complete: true, + entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }], + scope: { userId: 'owner-1', workspaceId: 'workspace-1' }, + }, + } + queueTableRows(schemaMock.workflowExecutionLogs, [ + { + executionId: 'source-execution', + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + executionData: { executionState: sourceState }, + }, + ]) + const request = createMockRequest( + 'POST', + { + input: { hello: 'world' }, + runFromBlock: { + startBlockId: 'start-block', + executionId: 'source-execution', + }, + }, + { + 'Content-Type': 'application/json', + Cookie: 'session=value', + } + ) + + const response = await POST(request, { params: Promise.resolve({ id: 'workflow-1' }) }) + + expect(response.status).toBe(200) + expect(mockExecuteWorkflowCore).toHaveBeenCalledWith( + expect.objectContaining({ + runFromBlock: { + startBlockId: 'start-block', + sourceSnapshot: sourceState, + sourceExecutionId: 'source-execution', + }, + }) + ) + }) + + it('returns encrypted resolution provenance only to an authenticated internal tool caller', async () => { + const caller = EXECUTION_CALLERS[4] + configureExecutionCaller(caller) + const provenance = { + version: 1, + complete: true, + entries: [{ name: 'CHILD_SECRET', encryptedValue: 'encrypted-child-secret' }], + } + mockExecuteWorkflowCore.mockResolvedValueOnce({ + success: true, + status: 'completed', + output: { ok: true }, + executionState: { resolvedSecretTraceProvenance: provenance }, + metadata: { + duration: 100, + startTime: '2026-01-01T00:00:00Z', + endTime: '2026-01-01T00:00:01Z', + }, + }) + const request = createCallerExecutionRequest(caller, undefined, 'sync') + request.headers.set('x-sim-request-private-tool-metadata', 'resolved-secret-provenance-v1') + + const response = await POST(request, { params: Promise.resolve({ id: 'workflow-1' }) }) + + expect(response.status).toBe(200) + expect(response.headers.get('x-sim-private-tool-metadata')).toBe( + 'resolved-secret-provenance-v1' + ) + await expect(response.json()).resolves.toMatchObject({ + output: { ok: true }, + __resolvedSecretTraceProvenance: provenance, + }) + }) + + it('does not expose private provenance metadata to non-internal callers', async () => { + const caller = EXECUTION_CALLERS[1] + configureExecutionCaller(caller) + mockExecuteWorkflowCore.mockResolvedValueOnce({ + success: true, + status: 'completed', + output: { ok: true }, + executionState: { + resolvedSecretTraceProvenance: { + version: 1, + complete: true, + entries: [{ name: 'SECRET', encryptedValue: 'encrypted-secret' }], + }, + }, + metadata: { + duration: 100, + startTime: '2026-01-01T00:00:00Z', + endTime: '2026-01-01T00:00:01Z', + }, + }) + const request = createCallerExecutionRequest(caller, undefined, 'sync') + request.headers.set('x-sim-request-private-tool-metadata', 'resolved-secret-provenance-v1') + + const response = await POST(request, { params: Promise.resolve({ id: 'workflow-1' }) }) + const body = await response.json() + + expect(response.status).toBe(200) + expect(response.headers.get('x-sim-private-tool-metadata')).toBeNull() + expect(body).not.toHaveProperty('__resolvedSecretTraceProvenance') + }) + it('releases the admission reservation when enqueue proves non-acceptance', async () => { mockEnqueue.mockRejectedValueOnce( new AsyncJobEnqueueError('queue rejected the job', { diff --git a/apps/sim/app/api/workflows/[id]/execute/route.ts b/apps/sim/app/api/workflows/[id]/execute/route.ts index f56a89504e6..d9712918d9e 100644 --- a/apps/sim/app/api/workflows/[id]/execute/route.ts +++ b/apps/sim/app/api/workflows/[id]/execute/route.ts @@ -58,6 +58,12 @@ import { import { containsLargeValueRef } from '@/lib/execution/payloads/large-value-ref' import { compactBlockLogs, compactExecutionPayload } from '@/lib/execution/payloads/serializer' import { type PreprocessExecutionSuccess, preprocessExecution } from '@/lib/execution/preprocessing' +import { + PRIVATE_TOOL_METADATA_RESPONSE_HEADER, + RESOLVED_SECRET_PROVENANCE_FIELD, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, + requestsPrivateToolMetadata, +} from '@/lib/execution/private-tool-metadata' import { LoggingSession } from '@/lib/logs/execution/logging-session' import { MAX_MCP_WORKFLOW_RESPONSE_BYTES, @@ -114,12 +120,18 @@ import { import { normalizeName } from '@/executor/constants' import { ExecutionSnapshot } from '@/executor/execution/snapshot' import type { + BlockCompletionCallbackData, ChildWorkflowContext, ExecutionMetadata, IterationContext, SerializableExecutionState, } from '@/executor/execution/types' -import type { BlockLog, NormalizedBlockOutput, StreamingExecution } from '@/executor/types' +import type { + BlockLog, + ExecutionResult, + NormalizedBlockOutput, + StreamingExecution, +} from '@/executor/types' import { getExecutionErrorStatus, hasExecutionResult } from '@/executor/utils/errors' import { Serializer } from '@/serializer' import { CORE_TRIGGER_TYPES, type CoreTriggerType } from '@/stores/logs/filters/types' @@ -133,6 +145,31 @@ const WORKFLOW_EXECUTION_JOB_ID_PREFIX = 'workflow-execution:' export const runtime = 'nodejs' export const dynamic = 'force-dynamic' +function createExecutionJsonResponse( + body: Record, + init: ResponseInit | undefined, + includePrivateProvenance: boolean, + result?: ExecutionResult +): NextResponse { + if (!includePrivateProvenance) { + return NextResponse.json(body, init) + } + + const headers = new Headers(init?.headers) + headers.set(PRIVATE_TOOL_METADATA_RESPONSE_HEADER, RESOLVED_SECRET_PROVENANCE_METADATA_V1) + return NextResponse.json( + { + ...body, + [RESOLVED_SECRET_PROVENANCE_FIELD]: result?.executionState?.resolvedSecretTraceProvenance ?? { + version: 1, + complete: false, + entries: [], + }, + }, + { ...init, headers } + ) +} + async function compactRoutePayload( value: T, context: { @@ -573,6 +610,10 @@ async function handleExecutePost( const isMcpBridgeRequest = auth.authType === AuthType.INTERNAL_JWT && req.headers.get(MCP_TOOL_BRIDGE_HEADER) === 'true' + const includePrivateTraceProvenance = + auth.success && + auth.authType === AuthType.INTERNAL_JWT && + requestsPrivateToolMetadata(req.headers, RESOLVED_SECRET_PROVENANCE_METADATA_V1) const useMcpBridgeAuthenticatedUserAsActor = isMcpBridgeRequest && req.headers.get(MCP_TOOL_BRIDGE_ACTOR_HEADER) === 'authenticated-user' @@ -682,6 +723,7 @@ async function handleExecutePost( includeToolCalls: requestedIncludeToolCalls, useDraftState, input: validatedInput, + inputFromExecutionId, isClientSession = false, includeFileBase64, base64MaxBytes, @@ -730,6 +772,20 @@ async function handleExecutePost( ) } + if (inputFromExecutionId && (isPublicApiAccess || auth.authType !== AuthType.SESSION)) { + return NextResponse.json( + { error: 'Stored execution input can only be reused by an authenticated session' }, + { status: 403 } + ) + } + + if (inputFromExecutionId && validatedInput !== undefined) { + return NextResponse.json( + { error: 'Provide either input or inputFromExecutionId, not both' }, + { status: 400 } + ) + } + if (auth.authType === 'api_key') { if (isClientSession) { return NextResponse.json( @@ -776,14 +832,7 @@ async function handleExecutePost( ) } - if (rawRunFromBlock.sourceSnapshot && !isPublicApiAccess) { - // Public API callers cannot inject arbitrary block state via sourceSnapshot. - // They must use executionId to resume from a server-stored execution state. - resolvedRunFromBlock = { - startBlockId: rawRunFromBlock.startBlockId, - sourceSnapshot: rawRunFromBlock.sourceSnapshot as SerializableExecutionState, - } - } else if (rawRunFromBlock.executionId) { + if (rawRunFromBlock.executionId) { const { getExecutionStateForWorkflow, getLatestExecutionStateWithExecutionId } = await import('@/lib/workflows/executor/execution-state') const sourceExecution = @@ -795,17 +844,32 @@ async function handleExecutePost( } const snapshot = sourceExecution?.state if (!snapshot) { - return NextResponse.json( - { - error: `No execution state found for ${rawRunFromBlock.executionId === 'latest' ? 'workflow' : `execution ${rawRunFromBlock.executionId}`}. Run the full workflow first.`, - }, - { status: 400 } - ) + if (rawRunFromBlock.sourceSnapshot && !isPublicApiAccess) { + resolvedRunFromBlock = { + startBlockId: rawRunFromBlock.startBlockId, + sourceSnapshot: rawRunFromBlock.sourceSnapshot as SerializableExecutionState, + } + } else { + return NextResponse.json( + { + error: `No execution state found for ${rawRunFromBlock.executionId === 'latest' ? 'workflow' : `execution ${rawRunFromBlock.executionId}`}. Run the full workflow first.`, + }, + { status: 400 } + ) + } + } else { + resolvedRunFromBlock = { + startBlockId: rawRunFromBlock.startBlockId, + sourceSnapshot: snapshot, + sourceExecutionId: sourceExecution.executionId, + } } + } else if (rawRunFromBlock.sourceSnapshot && !isPublicApiAccess) { + // Public API callers cannot inject arbitrary block state via sourceSnapshot. + // They must use executionId to resume from a server-stored execution state. resolvedRunFromBlock = { startBlockId: rawRunFromBlock.startBlockId, - sourceSnapshot: snapshot, - sourceExecutionId: sourceExecution.executionId, + sourceSnapshot: rawRunFromBlock.sourceSnapshot as SerializableExecutionState, } } else { return NextResponse.json( @@ -817,7 +881,7 @@ async function handleExecutePost( // For API key and internal JWT auth, the entire body is the input (except for our control fields) // For session auth, the input is explicitly provided in the input field - const input = isMcpBridgeRequest + let input = isMcpBridgeRequest ? validatedInput : isPublicApiAccess || auth.authType === AuthType.API_KEY || @@ -828,6 +892,7 @@ async function handleExecutePost( triggerType, stream, useDraftState, + inputFromExecutionId: _inputFromExecutionId, includeFileBase64, base64MaxBytes, workflowStateOverride, @@ -962,6 +1027,20 @@ async function handleExecutePost( ) } + if (inputFromExecutionId) { + const { getExecutionInputForWorkflow } = await import( + '@/lib/workflows/executor/execution-state' + ) + const sourceExecution = await getExecutionInputForWorkflow(inputFromExecutionId, workflowId) + if (!sourceExecution.found) { + return NextResponse.json( + { error: 'Source workflow execution was not found' }, + { status: 404 } + ) + } + input = sourceExecution.input + } + const upstreamBillingAttribution = auth.authType === AuthType.INTERNAL_JWT && workflowAuthorization.workflow?.workspaceId ? requireBillingAttributionHeader(req.headers, { @@ -1287,7 +1366,7 @@ async function handleExecutePost( workflowResponseCompaction ) - return NextResponse.json( + return createExecutionJsonResponse( { success: false, output: compactResultOutput, @@ -1300,7 +1379,9 @@ async function handleExecutePost( } : undefined, }, - { status: 408 } + { status: 408 }, + includePrivateTraceProvenance, + result ) } @@ -1367,7 +1448,12 @@ async function handleExecutePost( : undefined, } - return NextResponse.json(filteredResult) + return createExecutionJsonResponse( + filteredResult, + undefined, + includePrivateTraceProvenance, + result + ) } catch (error: unknown) { const errorMessage = getErrorMessage(error, 'Unknown error') @@ -1405,7 +1491,7 @@ async function handleExecutePost( throw compactError } } - return NextResponse.json( + return createExecutionJsonResponse( { success: false, output: compactErrorOutput, @@ -1418,7 +1504,9 @@ async function handleExecutePost( } : undefined, }, - { status } + { status }, + includePrivateTraceProvenance, + executionResult ) } finally { requestAbort.cleanup() @@ -1663,7 +1751,7 @@ async function handleExecutePost( blockId: string, blockName: string, blockType: string, - callbackData: any, + callbackData: BlockCompletionCallbackData, iterationContext?: IterationContext, childWorkflowContext?: ChildWorkflowContext ) => { @@ -1686,7 +1774,13 @@ async function handleExecutePost( preserveRoot: true, }), } - const hasError = compactCallbackData.output?.error + const callbackError = compactCallbackData.output?.error + const hasError = typeof callbackError === 'string' && callbackError.length > 0 + const display = await loggingSession.projectDisplayContent({ + input: compactCallbackData.input, + output: compactCallbackData.output, + ...(hasError ? { error: callbackError } : {}), + }) const childWorkflowData = childWorkflowContext ? { childWorkflowBlockId: childWorkflowContext.parentBlockId, @@ -1703,7 +1797,7 @@ async function handleExecutePost( blockId, blockName, blockType, - error: compactCallbackData.output.error, + error: display.error, }) await sendEvent({ type: 'block:error', @@ -1715,7 +1809,12 @@ async function handleExecutePost( blockName, blockType, input: compactCallbackData.input, - error: compactCallbackData.output.error, + error: callbackError, + display: { + ...(Object.hasOwn(display, 'input') ? { input: display.input } : {}), + ...(display.error !== undefined ? { error: display.error } : {}), + ...(display.clearLiveDisplay ? { clearLiveDisplay: true as const } : {}), + }, durationMs: compactCallbackData.executionTime || 0, startedAt: compactCallbackData.startedAt, executionOrder: compactCallbackData.executionOrder, @@ -1750,6 +1849,11 @@ async function handleExecutePost( blockType, input: compactCallbackData.input, output: compactCallbackData.output, + display: { + ...(Object.hasOwn(display, 'input') ? { input: display.input } : {}), + ...(Object.hasOwn(display, 'output') ? { output: display.output } : {}), + ...(display.clearLiveDisplay ? { clearLiveDisplay: true as const } : {}), + }, durationMs: compactCallbackData.executionTime || 0, startedAt: compactCallbackData.startedAt, executionOrder: compactCallbackData.executionOrder, @@ -1786,6 +1890,7 @@ async function handleExecutePost( workflowId, sendEvent, forwardAnswerText: answerTextFromSink, + projectDisplay: (field, value) => loggingSession.projectLiveDisplayText(field, value), }) const reader = streamingExec.stream.getReader() @@ -1807,12 +1912,13 @@ async function handleExecutePost( if (answerTextFromSink) continue const chunk = decoder.decode(value, { stream: true }) + const display = await loggingSession.projectLiveDisplayText('chunk', chunk) await sendEvent({ type: 'stream:chunk', timestamp: new Date().toISOString(), executionId, workflowId, - data: { blockId, chunk }, + data: { blockId, chunk, display }, }) } @@ -1931,7 +2037,10 @@ async function handleExecutePost( * object storage, so doing it twice would double the latency and storage * load on the happy path. */ - const compactedBlockLogs = await compactBlockLogs(result.logs, { + const displayBlockLogs = await loggingSession.projectBlockLogsForDisplay( + result.logs ?? [] + ) + const compactedBlockLogs = await compactBlockLogs(displayBlockLogs, { workspaceId, workflowId, executionId, @@ -1947,6 +2056,9 @@ async function handleExecutePost( }) await loggingSession.markAsFailed(timeoutErrorMessage) + const timeoutDisplay = await loggingSession.projectDisplayContent({ + error: timeoutErrorMessage, + }) finalMetaStatus = 'error' await sendEvent( @@ -1957,6 +2069,11 @@ async function handleExecutePost( workflowId, data: { error: timeoutErrorMessage, + display: { + ...(timeoutDisplay.error !== undefined + ? { error: timeoutDisplay.error } + : {}), + }, duration: result.metadata?.duration || 0, finalBlockLogs: compactedBlockLogs, }, @@ -2061,13 +2178,16 @@ async function handleExecutePost( let compactErrorLogs: BlockLog[] | undefined try { compactErrorLogs = executionResult?.logs - ? await compactBlockLogs(executionResult.logs, { - workspaceId, - workflowId, - executionId, - userId: actorUserId, - requireDurable: true, - }) + ? await compactBlockLogs( + await loggingSession.projectBlockLogsForDisplay(executionResult.logs), + { + workspaceId, + workflowId, + executionId, + userId: actorUserId, + requireDurable: true, + } + ) : undefined } catch (compactionError) { reqLogger.warn('Failed to compact SSE error logs, omitting oversized error details', { @@ -2076,6 +2196,10 @@ async function handleExecutePost( } finalMetaStatus = 'error' + const terminalError = executionResult?.error || errorMessage + const terminalDisplay = await loggingSession.projectDisplayContent({ + error: terminalError, + }) await sendEvent( { type: 'execution:error', @@ -2083,7 +2207,10 @@ async function handleExecutePost( executionId, workflowId, data: { - error: executionResult?.error || errorMessage, + error: terminalError, + display: { + ...(terminalDisplay.error !== undefined ? { error: terminalDisplay.error } : {}), + }, duration: executionResult?.metadata?.duration || 0, finalBlockLogs: compactErrorLogs, }, diff --git a/apps/sim/app/api/workflows/[id]/executions/[executionId]/route.ts b/apps/sim/app/api/workflows/[id]/executions/[executionId]/route.ts index d3ff2d2a8a6..d7923d662bc 100644 --- a/apps/sim/app/api/workflows/[id]/executions/[executionId]/route.ts +++ b/apps/sim/app/api/workflows/[id]/executions/[executionId]/route.ts @@ -9,6 +9,12 @@ import { } from '@/lib/api/contracts/workflows' import { parseRequest } from '@/lib/api/server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { + collectFunctionalBlockOutputs, + FUNCTIONAL_OUTPUTS_UNAVAILABLE_MESSAGE, + type FunctionalExecutionDataSource, + FunctionalOutputsUnavailableError, +} from '@/lib/logs/execution/functional-outputs' import { materializeExecutionData } from '@/lib/logs/execution/trace-store' import { getAutomaticResumeWaitingMetadata } from '@/lib/workflows/executor/paused-execution-metadata' import { validateWorkflowAccess } from '@/app/api/workflows/middleware' @@ -18,32 +24,10 @@ const logger = createLogger('WorkflowExecutionStatusAPI') type LogStatus = 'pending' | 'running' | 'completed' | 'failed' | 'cancelled' -interface TraceSpanShape { - blockId?: string - output?: Record - children?: TraceSpanShape[] -} - -interface ExecutionDataShape { +interface ExecutionDataShape extends FunctionalExecutionDataSource { finalOutput?: { error?: string } & Record error?: { message?: string } | string completionFailure?: string - traceSpans?: TraceSpanShape[] -} - -function collectBlockOutputs(spans: TraceSpanShape[] | undefined): Map { - const map = new Map() - const visit = (list?: TraceSpanShape[]): void => { - if (!list) return - for (const span of list) { - if (span.blockId && span.output !== undefined && !map.has(span.blockId)) { - map.set(span.blockId, span.output) - } - if (span.children) visit(span.children) - } - } - visit(spans) - return map } function resolvePath(value: unknown, path: string[]): unknown { @@ -204,10 +188,23 @@ export const GET = withRouteHandler( ? (executionData.finalOutput ?? null) : null - const blockOutputs = - selectedOutputs.length > 0 - ? pickSelectedOutputs(selectedOutputs, collectBlockOutputs(executionData?.traceSpans)) - : null + let blockOutputs: Record | null = null + if (selectedOutputs.length > 0) { + try { + blockOutputs = pickSelectedOutputs( + selectedOutputs, + collectFunctionalBlockOutputs(executionData) + ) + } catch (error) { + if (error instanceof FunctionalOutputsUnavailableError) { + return NextResponse.json( + { error: FUNCTIONAL_OUTPUTS_UNAVAILABLE_MESSAGE }, + { status: 409 } + ) + } + throw error + } + } const response: WorkflowExecutionStatusResponse = { executionId: logRow.executionId, diff --git a/apps/sim/app/api/workflows/[id]/log/route.test.ts b/apps/sim/app/api/workflows/[id]/log/route.test.ts index 9b6c35578d5..8611a1302ba 100644 --- a/apps/sim/app/api/workflows/[id]/log/route.test.ts +++ b/apps/sim/app/api/workflows/[id]/log/route.test.ts @@ -13,6 +13,7 @@ const { mockResolveBillingAttribution, mockAssertBillingAttributionSnapshot, mockStart, + mockSetResolvedSecretTraceRegistry, mockSafeComplete, mockSafeCompleteWithError, } = vi.hoisted(() => ({ @@ -21,6 +22,7 @@ const { mockResolveBillingAttribution: vi.fn(), mockAssertBillingAttributionSnapshot: vi.fn((value) => value), mockStart: vi.fn().mockResolvedValue(undefined), + mockSetResolvedSecretTraceRegistry: vi.fn(), mockSafeComplete: vi.fn().mockResolvedValue(undefined), mockSafeCompleteWithError: vi.fn().mockResolvedValue(undefined), })) @@ -42,6 +44,7 @@ vi.mock('@/lib/logs/execution/logging-session', () => ({ LoggingSession: vi.fn(function LoggingSession() { return { start: mockStart, + setResolvedSecretTraceRegistry: mockSetResolvedSecretTraceRegistry, markAsFailed: vi.fn().mockResolvedValue(undefined), safeCompleteWithError: mockSafeCompleteWithError, safeComplete: mockSafeComplete, @@ -307,4 +310,59 @@ describe('POST /api/workflows/[id]/log completion attribution', () => { ) expect(mockSafeComplete).toHaveBeenCalledOnce() }) + + it('restores trusted Secrets provenance before projecting legacy completion traces', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + workflowId: OWNER_WORKFLOW_ID, + workspaceId: 'workspace-1', + executionData: { + billingAttribution: storedBillingAttribution, + executionState: { + resolvedSecretTraceProvenance: { version: 1, complete: true, entries: [] }, + }, + }, + }, + ]) + + const res = await POST( + makeRequest(OWNER_WORKFLOW_ID, { + executionId: 'trusted-provenance-execution-id', + result: validResult, + }), + { params: Promise.resolve({ id: OWNER_WORKFLOW_ID }) } + ) + + expect(res.status).toBe(200) + const registry = mockSetResolvedSecretTraceRegistry.mock.calls[0]?.[0] + expect(registry?.isComplete()).toBe(true) + expect(registry?.exportProvenance()).toEqual({ + version: 1, + complete: true, + entries: [], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + }) + }) + + it('forces structural-only traces when trusted stored provenance is unavailable', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + workflowId: OWNER_WORKFLOW_ID, + workspaceId: 'workspace-1', + executionData: { billingAttribution: storedBillingAttribution }, + }, + ]) + + const res = await POST( + makeRequest(OWNER_WORKFLOW_ID, { + executionId: 'legacy-no-provenance-execution-id', + result: validResult, + }), + { params: Promise.resolve({ id: OWNER_WORKFLOW_ID }) } + ) + + expect(res.status).toBe(200) + const registry = mockSetResolvedSecretTraceRegistry.mock.calls[0]?.[0] + expect(registry?.isComplete()).toBe(false) + }) }) diff --git a/apps/sim/app/api/workflows/[id]/log/route.ts b/apps/sim/app/api/workflows/[id]/log/route.ts index f79888ca363..271ab98bed1 100644 --- a/apps/sim/app/api/workflows/[id]/log/route.ts +++ b/apps/sim/app/api/workflows/[id]/log/route.ts @@ -13,9 +13,11 @@ import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { LoggingSession } from '@/lib/logs/execution/logging-session' import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans' +import { materializeExecutionData } from '@/lib/logs/execution/trace-store' import { validateWorkflowAccess } from '@/app/api/workflows/middleware' import { createErrorResponse, createSuccessResponse } from '@/app/api/workflows/utils' import type { ExecutionResult } from '@/executor/types' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' const logger = createLogger('WorkflowLogAPI') @@ -119,6 +121,29 @@ export const POST = withRouteHandler( const isChatExecution = result.metadata?.source === 'chat' const triggerType = isChatExecution ? 'chat' : 'manual' const loggingSession = new LoggingSession(id, executionId, triggerType, requestId) + const resolvedSecretTraceRegistry = new ResolvedSecretTraceRegistry([], { + userId: actorUserId, + workspaceId: existingLog.workspaceId, + }) + const trustedExecutionData = await materializeExecutionData( + existingLog.executionData as Record, + { + workspaceId: existingLog.workspaceId, + workflowId: existingLog.workflowId, + executionId, + } + ) + const trustedExecutionState = trustedExecutionData.executionState + const trustedProvenance = + trustedExecutionState && typeof trustedExecutionState === 'object' + ? (trustedExecutionState as Record).resolvedSecretTraceProvenance + : undefined + if (trustedProvenance === undefined) { + resolvedSecretTraceRegistry.markIncomplete() + } else { + await resolvedSecretTraceRegistry.importProvenance(trustedProvenance, { trusted: true }) + } + loggingSession.setResolvedSecretTraceRegistry(resolvedSecretTraceRegistry) await loggingSession.start({ userId: actorUserId, diff --git a/apps/sim/app/workspace/[workspaceId]/logs/logs.tsx b/apps/sim/app/workspace/[workspaceId]/logs/logs.tsx index f161f672f78..7bf6142e139 100644 --- a/apps/sim/app/workspace/[workspaceId]/logs/logs.tsx +++ b/apps/sim/app/workspace/[workspaceId]/logs/logs.tsx @@ -72,8 +72,6 @@ import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/provide import { getBlock } from '@/blocks/registry' import { useFolderMap, useFolders } from '@/hooks/queries/folders' import { - fetchLogDetail, - logKeys, prefetchLogDetail, useCancelExecution, useDashboardStats, @@ -90,7 +88,6 @@ import { CORE_TRIGGER_TYPES } from '@/stores/logs/filters/types' import { Dashboard, ExecutionSnapshot, LogDetails, LogRowContextMenu } from './components' import { DELETED_WORKFLOW_LABEL, - extractRetryInput, formatDate, getDisplayStatus, type LogStatus, @@ -573,17 +570,11 @@ export default function Logs() { const retryLog = useCallback( async (log: WorkflowLogRow | null) => { const workflowId = log?.workflow?.id || log?.workflowId - const logId = log?.id - if (!workflowId || !logId) return + const executionId = log?.executionId + if (!workflowId || !executionId) return try { - const detailLog = await queryClient.fetchQuery({ - queryKey: logKeys.detail(workspaceId, logId), - queryFn: ({ signal }) => fetchLogDetail(logId, workspaceId, signal), - staleTime: 30 * 1000, - }) - const input = extractRetryInput(detailLog) - await retryExecution.mutateAsync({ workflowId, input }) + await retryExecution.mutateAsync({ workflowId, executionId }) toast.success('Retry started') } catch { toast.error('Failed to retry execution') diff --git a/apps/sim/app/workspace/[workspaceId]/logs/utils.ts b/apps/sim/app/workspace/[workspaceId]/logs/utils.ts index cadb003f19a..fbffba81702 100644 --- a/apps/sim/app/workspace/[workspaceId]/logs/utils.ts +++ b/apps/sim/app/workspace/[workspaceId]/logs/utils.ts @@ -2,7 +2,6 @@ import React from 'react' import { Badge } from '@sim/emcn' import { formatDuration, formatRelativeTime } from '@sim/utils/formatting' import { format } from 'date-fns' -import type { WorkflowLogDetail } from '@/lib/api/contracts/logs' import { getIntegrationMetadata } from '@/lib/logs/get-trigger-options' import { getBlock } from '@/blocks/registry' import { CORE_TRIGGER_TYPES } from '@/stores/logs/filters/types' @@ -213,37 +212,3 @@ export const formatDate = (dateString: string) => { relative: formatRelativeTime(dateString), } } - -/** - * Extracts the original workflow input from a log entry for retry. - * Prefers the persisted `workflowInput` field (new logs), falls back to - * reconstructing from `executionState.blockStates` (old logs). - */ -export function extractRetryInput(log: WorkflowLogDetail): unknown | undefined { - const execData = log.executionData - if (!execData) return undefined - - if (execData.workflowInput !== undefined) { - return execData.workflowInput - } - - const executionState = (execData as Record).executionState as - | { - blockStates?: Record< - string, - { output?: unknown; executed?: boolean; executionTime?: number } - > - } - | undefined - if (!executionState?.blockStates) return undefined - - // Starter/trigger blocks are pre-populated with executed: false and - // executionTime: 0, which distinguishes them from blocks that actually ran. - for (const state of Object.values(executionState.blockStates)) { - if (state.executed === false && state.executionTime === 0 && state.output != null) { - return state.output - } - } - - return undefined -} diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-workflow-execution.test.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-workflow-execution.test.tsx index f62015495af..bccb79120c9 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-workflow-execution.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-workflow-execution.test.tsx @@ -11,7 +11,9 @@ const { executionStoreState, mockExecute, mockFetch, + mockResolveStartCandidates, mockRunUploadStrategy, + mockSelectBestTrigger, terminalStoreState, workflowBlocks, workflowStoreState, @@ -50,7 +52,7 @@ const { const executionStoreState = { workflowExecutions: new Map([['workflow-1', idleExecution]]), getWorkflowExecution: vi.fn(() => idleExecution), - getCurrentExecutionId: vi.fn(() => null), + getCurrentExecutionId: vi.fn<() => string | null>(() => null), getLastExecutionSnapshot: vi.fn(() => null), setCurrentExecutionId: vi.fn(), setIsExecuting: vi.fn(), @@ -89,7 +91,9 @@ const { executionStoreState, mockExecute: vi.fn(), mockFetch: vi.fn(), + mockResolveStartCandidates: vi.fn(), mockRunUploadStrategy: vi.fn(), + mockSelectBestTrigger: vi.fn(), terminalStoreState, workflowBlocks, workflowStoreState, @@ -133,12 +137,12 @@ vi.mock('@/lib/workflows/input-format', () => ({ vi.mock('@/lib/workflows/triggers/trigger-utils', () => ({ extractTriggerMockPayload: () => ({}), - selectBestTrigger: () => [], + selectBestTrigger: mockSelectBestTrigger, triggerNeedsMockPayload: () => false, })) vi.mock('@/lib/workflows/triggers/triggers', () => ({ - resolveStartCandidates: () => [], + resolveStartCandidates: mockResolveStartCandidates, StartBlockPath: { SPLIT_API: 'split-api', SPLIT_INPUT: 'split-input', @@ -344,6 +348,9 @@ async function drainStream(value: unknown): Promise { describe('useWorkflowExecution attachment uploads', () => { beforeEach(() => { vi.clearAllMocks() + executionStoreState.getCurrentExecutionId.mockReturnValue(null) + mockResolveStartCandidates.mockReturnValue([]) + mockSelectBestTrigger.mockReturnValue([]) vi.stubGlobal('fetch', mockFetch) mockRunUploadStrategy.mockRejectedValue( new DirectUploadErrorMock('Server signaled fallback to API upload', 'FALLBACK_REQUIRED') @@ -470,4 +477,143 @@ describe('useWorkflowExecution attachment uploads', () => { unmount() }) + + it('uses only projected live thinking without changing normal settle behavior', async () => { + mockExecute.mockImplementationOnce(async (options) => { + options.onExecutionId?.('execution-1') + await options.callbacks?.onStreamThinking?.({ + blockId: 'agent-1', + text: 'sk-resolved-secret', + display: { text: '{{OPENAI_API_KEY}}' }, + }) + await options.callbacks?.onStreamDone?.({ blockId: 'agent-1' }) + await options.callbacks?.onBlockCompleted?.({ + blockId: 'agent-1', + blockName: 'Agent 1', + blockType: 'agent', + executionOrder: 1, + output: { content: 'sk-resolved-secret' }, + display: { output: { content: '{{OPENAI_API_KEY}}' } }, + durationMs: 10, + startedAt: '2026-07-31T00:00:00.000Z', + endedAt: '2026-07-31T00:00:00.010Z', + }) + }) + + const { result, unmount } = renderWorkflowExecutionHook() + + await act(async () => { + const runResult = await result().handleRunWorkflow({ input: 'chat input' }) + await drainStream(runResult) + }) + + expect(terminalStoreState.updateConsole).toHaveBeenCalledWith( + 'agent-1', + expect.objectContaining({ agentStreamThinking: '{{OPENAI_API_KEY}}' }), + 'execution-1' + ) + expect(terminalStoreState.updateConsole).not.toHaveBeenCalledWith( + 'agent-1', + expect.objectContaining({ clearAgentStreamThinking: true }), + 'execution-1' + ) + expect(JSON.stringify(terminalStoreState.updateConsole.mock.calls)).not.toContain( + 'sk-resolved-secret' + ) + + unmount() + }) + + it('preserves legacy live thinking when no display projection field is sent', async () => { + mockExecute.mockImplementationOnce(async (options) => { + options.onExecutionId?.('execution-1') + await options.callbacks?.onStreamThinking?.({ + blockId: 'agent-1', + text: 'sk-resolved-secret', + }) + await options.callbacks?.onStreamDone?.({ blockId: 'agent-1' }) + }) + + const { result, unmount } = renderWorkflowExecutionHook() + + await act(async () => { + const runResult = await result().handleRunWorkflow({ input: 'chat input' }) + await drainStream(runResult) + }) + + expect(terminalStoreState.updateConsole).toHaveBeenCalledWith( + 'agent-1', + expect.objectContaining({ agentStreamThinking: 'sk-resolved-secret' }), + 'execution-1' + ) + + unmount() + }) + + it('clears live thinking when the server sends an empty display projection', async () => { + mockExecute.mockImplementationOnce(async (options) => { + options.onExecutionId?.('execution-1') + await options.callbacks?.onStreamThinking?.({ + blockId: 'agent-1', + text: 'sk-resolved-secret', + display: {}, + }) + }) + + const { result, unmount } = renderWorkflowExecutionHook() + + await act(async () => { + const runResult = await result().handleRunWorkflow({ input: 'chat input' }) + await drainStream(runResult) + }) + + expect(terminalStoreState.updateConsole).toHaveBeenCalledWith( + 'agent-1', + { clearAgentStreamThinking: true }, + 'execution-1' + ) + expect(JSON.stringify(terminalStoreState.updateConsole.mock.calls)).not.toContain( + 'sk-resolved-secret' + ) + + unmount() + }) + + it('keeps the trusted execution ID when storing a run-until-block snapshot', async () => { + const startCandidate = { + blockId: 'start', + block: workflowBlocks.start, + path: 'legacy-starter', + } + mockResolveStartCandidates.mockReturnValue([startCandidate]) + mockSelectBestTrigger.mockReturnValue([startCandidate]) + mockExecute.mockImplementationOnce(async (options) => { + const executionId = options.executionId as string + executionStoreState.getCurrentExecutionId.mockReturnValue(executionId) + options.onExecutionId?.(executionId) + await options.callbacks?.onExecutionCompleted?.({ + success: true, + output: {}, + duration: 10, + startTime: '2026-07-31T00:00:00.000Z', + endTime: '2026-07-31T00:00:00.010Z', + finalBlockLogs: [], + }) + }) + + const { result, unmount } = renderWorkflowExecutionHook() + + await act(async () => { + await result().handleRunUntilBlock('function-1', 'workflow-1') + }) + + const executionId = mockExecute.mock.calls[0]?.[0]?.executionId + expect(executionId).toEqual(expect.any(String)) + expect(executionStoreState.setLastExecutionSnapshot).toHaveBeenCalledWith( + 'workflow-1', + expect.objectContaining({ sourceExecutionId: executionId }) + ) + + unmount() + }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-workflow-execution.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-workflow-execution.ts index 7c708170f14..f1b1c129b5d 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-workflow-execution.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-workflow-execution.ts @@ -16,6 +16,7 @@ import { } from '@/components/agent-stream/tool-call-lifecycle' import { requestJson } from '@/lib/api/client/request' import { cancelWorkflowExecutionContract, workflowLogContract } from '@/lib/api/contracts/workflows' +import type { SecretSafeBlockLog } from '@/lib/logs/execution/display-types' import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans' import { processStreamingBlockLogs } from '@/lib/tokenization' import type { @@ -103,6 +104,20 @@ interface DebugValidationResult { const WORKFLOW_EXECUTION_FAILURE_MESSAGE = 'Workflow execution failed' +function getExecutionDisplayError(data: unknown): { + displayError?: string + hasDisplayProjection: boolean +} { + if (!data || typeof data !== 'object' || !Object.hasOwn(data, 'display')) { + return { hasDisplayProjection: false } + } + const display = (data as { display?: { error?: unknown } }).display + return { + hasDisplayProjection: true, + ...(typeof display?.error === 'string' ? { displayError: display.error } : {}), + } +} + async function persistExecutionPointerProgress( workflowId: string, executionId: string, @@ -261,6 +276,16 @@ function createAgentStreamChrome({ executionIdRef, updateConsole }: AgentStreamC ) } + const clearThinking = (blockId: string) => { + const timer = thinkingFlushTimers.get(blockId) + if (timer !== undefined) { + clearTimeout(timer) + thinkingFlushTimers.delete(blockId) + } + thinkingByBlock.delete(blockId) + updateConsole(blockId, { clearAgentStreamThinking: true }, executionIdRef.current) + } + const settleBlock = (blockId: string, status: 'success' | 'error' | 'cancelled') => { flushThinking(blockId) const map = toolCallsByBlock.get(blockId) @@ -288,8 +313,21 @@ function createAgentStreamChrome({ executionIdRef, updateConsole }: AgentStreamC } const onStreamThinking = (data: StreamThinkingData) => { + const display = ( + data as StreamThinkingData & { + display?: { text?: string; clearLiveDisplay?: true } + } + ).display + const hasDisplayProjection = Object.hasOwn(data, 'display') + const text = hasDisplayProjection ? display?.text : data.text + if (display?.clearLiveDisplay || (hasDisplayProjection && typeof text !== 'string')) { + clearThinking(data.blockId) + return + } + if (!text) return + const prev = thinkingByBlock.get(data.blockId) ?? '' - thinkingByBlock.set(data.blockId, prev + data.text) + thinkingByBlock.set(data.blockId, prev + text) if (!thinkingFlushTimers.has(data.blockId)) { thinkingFlushTimers.set( data.blockId, @@ -334,7 +372,14 @@ function createAgentStreamChrome({ executionIdRef, updateConsole }: AgentStreamC settleBlock(data.blockId, 'success') } - return { flushThinking, settleBlock, settleAll, onStreamThinking, onStreamTool, onStreamDone } + return { + flushThinking, + settleBlock, + settleAll, + onStreamThinking, + onStreamTool, + onStreamDone, + } } export function useWorkflowExecution() { @@ -464,10 +509,12 @@ export function useWorkflowExecution() { workflowId?: string executionId?: string error?: string + displayError?: string + hasDisplayProjection?: boolean durationMs?: number blockLogs: BlockLog[] isPreExecutionError?: boolean - finalBlockLogs?: BlockLog[] + finalBlockLogs?: SecretSafeBlockLog[] }) => { if (!params.workflowId) return sharedHandleExecutionErrorConsole( @@ -483,7 +530,7 @@ export function useWorkflowExecution() { workflowId?: string executionId?: string durationMs?: number - finalBlockLogs?: BlockLog[] + finalBlockLogs?: SecretSafeBlockLog[] }) => { if (!params.workflowId) return sharedHandleExecutionCancelledConsole( @@ -862,27 +909,7 @@ export function useWorkflowExecution() { streamedContent.set(id, chunks.join('')) } - // Update streamed content and apply tokenization if (result.logs) { - result.logs.forEach((log: BlockLog) => { - if (streamedContent.has(log.blockId)) { - // For console display, show the actual structured block output instead of formatted streaming content - // This ensures console logs match the block state structure - // Use replaceOutput to completely replace the output instead of merging - // Use the executionId from this execution context - useTerminalConsoleStore.getState().updateConsole( - log.blockId, - { - executionOrder: log.executionOrder, - replaceOutput: log.output, - success: true, - }, - executionId - ) - } - }) - - // Process all logs for streaming tokenization const processedCount = processStreamingBlockLogs(result.logs, streamedContent) logger.info(`Processed ${processedCount} blocks for streaming tokenization`) } @@ -1208,7 +1235,6 @@ export function useWorkflowExecution() { const activeBlockRefCounts = new Map() const streamedChunks = new Map() const agentStreamChrome = createAgentStreamChrome({ executionIdRef, updateConsole }) - const settleAgentStreamChrome = agentStreamChrome.settleBlock const settleAllAgentStreamChrome = agentStreamChrome.settleAll const accumulatedBlockLogs: BlockLog[] = [] const accumulatedBlockStates = new Map() @@ -1282,8 +1308,7 @@ export function useWorkflowExecution() { onBlockStarted: blockHandlers.onBlockStarted, onBlockCompleted: blockHandlers.onBlockCompleted, onBlockError: (data) => { - // Failures often skip stream:done — settle thinking/tool chrome here. - settleAgentStreamChrome(data.blockId, 'error') + agentStreamChrome.settleBlock(data.blockId, 'error') blockHandlers.onBlockError(data) }, onBlockChildWorkflowStarted: blockHandlers.onBlockChildWorkflowStarted, @@ -1388,6 +1413,7 @@ export function useWorkflowExecution() { decisions: existingSnapshot?.decisions || { router: {}, condition: {} }, completedLoops: existingSnapshot?.completedLoops || [], activeExecutionPath: Array.from(mergedExecutedBlocks), + sourceExecutionId: executionIdRef.current, } setLastExecutionSnapshot(activeWorkflowId, snapshot) logger.info('Merged execution snapshot after run-until-block', { @@ -1403,6 +1429,7 @@ export function useWorkflowExecution() { decisions: { router: {}, condition: {} }, completedLoops: [], activeExecutionPath: Array.from(executedBlockIds), + sourceExecutionId: executionIdRef.current, } setLastExecutionSnapshot(activeWorkflowId, snapshot) logger.info('Stored execution snapshot for run-from-block', { @@ -1508,6 +1535,7 @@ export function useWorkflowExecution() { workflowId: activeWorkflowId, executionId: executionIdRef.current, error: data.error, + ...getExecutionDisplayError(data), durationMs: data.duration, blockLogs: accumulatedBlockLogs, isPreExecutionError, @@ -1927,6 +1955,7 @@ export function useWorkflowExecution() { const effectiveSnapshot: SerializableExecutionState = isTriggerBlock ? emptySnapshot : snapshot || emptySnapshot + const sourceExecutionId = isTriggerBlock ? undefined : effectiveSnapshot.sourceExecutionId // Extract mock payload for trigger blocks let workflowInput: any @@ -2008,7 +2037,7 @@ export function useWorkflowExecution() { await executionStream.executeFromBlock({ workflowId, startBlockId: blockId, - sourceSnapshot: effectiveSnapshot, + ...(sourceExecutionId ? { sourceExecutionId } : { sourceSnapshot: effectiveSnapshot }), input: workflowInput, onExecutionId: (id) => { if (runFromBlockOwnerRef.current !== runOwnerId) return @@ -2031,7 +2060,6 @@ export function useWorkflowExecution() { onBlockStarted: blockHandlers.onBlockStarted, onBlockCompleted: blockHandlers.onBlockCompleted, onBlockError: (data) => { - // Failures often skip stream:done — settle thinking/tool chrome here. agentStreamChrome.settleBlock(data.blockId, 'error') blockHandlers.onBlockError(data) }, @@ -2065,6 +2093,7 @@ export function useWorkflowExecution() { const updatedSnapshot: SerializableExecutionState = { ...effectiveSnapshot, + sourceExecutionId: executionId, blockStates: mergedBlockStates, executedBlocks: Array.from(mergedExecutedBlocks), blockLogs: [...effectiveSnapshot.blockLogs, ...accumulatedBlockLogs], @@ -2116,6 +2145,7 @@ export function useWorkflowExecution() { workflowId, executionId, error: data.error, + ...getExecutionDisplayError(data), durationMs: data.duration, blockLogs: accumulatedBlockLogs, finalBlockLogs: data.finalBlockLogs, @@ -2473,6 +2503,7 @@ export function useWorkflowExecution() { workflowId: reconnectWorkflowId, executionId: capturedExecutionId, error: data.error, + ...getExecutionDisplayError(data), blockLogs: accumulatedBlockLogs, finalBlockLogs: data.finalBlockLogs, }) diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.integration.test.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.integration.test.ts index f3b3d7cde6c..642b6327c58 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.integration.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.integration.test.ts @@ -30,6 +30,49 @@ describe('reconcileFinalBlockLogs (real store)', () => { } as any) }) + it('replaces completed terminal content with the final server projection', () => { + const store = useTerminalConsoleStore.getState() + store.addConsole({ + workflowId: 'wf-1', + blockId: 'function-1', + blockName: 'Function 1', + blockType: 'function', + executionId: 'exec-1', + executionOrder: 1, + isRunning: false, + success: false, + input: { code: 'return resolved-secret-value-123' }, + output: { error: 'resolved-secret-value-123' }, + error: 'SyntaxError: resolved-secret-value-123', + agentStreamThinking: 'resolved-secret-value-123', + }) + + const startedAt = new Date().toISOString() + reconcileFinalBlockLogs(store.updateConsole, 'wf-1', 'exec-1', [ + { + blockId: 'function-1', + blockName: 'Function 1', + blockType: 'function', + executionOrder: 1, + success: false, + input: { code: 'return {{SECRET_NAME}}' }, + output: { error: '{{SECRET_NAME}}' }, + error: 'SyntaxError: {{SECRET_NAME}}', + clearLiveDisplay: true, + startedAt, + endedAt: startedAt, + durationMs: 1, + } as any, + ]) + + const entry = useTerminalConsoleStore.getState().getWorkflowEntries('wf-1')[0] + expect(entry.input).toEqual({ code: 'return {{SECRET_NAME}}' }) + expect(entry.output).toEqual({ error: '{{SECRET_NAME}}' }) + expect(entry.error).toBe('SyntaxError: {{SECRET_NAME}}') + expect(entry.agentStreamThinking).toBeUndefined() + expect(JSON.stringify(entry)).not.toContain('resolved-secret-value-123') + }) + it('actually flips a child-workflow inner block from running to success', () => { const store = useTerminalConsoleStore.getState() store.addConsole({ diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.test.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.test.ts index e7272f901cf..3b9739a99c3 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.test.ts @@ -5,6 +5,7 @@ import { resetTerminalConsoleMock, terminalConsoleMockFns } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' import { addExecutionErrorConsoleEntry, + addHttpErrorConsoleEntry, createBlockEventHandlers, handleExecutionErrorConsole, reconcileFinalBlockLogs, @@ -195,6 +196,171 @@ describe('workflow-execution-utils', () => { 'exec-1', ]) }) + + it('keeps raw completion data functional while writing only the display projection', async () => { + const accumulatedBlockLogs: BlockLog[] = [] + const accumulatedBlockStates = new Map() + const updateConsole = vi.fn() + const onBlockCompleteCallback = vi.fn().mockResolvedValue(undefined) + const handlers = createBlockEventHandlers( + { + workflowId: 'wf-1', + executionIdRef: { current: 'exec-1' }, + workflowEdges: [], + activeBlocksSet: new Set(), + activeBlockRefCounts: new Map(), + accumulatedBlockLogs, + accumulatedBlockStates, + executedBlockIds: new Set(), + includeStartConsoleEntry: true, + onBlockCompleteCallback, + }, + { + addConsole: vi.fn(), + updateConsole, + setActiveBlocks: vi.fn(), + setBlockRunStatus: vi.fn(), + setEdgeRunStatus: vi.fn(), + } + ) + + handlers.onBlockCompleted({ + blockId: 'fn-1', + blockName: 'Function 1', + blockType: 'function', + executionOrder: 1, + input: { code: 'return sk-resolved-secret' }, + output: { result: 'sk-resolved-secret' }, + display: { + input: { code: 'return {{OPENAI_API_KEY}}' }, + output: { result: '{{OPENAI_API_KEY}}' }, + }, + durationMs: 10, + startedAt: '2026-07-31T00:00:00.000Z', + endedAt: '2026-07-31T00:00:00.010Z', + } as any) + + expect(accumulatedBlockLogs[0]).toMatchObject({ + input: { code: 'return sk-resolved-secret' }, + output: { result: 'sk-resolved-secret' }, + }) + expect(accumulatedBlockStates.get('fn-1')?.output).toEqual({ + result: 'sk-resolved-secret', + }) + expect(onBlockCompleteCallback).toHaveBeenCalledWith('fn-1', { + result: 'sk-resolved-secret', + }) + expect(updateConsole).toHaveBeenCalledWith( + 'fn-1', + expect.objectContaining({ + input: { code: 'return {{OPENAI_API_KEY}}' }, + replaceOutput: { result: '{{OPENAI_API_KEY}}' }, + }), + 'exec-1' + ) + expect(updateConsole.mock.calls[0][1]).not.toHaveProperty('clearAgentStreamThinking') + expect(JSON.stringify(updateConsole.mock.calls)).not.toContain('sk-resolved-secret') + }) + + it('does not fall back to a raw block error when the display projection is empty', () => { + const accumulatedBlockLogs: BlockLog[] = [] + const accumulatedBlockStates = new Map() + const updateConsole = vi.fn() + const handlers = createBlockEventHandlers( + { + workflowId: 'wf-1', + executionIdRef: { current: 'exec-1' }, + workflowEdges: [], + activeBlocksSet: new Set(), + activeBlockRefCounts: new Map(), + accumulatedBlockLogs, + accumulatedBlockStates, + executedBlockIds: new Set(), + includeStartConsoleEntry: true, + }, + { + addConsole: vi.fn(), + updateConsole, + setActiveBlocks: vi.fn(), + setBlockRunStatus: vi.fn(), + setEdgeRunStatus: vi.fn(), + } + ) + + handlers.onBlockError({ + blockId: 'fn-1', + blockName: 'Function 1', + blockType: 'function', + executionOrder: 1, + input: { code: 'return sk-resolved-secret' }, + error: 'SyntaxError: sk-resolved-secret', + display: {}, + durationMs: 10, + startedAt: '2026-07-31T00:00:00.000Z', + endedAt: '2026-07-31T00:00:00.010Z', + }) + + expect(accumulatedBlockLogs[0]?.error).toBe('SyntaxError: sk-resolved-secret') + expect(accumulatedBlockStates.get('fn-1')?.output).toEqual({ + error: 'SyntaxError: sk-resolved-secret', + }) + expect(updateConsole).toHaveBeenCalledWith( + 'fn-1', + expect.objectContaining({ + input: {}, + replaceOutput: {}, + error: 'Block failed', + clearAgentStreamThinking: true, + }), + 'exec-1' + ) + expect(JSON.stringify(updateConsole.mock.calls)).not.toContain('sk-resolved-secret') + }) + + it('preserves legacy block error display when the server sends no projection', () => { + const updateConsole = vi.fn() + const handlers = createBlockEventHandlers( + { + workflowId: 'wf-1', + executionIdRef: { current: 'exec-1' }, + workflowEdges: [], + activeBlocksSet: new Set(), + activeBlockRefCounts: new Map(), + accumulatedBlockLogs: [], + accumulatedBlockStates: new Map(), + executedBlockIds: new Set(), + includeStartConsoleEntry: true, + }, + { + addConsole: vi.fn(), + updateConsole, + setActiveBlocks: vi.fn(), + setBlockRunStatus: vi.fn(), + setEdgeRunStatus: vi.fn(), + } + ) + + handlers.onBlockError({ + blockId: 'fn-1', + blockName: 'Function 1', + blockType: 'function', + executionOrder: 1, + input: { code: 'return ordinary-value' }, + error: 'SyntaxError: ordinary-value', + durationMs: 10, + startedAt: '2026-07-31T00:00:00.000Z', + endedAt: '2026-07-31T00:00:00.010Z', + }) + + expect(updateConsole).toHaveBeenCalledWith( + 'fn-1', + expect.objectContaining({ + input: { code: 'return ordinary-value' }, + error: 'SyntaxError: ordinary-value', + }), + 'exec-1' + ) + }) }) describe('addExecutionErrorConsoleEntry', () => { @@ -204,6 +370,7 @@ describe('workflow-execution-utils', () => { workflowId: 'wf-1', executionId: 'exec-1', error: 'Run failed', + displayError: 'Safe run failure', durationMs: 1234, blockLogs: [], }) @@ -212,7 +379,45 @@ describe('workflow-execution-utils', () => { const entry = addConsole.mock.calls[0][0] expect(entry.blockName).toBe('Run Error') expect(entry.blockType).toBe('error') - expect(entry.error).toBe('Run failed') + expect(entry.error).toBe('Safe run failure') + }) + + it('does not use the raw execution error when the server projection is empty', () => { + const addConsole = vi.fn() + addExecutionErrorConsoleEntry(addConsole, { + workflowId: 'wf-1', + executionId: 'exec-1', + error: 'SyntaxError: sk-resolved-secret', + hasDisplayProjection: true, + blockLogs: [], + }) + + expect(addConsole.mock.calls[0][0].error).toBe('Run failed') + expect(JSON.stringify(addConsole.mock.calls)).not.toContain('sk-resolved-secret') + }) + + it('preserves legacy execution errors when the server sends no projection', () => { + const addConsole = vi.fn() + addExecutionErrorConsoleEntry(addConsole, { + workflowId: 'wf-1', + executionId: 'exec-1', + error: 'Legacy run failure', + blockLogs: [], + }) + + expect(addConsole.mock.calls[0][0].error).toBe('Legacy run failure') + }) + + it('preserves HTTP error detail before SSE projection is available', () => { + const addConsole = vi.fn() + addHttpErrorConsoleEntry(addConsole, { + workflowId: 'wf-1', + executionId: 'exec-1', + error: 'Workflow is archived', + httpStatus: 409, + }) + + expect(addConsole.mock.calls[0][0].error).toBe('Workflow is archived') }) it('skips when blockLogs already contain a block-level error', () => { @@ -407,6 +612,74 @@ describe('workflow-execution-utils', () => { expect(updateConsole).not.toHaveBeenCalled() }) + it('reprojects completed content without deep-comparing authoritative finalBlockLogs', () => { + terminalConsoleMockFns.mockAddConsole({ + workflowId: 'wf-1', + blockId: 'fn-1', + blockName: 'Function', + blockType: 'function', + executionId: 'exec-1', + executionOrder: 1, + isRunning: false, + success: false, + input: { code: 'return sk-resolved-secret' }, + output: { error: 'sk-resolved-secret' }, + error: 'SyntaxError: sk-resolved-secret', + agentStreamThinking: 'sk-resolved-secret', + }) + + const updateConsole = vi.fn() + reconcileFinalBlockLogs(updateConsole, 'wf-1', 'exec-1', [ + makeLog({ + blockId: 'fn-1', + input: { code: 'return {{OPENAI_API_KEY}}' }, + output: { error: '{{OPENAI_API_KEY}}' }, + error: 'SyntaxError: {{OPENAI_API_KEY}}', + success: false, + }), + ]) + + expect(updateConsole).toHaveBeenCalledWith( + 'fn-1', + expect.objectContaining({ + input: { code: 'return {{OPENAI_API_KEY}}' }, + replaceOutput: { error: '{{OPENAI_API_KEY}}' }, + error: 'SyntaxError: {{OPENAI_API_KEY}}', + }), + 'exec-1' + ) + expect(updateConsole.mock.calls[0][1]).not.toHaveProperty('clearAgentStreamThinking') + expect(JSON.stringify(updateConsole.mock.calls)).not.toContain('sk-resolved-secret') + }) + + it('clears live content when the final projection is structural-only', () => { + terminalConsoleMockFns.mockAddConsole({ + workflowId: 'wf-1', + blockId: 'fn-1', + blockName: 'Function', + blockType: 'function', + executionId: 'exec-1', + executionOrder: 1, + isRunning: false, + success: false, + input: { code: 'return sk-resolved-secret' }, + output: { error: 'sk-resolved-secret' }, + error: 'SyntaxError: sk-resolved-secret', + }) + + const updateConsole = vi.fn() + reconcileFinalBlockLogs(updateConsole, 'wf-1', 'exec-1', [ + makeLog({ blockId: 'fn-1', success: false }), + ]) + + expect(updateConsole.mock.calls[0][1]).toMatchObject({ + input: {}, + replaceOutput: {}, + error: null, + clearAgentStreamThinking: true, + }) + }) + it('reconciles child workflow spans before running entries are swept to canceled', () => { terminalConsoleMockFns.mockAddConsole({ workflowId: 'wf-1', diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.ts index 782e00428f9..c344decafaf 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' +import type { SecretSafeBlockLog } from '@/lib/logs/execution/display-types' import type { TraceSpan } from '@/lib/logs/types' import type { BlockChildWorkflowStartedData, @@ -15,9 +16,6 @@ import { SSEEventHandlerError, SSEStreamInterruptedError, } from '@/hooks/use-execution-stream' - -const logger = createLogger('workflow-execution-utils') - import { useExecutionStore } from '@/stores/execution' import type { ConsoleEntry, ConsoleUpdate } from '@/stores/terminal' import { @@ -29,6 +27,11 @@ import { import { useWorkflowRegistry } from '@/stores/workflows/registry/store' import { useWorkflowStore } from '@/stores/workflows/workflow/store' +const logger = createLogger('workflow-execution-utils') +const BLOCK_FAILURE_DISPLAY_MESSAGE = 'Block failed' +const RUN_FAILURE_DISPLAY_MESSAGE = 'Run failed' +const VALIDATION_FAILURE_DISPLAY_MESSAGE = 'Workflow validation failed' + /** * Updates the active blocks set and ref counts for a single block. * Ref counting ensures a block stays active until all parallel branches for it complete. @@ -132,6 +135,46 @@ interface BlockEventHandlerDeps { type BlockChildWorkflowStartedUpdate = BlockChildWorkflowStartedData +interface BlockCompletedDisplayProjection { + input?: unknown + output?: unknown + clearLiveDisplay?: true +} + +interface BlockErrorDisplayProjection { + input?: unknown + error?: string + clearLiveDisplay?: true +} + +interface ExecutionErrorDisplayProjection { + present: boolean + error?: string +} + +function getBlockCompletedDisplay(data: BlockCompletedData): BlockCompletedDisplayProjection { + const event = data as BlockCompletedData & { display?: BlockCompletedDisplayProjection } + if (!Object.hasOwn(event, 'display')) { + return { input: data.input, output: data.output } + } + return event.display ?? {} +} + +function getBlockErrorDisplay(data: BlockErrorData): BlockErrorDisplayProjection { + const event = data as BlockErrorData & { display?: BlockErrorDisplayProjection } + if (!Object.hasOwn(event, 'display')) { + return { input: data.input, error: data.error } + } + return event.display ?? {} +} + +function getExecutionErrorDisplay(data: { error: string }): ExecutionErrorDisplayProjection { + const event = data as typeof data & { display?: Omit } + if (!Object.hasOwn(event, 'display')) return { present: false } + const error = event.display?.error + return { present: true, ...(typeof error === 'string' ? { error } : {}) } +} + /** * Creates block event handlers for SSE execution events. * Shared by the workflow execution hook and standalone execution utilities. @@ -299,19 +342,26 @@ export function createBlockEventHandlers( }) const updateConsoleEntry = (data: BlockCompletedData) => { + const display = getBlockCompletedDisplay(data) + const projectionOmittedContent = + display.clearLiveDisplay === true || + (Object.hasOwn(data, 'display') && + ((!Object.hasOwn(display, 'input') && data.input !== undefined) || + (!Object.hasOwn(display, 'output') && data.output !== undefined))) updateConsole( data.blockId, { blockName: data.blockName, blockType: data.blockType, executionOrder: data.executionOrder, - input: data.input || {}, - replaceOutput: data.output, + input: display.input ?? {}, + replaceOutput: (display.output ?? {}) as Record, success: true, durationMs: data.durationMs, startedAt: data.startedAt, endedAt: data.endedAt, isRunning: false, + ...(projectionOmittedContent ? { clearAgentStreamThinking: true } : {}), ...extractIterationFields(data), }, executionIdRef.current @@ -319,20 +369,27 @@ export function createBlockEventHandlers( } const updateConsoleErrorEntry = (data: BlockErrorData) => { + const display = getBlockErrorDisplay(data) + const projectionOmittedContent = + display.clearLiveDisplay === true || + (Object.hasOwn(data, 'display') && + ((!Object.hasOwn(display, 'input') && data.input !== undefined) || + (!Object.hasOwn(display, 'error') && data.error !== undefined))) updateConsole( data.blockId, { blockName: data.blockName, blockType: data.blockType, executionOrder: data.executionOrder, - input: data.input || {}, + input: display.input ?? {}, replaceOutput: {}, success: false, - error: data.error, + error: display.error || BLOCK_FAILURE_DISPLAY_MESSAGE, durationMs: data.durationMs, startedAt: data.startedAt, endedAt: data.endedAt, isRunning: false, + ...(projectionOmittedContent ? { clearAgentStreamThinking: true } : {}), ...extractIterationFields(data), }, executionIdRef.current @@ -472,16 +529,16 @@ interface ExecutionConsoleDeps { } /** - * Reconciles still-running console entries with the server's authoritative - * `finalBlockLogs` so that any block whose terminal `block:completed`/`block:error` - * SSE event was lost gets the correct success/error state instead of being - * swept to "canceled". + * Reconciles console entries with the server's authoritative, secret-safe + * `finalBlockLogs`. Reapplying running or content-bearing rows both recovers + * dropped terminal events and replaces an earlier live projection after late + * secret activation. */ export function reconcileFinalBlockLogs( updateConsole: UpdateConsoleFn, workflowId: string, executionId: string | undefined, - finalBlockLogs: BlockLog[] | undefined + finalBlockLogs: SecretSafeBlockLog[] | undefined ): void { if (!finalBlockLogs?.length || !executionId) return for (const log of finalBlockLogs) { @@ -491,8 +548,17 @@ export function reconcileFinalBlockLogs( entry.executionId === executionId && entry.executionOrder === log.executionOrder const matchingEntry = entries.find(matchesFinalLog) - const runningEntry = entries.find((entry) => matchesFinalLog(entry) && entry.isRunning) - if (runningEntry) { + const hasExistingContent = + matchingEntry?.input !== undefined || + matchingEntry?.output !== undefined || + matchingEntry?.error !== undefined || + matchingEntry?.agentStreamThinking !== undefined + if (matchingEntry && (matchingEntry.isRunning || hasExistingContent)) { + const projectionOmittedContent = + log.clearLiveDisplay === true || + (log.input === undefined && matchingEntry.input !== undefined) || + (log.output === undefined && matchingEntry.output !== undefined) || + (log.error === undefined && matchingEntry.error !== undefined) updateConsole( log.blockId, { @@ -500,14 +566,15 @@ export function reconcileFinalBlockLogs( blockName: log.blockName, blockType: log.blockType, replaceOutput: (log.output ?? {}) as Record, - ...(log.input ? { input: log.input } : {}), + input: log.input ?? {}, success: log.success, - ...(log.error ? { error: log.error } : {}), + error: log.error ?? null, durationMs: log.durationMs, startedAt: log.startedAt, endedAt: log.endedAt, isRunning: false, isCanceled: false, + ...(projectionOmittedContent ? { clearAgentStreamThinking: true } : {}), }, executionId ) @@ -549,19 +616,26 @@ function reconcileChildTraceSpans( ? findConsoleEntryForSpan(workflowId, executionId, childWorkflowInstanceId, span) : undefined if (span.blockId) { - const errorMessage = normalizeSpanError(span.output?.error) + const errorMessage = normalizeSpanError(span.errorMessage ?? span.output?.error) + const projectionOmittedContent = matchingEntry + ? (span.input === undefined && matchingEntry.input !== undefined) || + (span.output === undefined && matchingEntry.output !== undefined) || + (errorMessage === undefined && matchingEntry.error !== undefined) + : false updateConsole( span.blockId, { ...spanConsoleIdentity(span, childWorkflowInstanceId), + input: span.input ?? {}, replaceOutput: (span.output ?? {}) as Record, success: span.status !== 'error', - ...(errorMessage !== undefined ? { error: errorMessage } : {}), + error: errorMessage ?? null, durationMs: span.duration, startedAt: span.startTime, endedAt: span.endTime, isRunning: false, isCanceled: false, + ...(projectionOmittedContent ? { clearAgentStreamThinking: true } : {}), }, executionId ) @@ -670,12 +744,17 @@ export function buildExecutionTiming(durationMs?: number): ExecutionTimingFields interface ExecutionErrorConsoleParams { workflowId: string executionId?: string + /** Raw runtime error used only for functional classification and result propagation. */ error?: string + /** Server-projected error text safe for terminal display. */ + displayError?: string + /** Distinguishes an intentionally empty projection from a legacy event with no projection. */ + hasDisplayProjection?: boolean durationMs?: number blockLogs: BlockLog[] isPreExecutionError?: boolean /** Server's authoritative per-block terminal states, used to reconcile lost SSE events. */ - finalBlockLogs?: BlockLog[] + finalBlockLogs?: SecretSafeBlockLog[] } /** @@ -702,8 +781,14 @@ export function addExecutionErrorConsoleEntry( const isPreExecutionError = params.isPreExecutionError ?? false if (!isPreExecutionError && hasBlockError) return - const errorMessage = params.error || 'Run failed' - const isTimeout = errorMessage.toLowerCase().includes('timed out') + const hasDisplayProjection = params.hasDisplayProjection ?? params.displayError !== undefined + const fallbackMessage = isPreExecutionError + ? VALIDATION_FAILURE_DISPLAY_MESSAGE + : RUN_FAILURE_DISPLAY_MESSAGE + const errorMessage = hasDisplayProjection + ? params.displayError || fallbackMessage + : params.error || fallbackMessage + const isTimeout = (params.error ?? params.displayError ?? '').toLowerCase().includes('timed out') const timing = buildExecutionTiming(params.durationMs) addConsole({ @@ -784,7 +869,7 @@ interface CancelledConsoleParams { executionId?: string durationMs?: number /** Server's authoritative per-block terminal states, used to reconcile lost SSE events. */ - finalBlockLogs?: BlockLog[] + finalBlockLogs?: SecretSafeBlockLog[] } /** @@ -1060,6 +1145,7 @@ export async function executeWorkflowWithFullLogging( if (!isCurrentExecution()) return executionFinished = true const errorMessage = data.error || 'Run failed' + const display = getExecutionErrorDisplay(data) executionResult = { success: false, output: {}, @@ -1074,6 +1160,8 @@ export async function executeWorkflowWithFullLogging( workflowId: wfId, executionId: executionIdRef.current, error: errorMessage, + displayError: display.error, + hasDisplayProjection: display.present, durationMs: data.duration || 0, blockLogs: accumulatedBlockLogs, isPreExecutionError: accumulatedBlockLogs.length === 0, diff --git a/apps/sim/background/async-preprocessing-correlation.test.ts b/apps/sim/background/async-preprocessing-correlation.test.ts index e86d3bd66de..40832442576 100644 --- a/apps/sim/background/async-preprocessing-correlation.test.ts +++ b/apps/sim/background/async-preprocessing-correlation.test.ts @@ -9,6 +9,7 @@ import { executionPreprocessingMockFns, LoggingSessionMock, loggingSessionMock, + loggingSessionMockFns, resetDbChainMock, workflowsPersistenceUtilsMock, workflowsPersistenceUtilsMockFns, @@ -16,13 +17,21 @@ import { import { beforeEach, describe, expect, it, vi } from 'vitest' import { ADMISSION_ERROR_CODE } from '@/lib/core/admission/transient-failure' -const { mockTask, mockExecuteWorkflowCore, mockGetScheduleTimeValues, mockGetSubBlockValue } = - vi.hoisted(() => ({ - mockTask: vi.fn((config) => config), - mockExecuteWorkflowCore: vi.fn(), - mockGetScheduleTimeValues: vi.fn(), - mockGetSubBlockValue: vi.fn(), - })) +const { + mockTask, + mockExecuteWorkflowCore, + mockWasExecutionFinalizedByCore, + mockHasExecutionResult, + mockGetScheduleTimeValues, + mockGetSubBlockValue, +} = vi.hoisted(() => ({ + mockTask: vi.fn((config) => config), + mockExecuteWorkflowCore: vi.fn(), + mockWasExecutionFinalizedByCore: vi.fn(), + mockHasExecutionResult: vi.fn(), + mockGetScheduleTimeValues: vi.fn(), + mockGetSubBlockValue: vi.fn(), +})) const mockPreprocessExecution = executionPreprocessingMockFns.mockPreprocessExecution const mockLoadDeployedWorkflowState = workflowsPersistenceUtilsMockFns.mockLoadDeployedWorkflowState @@ -55,7 +64,7 @@ vi.mock('@/lib/logs/execution/trace-spans/trace-spans', () => ({ vi.mock('@/lib/workflows/executor/execution-core', () => ({ executeWorkflowCore: mockExecuteWorkflowCore, - wasExecutionFinalizedByCore: vi.fn().mockReturnValue(false), + wasExecutionFinalizedByCore: mockWasExecutionFinalizedByCore, })) vi.mock('@/lib/workflows/executor/human-in-the-loop-manager', () => ({ @@ -78,7 +87,7 @@ vi.mock('@/executor/execution/snapshot', () => ({ })) vi.mock('@/executor/utils/errors', () => ({ - hasExecutionResult: vi.fn().mockReturnValue(false), + hasExecutionResult: mockHasExecutionResult, })) import { executeScheduleJob } from './schedule-execution' @@ -100,6 +109,8 @@ const billingAttribution = { describe('async preprocessing correlation threading', () => { beforeEach(() => { vi.clearAllMocks() + mockWasExecutionFinalizedByCore.mockReturnValue(false) + mockHasExecutionResult.mockReturnValue(false) resetDbChainMock() dbChainMockFns.limit.mockResolvedValue([ { @@ -166,6 +177,84 @@ describe('async preprocessing correlation threading', () => { ) }) + it('preserves a core-finalized execution error for task failure semantics', async () => { + const rawError = Object.assign(new Error('Function 1 failed with activated-secret-value'), { + executionResult: { + success: false, + output: { error: 'Function failed' }, + logs: [], + }, + }) + mockPreprocessExecution.mockResolvedValueOnce({ + success: true, + actorUserId: 'actor-1', + workflowRecord: { + id: 'workflow-1', + userId: 'owner-1', + workspaceId: 'workspace-1', + variables: {}, + }, + billingAttribution, + executionTimeout: {}, + }) + mockExecuteWorkflowCore.mockRejectedValueOnce(rawError) + mockHasExecutionResult.mockImplementation((error) => error === rawError) + mockWasExecutionFinalizedByCore.mockReturnValue(true) + + await expect( + executeWorkflowJob({ + workflowId: 'workflow-1', + userId: 'actor-1', + workspaceId: 'workspace-1', + billingAttribution, + triggerType: 'api', + executionId: 'execution-finalized', + requestId: 'request-finalized', + }) + ).rejects.toBe(rawError) + + expect(loggingSessionMockFns.mockWaitForPostExecution).not.toHaveBeenCalled() + expect(mockWasExecutionFinalizedByCore).toHaveBeenCalledWith(rawError, 'execution-finalized') + expect(loggingSessionMockFns.mockSafeCompleteWithError).not.toHaveBeenCalled() + }) + + it('persists and rethrows the original unfinalized execution error', async () => { + const rawError = new Error('Function 1 failed with activated-secret-value') + mockPreprocessExecution.mockResolvedValueOnce({ + success: true, + actorUserId: 'actor-1', + workflowRecord: { + id: 'workflow-1', + userId: 'owner-1', + workspaceId: 'workspace-1', + variables: {}, + }, + billingAttribution, + executionTimeout: {}, + }) + mockExecuteWorkflowCore.mockRejectedValueOnce(rawError) + + await expect( + executeWorkflowJob({ + workflowId: 'workflow-1', + userId: 'actor-1', + workspaceId: 'workspace-1', + billingAttribution, + triggerType: 'api', + executionId: 'execution-fault', + requestId: 'request-fault', + }) + ).rejects.toBe(rawError) + + expect(loggingSessionMockFns.mockSafeCompleteWithError).toHaveBeenCalledWith( + expect.objectContaining({ + error: expect.objectContaining({ + message: 'Function 1 failed with activated-secret-value', + }), + }) + ) + }) + it('does not pre-start schedule logging before core execution', async () => { mockPreprocessExecution.mockResolvedValueOnce({ success: true, diff --git a/apps/sim/background/resume-execution.test.ts b/apps/sim/background/resume-execution.test.ts new file mode 100644 index 00000000000..fe3dacc1439 --- /dev/null +++ b/apps/sim/background/resume-execution.test.ts @@ -0,0 +1,94 @@ +/** + * @vitest-environment node + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockTask, + mockGetPausedExecutionById, + mockStartResumeExecution, + mockFindCellContextByExecutionId, + mockSnapshotFromJson, +} = vi.hoisted(() => ({ + mockTask: vi.fn((config) => config), + mockGetPausedExecutionById: vi.fn(), + mockStartResumeExecution: vi.fn(), + mockFindCellContextByExecutionId: vi.fn(), + mockSnapshotFromJson: vi.fn(), +})) + +vi.mock('@trigger.dev/sdk', () => ({ task: mockTask })) + +vi.mock('@/lib/billing/core/billing-attribution', () => ({ + assertBillingAttributionSnapshot: vi.fn((value) => value), +})) + +vi.mock('@/lib/table/cascade-lock', () => ({ withCascadeLock: vi.fn() })) +vi.mock('@/lib/table/deps', () => ({ isExecCancelled: vi.fn(() => false) })) + +vi.mock('@/lib/table/workflow-columns', () => ({ + findCellContextByExecutionId: mockFindCellContextByExecutionId, +})) + +vi.mock('@/lib/workflows/executor/human-in-the-loop-manager', () => ({ + PauseResumeManager: { + getPausedExecutionById: mockGetPausedExecutionById, + startResumeExecution: mockStartResumeExecution, + }, +})) + +vi.mock('@/executor/execution/snapshot', () => ({ + ExecutionSnapshot: { fromJSON: mockSnapshotFromJson }, +})) + +import { executeResumeJob, type ResumeExecutionPayload } from '@/background/resume-execution' + +const payload: ResumeExecutionPayload = { + resumeEntryId: 'resume-entry-1', + resumeExecutionId: 'resume-execution-1', + pausedExecutionId: 'paused-execution-1', + contextId: 'context-1', + resumeInput: {}, + userId: 'user-1', + workflowId: 'workflow-1', + parentExecutionId: 'parent-execution-1', +} + +describe('executeResumeJob terminal errors', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetPausedExecutionById.mockResolvedValue({ + executionSnapshot: { snapshot: {} }, + }) + mockSnapshotFromJson.mockReturnValue({ + metadata: { + billingAttribution: { + actorUserId: 'user-1', + workspaceId: 'workspace-1', + }, + }, + }) + mockFindCellContextByExecutionId.mockResolvedValue(null) + }) + + it('rethrows the original core-finalized resume error', async () => { + const rawError = Object.assign(new Error('Agent tool exposed activated-secret-value'), { + executionResult: { + success: false, + output: { error: 'Agent tool failed' }, + logs: [], + }, + }) + mockStartResumeExecution.mockRejectedValue(rawError) + + await expect(executeResumeJob(payload)).rejects.toBe(rawError) + }) + + it('rethrows the original genuine resume fault', async () => { + const rawError = new Error('MCP setup exposed activated-secret-value') + mockStartResumeExecution.mockRejectedValue(rawError) + + await expect(executeResumeJob(payload)).rejects.toBe(rawError) + }) +}) diff --git a/apps/sim/background/schedule-execution.test.ts b/apps/sim/background/schedule-execution.test.ts index 8ebb5cdd8c4..771da4f2f98 100644 --- a/apps/sim/background/schedule-execution.test.ts +++ b/apps/sim/background/schedule-execution.test.ts @@ -21,7 +21,13 @@ vi.mock('@/lib/workflows/schedules/disable-notifications', () => ({ // does not re-export. Widen it rather than rewriting the source's imports. vi.mock('@sim/db', () => ({ ...databaseMock, ...schemaMock })) -import { applyScheduleFailureUpdate, releaseScheduleLock } from '@/background/schedule-execution' +import { + applyScheduleFailureUpdate, + readScheduledMothershipErrorResponse, + readScheduledMothershipJsonResponse, + releaseScheduleLock, +} from '@/background/schedule-execution' +import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' const BASE = { scheduleId: 'schedule-1', @@ -99,3 +105,71 @@ describe('releaseScheduleLock', () => { expect(notifyMock).not.toHaveBeenCalled() }) }) + +describe('scheduled Mothership response handling', () => { + it('parses JSON responses', async () => { + const response = new Response(JSON.stringify({ content: 'ok' }), { + headers: { 'content-type': 'application/json' }, + }) + + await expect(readScheduledMothershipJsonResponse(response)).resolves.toEqual({ content: 'ok' }) + }) + + it('preserves staging acceptance for responses above the generic tool cap', async () => { + const response = new Response(JSON.stringify({ content: 'unchanged' }), { + headers: { + 'content-length': String(10 * 1024 * 1024 + 1), + 'content-type': 'application/json', + }, + }) + + await expect(readScheduledMothershipJsonResponse(response)).resolves.toEqual({ + content: 'unchanged', + }) + }) + + it('does not include malformed response content in parse failures', async () => { + const response = new Response('secret-bearing-non-json') + const error = await readScheduledMothershipJsonResponse(response).catch((caught) => caught) + + expect(error).toBeInstanceOf(Error) + expect(error.message).toBe('Sim execution returned an invalid response') + expect(error.message).not.toContain('secret-bearing-non-json') + }) + + it('preserves the functional error body when no private metadata is present', async () => { + const registry = { + markIncomplete: vi.fn(), + importProvenance: vi.fn(), + } as unknown as ResolvedSecretTraceRegistry + const response = new Response('secret-bearing-error-body', { status: 500 }) + + const message = await readScheduledMothershipErrorResponse(response, registry) + + expect(message).toBe('secret-bearing-error-body') + expect(registry.markIncomplete).toHaveBeenCalledTimes(1) + }) + + it('strips private provenance metadata without replacing the provider error', async () => { + const registry = { + markIncomplete: vi.fn(), + importProvenance: vi.fn().mockReturnValue(true), + } as unknown as ResolvedSecretTraceRegistry + const response = new Response( + JSON.stringify({ + error: 'provider error detail', + __resolvedSecretTraceProvenance: { version: 1, complete: true, entries: [] }, + }), + { + status: 500, + headers: { 'x-sim-private-tool-metadata': 'resolved-secret-provenance-v1' }, + } + ) + + const message = await readScheduledMothershipErrorResponse(response, registry) + + expect(JSON.parse(message)).toEqual({ error: 'provider error detail' }) + expect(message).not.toContain('__resolvedSecretTraceProvenance') + expect(registry.importProvenance).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/background/schedule-execution.ts b/apps/sim/background/schedule-execution.ts index b684f180dc7..bd537a81359 100644 --- a/apps/sim/background/schedule-execution.ts +++ b/apps/sim/background/schedule-execution.ts @@ -9,6 +9,7 @@ import { import { createLogger, runWithRequestContext } from '@sim/logger' import { describeError, toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' +import { isPlainRecord } from '@sim/utils/object' import { task } from '@trigger.dev/sdk' import { Cron } from 'croner' import { and, eq, isNull, ne, type SQL, sql } from 'drizzle-orm' @@ -32,8 +33,16 @@ import { } from '@/lib/core/execution-limits' import type { DbOrTx } from '@/lib/db/types' import { preprocessExecution } from '@/lib/execution/preprocessing' +import { + PRIVATE_TOOL_METADATA_REQUEST_HEADER, + RESOLVED_SECRET_PROVENANCE_FIELD, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, + responseHasPrivateToolMetadata, +} from '@/lib/execution/private-tool-metadata' import { LoggingSession } from '@/lib/logs/execution/logging-session' +import { projectTraceSpansForSecrets } from '@/lib/logs/execution/trace-secret-projection' import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans' +import type { TraceSpan } from '@/lib/logs/types' import { cleanupExecutionBase64Cache } from '@/lib/uploads/utils/user-file-base64.server' import { executeWorkflowCore, @@ -61,6 +70,7 @@ import { ExecutionSnapshot } from '@/executor/execution/snapshot' import type { ExecutionMetadata } from '@/executor/execution/types' import { hasExecutionResult } from '@/executor/utils/errors' import { buildAPIUrl, buildAuthHeaders } from '@/executor/utils/http' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' import { MAX_CONSECUTIVE_FAILURES } from '@/triggers/constants' const logger = createLogger('ScheduleExecution') @@ -660,6 +670,7 @@ async function runWorkflowExecution({ stackTrace: error instanceof Error ? error.stack : undefined, }, traceSpans, + executionState: executionResult?.executionState, }) throw error @@ -1207,14 +1218,94 @@ function buildJobPrompt(jobRecord: { return parts.join('\n') } +async function consumeJobTraceProvenance( + response: Response, + payload: Record, + registry: ResolvedSecretTraceRegistry +): Promise { + const hasProvenance = Object.hasOwn(payload, RESOLVED_SECRET_PROVENANCE_FIELD) + const provenance = payload[RESOLVED_SECRET_PROVENANCE_FIELD] + delete payload[RESOLVED_SECRET_PROVENANCE_FIELD] + + if ( + !responseHasPrivateToolMetadata(response.headers, RESOLVED_SECRET_PROVENANCE_METADATA_V1) || + !hasProvenance + ) { + registry.markIncomplete() + return false + } + + return registry.importProvenance(provenance, { trusted: true }) +} + +/** Reads and validates a scheduled Mothership JSON response without changing its size contract. */ +export async function readScheduledMothershipJsonResponse( + response: Response +): Promise> { + let payload: unknown + + try { + payload = await response.json() + } catch { + throw new Error('Sim execution returned an invalid response') + } + + if (!isPlainRecord(payload)) { + throw new Error('Sim execution returned an invalid response') + } + return payload +} + +/** Reads the functional error body and strips private provenance metadata when present. */ +export async function readScheduledMothershipErrorResponse( + response: Response, + registry: ResolvedSecretTraceRegistry +): Promise { + const responseText = await response.text() + const hasPrivateMarker = responseHasPrivateToolMetadata( + response.headers, + RESOLVED_SECRET_PROVENANCE_METADATA_V1 + ) + const mayContainPrivateProvenance = responseText.includes(`"${RESOLVED_SECRET_PROVENANCE_FIELD}"`) + + if (!hasPrivateMarker && !mayContainPrivateProvenance) { + registry.markIncomplete() + return responseText + } + + let payload: unknown + + try { + payload = JSON.parse(responseText) + } catch { + registry.markIncomplete() + return responseText + } + + if (!isPlainRecord(payload)) { + registry.markIncomplete() + return responseText + } + + const hadPrivateProvenance = Object.hasOwn(payload, RESOLVED_SECRET_PROVENANCE_FIELD) + try { + await consumeJobTraceProvenance(response, payload, registry) + } catch { + registry.markIncomplete() + } + return hadPrivateProvenance ? JSON.stringify(payload) : responseText +} + async function createJobLogEntry(params: { scheduleId: string workspaceId: string + userId: string jobTitle: string | null startTime: Date endTime: Date durationMs: number success: boolean + resolvedSecretTraceRegistry: ResolvedSecretTraceRegistry responseBody?: Record errorMessage?: string }): Promise { @@ -1222,14 +1313,17 @@ async function createJobLogEntry(params: { const { scheduleId, workspaceId, + userId, jobTitle, startTime, endTime, durationMs, success, + resolvedSecretTraceRegistry, responseBody, } = params const name = jobTitle || 'Sim Job' + const executionId = generateId() const toolCallsList = (responseBody?.toolCalls || []).map((tc: Record) => ({ name: tc.name, @@ -1246,7 +1340,7 @@ async function createJobLogEntry(params: { status: tc.error ? 'error' : 'success', })) - const traceSpan = { + const traceSpan: TraceSpan = { id: generateId(), name, type: 'mothership', @@ -1263,12 +1357,20 @@ async function createJobLogEntry(params: { cost: responseBody?.cost || undefined, tokens: responseBody?.tokens || undefined, } + const [projectedTraceSpan] = await projectTraceSpansForSecrets([traceSpan], { + registry: resolvedSecretTraceRegistry, + store: { + workspaceId, + executionId, + userId, + }, + }) await db.insert(jobExecutionLogs).values({ id: generateId(), scheduleId, workspaceId, - executionId: generateId(), + executionId, level: success ? 'info' : 'error', status: success ? 'completed' : 'failed', trigger: 'mothership', @@ -1277,8 +1379,11 @@ async function createJobLogEntry(params: { totalDurationMs: durationMs, executionData: { enhanced: true, - traceSpans: [traceSpan], + traceSpans: [projectedTraceSpan], finalOutput: responseBody?.content ? { content: responseBody.content } : undefined, + executionState: { + resolvedSecretTraceProvenance: resolvedSecretTraceRegistry.exportProvenance(), + }, trigger: { type: 'mothership', source: name, @@ -1369,6 +1474,10 @@ export async function executeJobInline(payload: JobExecutionPayload) { } const promptText = buildJobPrompt(jobRecord) + const resolvedSecretTraceRegistry = new ResolvedSecretTraceRegistry([], { + userId: jobRecord.sourceUserId, + workspaceId: jobRecord.sourceWorkspaceId, + }) try { const billingAttribution = await resolveBillingAttribution({ @@ -1382,6 +1491,7 @@ export async function executeJobInline(payload: JobExecutionPayload) { const url = buildAPIUrl('/api/mothership/execute') const headers = await buildAuthHeaders(jobRecord.sourceUserId) headers[BILLING_ATTRIBUTION_HEADER] = serializeBillingAttributionHeader(billingAttribution) + headers[PRIVATE_TOOL_METADATA_REQUEST_HEADER] = RESOLVED_SECRET_PROVENANCE_METADATA_V1 const body = { messages: [{ role: 'user', content: promptText }], @@ -1404,7 +1514,11 @@ export async function executeJobInline(payload: JobExecutionPayload) { }) if (!response.ok) { - const errorText = await response.text().catch(() => { + const errorText = await readScheduledMothershipErrorResponse( + response, + resolvedSecretTraceRegistry + ).catch(() => { + resolvedSecretTraceRegistry.markIncomplete() if (timeoutController.isTimedOut()) { throw new Error(getTimeoutErrorMessage(null, timeoutController.timeoutMs)) } @@ -1416,11 +1530,13 @@ export async function executeJobInline(payload: JobExecutionPayload) { await createJobLogEntry({ scheduleId: payload.scheduleId, workspaceId: jobRecord.sourceWorkspaceId, + userId: jobRecord.sourceUserId, jobTitle: jobRecord.jobTitle, startTime, endTime, durationMs, success: false, + resolvedSecretTraceRegistry, errorMessage: errorText, }) @@ -1430,10 +1546,17 @@ export async function executeJobInline(payload: JobExecutionPayload) { let responseBody: Record = {} let wasCompletedByTool = false try { - responseBody = await response.json() + const payload = await readScheduledMothershipJsonResponse(response) + responseBody = payload + try { + await consumeJobTraceProvenance(response, payload, resolvedSecretTraceRegistry) + } catch { + resolvedSecretTraceRegistry.markIncomplete() + } const toolCalls = responseBody?.toolCalls as Array<{ name?: string }> | undefined wasCompletedByTool = toolCalls?.some((tc) => tc.name === 'complete_scheduled_task') ?? false } catch { + resolvedSecretTraceRegistry.markIncomplete() if (timeoutController.isTimedOut()) { throw new Error(getTimeoutErrorMessage(null, timeoutController.timeoutMs)) } @@ -1444,11 +1567,13 @@ export async function executeJobInline(payload: JobExecutionPayload) { await createJobLogEntry({ scheduleId: payload.scheduleId, workspaceId: jobRecord.sourceWorkspaceId, + userId: jobRecord.sourceUserId, jobTitle: jobRecord.jobTitle, startTime, endTime, durationMs, success: true, + resolvedSecretTraceRegistry, responseBody, }) diff --git a/apps/sim/background/webhook-execution.test.ts b/apps/sim/background/webhook-execution.test.ts index fa44602b09b..a72f241d854 100644 --- a/apps/sim/background/webhook-execution.test.ts +++ b/apps/sim/background/webhook-execution.test.ts @@ -4,30 +4,33 @@ import { dbChainMockFns, + environmentUtilsMockFns, executionPreprocessingMock, executionPreprocessingMockFns, + LoggingSessionMock, loggingSessionMock, loggingSessionMockFns, + resetEnvironmentUtilsMock, } from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { mockResolveWebhookRecordProviderConfig, mockExecuteWorkflowCore, mockWasExecutionFinalizedByCore, - mockRecordException, - mockGetActiveSpan, mockExecuteWithIdempotency, mockReleaseExecutionSlot, mockLoadDeploymentVersionState, + mockGetProviderHandler, + mockSetResolvedSecretTraceRegistry, } = vi.hoisted(() => ({ mockResolveWebhookRecordProviderConfig: vi.fn(), mockExecuteWorkflowCore: vi.fn(), mockWasExecutionFinalizedByCore: vi.fn(), - mockRecordException: vi.fn(), - mockGetActiveSpan: vi.fn(), mockExecuteWithIdempotency: vi.fn(), mockReleaseExecutionSlot: vi.fn(), + mockGetProviderHandler: vi.fn(() => ({})), + mockSetResolvedSecretTraceRegistry: vi.fn(), mockLoadDeploymentVersionState: vi.fn( async (_workflowId: string, deploymentVersionId: string) => ({ blocks: {}, @@ -39,9 +42,9 @@ const { ), })) -vi.mock('@opentelemetry/api', () => ({ - trace: { getActiveSpan: mockGetActiveSpan }, -})) +const mockGetPersonalAndWorkspaceEnv = environmentUtilsMockFns.mockGetPersonalAndWorkspaceEnv + +afterAll(resetEnvironmentUtilsMock) vi.mock('@/lib/execution/preprocessing', () => executionPreprocessingMock) vi.mock('@/lib/logs/execution/logging-session', () => loggingSessionMock) @@ -77,9 +80,7 @@ vi.mock('@/lib/workflows/persistence/utils', () => ({ loadWorkflowDeploymentVersionState: mockLoadDeploymentVersionState, })) -vi.mock('@/lib/webhooks/providers', () => ({ - getProviderHandler: vi.fn(() => ({})), -})) +vi.mock('@/lib/webhooks/providers', () => ({ getProviderHandler: mockGetProviderHandler })) vi.mock('@/lib/logs/execution/trace-spans/trace-spans', () => ({ buildTraceSpans: vi.fn(() => ({ traceSpans: [] })), @@ -208,6 +209,17 @@ describe('executeWebhookJob fault vs error handling', () => { beforeEach(() => { vi.clearAllMocks() + LoggingSessionMock.mockImplementation(function LoggingSession() { + return { + safeStart: loggingSessionMockFns.mockSafeStart, + safeComplete: loggingSessionMockFns.mockSafeComplete, + safeCompleteWithError: loggingSessionMockFns.mockSafeCompleteWithError, + waitForPostExecution: loggingSessionMockFns.mockWaitForPostExecution, + markAsFailed: loggingSessionMockFns.mockMarkAsFailed, + setResolvedSecretTraceRegistry: mockSetResolvedSecretTraceRegistry, + } + }) + mockGetProviderHandler.mockReturnValue({}) mockExecuteWithIdempotency.mockImplementation( (_provider: string, _key: string, operation: () => Promise) => operation() ) @@ -225,8 +237,15 @@ describe('executeWebhookJob fault vs error handling', () => { executionTimeout: { async: 120_000 }, }) mockResolveWebhookRecordProviderConfig.mockImplementation(async (record) => record) + mockGetPersonalAndWorkspaceEnv.mockResolvedValue({ + personalEncrypted: {}, + workspaceEncrypted: {}, + personalDecrypted: {}, + workspaceDecrypted: {}, + conflicts: [], + decryptionFailures: [], + }) dbChainMockFns.limit.mockResolvedValue([{ id: 'webhook-1' }]) - mockGetActiveSpan.mockReturnValue({ recordException: mockRecordException }) }) it('completes the run (does not throw) when the failure was finalized by core', async () => { @@ -246,17 +265,14 @@ describe('executeWebhookJob fault vs error handling', () => { expect(loggingSessionMockFns.mockWaitForPostExecution).toHaveBeenCalled() // User/workflow errors are already recorded by core — the catch must not re-log them. expect(loggingSessionMockFns.mockSafeCompleteWithError).not.toHaveBeenCalled() - // The error is still recorded on the run span so it stays visible in traces. - expect(mockRecordException).toHaveBeenCalledWith( - expect.objectContaining({ message: 'Gmail 2 is missing required fields: Label' }) - ) }) it('faults the run (re-throws) when the failure was not finalized by core', async () => { - mockExecuteWorkflowCore.mockRejectedValue(new Error('Workflow state not found')) + const rawError = new Error('Workflow state not found') + mockExecuteWorkflowCore.mockRejectedValue(rawError) mockWasExecutionFinalizedByCore.mockReturnValue(false) - await expect(executeWebhookJob(payload)).rejects.toThrow('Workflow state not found') + await expect(executeWebhookJob(payload)).rejects.toBe(rawError) // waitForPostExecution must run on every path so the finalized-by-core signal is always reliable. expect(loggingSessionMockFns.mockWaitForPostExecution).toHaveBeenCalled() // Pipeline/infra errors are recorded here before re-throwing to fault the trigger.dev run. @@ -291,6 +307,92 @@ describe('executeWebhookJob fault vs error handling', () => { ) }) + it('passes encrypted webhook resolution provenance into workflow execution', async () => { + mockGetPersonalAndWorkspaceEnv.mockResolvedValue({ + personalEncrypted: { WEBHOOK_SECRET: 'personal-ciphertext' }, + workspaceEncrypted: { WEBHOOK_SECRET: 'workspace-ciphertext' }, + personalDecrypted: { WEBHOOK_SECRET: 'personal-value' }, + workspaceDecrypted: { WEBHOOK_SECRET: 'workspace-value' }, + conflicts: ['WEBHOOK_SECRET'], + decryptionFailures: [], + }) + mockResolveWebhookRecordProviderConfig.mockImplementation( + async (record, _userId, _workspaceId, options) => { + options.onResolved('WEBHOOK_SECRET', options.envVars.WEBHOOK_SECRET) + return record + } + ) + mockExecuteWorkflowCore.mockResolvedValue({ + success: true, + status: 'completed', + output: {}, + logs: [], + executionState: { + blockStates: {}, + executedBlocks: [], + blockLogs: [], + decisions: {}, + completedLoops: [], + activeExecutionPath: [], + }, + }) + + await executeWebhookJob(payload) + + expect(mockResolveWebhookRecordProviderConfig).toHaveBeenCalledWith( + { id: 'webhook-1' }, + 'user-1', + 'workspace-1', + expect.objectContaining({ + envVars: { WEBHOOK_SECRET: 'workspace-value' }, + onResolved: expect.any(Function), + }) + ) + expect(mockExecuteWorkflowCore).toHaveBeenCalledWith( + expect.objectContaining({ + trustedInitialResolvedSecretTraceProvenance: { + version: 1, + complete: true, + entries: [{ name: 'WEBHOOK_SECRET', encryptedValue: 'workspace-ciphertext' }], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + }, + }) + ) + expect(mockSetResolvedSecretTraceRegistry).toHaveBeenCalledOnce() + }) + + it('installs provenance before a post-resolution webhook setup failure', async () => { + const rawMessage = 'Webhook handler exposed activated-secret-value' + const rawError = new Error(rawMessage) + mockGetPersonalAndWorkspaceEnv.mockResolvedValue({ + personalEncrypted: {}, + workspaceEncrypted: { WEBHOOK_SECRET: 'workspace-ciphertext' }, + personalDecrypted: {}, + workspaceDecrypted: { WEBHOOK_SECRET: 'activated-secret-value' }, + conflicts: [], + decryptionFailures: [], + }) + mockResolveWebhookRecordProviderConfig.mockImplementation( + async (record, _userId, _workspaceId, options) => { + options.onResolved('WEBHOOK_SECRET', options.envVars.WEBHOOK_SECRET) + return record + } + ) + mockGetProviderHandler.mockReturnValue({ + formatInput: vi.fn().mockRejectedValue(rawError), + }) + + await expect(executeWebhookJob(payload)).rejects.toBe(rawError) + + expect(mockSetResolvedSecretTraceRegistry).toHaveBeenCalledOnce() + expect(loggingSessionMockFns.mockSafeCompleteWithError).toHaveBeenCalledWith( + expect.objectContaining({ + error: expect.objectContaining({ message: rawMessage }), + }) + ) + expect(mockExecuteWorkflowCore).not.toHaveBeenCalled() + }) + it('acknowledges and skips queued webhook work after the workflow is undeployed', async () => { executionPreprocessingMockFns.mockPreprocessExecution.mockResolvedValueOnce({ success: true, diff --git a/apps/sim/background/webhook-execution.ts b/apps/sim/background/webhook-execution.ts index 2ffb0dcaab8..82f1518a78c 100644 --- a/apps/sim/background/webhook-execution.ts +++ b/apps/sim/background/webhook-execution.ts @@ -1,4 +1,3 @@ -import { trace } from '@opentelemetry/api' import { db } from '@sim/db' import { account, webhook } from '@sim/db/schema' import { createLogger, runWithRequestContext } from '@sim/logger' @@ -15,6 +14,7 @@ import { import type { AsyncExecutionCorrelation } from '@/lib/core/async-jobs/types' import { createTimeoutAbortController, getTimeoutErrorMessage } from '@/lib/core/execution-limits' import { IdempotencyService, webhookIdempotency } from '@/lib/core/idempotency' +import { getPersonalAndWorkspaceEnv } from '@/lib/environment/utils' import { preprocessExecution } from '@/lib/execution/preprocessing' import { LoggingSession } from '@/lib/logs/execution/logging-session' import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans' @@ -22,7 +22,10 @@ import { type WebhookAttachment, WebhookAttachmentProcessor, } from '@/lib/webhooks/attachment-processor' -import { resolveWebhookRecordProviderConfig } from '@/lib/webhooks/env-resolver' +import { + resolveWebhookRecordProviderConfig, + type WebhookEnvResolutionOptions, +} from '@/lib/webhooks/env-resolver' import { getProviderHandler } from '@/lib/webhooks/providers' import { executeWorkflowCore, @@ -40,6 +43,7 @@ import { ExecutionSnapshot } from '@/executor/execution/snapshot' import type { ExecutionMetadata } from '@/executor/execution/types' import type { ExecutionResult } from '@/executor/types' import { hasExecutionResult } from '@/executor/utils/errors' +import { createResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' import { safeAssign } from '@/tools/safe-assign' import { getTrigger, isTriggerValid } from '@/triggers' @@ -316,10 +320,13 @@ export async function resolveWebhookExecutionProviderConfig< webhookRecord: T, provider: string, userId: string, - workspaceId?: string + workspaceId?: string, + options?: WebhookEnvResolutionOptions ): Promise }> { try { - return await resolveWebhookRecordProviderConfig(webhookRecord, userId, workspaceId) + return options + ? await resolveWebhookRecordProviderConfig(webhookRecord, userId, workspaceId, options) + : await resolveWebhookRecordProviderConfig(webhookRecord, userId, workspaceId) } catch (error) { const errorMessage = toError(error).message throw new Error( @@ -489,11 +496,29 @@ async function executeWebhookJobInternal( throw new Error(`Webhook record not found: ${payload.webhookId}`) } + const secretEnvironment = await getPersonalAndWorkspaceEnv(workflowRecord.userId, workspaceId) + const resolvedSecretTraceRegistry = await createResolvedSecretTraceRegistry({ + personalEncrypted: secretEnvironment.personalEncrypted, + workspaceEncrypted: secretEnvironment.workspaceEncrypted, + personalDecrypted: secretEnvironment.personalDecrypted, + workspaceDecrypted: secretEnvironment.workspaceDecrypted, + decryptionFailures: secretEnvironment.decryptionFailures, + scope: { userId: workflowRecord.userId, workspaceId }, + }) + loggingSession.setResolvedSecretTraceRegistry(resolvedSecretTraceRegistry) + const secretEnvVars = { + ...secretEnvironment.personalDecrypted, + ...secretEnvironment.workspaceDecrypted, + } const resolvedWebhookRecord = await resolveWebhookExecutionProviderConfig( webhookRecord, payload.provider, workflowRecord.userId, - workspaceId + workspaceId, + { + envVars: secretEnvVars, + onResolved: (name, value) => resolvedSecretTraceRegistry.recordResolved(name, value), + } ) if (handler.formatInput) { @@ -665,6 +690,7 @@ async function executeWebhookJobInternal( snapshot, callbacks: {}, loggingSession, + trustedInitialResolvedSecretTraceProvenance: resolvedSecretTraceRegistry.exportProvenance(), includeFileBase64: false, base64MaxBytes: undefined, abortSignal: timeoutController.signal, @@ -712,10 +738,6 @@ async function executeWebhookJobInternal( // not a trigger.dev job fault — complete the run normally so we don't fire a false alert. Errors // that were not finalized came from the webhook pipeline itself, so we re-throw to fault below. if (wasExecutionFinalizedByCore(error, executionId)) { - // Record the exception on the run span so it stays visible in traces without - // marking the span as ERROR — that status is what faults the trigger.dev run. - trace.getActiveSpan()?.recordException(toError(error)) - return { success: false, workflowId: payload.workflowId, @@ -757,6 +779,7 @@ async function executeWebhookJobInternal( stackTrace: errorStack, }, traceSpans, + executionState: executionResult.executionState, }) } catch (loggingError) { logger.error(`[${requestId}] Failed to complete logging session`, loggingError) diff --git a/apps/sim/background/workflow-execution.ts b/apps/sim/background/workflow-execution.ts index cce7d55dd3e..cf7a6b09994 100644 --- a/apps/sim/background/workflow-execution.ts +++ b/apps/sim/background/workflow-execution.ts @@ -219,6 +219,7 @@ export async function executeWorkflowJob(payload: WorkflowExecutionPayload) { stackTrace: error instanceof Error ? error.stack : undefined, }, traceSpans, + executionState: executionResult?.executionState, }) throw error diff --git a/apps/sim/executor/execution/block-executor.test.ts b/apps/sim/executor/execution/block-executor.test.ts index b6fcb7a2f97..e267183e144 100644 --- a/apps/sim/executor/execution/block-executor.test.ts +++ b/apps/sim/executor/execution/block-executor.test.ts @@ -4,15 +4,14 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { clearLargeValueCacheForTests } from '@/lib/execution/payloads/cache' import { isLargeArrayManifest } from '@/lib/execution/payloads/large-array-manifest-metadata' +import { projectTraceSpansForSecrets } from '@/lib/logs/execution/trace-secret-projection' import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans' -import { calculateStreamingCost } from '@/lib/tokenization' import { BlockType } from '@/executor/constants' import type { DAGNode } from '@/executor/dag/builder' import { BlockExecutor } from '@/executor/execution/block-executor' -import { serializePauseSnapshot } from '@/executor/execution/snapshot-serializer' import { ExecutionState } from '@/executor/execution/state' -import type { ContextExtensions } from '@/executor/execution/types' -import type { BlockHandler, ExecutionContext, ExecutionResult } from '@/executor/types' +import type { BlockHandler, ExecutionContext } from '@/executor/types' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' import { VariableResolver } from '@/executor/variables/resolver' import type { SerializedBlock, SerializedWorkflow } from '@/serializer/types' @@ -79,44 +78,6 @@ function createNode(block: SerializedBlock): DAGNode { } } -function createExecutorForTest( - block: SerializedBlock, - state: ExecutionState, - handler: BlockHandler, - extensions: Partial = {} -): BlockExecutor { - const workflow: SerializedWorkflow = { - version: '1', - blocks: [block], - connections: [], - loops: {}, - parallels: {}, - } - const resolver = new VariableResolver(workflow, {}, state) - - return new BlockExecutor( - [handler], - resolver, - { - workspaceId: 'workspace-1', - executionId: 'execution-1', - userId: 'user-1', - metadata: { - requestId: 'request-1', - executionId: 'execution-1', - workflowId: 'workflow-1', - workspaceId: 'workspace-1', - userId: 'user-1', - triggerType: 'manual', - useDraftState: false, - startTime: new Date().toISOString(), - }, - ...extensions, - }, - state - ) -} - describe('BlockExecutor', () => { beforeEach(() => { vi.clearAllMocks() @@ -124,304 +85,6 @@ describe('BlockExecutor', () => { mockUploadFile.mockImplementation(async ({ customKey }) => ({ key: customKey })) }) - it.each([ - { label: 'provider time segments', includeTimeSegments: true }, - { label: 'fallback tool calls', includeTimeSegments: false }, - ])( - 'sanitizes agent trace I/O while preserving runtime values with $label', - async ({ includeTimeSegments }) => { - const secret = 'trace-secret/value' - const unreferencedValue = 'us-east-1' - const block: SerializedBlock = { - ...createBlock(), - id: 'agent-block-1', - metadata: { id: BlockType.AGENT, name: 'Agent' }, - config: { - tool: BlockType.AGENT, - params: { - prompt: 'Use {{TRACE_SECRET}}', - }, - }, - } - const state = new ExecutionState() - const execute = vi.fn(async (_ctx, _block, inputs) => { - expect(inputs.prompt).toBe(`Use ${secret}`) - return { - content: `Echoed ${secret}`, - region: unreferencedValue, - providerTiming: { - startTime: '2024-01-01T10:00:00.000Z', - endTime: '2024-01-01T10:00:02.000Z', - duration: 2000, - ...(includeTimeSegments && { - timeSegments: [ - { - type: 'model' as const, - name: 'Model', - startTime: 1704103200000, - endTime: 1704103201000, - duration: 1000, - assistantContent: `Calling with ${secret}`, - toolCalls: [ - { - id: 'call-1', - name: 'lookup', - arguments: { query: secret }, - }, - ], - }, - { - type: 'tool' as const, - name: 'lookup', - startTime: 1704103201000, - endTime: 1704103202000, - duration: 1000, - }, - ], - }), - }, - toolCalls: { - list: [ - { - name: 'lookup', - arguments: { query: secret }, - result: { echoed: secret }, - duration: 1000, - }, - ], - count: 1, - }, - childTraceSpans: [ - { - id: 'child-1', - name: 'Child', - type: 'function', - duration: 1, - startTime: '2024-01-01T10:00:00.000Z', - endTime: '2024-01-01T10:00:00.001Z', - input: { query: secret }, - output: { echoed: secret }, - }, - ], - } - }) - const handler: BlockHandler = { - canHandle: () => true, - execute, - } - const onBlockComplete = vi.fn(async () => {}) - const executor = createExecutorForTest(block, state, handler, { onBlockComplete }) - const ctx = createContext(state) - ctx.environmentVariables = { - TRACE_SECRET: secret, - UNREFERENCED_REGION: unreferencedValue, - } - - const output = await executor.execute(ctx, createNode(block), block) - - expect(output.content).toBe(`Echoed ${secret}`) - expect(output.toolCalls.list[0].arguments.query).toBe(secret) - expect(state.getBlockOutput(block.id)?.content).toBe(`Echoed ${secret}`) - - await vi.waitFor(() => { - expect(onBlockComplete).toHaveBeenCalled() - }) - - const serializedLog = JSON.stringify(ctx.blockLogs[0]) - const serializedCallback = JSON.stringify(onBlockComplete.mock.calls[0]) - expect(serializedLog).not.toContain(secret) - expect(serializedLog).toContain('{{TRACE_SECRET}}') - expect(serializedLog).toContain(unreferencedValue) - expect(serializedCallback).not.toContain(secret) - expect(serializedCallback).toContain('{{TRACE_SECRET}}') - - const { traceSpans } = buildTraceSpans({ - success: true, - output: {}, - logs: ctx.blockLogs, - } as ExecutionResult) - const serializedSpans = JSON.stringify(traceSpans) - expect(serializedSpans).not.toContain(secret) - expect(serializedSpans).toContain('{{TRACE_SECRET}}') - } - ) - - it('sanitizes failed logs and callbacks while preserving runtime errors', async () => { - const secret = 'failure-secret' - const block: SerializedBlock = { - ...createBlock(), - config: { - tool: BlockType.FUNCTION, - params: { - code: 'throw new Error("{{TRACE_SECRET}}")', - }, - }, - } - const state = new ExecutionState() - const handler: BlockHandler = { - canHandle: () => true, - execute: async (_ctx, _block, inputs) => { - expect(inputs.code).toContain(secret) - throw new Error(`Execution failed with ${secret}`) - }, - } - const onBlockComplete = vi.fn(async () => {}) - const executor = createExecutorForTest(block, state, handler, { onBlockComplete }) - const ctx = createContext(state) - ctx.environmentVariables = { TRACE_SECRET: secret } - - await expect(executor.execute(ctx, createNode(block), block)).rejects.toThrow(secret) - - expect(state.getBlockOutput(block.id)?.error).toContain(secret) - expect(ctx.blockLogs[0].error).toBe('Execution failed with {{TRACE_SECRET}}') - expect(JSON.stringify(ctx.blockLogs[0])).not.toContain(secret) - await vi.waitFor(() => { - expect(onBlockComplete).toHaveBeenCalled() - }) - expect(JSON.stringify(onBlockComplete.mock.calls[0])).not.toContain(secret) - }) - - it('sanitizes error-handler logs without changing the handled runtime output', async () => { - const secret = 'handled-secret' - const block: SerializedBlock = { - ...createBlock(), - config: { - tool: BlockType.FUNCTION, - params: { - code: 'throw new Error("{{TRACE_SECRET}}")', - }, - }, - } - const state = new ExecutionState() - const handler: BlockHandler = { - canHandle: () => true, - execute: async () => { - throw new Error(`Handled ${secret}`) - }, - } - const executor = createExecutorForTest(block, state, handler) - const infoSpy = vi.spyOn( - ( - executor as unknown as { - execLogger: { info: (message: string, metadata?: Record) => void } - } - ).execLogger, - 'info' - ) - const ctx = createContext(state) - ctx.environmentVariables = { TRACE_SECRET: secret } - const node = createNode(block) - node.outgoingEdges.set('error-edge', { - id: 'error-edge', - source: block.id, - target: 'error-handler', - sourceHandle: 'error', - targetHandle: 'target', - }) - - const output = await executor.execute(ctx, node, block) - - expect(output.error).toBe(`Handled ${secret}`) - expect(state.getBlockOutput(block.id)?.error).toBe(`Handled ${secret}`) - expect(ctx.blockLogs[0].errorHandled).toBe(true) - expect(JSON.stringify(ctx.blockLogs[0])).not.toContain(secret) - expect(JSON.stringify(ctx.blockLogs[0])).toContain('{{TRACE_SECRET}}') - expect(infoSpy).toHaveBeenCalledWith( - 'Block has error port - returning error output instead of throwing', - expect.objectContaining({ error: 'Handled {{TRACE_SECRET}}' }) - ) - }) - - it('sanitizes soft-abort agent inputs while leaving execution resolution unchanged', async () => { - const secret = 'abort-secret' - const block: SerializedBlock = { - ...createBlock(), - id: 'agent-block-1', - metadata: { id: BlockType.AGENT, name: 'Agent' }, - config: { - tool: BlockType.AGENT, - params: { - prompt: 'Use {{TRACE_SECRET}}', - }, - }, - } - const state = new ExecutionState() - const abortController = new AbortController() - const handler: BlockHandler = { - canHandle: () => true, - execute: async (_ctx, _block, inputs) => { - expect(inputs.prompt).toBe(`Use ${secret}`) - abortController.abort('user') - throw new DOMException(`Stopped ${secret}`, 'AbortError') - }, - } - const onBlockComplete = vi.fn(async () => {}) - const executor = createExecutorForTest(block, state, handler, { onBlockComplete }) - const ctx = createContext(state) - ctx.environmentVariables = { TRACE_SECRET: secret } - ctx.abortSignal = abortController.signal - - const output = await executor.execute(ctx, createNode(block), block) - - expect(output).toEqual({ content: '' }) - expect(ctx.blockLogs[0].success).toBe(true) - expect(JSON.stringify(ctx.blockLogs[0])).not.toContain(secret) - expect(JSON.stringify(ctx.blockLogs[0])).toContain('{{TRACE_SECRET}}') - await vi.waitFor(() => { - expect(onBlockComplete).toHaveBeenCalled() - }) - expect(JSON.stringify(onBlockComplete.mock.calls[0])).not.toContain(secret) - }) - - it('keeps snapshot state executable and re-resolves current values on retry', async () => { - const firstSecret = 'first-runtime-secret' - const secondSecret = 'second-runtime-secret' - const block: SerializedBlock = { - ...createBlock(), - config: { - tool: BlockType.FUNCTION, - params: { - code: 'return "{{TRACE_SECRET}}"', - }, - }, - } - const state = new ExecutionState() - const receivedRuntimeCode: string[] = [] - const handler: BlockHandler = { - canHandle: () => true, - execute: async (_ctx, _block, inputs) => { - receivedRuntimeCode.push(inputs.code) - return { result: inputs.code } - }, - } - const executor = createExecutorForTest(block, state, handler) - const firstContext = createContext(state) - firstContext.environmentVariables = { TRACE_SECRET: firstSecret } - - await executor.execute(firstContext, createNode(block), block) - const snapshot = JSON.parse(serializePauseSnapshot(firstContext, ['next-block']).snapshot) as { - state: { - blockStates: Record - blockLogs: ExecutionContext['blockLogs'] - } - } - - expect(snapshot.state.blockStates[block.id].output.result).toContain(firstSecret) - expect(JSON.stringify(snapshot.state.blockLogs)).not.toContain(firstSecret) - expect(JSON.stringify(snapshot.state.blockLogs)).toContain('{{TRACE_SECRET}}') - - const resumedContext = createContext(state) - resumedContext.blockLogs.push(...snapshot.state.blockLogs) - resumedContext.environmentVariables = { TRACE_SECRET: secondSecret } - - await executor.execute(resumedContext, createNode(block), block) - - expect(receivedRuntimeCode).toEqual([`return "${firstSecret}"`, `return "${secondSecret}"`]) - expect(state.getBlockOutput(block.id)?.result).toContain(secondSecret) - expect(JSON.stringify(resumedContext.blockLogs)).not.toContain(firstSecret) - expect(JSON.stringify(resumedContext.blockLogs)).not.toContain(secondSecret) - }) - it('persists function output arrays as manifests in execution state', async () => { const block = createBlock() const workflow: SerializedWorkflow = { @@ -776,23 +439,125 @@ describe('BlockExecutor', () => { expect(output?.error).toBeTruthy() expect(output).not.toEqual({ content: '' }) }) + + it('projects a resolved secret out of Function syntax-error TraceSpans only', async () => { + const secret = 'function-secret-literal-7f3a91' + const block = createBlock() + block.metadata.name = 'Function 1' + block.config.params = { + code: 'return {{OPENAI_API_KEY}}', + language: 'javascript', + } + const workflow: SerializedWorkflow = { + version: '1', + blocks: [block], + connections: [], + loops: {}, + parallels: {}, + } + const state = new ExecutionState() + const registry = new ResolvedSecretTraceRegistry([ + { + name: 'OPENAI_API_KEY', + plaintext: secret, + encryptedValue: 'encrypted-openai-api-key', + }, + ]) + const resolver = new VariableResolver(workflow, {}, state) + const syntaxError = `Syntax Error: Line 1: \`return ${secret}\` - Invalid or unexpected token` + const handler: BlockHandler = { + canHandle: () => true, + execute: async (_ctx, _block, inputs) => { + expect(inputs.code).toBe(`return ${secret}`) + throw new Error(syntaxError) + }, + } + const executor = new BlockExecutor( + [handler], + resolver, + { + workspaceId: 'workspace-1', + executionId: 'execution-1', + userId: 'user-1', + metadata: { + requestId: 'request-1', + executionId: 'execution-1', + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + userId: 'user-1', + triggerType: 'manual', + useDraftState: false, + startTime: new Date().toISOString(), + }, + }, + state + ) + const ctx = createContext(state) + ctx.environmentVariables = { OPENAI_API_KEY: secret } + ctx.resolvedSecretTraceRegistry = registry + + await expect(executor.execute(ctx, createNode(block), block)).rejects.toThrow( + `Function 1: ${syntaxError}` + ) + + expect(registry.getActiveMatches()).toEqual([ + { plaintext: secret, replacement: '{{OPENAI_API_KEY}}' }, + ]) + expect(state.getBlockOutput(block.id)).toEqual({ error: syntaxError }) + expect(ctx.blockLogs[0]).toMatchObject({ + input: { code: `return ${secret}` }, + output: { error: syntaxError }, + error: syntaxError, + }) + + const rawLogs = structuredClone(ctx.blockLogs) + const { traceSpans: rawTraceSpans } = buildTraceSpans({ + success: false, + output: { error: syntaxError }, + error: `Function 1: ${syntaxError}`, + logs: ctx.blockLogs, + }) + const rawTraceSnapshot = structuredClone(rawTraceSpans) + const projectedTraceSpans = await projectTraceSpansForSecrets(rawTraceSpans, { + registry, + store: { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + userId: 'user-1', + }, + }) + + expect(ctx.blockLogs).toEqual(rawLogs) + expect(rawTraceSpans).toEqual(rawTraceSnapshot) + expect(projectedTraceSpans).toEqual([ + expect.objectContaining({ + name: 'Function 1', + input: expect.objectContaining({ code: 'return {{OPENAI_API_KEY}}' }), + output: { + error: 'Syntax Error: Line 1: `return {{OPENAI_API_KEY}}` - Invalid or unexpected token', + }, + }), + ]) + expect(JSON.stringify(projectedTraceSpans)).not.toContain(secret) + }) }) describe('BlockExecutor streaming pump', () => { - function createAgentBlock(params: Record = {}): SerializedBlock { + function createAgentBlock(): SerializedBlock { return { id: 'agent-block-1', metadata: { id: BlockType.AGENT, name: 'Agent' }, position: { x: 0, y: 0 }, - config: { tool: BlockType.AGENT, params }, + config: { tool: BlockType.AGENT, params: {} }, inputs: {}, outputs: {}, enabled: true, } } - function createExecutor(handler: BlockHandler, params: Record = {}) { - const block = createAgentBlock(params) + function createExecutor(handler: BlockHandler) { + const block = createAgentBlock() const workflow: SerializedWorkflow = { version: '1', blocks: [block], @@ -955,51 +720,6 @@ describe('BlockExecutor streaming pump', () => { expect(state.getBlockOutput(block.id)?.content).toBe('offline answer') }) - it('estimates missing streaming usage from resolved input before sanitizing logs', async () => { - const secret = `sk-${'resolved-secret-'.repeat(20)}` - const params = { prompt: '{{OPENAI_API_KEY}}', model: 'gpt-4o' } - const handler: BlockHandler = { - canHandle: () => true, - execute: async (_ctx, _block, resolvedInputs) => ({ - stream: new ReadableStream({ - start(controller) { - controller.enqueue(new TextEncoder().encode('streamed answer')) - controller.close() - }, - }), - execution: { - success: true, - output: { content: '', model: 'gpt-4o' }, - logs: [], - metadata: { - startTime: new Date().toISOString(), - endTime: new Date().toISOString(), - duration: 1, - }, - }, - }), - } - const { executor, block, state } = createExecutor(handler, params) - const ctx = createContext(state) - ctx.environmentVariables = { OPENAI_API_KEY: secret } - - await executor.execute(ctx, createNode(block), block) - - const expected = calculateStreamingCost( - 'gpt-4o', - JSON.stringify({ prompt: secret, model: 'gpt-4o' }), - 'streamed answer' - ) - expect(state.getBlockOutput(block.id)?.tokens).toEqual(expected.tokens) - expect(state.getBlockOutput(block.id)?.cost).toEqual(expected.cost) - expect(ctx.blockLogs[0].input).toEqual({ - prompt: '{{OPENAI_API_KEY}}', - model: 'gpt-4o', - }) - expect(ctx.blockLogs[0].output?.tokens).toEqual(expected.tokens) - expect(JSON.stringify(ctx.blockLogs[0])).not.toContain(secret) - }) - it('throws on mid-stream provider error (no truncated success)', async () => { const handler = createAgentEventsStreamingHandler({ failAfterText: 'partial', @@ -1061,20 +781,16 @@ describe('BlockExecutor streaming pump', () => { }) it('fails on timeout but keeps drained answer text in block output', async () => { - const secret = 'partial-stream-secret' const abortController = new AbortController() const handler = createAgentEventsStreamingHandler({ events: [ - { type: 'text_delta', text: `partial ${secret} before timeout`, turn: 'final' }, + { type: 'text_delta', text: 'partial before timeout', turn: 'final' }, { type: 'thinking_delta', text: 'more' }, ], }) - const { executor, block, state } = createExecutor(handler, { - prompt: '{{TRACE_SECRET}}', - }) + const { executor, block, state } = createExecutor(handler) const ctx = createContext(state) - ctx.environmentVariables = { TRACE_SECRET: secret } ctx.abortSignal = abortController.signal ctx.onStream = async (streamingExec) => { streamingExec.subscribe?.({ onEvent: async () => {} }) @@ -1096,9 +812,7 @@ describe('BlockExecutor streaming pump', () => { const output = state.getBlockOutput(block.id) expect(output?.error).toBeTruthy() - expect(output?.content).toBe(`partial ${secret} before timeout`) - expect(JSON.stringify(ctx.blockLogs[0])).not.toContain(secret) - expect(ctx.blockLogs[0].output?.content).toBe('partial {{TRACE_SECRET}} before timeout') + expect(output?.content).toBe('partial before timeout') }) it('with PII redaction: no live forward and strips thinking from traces', async () => { diff --git a/apps/sim/executor/execution/block-executor.ts b/apps/sim/executor/execution/block-executor.ts index 22108c8c862..105eb517863 100644 --- a/apps/sim/executor/execution/block-executor.ts +++ b/apps/sim/executor/execution/block-executor.ts @@ -7,7 +7,6 @@ import { getBaseUrl } from '@/lib/core/utils/urls' import { compactExecutionPayload } from '@/lib/execution/payloads/serializer' import { redactLargeValueRefsInValue } from '@/lib/logs/execution/pii-large-values' import { redactObjectStrings } from '@/lib/logs/execution/pii-redaction' -import { processStreamingBlockLog } from '@/lib/tokenization' import { containsUserFileWithMetadata, hydrateUserFilesWithBase64, @@ -45,10 +44,6 @@ import { type StreamingExecution, } from '@/executor/types' import { streamingResponseFormatProcessor } from '@/executor/utils' -import { - createEnvironmentSecretSanitizer, - type EnvironmentSecretSanitizer, -} from '@/executor/utils/environment-secret-sanitizer' import { buildBlockExecutionError, normalizeError } from '@/executor/utils/errors' import { buildUnifiedParentIterations, @@ -106,10 +101,6 @@ export class BlockExecutor { const blockType = block.metadata?.id ?? '' const isSentinel = isSentinelBlockType(blockType) - const sanitizeEnvironmentSecrets = createEnvironmentSecretSanitizer( - block.config, - ctx.environmentVariables - ) // Capture startedAt and startTime at the same synchronous instant so // blockLog.startedAt and performance.now()-derived durationMs share a @@ -169,11 +160,7 @@ export class BlockExecutor { } if (blockLog) { - blockLog.input = this.sanitizeInputsForLog( - inputsForLog, - sanitizeEnvironmentSecrets, - block.metadata?.id - ) + blockLog.input = this.sanitizeInputsForLog(inputsForLog, block.metadata?.id) } } catch (error) { cleanupSelfReference?.() @@ -187,7 +174,6 @@ export class BlockExecutor { blockLog, inputsForLog, isSentinel, - sanitizeEnvironmentSecrets, 'input_resolution' ) } @@ -292,13 +278,9 @@ export class BlockExecutor { blockLog.endedAt = endedAt blockLog.durationMs = duration blockLog.success = true - blockLog.output = this.sanitizeOutputForLog( - block, - normalizedOutput, - sanitizeEnvironmentSecrets - ) + blockLog.output = filterOutputForLog(block.metadata?.id || '', normalizedOutput, { block }) if (normalizedOutput.childTraceSpans && Array.isArray(normalizedOutput.childTraceSpans)) { - blockLog.childTraceSpans = sanitizeEnvironmentSecrets(normalizedOutput.childTraceSpans) + blockLog.childTraceSpans = normalizedOutput.childTraceSpans } } @@ -310,17 +292,15 @@ export class BlockExecutor { typeof normalizedOutput._childWorkflowInstanceId === 'string' ? normalizedOutput._childWorkflowInstanceId : undefined - const displayOutput = this.sanitizeOutputForLog( + const displayOutput = filterOutputForLog(block.metadata?.id || '', normalizedOutput, { block, - normalizedOutput, - sanitizeEnvironmentSecrets - ) + }) this.fireBlockCompleteCallback( blockStartPromise, ctx, node, block, - this.sanitizeInputsForLog(inputsForLog, sanitizeEnvironmentSecrets, block.metadata?.id), + this.sanitizeInputsForLog(inputsForLog, block.metadata?.id), displayOutput, duration, blockLog.startedAt, @@ -342,7 +322,6 @@ export class BlockExecutor { blockLog, inputsForLog, isSentinel, - sanitizeEnvironmentSecrets, 'execution', streamingPartialOutput ) @@ -401,14 +380,12 @@ export class BlockExecutor { blockLog: BlockLog | undefined, inputsForLog: Record, isSentinel: boolean, - sanitizeEnvironmentSecrets: EnvironmentSecretSanitizer, phase: 'input_resolution' | 'execution', streamingPartialOutput?: Record ): Promise { const endedAt = new Date().toISOString() const duration = performance.now() - startTime const errorMessage = normalizeError(error) - const sanitizedErrorMessage = sanitizeEnvironmentSecrets(errorMessage) const hasLogInputs = inputsForLog && typeof inputsForLog === 'object' && Object.keys(inputsForLog).length > 0 const input = hasLogInputs @@ -436,12 +413,8 @@ export class BlockExecutor { blockLog.durationMs = duration blockLog.success = true blockLog.error = undefined - blockLog.input = this.sanitizeInputsForLog( - input, - sanitizeEnvironmentSecrets, - block.metadata?.id - ) - blockLog.output = this.sanitizeOutputForLog(block, softOutput, sanitizeEnvironmentSecrets) + blockLog.input = this.sanitizeInputsForLog(input, block.metadata?.id) + blockLog.output = filterOutputForLog(block.metadata?.id || '', softOutput, { block }) } this.execLogger.info('Block stream aborted by client; soft-completing', { @@ -455,8 +428,8 @@ export class BlockExecutor { ctx, node, block, - this.sanitizeInputsForLog(input, sanitizeEnvironmentSecrets, block.metadata?.id), - this.sanitizeOutputForLog(block, softOutput, sanitizeEnvironmentSecrets), + this.sanitizeInputsForLog(input, block.metadata?.id), + filterOutputForLog(block.metadata?.id || '', softOutput, { block }), duration, blockLog.startedAt, blockLog.executionOrder, @@ -506,16 +479,12 @@ export class BlockExecutor { blockLog.endedAt = endedAt blockLog.durationMs = duration blockLog.success = false - blockLog.error = sanitizedErrorMessage - blockLog.input = this.sanitizeInputsForLog( - input, - sanitizeEnvironmentSecrets, - block.metadata?.id - ) - blockLog.output = this.sanitizeOutputForLog(block, errorOutput, sanitizeEnvironmentSecrets) + blockLog.error = errorMessage + blockLog.input = this.sanitizeInputsForLog(input, block.metadata?.id) + blockLog.output = filterOutputForLog(block.metadata?.id || '', errorOutput, { block }) if (ChildWorkflowError.isChildWorkflowError(error) && error.childTraceSpans.length > 0) { - blockLog.childTraceSpans = sanitizeEnvironmentSecrets(error.childTraceSpans) + blockLog.childTraceSpans = error.childTraceSpans } } @@ -524,7 +493,7 @@ export class BlockExecutor { { blockId: node.id, blockType: block.metadata?.id, - error: sanitizedErrorMessage, + error: errorMessage, } ) @@ -532,17 +501,13 @@ export class BlockExecutor { const childWorkflowInstanceId = ChildWorkflowError.isChildWorkflowError(error) ? error.childWorkflowInstanceId : undefined - const displayOutput = this.sanitizeOutputForLog( - block, - errorOutput, - sanitizeEnvironmentSecrets - ) + const displayOutput = filterOutputForLog(block.metadata?.id || '', errorOutput, { block }) this.fireBlockCompleteCallback( blockStartPromise, ctx, node, block, - this.sanitizeInputsForLog(input, sanitizeEnvironmentSecrets, block.metadata?.id), + this.sanitizeInputsForLog(input, block.metadata?.id), displayOutput, duration, blockLog.startedAt, @@ -559,7 +524,7 @@ export class BlockExecutor { } this.execLogger.info('Block has error port - returning error output instead of throwing', { blockId: node.id, - error: sanitizedErrorMessage, + error: errorMessage, }) return errorOutput } @@ -666,7 +631,6 @@ export class BlockExecutor { */ private sanitizeInputsForLog( inputs: Record, - sanitizeEnvironmentSecrets: EnvironmentSecretSanitizer, blockType?: string ): Record { // Custom (deploy-as-block) blocks run via an internal `workflow_executor`; the @@ -723,20 +687,7 @@ export class BlockExecutor { } } - return sanitizeEnvironmentSecrets(redactApiKeys(result)) - } - - /** - * Builds the display-only output shared by persisted logs and live callbacks. - * Runtime output remains untouched for state, handlers, retries, and resume. - */ - private sanitizeOutputForLog( - block: SerializedBlock, - output: NormalizedBlockOutput, - sanitizeEnvironmentSecrets: EnvironmentSecretSanitizer - ): NormalizedBlockOutput { - const filteredOutput = filterOutputForLog(block.metadata?.id ?? '', output, { block }) - return sanitizeEnvironmentSecrets(redactApiKeys(filteredOutput)) as NormalizedBlockOutput + return redactApiKeys(result) } /** @@ -1054,19 +1005,6 @@ export class BlockExecutor { if (!parsedForFormat) { executionOutput.content = fullContent } - - // Fallback usage estimation must happen while the resolved input is - // still available. The log copy is sanitized later, and estimating from - // `{{ENV_VAR}}` placeholders would skew token counts and cost. - processStreamingBlockLog( - { - blockId, - blockType: block.metadata?.id, - input: resolvedInputs, - output: streamingExec.execution.output, - }, - fullContent - ) } if (streamingExec.onFullContent) { diff --git a/apps/sim/executor/execution/engine.ts b/apps/sim/executor/execution/engine.ts index 9c65779a8eb..bebdd37db72 100644 --- a/apps/sim/executor/execution/engine.ts +++ b/apps/sim/executor/execution/engine.ts @@ -180,6 +180,7 @@ export class ExecutionEngine { output: this.finalOutput, error: errorMessage, logs: this.context.blockLogs, + executionState: this.getSerializableExecutionState(), metadata: this.context.metadata, } diff --git a/apps/sim/executor/execution/executor.ts b/apps/sim/executor/execution/executor.ts index d73481f6b95..92058b99ed7 100644 --- a/apps/sim/executor/execution/executor.ts +++ b/apps/sim/executor/execution/executor.ts @@ -438,6 +438,7 @@ export class DAGExecutor { }, startRunMetadata: this.contextExtensions.startRunMetadata, environmentVariables: this.environmentVariables, + resolvedSecretTraceRegistry: this.contextExtensions.resolvedSecretTraceRegistry, workflowVariables: this.workflowVariables, decisions: { router: snapshotState?.decisions?.router diff --git a/apps/sim/executor/execution/snapshot-serializer.test.ts b/apps/sim/executor/execution/snapshot-serializer.test.ts index c8630a77748..2c44d6c2a0c 100644 --- a/apps/sim/executor/execution/snapshot-serializer.test.ts +++ b/apps/sim/executor/execution/snapshot-serializer.test.ts @@ -6,6 +6,7 @@ import type { DAG, DAGNode } from '@/executor/dag/builder' import { EdgeManager } from '@/executor/execution/edge-manager' import { serializePauseSnapshot } from '@/executor/execution/snapshot-serializer' import type { ExecutionContext } from '@/executor/types' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' function createContext(overrides: Partial = {}): ExecutionContext { return { @@ -38,6 +39,45 @@ function createContext(overrides: Partial = {}): ExecutionCont } describe('serializePauseSnapshot', () => { + it('persists encrypted resolved-secret provenance and the source execution id', () => { + const registry = new ResolvedSecretTraceRegistry([ + { name: 'TOKEN', plaintext: 'raw-secret', encryptedValue: 'ciphertext' }, + ]) + registry.recordResolved('TOKEN', 'raw-secret') + const context = createContext({ resolvedSecretTraceRegistry: registry }) + + const snapshot = serializePauseSnapshot(context, ['next-block']) + const serialized = JSON.parse(snapshot.snapshot) + + expect(serialized.state.sourceExecutionId).toBe('execution-1') + expect(serialized.state.resolvedSecretTraceProvenance).toEqual({ + version: 1, + complete: true, + entries: [{ name: 'TOKEN', encryptedValue: 'ciphertext' }], + }) + expect(snapshot.snapshot).not.toContain('raw-secret') + }) + + it('persists a complete zero-entry provenance state for a fresh execution', () => { + const registry = new ResolvedSecretTraceRegistry([], { + userId: 'user-1', + workspaceId: 'workspace-1', + }) + + const snapshot = serializePauseSnapshot( + createContext({ resolvedSecretTraceRegistry: registry }), + ['next-block'] + ) + const serialized = JSON.parse(snapshot.snapshot) + + expect(serialized.state.resolvedSecretTraceProvenance).toEqual({ + version: 1, + complete: true, + entries: [], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + }) + }) + it('serializes batched parallel accumulated outputs for cross-process resume', () => { const context = createContext({ parallelExecutions: new Map([ diff --git a/apps/sim/executor/execution/snapshot-serializer.ts b/apps/sim/executor/execution/snapshot-serializer.ts index d6025fcc0fd..fe8721a4875 100644 --- a/apps/sim/executor/execution/snapshot-serializer.ts +++ b/apps/sim/executor/execution/snapshot-serializer.ts @@ -224,6 +224,19 @@ export function serializePauseSnapshot( dagIncomingEdges, deactivatedEdges: edgeManager?.getDeactivatedEdges(), nodesWithActivatedEdge: edgeManager?.getNodesWithActivatedEdge(), + sourceExecutionId: context.executionId, + trustedLargeValueAccess: { + executionIds: Array.from( + new Set( + [context.executionId, ...(context.largeValueExecutionIds ?? [])].filter( + (id): id is string => Boolean(id) + ) + ) + ), + largeValueKeys: Array.from(new Set(context.largeValueKeys ?? [])), + fileKeys: Array.from(new Set(context.fileKeys ?? [])), + }, + resolvedSecretTraceProvenance: context.resolvedSecretTraceRegistry?.exportProvenance(), } assertSnapshotValueIsCompact(context.workflowVariables, 'workflow variables') diff --git a/apps/sim/executor/execution/types.ts b/apps/sim/executor/execution/types.ts index 02723e9fcdf..23a03cbd7a0 100644 --- a/apps/sim/executor/execution/types.ts +++ b/apps/sim/executor/execution/types.ts @@ -10,6 +10,10 @@ import type { StartBlockRunMetadata, StreamingExecution, } from '@/executor/types' +import type { + ResolvedSecretTraceProvenanceV1, + ResolvedSecretTraceRegistry, +} from '@/executor/utils/resolved-secret-trace-registry' import type { RunFromBlockContext } from '@/executor/utils/run-from-block' import type { SubflowType } from '@/stores/workflows/workflow/types' @@ -88,6 +92,16 @@ export interface SerializableExecutionState { deactivatedEdges?: string[] nodesWithActivatedEdge?: string[] completedPauseContexts?: string[] + /** Server execution that produced this state; callers must still verify it against storage. */ + sourceExecutionId?: string + /** Server-only closure authorizing offloaded values carried by trusted restored state. */ + trustedLargeValueAccess?: { + executionIds: string[] + largeValueKeys: string[] + fileKeys: string[] + } + /** Encrypted-only provenance for Secrets-tab values resolved during this execution. */ + resolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceV1 } /** @@ -144,6 +158,17 @@ export interface ChildWorkflowContext { depth: number } +export interface BlockCompletionCallbackData { + input?: unknown + output: NormalizedBlockOutput + executionTime: number + startedAt: string + executionOrder: number + endedAt: string + /** Per-invocation unique ID linking this workflow block execution to its child block events. */ + childWorkflowInstanceId?: string +} + export interface ExecutionCallbacks { onStream?: (streamingExec: StreamingExecution) => Promise onBlockStart?: ( @@ -158,7 +183,7 @@ export interface ExecutionCallbacks { blockId: string, blockName: string, blockType: string, - output: any, + output: BlockCompletionCallbackData, iterationContext?: IterationContext, childWorkflowContext?: ChildWorkflowContext ) => Promise @@ -214,6 +239,7 @@ export interface ContextExtensions { }> dagIncomingEdges?: Record snapshotState?: SerializableExecutionState + resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry metadata?: ExecutionMetadata /** * Trusted run metadata injected into the Start block output when its @@ -247,16 +273,7 @@ export interface ContextExtensions { blockId: string, blockName: string, blockType: string, - output: { - input?: any - output: NormalizedBlockOutput - executionTime: number - startedAt: string - executionOrder: number - endedAt: string - /** Per-invocation unique ID linking this workflow block execution to its child block events. */ - childWorkflowInstanceId?: string - }, + output: BlockCompletionCallbackData, iterationContext?: IterationContext, childWorkflowContext?: ChildWorkflowContext ) => Promise diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index 968097b2014..43ab9d7c1b0 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -1182,48 +1182,55 @@ export class AgentBlockHandler implements BlockHandler { const { blockData, blockNameMapping } = collectBlockData(ctx) - const response = await executeProviderRequest(providerId, { - model, - systemPrompt: 'systemPrompt' in providerRequest ? providerRequest.systemPrompt : undefined, - context: 'context' in providerRequest ? providerRequest.context : undefined, - tools: providerRequest.tools, - temperature: providerRequest.temperature, - maxTokens: providerRequest.maxTokens, - apiKey: finalApiKey, - azureEndpoint: providerRequest.azureEndpoint, - azureApiVersion: providerRequest.azureApiVersion, - vertexProject: providerRequest.vertexProject, - vertexLocation: providerRequest.vertexLocation, - bedrockAccessKeyId: providerRequest.bedrockAccessKeyId, - bedrockSecretKey: providerRequest.bedrockSecretKey, - bedrockRegion: providerRequest.bedrockRegion, - responseFormat: providerRequest.responseFormat, - workflowId: providerRequest.workflowId, - workspaceId: ctx.workspaceId, - userId: ctx.userId, - stream: providerRequest.stream, - messages: 'messages' in providerRequest ? providerRequest.messages : undefined, - environmentVariables: normalizeStringRecord(ctx.environmentVariables), - workflowVariables: normalizeWorkflowVariables(ctx.workflowVariables), - blockData, - blockNameMapping, - isDeployedContext: ctx.isDeployedContext, - callChain: ctx.callChain, - billingAttribution: ctx.metadata.billingAttribution, - // Reaches tool `_context` via `prepareToolExecution`, so a tool that starts - // its own child execution (a custom block) correlates and cancels against - // this real run instead of minting a phantom id. - executionId: ctx.executionId, - reasoningEffort: providerRequest.reasoningEffort, - verbosity: providerRequest.verbosity, - thinkingLevel: providerRequest.thinkingLevel, - promptCaching: providerRequest.promptCaching, - // Stable per-block identity; providers use it to route cache lookups. - blockId: block.id, - previousInteractionId: providerRequest.previousInteractionId, - agentEvents: providerRequest.agentEvents, - abortSignal: ctx.abortSignal, - }) + const response = await executeProviderRequest( + providerId, + { + model, + systemPrompt: + 'systemPrompt' in providerRequest ? providerRequest.systemPrompt : undefined, + context: 'context' in providerRequest ? providerRequest.context : undefined, + tools: providerRequest.tools, + temperature: providerRequest.temperature, + maxTokens: providerRequest.maxTokens, + apiKey: finalApiKey, + azureEndpoint: providerRequest.azureEndpoint, + azureApiVersion: providerRequest.azureApiVersion, + vertexProject: providerRequest.vertexProject, + vertexLocation: providerRequest.vertexLocation, + bedrockAccessKeyId: providerRequest.bedrockAccessKeyId, + bedrockSecretKey: providerRequest.bedrockSecretKey, + bedrockRegion: providerRequest.bedrockRegion, + responseFormat: providerRequest.responseFormat, + workflowId: providerRequest.workflowId, + workspaceId: ctx.workspaceId, + userId: ctx.userId, + stream: providerRequest.stream, + messages: 'messages' in providerRequest ? providerRequest.messages : undefined, + environmentVariables: normalizeStringRecord(ctx.environmentVariables), + workflowVariables: normalizeWorkflowVariables(ctx.workflowVariables), + blockData, + blockNameMapping, + isDeployedContext: ctx.isDeployedContext, + callChain: ctx.callChain, + billingAttribution: ctx.metadata.billingAttribution, + // Reaches tool `_context` via `prepareToolExecution`, so a tool that starts + // its own child execution (a custom block) correlates and cancels against + // this real run instead of minting a phantom id. + executionId: ctx.executionId, + reasoningEffort: providerRequest.reasoningEffort, + verbosity: providerRequest.verbosity, + thinkingLevel: providerRequest.thinkingLevel, + promptCaching: providerRequest.promptCaching, + // Stable per-block identity; providers use it to route cache lookups. + blockId: block.id, + previousInteractionId: providerRequest.previousInteractionId, + agentEvents: providerRequest.agentEvents, + abortSignal: ctx.abortSignal, + }, + { + resolvedSecretTraceRegistry: ctx.resolvedSecretTraceRegistry, + } + ) return this.processProviderResponse(response, block, responseFormat) } catch (error) { diff --git a/apps/sim/executor/handlers/mothership/mothership-handler.test.ts b/apps/sim/executor/handlers/mothership/mothership-handler.test.ts index 69098fae148..c70776aba80 100644 --- a/apps/sim/executor/handlers/mothership/mothership-handler.test.ts +++ b/apps/sim/executor/handlers/mothership/mothership-handler.test.ts @@ -5,6 +5,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { BlockType } from '@/executor/constants' import { MothershipBlockHandler } from '@/executor/handlers/mothership/mothership-handler' import type { ExecutionContext, StreamingExecution } from '@/executor/types' +import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' import type { SerializedBlock } from '@/serializer/types' const BILLING_ATTRIBUTION = { @@ -20,6 +21,13 @@ const BILLING_ATTRIBUTION = { payerSubscription: null, } as const +const PRIVATE_PROVENANCE_TYPE = 'resolved-secret-provenance-v1' +const PRIVATE_PROVENANCE = { + version: 1, + complete: true, + entries: [{ name: 'API_KEY', encryptedValue: 'encrypted-secret' }], +} + const { mockBuildAuthHeaders, mockBuildAPIUrl, @@ -96,6 +104,19 @@ async function readStreamText(stream: ReadableStream): Promise { return text } +function createTraceRegistryMock(): ResolvedSecretTraceRegistry & { + importProvenance: ReturnType + markIncomplete: ReturnType +} { + return { + importProvenance: vi.fn().mockResolvedValue(true), + markIncomplete: vi.fn(), + } as unknown as ResolvedSecretTraceRegistry & { + importProvenance: ReturnType + markIncomplete: ReturnType + } +} + describe('MothershipBlockHandler', () => { let handler: MothershipBlockHandler let block: SerializedBlock @@ -152,7 +173,7 @@ describe('MothershipBlockHandler', () => { resetEnvMock() }) - function createNdjsonResponse(events: unknown[]): Response { + function createNdjsonResponse(events: unknown[], headers: Record = {}): Response { const encoder = new TextEncoder() return new Response( new ReadableStream({ @@ -165,11 +186,146 @@ describe('MothershipBlockHandler', () => { }), { status: 200, - headers: { 'Content-Type': 'application/x-ndjson; charset=utf-8' }, + headers: { 'Content-Type': 'application/x-ndjson; charset=utf-8', ...headers }, } ) } + it('imports marker-gated JSON provenance without exposing it in block output', async () => { + const registry = createTraceRegistryMock() + context.resolvedSecretTraceRegistry = registry + mockGenerateId + .mockReturnValueOnce('chat-uuid') + .mockReturnValueOnce('message-uuid') + .mockReturnValueOnce('request-uuid') + fetchMock.mockResolvedValue( + new Response( + JSON.stringify({ + content: 'raw secret remains functional', + toolCalls: [], + __resolvedSecretTraceProvenance: PRIVATE_PROVENANCE, + }), + { + status: 200, + headers: { + 'Content-Type': 'application/json', + 'x-sim-private-tool-metadata': PRIVATE_PROVENANCE_TYPE, + }, + } + ) + ) + + const result = await handler.execute(context, block, { prompt: 'Hello' }) + + const [, options] = fetchMock.mock.calls[0] as [string, RequestInit] + expect(options.headers).toMatchObject({ + 'x-sim-request-private-tool-metadata': PRIVATE_PROVENANCE_TYPE, + }) + expect(registry.importProvenance).toHaveBeenCalledWith(PRIVATE_PROVENANCE, { + trusted: true, + }) + expect(registry.markIncomplete).not.toHaveBeenCalled() + expect(result).toMatchObject({ content: 'raw secret remains functional' }) + expect(JSON.stringify(result)).not.toContain('__resolvedSecretTraceProvenance') + expect(JSON.stringify(result)).not.toContain('encrypted-secret') + }) + + it('rejects unmarked provenance while keeping private metadata out of block output', async () => { + const registry = createTraceRegistryMock() + context.resolvedSecretTraceRegistry = registry + mockGenerateId + .mockReturnValueOnce('chat-uuid') + .mockReturnValueOnce('message-uuid') + .mockReturnValueOnce('request-uuid') + fetchMock.mockResolvedValue( + new Response( + JSON.stringify({ + content: 'unchanged output', + toolCalls: [], + __resolvedSecretTraceProvenance: PRIVATE_PROVENANCE, + }), + { status: 200, headers: { 'Content-Type': 'application/json' } } + ) + ) + + const result = await handler.execute(context, block, { prompt: 'Hello' }) + + expect(registry.importProvenance).not.toHaveBeenCalled() + expect(registry.markIncomplete).toHaveBeenCalledOnce() + expect(result).toMatchObject({ content: 'unchanged output' }) + expect(JSON.stringify(result)).not.toContain('__resolvedSecretTraceProvenance') + expect(JSON.stringify(result)).not.toContain('encrypted-secret') + }) + + it('imports provenance from a terminal NDJSON error without forcing structural fallback', async () => { + const registry = createTraceRegistryMock() + context.resolvedSecretTraceRegistry = registry + mockGenerateId + .mockReturnValueOnce('chat-uuid') + .mockReturnValueOnce('message-uuid') + .mockReturnValueOnce('request-uuid') + fetchMock.mockResolvedValue( + createNdjsonResponse( + [ + { + type: 'error', + error: 'secret-backed failure', + __resolvedSecretTraceProvenance: PRIVATE_PROVENANCE, + }, + ], + { 'x-sim-private-tool-metadata': PRIVATE_PROVENANCE_TYPE } + ) + ) + + await expect(handler.execute(context, block, { prompt: 'Hello' })).rejects.toThrow( + 'Sim execution failed: secret-backed failure' + ) + + expect(registry.importProvenance).toHaveBeenCalledWith(PRIVATE_PROVENANCE, { + trusted: true, + }) + expect(registry.markIncomplete).not.toHaveBeenCalled() + }) + + it('imports final provenance for selected-output streaming without adding it to output', async () => { + const registry = createTraceRegistryMock() + context.resolvedSecretTraceRegistry = registry + context.stream = true + context.selectedOutputs = [`${block.id}_content`] + mockGenerateId + .mockReturnValueOnce('chat-uuid') + .mockReturnValueOnce('message-uuid') + .mockReturnValueOnce('request-uuid') + fetchMock.mockResolvedValue( + createNdjsonResponse( + [ + { type: 'chunk', content: 'unchanged' }, + { + type: 'final', + data: { + content: 'unchanged', + toolCalls: [], + __resolvedSecretTraceProvenance: PRIVATE_PROVENANCE, + }, + }, + ], + { 'x-sim-private-tool-metadata': PRIVATE_PROVENANCE_TYPE } + ) + ) + + const result = (await handler.execute(context, block, { + prompt: 'Hello', + })) as StreamingExecution + + await expect(readStreamText(result.stream)).resolves.toBe('unchanged') + expect(registry.importProvenance).toHaveBeenCalledWith(PRIVATE_PROVENANCE, { + trusted: true, + }) + expect(registry.markIncomplete).not.toHaveBeenCalled() + expect(JSON.stringify(result.execution.output)).not.toContain('__resolvedSecretTraceProvenance') + expect(JSON.stringify(result.execution.output)).not.toContain('encrypted-secret') + }) + it('forwards workflow and execution metadata with generated UUID ids', async () => { mockGenerateId.mockReturnValueOnce('chat-uuid') mockGenerateId.mockReturnValueOnce('message-uuid') diff --git a/apps/sim/executor/handlers/mothership/mothership-handler.ts b/apps/sim/executor/handlers/mothership/mothership-handler.ts index 7a082b731e1..b12dea62f66 100644 --- a/apps/sim/executor/handlers/mothership/mothership-handler.ts +++ b/apps/sim/executor/handlers/mothership/mothership-handler.ts @@ -8,6 +8,12 @@ import { import { env } from '@/lib/core/config/env' import { isExecutionCancelled, isRedisCancellationEnabled } from '@/lib/execution/cancellation' import { readUserFileContent } from '@/lib/execution/payloads/materialization.server' +import { + PRIVATE_TOOL_METADATA_REQUEST_HEADER, + RESOLVED_SECRET_PROVENANCE_FIELD, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, + responseHasPrivateToolMetadata, +} from '@/lib/execution/private-tool-metadata' import { createFileContentFromBase64, type MessageContent, @@ -24,6 +30,7 @@ import type { StreamingExecution, } from '@/executor/types' import { buildAPIUrl, buildAuthHeaders, extractAPIErrorMessage } from '@/executor/utils/http' +import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' import type { SerializedBlock } from '@/serializer/types' const logger = createLogger('MothershipBlockHandler') @@ -43,13 +50,31 @@ type MothershipExecuteResult = { tokens?: Record toolCalls?: Array> cost?: unknown -} +} & Partial> type MothershipExecuteStreamEvent = | { type: 'heartbeat'; timestamp?: string } | { type: 'chunk'; content?: string } | { type: 'final'; data: MothershipExecuteResult } - | { type: 'error'; error?: string } + | ({ type: 'error'; error?: string } & Partial< + Record + >) + +async function consumeMothershipProvenance( + payload: Partial>, + response: Response, + registry?: ResolvedSecretTraceRegistry +): Promise { + if (!registry) return true + if ( + !responseHasPrivateToolMetadata(response.headers, RESOLVED_SECRET_PROVENANCE_METADATA_V1) || + !Object.hasOwn(payload, RESOLVED_SECRET_PROVENANCE_FIELD) + ) { + registry.markIncomplete() + return false + } + return registry.importProvenance(payload[RESOLVED_SECRET_PROVENANCE_FIELD], { trusted: true }) +} function parseMothershipExecuteStreamLine(line: string): MothershipExecuteStreamEvent | undefined { const trimmed = line.trim() @@ -100,10 +125,15 @@ function isContentSelectedForStreaming(ctx: ExecutionContext, block: SerializedB ) } -async function readMothershipExecuteResponse(response: Response): Promise { +async function readMothershipExecuteResponse( + response: Response, + registry?: ResolvedSecretTraceRegistry +): Promise { const contentType = response.headers.get('content-type') || '' if (!contentType.includes('application/x-ndjson')) { - return response.json() + const result = (await response.json()) as MothershipExecuteResult + await consumeMothershipProvenance(result, response, registry) + return result } if (!response.body) { @@ -114,8 +144,9 @@ async function readMothershipExecuteResponse(response: Response): Promise { + const processLine = async (line: string): Promise => { const event = parseMothershipExecuteStreamLine(line) if (!event) return @@ -124,10 +155,12 @@ async function readMothershipExecuteResponse(response: Response): Promise void onDone?: () => void + registry?: ResolvedSecretTraceRegistry } = {} ): StreamingExecution { if (!response.body) { @@ -191,8 +226,9 @@ function createMothershipStreamingExecution( const encoder = new TextEncoder() let buffer = '' let sawFinal = false + let receivedTerminalProvenance = false - const processLine = (line: string) => { + const processLine = async (line: string): Promise => { const event = parseMothershipExecuteStreamLine(line) if (!event) return @@ -208,10 +244,16 @@ function createMothershipStreamingExecution( } if (event.type === 'error') { + receivedTerminalProvenance = await consumeMothershipProvenance( + event, + response, + options.registry + ) throw new Error(`Sim execution failed: ${event.error || 'Unknown error'}`) } if (event.type === 'final') { + await consumeMothershipProvenance(event.data, response, options.registry) sawFinal = true Object.assign(output, formatMothershipBlockOutput(event.data, fallbackChatId)) return @@ -230,12 +272,12 @@ function createMothershipStreamingExecution( const lines = buffer.split('\n') buffer = lines.pop() ?? '' for (const line of lines) { - processLine(line) + await processLine(line) } } buffer += decoder.decode() - processLine(buffer) + await processLine(buffer) if (!sawFinal) { throw new Error('Sim execution stream ended without a final result') @@ -249,6 +291,7 @@ function createMothershipStreamingExecution( controller.error(error) } } finally { + if (!sawFinal && !receivedTerminalProvenance) options.registry?.markIncomplete() cleanup() reader?.releaseLock() } @@ -382,6 +425,9 @@ export class MothershipBlockHandler implements BlockHandler { const headers = await buildAuthHeaders(ctx.userId) headers.Accept = 'application/x-ndjson' headers[MOTHERSHIP_EXECUTE_STREAM_HEADER] = MOTHERSHIP_EXECUTE_STREAM_VALUE + if (ctx.resolvedSecretTraceRegistry) { + headers[PRIVATE_TOOL_METADATA_REQUEST_HEADER] = RESOLVED_SECRET_PROVENANCE_METADATA_V1 + } if (!ctx.metadata.billingAttribution) { throw new Error('Billing attribution is required for Mothership execution') } @@ -474,6 +520,14 @@ export class MothershipBlockHandler implements BlockHandler { }) if (!response.ok) { + if (ctx.resolvedSecretTraceRegistry) { + try { + const payload = (await response.clone().json()) as MothershipExecuteResult + await consumeMothershipProvenance(payload, response, ctx.resolvedSecretTraceRegistry) + } catch { + ctx.resolvedSecretTraceRegistry.markIncomplete() + } + } const errorMsg = await extractAPIErrorMessage(response) throw new Error(`Sim execution failed: ${errorMsg}`) } @@ -486,12 +540,13 @@ export class MothershipBlockHandler implements BlockHandler { } }, onDone: cleanupAbortListeners, + registry: ctx.resolvedSecretTraceRegistry, }) cleanupImmediately = false return streamingExecution } - const result = await readMothershipExecuteResponse(response) + const result = await readMothershipExecuteResponse(response, ctx.resolvedSecretTraceRegistry) return formatMothershipBlockOutput(result, chatId) } finally { if (cleanupImmediately) { diff --git a/apps/sim/executor/handlers/workflow/custom-block-tool-runner.test.ts b/apps/sim/executor/handlers/workflow/custom-block-tool-runner.test.ts index 39898e9148d..ff7a3692004 100644 --- a/apps/sim/executor/handlers/workflow/custom-block-tool-runner.test.ts +++ b/apps/sim/executor/handlers/workflow/custom-block-tool-runner.test.ts @@ -18,6 +18,7 @@ import { buildCustomBlockExecutionContext, runCustomBlockTool, } from '@/executor/handlers/workflow/custom-block-tool-runner' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' describe('buildCustomBlockExecutionContext', () => { it('carries consumer identity, inherits the call chain, and is fully scaffolded', () => { @@ -145,3 +146,16 @@ describe('buildCustomBlockExecutionContext cancellation', () => { expect(buildCustomBlockExecutionContext({ workspaceId: 'ws-1' }).abortSignal).toBeUndefined() }) }) + +describe('buildCustomBlockExecutionContext secret provenance', () => { + it('carries the server-only parent registry without putting it in model parameters', () => { + const registry = new ResolvedSecretTraceRegistry() + + const ctx = buildCustomBlockExecutionContext( + { workspaceId: 'ws-1' }, + { resolvedSecretTraceRegistry: registry } + ) + + expect(ctx.resolvedSecretTraceRegistry).toBe(registry) + }) +}) diff --git a/apps/sim/executor/handlers/workflow/custom-block-tool-runner.ts b/apps/sim/executor/handlers/workflow/custom-block-tool-runner.ts index ce8ca3b67c8..79539ced526 100644 --- a/apps/sim/executor/handlers/workflow/custom-block-tool-runner.ts +++ b/apps/sim/executor/handlers/workflow/custom-block-tool-runner.ts @@ -4,6 +4,7 @@ import { generateId } from '@sim/utils/id' import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' import { WorkflowBlockHandler } from '@/executor/handlers/workflow/workflow-handler' import type { ExecutionContext } from '@/executor/types' +import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' import type { SerializedBlock } from '@/serializer/types' import type { ToolResponse } from '@/tools/types' @@ -49,7 +50,10 @@ interface CustomBlockToolParams { */ export function buildCustomBlockExecutionContext( context: CustomBlockExecutorContext, - options: { abortSignal?: AbortSignal } = {} + options: { + abortSignal?: AbortSignal + resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry + } = {} ): ExecutionContext { // Prefer the invoking agent run's ids so correlation and cancellation both // point at a real execution; fall back only when a caller could not supply them. @@ -66,6 +70,7 @@ export function buildCustomBlockExecutionContext( // Without this the child's cancellation bridge has nothing to abort on: // the agent tool loop owns the only signal reaching this path. abortSignal: options.abortSignal, + resolvedSecretTraceRegistry: options.resolvedSecretTraceRegistry, environmentVariables: {}, blockStates: new Map(), executedBlocks: new Set(), @@ -102,7 +107,10 @@ export function buildCustomBlockExecutionContext( */ export async function runCustomBlockTool( params: CustomBlockToolParams, - options: { abortSignal?: AbortSignal } = {} + options: { + abortSignal?: AbortSignal + resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry + } = {} ): Promise { if (!params.blockType) { return { success: false, output: {}, error: 'Missing custom block type' } @@ -110,6 +118,7 @@ export async function runCustomBlockTool( const ctx = buildCustomBlockExecutionContext(params._context ?? {}, { abortSignal: options.abortSignal, + resolvedSecretTraceRegistry: options.resolvedSecretTraceRegistry, }) const block: SerializedBlock = { id: generateId(), diff --git a/apps/sim/executor/handlers/workflow/workflow-handler.test.ts b/apps/sim/executor/handlers/workflow/workflow-handler.test.ts index 9acbc2a1905..cb8a60737dc 100644 --- a/apps/sim/executor/handlers/workflow/workflow-handler.test.ts +++ b/apps/sim/executor/handlers/workflow/workflow-handler.test.ts @@ -1,4 +1,4 @@ -import { environmentUtilsMockFns, resetEnvironmentUtilsMock } from '@sim/testing' +import { encryptionMockFns, environmentUtilsMockFns, resetEnvironmentUtilsMock } from '@sim/testing' import { afterAll, beforeAll, beforeEach, describe, expect, it, type Mock, vi } from 'vitest' import { getBlock } from '@/blocks/registry' import { BlockType } from '@/executor/constants' @@ -9,6 +9,10 @@ import { WorkflowBlockHandler, } from '@/executor/handlers/workflow/workflow-handler' import type { ExecutionContext } from '@/executor/types' +import { + ANONYMOUS_SECRET_TRACE_REPLACEMENT, + ResolvedSecretTraceRegistry, +} from '@/executor/utils/resolved-secret-trace-registry' import type { SerializedBlock } from '@/serializer/types' const { @@ -24,6 +28,8 @@ const { mockSafeComplete, mockSafeCompleteWithError, mockSafeCompleteWithCancellation, + mockSetResolvedSecretTraceRegistry, + mockSetTraceLargeValueAccess, mockDispose, executorOptions, loggingSessionArgs, @@ -40,6 +46,8 @@ const { mockSafeComplete: vi.fn(), mockSafeCompleteWithError: vi.fn(), mockSafeCompleteWithCancellation: vi.fn(), + mockSetResolvedSecretTraceRegistry: vi.fn(), + mockSetTraceLargeValueAccess: vi.fn(), mockDispose: vi.fn(), executorOptions: [] as Array>, loggingSessionArgs: [] as Array, @@ -54,6 +62,8 @@ vi.mock('@/lib/logs/execution/logging-session', () => ({ safeComplete = mockSafeComplete safeCompleteWithError = mockSafeCompleteWithError safeCompleteWithCancellation = mockSafeCompleteWithCancellation + setResolvedSecretTraceRegistry = mockSetResolvedSecretTraceRegistry + setTraceLargeValueAccess = mockSetTraceLargeValueAccess onBlockStart = vi.fn() onBlockComplete = vi.fn() }, @@ -63,6 +73,11 @@ vi.mock('@/lib/logs/execution/trace-spans/trace-spans', () => ({ buildTraceSpans: mockBuildTraceSpans, })) +vi.mock('@/lib/core/security/encryption', () => ({ + decryptSecret: encryptionMockFns.mockDecryptSecret, + encryptSecret: encryptionMockFns.mockEncryptSecret, +})) + vi.mock('@/lib/workflows/custom-blocks/child-execution', () => ({ admitCustomBlockChildExecution: mockAdmitCustomBlockChildExecution, trackChildRun: mockTrackChildRun, @@ -1147,6 +1162,54 @@ describe('WorkflowBlockHandler', () => { expect(ctx.largeValueExecutionIds).toContain('grandchild-execution-id') }) + it('imports only publisher secret provenance that crosses the curated output boundary', async () => { + encryptionMockFns.mockDecryptSecret.mockResolvedValueOnce({ + decrypted: 'publisher-secret', + }) + mockGetPersonalAndWorkspaceEnv.mockResolvedValueOnce({ + personalDecrypted: { SECRET: 'publisher-secret', UNUSED: 'unused-secret' }, + workspaceDecrypted: {}, + personalEncrypted: { + SECRET: 'publisher-ciphertext', + UNUSED: 'unused-ciphertext', + }, + workspaceEncrypted: {}, + decryptionFailures: [], + }) + mockExecutorExecute.mockImplementationOnce(async () => { + const childRegistry = executorOptions.at(-1)?.contextExtensions + .resolvedSecretTraceRegistry as ResolvedSecretTraceRegistry + childRegistry.recordResolved('SECRET', 'publisher-secret') + childRegistry.recordResolved('UNUSED', 'unused-secret') + return { + success: true, + output: {}, + logs: [ + { + blockId: 'b1', + success: true, + output: { content: 'value=publisher-secret' }, + }, + ], + } + }) + const parentRegistry = new ResolvedSecretTraceRegistry() + const result = await handler.execute( + customBlockContext({ resolvedSecretTraceRegistry: parentRegistry }), + customBlock(), + {} + ) + + expect(result).toMatchObject({ answer: 'value=publisher-secret', success: true }) + expect(parentRegistry.getActiveMatches()).toEqual([ + { + plaintext: 'publisher-secret', + replacement: ANONYMOUS_SECRET_TRACE_REPLACEMENT, + }, + ]) + expect(mockSetResolvedSecretTraceRegistry).toHaveBeenCalledTimes(1) + }) + it('does not duplicate ids across repeated invocations', async () => { const ctx = customBlockContext() await handler.execute(ctx, customBlock(), {}) @@ -1407,12 +1470,14 @@ describe('WorkflowBlockHandler', () => { }) it('leaves regular workflow blocks entirely alone', async () => { + const registry = new ResolvedSecretTraceRegistry() const ctx = { ...mockContext, workspaceId: 'workspace-1', executionId: 'parent-execution-id', onBlockStart: vi.fn(), onStream: vi.fn(), + resolvedSecretTraceRegistry: registry, } as unknown as ExecutionContext mockFetch.mockResolvedValue({ ok: true, @@ -1431,6 +1496,7 @@ describe('WorkflowBlockHandler', () => { expect(loggingSessionArgs).toHaveLength(0) const extensions = executorOptions[0].contextExtensions expect(extensions.executionId).toBe('parent-execution-id') + expect(extensions.resolvedSecretTraceRegistry).toBe(registry) expect(extensions.onStream).toBe(ctx.onStream) expect(extensions.childWorkflowContext).toBeDefined() }) diff --git a/apps/sim/executor/handlers/workflow/workflow-handler.ts b/apps/sim/executor/handlers/workflow/workflow-handler.ts index 07648a6c368..2ae125b9499 100644 --- a/apps/sim/executor/handlers/workflow/workflow-handler.ts +++ b/apps/sim/executor/handlers/workflow/workflow-handler.ts @@ -45,6 +45,7 @@ import { buildAPIUrl, buildAuthHeaders } from '@/executor/utils/http' import { getIterationContext } from '@/executor/utils/iteration-context' import { parseJSON } from '@/executor/utils/json' import { lazyCleanupInputMapping } from '@/executor/utils/lazy-cleanup' +import { createResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' import { isRunMetadataEnabled, resolveExecutorStartBlock } from '@/executor/utils/start-block' import { Serializer } from '@/serializer' import type { SerializedBlock } from '@/serializer/types' @@ -269,6 +270,7 @@ export class WorkflowBlockHandler implements BlockHandler { /** Set for custom blocks only: the child's own execution id / log row. */ let childExecutionId: string | undefined let childSession: LoggingSession | undefined + let childResolvedSecretTraceRegistry = ctx.resolvedSecretTraceRegistry let childSessionStarted = false /** Set once the child's session reached a terminal state, so the catch doesn't re-complete it. */ let childSessionFinalized = false @@ -431,6 +433,24 @@ export class WorkflowBlockHandler implements BlockHandler { ...ownerEnv.personalEncrypted, ...ownerEnv.workspaceEncrypted, } + childResolvedSecretTraceRegistry = await createResolvedSecretTraceRegistry({ + personalEncrypted: ownerEnv.personalEncrypted, + workspaceEncrypted: ownerEnv.workspaceEncrypted, + personalDecrypted: ownerEnv.personalDecrypted, + workspaceDecrypted: ownerEnv.workspaceDecrypted, + decryptionFailures: ownerEnv.decryptionFailures, + scope: { userId: loadUserId, workspaceId: sourceWorkspaceId }, + }) + if (ctx.resolvedSecretTraceRegistry) { + const crossingProvenance = ctx.resolvedSecretTraceRegistry.exportProvenanceForValue( + childWorkflowInput, + { anonymous: true } + ) + await childResolvedSecretTraceRegistry.importProvenance(crossingProvenance, { + trusted: true, + anonymous: true, + }) + } // Custom-block children authenticate internal tool calls as the source // owner in the source workspace, so the consumer's snapshot would fail // the internal routes' actor/workspace scope match. Resolve the @@ -459,6 +479,7 @@ export class WorkflowBlockHandler implements BlockHandler { // child is part of that same logical run and must not add a second. { baseExecutionCharge: 0 } ) + childSession.setResolvedSecretTraceRegistry(childResolvedSecretTraceRegistry) const correlation = buildCustomBlockCorrelation({ invokerExecutionId: ctx.executionId, invokerRequestId: ctx.metadata.requestId, @@ -509,6 +530,12 @@ export class WorkflowBlockHandler implements BlockHandler { for (const id of [ctx.executionId, childExecutionId]) { if (id && !sharedLargeValueIds.includes(id)) sharedLargeValueIds.push(id) } + childSession.setTraceLargeValueAccess({ + largeValueExecutionIds: sharedLargeValueIds, + largeValueKeys: ctx.largeValueKeys, + fileKeys: ctx.fileKeys, + allowLargeValueWorkflowScope: ctx.allowLargeValueWorkflowScope, + }) } // Trusted run metadata for the child's Start block. Every field describes @@ -570,6 +597,7 @@ export class WorkflowBlockHandler implements BlockHandler { // internal tool calls (knowledge, guardrails, MCP, Mothership) can // attach the required billing attribution header. billingAttribution: childBillingAttribution, + resolvedSecretTraceRegistry: childResolvedSecretTraceRegistry, // Fall back to the inherited metadata so a toggle-off intermediate // child still carries the trusted identity chain to deeper children. startRunMetadata: childStartRunMetadata ?? inherited, @@ -672,7 +700,18 @@ export class WorkflowBlockHandler implements BlockHandler { // failures surface identically; we just reshape the successful output. The // child's spend is billed by its own session, not rolled onto this block. if (isCustomBlock) { - return this.projectCustomBlockOutput(executionResult, exposedOutputs) + const exposedOutput = this.projectCustomBlockOutput(executionResult, exposedOutputs) + if (ctx.resolvedSecretTraceRegistry && childResolvedSecretTraceRegistry) { + const crossingProvenance = childResolvedSecretTraceRegistry.exportProvenanceForValue( + exposedOutput, + { anonymous: true } + ) + await ctx.resolvedSecretTraceRegistry.importProvenance(crossingProvenance, { + trusted: true, + anonymous: true, + }) + } + return exposedOutput } return mappedResult @@ -792,6 +831,7 @@ export class WorkflowBlockHandler implements BlockHandler { totalDurationMs: totalDuration ?? 0, error: { message: normalized.message, stackTrace: normalized.stack }, traceSpans, + executionState: executionResult?.executionState, }) } diff --git a/apps/sim/executor/types.ts b/apps/sim/executor/types.ts index 653d1c51d8e..ef5afe1ab6b 100644 --- a/apps/sim/executor/types.ts +++ b/apps/sim/executor/types.ts @@ -9,6 +9,7 @@ import type { PiiBlockOutputRedaction, SerializableExecutionState, } from '@/executor/execution/types' +import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' import type { RunFromBlockContext } from '@/executor/utils/run-from-block' import type { AgentStreamSink, UnsubscribeAgentStreamSink } from '@/providers/stream-events' import type { SerializedBlock, SerializedWorkflow } from '@/serializer/types' @@ -354,6 +355,7 @@ export interface ExecutionContext { /** Trusted run metadata for the Start block's "Add run metadata" toggle. */ startRunMetadata?: StartBlockRunMetadata environmentVariables: Record + resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry workflowVariables?: Record decisions: { diff --git a/apps/sim/executor/utils/environment-secret-sanitizer.test.ts b/apps/sim/executor/utils/environment-secret-sanitizer.test.ts deleted file mode 100644 index 6f74eade253..00000000000 --- a/apps/sim/executor/utils/environment-secret-sanitizer.test.ts +++ /dev/null @@ -1,144 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { createEnvironmentSecretSanitizer } from '@/executor/utils/environment-secret-sanitizer' - -describe('createEnvironmentSecretSanitizer', () => { - it('sanitizes referenced values recursively without mutating the source', () => { - const secret = 'top-secret/value' - const source = { - message: `Bearer ${secret}`, - nested: [{ [secret]: secret }], - } - const sanitize = createEnvironmentSecretSanitizer( - { credential: '{{API_SECRET}}' }, - { API_SECRET: secret } - ) - - const sanitized = sanitize(source) - - expect(sanitized).toEqual({ - message: 'Bearer {{API_SECRET}}', - nested: [{ '{{API_SECRET}}': '{{API_SECRET}}' }], - }) - expect(source).toEqual({ - message: `Bearer ${secret}`, - nested: [{ [secret]: secret }], - }) - expect(sanitized).not.toBe(source) - expect(sanitized.nested).not.toBe(source.nested) - }) - - it('sanitizes URL-encoded secret values', () => { - const secret = 'secret/value with spaces' - const sanitize = createEnvironmentSecretSanitizer('{{API_SECRET}}', { API_SECRET: secret }) - - expect(sanitize(`token=${encodeURIComponent(secret)}`)).toBe('token={{API_SECRET}}') - }) - - it('sanitizes mixed-case percent escapes without folding ordinary characters', () => { - const secret = 'CaseSensitive/value:next' - const sanitize = createEnvironmentSecretSanitizer('{{API_SECRET}}', { API_SECRET: secret }) - - expect(sanitize('token=CaseSensitive%2fvalue%3Anext')).toBe('token={{API_SECRET}}') - expect(sanitize('token=casesensitive%2fvalue%3Anext')).toBe( - 'token=casesensitive%2fvalue%3Anext' - ) - }) - - it('sanitizes form-encoded secrets that use plus for spaces', () => { - const secret = 'secret value/next' - const sanitize = createEnvironmentSecretSanitizer('{{API_SECRET}}', { API_SECRET: secret }) - - expect(sanitize('token=secret+value%2fnext')).toBe('token={{API_SECRET}}') - }) - - it('still sanitizes literal values that cannot be URI encoded', () => { - const secret = `secret-${String.fromCharCode(0xd800)}` - const sanitize = createEnvironmentSecretSanitizer('{{API_SECRET}}', { API_SECRET: secret }) - - expect(sanitize(secret)).toBe('{{API_SECRET}}') - }) - - it('only uses environment variables referenced by the block configuration', () => { - const sanitize = createEnvironmentSecretSanitizer( - { credential: '{{REFERENCED}}' }, - { - REFERENCED: 'replace-me', - UNREFERENCED: 'ordinary-value', - } - ) - - expect(sanitize('replace-me ordinary-value')).toBe('{{REFERENCED}} ordinary-value') - }) - - it('ignores empty and missing referenced values', () => { - const sanitize = createEnvironmentSecretSanitizer(['{{EMPTY}}', '{{MISSING}}'], { EMPTY: '' }) - - expect(sanitize('unchanged')).toBe('unchanged') - }) - - it('replaces overlapping values longest first', () => { - const sanitize = createEnvironmentSecretSanitizer(['{{SHORT}}', '{{LONG}}'], { - SHORT: 'secret', - LONG: 'secret-suffix', - }) - - expect(sanitize('secret-suffix secret')).toBe('{{LONG}} {{SHORT}}') - }) - - it('uses the lexicographically first name for duplicate values', () => { - const sanitize = createEnvironmentSecretSanitizer(['{{Z_SECRET}}', '{{A_SECRET}}'], { - Z_SECRET: 'same-value', - A_SECRET: 'same-value', - }) - - expect(sanitize('same-value')).toBe('{{A_SECRET}}') - }) - - it('does not re-sanitize placeholders inserted earlier in the same string', () => { - const sanitize = createEnvironmentSecretSanitizer(['{{A_SECRET}}', '{{B_SECRET}}'], { - A_SECRET: 'secret-value', - B_SECRET: 'A_SECRET', - }) - - expect(sanitize('secret-value A_SECRET')).toBe('{{A_SECRET}} {{B_SECRET}}') - }) - - it('handles special object keys without changing the object prototype', () => { - const source = Object.create(null) as Record - Object.defineProperty(source, '__proto__', { - value: 'secret', - enumerable: true, - configurable: true, - writable: true, - }) - source.constructor = 'secret' - const sanitize = createEnvironmentSecretSanitizer('{{SECRET}}', { SECRET: 'secret' }) - - const sanitized = sanitize(source) - - expect(Object.getPrototypeOf(sanitized)).toBeNull() - expect(Object.keys(sanitized)).toEqual(['__proto__', 'constructor']) - expect(sanitized.__proto__).toBe('{{SECRET}}') - expect(sanitized.constructor).toBe('{{SECRET}}') - }) - - it('leaves non-plain runtime objects unchanged', () => { - const date = new Date() - const sanitize = createEnvironmentSecretSanitizer('{{SECRET}}', { SECRET: 'secret' }) - - expect(sanitize(date)).toBe(date) - }) - - it('finds references in nested configuration keys and tolerates cycles', () => { - const configured: Record = {} - configured['prefix-{{SECRET}}'] = configured - const source: Record = { value: 'secret' } - source.self = source - const sanitize = createEnvironmentSecretSanitizer(configured, { SECRET: 'secret' }) - - const sanitized = sanitize(source) - - expect(sanitized.value).toBe('{{SECRET}}') - expect(sanitized.self).toBe(sanitized) - }) -}) diff --git a/apps/sim/executor/utils/environment-secret-sanitizer.ts b/apps/sim/executor/utils/environment-secret-sanitizer.ts deleted file mode 100644 index 5daff377108..00000000000 --- a/apps/sim/executor/utils/environment-secret-sanitizer.ts +++ /dev/null @@ -1,223 +0,0 @@ -import { createEnvVarPattern } from '@/executor/utils/reference-validation' - -interface SecretReplacement { - value: string - comparisonValue: string - normalizePercentEscapes: boolean - placeholder: string -} - -export type EnvironmentSecretSanitizer = (value: T) => T - -function isPlainObject(value: object): value is Record { - const prototype = Object.getPrototypeOf(value) - return prototype === Object.prototype || prototype === null -} - -function collectReferencedEnvironmentVariables( - value: unknown, - referencedNames: Set, - visited: WeakSet -): void { - if (typeof value === 'string') { - for (const match of value.matchAll(createEnvVarPattern())) { - const name = match[1]?.trim() - if (name) { - referencedNames.add(name) - } - } - return - } - - if (value === null || typeof value !== 'object' || visited.has(value)) { - return - } - visited.add(value) - - if (Array.isArray(value)) { - for (const item of value) { - collectReferencedEnvironmentVariables(item, referencedNames, visited) - } - return - } - - if (!isPlainObject(value)) { - return - } - - for (const [key, item] of Object.entries(value)) { - collectReferencedEnvironmentVariables(key, referencedNames, visited) - collectReferencedEnvironmentVariables(item, referencedNames, visited) - } -} - -function compareStrings(left: string, right: string): number { - if (left < right) return -1 - if (left > right) return 1 - return 0 -} - -function normalizePercentEscapeCase(value: string): string { - return value.replace(/%[0-9a-f]{2}/gi, (percentEscape) => percentEscape.toUpperCase()) -} - -function buildSecretReplacements( - configuredValue: unknown, - environmentVariables: Record -): SecretReplacement[] { - const referencedNames = new Set() - collectReferencedEnvironmentVariables(configuredValue, referencedNames, new WeakSet()) - - const replacementsByValue = new Map() - for (const name of [...referencedNames].sort()) { - const value = environmentVariables[name] - if (!value) { - continue - } - - const placeholder = `{{${name}}}` - const exactKey = `exact:${value}` - if (!replacementsByValue.has(exactKey)) { - replacementsByValue.set(exactKey, { - value, - comparisonValue: value, - normalizePercentEscapes: false, - placeholder, - }) - } - - let encodedValue: string - try { - encodedValue = encodeURIComponent(value) - } catch { - continue - } - - const encodedVariants = new Set([encodedValue, encodedValue.replaceAll('%20', '+')]) - for (const encodedVariant of encodedVariants) { - if (encodedVariant === value) { - continue - } - - const comparisonValue = normalizePercentEscapeCase(encodedVariant) - const encodedKey = `encoded:${comparisonValue}` - if (!replacementsByValue.has(encodedKey)) { - replacementsByValue.set(encodedKey, { - value: encodedVariant, - comparisonValue, - normalizePercentEscapes: true, - placeholder, - }) - } - } - } - - return [...replacementsByValue.values()].sort( - (left, right) => - right.value.length - left.value.length || - compareStrings(left.placeholder, right.placeholder) || - compareStrings(left.comparisonValue, right.comparisonValue) - ) -} - -function sanitizeString(value: string, replacements: SecretReplacement[]): string { - let cursor = 0 - let sanitized = '' - const percentNormalizedValue = replacements.some( - (replacement) => replacement.normalizePercentEscapes - ) - ? normalizePercentEscapeCase(value) - : value - - while (cursor < value.length) { - let nextIndex = -1 - let nextReplacement: SecretReplacement | undefined - - for (const replacement of replacements) { - const source = replacement.normalizePercentEscapes ? percentNormalizedValue : value - const index = source.indexOf(replacement.comparisonValue, cursor) - if (index !== -1 && (nextIndex === -1 || index < nextIndex)) { - nextIndex = index - nextReplacement = replacement - } - } - - if (!nextReplacement) { - sanitized += value.slice(cursor) - break - } - - sanitized += value.slice(cursor, nextIndex) - sanitized += nextReplacement.placeholder - cursor = nextIndex + nextReplacement.value.length - } - - return sanitized -} - -function sanitizeValue( - value: unknown, - replacements: SecretReplacement[], - visited: WeakMap -): unknown { - if (typeof value === 'string') { - return sanitizeString(value, replacements) - } - - if (value === null || typeof value !== 'object') { - return value - } - - const existing = visited.get(value) - if (existing !== undefined) { - return existing - } - - if (Array.isArray(value)) { - const sanitized: unknown[] = [] - visited.set(value, sanitized) - for (const item of value) { - sanitized.push(sanitizeValue(item, replacements, visited)) - } - return sanitized - } - - if (!isPlainObject(value)) { - return value - } - - const sanitized = Object.create(Object.getPrototypeOf(value)) as Record - visited.set(value, sanitized) - - for (const [key, item] of Object.entries(value)) { - const sanitizedKey = sanitizeString(key, replacements) - Object.defineProperty(sanitized, sanitizedKey, { - value: sanitizeValue(item, replacements, visited), - enumerable: true, - configurable: true, - writable: true, - }) - } - - return sanitized -} - -/** - * Creates a sanitizer for observability values using environment variables - * explicitly referenced by the unresolved block configuration. - * - * Runtime values are never changed. Sanitized arrays and plain objects are - * copied, while unsupported object types are returned unchanged. - */ -export function createEnvironmentSecretSanitizer( - configuredValue: unknown, - environmentVariables: Record -): EnvironmentSecretSanitizer { - const replacements = buildSecretReplacements(configuredValue, environmentVariables) - - if (replacements.length === 0) { - return (value: T): T => value - } - - return (value: T): T => sanitizeValue(value, replacements, new WeakMap()) as T -} diff --git a/apps/sim/executor/utils/reference-validation.test.ts b/apps/sim/executor/utils/reference-validation.test.ts new file mode 100644 index 00000000000..5dcf8e24059 --- /dev/null +++ b/apps/sim/executor/utils/reference-validation.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it, vi } from 'vitest' +import { resolveEnvVarReferences } from '@/executor/utils/reference-validation' + +describe('resolveEnvVarReferences provenance', () => { + it('reports each successful exact, embedded, and deep substitution', () => { + const onResolved = vi.fn() + + const result = resolveEnvVarReferences( + { + exact: '{{TOKEN}}', + embedded: 'Bearer {{TOKEN}} and {{OTHER}}', + missing: '{{MISSING}}', + }, + { TOKEN: 'secret', OTHER: 'other-secret' }, + { deep: true, onResolved } + ) + + expect(result).toEqual({ + exact: 'secret', + embedded: 'Bearer secret and other-secret', + missing: '{{MISSING}}', + }) + expect(onResolved.mock.calls).toEqual([ + ['TOKEN', 'secret'], + ['TOKEN', 'secret'], + ['OTHER', 'other-secret'], + ]) + }) + + it('does not report disabled embedded or missing references', () => { + const onResolved = vi.fn() + expect( + resolveEnvVarReferences( + 'prefix {{TOKEN}} {{MISSING}}', + { TOKEN: 'secret' }, + { + allowEmbedded: false, + onResolved, + } + ) + ).toBe('prefix {{TOKEN}} {{MISSING}}') + expect(onResolved).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/executor/utils/reference-validation.ts b/apps/sim/executor/utils/reference-validation.ts index 18b2b76d89f..91130a19683 100644 --- a/apps/sim/executor/utils/reference-validation.ts +++ b/apps/sim/executor/utils/reference-validation.ts @@ -26,6 +26,7 @@ export interface EnvVarResolveOptions { onMissing?: 'keep' | 'throw' | 'empty' deep?: boolean missingKeys?: string[] + onResolved?: (name: string, value: string) => void } /** @@ -37,7 +38,9 @@ export interface EnvVarResolveOptions { * - `onMissing: 'keep'` - Unknown patterns pass through (e.g., Grafana's `{{instance}}`) * - `deep: false` - Only processes strings by default; set `true` for nested objects */ -export const ENV_VAR_RESOLVE_DEFAULTS: Required> = { +export const ENV_VAR_RESOLVE_DEFAULTS: Required< + Omit +> = { resolveExactMatch: true, allowEmbedded: true, trimKeys: true, @@ -70,7 +73,10 @@ export function resolveEnvVarReferences( if (exactMatch) { const envKey = trimKeys ? exactMatch[1].trim() : exactMatch[1] const envValue = envVars[envKey] - if (envValue !== undefined) return envValue + if (envValue !== undefined) { + if (Object.hasOwn(envVars, envKey)) options.onResolved?.(envKey, envValue) + return envValue + } if (options.missingKeys) options.missingKeys.push(envKey) if (onMissing === 'throw') { throw new Error(`Environment variable "${envKey}" was not found`) @@ -88,7 +94,10 @@ export function resolveEnvVarReferences( return value.replace(envVarPattern, (match, varName) => { const envKey = trimKeys ? String(varName).trim() : String(varName) const envValue = envVars[envKey] - if (envValue !== undefined) return envValue + if (envValue !== undefined) { + if (Object.hasOwn(envVars, envKey)) options.onResolved?.(envKey, envValue) + return envValue + } if (options.missingKeys) options.missingKeys.push(envKey) if (onMissing === 'throw') { throw new Error(`Environment variable "${envKey}" was not found`) diff --git a/apps/sim/executor/utils/resolved-secret-trace-registry.test.ts b/apps/sim/executor/utils/resolved-secret-trace-registry.test.ts new file mode 100644 index 00000000000..70487660b6d --- /dev/null +++ b/apps/sim/executor/utils/resolved-secret-trace-registry.test.ts @@ -0,0 +1,551 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockDecryptSecret } = vi.hoisted(() => ({ + mockDecryptSecret: vi.fn(), +})) + +vi.mock('@/lib/core/security/encryption', () => ({ + decryptSecret: mockDecryptSecret, +})) + +import { + ANONYMOUS_SECRET_TRACE_REPLACEMENT, + createResolvedSecretTraceRegistry, + ResolvedSecretTraceProvenanceAccumulator, + type ResolvedSecretTraceProvenanceV1, + ResolvedSecretTraceRegistry, +} from '@/executor/utils/resolved-secret-trace-registry' + +describe('ResolvedSecretTraceProvenanceAccumulator', () => { + const scope = { userId: 'user-1', workspaceId: 'workspace-1' } + + it('unions cold, warm, and retry reports while keeping completeness sticky', () => { + const accumulator = new ResolvedSecretTraceProvenanceAccumulator(scope) + + expect( + accumulator.record({ + version: 1, + complete: false, + entries: [{ name: 'OLD_TOKEN', encryptedValue: 'encrypted-v1' }], + scope, + }) + ).toBe(true) + expect( + accumulator.record({ + version: 1, + complete: true, + entries: [{ name: 'NEW_TOKEN', encryptedValue: 'encrypted-v2' }], + scope, + }) + ).toBe(true) + accumulator.record({ + version: 1, + complete: true, + entries: [{ name: 'OLD_TOKEN', encryptedValue: 'encrypted-v1' }], + scope, + }) + + expect(accumulator.exportProvenance()).toEqual({ + version: 1, + complete: false, + entries: [ + { name: 'OLD_TOKEN', encryptedValue: 'encrypted-v1' }, + { name: 'NEW_TOKEN', encryptedValue: 'encrypted-v2' }, + ], + scope, + }) + }) + + it('anonymizes mismatched-scope reports and marks them incomplete', () => { + const accumulator = new ResolvedSecretTraceProvenanceAccumulator(scope) + + expect( + accumulator.record({ + version: 1, + complete: true, + entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-value' }], + scope: { userId: 'user-1', workspaceId: 'workspace-2' }, + }) + ).toBe(true) + + expect(accumulator.exportProvenance()).toEqual({ + version: 1, + complete: false, + entries: [{ encryptedValue: 'encrypted-value' }], + scope, + }) + }) + + it('fails closed for malformed reports and supports terminal entry discard', () => { + const accumulator = new ResolvedSecretTraceProvenanceAccumulator(scope) + accumulator.record({ + version: 1, + complete: true, + entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-value' }], + scope, + }) + + expect(accumulator.record({ version: 1 })).toBe(false) + expect(accumulator.exportProvenance()).toEqual({ + version: 1, + complete: false, + entries: [], + scope, + }) + + accumulator.record({ + version: 1, + complete: true, + entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-value' }], + scope, + }) + accumulator.markIncomplete({ discardEntries: true }) + expect(accumulator.exportProvenance().entries).toEqual([]) + }) +}) + +describe('ResolvedSecretTraceRegistry', () => { + beforeEach(() => { + vi.clearAllMocks() + mockDecryptSecret.mockImplementation(async (encryptedValue: string) => ({ + decrypted: `decrypted:${encryptedValue}`, + })) + }) + + it('starts with an inert catalog and activates only an exact successful resolution', () => { + const registry = new ResolvedSecretTraceRegistry([ + { name: 'API_KEY', plaintext: 'secret-value', encryptedValue: 'encrypted-value' }, + ]) + + expect(registry.getActiveMatches()).toEqual([]) + expect(registry.recordResolved('API_KEY', 'wrong-value')).toBe(false) + expect(registry.recordResolved('MISSING', 'secret-value')).toBe(false) + expect(registry.getActiveMatches()).toEqual([]) + + expect(registry.recordResolved('API_KEY', 'secret-value')).toBe(true) + expect(registry.getActiveMatches()).toEqual([ + { plaintext: 'secret-value', replacement: '{{API_KEY}}' }, + ]) + }) + + it('uses the workspace catalog entry when personal and workspace names conflict', async () => { + const registry = await createResolvedSecretTraceRegistry({ + personalEncrypted: { SHARED: 'personal-encrypted' }, + workspaceEncrypted: { SHARED: 'workspace-encrypted' }, + personalDecrypted: { SHARED: 'personal-secret' }, + workspaceDecrypted: { SHARED: 'workspace-secret' }, + }) + + expect(registry.recordResolved('SHARED', 'personal-secret')).toBe(false) + expect(registry.recordResolved('SHARED', 'workspace-secret')).toBe(true) + expect(registry.exportProvenance()).toEqual({ + version: 1, + complete: true, + entries: [{ name: 'SHARED', encryptedValue: 'workspace-encrypted' }], + }) + }) + + it('does not make current decryption failures or decrypted-only keys globally incomplete', async () => { + const registry = await createResolvedSecretTraceRegistry({ + personalEncrypted: { FAILED: 'failed-ciphertext' }, + workspaceEncrypted: {}, + personalDecrypted: { FAILED: '', DECRYPTED_ONLY: 'not-catalogued' }, + workspaceDecrypted: {}, + decryptionFailures: ['FAILED'], + }) + + expect(registry.isComplete()).toBe(true) + expect(registry.recordResolved('FAILED', '')).toBe(false) + expect(registry.recordResolved('DECRYPTED_ONLY', 'not-catalogued')).toBe(false) + }) + + it('keeps a successful workspace override when the shadowed personal value failed', async () => { + const registry = await createResolvedSecretTraceRegistry({ + personalEncrypted: { SHARED: 'broken-personal-ciphertext' }, + workspaceEncrypted: { SHARED: 'workspace-ciphertext' }, + personalDecrypted: { SHARED: '' }, + workspaceDecrypted: { SHARED: 'workspace-secret' }, + decryptionFailures: ['SHARED'], + }) + + expect(registry.recordResolved('SHARED', 'workspace-secret')).toBe(true) + expect(registry.exportProvenance().entries).toEqual([ + { name: 'SHARED', encryptedValue: 'workspace-ciphertext' }, + ]) + }) + + it('exports encrypted active provenance without plaintext', () => { + const registry = new ResolvedSecretTraceRegistry([ + { name: 'TOKEN', plaintext: 'raw-secret', encryptedValue: 'ciphertext' }, + ]) + registry.recordResolved('TOKEN', 'raw-secret') + + const serialized = JSON.stringify(registry.exportProvenance()) + expect(serialized).toContain('ciphertext') + expect(serialized).not.toContain('raw-secret') + }) + + it('restores old encrypted values alongside the current catalog after rotation', async () => { + mockDecryptSecret.mockResolvedValueOnce({ decrypted: 'old-secret' }) + const oldProvenance: ResolvedSecretTraceProvenanceV1 = { + version: 1, + complete: true, + entries: [{ name: 'TOKEN', encryptedValue: 'old-ciphertext' }], + } + const registry = await createResolvedSecretTraceRegistry({ + personalEncrypted: { TOKEN: 'new-ciphertext' }, + workspaceEncrypted: {}, + personalDecrypted: { TOKEN: 'new-secret' }, + workspaceDecrypted: {}, + restoredProvenance: oldProvenance, + restoreTrusted: true, + requireRestoredProvenance: true, + }) + + registry.recordResolved('TOKEN', 'new-secret') + + expect(registry.getActiveMatches()).toEqual([ + { plaintext: 'new-secret', replacement: '{{TOKEN}}' }, + { plaintext: 'old-secret', replacement: '{{TOKEN}}' }, + ]) + expect(registry.exportProvenance().entries).toEqual([ + { name: 'TOKEN', encryptedValue: 'new-ciphertext' }, + { name: 'TOKEN', encryptedValue: 'old-ciphertext' }, + ]) + }) + + it('marks untrusted, missing, malformed, and undecryptable restoration incomplete', async () => { + const provenance: ResolvedSecretTraceProvenanceV1 = { + version: 1, + complete: true, + entries: [{ name: 'TOKEN', encryptedValue: 'ciphertext' }], + } + const untrusted = new ResolvedSecretTraceRegistry() + expect(await untrusted.importProvenance(provenance, { trusted: false })).toBe(false) + expect(untrusted.isComplete()).toBe(false) + expect(mockDecryptSecret).not.toHaveBeenCalled() + + const missing = await createResolvedSecretTraceRegistry({ + personalEncrypted: {}, + workspaceEncrypted: {}, + personalDecrypted: {}, + workspaceDecrypted: {}, + requireRestoredProvenance: true, + restoreTrusted: true, + }) + expect(missing.isComplete()).toBe(false) + + const malformed = new ResolvedSecretTraceRegistry() + expect(await malformed.importProvenance({ version: 1 }, { trusted: true })).toBe(false) + expect(malformed.isComplete()).toBe(false) + + mockDecryptSecret.mockRejectedValueOnce(new Error('cannot decrypt')) + const undecryptable = new ResolvedSecretTraceRegistry() + expect(await undecryptable.importProvenance(provenance, { trusted: true })).toBe(false) + expect(undecryptable.isComplete()).toBe(false) + }) + + it('uses anonymous replacements for cross-scope provenance', async () => { + mockDecryptSecret.mockResolvedValueOnce({ decrypted: 'publisher-secret' }) + const registry = new ResolvedSecretTraceRegistry() + await registry.importProvenance( + { + version: 1, + complete: true, + entries: [{ name: 'PUBLISHER_TOKEN', encryptedValue: 'publisher-ciphertext' }], + }, + { trusted: true, anonymous: true } + ) + + expect(registry.getActiveMatches()).toEqual([ + { plaintext: 'publisher-secret', replacement: ANONYMOUS_SECRET_TRACE_REPLACEMENT }, + ]) + expect(registry.exportProvenance().entries).toEqual([ + { encryptedValue: 'publisher-ciphertext' }, + ]) + }) + + it('preserves labels only when imported provenance has the same complete scope', async () => { + const provenance: ResolvedSecretTraceProvenanceV1 = { + version: 1, + complete: true, + entries: [{ name: 'TOKEN', encryptedValue: 'ciphertext' }], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + } + const sameScope = new ResolvedSecretTraceRegistry([], { + userId: 'user-1', + workspaceId: 'workspace-1', + }) + const mismatchedScope = new ResolvedSecretTraceRegistry([], { + userId: 'user-1', + workspaceId: 'workspace-2', + }) + const missingReceiverScope = new ResolvedSecretTraceRegistry() + const missingSourceScope = new ResolvedSecretTraceRegistry([], { + userId: 'user-1', + workspaceId: 'workspace-1', + }) + + expect(await sameScope.importProvenance(provenance, { trusted: true })).toBe(true) + expect(await mismatchedScope.importProvenance(provenance, { trusted: true })).toBe(true) + expect(await missingReceiverScope.importProvenance(provenance, { trusted: true })).toBe(true) + expect( + await missingSourceScope.importProvenance( + { version: 1, complete: true, entries: provenance.entries }, + { trusted: true } + ) + ).toBe(true) + + expect(sameScope.getActiveMatches()).toEqual([ + { plaintext: 'decrypted:ciphertext', replacement: '{{TOKEN}}' }, + ]) + for (const registry of [mismatchedScope, missingReceiverScope, missingSourceScope]) { + expect(registry.getActiveMatches()).toEqual([ + { + plaintext: 'decrypted:ciphertext', + replacement: ANONYMOUS_SECRET_TRACE_REPLACEMENT, + }, + ]) + } + }) + + it('imports all same-scope provenance across an in-process boundary', async () => { + const registry = new ResolvedSecretTraceRegistry([], { + userId: 'user-1', + workspaceId: 'workspace-1', + }) + const provenance: ResolvedSecretTraceProvenanceV1 = { + version: 1, + complete: true, + entries: [ + { name: 'PRESENT', encryptedValue: 'present-ciphertext' }, + { name: 'DORMANT_IN_OUTPUT', encryptedValue: 'other-ciphertext' }, + ], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + } + + expect( + await registry.importCrossingProvenance( + provenance, + { output: 'decrypted:present-ciphertext' }, + { trusted: true } + ) + ).toBe(true) + + expect(registry.getActiveMatches()).toEqual([ + { plaintext: 'decrypted:present-ciphertext', replacement: '{{PRESENT}}' }, + { plaintext: 'decrypted:other-ciphertext', replacement: '{{DORMANT_IN_OUTPUT}}' }, + ]) + }) + + it('filters and anonymizes provenance crossing from another scope', async () => { + const registry = new ResolvedSecretTraceRegistry([], { + userId: 'user-1', + workspaceId: 'workspace-1', + }) + const provenance: ResolvedSecretTraceProvenanceV1 = { + version: 1, + complete: true, + entries: [ + { name: 'PRESENT', encryptedValue: 'present-ciphertext' }, + { name: 'ABSENT', encryptedValue: 'absent-ciphertext' }, + ], + scope: { userId: 'user-1', workspaceId: 'workspace-2' }, + } + + expect( + await registry.importCrossingProvenance( + provenance, + { output: 'decrypted:present-ciphertext' }, + { trusted: true } + ) + ).toBe(true) + + expect(registry.getActiveMatches()).toEqual([ + { + plaintext: 'decrypted:present-ciphertext', + replacement: ANONYMOUS_SECRET_TRACE_REPLACEMENT, + }, + ]) + }) + + it('exports only active secrets whose exact literals cross a value boundary', () => { + const registry = new ResolvedSecretTraceRegistry([ + { name: 'PRESENT', plaintext: 'present-secret', encryptedValue: 'present-ciphertext' }, + { name: 'ABSENT', plaintext: 'absent-secret', encryptedValue: 'absent-ciphertext' }, + { name: 'UNUSED', plaintext: 'unused-secret', encryptedValue: 'unused-ciphertext' }, + ]) + registry.recordResolved('PRESENT', 'present-secret') + registry.recordResolved('ABSENT', 'absent-secret') + + const provenance = registry.exportProvenanceForValue( + { nested: [{ 'key-present-secret': new Error('failed with present-secret') }] }, + { anonymous: true } + ) + + expect(provenance).toEqual({ + version: 1, + complete: true, + entries: [{ encryptedValue: 'present-ciphertext' }], + }) + }) + + it('exports active numeric, boolean, and null literals crossing a value boundary', () => { + const registry = new ResolvedSecretTraceRegistry([ + { name: 'NUMBER', plaintext: '1234', encryptedValue: 'number-ciphertext' }, + { name: 'BOOLEAN', plaintext: 'false', encryptedValue: 'boolean-ciphertext' }, + { name: 'NULL', plaintext: 'null', encryptedValue: 'null-ciphertext' }, + { name: 'ABSENT', plaintext: '5678', encryptedValue: 'absent-ciphertext' }, + ]) + registry.recordResolved('NUMBER', '1234') + registry.recordResolved('BOOLEAN', 'false') + registry.recordResolved('NULL', 'null') + registry.recordResolved('ABSENT', '5678') + + expect( + registry.exportProvenanceForValue( + { number: 1234, boolean: false, nullable: null }, + { anonymous: true } + ) + ).toEqual({ + version: 1, + complete: true, + entries: [ + { encryptedValue: 'boolean-ciphertext' }, + { encryptedValue: 'null-ciphertext' }, + { encryptedValue: 'number-ciphertext' }, + ], + }) + }) + + it('marks a bounded cross-boundary scan incomplete when an enumerable accessor is opaque', () => { + const registry = new ResolvedSecretTraceRegistry([ + { name: 'TOKEN', plaintext: 'secret', encryptedValue: 'ciphertext' }, + ]) + registry.recordResolved('TOKEN', 'secret') + const value = {} + Object.defineProperty(value, 'opaque', { + enumerable: true, + get: () => 'secret', + }) + + expect(registry.exportProvenanceForValue(value, { anonymous: true })).toEqual({ + version: 1, + complete: false, + entries: [], + }) + }) + + it('does not claim a complete cross-boundary scan for opaque large-value refs', () => { + const registry = new ResolvedSecretTraceRegistry([ + { name: 'TOKEN', plaintext: 'secret', encryptedValue: 'ciphertext' }, + ]) + registry.recordResolved('TOKEN', 'secret') + + expect( + registry.exportProvenanceForValue( + { + __simLargeValueRef: true, + version: 1, + id: 'lv_ABCDEFGHIJKL', + kind: 'object', + size: 1024, + }, + { anonymous: true } + ) + ).toEqual({ version: 1, complete: false, entries: [] }) + }) + + it('does not snapshot or enqueue an entire wide crossing object', () => { + const registry = new ResolvedSecretTraceRegistry([ + { name: 'TOKEN', plaintext: 'needle-value', encryptedValue: 'ciphertext' }, + ]) + registry.recordResolved('TOKEN', 'needle-value') + const wideValue: Record = {} + for (let index = 0; index < 30_000; index++) { + wideValue[`field_${index}`] = 'safe-value' + } + const descriptorSnapshotSpy = vi.spyOn(Object, 'getOwnPropertyDescriptors') + const provenance = registry.exportProvenanceForValue(wideValue, { anonymous: true }) + const descriptorSnapshotCalls = descriptorSnapshotSpy.mock.calls.length + descriptorSnapshotSpy.mockRestore() + + expect(provenance).toEqual({ + version: 1, + complete: false, + entries: [], + }) + expect(descriptorSnapshotCalls).toBe(0) + }) + + it('handles duplicate and empty values deterministically', () => { + const registry = new ResolvedSecretTraceRegistry([ + { name: 'Z_TOKEN', plaintext: 'same', encryptedValue: 'z-ciphertext' }, + { name: 'A_TOKEN', plaintext: 'same', encryptedValue: 'a-ciphertext' }, + { name: 'EMPTY', plaintext: '', encryptedValue: 'empty-ciphertext' }, + { name: 'A', plaintext: 'A', encryptedValue: 'short-ciphertext' }, + ]) + registry.recordResolved('Z_TOKEN', 'same') + registry.recordResolved('A_TOKEN', 'same') + registry.recordResolved('EMPTY', '') + + expect(registry.getActiveMatches()).toEqual([{ plaintext: 'same', replacement: '{{A_TOKEN}}' }]) + + registry.recordResolved('A', 'A') + expect(registry.getActiveMatches()).toEqual([ + { plaintext: 'same', replacement: '{{A_TOKEN}}' }, + { plaintext: 'A', replacement: '{{A}}' }, + ]) + }) + + it('marks the registry incomplete when active provenance exceeds its hard cap', () => { + const entries = Array.from({ length: 10_001 }, (_, index) => ({ + name: `SECRET_${index}`, + plaintext: `value-${index}`, + encryptedValue: `ciphertext-${index}`, + })) + const registry = new ResolvedSecretTraceRegistry(entries) + + for (const entry of entries) { + registry.recordResolved(entry.name, entry.plaintext) + } + + expect(registry.isComplete()).toBe(false) + expect(registry.exportProvenance().entries).toEqual([]) + }) + + it('stops consuming a dormant catalog when its entry cap is exceeded', () => { + let yieldedEntries = 0 + function* catalogEntries() { + for (let index = 0; index < 20_000; index++) { + yieldedEntries++ + yield { + name: `SECRET_${index}`, + plaintext: `value-${index}`, + encryptedValue: `ciphertext-${index}`, + } + } + } + + const registry = new ResolvedSecretTraceRegistry(catalogEntries()) + + expect(yieldedEntries).toBe(10_001) + expect(registry.isComplete()).toBe(false) + expect(registry.exportProvenance().entries).toEqual([]) + }) + + it('marks an oversized dormant catalog value incomplete without retaining it', () => { + const oversizedPlaintext = 'x'.repeat(8 * 1024 * 1024) + const registry = new ResolvedSecretTraceRegistry([ + { + name: 'OVERSIZED', + plaintext: oversizedPlaintext, + encryptedValue: 'ciphertext', + }, + ]) + + expect(registry.isComplete()).toBe(false) + expect(registry.recordResolved('OVERSIZED', oversizedPlaintext)).toBe(false) + expect(registry.getActiveMatches()).toEqual([]) + }) +}) diff --git a/apps/sim/executor/utils/resolved-secret-trace-registry.ts b/apps/sim/executor/utils/resolved-secret-trace-registry.ts new file mode 100644 index 00000000000..6d7b9605040 --- /dev/null +++ b/apps/sim/executor/utils/resolved-secret-trace-registry.ts @@ -0,0 +1,620 @@ +import { decryptSecret } from '@/lib/core/security/encryption' +import { isLargeArrayManifest } from '@/lib/execution/payloads/large-array-manifest-metadata' +import { isLargeValueRef } from '@/lib/execution/payloads/large-value-ref' + +export const ANONYMOUS_SECRET_TRACE_REPLACEMENT = '[REDACTED_SECRET]' + +const MAX_PROVENANCE_ENTRIES = 10_000 +const MAX_PROVENANCE_BYTES = 8 * 1024 * 1024 +const MAX_TRACE_CATALOG_ENTRIES = MAX_PROVENANCE_ENTRIES +const MAX_TRACE_CATALOG_BYTES = MAX_PROVENANCE_BYTES +const MAX_PROVENANCE_FILTER_NODES = 50_000 +const MAX_PROVENANCE_FILTER_CHARACTERS = 16 * 1024 * 1024 +const MAX_PROVENANCE_FILTER_COMPARISONS = 1_000_000 +const ERROR_CONTENT_PROPERTY_NAMES = ['name', 'message', 'stack', 'cause', 'errors'] as const + +export interface ResolvedSecretTraceCatalogEntry { + name: string + plaintext: string + encryptedValue: string +} + +export interface ResolvedSecretTraceMatch { + plaintext: string + replacement: string +} + +export interface ResolvedSecretTraceProvenanceEntryV1 { + encryptedValue: string + name?: string +} + +export interface ResolvedSecretTraceScopeV1 { + userId: string + workspaceId?: string +} + +export interface ResolvedSecretTraceProvenanceV1 { + version: 1 + complete: boolean + entries: ResolvedSecretTraceProvenanceEntryV1[] + scope?: ResolvedSecretTraceScopeV1 +} + +interface ActiveSecretEntry extends ResolvedSecretTraceCatalogEntry { + anonymous: boolean +} + +export interface ImportResolvedSecretTraceProvenanceOptions { + trusted: boolean + anonymous?: boolean +} + +export interface ExportResolvedSecretTraceProvenanceForValueOptions { + anonymous?: boolean +} + +export interface CreateResolvedSecretTraceRegistryOptions { + personalEncrypted: Record + workspaceEncrypted: Record + personalDecrypted: Record + workspaceDecrypted: Record + decryptionFailures?: readonly string[] + restoredProvenance?: unknown + restoreTrusted?: boolean + requireRestoredProvenance?: boolean + scope?: ResolvedSecretTraceScopeV1 +} + +function compareStrings(left: string, right: string): number { + if (left < right) return -1 + if (left > right) return 1 + return 0 +} + +function hasOwn(record: Record, name: string): boolean { + return Object.hasOwn(record, name) +} + +function catalogEntryByteSize(entry: ResolvedSecretTraceCatalogEntry): number { + return ( + Buffer.byteLength(entry.name, 'utf8') + + Buffer.byteLength(entry.plaintext, 'utf8') + + Buffer.byteLength(entry.encryptedValue, 'utf8') + ) +} + +function buildEffectiveCatalogEntry( + options: CreateResolvedSecretTraceRegistryOptions, + failedNames: ReadonlySet, + name: string, + encryptedValue: string +): ResolvedSecretTraceCatalogEntry | undefined { + const plaintext = hasOwn(options.workspaceDecrypted, name) + ? options.workspaceDecrypted[name] + : options.personalDecrypted[name] + if (plaintext === undefined || (plaintext.length === 0 && failedNames.has(name))) { + return undefined + } + return { name, plaintext, encryptedValue } +} + +function* iterateEffectiveCatalogEntries( + options: CreateResolvedSecretTraceRegistryOptions, + failedNames: ReadonlySet +): Generator { + for (const name in options.personalEncrypted) { + if (!hasOwn(options.personalEncrypted, name)) continue + const encryptedValue = hasOwn(options.workspaceEncrypted, name) + ? options.workspaceEncrypted[name] + : options.personalEncrypted[name] + const entry = buildEffectiveCatalogEntry(options, failedNames, name, encryptedValue) + if (entry) yield entry + } + + for (const name in options.workspaceEncrypted) { + if (!hasOwn(options.workspaceEncrypted, name) || hasOwn(options.personalEncrypted, name)) { + continue + } + const entry = buildEffectiveCatalogEntry( + options, + failedNames, + name, + options.workspaceEncrypted[name] + ) + if (entry) yield entry + } +} + +function isProvenanceEntry(value: unknown): value is ResolvedSecretTraceProvenanceEntryV1 { + if (value === null || typeof value !== 'object') return false + const entry = value as Record + return ( + typeof entry.encryptedValue === 'string' && + entry.encryptedValue.length > 0 && + (entry.name === undefined || (typeof entry.name === 'string' && entry.name.length > 0)) + ) +} + +function isProvenanceScope(value: unknown): value is ResolvedSecretTraceScopeV1 { + if (value === null || typeof value !== 'object') return false + const scope = value as Record + return ( + typeof scope.userId === 'string' && + scope.userId.length > 0 && + (scope.workspaceId === undefined || + (typeof scope.workspaceId === 'string' && scope.workspaceId.length > 0)) + ) +} + +function scopesMatch( + left: ResolvedSecretTraceScopeV1 | undefined, + right: ResolvedSecretTraceScopeV1 | undefined +): boolean { + return ( + (left === undefined && right === undefined) || + (left !== undefined && + right !== undefined && + left.userId === right.userId && + left.workspaceId === right.workspaceId) + ) +} + +export function isResolvedSecretTraceProvenanceV1( + value: unknown +): value is ResolvedSecretTraceProvenanceV1 { + if (value === null || typeof value !== 'object') return false + const provenance = value as Record + if ( + provenance.version !== 1 || + typeof provenance.complete !== 'boolean' || + !Array.isArray(provenance.entries) || + provenance.entries.length > MAX_PROVENANCE_ENTRIES || + !provenance.entries.every(isProvenanceEntry) || + (provenance.scope !== undefined && !isProvenanceScope(provenance.scope)) + ) { + return false + } + + let byteSize = 0 + for (const entry of provenance.entries) { + byteSize += Buffer.byteLength(entry.encryptedValue, 'utf8') + if (entry.name) byteSize += Buffer.byteLength(entry.name, 'utf8') + if (byteSize > MAX_PROVENANCE_BYTES) return false + } + if (provenance.scope) { + byteSize += Buffer.byteLength(provenance.scope.userId, 'utf8') + if (provenance.scope.workspaceId) { + byteSize += Buffer.byteLength(provenance.scope.workspaceId, 'utf8') + } + } + return byteSize <= MAX_PROVENANCE_BYTES +} + +/** + * Unions encrypted provenance reports emitted during one internal transport invocation. + * It never decrypts values or projects trace content; the execution registry remains the + * only owner of plaintext matches and replacement policy. + */ +export class ResolvedSecretTraceProvenanceAccumulator { + private readonly scope?: ResolvedSecretTraceScopeV1 + private provenance: ResolvedSecretTraceProvenanceV1 + + constructor(scope?: ResolvedSecretTraceScopeV1) { + this.scope = scope ? { ...scope } : undefined + this.provenance = this.emptyProvenance(true) + } + + /** Adds one cold, warm-pool, or retry report without decrypting its entries. */ + record(provenance: unknown): boolean { + if (!isResolvedSecretTraceProvenanceV1(provenance)) { + this.provenance = this.emptyProvenance(false) + return false + } + + const sameScope = scopesMatch(provenance.scope, this.scope) + const entries = new Map( + this.provenance.entries.map((entry) => [ + `${entry.name ?? ''}\u0000${entry.encryptedValue}`, + entry, + ]) + ) + for (const entry of provenance.entries) { + const scopedEntry = sameScope ? entry : { encryptedValue: entry.encryptedValue } + entries.set(`${scopedEntry.name ?? ''}\u0000${scopedEntry.encryptedValue}`, scopedEntry) + } + + const merged: ResolvedSecretTraceProvenanceV1 = { + version: 1, + complete: this.provenance.complete && provenance.complete && sameScope, + entries: [...entries.values()], + ...(this.scope ? { scope: { ...this.scope } } : {}), + } + if (!isResolvedSecretTraceProvenanceV1(merged)) { + this.provenance = this.emptyProvenance(false) + return false + } + + this.provenance = merged + return true + } + + /** Marks the invocation incomplete, optionally discarding entries on a fail-closed boundary. */ + markIncomplete(options: { discardEntries?: boolean } = {}): void { + this.provenance = { + ...this.provenance, + complete: false, + ...(options.discardEntries ? { entries: [] } : {}), + } + } + + exportProvenance(): ResolvedSecretTraceProvenanceV1 { + return structuredClone(this.provenance) + } + + private emptyProvenance(complete: boolean): ResolvedSecretTraceProvenanceV1 { + return { + version: 1, + complete, + entries: [], + ...(this.scope ? { scope: { ...this.scope } } : {}), + } + } +} + +/** + * Tracks Secrets-tab values that were actually substituted through `{{NAME}}` during one run. + * The current catalog is inert until a successful resolution activates an entry. + */ +export class ResolvedSecretTraceRegistry { + private readonly catalog = new Map() + private catalogBytes = 0 + private readonly activeEntries = new Map() + private activeProvenanceBytes = 0 + private complete = true + private readonly scope?: ResolvedSecretTraceScopeV1 + + constructor( + catalogEntries: Iterable = [], + scope?: ResolvedSecretTraceScopeV1 + ) { + this.scope = scope ? { ...scope } : undefined + let catalogEntriesSeen = 0 + for (const entry of catalogEntries) { + catalogEntriesSeen++ + if (catalogEntriesSeen > MAX_TRACE_CATALOG_ENTRIES) { + this.markIncomplete() + break + } + if (!this.addCatalogEntry(entry)) break + } + } + + /** Activates a configured secret only when the resolved runtime value matches its catalog value. */ + recordResolved(name: string, resolvedValue: string): boolean { + if (resolvedValue.length === 0) return false + const catalogEntry = this.catalog.get(name) + if (!catalogEntry || catalogEntry.plaintext !== resolvedValue) { + return false + } + + this.addActiveEntry({ ...catalogEntry, anonymous: false }) + return true + } + + /** Imports encrypted provenance only from a boundary that has already established trust. */ + async importProvenance( + provenance: unknown, + options: ImportResolvedSecretTraceProvenanceOptions + ): Promise { + if (!options.trusted || !isResolvedSecretTraceProvenanceV1(provenance)) { + this.markIncomplete() + return false + } + + if (!provenance.complete) { + this.markIncomplete() + } + + const sameScope = scopesMatch(provenance.scope, this.scope) + let importedAll = true + for (const entry of provenance.entries) { + try { + const { decrypted } = await decryptSecret(entry.encryptedValue) + this.addActiveEntry({ + name: entry.name ?? '', + plaintext: decrypted, + encryptedValue: entry.encryptedValue, + anonymous: options.anonymous === true || !sameScope || entry.name === undefined, + }) + } catch { + importedAll = false + this.markIncomplete() + } + } + + return importedAll + } + + /** Imports all same-scope provenance, or only anonymous literals crossing a trust boundary. */ + async importCrossingProvenance( + provenance: unknown, + crossingValue: unknown, + options: { trusted: boolean } + ): Promise { + if (!options.trusted || !isResolvedSecretTraceProvenanceV1(provenance)) { + return this.importProvenance(provenance, { trusted: options.trusted }) + } + + if (scopesMatch(provenance.scope, this.scope)) { + return this.importProvenance(provenance, { trusted: true }) + } + + const sourceRegistry = new ResolvedSecretTraceRegistry([], provenance.scope) + const sourceImported = await sourceRegistry.importProvenance(provenance, { trusted: true }) + const crossingProvenance = sourceRegistry.exportProvenanceForValue(crossingValue, { + anonymous: true, + }) + const crossingImported = await this.importProvenance(crossingProvenance, { trusted: true }) + return sourceImported && crossingImported + } + + /** Returns deterministic literal replacements for the terminal TraceSpan projection. */ + getActiveMatches(): readonly ResolvedSecretTraceMatch[] { + const candidatesByPlaintext = new Map() + for (const entry of this.activeEntries.values()) { + if (entry.plaintext.length === 0) continue + const candidates = candidatesByPlaintext.get(entry.plaintext) ?? [] + candidates.push(entry) + candidatesByPlaintext.set(entry.plaintext, candidates) + } + + const matches = [...candidatesByPlaintext.keys()].map((plaintext) => { + const candidates = candidatesByPlaintext.get(plaintext) ?? [] + const anonymous = candidates.some((candidate) => candidate.anonymous) + const firstNamed = candidates + .filter((candidate) => !candidate.anonymous) + .sort((left, right) => compareStrings(left.name, right.name))[0] + const replacement = anonymous + ? ANONYMOUS_SECRET_TRACE_REPLACEMENT + : `{{${firstNamed?.name ?? ''}}}` + + return { plaintext, replacement } + }) + + return matches.sort( + (left, right) => + right.plaintext.length - left.plaintext.length || + compareStrings(left.plaintext, right.plaintext) || + compareStrings(left.replacement, right.replacement) + ) + } + + isComplete(): boolean { + return this.complete + } + + markIncomplete(): void { + this.complete = false + } + + /** Serializes only encrypted active values; plaintext never enters execution state. */ + exportProvenance(): ResolvedSecretTraceProvenanceV1 { + const entries = this.complete + ? this.buildProvenanceEntries([...this.activeEntries.values()]) + : [] + + return { + version: 1, + complete: this.complete, + entries, + ...(this.scope ? { scope: { ...this.scope } } : {}), + } + } + + /** + * Exports only active provenance whose exact plaintext occurs in a cross-boundary value. + * Bounded traversal returns incomplete provenance instead of broadening to unrelated secrets. + */ + exportProvenanceForValue( + value: unknown, + options: ExportResolvedSecretTraceProvenanceForValueOptions = {} + ): ResolvedSecretTraceProvenanceV1 { + if (!this.complete) return { version: 1, complete: false, entries: [] } + + const candidatesByPlaintext = new Map() + const sortedActiveEntries = [...this.activeEntries.values()].sort( + (left, right) => + compareStrings(left.name, right.name) || + compareStrings(left.encryptedValue, right.encryptedValue) + ) + for (const entry of sortedActiveEntries) { + if (entry.plaintext.length > 0 && !candidatesByPlaintext.has(entry.plaintext)) { + candidatesByPlaintext.set(entry.plaintext, entry) + } + } + + const matchedEntries = new Map() + const pendingValues: unknown[] = [value] + const visited = new WeakSet() + let scannedNodes = 0 + let scannedCharacters = 0 + let comparisons = 0 + let enumeratedProperties = 0 + let scanComplete = true + + const scanString = (candidate: string): boolean => { + scannedCharacters += candidate.length + if (scannedCharacters > MAX_PROVENANCE_FILTER_CHARACTERS) return false + + for (const [plaintext, entry] of candidatesByPlaintext) { + if (matchedEntries.has(plaintext)) continue + comparisons++ + if (comparisons > MAX_PROVENANCE_FILTER_COMPARISONS) return false + if (candidate.includes(plaintext)) { + matchedEntries.set(plaintext, entry) + } + } + return true + } + + const scanProperty = (key: string, descriptor: PropertyDescriptor): boolean => { + if (scannedNodes + pendingValues.length >= MAX_PROVENANCE_FILTER_NODES) return false + scannedNodes++ + if (!scanString(key)) return false + if (matchedEntries.size >= candidatesByPlaintext.size) return true + + if ('value' in descriptor) { + if (scannedNodes + pendingValues.length >= MAX_PROVENANCE_FILTER_NODES) return false + pendingValues.push(descriptor.value) + } else if (descriptor.enumerable) { + return false + } + return true + } + + while (pendingValues.length > 0 && matchedEntries.size < candidatesByPlaintext.size) { + const current = pendingValues.pop() + scannedNodes++ + if (scannedNodes > MAX_PROVENANCE_FILTER_NODES) { + scanComplete = false + break + } + + if ( + typeof current === 'string' || + typeof current === 'number' || + typeof current === 'boolean' || + current === null + ) { + if (!scanString(String(current))) scanComplete = false + if (!scanComplete) break + continue + } + + if (typeof current !== 'object' || visited.has(current)) { + continue + } + visited.add(current) + + if (isLargeValueRef(current) || isLargeArrayManifest(current)) { + scanComplete = false + break + } + + try { + for (const key of ERROR_CONTENT_PROPERTY_NAMES) { + const descriptor = Object.getOwnPropertyDescriptor(current, key) + if (descriptor && !descriptor.enumerable && !scanProperty(key, descriptor)) { + scanComplete = false + break + } + if (matchedEntries.size >= candidatesByPlaintext.size) break + } + if (!scanComplete || matchedEntries.size >= candidatesByPlaintext.size) break + + for (const key in current as Record) { + enumeratedProperties++ + if (enumeratedProperties > MAX_PROVENANCE_FILTER_NODES) { + scanComplete = false + break + } + + const descriptor = Object.getOwnPropertyDescriptor(current, key) + if (!descriptor) continue + if (!scanProperty(key, descriptor)) { + scanComplete = false + break + } + if (matchedEntries.size >= candidatesByPlaintext.size) break + } + if (!scanComplete) break + } catch { + scanComplete = false + break + } + } + + const entries = this.buildProvenanceEntries([...matchedEntries.values()], options.anonymous) + return { + version: 1, + complete: this.complete && scanComplete, + entries, + ...(this.scope ? { scope: { ...this.scope } } : {}), + } + } + + private addActiveEntry(entry: ActiveSecretEntry): void { + const key = `${entry.anonymous ? 'anonymous' : entry.name}\u0000${entry.encryptedValue}` + if (this.activeEntries.has(key)) { + this.activeEntries.set(key, entry) + return + } + + const entryBytes = + Buffer.byteLength(entry.encryptedValue, 'utf8') + Buffer.byteLength(entry.name, 'utf8') + if ( + this.activeEntries.size >= MAX_PROVENANCE_ENTRIES || + this.activeProvenanceBytes + entryBytes > MAX_PROVENANCE_BYTES + ) { + this.markIncomplete() + return + } + this.activeEntries.set(key, entry) + this.activeProvenanceBytes += entryBytes + } + + private addCatalogEntry(entry: ResolvedSecretTraceCatalogEntry): boolean { + const existing = this.catalog.get(entry.name) + const nextCatalogBytes = + this.catalogBytes - + (existing ? catalogEntryByteSize(existing) : 0) + + catalogEntryByteSize(entry) + const nextCatalogSize = this.catalog.size + (existing ? 0 : 1) + if (nextCatalogSize > MAX_TRACE_CATALOG_ENTRIES || nextCatalogBytes > MAX_TRACE_CATALOG_BYTES) { + this.markIncomplete() + return false + } + + this.catalog.set(entry.name, { ...entry }) + this.catalogBytes = nextCatalogBytes + return true + } + + private buildProvenanceEntries( + activeEntries: ActiveSecretEntry[], + forceAnonymous = false + ): ResolvedSecretTraceProvenanceEntryV1[] { + return activeEntries + .sort( + (left, right) => + compareStrings(left.name, right.name) || + compareStrings(left.encryptedValue, right.encryptedValue) + ) + .map((entry) => ({ + encryptedValue: entry.encryptedValue, + ...(!forceAnonymous && !entry.anonymous && entry.name ? { name: entry.name } : {}), + })) + } +} + +/** Builds the effective workspace-over-personal catalog and restores trusted active provenance. */ +export async function createResolvedSecretTraceRegistry( + options: CreateResolvedSecretTraceRegistryOptions +): Promise { + const failedNames = new Set(options.decryptionFailures ?? []) + const registry = new ResolvedSecretTraceRegistry( + iterateEffectiveCatalogEntries(options, failedNames), + options.scope + ) + + if (options.restoredProvenance !== undefined) { + await registry.importProvenance(options.restoredProvenance, { + trusted: options.restoreTrusted === true, + }) + } else if (options.requireRestoredProvenance) { + registry.markIncomplete() + } + + return registry +} diff --git a/apps/sim/executor/variables/resolver.test.ts b/apps/sim/executor/variables/resolver.test.ts index 92771e5a156..bd512bfa464 100644 --- a/apps/sim/executor/variables/resolver.test.ts +++ b/apps/sim/executor/variables/resolver.test.ts @@ -9,6 +9,7 @@ import { import { BlockType } from '@/executor/constants' import { ExecutionState } from '@/executor/execution/state' import type { ExecutionContext } from '@/executor/types' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' import { VariableResolver } from '@/executor/variables/resolver' import { navigatePathAsync } from '@/executor/variables/resolvers/reference-async.server' import type { SerializedBlock, SerializedWorkflow } from '@/serializer/types' @@ -114,6 +115,27 @@ function createResolver( } describe('VariableResolver function block inputs', () => { + it('records a secret reached through workflow-variable indirection', async () => { + const { ctx, resolver } = createResolver() + const registry = new ResolvedSecretTraceRegistry([ + { name: 'TOKEN', plaintext: 'resolved-secret', encryptedValue: 'ciphertext' }, + ]) + ctx.workflowVariables = { + 'var-1': { id: 'var-1', name: 'indirect', type: 'string', value: '{{TOKEN}}' }, + } + ctx.environmentVariables = { TOKEN: 'resolved-secret' } + ctx.resolvedSecretTraceRegistry = registry + + const result = await resolver.resolveInputs(ctx, 'function', { + value: '', + }) + + expect(result.value).toBe('resolved-secret') + expect(registry.getActiveMatches()).toEqual([ + { plaintext: 'resolved-secret', replacement: '{{TOKEN}}' }, + ]) + }) + it('returns empty inputs when params are missing', async () => { const { block, ctx, resolver } = createResolver() @@ -137,47 +159,6 @@ describe('VariableResolver function block inputs', () => { expect(result.contextVariables).toEqual({ __blockRef_0: 'hello world' }) }) - it('resolves environment variables only in runtime function code', async () => { - const { block, ctx, resolver } = createResolver('javascript') - ctx.environmentVariables = { API_SECRET: 'resolved-secret' } - - const result = await resolver.resolveInputsForFunctionBlock( - ctx, - 'function', - { code: 'return "{{API_SECRET}}"' }, - block - ) - - expect(result.resolvedInputs.code).toBe('return "resolved-secret"') - expect(result.displayInputs.code).toBe('return "{{API_SECRET}}"') - }) - - it('preserves environment references in multi-part function display code', async () => { - const { block, ctx, resolver } = createResolver('javascript') - ctx.environmentVariables = { API_SECRET: 'resolved-secret' } - - const result = await resolver.resolveInputsForFunctionBlock( - ctx, - 'function', - { - code: [ - { language: 'javascript', content: 'const key = "{{API_SECRET}}"' }, - { language: 'javascript', content: 'return "{{API_SECRET}}"' }, - ], - }, - block - ) - - expect(result.resolvedInputs.code).toEqual([ - { language: 'javascript', content: 'const key = "resolved-secret"' }, - { language: 'javascript', content: 'return "resolved-secret"' }, - ]) - expect(result.displayInputs.code).toEqual([ - { language: 'javascript', content: 'const key = "{{API_SECRET}}"' }, - { language: 'javascript', content: 'return "{{API_SECRET}}"' }, - ]) - }) - it('allows Variables block assignments to receive whole large refs', async () => { const producer = createBlock('producer', 'Producer', BlockType.API) const variablesBlock = createBlock('variables', 'Variables', BlockType.VARIABLES, { diff --git a/apps/sim/executor/variables/resolver.ts b/apps/sim/executor/variables/resolver.ts index c4029c39384..851358cd2c2 100644 --- a/apps/sim/executor/variables/resolver.ts +++ b/apps/sim/executor/variables/resolver.ts @@ -367,9 +367,9 @@ export class VariableResolver { /** * Resolves a code template for a function block. Block output references are stored * in `contextVarAccumulator` as named variables (e.g. `__blockRef_0`) and replaced - * with those variable names in the returned code string. Environment variables are - * inlined only in runtime code; display code keeps their unresolved references so - * logs and function errors do not expose their values. + * with those variable names in the returned code string. Non-block references (loop + * items, workflow variables, env vars) are still inlined as literals so they remain + * available without any extra passing mechanism. */ private async resolveCodeWithContextVars( ctx: ExecutionContext, @@ -598,6 +598,10 @@ export class VariableResolver { const resolved = await this.resolveReference(match, resolutionContext) return typeof resolved === 'string' ? resolved : match }) + displayResult = await replaceEnvVarsAsync(displayResult, async (match) => { + const resolved = await this.resolveReference(match, resolutionContext) + return typeof resolved === 'string' ? resolved : match + }) return { resolvedCode: result, displayCode: displayResult } } diff --git a/apps/sim/executor/variables/resolvers/env.test.ts b/apps/sim/executor/variables/resolvers/env.test.ts index b9fa9658343..72f72e65b30 100644 --- a/apps/sim/executor/variables/resolvers/env.test.ts +++ b/apps/sim/executor/variables/resolvers/env.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' import { EnvResolver } from './env' import type { ResolutionContext } from './reference' @@ -48,6 +49,23 @@ describe('EnvResolver', () => { }) describe('resolve', () => { + it('records only successful Secrets-tab substitutions', () => { + const resolver = new EnvResolver() + const registry = new ResolvedSecretTraceRegistry([ + { name: 'API_KEY', plaintext: 'secret-api-key', encryptedValue: 'ciphertext' }, + ]) + const ctx = createTestContext({ API_KEY: 'secret-api-key' }) + ctx.executionContext.resolvedSecretTraceRegistry = registry + + expect(resolver.resolve('{{MISSING}}', ctx)).toBe('{{MISSING}}') + expect(registry.getActiveMatches()).toEqual([]) + + expect(resolver.resolve('{{API_KEY}}', ctx)).toBe('secret-api-key') + expect(registry.getActiveMatches()).toEqual([ + { plaintext: 'secret-api-key', replacement: '{{API_KEY}}' }, + ]) + }) + it.concurrent('should resolve existing environment variable', () => { const resolver = new EnvResolver() const ctx = createTestContext({ API_KEY: 'secret-api-key' }) diff --git a/apps/sim/executor/variables/resolvers/env.ts b/apps/sim/executor/variables/resolvers/env.ts index fb5ee3ac6e5..c0ddfeef259 100644 --- a/apps/sim/executor/variables/resolvers/env.ts +++ b/apps/sim/executor/variables/resolvers/env.ts @@ -16,6 +16,9 @@ export class EnvResolver implements Resolver { if (value === undefined) { return reference } + if (Object.hasOwn(context.executionContext.environmentVariables, varName)) { + context.executionContext.resolvedSecretTraceRegistry?.recordResolved(varName, value) + } return value } } diff --git a/apps/sim/hooks/queries/logs.ts b/apps/sim/hooks/queries/logs.ts index 772fb28de48..76bb3fbd255 100644 --- a/apps/sim/hooks/queries/logs.ts +++ b/apps/sim/hooks/queries/logs.ts @@ -375,12 +375,22 @@ export function useCancelExecution(workspaceId: string) { export function useRetryExecution() { const queryClient = useQueryClient() return useMutation({ - mutationFn: async ({ workflowId, input }: { workflowId: string; input?: unknown }) => { + mutationFn: async ({ + workflowId, + executionId, + }: { + workflowId: string + executionId: string + }) => { // boundary-raw-fetch: stream response, body is a ReadableStream consumed one chunk at a time const res = await fetch(`/api/workflows/${workflowId}/execute`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ input, triggerType: 'manual', stream: true }), + body: JSON.stringify({ + inputFromExecutionId: executionId, + triggerType: 'manual', + stream: true, + }), }) if (!res.ok) { const data = await res.json().catch(() => ({})) diff --git a/apps/sim/hooks/use-execution-stream.ts b/apps/sim/hooks/use-execution-stream.ts index a27c1ab4ce4..100b6c0e649 100644 --- a/apps/sim/hooks/use-execution-stream.ts +++ b/apps/sim/hooks/use-execution-stream.ts @@ -210,7 +210,8 @@ export interface ExecuteStreamOptions { export interface ExecuteFromBlockOptions { workflowId: string startBlockId: string - sourceSnapshot: SerializableExecutionState + sourceSnapshot?: SerializableExecutionState + sourceExecutionId?: string input?: any onExecutionId?: (executionId: string) => void callbacks?: ExecutionStreamCallbacks @@ -348,6 +349,7 @@ export function useExecutionStream() { workflowId, startBlockId, sourceSnapshot, + sourceExecutionId, input, onExecutionId, callbacks = {}, @@ -370,7 +372,11 @@ export function useExecutionStream() { body: JSON.stringify({ stream: true, input, - runFromBlock: { startBlockId, sourceSnapshot }, + runFromBlock: { + startBlockId, + ...(sourceExecutionId ? { executionId: sourceExecutionId } : {}), + ...(sourceSnapshot ? { sourceSnapshot } : {}), + }, }), signal: abortController.signal, }) diff --git a/apps/sim/lib/api/contracts/workflows.test.ts b/apps/sim/lib/api/contracts/workflows.test.ts index 7b488d9d631..5880d7730df 100644 --- a/apps/sim/lib/api/contracts/workflows.test.ts +++ b/apps/sim/lib/api/contracts/workflows.test.ts @@ -28,6 +28,12 @@ describe('workflow contracts', () => { }) }) + it('accepts a trusted prior-execution input reference', () => { + expect( + executeWorkflowBodySchema.parse({ inputFromExecutionId: 'execution-123' }) + ).toMatchObject({ inputFromExecutionId: 'execution-123' }) + }) + it('normalizes null React Flow edge handles in execution overrides', () => { const parsed = executeWorkflowBodySchema.parse({ workflowStateOverride: { diff --git a/apps/sim/lib/api/contracts/workflows.ts b/apps/sim/lib/api/contracts/workflows.ts index 60db7b61c7c..c254daf74ff 100644 --- a/apps/sim/lib/api/contracts/workflows.ts +++ b/apps/sim/lib/api/contracts/workflows.ts @@ -360,6 +360,8 @@ export const executeWorkflowBodySchema = z.object({ includeToolCalls: z.boolean().optional().default(false), useDraftState: z.boolean().optional(), input: z.any().optional(), + /** Trusted server-side reuse of a prior execution's raw workflow input. */ + inputFromExecutionId: executionIdSchema.optional(), isClientSession: z.boolean().optional(), includeFileBase64: z.boolean().optional().default(true), base64MaxBytes: z.number().int().positive().optional(), diff --git a/apps/sim/lib/copilot/mcp-tools.test.ts b/apps/sim/lib/copilot/mcp-tools.test.ts index 60c7de69f11..19583f5e92c 100644 --- a/apps/sim/lib/copilot/mcp-tools.test.ts +++ b/apps/sim/lib/copilot/mcp-tools.test.ts @@ -11,7 +11,7 @@ const { discoverServerTools, validateMcpToolsAllowed } = vi.hoisted(() => ({ vi.mock('@/lib/mcp/service', () => ({ mcpService: { discoverServerTools } })) vi.mock('@/ee/access-control/utils/permission-check', () => ({ validateMcpToolsAllowed })) -import { buildSelectedMcpToolSchemas, buildTaggedMcpToolSchemas } from './mcp-tools' +import { buildSelectedMcpToolSchemas, buildTaggedMcpToolSchemas } from '@/lib/copilot/mcp-tools' describe('mothership MCP tool schemas', () => { beforeEach(() => { @@ -50,19 +50,130 @@ describe('mothership MCP tool schemas', () => { ]) }) + it('reports tagged-server discovery provenance without placing it in tool schemas', async () => { + const provenance = { + version: 1, + complete: true, + entries: [{ name: 'MCP_TOKEN', encryptedValue: 'encrypted-token' }], + scope: { userId: 'user-1', workspaceId: 'ws-1' }, + } + discoverServerTools.mockImplementationOnce( + async ( + _userId: string, + _serverId: string, + _workspaceId: string, + _forceRefresh: boolean, + report: (value: unknown) => void + ) => { + report(provenance) + return [ + { + serverId: 'mcp-server-1', + name: 'search', + description: 'Search docs', + inputSchema: { type: 'object' }, + }, + ] + } + ) + const recordProvenance = vi.fn() + + const tools = await buildTaggedMcpToolSchemas( + 'user-1', + 'ws-1', + ['mcp-server-1'], + recordProvenance + ) + + expect(discoverServerTools).toHaveBeenCalledWith( + 'user-1', + 'mcp-server-1', + 'ws-1', + false, + expect.any(Function) + ) + expect(recordProvenance).toHaveBeenCalledWith(provenance) + expect(JSON.stringify(tools)).not.toContain('encrypted-token') + expect(JSON.stringify(tools)).not.toContain('resolvedSecretTraceProvenance') + }) + + it('reports incomplete provenance when tagged-server discovery returns no report', async () => { + discoverServerTools.mockResolvedValue([]) + const recordProvenance = vi.fn() + + await buildTaggedMcpToolSchemas('user-1', 'ws-1', ['mcp-server-1'], recordProvenance) + + expect(recordProvenance).toHaveBeenCalledWith({ + version: 1, + complete: false, + entries: [], + scope: { userId: 'user-1', workspaceId: 'ws-1' }, + }) + }) + it('uses a selected block tool cached schema without discovering the server', async () => { - const tools = await buildSelectedMcpToolSchemas('user-1', 'ws-1', [ - { - type: 'mcp', - params: { serverId: 'mcp-server-1', toolName: 'search', serverName: 'Docs' }, - schema: { type: 'object', properties: { query: { type: 'string' } } }, - }, - ]) + const recordProvenance = vi.fn() + const tools = await buildSelectedMcpToolSchemas( + 'user-1', + 'ws-1', + [ + { + type: 'mcp', + params: { serverId: 'mcp-server-1', toolName: 'search', serverName: 'Docs' }, + schema: { type: 'object', properties: { query: { type: 'string' } } }, + }, + ], + recordProvenance + ) expect(discoverServerTools).not.toHaveBeenCalled() + expect(recordProvenance).not.toHaveBeenCalled() expect(tools[0]).toMatchObject({ name: 'mcp-server-1-search', input_schema: { type: 'object', properties: { query: { type: 'string' } } }, }) }) + + it('reports provenance from selected tools that require server discovery', async () => { + const provenance = { + version: 1, + complete: true, + entries: [{ name: 'MCP_TOKEN', encryptedValue: 'encrypted-token' }], + scope: { userId: 'user-1', workspaceId: 'ws-1' }, + } + discoverServerTools.mockImplementationOnce( + async ( + _userId: string, + _serverId: string, + _workspaceId: string, + _forceRefresh: boolean, + report: (value: unknown) => void + ) => { + report(provenance) + return [ + { + serverId: 'mcp-server-1', + name: 'search', + inputSchema: { type: 'object' }, + }, + ] + } + ) + const recordProvenance = vi.fn() + + const tools = await buildSelectedMcpToolSchemas( + 'user-1', + 'ws-1', + [ + { + type: 'mcp', + params: { serverId: 'mcp-server-1', toolName: 'search' }, + }, + ], + recordProvenance + ) + + expect(recordProvenance).toHaveBeenCalledWith(provenance) + expect(tools[0]).toMatchObject({ name: 'mcp-server-1-search' }) + }) }) diff --git a/apps/sim/lib/copilot/mcp-tools.ts b/apps/sim/lib/copilot/mcp-tools.ts index 24856365e08..7d915035ca7 100644 --- a/apps/sim/lib/copilot/mcp-tools.ts +++ b/apps/sim/lib/copilot/mcp-tools.ts @@ -8,6 +8,8 @@ import type { ToolInput } from '@/executor/handlers/agent/types' const logger = createLogger('CopilotMcpTools') +type ResolvedSecretTraceProvenanceCallback = (provenance: unknown) => void + function toMothershipMcpTool(tool: { serverId: string serverName?: string @@ -51,11 +53,26 @@ function dedupeMcpTools(tools: ToolSchema[]): ToolSchema[] { async function discoverServerTools( userId: string, workspaceId: string, - serverId: string + serverId: string, + onResolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceCallback ): Promise { + let provenanceReported = false try { const { mcpService } = await import('@/lib/mcp/service') - return await mcpService.discoverServerTools(userId, serverId, workspaceId) + if (!onResolvedSecretTraceProvenance) { + return await mcpService.discoverServerTools(userId, serverId, workspaceId) + } + + return await mcpService.discoverServerTools( + userId, + serverId, + workspaceId, + false, + (provenance) => { + provenanceReported = true + onResolvedSecretTraceProvenance(provenance) + } + ) } catch (error) { logger.warn('Failed to resolve tagged MCP server tools', { serverId, @@ -63,6 +80,15 @@ async function discoverServerTools( error: toError(error).message, }) return [] + } finally { + if (onResolvedSecretTraceProvenance && !provenanceReported) { + onResolvedSecretTraceProvenance({ + version: 1, + complete: false, + entries: [], + scope: { userId, workspaceId }, + }) + } } } @@ -73,14 +99,17 @@ async function discoverServerTools( export async function buildTaggedMcpToolSchemas( userId: string, workspaceId: string, - serverIds: string[] + serverIds: string[], + onResolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceCallback ): Promise { const uniqueServerIds = [...new Set(serverIds.filter(Boolean))] if (uniqueServerIds.length === 0) return [] await validateMcpToolsAllowed(userId, workspaceId) const discovered = await Promise.all( - uniqueServerIds.map((serverId) => discoverServerTools(userId, workspaceId, serverId)) + uniqueServerIds.map((serverId) => + discoverServerTools(userId, workspaceId, serverId, onResolvedSecretTraceProvenance) + ) ) return dedupeMcpTools(discovered.flat().map(toMothershipMcpTool)) } @@ -93,7 +122,8 @@ export async function buildTaggedMcpToolSchemas( export async function buildSelectedMcpToolSchemas( userId: string, workspaceId: string, - selections: ToolInput[] + selections: ToolInput[], + onResolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceCallback ): Promise { const selected = selections.filter( (tool) => @@ -130,7 +160,12 @@ export async function buildSelectedMcpToolSchemas( let discovery = discoveredByServer.get(serverId) if (!discovery) { - discovery = discoverServerTools(userId, workspaceId, serverId) + discovery = discoverServerTools( + userId, + workspaceId, + serverId, + onResolvedSecretTraceProvenance + ) discoveredByServer.set(serverId, discovery) } const match = (await discovery).find((tool) => tool.name === toolName) diff --git a/apps/sim/lib/copilot/request/lifecycle/run.test.ts b/apps/sim/lib/copilot/request/lifecycle/run.test.ts index 859ae7a7d44..46cfd2f19c2 100644 --- a/apps/sim/lib/copilot/request/lifecycle/run.test.ts +++ b/apps/sim/lib/copilot/request/lifecycle/run.test.ts @@ -10,6 +10,7 @@ import { } from '@sim/testing' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' import type { ExecutionContext, StreamingContext } from '@/lib/copilot/request/types' +import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' const mockGetEffectiveDecryptedEnv = environmentUtilsMockFns.mockGetEffectiveDecryptedEnv @@ -148,6 +149,44 @@ describe('runCopilotLifecycle', () => { mockGetMothershipSourceEnvHeaders.mockReturnValue({}) }) + it('threads trace provenance through server execution context only', async () => { + const registry = {} as ResolvedSecretTraceRegistry + const executionContext: ExecutionContext = { + userId: 'user-1', + workflowId: '', + workspaceId: 'ws-1', + decryptedEnvVars: {}, + } + let capturedExecutionContext: ExecutionContext | undefined + let capturedRequestBody = '' + mockRunStreamLoop.mockImplementationOnce( + async ( + _url: string, + request: RequestInit, + _streamingContext: StreamingContext, + context: ExecutionContext + ) => { + capturedExecutionContext = context + capturedRequestBody = String(request.body) + } + ) + + await runCopilotLifecycle( + { message: 'hello', messageId: 'stream-private-context' }, + { + userId: 'user-1', + workspaceId: 'ws-1', + executionContext, + resolvedSecretTraceRegistry: registry, + } + ) + + expect(capturedExecutionContext?.resolvedSecretTraceRegistry).toBe(registry) + expect(capturedRequestBody).not.toContain('resolvedSecretTraceRegistry') + expect(capturedRequestBody).not.toContain('resolved-secret-provenance') + expect(executionContext).not.toHaveProperty('resolvedSecretTraceRegistry') + }) + describe('tool permission feature flag', () => { const runMothershipTurn = () => runCopilotLifecycle( diff --git a/apps/sim/lib/copilot/request/lifecycle/run.ts b/apps/sim/lib/copilot/request/lifecycle/run.ts index 40d100148a9..6eb5888b630 100644 --- a/apps/sim/lib/copilot/request/lifecycle/run.ts +++ b/apps/sim/lib/copilot/request/lifecycle/run.ts @@ -62,6 +62,7 @@ import { isHosted, } from '@/lib/core/config/env-flags' import { getEffectiveDecryptedEnv } from '@/lib/environment/utils' +import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' const logger = createLogger('CopilotLifecycle') @@ -95,6 +96,7 @@ export interface CopilotLifecycleOptions extends OrchestratorOptions { onGoTraceId?: (goTraceId: string) => void executionContext?: ExecutionContext billingAttribution?: BillingAttributionSnapshot + resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry } /** @@ -161,6 +163,9 @@ export async function runCopilotLifecycle( abortSignal: options.abortSignal, billingAttribution: options.billingAttribution ?? options.executionContext.billingAttribution, + ...(options.resolvedSecretTraceRegistry + ? { resolvedSecretTraceRegistry: options.resolvedSecretTraceRegistry } + : {}), }, } : {}), @@ -177,6 +182,7 @@ export async function runCopilotLifecycle( runId: resolvedRunId, abortSignal: lifecycleOptions.abortSignal, billingAttribution: lifecycleOptions.billingAttribution, + resolvedSecretTraceRegistry: lifecycleOptions.resolvedSecretTraceRegistry, })) const shouldUseHostedBillingProtocol = isHosted && isCopilotBillingAttributionV1Enabled if ( @@ -993,6 +999,7 @@ async function buildExecutionContext( runId?: string abortSignal?: AbortSignal billingAttribution?: BillingAttributionSnapshot + resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry } ): Promise { const { @@ -1004,6 +1011,7 @@ async function buildExecutionContext( runId, abortSignal, billingAttribution, + resolvedSecretTraceRegistry, } = params const userTimezone = typeof requestPayload?.userTimezone === 'string' ? requestPayload.userTimezone : undefined @@ -1039,6 +1047,9 @@ async function buildExecutionContext( execContext.runId = runId execContext.abortSignal = abortSignal if (billingAttribution) execContext.billingAttribution = billingAttribution + if (resolvedSecretTraceRegistry) { + execContext.resolvedSecretTraceRegistry = resolvedSecretTraceRegistry + } return execContext } diff --git a/apps/sim/lib/copilot/tool-executor/executor.test.ts b/apps/sim/lib/copilot/tool-executor/executor.test.ts index b3cb29d508c..2342f31efbe 100644 --- a/apps/sim/lib/copilot/tool-executor/executor.test.ts +++ b/apps/sim/lib/copilot/tool-executor/executor.test.ts @@ -4,6 +4,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/execution/constants' +import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' const { isKnownTool, isSimExecuted, isClientExecuted } = vi.hoisted(() => ({ isKnownTool: vi.fn(), @@ -111,6 +112,35 @@ describe('copilot tool executor fallback', () => { expect(appParams._context).not.toHaveProperty('billingAttribution') }) + it('passes trace provenance out-of-band without exposing it in app tool parameters', async () => { + isKnownTool.mockReturnValue(false) + isSimExecuted.mockReturnValue(false) + executeAppTool.mockResolvedValue({ success: true, output: { result: 'unchanged' } }) + const registry = {} as ResolvedSecretTraceRegistry + + const result = await executeTool( + 'gmail_read', + { query: 'hello' }, + { + userId: 'user-1', + workflowId: 'workflow-1', + resolvedSecretTraceRegistry: registry, + } + ) + + expect(executeAppTool).toHaveBeenCalledWith( + 'gmail_read', + expect.objectContaining({ + query: 'hello', + _context: expect.not.objectContaining({ resolvedSecretTraceRegistry: expect.anything() }), + }), + { resolvedSecretTraceRegistry: registry } + ) + const appParams = executeAppTool.mock.calls[0]?.[1] + expect(JSON.stringify(appParams)).not.toContain('resolvedSecretTraceRegistry') + expect(result).toEqual({ success: true, output: { result: 'unchanged' } }) + }) + it('uses the registered handler for client-routed tools when running headless (Mothership block)', async () => { isKnownTool.mockReturnValue(true) isSimExecuted.mockReturnValue(false) diff --git a/apps/sim/lib/copilot/tool-executor/executor.ts b/apps/sim/lib/copilot/tool-executor/executor.ts index 0a5323f5e16..3b9efa8438b 100644 --- a/apps/sim/lib/copilot/tool-executor/executor.ts +++ b/apps/sim/lib/copilot/tool-executor/executor.ts @@ -55,7 +55,11 @@ export async function executeTool( (isSimExecuted(toolId) || (isClientExecuted(toolId) && hasHandler(toolId))) if (!canUseRegisteredHandler) { const appParams = buildAppToolParams(normalizedParams, context) - return executeAppTool(toolId, appParams) + return context.resolvedSecretTraceRegistry + ? executeAppTool(toolId, appParams, { + resolvedSecretTraceRegistry: context.resolvedSecretTraceRegistry, + }) + : executeAppTool(toolId, appParams) } if (context.abortSignal?.aborted) { diff --git a/apps/sim/lib/copilot/tool-executor/types.ts b/apps/sim/lib/copilot/tool-executor/types.ts index 4bc8c9835ec..a08fda51758 100644 --- a/apps/sim/lib/copilot/tool-executor/types.ts +++ b/apps/sim/lib/copilot/tool-executor/types.ts @@ -1,5 +1,6 @@ import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' import type { MothershipResource } from '@/lib/copilot/resources/types' +import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' export interface ToolExecutionContext { userId: string @@ -24,6 +25,7 @@ export interface ToolExecutionContext { userTimezone?: string userPermission?: string decryptedEnvVars?: Record + resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry } export interface ToolExecutionResult { diff --git a/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts b/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts index 3fc90655188..0320fb41f35 100644 --- a/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts +++ b/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts @@ -89,6 +89,32 @@ function mountedFiles() { const snapshotCacheOn = (flag: string) => Promise.resolve(flag === 'table-snapshot-cache') +describe('executeFunctionExecute trace-secret provenance', () => { + beforeEach(() => { + vi.clearAllMocks() + mockExecuteTool.mockResolvedValue({ success: true }) + }) + + it('forwards the registry only through server execution options', async () => { + const resolvedSecretTraceRegistry = { recordResolved: vi.fn() } + + await executeFunctionExecute({ code: 'return {{API_KEY}}' }, { + userId: 'u1', + workspaceId: 'ws_1', + resolvedSecretTraceRegistry, + } as never) + + expect(mockExecuteTool).toHaveBeenCalledWith( + 'function_execute', + expect.not.objectContaining({ resolvedSecretTraceRegistry: expect.anything() }), + { resolvedSecretTraceRegistry } + ) + const appParams = mockExecuteTool.mock.calls[0]?.[1] as Record + expect(appParams._context).not.toHaveProperty('resolvedSecretTraceRegistry') + expect(JSON.stringify(appParams)).not.toContain('resolvedSecretTraceRegistry') + }) +}) + describe('executeFunctionExecute table mounts', () => { beforeEach(() => { vi.clearAllMocks() diff --git a/apps/sim/lib/copilot/tools/handlers/function-execute.ts b/apps/sim/lib/copilot/tools/handlers/function-execute.ts index ecb3c630ff0..e064b40d579 100644 --- a/apps/sim/lib/copilot/tools/handlers/function-execute.ts +++ b/apps/sim/lib/copilot/tools/handlers/function-execute.ts @@ -509,5 +509,9 @@ export async function executeFunctionExecute( enforceCredentialAccess: true, } - return executeAppTool('function_execute', enrichedParams) + return context.resolvedSecretTraceRegistry + ? executeAppTool('function_execute', enrichedParams, { + resolvedSecretTraceRegistry: context.resolvedSecretTraceRegistry, + }) + : executeAppTool('function_execute', enrichedParams) } diff --git a/apps/sim/lib/copilot/tools/handlers/workflow/mutations.test.ts b/apps/sim/lib/copilot/tools/handlers/workflow/mutations.test.ts index 37d2ee42a92..097794dac06 100644 --- a/apps/sim/lib/copilot/tools/handlers/workflow/mutations.test.ts +++ b/apps/sim/lib/copilot/tools/handlers/workflow/mutations.test.ts @@ -23,6 +23,10 @@ afterAll(() => { import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' import type { ExecutionContext } from '@/lib/copilot/request/types' +import { + ANONYMOUS_SECRET_TRACE_REPLACEMENT, + ResolvedSecretTraceRegistry, +} from '@/executor/utils/resolved-secret-trace-registry' const { ensureWorkflowAccessMock, @@ -39,6 +43,7 @@ const { checkAttributedUsageLimitsMock, reserveExecutionSlotMock, releaseExecutionSlotMock, + decryptSecretMock, } = vi.hoisted(() => ({ ensureWorkflowAccessMock: vi.fn(), ensureWorkspaceAccessMock: vi.fn(), @@ -54,6 +59,7 @@ const { checkAttributedUsageLimitsMock: vi.fn(), reserveExecutionSlotMock: vi.fn(), releaseExecutionSlotMock: vi.fn(), + decryptSecretMock: vi.fn(), })) vi.mock('@sim/audit', () => ({ @@ -79,6 +85,11 @@ vi.mock('@/lib/billing/calculations/usage-reservation', () => ({ UsageReservationUnavailableError: class UsageReservationUnavailableError extends Error {}, })) +vi.mock('@/lib/core/security/encryption', () => ({ + decryptSecret: decryptSecretMock, + encryptSecret: vi.fn(), +})) + vi.mock('@/lib/workflows/executor/execute-workflow', () => ({ executeWorkflow: executeWorkflowMock, })) @@ -297,6 +308,7 @@ describe('executeCreateWorkflow billing attribution', () => { payerUsage: { currentUsage: 1, limit: 10 }, }) reserveExecutionSlotMock.mockResolvedValue({ reserved: true, created: true }) + decryptSecretMock.mockResolvedValue({ decrypted: 'secret-value' }) listFoldersMock.mockResolvedValue([]) }) @@ -580,6 +592,7 @@ describe('Copilot workflow execution billing attribution', () => { payerUsage: { currentUsage: 1, limit: 10 }, }) reserveExecutionSlotMock.mockResolvedValue({ reserved: true, created: true }) + decryptSecretMock.mockResolvedValue({ decrypted: 'secret-value' }) }) async function expectBillingAttributionForwarded( @@ -602,6 +615,178 @@ describe('Copilot workflow execution billing attribution', () => { ) }) + it('passes only input-crossing parent provenance to the child execution', async () => { + const registry = new ResolvedSecretTraceRegistry( + [ + { + name: 'INPUT_SECRET', + plaintext: 'input-secret', + encryptedValue: 'input-ciphertext', + }, + { + name: 'UNRELATED_SECRET', + plaintext: 'unrelated-secret', + encryptedValue: 'unrelated-ciphertext', + }, + ], + { userId: 'user-1', workspaceId: 'workspace-1' } + ) + registry.recordResolved('INPUT_SECRET', 'input-secret') + registry.recordResolved('UNRELATED_SECRET', 'unrelated-secret') + resolveTriggerRunOptionsMock.mockReturnValueOnce([ + { + triggerBlockId: 'trigger-1', + blockName: 'Start', + mockPayload: { value: 'input-secret' }, + }, + ]) + executeWorkflowMock.mockResolvedValueOnce({ + success: true, + output: { ok: true }, + logs: [], + metadata: { executionId: 'new-execution-1' }, + executionState: { + resolvedSecretTraceProvenance: { + version: 1, + complete: true, + entries: [], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + }, + }, + }) + + const result = await executeRunWorkflow( + { workflowId: 'workflow-1', useMockPayload: true }, + { ...executionContext, resolvedSecretTraceRegistry: registry } + ) + + expect(result.success).toBe(true) + expect(executeWorkflowMock.mock.calls[0]?.[2]).toEqual({ value: 'input-secret' }) + expect(executeWorkflowMock.mock.calls[0]?.[4]).toEqual( + expect.objectContaining({ + trustedInitialResolvedSecretTraceProvenance: { + version: 1, + complete: true, + entries: [{ name: 'INPUT_SECRET', encryptedValue: 'input-ciphertext' }], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + }, + }) + ) + expect(JSON.stringify(result)).not.toContain('input-ciphertext') + expect(JSON.stringify(result)).not.toContain('unrelated-ciphertext') + }) + + it('imports child provenance without returning private metadata to the model', async () => { + const registry = new ResolvedSecretTraceRegistry([], { + userId: 'user-1', + workspaceId: 'workspace-1', + }) + const context: ExecutionContext = { + ...executionContext, + resolvedSecretTraceRegistry: registry, + } + executeWorkflowMock.mockResolvedValueOnce({ + success: true, + output: { value: 'secret-value' }, + logs: [], + metadata: { executionId: 'new-execution-1' }, + executionState: { + resolvedSecretTraceProvenance: { + version: 1, + complete: true, + entries: [{ name: 'API_KEY', encryptedValue: 'encrypted-secret' }], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + }, + }, + }) + + const result = await executeRunWorkflow( + { workflowId: 'workflow-1', useMockPayload: true }, + context + ) + + expect(result).toMatchObject({ + success: true, + output: { output: { value: 'secret-value' } }, + }) + expect(registry.getActiveMatches()).toEqual([ + { plaintext: 'secret-value', replacement: '{{API_KEY}}' }, + ]) + expect(JSON.stringify(result)).not.toContain('__resolvedSecretTraceProvenance') + expect(JSON.stringify(result)).not.toContain('encrypted-secret') + }) + + it('marks provenance incomplete when child execution returns no trusted state', async () => { + const registry = new ResolvedSecretTraceRegistry() + const context: ExecutionContext = { + ...executionContext, + resolvedSecretTraceRegistry: registry, + } + + const result = await executeRunWorkflow( + { workflowId: 'workflow-1', useMockPayload: true }, + context + ) + + expect(result.success).toBe(true) + expect(registry.isComplete()).toBe(false) + }) + + it('filters and anonymizes cross-workspace child provenance to values that cross back', async () => { + const registry = new ResolvedSecretTraceRegistry([], { + userId: 'user-1', + workspaceId: 'workspace-1', + }) + const context: ExecutionContext = { + ...executionContext, + resolvedSecretTraceRegistry: registry, + } + ensureWorkflowAccessMock.mockResolvedValueOnce({ + workflow: { + id: 'workflow-2', + userId: 'owner-2', + workspaceId: 'workspace-2', + variables: {}, + }, + }) + resolveBillingAttributionMock.mockResolvedValueOnce(childBillingAttribution) + decryptSecretMock.mockImplementation(async (encryptedValue: string) => ({ + decrypted: encryptedValue === 'used-ciphertext' ? 'used-secret' : 'workspace-only-secret', + })) + executeWorkflowMock.mockResolvedValueOnce({ + success: true, + output: { value: 'used-secret' }, + logs: [], + metadata: { executionId: 'child-execution' }, + executionState: { + resolvedSecretTraceProvenance: { + version: 1, + complete: true, + entries: [ + { name: 'USED', encryptedValue: 'used-ciphertext' }, + { name: 'WORKSPACE_ONLY', encryptedValue: 'workspace-only-ciphertext' }, + ], + scope: { userId: 'user-1', workspaceId: 'workspace-2' }, + }, + }, + }) + + const result = await executeRunWorkflow( + { workflowId: 'workflow-2', useMockPayload: true }, + context + ) + + expect(result).toMatchObject({ + success: true, + output: { output: { value: 'used-secret' } }, + }) + expect(registry.getActiveMatches()).toEqual([ + { plaintext: 'used-secret', replacement: ANONYMOUS_SECRET_TRACE_REPLACEMENT }, + ]) + expect(JSON.stringify(result)).not.toContain('workspace-only-ciphertext') + expect(JSON.stringify(result)).not.toContain('__resolvedSecretTraceProvenance') + }) + it('passes immutable attribution when running until a block', async () => { await expectBillingAttributionForwarded(() => executeRunWorkflowUntilBlock( diff --git a/apps/sim/lib/copilot/tools/handlers/workflow/mutations.ts b/apps/sim/lib/copilot/tools/handlers/workflow/mutations.ts index 8aea602d323..4a8823cd717 100644 --- a/apps/sim/lib/copilot/tools/handlers/workflow/mutations.ts +++ b/apps/sim/lib/copilot/tools/handlers/workflow/mutations.ts @@ -97,7 +97,9 @@ async function executeCopilotWorkflowTarget(params: { ) try { - return await executeWorkflow( + const trustedInitialResolvedSecretTraceProvenance = + params.context.resolvedSecretTraceRegistry?.exportProvenanceForValue(params.input) + const result = await executeWorkflow( params.workflow, generateRequestId(), params.input, @@ -105,10 +107,34 @@ async function executeCopilotWorkflowTarget(params: { { ...params.options, billingAttribution: admission.billingAttribution, + ...(trustedInitialResolvedSecretTraceProvenance + ? { trustedInitialResolvedSecretTraceProvenance } + : {}), }, childExecutionId ) + if (params.context.resolvedSecretTraceRegistry) { + await params.context.resolvedSecretTraceRegistry.importCrossingProvenance( + result.executionState?.resolvedSecretTraceProvenance, + { output: result.output, logs: result.logs, error: result.error }, + { trusted: true } + ) + } + return result } catch (error) { + if (params.context.resolvedSecretTraceRegistry) { + const executionResult = hasExecutionResult(error) ? error.executionResult : undefined + await params.context.resolvedSecretTraceRegistry.importCrossingProvenance( + executionResult?.executionState?.resolvedSecretTraceProvenance, + { + output: executionResult?.output, + logs: executionResult?.logs, + error: executionResult?.error, + thrownMessage: toError(error).message, + }, + { trusted: true } + ) + } if (admission.targetReservation) { await releaseExecutionSlot(childExecutionId) } diff --git a/apps/sim/lib/copilot/tools/server/jobs/get-job-logs.test.ts b/apps/sim/lib/copilot/tools/server/jobs/get-job-logs.test.ts new file mode 100644 index 00000000000..a77b0a606cb --- /dev/null +++ b/apps/sim/lib/copilot/tools/server/jobs/get-job-logs.test.ts @@ -0,0 +1,131 @@ +/** + * @vitest-environment node + */ +import { dbChainMockFns, resetDbChainMock } from '@sim/testing' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { checkWorkspaceAccessMock, materializeExecutionDataForDisplayMock } = vi.hoisted(() => ({ + checkWorkspaceAccessMock: vi.fn(), + materializeExecutionDataForDisplayMock: vi.fn(), +})) + +vi.mock('@/lib/workspaces/permissions/utils', () => ({ + checkWorkspaceAccess: checkWorkspaceAccessMock, +})) + +vi.mock('@/lib/logs/execution/trace-store', () => ({ + materializeExecutionDataForDisplay: materializeExecutionDataForDisplayMock, +})) + +import { getJobLogsServerTool } from './get-job-logs' + +const RAW_SECRET = 'sk-job-log-secret' +const MASKED_SECRET = '{{OPENAI_API_KEY}}' +const CONTEXT = { userId: 'user-1', workspaceId: 'workspace-1' } + +function jobLogRow(overrides: Record = {}) { + return { + id: 'log-1', + executionId: 'execution-1', + status: 'success', + level: 'info', + trigger: 'schedule', + startedAt: new Date('2026-07-31T00:00:00.000Z'), + endedAt: new Date('2026-07-31T00:00:01.000Z'), + totalDurationMs: 1000, + executionData: { + finalOutput: { result: RAW_SECRET }, + traceSpans: [{ id: 'span-1', output: { result: RAW_SECRET } }], + }, + cost: null, + ...overrides, + } +} + +describe('getJobLogsServerTool', () => { + afterAll(resetDbChainMock) + + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + checkWorkspaceAccessMock.mockResolvedValue({ hasAccess: true }) + }) + + it('returns the secret-safe log projection instead of raw successful output', async () => { + const row = jobLogRow() + dbChainMockFns.limit.mockResolvedValueOnce([row]) + materializeExecutionDataForDisplayMock.mockResolvedValueOnce({ + finalOutput: { result: MASKED_SECRET }, + traceSpans: [ + { + id: 'span-1', + name: 'Function 1', + type: 'function', + status: 'success', + duration: 1000, + startTime: '2026-07-31T00:00:00.000Z', + endTime: '2026-07-31T00:00:01.000Z', + output: { result: MASKED_SECRET }, + }, + ], + }) + + const result = await getJobLogsServerTool.execute( + { jobId: 'schedule-1', includeDetails: true }, + CONTEXT + ) + + expect(result).toEqual([ + expect.objectContaining({ + executionId: 'execution-1', + output: { result: MASKED_SECRET }, + }), + ]) + expect(JSON.stringify(result)).not.toContain(RAW_SECRET) + expect(materializeExecutionDataForDisplayMock).toHaveBeenCalledWith(row.executionData, { + workspaceId: 'workspace-1', + workflowId: null, + executionId: 'execution-1', + userId: 'user-1', + }) + }) + + it('does not fall back to raw output or errors when provenance is incomplete', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + jobLogRow({ + status: 'error', + executionData: { + finalOutput: { error: RAW_SECRET }, + error: RAW_SECRET, + traceSpans: [{ id: 'span-1', output: { error: RAW_SECRET } }], + }, + }), + ]) + materializeExecutionDataForDisplayMock.mockResolvedValueOnce({ + traceSpans: [ + { + id: 'span-1', + name: 'Function 1', + type: 'function', + status: 'error', + duration: 1000, + startTime: '2026-07-31T00:00:00.000Z', + endTime: '2026-07-31T00:00:01.000Z', + }, + ], + }) + + const result = await getJobLogsServerTool.execute( + { jobId: 'schedule-1', includeDetails: true }, + CONTEXT + ) + + expect(result).toEqual([ + expect.not.objectContaining({ + output: expect.anything(), + error: expect.anything(), + }), + ]) + expect(JSON.stringify(result)).not.toContain(RAW_SECRET) + }) +}) diff --git a/apps/sim/lib/copilot/tools/server/jobs/get-job-logs.ts b/apps/sim/lib/copilot/tools/server/jobs/get-job-logs.ts index bdead9b45bc..f8c9c60c9bb 100644 --- a/apps/sim/lib/copilot/tools/server/jobs/get-job-logs.ts +++ b/apps/sim/lib/copilot/tools/server/jobs/get-job-logs.ts @@ -4,6 +4,7 @@ import { createLogger } from '@sim/logger' import { and, desc, eq } from 'drizzle-orm' import { GetScheduledTaskLogs } from '@/lib/copilot/generated/tool-catalog-v1' import type { BaseServerTool, ServerToolContext } from '@/lib/copilot/tools/server/base-tool' +import { materializeExecutionDataForDisplay } from '@/lib/logs/execution/trace-store' import type { TraceSpan } from '@/lib/logs/types' import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils' @@ -188,34 +189,47 @@ export const getJobLogsServerTool: BaseServerTool .orderBy(desc(jobExecutionLogs.startedAt)) .limit(executionId ? 1 : clampedLimit) - const entries: JobLogEntry[] = rows.map((row) => { - const executionData = row.executionData as any - const details = includeDetails ? extractOutputAndError(executionData) : null - - const entry: JobLogEntry = { - executionId: row.executionId, - status: row.status, - trigger: row.trigger, - startedAt: row.startedAt.toISOString(), - endedAt: row.endedAt ? row.endedAt.toISOString() : null, - durationMs: row.totalDurationMs ?? null, - } + const entries: JobLogEntry[] = await Promise.all( + rows.map(async (row) => { + const executionData = await materializeExecutionDataForDisplay( + row.executionData as Record | null, + { + workspaceId: wsId, + workflowId: null, + executionId: row.executionId, + userId: context.userId, + } + ) + const details = includeDetails ? extractOutputAndError(executionData) : null + + const entry: JobLogEntry = { + executionId: row.executionId, + status: row.status, + trigger: row.trigger, + startedAt: row.startedAt.toISOString(), + endedAt: row.endedAt ? row.endedAt.toISOString() : null, + durationMs: row.totalDurationMs ?? null, + } - if (details) { - if (details.error) entry.error = details.error - if (details.toolCalls.length > 0) entry.toolCalls = details.toolCalls - if (details.output) entry.output = details.output - if (details.cost) entry.cost = details.cost - if (details.tokens) entry.tokens = details.tokens - } else { - const errorMsg = executionData?.error || executionData?.traceSpans?.[0]?.output?.error - if (row.status === 'error' && errorMsg) { - entry.error = typeof errorMsg === 'string' ? errorMsg : JSON.stringify(errorMsg) + if (details) { + if (details.error) entry.error = details.error + if (details.toolCalls.length > 0) entry.toolCalls = details.toolCalls + if (details.output) entry.output = details.output + if (details.cost) entry.cost = details.cost + if (details.tokens) entry.tokens = details.tokens + } else { + const traceSpans = Array.isArray(executionData.traceSpans) + ? (executionData.traceSpans as TraceSpan[]) + : [] + const errorMsg = executionData.error || traceSpans[0]?.output?.error + if (row.status === 'error' && errorMsg) { + entry.error = typeof errorMsg === 'string' ? errorMsg : JSON.stringify(errorMsg) + } } - } - return entry - }) + return entry + }) + ) logger.info('Job logs prepared', { jobId, diff --git a/apps/sim/lib/core/telemetry.ts b/apps/sim/lib/core/telemetry.ts index 125592d0ad7..a36c7ff28b2 100644 --- a/apps/sim/lib/core/telemetry.ts +++ b/apps/sim/lib/core/telemetry.ts @@ -18,7 +18,6 @@ import { context, type Span, SpanStatusCode, trace } from '@opentelemetry/api' import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1' import type { TraceSpan } from '@/lib/logs/types' import { hostedKeyMetrics } from '@/lib/monitoring/metrics' @@ -335,7 +334,6 @@ export function createOTelSpansForWorkflowExecution(params: { endTime: string totalDurationMs: number status: 'success' | 'error' - error?: string }): void { try { const tracer = getTracer() @@ -358,11 +356,8 @@ export function createOTelSpansForWorkflowExecution(params: { if (params.status === 'error') { rootSpan.setStatus({ code: SpanStatusCode.ERROR, - message: params.error || 'Workflow execution failed', + message: 'Workflow execution failed', }) - if (params.error) { - rootSpan.recordException(new Error(params.error)) - } } else { rootSpan.setStatus({ code: SpanStatusCode.OK }) } @@ -387,52 +382,6 @@ export function createOTelSpansForWorkflowExecution(params: { } } -/** - * Create a real-time OpenTelemetry span for a block execution - * Can be called from block handlers during execution for real-time tracing - */ -export async function traceBlockExecution( - blockType: string, - blockId: string, - blockName: string, - fn: (span: Span) => Promise -): Promise { - const tracer = getTracer() - - const blockMapping = BLOCK_TYPE_MAPPING[blockType] || { - spanName: `block.${blockType}`, - spanKind: 'internal', - getAttributes: () => ({}), - } - - return tracer.startActiveSpan( - blockMapping.spanName, - { - attributes: { - [TraceAttr.BlockType]: blockType, - [TraceAttr.BlockId]: blockId, - [TraceAttr.BlockName]: blockName, - }, - }, - async (span) => { - try { - const result = await fn(span) - span.setStatus({ code: SpanStatusCode.OK }) - return result - } catch (error) { - span.setStatus({ - code: SpanStatusCode.ERROR, - message: getErrorMessage(error, 'Block execution failed'), - }) - span.recordException(toError(error)) - throw error - } finally { - span.end() - } - } - ) -} - /** * Track platform events (workflow creation, knowledge base operations, etc.) */ @@ -672,7 +621,6 @@ export const PlatformEvents = { blocksExecuted: number hasErrors: boolean totalCost?: number - errorMessage?: string }) => { trackPlatformEvent('platform.workflow.executed', { 'workflow.id': attrs.workflowId, @@ -682,7 +630,6 @@ export const PlatformEvents = { 'execution.blocks_executed': attrs.blocksExecuted, 'execution.has_errors': attrs.hasErrors, ...(attrs.totalCost !== undefined && { 'execution.total_cost': attrs.totalCost }), - ...(attrs.errorMessage && { 'execution.error_message': attrs.errorMessage }), }) }, diff --git a/apps/sim/lib/data-drains/sources/job-logs.ts b/apps/sim/lib/data-drains/sources/job-logs.ts index dcb2d9c98c5..d842fefbfa8 100644 --- a/apps/sim/lib/data-drains/sources/job-logs.ts +++ b/apps/sim/lib/data-drains/sources/job-logs.ts @@ -1,6 +1,7 @@ import { dbReplica } from '@sim/db' import { jobExecutionLogs } from '@sim/db/schema' import { and, inArray, isNotNull } from 'drizzle-orm' +import { MATERIALIZE_CONCURRENCY, mapWithConcurrency } from '@/lib/core/utils/concurrency' import { decodeTimeCursor, encodeTimeCursor, @@ -10,6 +11,7 @@ import { } from '@/lib/data-drains/sources/cursor' import { getOrganizationWorkspaceIds } from '@/lib/data-drains/sources/helpers' import type { Cursor, DrainSource, SourcePageInput } from '@/lib/data-drains/types' +import { materializeExecutionDataForDisplay } from '@/lib/logs/execution/trace-store' type JobLogRow = typeof jobExecutionLogs.$inferSelect @@ -40,6 +42,16 @@ async function* pages(input: SourcePageInput): AsyncIterable { .limit(input.chunkSize) if (rows.length === 0) return + const displayExecutionData = await mapWithConcurrency(rows, MATERIALIZE_CONCURRENCY, (row) => + materializeExecutionDataForDisplay(row.executionData as Record | null, { + workspaceId: row.workspaceId, + workflowId: null, + executionId: row.executionId, + }) + ) + for (let index = 0; index < rows.length; index += 1) { + rows[index].executionData = displayExecutionData[index] as JobLogRow['executionData'] + } yield rows const last = rows[rows.length - 1] cursor = { ts: last.endedAt!.toISOString(), id: last.id } diff --git a/apps/sim/lib/data-drains/sources/workflow-logs.ts b/apps/sim/lib/data-drains/sources/workflow-logs.ts index 4466c5a4c60..a85cd21349a 100644 --- a/apps/sim/lib/data-drains/sources/workflow-logs.ts +++ b/apps/sim/lib/data-drains/sources/workflow-logs.ts @@ -11,7 +11,7 @@ import { } from '@/lib/data-drains/sources/cursor' import { getOrganizationWorkspaceIds } from '@/lib/data-drains/sources/helpers' import type { Cursor, DrainSource, SourcePageInput } from '@/lib/data-drains/types' -import { materializeExecutionData } from '@/lib/logs/execution/trace-store' +import { materializeExecutionDataForDisplay } from '@/lib/logs/execution/trace-store' type WorkflowLogRow = typeof workflowExecutionLogs.$inferSelect @@ -55,7 +55,7 @@ async function* pages(input: SourcePageInput): AsyncIterable { // Use the order-preserving returned array (the util's documented contract) // and write back, rather than mutating rows inside the mapper. const materialized = await mapWithConcurrency(rows, MATERIALIZE_CONCURRENCY, (row) => - materializeExecutionData(row.executionData as Record | null, { + materializeExecutionDataForDisplay(row.executionData as Record | null, { workspaceId: row.workspaceId, workflowId: row.workflowId, executionId: row.executionId, diff --git a/apps/sim/lib/execution/payloads/serializer.ts b/apps/sim/lib/execution/payloads/serializer.ts index c3d0079defb..e8239f9b141 100644 --- a/apps/sim/lib/execution/payloads/serializer.ts +++ b/apps/sim/lib/execution/payloads/serializer.ts @@ -23,6 +23,8 @@ interface CompactState { seen: WeakSet } +const BLOCK_LOG_COMPACTION_CONCURRENCY = 4 + function getJsonAndSize(value: unknown): { json: string; size: number } | null { try { const json = JSON.stringify(value) @@ -260,8 +262,15 @@ export async function compactBlockLogs( return logs } - return Promise.all( - logs.map(async (log) => { + const compactedLogs = new Array(logs.length) + let cursor = 0 + const worker = async (): Promise => { + while (true) { + const index = cursor + cursor += 1 + if (index >= logs.length) return + + const log = logs[index] const compactedLog = { ...log } if ('input' in compactedLog) { compactedLog.input = await compactExecutionPayload(compactedLog.input, options) @@ -275,7 +284,12 @@ export async function compactBlockLogs( options ) } - return compactedLog - }) + compactedLogs[index] = compactedLog + } + } + + await Promise.all( + Array.from({ length: Math.min(BLOCK_LOG_COMPACTION_CONCURRENCY, logs.length) }, worker) ) + return compactedLogs } diff --git a/apps/sim/lib/execution/private-tool-metadata.test.ts b/apps/sim/lib/execution/private-tool-metadata.test.ts new file mode 100644 index 00000000000..5590763f14d --- /dev/null +++ b/apps/sim/lib/execution/private-tool-metadata.test.ts @@ -0,0 +1,62 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + getPrivateToolMetadataField, + isPrivateToolMetadataType, + PRIVATE_TOOL_METADATA_REQUEST_HEADER, + PRIVATE_TOOL_METADATA_RESPONSE_HEADER, + RESOLVED_SECRET_NAMES_FIELD, + RESOLVED_SECRET_NAMES_METADATA_V1, + RESOLVED_SECRET_PROVENANCE_FIELD, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, + requestsPrivateToolMetadata, + responseHasPrivateToolMetadata, +} from '@/lib/execution/private-tool-metadata' + +describe('private tool metadata protocol', () => { + it('keeps the versioned wire markers stable', () => { + expect(PRIVATE_TOOL_METADATA_REQUEST_HEADER).toBe('x-sim-request-private-tool-metadata') + expect(PRIVATE_TOOL_METADATA_RESPONSE_HEADER).toBe('x-sim-private-tool-metadata') + expect(RESOLVED_SECRET_NAMES_METADATA_V1).toBe('resolved-secret-names-v1') + expect(RESOLVED_SECRET_PROVENANCE_METADATA_V1).toBe('resolved-secret-provenance-v1') + expect(RESOLVED_SECRET_NAMES_FIELD).toBe('__resolvedSecretNames') + expect(RESOLVED_SECRET_PROVENANCE_FIELD).toBe('__resolvedSecretTraceProvenance') + }) + + it('accepts only exact request and response markers', () => { + const requestHeaders = new Headers({ + [PRIVATE_TOOL_METADATA_REQUEST_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1, + }) + const responseHeaders = new Headers({ + [PRIVATE_TOOL_METADATA_RESPONSE_HEADER]: RESOLVED_SECRET_NAMES_METADATA_V1, + }) + + expect( + requestsPrivateToolMetadata(requestHeaders, RESOLVED_SECRET_PROVENANCE_METADATA_V1) + ).toBe(true) + expect(requestsPrivateToolMetadata(requestHeaders, RESOLVED_SECRET_NAMES_METADATA_V1)).toBe( + false + ) + expect(responseHasPrivateToolMetadata(responseHeaders, RESOLVED_SECRET_NAMES_METADATA_V1)).toBe( + true + ) + expect( + responseHasPrivateToolMetadata(responseHeaders, RESOLVED_SECRET_PROVENANCE_METADATA_V1) + ).toBe(false) + }) + + it('maps each marker to its private payload field', () => { + expect(isPrivateToolMetadataType(RESOLVED_SECRET_NAMES_METADATA_V1)).toBe(true) + expect(isPrivateToolMetadataType(RESOLVED_SECRET_PROVENANCE_METADATA_V1)).toBe(true) + expect(isPrivateToolMetadataType('resolved-secret-provenance-v2')).toBe(false) + expect(isPrivateToolMetadataType(null)).toBe(false) + expect(getPrivateToolMetadataField(RESOLVED_SECRET_NAMES_METADATA_V1)).toBe( + RESOLVED_SECRET_NAMES_FIELD + ) + expect(getPrivateToolMetadataField(RESOLVED_SECRET_PROVENANCE_METADATA_V1)).toBe( + RESOLVED_SECRET_PROVENANCE_FIELD + ) + }) +}) diff --git a/apps/sim/lib/execution/private-tool-metadata.ts b/apps/sim/lib/execution/private-tool-metadata.ts new file mode 100644 index 00000000000..9f61b9098de --- /dev/null +++ b/apps/sim/lib/execution/private-tool-metadata.ts @@ -0,0 +1,44 @@ +export const PRIVATE_TOOL_METADATA_REQUEST_HEADER = 'x-sim-request-private-tool-metadata' +export const PRIVATE_TOOL_METADATA_RESPONSE_HEADER = 'x-sim-private-tool-metadata' + +export const RESOLVED_SECRET_NAMES_METADATA_V1 = 'resolved-secret-names-v1' +export const RESOLVED_SECRET_PROVENANCE_METADATA_V1 = 'resolved-secret-provenance-v1' + +export const RESOLVED_SECRET_NAMES_FIELD = '__resolvedSecretNames' +export const RESOLVED_SECRET_PROVENANCE_FIELD = '__resolvedSecretTraceProvenance' + +export type PrivateToolMetadataType = + | typeof RESOLVED_SECRET_NAMES_METADATA_V1 + | typeof RESOLVED_SECRET_PROVENANCE_METADATA_V1 + +interface HeaderReader { + get(name: string): string | null +} + +export function isPrivateToolMetadataType(value: string | null): value is PrivateToolMetadataType { + return ( + value === RESOLVED_SECRET_NAMES_METADATA_V1 || value === RESOLVED_SECRET_PROVENANCE_METADATA_V1 + ) +} + +export function requestsPrivateToolMetadata( + headers: HeaderReader, + expectedType: PrivateToolMetadataType +): boolean { + return headers.get(PRIVATE_TOOL_METADATA_REQUEST_HEADER) === expectedType +} + +export function responseHasPrivateToolMetadata( + headers: HeaderReader, + expectedType: PrivateToolMetadataType +): boolean { + return headers.get(PRIVATE_TOOL_METADATA_RESPONSE_HEADER) === expectedType +} + +export function getPrivateToolMetadataField( + type: PrivateToolMetadataType +): typeof RESOLVED_SECRET_NAMES_FIELD | typeof RESOLVED_SECRET_PROVENANCE_FIELD { + return type === RESOLVED_SECRET_NAMES_METADATA_V1 + ? RESOLVED_SECRET_NAMES_FIELD + : RESOLVED_SECRET_PROVENANCE_FIELD +} diff --git a/apps/sim/lib/logs/execution/display-types.ts b/apps/sim/lib/logs/execution/display-types.ts new file mode 100644 index 00000000000..77211011a0a --- /dev/null +++ b/apps/sim/lib/logs/execution/display-types.ts @@ -0,0 +1,6 @@ +import type { BlockLog } from '@/executor/types' + +/** A server-projected BlockLog copy carrying presentation-only reconciliation hints. */ +export interface SecretSafeBlockLog extends BlockLog { + clearLiveDisplay?: true +} diff --git a/apps/sim/lib/logs/execution/functional-outputs.test.ts b/apps/sim/lib/logs/execution/functional-outputs.test.ts new file mode 100644 index 00000000000..d52719d383f --- /dev/null +++ b/apps/sim/lib/logs/execution/functional-outputs.test.ts @@ -0,0 +1,66 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + collectFunctionalBlockOutputs, + FUNCTIONAL_OUTPUTS_UNAVAILABLE_MESSAGE, + FunctionalOutputsUnavailableError, + getFunctionalBlockOutput, +} from '@/lib/logs/execution/functional-outputs' + +describe('functional execution outputs', () => { + it('prefers raw execution state over projected TraceSpan output', () => { + const data = { + executionState: { + blockStates: { + 'function-1': { output: { result: 1234 } }, + }, + }, + traceSpans: [ + { + blockId: 'function-1', + output: { result: '{{OPENAI_API_KEY}}' }, + }, + ], + } + + expect(getFunctionalBlockOutput(data, 'function-1')).toEqual({ result: 1234 }) + }) + + it('uses nested TraceSpan output for legacy rows without execution state', () => { + const outputs = collectFunctionalBlockOutputs({ + traceSpans: [ + { + children: [{ blockId: 'function-1', output: { result: 'legacy' } }], + }, + ], + }) + + expect(outputs.get('function-1')).toEqual({ result: 'legacy' }) + }) + + it('does not mix projected TraceSpan output into an available raw state', () => { + const outputs = collectFunctionalBlockOutputs({ + executionState: { + blockStates: { + 'function-1': { output: { result: 1234 } }, + }, + }, + traceSpans: [{ blockId: 'function-2', output: { result: '{{OPENAI_API_KEY}}' } }], + }) + + expect(outputs.get('function-1')).toEqual({ result: 1234 }) + expect(outputs.has('function-2')).toBe(false) + }) + + it('fails instead of returning projected traces when raw state was truncated', () => { + expect(() => + collectFunctionalBlockOutputs({ + executionDataTruncated: true, + traceSpans: [{ blockId: 'function-1', output: { result: '{{OPENAI_API_KEY}}' } }], + }) + ).toThrowError(new FunctionalOutputsUnavailableError()) + expect(FUNCTIONAL_OUTPUTS_UNAVAILABLE_MESSAGE).toContain('could not be retained in full') + }) +}) diff --git a/apps/sim/lib/logs/execution/functional-outputs.ts b/apps/sim/lib/logs/execution/functional-outputs.ts new file mode 100644 index 00000000000..3c50ea138e5 --- /dev/null +++ b/apps/sim/lib/logs/execution/functional-outputs.ts @@ -0,0 +1,62 @@ +export interface FunctionalTraceSpanSource { + blockId?: string + output?: unknown + children?: FunctionalTraceSpanSource[] +} + +export interface FunctionalExecutionDataSource { + traceSpans?: FunctionalTraceSpanSource[] + executionState?: { + blockStates?: Record + } + executionDataTruncated?: boolean +} + +export const FUNCTIONAL_OUTPUTS_UNAVAILABLE_MESSAGE = + 'Raw block outputs are unavailable because the execution data could not be retained in full.' + +export class FunctionalOutputsUnavailableError extends Error { + constructor() { + super(FUNCTIONAL_OUTPUTS_UNAVAILABLE_MESSAGE) + this.name = 'FunctionalOutputsUnavailableError' + } +} + +function collectTraceBlockOutputs( + spans: FunctionalTraceSpanSource[] | undefined, + outputs: Map +): void { + if (!spans) return + for (const span of spans) { + if (span.blockId && span.output !== undefined && !outputs.has(span.blockId)) { + outputs.set(span.blockId, span.output) + } + collectTraceBlockOutputs(span.children, outputs) + } +} + +/** Returns functional block outputs, preferring raw execution state over display TraceSpans. */ +export function collectFunctionalBlockOutputs( + data: FunctionalExecutionDataSource | undefined +): Map { + const outputs = new Map() + const blockStates = data?.executionState?.blockStates + if (blockStates) { + for (const [blockId, blockState] of Object.entries(blockStates)) { + if (blockState?.output !== undefined) outputs.set(blockId, blockState.output) + } + return outputs + } + + if (data?.executionDataTruncated) throw new FunctionalOutputsUnavailableError() + + collectTraceBlockOutputs(data?.traceSpans, outputs) + return outputs +} + +export function getFunctionalBlockOutput( + data: FunctionalExecutionDataSource | undefined, + blockId: string +): unknown { + return collectFunctionalBlockOutputs(data).get(blockId) +} diff --git a/apps/sim/lib/logs/execution/logger.test.ts b/apps/sim/lib/logs/execution/logger.test.ts index 5ae0036ed00..f79f83d915d 100644 --- a/apps/sim/lib/logs/execution/logger.test.ts +++ b/apps/sim/lib/logs/execution/logger.test.ts @@ -3,6 +3,8 @@ import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' import { afterAll, beforeEach, describe, expect, test, vi } from 'vitest' import { recordUsage } from '@/lib/billing/core/usage-log' import { ExecutionLogger } from '@/lib/logs/execution/logger' +import type { WorkflowExecutionLog } from '@/lib/logs/types' +import type { SerializableExecutionState } from '@/executor/execution/types' afterAll(resetDbChainMock) @@ -242,6 +244,56 @@ describe('ExecutionLogger', () => { expect(completedData.billingAttribution).toEqual(billingAttribution) }) + test('preserves server-only lifecycle metadata after execution-state PII masking', () => { + const loggerInstance = new ExecutionLogger() as unknown as { + preservePrivateExecutionStateMetadata( + redactedState: SerializableExecutionState | undefined, + originalState: SerializableExecutionState | undefined + ): SerializableExecutionState | undefined + } + const provenance = { + version: 1 as const, + complete: true, + entries: [{ name: 'API_SECRET', encryptedValue: 'enc:original-ciphertext' }], + } + const trustedLargeValueAccess = { + executionIds: ['execution-1'], + largeValueKeys: ['execution/workspace-1/workflow-1/execution-1/value.json'], + fileKeys: ['workspace-1/file-1'], + } + const originalState: SerializableExecutionState = { + blockStates: {}, + executedBlocks: [], + blockLogs: [], + decisions: { router: {}, condition: {} }, + completedLoops: [], + activeExecutionPath: [], + resolvedSecretTraceProvenance: provenance, + trustedLargeValueAccess, + } + const redactedState: SerializableExecutionState = { + ...originalState, + resolvedSecretTraceProvenance: { + ...provenance, + entries: [{ name: 'API_SECRET', encryptedValue: '[MASKED]' }], + }, + trustedLargeValueAccess: { + executionIds: [], + largeValueKeys: [], + fileKeys: [], + }, + } + + const preserved = loggerInstance.preservePrivateExecutionStateMetadata( + redactedState, + originalState + ) + + expect(preserved?.resolvedSecretTraceProvenance).toBe(provenance) + expect(preserved?.trustedLargeValueAccess).toBe(trustedLargeValueAccess) + expect(preserved?.blockStates).toBe(redactedState.blockStates) + }) + test('summarizes oversized execution data before storage', () => { const loggerInstance = new ExecutionLogger() as any const largePayload = 'x'.repeat(1_100_000) @@ -325,12 +377,71 @@ describe('ExecutionLogger', () => { activeExecutionPathLength: 0, pendingQueueLength: 0, }) - expect(compacted.traceSpans?.[0]?.children?.[0]?.input).toEqual({ - _truncated: true, - reason: 'execution_data_size_limit', - originalBytes: expect.any(Number), - summary: 'object with 2 keys', + expect(compacted.traceSpans?.[0]?.children?.[0]).not.toHaveProperty('input') + }) + + test('retains tool-call structure when aggregate trace content exceeds the compaction cap', () => { + const loggerInstance = new ExecutionLogger() as unknown as { + compactExecutionDataForStorage( + executionData: WorkflowExecutionLog['executionData'], + executionId: string + ): WorkflowExecutionLog['executionData'] + } + const oversizedContent = 'x'.repeat(9_000) + const modelToolCalls = Array.from({ length: 200 }, (_, index) => ({ + id: `model-call-${index}-${'m'.repeat(40)}`, + name: 'lookup', + arguments: oversizedContent, + })) + const toolCalls = Array.from({ length: 200 }, (_, index) => ({ + id: `legacy-call-${index}-${'l'.repeat(40)}`, + name: 'legacy_lookup', + duration: 1, + startTime: '2025-01-01T00:00:00.000Z', + endTime: '2025-01-01T00:00:00.001Z', + status: 'success' as const, + input: oversizedContent, + output: oversizedContent, + error: oversizedContent, + })) + + const compacted = loggerInstance.compactExecutionDataForStorage( + { + traceSpans: [ + { + id: 'span-1', + name: 'Agent', + type: 'agent', + duration: 1, + startTime: '2025-01-01T00:00:00.000Z', + endTime: '2025-01-01T00:00:00.001Z', + status: 'success', + modelToolCalls, + toolCalls, + }, + ], + finalOutput: { data: 'y'.repeat(1_100_000) }, + }, + 'execution-tool-structure' + ) + + expect(compacted.executionDataTruncated).toBe(true) + expect(compacted.traceSpans?.[0]?.modelToolCalls).toHaveLength(modelToolCalls.length) + expect(compacted.traceSpans?.[0]?.modelToolCalls?.[0]).toEqual({ + id: modelToolCalls[0].id, + name: 'lookup', }) + expect(compacted.traceSpans?.[0]?.toolCalls).toHaveLength(toolCalls.length) + expect(compacted.traceSpans?.[0]?.toolCalls?.[0]).toEqual( + expect.objectContaining({ + id: toolCalls[0].id, + name: 'legacy_lookup', + status: 'success', + }) + ) + expect(compacted.traceSpans?.[0]?.toolCalls?.[0]).not.toHaveProperty('input') + expect(compacted.traceSpans?.[0]?.toolCalls?.[0]).not.toHaveProperty('output') + expect(compacted.traceSpans?.[0]?.toolCalls?.[0]).not.toHaveProperty('error') }) }) diff --git a/apps/sim/lib/logs/execution/logger.ts b/apps/sim/lib/logs/execution/logger.ts index 78729e44783..1fb46351681 100644 --- a/apps/sim/lib/logs/execution/logger.ts +++ b/apps/sim/lib/logs/execution/logger.ts @@ -51,8 +51,9 @@ import { import { snapshotService } from '@/lib/logs/execution/snapshot/service' import { traceSpansIndicateFailure } from '@/lib/logs/execution/trace-spans/trace-spans' import { + copyTraceSpansWithoutCosts, externalizeExecutionData, - stripSpanCosts, + materializeExecutionData, TRACE_STORE_REF_KEY, } from '@/lib/logs/execution/trace-store' import type { @@ -186,13 +187,87 @@ function summarizeValueForExecutionData(value: unknown, maxBytes: number): unkno } } -function summarizeTextForExecutionData(value: string | undefined): string | undefined { - if (!value) return value - const size = getJsonByteSize(value, MAX_TRACE_IO_BYTES) - if (size === undefined || size <= MAX_TRACE_IO_BYTES) { - return value +function retainBoundedTraceContent(value: T, maxBytes = MAX_TRACE_IO_BYTES): T | undefined { + const size = getJsonByteSize(value, maxBytes) + return size !== undefined && size <= maxBytes ? value : undefined +} + +function stripModelToolCallArguments( + calls: NonNullable +): NonNullable { + return calls.map(({ arguments: _arguments, ...call }) => call as (typeof calls)[number]) +} + +function compactModelToolCalls( + calls: NonNullable +): NonNullable | undefined { + const compacted = calls.map(({ arguments: callArguments, ...call }) => { + const retainedArguments = retainBoundedTraceContent(callArguments) + return { + ...call, + ...(retainedArguments !== undefined ? { arguments: retainedArguments } : {}), + } as (typeof calls)[number] + }) + return retainBoundedTraceContent(compacted) +} + +function compactLegacyToolCalls( + calls: NonNullable +): NonNullable | undefined { + const compacted = calls.map(({ input, output, error, ...call }) => ({ + ...call, + ...(retainBoundedTraceContent(input) !== undefined ? { input } : {}), + ...(retainBoundedTraceContent(output) !== undefined ? { output } : {}), + ...(retainBoundedTraceContent(error) !== undefined ? { error } : {}), + })) + return retainBoundedTraceContent(compacted) +} + +function stripLegacyToolCallContent( + calls: NonNullable +): NonNullable { + return calls.map(({ input: _input, output: _output, error: _error, ...call }) => call) +} + +function compactProviderTiming( + providerTiming: NonNullable +): NonNullable { + return { + ...providerTiming, + segments: providerTiming.segments.map( + ({ assistantContent, thinkingContent, errorMessage, toolCalls, ...segment }) => ({ + ...segment, + ...(retainBoundedTraceContent(assistantContent) !== undefined ? { assistantContent } : {}), + ...(retainBoundedTraceContent(thinkingContent) !== undefined ? { thinkingContent } : {}), + ...(retainBoundedTraceContent(errorMessage) !== undefined ? { errorMessage } : {}), + ...(toolCalls + ? { + toolCalls: compactModelToolCalls(toolCalls) ?? stripModelToolCallArguments(toolCalls), + } + : {}), + }) + ), + } +} + +function stripProviderTimingContent( + providerTiming: NonNullable +): NonNullable { + return { + ...providerTiming, + segments: providerTiming.segments.map( + ({ + assistantContent: _assistantContent, + thinkingContent: _thinkingContent, + errorMessage: _errorMessage, + toolCalls, + ...segment + }) => ({ + ...segment, + ...(toolCalls ? { toolCalls: stripModelToolCallArguments(toolCalls) } : {}), + }) + ), } - return `[Truncated ${size} byte text value due to execution log size limit]` } function summarizeTraceSpansForExecutionData(traceSpans?: TraceSpan[]): TraceSpan[] | undefined { @@ -201,33 +276,39 @@ function summarizeTraceSpansForExecutionData(traceSpans?: TraceSpan[]): TraceSpa } return traceSpans.map((span) => { - const { input, output, children, thinking, modelToolCalls, ...rest } = span + const { + input, + output, + children, + thinking, + errorMessage, + modelToolCalls, + toolCalls, + providerTiming, + ...rest + } = span const summarized: TraceSpan = { ...rest } - if (input !== undefined) { - summarized.input = summarizeValueForExecutionData(input, MAX_TRACE_IO_BYTES) as Record< - string, - unknown - > - } - if (output !== undefined) { - summarized.output = summarizeValueForExecutionData(output, MAX_TRACE_IO_BYTES) as Record< - string, - unknown - > - } + const retainedInput = retainBoundedTraceContent(input) + if (retainedInput !== undefined) summarized.input = retainedInput + const retainedOutput = retainBoundedTraceContent(output) + if (retainedOutput !== undefined) summarized.output = retainedOutput if (children?.length) { summarized.children = summarizeTraceSpansForExecutionData(children) } - if (thinking !== undefined) { - summarized.thinking = summarizeTextForExecutionData(thinking) + const retainedThinking = retainBoundedTraceContent(thinking) + if (retainedThinking !== undefined) summarized.thinking = retainedThinking + const retainedError = retainBoundedTraceContent(errorMessage) + if (retainedError !== undefined) summarized.errorMessage = retainedError + if (modelToolCalls) { + summarized.modelToolCalls = + compactModelToolCalls(modelToolCalls) ?? stripModelToolCallArguments(modelToolCalls) } - if ( - modelToolCalls !== undefined && - (getJsonByteSize(modelToolCalls, MAX_TRACE_IO_BYTES) ?? 0) <= MAX_TRACE_IO_BYTES - ) { - summarized.modelToolCalls = modelToolCalls + if (toolCalls) { + summarized.toolCalls = + compactLegacyToolCalls(toolCalls) ?? stripLegacyToolCallContent(toolCalls) } + if (providerTiming) summarized.providerTiming = compactProviderTiming(providerTiming) return summarized }) @@ -244,11 +325,17 @@ function summarizeTraceSpansWithoutIo(traceSpans?: TraceSpan[]): TraceSpan[] | u output: _output, children, thinking: _thinking, - modelToolCalls: _modelToolCalls, + errorMessage: _errorMessage, + modelToolCalls, + toolCalls, + providerTiming, ...rest } = span return { ...rest, + ...(modelToolCalls ? { modelToolCalls: stripModelToolCallArguments(modelToolCalls) } : {}), + ...(toolCalls ? { toolCalls: stripLegacyToolCallContent(toolCalls) } : {}), + ...(providerTiming ? { providerTiming: stripProviderTimingContent(providerTiming) } : {}), ...(children?.length ? { children: summarizeTraceSpansWithoutIo(children) } : {}), } }) @@ -698,6 +785,64 @@ export class ExecutionLogger implements IExecutionLoggerService { }) } + /** Restores server-only lifecycle metadata after broad execution-state PII masking. */ + private preservePrivateExecutionStateMetadata( + redactedState: SerializableExecutionState | undefined, + originalState: SerializableExecutionState | undefined + ): SerializableExecutionState | undefined { + const provenance = originalState?.resolvedSecretTraceProvenance + const trustedLargeValueAccess = originalState?.trustedLargeValueAccess + return redactedState + ? { + ...redactedState, + ...(provenance !== undefined ? { resolvedSecretTraceProvenance: provenance } : {}), + ...(trustedLargeValueAccess !== undefined ? { trustedLargeValueAccess } : {}), + } + : redactedState + } + + async prepareTraceSpansForProjection(params: { + executionId: string + workflowId: string + workspaceId: string | null + userId?: string | null + traceSpans: TraceSpan[] + isResume?: boolean + }): Promise { + let sourceSpans = params.traceSpans + if (params.isResume && sourceSpans.length === 0) { + const [existingLog] = await execDb + .select({ executionData: workflowExecutionLogs.executionData }) + .from(workflowExecutionLogs) + .where(eq(workflowExecutionLogs.executionId, params.executionId)) + .limit(1) + const executionData = await materializeExecutionData( + existingLog?.executionData as Record | null, + { + workspaceId: params.workspaceId, + workflowId: params.workflowId, + executionId: params.executionId, + } + ) + if (Array.isArray(executionData.traceSpans)) { + sourceSpans = executionData.traceSpans as TraceSpan[] + } + } + + const filtered = filterForDisplay(sourceSpans) + const redacted = redactApiKeys(filtered) + const pii = await this.applyPiiRedaction( + params.workspaceId, + { traceSpans: redacted }, + { + workflowId: params.workflowId, + executionId: params.executionId, + userId: params.userId ?? undefined, + } + ) + return pii.traceSpans as TraceSpan[] + } + async completeWorkflowExecution(params: { executionId: string endedAt: string @@ -785,11 +930,7 @@ export class ExecutionLogger implements IExecutionLoggerService { const status = statusOverride ?? (hasErrors ? 'failed' : 'completed') // For resume executions, rebuild trace spans from the aggregated logs - const mergedTraceSpans = isResume - ? traceSpans && traceSpans.length > 0 - ? traceSpans - : existingExecutionData?.traceSpans || [] - : traceSpans + const mergedTraceSpans = traceSpans const executionCost = { total: costSummary.totalCost, @@ -826,13 +967,12 @@ export class ExecutionLogger implements IExecutionLoggerService { builtExecutionData.workflowInput ) - const filteredTraceSpans = filterForDisplay(builtExecutionData.traceSpans) + const preparedTraceSpans = builtExecutionData.traceSpans const filteredFinalOutput = filterForDisplay(builtExecutionData.finalOutput) const filteredWorkflowInput = builtExecutionData.workflowInput !== undefined ? filterForDisplay(builtExecutionData.workflowInput) : undefined - const redactedTraceSpans = redactApiKeys(filteredTraceSpans) const redactedFinalOutput = redactApiKeys(filteredFinalOutput) const redactedWorkflowInput = filteredWorkflowInput !== undefined ? redactApiKeys(filteredWorkflowInput) : undefined @@ -840,7 +980,7 @@ export class ExecutionLogger implements IExecutionLoggerService { const pii = await this.applyPiiRedaction( existingLog?.workspaceId ?? null, { - traceSpans: redactedTraceSpans, + traceSpans: [], finalOutput: redactedFinalOutput, ...(redactedWorkflowInput !== undefined ? { workflowInput: redactedWorkflowInput } : {}), ...(builtExecutionData.error !== undefined ? { error: builtExecutionData.error } : {}), @@ -899,9 +1039,14 @@ export class ExecutionLogger implements IExecutionLoggerService { ? Math.max(0, Math.round(rawDurationMs)) : 0 + const safeExecutionState = this.preservePrivateExecutionStateMetadata( + pii.executionState as SerializableExecutionState | undefined, + builtExecutionData.executionState + ) + const cleanExecutionData: ExecutionData = { ...builtExecutionData, - traceSpans: pii.traceSpans as TraceSpan[], + traceSpans: copyTraceSpansWithoutCosts(preparedTraceSpans), finalOutput: pii.finalOutput as BlockOutputData, ...(pii.workflowInput !== undefined ? { workflowInput: pii.workflowInput } : {}), ...(pii.error !== undefined ? { error: pii.error as string } : {}), @@ -909,9 +1054,7 @@ export class ExecutionLogger implements IExecutionLoggerService { ? { completionFailure: pii.completionFailure as string } : {}), ...(pii.trigger !== undefined ? { trigger: pii.trigger as ExecutionTrigger } : {}), - ...(pii.executionState !== undefined - ? { executionState: pii.executionState as SerializableExecutionState } - : {}), + ...(safeExecutionState !== undefined ? { executionState: safeExecutionState } : {}), ...(pii.environment !== undefined ? { environment: pii.environment as ExecutionEnvironment } : {}), @@ -920,8 +1063,6 @@ export class ExecutionLogger implements IExecutionLoggerService { : {}), } - stripSpanCosts((cleanExecutionData as Record).traceSpans) - // Bounded in-memory form. Returned to callers (notification delivery/events) // and reused as the inline-storage fallback below. This is a no-op for // payloads already within MAX_EXECUTION_DATA_BYTES. diff --git a/apps/sim/lib/logs/execution/logging-session.test.ts b/apps/sim/lib/logs/execution/logging-session.test.ts index 9881c628304..7fd9aa26b3b 100644 --- a/apps/sim/lib/logs/execution/logging-session.test.ts +++ b/apps/sim/lib/logs/execution/logging-session.test.ts @@ -9,14 +9,20 @@ const dbMocks = vi.hoisted(() => ({ const { completeWorkflowExecutionMock, + prepareTraceSpansForProjectionMock, startWorkflowExecutionMock, loadWorkflowStateForExecutionMock, releaseExecutionSlotMock, + createOTelSpansMock, + workflowExecutedMock, } = vi.hoisted(() => ({ completeWorkflowExecutionMock: vi.fn(), + prepareTraceSpansForProjectionMock: vi.fn(), startWorkflowExecutionMock: vi.fn(), loadWorkflowStateForExecutionMock: vi.fn(), releaseExecutionSlotMock: vi.fn(), + createOTelSpansMock: vi.fn(), + workflowExecutedMock: vi.fn(), })) vi.mock('drizzle-orm', () => ({ @@ -29,6 +35,7 @@ vi.mock('@/lib/logs/execution/logger', () => ({ executionLogger: { startWorkflowExecution: startWorkflowExecutionMock, completeWorkflowExecution: completeWorkflowExecutionMock, + prepareTraceSpansForProjection: prepareTraceSpansForProjectionMock, }, })) @@ -36,6 +43,11 @@ vi.mock('@/lib/billing/calculations/usage-reservation', () => ({ releaseExecutionSlot: releaseExecutionSlotMock, })) +vi.mock('@/lib/core/telemetry', () => ({ + createOTelSpansForWorkflowExecution: createOTelSpansMock, + PlatformEvents: { workflowExecuted: workflowExecutedMock }, +})) + const { setLastStartedBlockMock, setLastCompletedBlockMock, @@ -73,11 +85,67 @@ vi.mock('@/lib/logs/execution/logging-factory', () => ({ })) import { calculateCostSummary } from '@/lib/logs/execution/logging-factory' -import { createEnvironmentSecretSanitizer } from '@/executor/utils/environment-secret-sanitizer' +import type { + ResolvedSecretTraceMatch, + ResolvedSecretTraceRegistry, +} from '@/executor/utils/resolved-secret-trace-registry' import { LoggingSession } from './logging-session' afterAll(resetDbChainMock) +function createSecretRegistry( + matches: ResolvedSecretTraceMatch[], + complete = true +): ResolvedSecretTraceRegistry { + return { + isComplete: () => complete, + getActiveMatches: () => matches, + exportProvenance: () => ({ version: 1, complete, entries: [] }), + } as unknown as ResolvedSecretTraceRegistry +} + +describe('LoggingSession terminal provenance', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + dbChainMockFns.limit.mockResolvedValue([]) + completeWorkflowExecutionMock.mockResolvedValue({}) + releaseExecutionSlotMock.mockResolvedValue(undefined) + }) + + it.each([ + [ + 'error', + (session: LoggingSession) => session.completeWithError({ error: { message: 'failed' } }), + ], + ['cancellation', (session: LoggingSession) => session.completeWithCancellation()], + ['pause', (session: LoggingSession) => session.completeWithPause()], + ])('persists complete zero-entry provenance on %s finalization', async (_name, finalize) => { + const session = new LoggingSession('workflow-1', `execution-${_name}`, 'manual') + session.setResolvedSecretTraceRegistry(createSecretRegistry([])) + + await finalize(session) + + expect(completeWorkflowExecutionMock).toHaveBeenCalledWith( + expect.objectContaining({ + executionState: expect.objectContaining({ + resolvedSecretTraceProvenance: { + version: 1, + complete: true, + entries: [], + }, + }), + }) + ) + }) +}) + +beforeEach(() => { + prepareTraceSpansForProjectionMock.mockImplementation( + async ({ traceSpans }: { traceSpans: unknown[] }) => traceSpans + ) +}) + describe('LoggingSession start snapshots', () => { beforeEach(() => { vi.clearAllMocks() @@ -225,6 +293,7 @@ describe('LoggingSession completion retries', () => { it('starts a new error completion attempt after a non-error completion and fallback both fail', async () => { const session = new LoggingSession('workflow-1', 'execution-3', 'api', 'req-1') + session.setResolvedSecretTraceRegistry(createSecretRegistry([])) completeWorkflowExecutionMock .mockRejectedValueOnce(new Error('success finalize failed')) @@ -251,6 +320,7 @@ describe('LoggingSession completion retries', () => { it('preserves successful final output during fallback completion', async () => { const session = new LoggingSession('workflow-1', 'execution-5', 'api', 'req-1') + session.setResolvedSecretTraceRegistry(createSecretRegistry([])) completeWorkflowExecutionMock .mockRejectedValueOnce(new Error('success finalize failed')) @@ -269,7 +339,7 @@ describe('LoggingSession completion retries', () => { ) }) - it('sanitizes workflow output and final trace spans without mutating runtime values', async () => { + it('projects only TraceSpans while preserving functional completion values', async () => { const session = new LoggingSession('workflow-1', 'execution-safe', 'api', 'req-1') const secret = 'sk-demo / trace?token=7f3a91' const rawFinalOutput = { @@ -289,39 +359,32 @@ describe('LoggingSession completion retries', () => { startTime: '2026-07-01T00:00:00.000Z', endTime: '2026-07-01T00:00:00.001Z', status: 'success', - output: { echoed: secret }, + output: { echoed: secret, encoded: encodeURIComponent(secret) }, }, ] + const rawWorkflowInput = { prompt: `use ${secret}` } - session.setEnvironmentSecretSanitizer( - createEnvironmentSecretSanitizer( - { code: 'return "{{OPENAI_API_KEY}}"' }, - { - OPENAI_API_KEY: secret, - UNREFERENCED_REGION: 'us-east-1', - } - ) + session.setResolvedSecretTraceRegistry( + createSecretRegistry([{ plaintext: secret, replacement: '{{OPENAI_API_KEY}}' }]) ) completeWorkflowExecutionMock.mockResolvedValue({}) await session.safeComplete({ finalOutput: rawFinalOutput, traceSpans: rawTraceSpans as any, + workflowInput: rawWorkflowInput, }) expect(completeWorkflowExecutionMock).toHaveBeenCalledWith( expect.objectContaining({ - finalOutput: { - result: { - resolvedAtRuntime: true, - echoed: 'prefix:{{OPENAI_API_KEY}}:suffix', - encoded: '{{OPENAI_API_KEY}}', - ordinary: 'us-east-1', - }, - }, + finalOutput: rawFinalOutput, + workflowInput: rawWorkflowInput, traceSpans: [ expect.objectContaining({ - output: { echoed: '{{OPENAI_API_KEY}}' }, + output: { + echoed: '{{OPENAI_API_KEY}}', + encoded: encodeURIComponent(secret), + }, }), ], }) @@ -329,46 +392,64 @@ describe('LoggingSession completion retries', () => { expect(rawFinalOutput.result.echoed).toBe(`prefix:${secret}:suffix`) expect(rawTraceSpans[0].output.echoed).toBe(secret) expect(calculateCostSummary).toHaveBeenCalledWith(rawTraceSpans, undefined) + + const persistedSpans = completeWorkflowExecutionMock.mock.calls[0]?.[0].traceSpans + expect(createOTelSpansMock).toHaveBeenCalledWith( + expect.objectContaining({ traceSpans: persistedSpans }) + ) + expect(createOTelSpansMock.mock.calls[0]?.[0].traceSpans).toBe(persistedSpans) }) - it('sanitizes synthetic workflow errors and completion failure metadata', async () => { + it('projects synthetic error spans without copying the raw error into OTel metadata', async () => { const session = new LoggingSession('workflow-1', 'execution-error-safe', 'api', 'req-1') const secret = 'sk-demo-error-7f3a91' - session.setEnvironmentSecretSanitizer( - createEnvironmentSecretSanitizer( - { code: 'throw new Error("{{OPENAI_API_KEY}}")' }, - { OPENAI_API_KEY: secret } - ) + session.setResolvedSecretTraceRegistry( + createSecretRegistry([{ plaintext: secret, replacement: '{{OPENAI_API_KEY}}' }]) ) completeWorkflowExecutionMock.mockResolvedValue({}) + const rawExecutionState = { + blockStates: { 'function-1': { output: { result: secret } } }, + executedBlocks: ['function-1'], + blockLogs: [], + decisions: { router: {}, condition: {} }, + completedLoops: [], + activeExecutionPath: ['function-1'], + } await session.safeCompleteWithError({ error: { message: `Function failed with ${secret}` }, + executionState: rawExecutionState, }) expect(completeWorkflowExecutionMock).toHaveBeenCalledWith( expect.objectContaining({ - finalOutput: { error: 'Function failed with {{OPENAI_API_KEY}}' }, + finalOutput: { error: `Function failed with ${secret}` }, traceSpans: [ expect.objectContaining({ output: { error: 'Function failed with {{OPENAI_API_KEY}}' }, }), ], - completionFailure: 'Function failed with {{OPENAI_API_KEY}}', + completionFailure: `Function failed with ${secret}`, + executionState: expect.objectContaining({ + blockStates: { 'function-1': { output: { result: secret } } }, + }), }) ) + expect(createOTelSpansMock).toHaveBeenCalledWith( + expect.not.objectContaining({ error: expect.anything() }) + ) + expect(workflowExecutedMock).toHaveBeenCalledWith( + expect.not.objectContaining({ errorMessage: expect.anything() }) + ) }) - it('keeps workflow output sanitized when completion falls back to cost-only persistence', async () => { + it('keeps fallback functional output unchanged', async () => { const session = new LoggingSession('workflow-1', 'execution-fallback-safe', 'api', 'req-1') const secret = 'sk-demo-fallback-7f3a91' - session.setEnvironmentSecretSanitizer( - createEnvironmentSecretSanitizer( - { code: 'return "{{OPENAI_API_KEY}}"' }, - { OPENAI_API_KEY: secret } - ) + session.setResolvedSecretTraceRegistry( + createSecretRegistry([{ plaintext: secret, replacement: '{{OPENAI_API_KEY}}' }]) ) completeWorkflowExecutionMock .mockRejectedValueOnce(new Error('primary persistence failed')) @@ -380,12 +461,183 @@ describe('LoggingSession completion retries', () => { expect(completeWorkflowExecutionMock).toHaveBeenLastCalledWith( expect.objectContaining({ - finalOutput: { echoed: '{{OPENAI_API_KEY}}' }, + finalOutput: { echoed: secret }, finalizationPath: 'fallback_completed', }) ) }) + it('persists structural-only spans when installed provenance is incomplete', async () => { + const session = new LoggingSession('workflow-1', 'execution-incomplete', 'api', 'req-1') + session.setResolvedSecretTraceRegistry(createSecretRegistry([], false)) + completeWorkflowExecutionMock.mockResolvedValue({}) + + await session.safeComplete({ + finalOutput: { raw: 'functional-data' }, + traceSpans: [ + { + id: 'span-1', + name: 'Agent', + type: 'agent', + duration: 1, + startTime: '2026-07-01T00:00:00.000Z', + endTime: '2026-07-01T00:00:00.001Z', + status: 'success', + output: { raw: 'unknown-provenance' }, + }, + ], + }) + + expect(completeWorkflowExecutionMock).toHaveBeenCalledWith( + expect.objectContaining({ + finalOutput: { raw: 'functional-data' }, + traceSpans: [ + expect.not.objectContaining({ + output: expect.anything(), + }), + ], + }) + ) + }) + + it('fails closed to structural-only spans when provenance was not installed', async () => { + const session = new LoggingSession('workflow-1', 'execution-no-registry', 'api', 'req-1') + completeWorkflowExecutionMock.mockResolvedValue({}) + + await session.safeComplete({ + traceSpans: [ + { + id: 'span-1', + name: 'Function', + type: 'function', + duration: 1, + startTime: '2026-07-01T00:00:00.000Z', + endTime: '2026-07-01T00:00:00.001Z', + output: { unknown: 'provenance' }, + }, + ], + }) + + expect(completeWorkflowExecutionMock).toHaveBeenCalledWith( + expect.objectContaining({ + finalOutput: {}, + traceSpans: [expect.not.objectContaining({ output: expect.anything() })], + }) + ) + }) + + it('projects live block errors and terminal block logs without mutating raw callback data', async () => { + const session = new LoggingSession('workflow-1', 'execution-display-safe', 'manual', 'req-1') + const secret = '1234' + const rawError = `Reference Error: Line 1: return blah +${secret} - blah is not defined` + const rawLog = { + blockId: 'function-1', + blockName: 'Function 1', + blockType: 'function', + startedAt: '2026-07-01T00:00:00.000Z', + endedAt: '2026-07-01T00:00:00.001Z', + durationMs: 1, + success: false, + executionOrder: 1, + input: { code: `return blah +${secret}` }, + output: { error: rawError }, + error: rawError, + } + session.setResolvedSecretTraceRegistry( + createSecretRegistry([{ plaintext: secret, replacement: '{{NUMBER_SECRET}}' }]) + ) + + const display = await session.projectDisplayContent({ + input: rawLog.input, + output: rawLog.output, + error: rawError, + }) + const [displayLog] = await session.projectBlockLogsForDisplay([rawLog]) + + expect(display).toEqual({ + input: { code: 'return blah +{{NUMBER_SECRET}}' }, + output: { + error: 'Reference Error: Line 1: return blah +{{NUMBER_SECRET}} - blah is not defined', + }, + error: 'Reference Error: Line 1: return blah +{{NUMBER_SECRET}} - blah is not defined', + clearLiveDisplay: true, + }) + expect(displayLog.input).toEqual(display.input) + expect(displayLog.output).toEqual(display.output) + expect(displayLog.error).toBe(display.error) + expect(displayLog.clearLiveDisplay).toBe(true) + expect(rawLog.input.code).toBe(`return blah +${secret}`) + expect(rawLog.output.error).toBe(rawError) + expect(rawLog.error).toBe(rawError) + }) + + it('projects large terminal log sets in bounded batches without dropping rows', async () => { + const session = new LoggingSession('workflow-1', 'execution-display-batches', 'manual', 'req-1') + session.setResolvedSecretTraceRegistry( + createSecretRegistry([{ plaintext: 'raw-secret', replacement: '{{TOKEN}}' }]) + ) + const rawLogs = Array.from({ length: 129 }, (_, index) => ({ + blockId: `function-${index}`, + blockName: `Function ${index}`, + blockType: 'function', + startedAt: '2026-07-01T00:00:00.000Z', + endedAt: '2026-07-01T00:00:00.001Z', + durationMs: 1, + success: true, + executionOrder: index, + output: { value: `row-${index}:raw-secret` }, + })) + + const displayLogs = await session.projectBlockLogsForDisplay(rawLogs) + + expect(displayLogs).toHaveLength(rawLogs.length) + expect(displayLogs[0].output).toEqual({ value: 'row-0:{{TOKEN}}' }) + expect(displayLogs[128].output).toEqual({ value: 'row-128:{{TOKEN}}' }) + expect(rawLogs[128].output.value).toBe('row-128:raw-secret') + }) + + it('projects a numeric Function result produced by a resolved numeric secret', async () => { + const session = new LoggingSession('workflow-1', 'execution-numeric-secret', 'manual', 'req-1') + session.setResolvedSecretTraceRegistry( + createSecretRegistry([{ plaintext: '1234', replacement: '{{OPENAI_API_KEY}}' }]) + ) + const rawLog = { + blockId: 'function-1', + blockName: 'Function 1', + blockType: 'function', + startedAt: '2026-07-01T00:00:00.000Z', + endedAt: '2026-07-01T00:00:00.001Z', + durationMs: 1, + success: true, + executionOrder: 1, + input: { code: 'return 1234' }, + output: { result: 1234, stdout: '' }, + } + + const [displayLog] = await session.projectBlockLogsForDisplay([rawLog]) + + expect(displayLog.input).toEqual({ code: 'return {{OPENAI_API_KEY}}' }) + expect(displayLog.output).toEqual({ result: '{{OPENAI_API_KEY}}', stdout: '' }) + expect(rawLog.output.result).toBe(1234) + }) + + it('suppresses live deltas once a resolved secret is active', async () => { + const active = new LoggingSession('workflow-1', 'execution-live-active', 'manual', 'req-1') + active.setResolvedSecretTraceRegistry( + createSecretRegistry([{ plaintext: 'split-secret', replacement: '{{SECRET}}' }]) + ) + + const inactive = new LoggingSession('workflow-1', 'execution-live-inactive', 'manual', 'req-1') + inactive.setResolvedSecretTraceRegistry(createSecretRegistry([])) + + await expect(active.projectLiveDisplayText('chunk', 'split-')).resolves.toEqual({ + clearLiveDisplay: true, + }) + await expect(inactive.projectLiveDisplayText('chunk', 'ordinary text')).resolves.toEqual({ + chunk: 'ordinary text', + }) + }) + it('derives fallback cost from trace spans when the primary completion fails', async () => { const session = new LoggingSession('workflow-1', 'execution-6', 'api', 'req-1') as any @@ -446,6 +698,7 @@ describe('LoggingSession completion retries', () => { it('persists failed error semantics when completeWithError receives non-error trace spans', async () => { const session = new LoggingSession('workflow-1', 'execution-4', 'api', 'req-1') + session.setResolvedSecretTraceRegistry(createSecretRegistry([])) const traceSpans = [ { id: 'span-1', @@ -785,11 +1038,22 @@ describe('completeWithError cancelled-status guard', () => { dbChainMockFns.limit.mockRejectedValueOnce(new Error('DB connection lost')) completeWorkflowExecutionMock.mockResolvedValue({}) const session = new LoggingSession('workflow-1', 'execution-1', 'api', 'req-1') + const executionState = { + blockStates: { 'function-1': { output: { result: 'raw-secret-value' } } }, + executedBlocks: ['function-1'], + blockLogs: [], + decisions: { router: {}, condition: {} }, + completedLoops: [], + activeExecutionPath: ['function-1'], + } - await session.safeCompleteWithError({ error: { message: 'block failed' } }) + await session.safeCompleteWithError({ + error: { message: 'block failed' }, + executionState, + }) expect(completeWorkflowExecutionMock).toHaveBeenCalledWith( - expect.objectContaining({ finalizationPath: 'force_failed' }) + expect.objectContaining({ finalizationPath: 'force_failed', executionState }) ) expect(session.hasCompleted()).toBe(true) }) diff --git a/apps/sim/lib/logs/execution/logging-session.ts b/apps/sim/lib/logs/execution/logging-session.ts index cf1166ae540..5af39bfe4f9 100644 --- a/apps/sim/lib/logs/execution/logging-session.ts +++ b/apps/sim/lib/logs/execution/logging-session.ts @@ -6,6 +6,8 @@ import { and, eq, sql } from 'drizzle-orm' import { releaseExecutionSlot } from '@/lib/billing/calculations/usage-reservation' import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' import { isRetryableInfrastructureError } from '@/lib/core/errors/retryable-infrastructure' +import type { LargeValueStoreContext } from '@/lib/execution/payloads/store' +import type { SecretSafeBlockLog } from '@/lib/logs/execution/display-types' import { executionLogger } from '@/lib/logs/execution/logger' import { type CostSummaryOptions, @@ -21,6 +23,7 @@ import { setLastCompletedBlock, setLastStartedBlock, } from '@/lib/logs/execution/progress-markers' +import { projectTraceSpansForSecrets } from '@/lib/logs/execution/trace-secret-projection' import { traceSpansIndicateFailure } from '@/lib/logs/execution/trace-spans/trace-spans' import type { ExecutionEnvironment, @@ -32,7 +35,8 @@ import type { WorkflowState, } from '@/lib/logs/types' import type { SerializableExecutionState } from '@/executor/execution/types' -import type { EnvironmentSecretSanitizer } from '@/executor/utils/environment-secret-sanitizer' +import type { BlockLog } from '@/executor/types' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' type TriggerData = Record & { correlation?: NonNullable['correlation'] @@ -84,12 +88,31 @@ function buildCompletedMarkerPersistenceQuery(params: { /** Progress-marker and status writes on `workflow_execution_logs` use the exec pool. */ const execDb = dbFor('exec') +const BLOCK_LOG_PROJECTION_BATCH_SIZE = 64 + +function structuralBlockLog(log: BlockLog): BlockLog { + const { + input: _input, + output: _output, + error: _error, + childTraceSpans: _childTraceSpans, + ...structural + } = log + return structural +} const logger = createLogger('LoggingSession') type CompletionAttempt = 'complete' | 'error' | 'cancelled' | 'paused' -const identityEnvironmentSecretSanitizer: EnvironmentSecretSanitizer = (value: T): T => value +export interface SecretSafeDisplayContent { + input?: unknown + output?: unknown + error?: string + text?: string + chunk?: string + clearLiveDisplay?: true +} export interface SessionStartParams { userId?: string @@ -123,6 +146,7 @@ export interface SessionErrorCompleteParams { } traceSpans?: TraceSpan[] skipCost?: boolean + executionState?: SerializableExecutionState } export interface SessionCancelledParams { @@ -169,8 +193,8 @@ export class LoggingSession { private costOptions?: CostSummaryOptions private pendingProgressWrites = new Set>() private postExecutionPromise: Promise | null = null - private environmentSecretSanitizer: EnvironmentSecretSanitizer = - identityEnvironmentSecretSanitizer + private resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry + private traceLargeValueAccess: LargeValueStoreContext = {} constructor( workflowId: string, @@ -191,17 +215,174 @@ export class LoggingSession { : undefined } + /** Installs the run-scoped provenance used only at the terminal TraceSpan boundary. */ + setResolvedSecretTraceRegistry(registry: ResolvedSecretTraceRegistry): void { + this.resolvedSecretTraceRegistry = registry + } + + /** Adds the trusted execution-ref scope needed to rewrite offloaded trace content. */ + setTraceLargeValueAccess(context: LargeValueStoreContext): void { + this.traceLargeValueAccess = context + } + + private getSecretProjectionStore(): LargeValueStoreContext { + return { + ...this.traceLargeValueAccess, + workspaceId: this.environment?.workspaceId, + workflowId: this.workflowId, + executionId: this.executionId, + userId: this.actorUserId ?? this.environment?.userId, + } + } + + private async projectRawTraceSpans(traceSpans: TraceSpan[]): Promise { + return projectTraceSpansForSecrets(traceSpans, { + registry: this.resolvedSecretTraceRegistry, + store: this.getSecretProjectionStore(), + }) + } + + /** + * Produces a display-only copy of known observability content through the same + * projector used for persisted TraceSpans. Runtime values and callback payloads + * remain untouched, and an unavailable projection yields no content fields. + */ + async projectDisplayContent( + content: SecretSafeDisplayContent + ): Promise { + try { + const envelope: Record = {} + for (const key of ['input', 'output', 'error', 'text', 'chunk'] as const) { + if (Object.hasOwn(content, key)) envelope[key] = content[key] + } + + const now = new Date().toISOString() + const [projectedSpan] = await this.projectRawTraceSpans([ + { + id: 'secret-safe-display-projection', + name: 'Display Projection', + type: 'display', + duration: 0, + startTime: now, + endTime: now, + output: envelope, + }, + ]) + const projected = this.readProjectedDisplayContent(projectedSpan?.output) + return this.shouldClearLiveDisplay() ? { ...projected, clearLiveDisplay: true } : projected + } catch { + logger.warn('Display secret projection failed; omitting display content') + return {} + } + } + + private readProjectedDisplayContent( + projectedEnvelope: TraceSpan['output'] | undefined + ): SecretSafeDisplayContent { + if (!projectedEnvelope) return {} + + const projected: SecretSafeDisplayContent = {} + if (Object.hasOwn(projectedEnvelope, 'input')) projected.input = projectedEnvelope.input + if (Object.hasOwn(projectedEnvelope, 'output')) projected.output = projectedEnvelope.output + if (typeof projectedEnvelope.error === 'string') projected.error = projectedEnvelope.error + if (typeof projectedEnvelope.text === 'string') projected.text = projectedEnvelope.text + if (typeof projectedEnvelope.chunk === 'string') projected.chunk = projectedEnvelope.chunk + return projected + } + + /** + * Projects terminal reconciliation logs without changing the executor-owned + * BlockLogs. Child traces use the identical TraceSpan projection boundary. + */ + async projectBlockLogsForDisplay(blockLogs: BlockLog[]): Promise { + const now = new Date().toISOString() + const displayLogs: SecretSafeBlockLog[] = [] + const clearLiveDisplay = this.shouldClearLiveDisplay() + + for (let offset = 0; offset < blockLogs.length; offset += BLOCK_LOG_PROJECTION_BATCH_SIZE) { + const batch = blockLogs.slice(offset, offset + BLOCK_LOG_PROJECTION_BATCH_SIZE) + let projectedLogs: TraceSpan[] + try { + projectedLogs = await this.projectRawTraceSpans( + batch.map((log, index) => ({ + id: `secret-safe-block-log-${offset + index}`, + name: 'Block Log Display Projection', + type: 'display', + duration: 0, + startTime: now, + endTime: now, + output: { + ...(log.input !== undefined ? { input: log.input } : {}), + ...(log.output !== undefined ? { output: log.output } : {}), + ...(log.error !== undefined ? { error: log.error } : {}), + }, + ...(log.childTraceSpans ? { children: log.childTraceSpans } : {}), + })) + ) + } catch { + logger.warn('Block-log secret projection failed; retaining structural logs only') + displayLogs.push(...batch.map(structuralBlockLog)) + continue + } + + for (let index = 0; index < batch.length; index += 1) { + const log = batch[index] + const display = this.readProjectedDisplayContent(projectedLogs[index]?.output) + displayLogs.push({ + ...structuralBlockLog(log), + ...(clearLiveDisplay ? { clearLiveDisplay: true as const } : {}), + ...(Object.hasOwn(display, 'input') + ? { input: display.input as Record } + : {}), + ...(Object.hasOwn(display, 'output') + ? { output: display.output as BlockLog['output'] } + : {}), + ...(display.error !== undefined ? { error: display.error } : {}), + ...(projectedLogs[index]?.children + ? { childTraceSpans: projectedLogs[index].children } + : {}), + }) + } + } + + return displayLogs + } + /** - * Installs the workflow-scoped sanitizer used only for persisted and emitted - * observability values. The closure is retained in memory for this session - * and is never added to execution data. + * Live deltas may split one literal across multiple events. Once provenance is + * active (or incomplete), suppress their display copy instead of attempting a + * per-chunk replacement that could miss the split value. */ - setEnvironmentSecretSanitizer(sanitizer: EnvironmentSecretSanitizer): void { - this.environmentSecretSanitizer = sanitizer + async projectLiveDisplayText( + field: 'text' | 'chunk', + value: string + ): Promise { + if ( + !this.resolvedSecretTraceRegistry?.isComplete() || + this.resolvedSecretTraceRegistry.getActiveMatches().length > 0 + ) { + return { clearLiveDisplay: true } + } + return this.projectDisplayContent({ [field]: value }) } - private sanitizeForLog(value: T): T { - return this.environmentSecretSanitizer(value) + private shouldClearLiveDisplay(): boolean { + return ( + !this.resolvedSecretTraceRegistry?.isComplete() || + this.resolvedSecretTraceRegistry.getActiveMatches().length > 0 + ) + } + + private async projectTraceSpans(traceSpans: TraceSpan[]): Promise { + const preparedTraceSpans = await executionLogger.prepareTraceSpansForProjection({ + executionId: this.executionId, + workflowId: this.workflowId, + workspaceId: this.environment?.workspaceId ?? null, + userId: this.actorUserId ?? this.environment?.userId, + traceSpans, + isResume: this.isResume, + }) + return this.projectRawTraceSpans(preparedTraceSpans) } async onBlockStart( @@ -321,24 +502,18 @@ export class LoggingSession { level?: 'info' | 'error' status?: 'completed' | 'failed' | 'cancelled' | 'pending' }): Promise { - const finalOutput = this.sanitizeForLog(params.finalOutput) - const traceSpans = this.sanitizeForLog(params.traceSpans) - const completionFailure = - params.completionFailure === undefined - ? undefined - : this.sanitizeForLog(params.completionFailure) - + const executionState = this.withResolvedSecretTraceProvenance(params.executionState) await executionLogger.completeWorkflowExecution({ executionId: this.executionId, endedAt: params.endedAt, totalDurationMs: params.totalDurationMs, costSummary: params.costSummary, - finalOutput, - traceSpans, + finalOutput: params.finalOutput, + traceSpans: params.traceSpans, workflowInput: params.workflowInput, - executionState: params.executionState, + executionState, finalizationPath: params.finalizationPath, - completionFailure, + completionFailure: params.completionFailure, isResume: this.isResume, level: params.level, status: params.status, @@ -362,6 +537,27 @@ export class LoggingSession { } } + private withResolvedSecretTraceProvenance( + executionState?: SerializableExecutionState + ): SerializableExecutionState | undefined { + if (!this.resolvedSecretTraceRegistry) return executionState + + const resolvedSecretTraceProvenance = this.resolvedSecretTraceRegistry.exportProvenance() + if (executionState) { + return { ...executionState, resolvedSecretTraceProvenance } + } + + return { + blockStates: {}, + executedBlocks: [], + blockLogs: [], + decisions: { router: {}, condition: {} }, + completedLoops: [], + activeExecutionPath: [], + resolvedSecretTraceProvenance, + } + } + async onBlockComplete( blockId: string, blockName: string, @@ -396,6 +592,14 @@ export class LoggingSession { } = params this.actorUserId = billingAttribution?.actorUserId ?? actorUserId ?? userId ?? null this.billingAttribution = billingAttribution + if (!this.resolvedSecretTraceRegistry) { + const scopeUserId = userId ?? this.actorUserId + this.resolvedSecretTraceRegistry = new ResolvedSecretTraceRegistry( + [], + scopeUserId ? { userId: scopeUserId, workspaceId } : undefined + ) + if (skipLogCreation) this.resolvedSecretTraceRegistry.markIncomplete() + } try { this.trigger = createTriggerObject(this.triggerType, triggerData) @@ -445,14 +649,15 @@ export class LoggingSession { this.completing = true const { endedAt, totalDurationMs, workflowInput, executionState } = params - const finalOutput = this.sanitizeForLog(params.finalOutput || {}) + const finalOutput = params.finalOutput || {} const rawTraceSpans = params.traceSpans || [] - const traceSpans = this.sanitizeForLog(rawTraceSpans) try { const costSummary = calculateCostSummary(rawTraceSpans, this.costOptions) const endTime = endedAt || new Date().toISOString() const duration = totalDurationMs || 0 + const hasErrors = traceSpansIndicateFailure(rawTraceSpans) + const traceSpans = await this.projectTraceSpans(rawTraceSpans) await this.completeExecutionWithFinalization({ endedAt: endTime, @@ -463,6 +668,8 @@ export class LoggingSession { workflowInput, executionState, finalizationPath: 'completed', + level: hasErrors ? 'error' : 'info', + status: hasErrors ? 'failed' : 'completed', }) this.completed = true @@ -473,8 +680,6 @@ export class LoggingSession { '@/lib/core/telemetry' ) - const hasErrors = traceSpansIndicateFailure(rawTraceSpans) - PlatformEvents.workflowExecuted({ workflowId: this.workflowId, durationMs: duration, @@ -542,13 +747,12 @@ export class LoggingSession { const { endedAt, totalDurationMs, error, skipCost } = params const rawTraceSpans = params.traceSpans || [] - const traceSpans = this.sanitizeForLog(rawTraceSpans) const endTime = endedAt ? new Date(endedAt) : new Date() const durationMs = typeof totalDurationMs === 'number' ? totalDurationMs : 0 const startTime = new Date(endTime.getTime() - Math.max(1, durationMs)) - const hasProvidedSpans = traceSpans.length > 0 + const hasProvidedSpans = rawTraceSpans.length > 0 // calculateCostSummary([]) / (undefined) already returns the base-charge // summary, so the no-spans branch needs no separate literal. @@ -566,7 +770,7 @@ export class LoggingSession { } : calculateCostSummary(rawTraceSpans, this.costOptions) - const message = this.sanitizeForLog(error?.message || 'Run failed before starting blocks') + const message = error?.message || 'Run failed before starting blocks' const errorSpan: TraceSpan = { id: 'workflow-error-root', @@ -580,7 +784,7 @@ export class LoggingSession { output: { error: message }, } - const spans = hasProvidedSpans ? traceSpans : [errorSpan] + const spans = await this.projectTraceSpans(hasProvidedSpans ? rawTraceSpans : [errorSpan]) await this.completeExecutionWithFinalization({ endedAt: endTime.toISOString(), @@ -588,6 +792,7 @@ export class LoggingSession { costSummary, finalOutput: { error: message }, traceSpans: spans, + executionState: params.executionState, level: 'error', status: 'failed', finalizationPath: 'force_failed', @@ -607,7 +812,6 @@ export class LoggingSession { trigger: this.triggerType, blocksExecuted: spans.length, hasErrors: true, - errorMessage: message, }) createOTelSpansForWorkflowExecution({ @@ -620,7 +824,6 @@ export class LoggingSession { endTime: endTime.toISOString(), totalDurationMs: Math.max(1, durationMs), status: 'error', - error: message, }) } catch (_e) { // Silently fail @@ -653,7 +856,6 @@ export class LoggingSession { try { const { endedAt, totalDurationMs } = params const rawTraceSpans = params.traceSpans || [] - const traceSpans = this.sanitizeForLog(rawTraceSpans) const endTime = endedAt ? new Date(endedAt) : new Date() const durationMs = typeof totalDurationMs === 'number' ? totalDurationMs : 0 @@ -678,6 +880,7 @@ export class LoggingSession { // calculateCostSummary handles empty/undefined spans by returning the // base-charge summary, so no separate no-spans literal is needed. const costSummary = calculateCostSummary(rawTraceSpans, this.costOptions) + const traceSpans = await this.projectTraceSpans(rawTraceSpans) await this.completeExecutionWithFinalization({ endedAt: endTime.toISOString(), @@ -749,7 +952,6 @@ export class LoggingSession { try { const { endedAt, totalDurationMs, workflowInput } = params const rawTraceSpans = params.traceSpans || [] - const traceSpans = this.sanitizeForLog(rawTraceSpans) const endTime = endedAt ? new Date(endedAt) : new Date() const durationMs = typeof totalDurationMs === 'number' ? totalDurationMs : 0 @@ -774,6 +976,7 @@ export class LoggingSession { // calculateCostSummary handles empty/undefined spans by returning the // base-charge summary, so no separate no-spans literal is needed. const costSummary = calculateCostSummary(rawTraceSpans, this.costOptions) + const traceSpans = await this.projectTraceSpans(rawTraceSpans) await this.completeExecutionWithFinalization({ endedAt: endTime.toISOString(), @@ -1009,6 +1212,7 @@ export class LoggingSession { finalOutput: { error: params?.error?.message || `Execution failed to store trace spans: ${errorMsg}`, }, + executionState: params?.executionState, status: 'failed', }) } @@ -1161,6 +1365,7 @@ export class LoggingSession { isError: boolean finalizationPath: ExecutionFinalizationPath finalOutput?: Record + executionState?: SerializableExecutionState status?: 'completed' | 'failed' | 'cancelled' | 'pending' }): Promise { if (this.completed || this.completing) { @@ -1188,6 +1393,7 @@ export class LoggingSession { costSummary, finalOutput, traceSpans: [], + executionState: params.executionState, finalizationPath: params.finalizationPath, completionFailure: params.errorMessage, level: params.isError ? 'error' : 'info', diff --git a/apps/sim/lib/logs/execution/trace-secret-projection.test.ts b/apps/sim/lib/logs/execution/trace-secret-projection.test.ts new file mode 100644 index 00000000000..9353dfd8b78 --- /dev/null +++ b/apps/sim/lib/logs/execution/trace-secret-projection.test.ts @@ -0,0 +1,797 @@ +/** + * @vitest-environment node + */ +import { createHash } from 'node:crypto' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { materializeLargeValueRefMock, storeLargeValueMock } = vi.hoisted(() => ({ + materializeLargeValueRefMock: vi.fn(), + storeLargeValueMock: vi.fn(), +})) + +vi.mock('@/lib/execution/payloads/store', () => ({ + materializeLargeValueRef: materializeLargeValueRefMock, + storeLargeValue: storeLargeValueMock, +})) + +import { projectTraceSpansForSecrets } from '@/lib/logs/execution/trace-secret-projection' +import type { TraceSpan } from '@/lib/logs/types' +import { + type ResolvedSecretTraceMatch, + ResolvedSecretTraceRegistry, +} from '@/executor/utils/resolved-secret-trace-registry' + +const STORE = { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + userId: 'user-1', +} + +function createRegistry( + matches: ResolvedSecretTraceMatch[], + complete = true +): ResolvedSecretTraceRegistry { + return { + isComplete: () => complete, + getActiveMatches: () => matches, + } as unknown as ResolvedSecretTraceRegistry +} + +function createSpan(overrides: Partial = {}): TraceSpan { + return { + id: 'span-1', + name: 'Function', + type: 'function', + duration: 20, + startTime: '2026-07-01T00:00:00.000Z', + endTime: '2026-07-01T00:00:00.020Z', + status: 'success', + ...overrides, + } +} + +beforeEach(() => { + vi.clearAllMocks() +}) + +describe('projectTraceSpansForSecrets', () => { + it('protects only literals activated by a successful Secrets-tab substitution', async () => { + const plaintext = 'trace/secret+value' + const registry = new ResolvedSecretTraceRegistry([ + { + name: 'API_SECRET', + plaintext, + encryptedValue: 'encrypted-api-secret', + }, + ]) + const transformations = { + urlEncoded: encodeURIComponent(plaintext), + base64: Buffer.from(plaintext).toString('base64'), + sha256: createHash('sha256').update(plaintext).digest('hex'), + } + const source = createSpan({ + output: { + direct: `prefix:${plaintext}:suffix`, + ...transformations, + }, + }) + + const beforeResolution = await projectTraceSpansForSecrets([source], { + registry, + store: STORE, + }) + + expect(beforeResolution[0].output).toEqual(source.output) + expect(registry.recordResolved('API_SECRET', plaintext)).toBe(true) + + const afterResolution = await projectTraceSpansForSecrets([source], { + registry, + store: STORE, + }) + + expect(afterResolution[0].output).toEqual({ + direct: 'prefix:{{API_SECRET}}:suffix', + ...transformations, + }) + expect(source.output).toEqual({ + direct: `prefix:${plaintext}:suffix`, + ...transformations, + }) + }) + + it('projects exact secret literals only inside schema-defined content', async () => { + const secret = 'secret/value with spaces' + const encoded = encodeURIComponent(secret) + const source = createSpan({ + id: `structural-${secret}`, + name: `Structural ${secret}`, + blockId: secret, + input: { prompt: `prefix:${secret}:suffix`, encoded }, + output: { [secret]: secret }, + thinking: `thinking ${secret}`, + errorMessage: `error ${secret}`, + modelToolCalls: [{ id: 'call-1', name: 'lookup', arguments: { token: secret } }], + toolCalls: [ + { + name: 'legacy', + duration: 1, + startTime: '2026-07-01T00:00:00.000Z', + endTime: '2026-07-01T00:00:00.001Z', + status: 'error', + input: { token: secret }, + output: { echoed: secret }, + error: `failed ${secret}`, + }, + ], + providerTiming: { + duration: 10, + startTime: '2026-07-01T00:00:00.000Z', + endTime: '2026-07-01T00:00:00.010Z', + segments: [ + { + type: 'model', + name: 'model-1', + startTime: 0, + endTime: 10, + duration: 10, + assistantContent: secret, + thinkingContent: `why ${secret}`, + errorMessage: `bad ${secret}`, + toolCalls: [{ id: 'call-2', name: 'lookup', arguments: secret }], + }, + ], + }, + children: [createSpan({ id: 'child-1', output: { child: secret } })], + }) + + const result = await projectTraceSpansForSecrets([source], { + registry: createRegistry([{ plaintext: secret, replacement: '{{API_SECRET}}' }]), + store: STORE, + }) + + const span = result[0] + expect(span.id).toBe(`structural-${secret}`) + expect(span.name).toBe(`Structural ${secret}`) + expect(span.blockId).toBe(secret) + expect(span.input).toEqual({ + prompt: 'prefix:{{API_SECRET}}:suffix', + encoded, + }) + expect(span.output).toEqual({ '{{API_SECRET}}': '{{API_SECRET}}' }) + expect(span.thinking).toBe('thinking {{API_SECRET}}') + expect(span.errorMessage).toBe('error {{API_SECRET}}') + expect(span.modelToolCalls?.[0].arguments).toEqual({ token: '{{API_SECRET}}' }) + expect(span.toolCalls?.[0]).toEqual( + expect.objectContaining({ + input: { token: '{{API_SECRET}}' }, + output: { echoed: '{{API_SECRET}}' }, + error: 'failed {{API_SECRET}}', + }) + ) + expect(span.providerTiming?.segments[0]).toEqual( + expect.objectContaining({ + assistantContent: '{{API_SECRET}}', + thinkingContent: 'why {{API_SECRET}}', + errorMessage: 'bad {{API_SECRET}}', + toolCalls: [ + expect.objectContaining({ + arguments: '{{API_SECRET}}', + }), + ], + }) + ) + expect(span.children?.[0].output).toEqual({ child: '{{API_SECRET}}' }) + expect(source.output).toEqual({ [secret]: secret }) + expect(result[0]).not.toBe(source) + }) + + it('uses deterministic longest matches and does not rescan replacements', async () => { + const result = await projectTraceSpansForSecrets( + [createSpan({ output: { value: 'secret-suffix secret A_SECRET' } })], + { + registry: createRegistry([ + { plaintext: 'secret', replacement: '{{SHORT}}' }, + { plaintext: 'secret-suffix', replacement: '{{LONG}}' }, + { plaintext: 'A_SECRET', replacement: '{{A_SECRET}}' }, + ]), + store: STORE, + } + ) + + expect(result[0].output).toEqual({ + value: '{{LONG}} {{SHORT}} ', + }) + }) + + it('keeps matching bounded with a full 10,000-entry provenance set', async () => { + const dormant = Array.from({ length: 9_998 }, (_, index) => ({ + plaintext: `unused-secret-${index}`, + replacement: `{{UNUSED_${index}}}`, + })) + const result = await projectTraceSpansForSecrets( + [createSpan({ output: { value: 'prefix secret-suffix suffix' } })], + { + registry: createRegistry([ + ...dormant, + { plaintext: 'secret', replacement: '{{SHORT}}' }, + { plaintext: 'secret-suffix', replacement: '{{LONG}}' }, + ]), + store: STORE, + } + ) + + expect(result[0].output).toEqual({ value: 'prefix {{LONG}} suffix' }) + }) + + it('chooses the lexicographically first replacement for duplicate plaintext', async () => { + const result = await projectTraceSpansForSecrets( + [createSpan({ output: { value: 'same-value' } })], + { + registry: createRegistry([ + { plaintext: 'same-value', replacement: '{{Z_SECRET}}' }, + { plaintext: 'same-value', replacement: '{{A_SECRET}}' }, + { plaintext: '', replacement: '{{EMPTY}}' }, + ]), + store: STORE, + } + ) + + expect(result[0].output).toEqual({ value: '{{A_SECRET}}' }) + }) + + it('supports one-character case-sensitive secrets without altering structural fields', async () => { + const result = await projectTraceSpansForSecrets( + [createSpan({ id: 'A-structural', output: { value: 'A a' } })], + { + registry: createRegistry([{ plaintext: 'A', replacement: '{{LETTER}}' }]), + store: STORE, + } + ) + + expect(result[0].id).toBe('A-structural') + expect(result[0].output).toEqual({ value: '{{LETTER}} a' }) + }) + + it('projects a successfully resolved numeric Function result without mutating runtime output', async () => { + const runtimeOutput = { result: 1234, unchanged: 5678 } + const source = createSpan({ output: runtimeOutput }) + + const [result] = await projectTraceSpansForSecrets([source], { + registry: createRegistry([{ plaintext: '1234', replacement: '{{NUMERIC_SECRET}}' }]), + store: STORE, + }) + + expect(result.output).toEqual({ result: '{{NUMERIC_SECRET}}', unchanged: 5678 }) + expect(source.output).toBe(runtimeOutput) + expect(runtimeOutput).toEqual({ result: 1234, unchanged: 5678 }) + }) + + it('does not infer or redact values derived from a resolved secret', async () => { + const source = createSpan({ + input: { code: 'return 1234 + 5' }, + output: { result: 1239 }, + }) + + const [result] = await projectTraceSpansForSecrets([source], { + registry: createRegistry([{ plaintext: '1234', replacement: '{{NUMERIC_SECRET}}' }]), + store: STORE, + }) + + expect(result.input).toEqual({ code: 'return {{NUMERIC_SECRET}} + 5' }) + expect(result.output).toEqual({ result: 1239 }) + }) + + it('omits a content field when secret replacement collides object keys', async () => { + const result = await projectTraceSpansForSecrets( + [createSpan({ input: { safe: 'keep' }, output: { secret: 1, '{{TOKEN}}': 2 } })], + { + registry: createRegistry([{ plaintext: 'secret', replacement: '{{TOKEN}}' }]), + store: STORE, + } + ) + + expect(result[0].input).toEqual({ safe: 'keep' }) + expect(result[0]).not.toHaveProperty('output') + }) + + it('returns structural-only spans when provenance is incomplete', async () => { + const result = await projectTraceSpansForSecrets( + [ + createSpan({ + input: { secret: 'raw' }, + output: { secret: 'raw' }, + thinking: 'raw', + errorMessage: 'raw', + modelToolCalls: [{ id: 'call-1', name: 'lookup', arguments: 'raw' }], + toolCalls: [ + { + name: 'legacy', + duration: 1, + startTime: '2026-07-01T00:00:00.000Z', + endTime: '2026-07-01T00:00:00.001Z', + status: 'error', + error: 'raw', + }, + ], + providerTiming: { + duration: 1, + startTime: '2026-07-01T00:00:00.000Z', + endTime: '2026-07-01T00:00:00.001Z', + segments: [ + { + type: 'model', + startTime: 0, + endTime: 1, + duration: 1, + assistantContent: 'raw', + tokens: { input: 1, output: 2, total: 3 }, + }, + ], + }, + children: [createSpan({ id: 'child-1', output: { secret: 'raw' } })], + }), + ], + { registry: createRegistry([], false), store: STORE } + ) + + const span = result[0] + expect(span).toMatchObject({ + id: 'span-1', + name: 'Function', + status: 'success', + providerTiming: { + duration: 1, + segments: [expect.objectContaining({ tokens: { input: 1, output: 2, total: 3 } })], + }, + children: [expect.objectContaining({ id: 'child-1' })], + }) + expect(span).not.toHaveProperty('input') + expect(span).not.toHaveProperty('output') + expect(span).not.toHaveProperty('thinking') + expect(span).not.toHaveProperty('errorMessage') + expect(span.modelToolCalls).toEqual([{ id: 'call-1', name: 'lookup' }]) + expect(span.toolCalls).toEqual([expect.objectContaining({ name: 'legacy', status: 'error' })]) + expect(span.toolCalls?.[0]).not.toHaveProperty('error') + expect(span.providerTiming?.segments[0]).not.toHaveProperty('assistantContent') + expect(span.providerTiming?.segments[0].toolCalls).toBeUndefined() + expect(span.children?.[0]).not.toHaveProperty('output') + }) + + it('hydrates and re-stores large values without retaining the source ref', async () => { + const sourceRef = { + __simLargeValueRef: true, + version: 1, + id: 'lv_aaaaaaaaaaaa', + kind: 'object', + size: 9_000_000, + } as const + const safeRef = { + __simLargeValueRef: true, + version: 1, + id: 'lv_bbbbbbbbbbbb', + kind: 'object', + size: 32, + preview: { keys: ['token'] }, + } as const + materializeLargeValueRefMock.mockResolvedValue({ token: 'top-secret' }) + storeLargeValueMock.mockResolvedValue(safeRef) + + const result = await projectTraceSpansForSecrets( + [createSpan({ output: { payload: sourceRef } })], + { + registry: createRegistry([{ plaintext: 'top-secret', replacement: '{{API_SECRET}}' }]), + store: STORE, + } + ) + + expect(materializeLargeValueRefMock).toHaveBeenCalledWith( + sourceRef, + expect.objectContaining({ + trackReference: false, + maxBytes: 64 * 1024 * 1024, + }) + ) + expect(storeLargeValueMock).toHaveBeenCalledWith( + { token: '{{API_SECRET}}' }, + JSON.stringify({ token: '{{API_SECRET}}' }), + expect.any(Number), + expect.objectContaining({ requireDurable: true }) + ) + expect(result[0].output).toEqual({ payload: safeRef }) + expect(result[0].output).not.toEqual({ payload: sourceRef }) + }) + + it('omits ref-backed content without reading or writing during display projection', async () => { + const sourceRef = { + __simLargeValueRef: true, + version: 1, + id: 'lv_aaaaaaaaaaaa', + kind: 'object', + size: 9_000_000, + } as const + + const result = await projectTraceSpansForSecrets( + [createSpan({ input: { safe: true }, output: { payload: sourceRef } })], + { + registry: createRegistry([{ plaintext: 'top-secret', replacement: '{{API_SECRET}}' }]), + store: STORE, + allowLargeValueWrites: false, + } + ) + + expect(result[0].input).toEqual({ safe: true }) + expect(result[0]).not.toHaveProperty('output') + expect(materializeLargeValueRefMock).not.toHaveBeenCalled() + expect(storeLargeValueMock).not.toHaveBeenCalled() + }) + + it('omits only the affected field when a large value cannot be re-stored', async () => { + const sourceRef = { + __simLargeValueRef: true, + version: 1, + id: 'lv_aaaaaaaaaaaa', + kind: 'object', + size: 9_000_000, + } as const + materializeLargeValueRefMock.mockResolvedValue({ token: 'top-secret' }) + storeLargeValueMock.mockRejectedValue(new Error('storage unavailable')) + + const result = await projectTraceSpansForSecrets( + [createSpan({ input: { safe: true }, output: { payload: sourceRef } })], + { + registry: createRegistry([{ plaintext: 'top-secret', replacement: '{{API_SECRET}}' }]), + store: STORE, + } + ) + + expect(result[0].input).toEqual({ safe: true }) + expect(result[0]).not.toHaveProperty('output') + }) + + it('rewrites manifests chunk-by-chunk and derives a sanitized preview', async () => { + const chunkRef = { + __simLargeValueRef: true, + version: 1, + id: 'lv_aaaaaaaaaaaa', + kind: 'array', + size: 64, + } as const + const manifest = { + __simLargeArrayManifest: true, + version: 2, + kind: 'array', + totalCount: 1, + chunkCount: 1, + byteSize: 64, + chunks: [{ ref: chunkRef, count: 1, byteSize: 64 }], + preview: [{ token: 'top-secret' }], + } as const + materializeLargeValueRefMock.mockResolvedValue([{ token: 'top-secret' }]) + storeLargeValueMock.mockImplementation( + async (_value: unknown, _json: string, size: number) => ({ + __simLargeValueRef: true, + version: 1, + id: 'lv_bbbbbbbbbbbb', + kind: 'array', + size, + }) + ) + + const result = await projectTraceSpansForSecrets( + [createSpan({ output: { items: manifest } })], + { + registry: createRegistry([{ plaintext: 'top-secret', replacement: '{{API_SECRET}}' }]), + store: STORE, + } + ) + + expect(materializeLargeValueRefMock).toHaveBeenCalledTimes(1) + const rewritten = result[0].output?.items + expect(rewritten).toEqual( + expect.objectContaining({ + __simLargeArrayManifest: true, + totalCount: 1, + preview: [{ token: '{{API_SECRET}}' }], + }) + ) + expect(rewritten).not.toBe(manifest) + }) + + it('stores each sanitized manifest chunk before materializing the next chunk', async () => { + const sourceRefs = ['aaaaaaaaaaaa', 'bbbbbbbbbbbb'].map((suffix) => ({ + __simLargeValueRef: true as const, + version: 1 as const, + id: `lv_${suffix}`, + kind: 'array' as const, + size: 32, + })) + const events: string[] = [] + materializeLargeValueRefMock.mockImplementation(async (ref: { id: string }) => { + events.push(`materialize:${ref.id}`) + return [{ token: 'top-secret' }] + }) + storeLargeValueMock.mockImplementation(async (_value: unknown, _json: string, size: number) => { + const index = events.filter((event) => event.startsWith('store:')).length + const id = `lv_safechunk${index.toString().padStart(3, '0')}` + events.push(`store:${id}`) + return { + __simLargeValueRef: true, + version: 1, + id, + kind: 'array', + size, + } + }) + const manifest = { + __simLargeArrayManifest: true as const, + version: 2 as const, + kind: 'array' as const, + totalCount: 2, + chunkCount: 2, + byteSize: 64, + chunks: sourceRefs.map((ref) => ({ ref, count: 1, byteSize: 32 })), + preview: [], + } + + const result = await projectTraceSpansForSecrets( + [createSpan({ output: { items: manifest } })], + { + registry: createRegistry([{ plaintext: 'top-secret', replacement: '{{API_SECRET}}' }]), + store: STORE, + } + ) + + expect(events).toEqual([ + 'materialize:lv_aaaaaaaaaaaa', + 'store:lv_safechunk000', + 'materialize:lv_bbbbbbbbbbbb', + 'store:lv_safechunk001', + ]) + expect(result[0].output?.items).toEqual( + expect.objectContaining({ chunkCount: 2, totalCount: 2 }) + ) + }) + + it('fails closed before sanitized manifest chunks exceed the cumulative output budget', async () => { + const sourceRefs = Array.from({ length: 70 }, (_, index) => ({ + __simLargeValueRef: true as const, + version: 1 as const, + id: `lv_${index.toString().padStart(12, '0')}`, + kind: 'array' as const, + size: 1, + })) + const expandedValue = 'a'.repeat(1024) + materializeLargeValueRefMock.mockImplementation(async () => [expandedValue]) + storeLargeValueMock.mockImplementation( + async (_value: unknown, _json: string, size: number) => ({ + __simLargeValueRef: true, + version: 1, + id: `lv_safe${storeLargeValueMock.mock.calls.length.toString().padStart(8, '0')}`, + kind: 'array', + size, + }) + ) + const manifest = { + __simLargeArrayManifest: true as const, + version: 2 as const, + kind: 'array' as const, + totalCount: sourceRefs.length, + chunkCount: sourceRefs.length, + byteSize: sourceRefs.length, + chunks: sourceRefs.map((ref) => ({ ref, count: 1, byteSize: 1 })), + preview: [], + } + + const result = await projectTraceSpansForSecrets( + [createSpan({ input: { ok: true }, output: { items: manifest } })], + { + registry: createRegistry([{ plaintext: 'a', replacement: 'X'.repeat(1024) }]), + store: STORE, + } + ) + + expect(storeLargeValueMock).toHaveBeenCalledTimes(63) + expect(result[0].input).toEqual({ ok: true }) + expect(result[0]).not.toHaveProperty('output') + }) + + it('rejects a manifest when storage reuses one sanitized ref for multiple chunks', async () => { + const sourceRefs = ['aaaaaaaaaaaa', 'bbbbbbbbbbbb'].map((suffix) => ({ + __simLargeValueRef: true as const, + version: 1 as const, + id: `lv_${suffix}`, + kind: 'array' as const, + size: 32, + })) + const reusedRef = { + __simLargeValueRef: true as const, + version: 1 as const, + id: 'lv_reusedref000', + kind: 'array' as const, + size: 32, + } + materializeLargeValueRefMock.mockResolvedValue([{ token: 'top-secret' }]) + storeLargeValueMock.mockResolvedValue(reusedRef) + const manifest = { + __simLargeArrayManifest: true as const, + version: 2 as const, + kind: 'array' as const, + totalCount: 2, + chunkCount: 2, + byteSize: 64, + chunks: sourceRefs.map((ref) => ({ ref, count: 1, byteSize: 32 })), + preview: [], + } + + const result = await projectTraceSpansForSecrets( + [createSpan({ input: { safe: true }, output: { items: manifest } })], + { + registry: createRegistry([{ plaintext: 'top-secret', replacement: '{{API_SECRET}}' }]), + store: STORE, + } + ) + + expect(storeLargeValueMock).toHaveBeenCalledTimes(2) + expect(result[0].input).toEqual({ safe: true }) + expect(result[0]).not.toHaveProperty('output') + }) + + it('does not invoke array map while replacing refs and sanitizing inline content', async () => { + const sourceRef = { + __simLargeValueRef: true, + version: 1, + id: 'lv_aaaaaaaaaaaa', + kind: 'object', + size: 32, + } as const + const safeRef = { + __simLargeValueRef: true, + version: 1, + id: 'lv_bbbbbbbbbbbb', + kind: 'object', + size: 32, + } as const + const map = vi.fn(() => { + throw new Error('array map must not be invoked') + }) + const values: unknown[] = [sourceRef, 'top-secret'] + Object.defineProperty(values, 'map', { value: map }) + materializeLargeValueRefMock.mockResolvedValue({ token: 'top-secret' }) + storeLargeValueMock.mockResolvedValue(safeRef) + + const result = await projectTraceSpansForSecrets([createSpan({ output: values })], { + registry: createRegistry([{ plaintext: 'top-secret', replacement: '{{API_SECRET}}' }]), + store: STORE, + }) + + expect(map).not.toHaveBeenCalled() + expect(result[0].output).toEqual([safeRef, '{{API_SECRET}}']) + }) + + it('omits an oversized sparse content array without allocating its declared length', async () => { + const sparse: unknown[] = [] + sparse.length = 100_001 + sparse[0] = 'top-secret' + + const [result] = await projectTraceSpansForSecrets( + [createSpan({ input: { safe: true }, output: sparse })], + { + registry: createRegistry([{ plaintext: 'top-secret', replacement: '{{API_SECRET}}' }]), + store: STORE, + } + ) + + expect(result.input).toEqual({ safe: true }) + expect(result).not.toHaveProperty('output') + }) + + it('caps no-secret structural cloning by depth', async () => { + let source = createSpan({ id: 'depth-150', output: { safe: true } }) + for (let depth = 149; depth >= 0; depth -= 1) { + source = createSpan({ id: `depth-${depth}`, children: [source] }) + } + + const [result] = await projectTraceSpansForSecrets([source], { + registry: createRegistry([]), + store: STORE, + }) + + let projectedDepth = 0 + let cursor: TraceSpan | undefined = result + while (cursor?.children?.[0]) { + projectedDepth += 1 + cursor = cursor.children[0] + } + expect(projectedDepth).toBe(100) + expect(cursor?.children).toEqual([]) + }) + + it('caps structural fallback cardinality and strips every retained call argument', async () => { + const call = { id: 'call-1', name: 'lookup', arguments: { secret: 'raw' } } + const source = createSpan({ + output: { secret: 'raw' }, + modelToolCalls: Array(100_005).fill(call), + children: [createSpan({ id: 'unreached-child', output: { secret: 'raw' } })], + }) + + const [result] = await projectTraceSpansForSecrets([source], { + registry: createRegistry([], false), + store: STORE, + }) + + expect(result).not.toHaveProperty('output') + expect(result.modelToolCalls?.length).toBeLessThan(source.modelToolCalls?.length ?? 0) + expect(result.modelToolCalls?.every((retained) => !Object.hasOwn(retained, 'arguments'))).toBe( + true + ) + expect(result.children).toEqual([]) + }) + + it('bounds active-path model tool calls with the global structure budget', async () => { + const call = { id: 'call-1', name: 'lookup', arguments: { secret: 'top-secret' } } + const source = createSpan({ + output: { secret: 'top-secret' }, + modelToolCalls: Array(100_005).fill(call), + children: [createSpan({ id: 'unreached-child', output: { secret: 'top-secret' } })], + }) + + const [result] = await projectTraceSpansForSecrets([source], { + registry: createRegistry([{ plaintext: 'top-secret', replacement: '{{API_SECRET}}' }]), + store: STORE, + }) + + expect(result).not.toHaveProperty('output') + expect(result.modelToolCalls?.length).toBeLessThan(source.modelToolCalls?.length ?? 0) + expect(result.modelToolCalls?.every((retained) => !Object.hasOwn(retained, 'arguments'))).toBe( + true + ) + expect(result.children).toEqual([]) + }) + + it('iterates wide records without bulk property-descriptor materialization', async () => { + const descriptorSpy = vi.spyOn(Object, 'getOwnPropertyDescriptors').mockImplementation(() => { + throw new Error('bulk descriptor materialization is forbidden') + }) + + let result: TraceSpan[] = [] + try { + result = await projectTraceSpansForSecrets( + [createSpan({ output: { token: 'top-secret' } })], + { + registry: createRegistry([{ plaintext: 'top-secret', replacement: '{{API_SECRET}}' }]), + store: STORE, + } + ) + } finally { + descriptorSpy.mockRestore() + } + + expect(result[0].output).toEqual({ token: '{{API_SECRET}}' }) + }) + + it('uses bounded structural fallback when matcher construction fails', async () => { + let source = createSpan({ id: 'depth-150', output: { secret: 'raw' } }) + for (let depth = 149; depth >= 0; depth -= 1) { + source = createSpan({ id: `depth-${depth}`, children: [source] }) + } + + const [result] = await projectTraceSpansForSecrets([source], { + registry: createRegistry([ + { plaintext: 's'.repeat(64 * 1024 + 1), replacement: '{{TOO_LARGE}}' }, + ]), + store: STORE, + }) + + let projectedDepth = 0 + let cursor: TraceSpan | undefined = result + while (cursor?.children?.[0]) { + expect(cursor).not.toHaveProperty('output') + projectedDepth += 1 + cursor = cursor.children[0] + } + expect(projectedDepth).toBe(100) + expect(cursor).not.toHaveProperty('output') + expect(cursor?.children).toEqual([]) + }) +}) diff --git a/apps/sim/lib/logs/execution/trace-secret-projection.ts b/apps/sim/lib/logs/execution/trace-secret-projection.ts new file mode 100644 index 00000000000..16d40182b96 --- /dev/null +++ b/apps/sim/lib/logs/execution/trace-secret-projection.ts @@ -0,0 +1,1391 @@ +import { createLogger } from '@sim/logger' +import { isPlainRecord, omit } from '@sim/utils/object' +import { + isLargeArrayManifest, + LARGE_ARRAY_MANIFEST_MARKER, + LARGE_ARRAY_MANIFEST_PREVIEW_MAX_BYTES, + type LargeArrayManifest, +} from '@/lib/execution/payloads/large-array-manifest-metadata' +import { + isLargeValueRef, + LARGE_VALUE_REF_MARKER, + type LargeValueRef, +} from '@/lib/execution/payloads/large-value-ref' +import { + MAX_DURABLE_LARGE_VALUE_BYTES, + MAX_INLINE_MATERIALIZATION_BYTES, +} from '@/lib/execution/payloads/materialization.server' +import type { LargeValueStoreContext } from '@/lib/execution/payloads/store' +import { materializeLargeValueRef, storeLargeValue } from '@/lib/execution/payloads/store' +import type { ToolCall, TraceSpan } from '@/lib/logs/types' +import type { IterationToolCall, ProviderTimingSegment } from '@/executor/types' +import type { + ResolvedSecretTraceMatch, + ResolvedSecretTraceRegistry, +} from '@/executor/utils/resolved-secret-trace-registry' + +const logger = createLogger('TraceSecretProjection') +const REF_CONCURRENCY = 4 +const MAX_CONTENT_NODES = 100_000 +const MAX_CONTENT_DEPTH = 100 +const MAX_MATCHER_NODES = 250_000 +const MAX_SECRET_LITERAL_LENGTH = 64 * 1024 +const MAX_MATCH_EVENTS = 1_000_000 +const MAX_LARGE_VALUES = 1_024 +const MAX_LARGE_VALUE_CHAIN_DEPTH = 32 +const MAX_LARGE_MANIFEST_CHUNKS = MAX_LARGE_VALUES +const MAX_LARGE_MANIFEST_ITEMS = MAX_CONTENT_NODES +const OMIT = Symbol('omit-trace-content') + +interface SecretReplacement { + plaintext: string + replacement: string +} + +interface SecretTrieNode { + children: Map + failure?: SecretTrieNode + outputLink?: SecretTrieNode + replacement?: SecretReplacement +} + +interface SecretMatcher { + root: SecretTrieNode + maxPatternLength: number +} + +interface ProjectionContext { + matcher: SecretMatcher + store: LargeValueStoreContext + allowLargeValueWrites: boolean + safeLargeValues: WeakSet + oversizedGate: OversizedHydrationGate + refIoSemaphore: AsyncSemaphore + seenLargeValues: Set + largeValueCache: Map> + manifestIds: WeakMap + nextManifestId: number + largeValueCount: number + sourceLargeValueBytes: number + storedLargeValueBytes: number + storedLargeValueCount: number +} + +interface OversizedHydrationGate { + chain: Promise +} + +interface AsyncSemaphore { + active: number + waiters: Array<() => void> +} + +interface TraversalState { + nodes: number + ancestors: WeakSet +} + +interface SanitizationTraversalState extends TraversalState { + outputBytes: number + maxBytes: number +} + +export interface ProjectTraceSpansForSecretsOptions { + registry?: ResolvedSecretTraceRegistry + store: LargeValueStoreContext + /** Read-only display projections omit ref-backed content instead of writing another ref. */ + allowLargeValueWrites?: boolean +} + +class TraceSecretProjectionError extends Error { + constructor(message: string) { + super(message) + this.name = 'TraceSecretProjectionError' + } +} + +function compareStrings(left: string, right: string): number { + if (left < right) return -1 + if (left > right) return 1 + return 0 +} + +function normalizeReplacements(matches: readonly ResolvedSecretTraceMatch[]): SecretReplacement[] { + const replacementByPlaintext = new Map() + + for (const match of matches) { + if (!match.plaintext) continue + const current = replacementByPlaintext.get(match.plaintext) + if (current === undefined || compareStrings(match.replacement, current) < 0) { + replacementByPlaintext.set(match.plaintext, match.replacement) + } + } + + const provisional = [...replacementByPlaintext.keys()] + .map((plaintext) => { + const requested = replacementByPlaintext.get(plaintext) ?? '' + return { plaintext, replacement: requested } + }) + .sort( + (left, right) => + right.plaintext.length - left.plaintext.length || + compareStrings(left.replacement, right.replacement) || + compareStrings(left.plaintext, right.plaintext) + ) + + const detector = createSecretMatcher( + provisional.map(({ plaintext }) => ({ plaintext, replacement: '' })) + ) + return provisional.map(({ plaintext, replacement }) => ({ + plaintext, + replacement: containsSecret(replacement, detector) ? '' : replacement, + })) +} + +function createSecretMatcher(replacements: readonly SecretReplacement[]): SecretMatcher { + const root: SecretTrieNode = { children: new Map() } + root.failure = root + let nodeCount = 1 + let maxPatternLength = 0 + for (const replacement of replacements) { + if (replacement.plaintext.length > MAX_SECRET_LITERAL_LENGTH) { + throw new TraceSecretProjectionError('Secret literal exceeds the matcher size limit') + } + maxPatternLength = Math.max(maxPatternLength, replacement.plaintext.length) + let node = root + for (let index = 0; index < replacement.plaintext.length; index += 1) { + const character = replacement.plaintext[index] + let child = node.children.get(character) + if (!child) { + child = { children: new Map() } + node.children.set(character, child) + nodeCount += 1 + if (nodeCount > MAX_MATCHER_NODES) { + throw new TraceSecretProjectionError('Secret matcher node limit exceeded') + } + } + node = child + } + node.replacement = replacement + } + + const queue: SecretTrieNode[] = [] + for (const child of root.children.values()) { + child.failure = root + queue.push(child) + } + for (let cursor = 0; cursor < queue.length; cursor += 1) { + const node = queue[cursor] + for (const [character, child] of node.children) { + let fallback = node.failure ?? root + while (fallback !== root && !fallback.children.has(character)) { + fallback = fallback.failure ?? root + } + const transition = fallback.children.get(character) + child.failure = transition && transition !== child ? transition : root + child.outputLink = child.failure.replacement ? child.failure : child.failure.outputLink + queue.push(child) + } + } + + return { root, maxPatternLength } +} + +function advanceMatcher( + matcher: SecretMatcher, + node: SecretTrieNode, + character: string +): SecretTrieNode { + let current = node + while (current !== matcher.root && !current.children.has(character)) { + current = current.failure ?? matcher.root + } + return current.children.get(character) ?? matcher.root +} + +function containsSecret(value: string, matcher: SecretMatcher): boolean { + let node = matcher.root + for (let index = 0; index < value.length; index += 1) { + node = advanceMatcher(matcher, node, value[index]) + if (node.replacement || node.outputLink) return true + } + return false +} + +function sanitizeString( + value: string, + matcher: SecretMatcher, + maxBytes = MAX_INLINE_MATERIALIZATION_BYTES +): string { + if (maxBytes < 0) { + throw new TraceSecretProjectionError('Sanitized trace string exceeds the size limit') + } + if (Buffer.byteLength(value, 'utf8') > maxBytes) { + throw new TraceSecretProjectionError('Trace string exceeds the size limit') + } + if (matcher.maxPatternLength === 0 || value.length === 0) return value + + let emitCursor = 0 + let literalStart = 0 + let outputBytes = 0 + let matchEvents = 0 + const chunks: string[] = [] + const windowSize = matcher.maxPatternLength + const slotStarts = new Int32Array(windowSize) + const slotEnds = new Int32Array(windowSize) + slotStarts.fill(-1) + const slotReplacements = new Array(windowSize) + + const append = (chunk: string): void => { + if (!chunk) return + outputBytes += Buffer.byteLength(chunk, 'utf8') + if (outputBytes > maxBytes) { + throw new TraceSecretProjectionError('Sanitized trace string exceeds the size limit') + } + const lastIndex = chunks.length - 1 + if (lastIndex >= 0 && chunks[lastIndex].length + chunk.length <= 64 * 1024) { + chunks[lastIndex] += chunk + } else { + chunks.push(chunk) + } + } + + const finalizeThrough = (limit: number): void => { + while (emitCursor <= limit && emitCursor < value.length) { + const slot = emitCursor % windowSize + if (slotStarts[slot] === emitCursor && slotReplacements[slot] !== undefined) { + append(value.slice(literalStart, emitCursor)) + append(slotReplacements[slot] ?? '') + emitCursor = slotEnds[slot] + literalStart = emitCursor + } else { + emitCursor += 1 + } + } + } + + let node = matcher.root + for (let index = 0; index < value.length; index += 1) { + node = advanceMatcher(matcher, node, value[index]) + let outputNode: SecretTrieNode | undefined = node.replacement ? node : node.outputLink + while (outputNode?.replacement) { + matchEvents += 1 + if (matchEvents > MAX_MATCH_EVENTS) { + throw new TraceSecretProjectionError('Secret matcher event limit exceeded') + } + const start = index - outputNode.replacement.plaintext.length + 1 + if (start >= emitCursor) { + const slot = start % windowSize + const end = index + 1 + if (slotStarts[slot] !== start || end > slotEnds[slot]) { + slotStarts[slot] = start + slotEnds[slot] = end + slotReplacements[slot] = outputNode.replacement.replacement + } + } + outputNode = outputNode.outputLink + } + finalizeThrough(index - matcher.maxPatternLength + 1) + } + + finalizeThrough(value.length - 1) + append(value.slice(literalStart)) + return chunks.join('') +} + +function visitNode(state: TraversalState, depth: number): void { + state.nodes += 1 + if (state.nodes > MAX_CONTENT_NODES) { + throw new TraceSecretProjectionError('Trace content node limit exceeded') + } + if (depth > MAX_CONTENT_DEPTH) { + throw new TraceSecretProjectionError('Trace content depth limit exceeded') + } +} + +function assertArrayFitsTraversal(value: readonly unknown[], state: TraversalState): void { + if (value.length > MAX_CONTENT_NODES - state.nodes) { + throw new TraceSecretProjectionError('Trace content array exceeds the traversal limit') + } +} + +function* arrayDataEntries(value: readonly unknown[]): Generator<[number, unknown]> { + for (let index = 0; index < value.length; index += 1) { + const descriptor = Object.getOwnPropertyDescriptor(value, index) + if (!descriptor) continue + if (!('value' in descriptor)) { + throw new TraceSecretProjectionError('Trace content accessors are not supported') + } + yield [index, descriptor.value] + } +} + +function enterObject(value: object, state: TraversalState): void { + if (state.ancestors.has(value)) { + throw new TraceSecretProjectionError('Cyclic trace content is not supported') + } + state.ancestors.add(value) +} + +function leaveObject(value: object, state: TraversalState): void { + state.ancestors.delete(value) +} + +function* enumerableDataEntries(value: object): Generator<[string, unknown]> { + for (const key in value) { + if (!Object.hasOwn(value, key)) continue + const descriptor = Object.getOwnPropertyDescriptor(value, key) + if (!descriptor?.enumerable) continue + if (!('value' in descriptor)) { + throw new TraceSecretProjectionError('Trace content accessors are not supported') + } + yield [key, descriptor.value] + } +} + +type LargeValueCandidate = LargeValueRef | LargeArrayManifest + +function readOwnDataProperty(value: object, key: string): unknown { + const descriptor = Object.getOwnPropertyDescriptor(value, key) + if (!descriptor) return undefined + if (!('value' in descriptor)) { + throw new TraceSecretProjectionError('Trace content accessors are not supported') + } + return descriptor.value +} + +function getLargeValueCandidate(value: unknown): LargeValueCandidate | undefined { + if (!value || typeof value !== 'object') return undefined + + const largeValueMarker = readOwnDataProperty(value, LARGE_VALUE_REF_MARKER) + const manifestMarker = readOwnDataProperty(value, LARGE_ARRAY_MANIFEST_MARKER) + if (largeValueMarker === true && manifestMarker === true) { + throw new TraceSecretProjectionError('Trace content has ambiguous large-value metadata') + } + if (largeValueMarker === true) { + if (!isLargeValueRef(value) || value.size > MAX_DURABLE_LARGE_VALUE_BYTES) { + throw new TraceSecretProjectionError('Trace content has invalid large-value metadata') + } + return value + } + if (manifestMarker !== true) return undefined + + const chunks = readOwnDataProperty(value, 'chunks') + const chunkCount = readOwnDataProperty(value, 'chunkCount') + const totalCount = readOwnDataProperty(value, 'totalCount') + const byteSize = readOwnDataProperty(value, 'byteSize') + if ( + !Array.isArray(chunks) || + chunks.length > MAX_LARGE_MANIFEST_CHUNKS || + chunkCount !== chunks.length || + typeof totalCount !== 'number' || + totalCount > MAX_LARGE_MANIFEST_ITEMS || + typeof byteSize !== 'number' || + byteSize > MAX_DURABLE_LARGE_VALUE_BYTES || + !isLargeArrayManifest(value) + ) { + throw new TraceSecretProjectionError('Trace content has invalid large-array metadata') + } + return value +} + +function sanitizeInlineValue( + value: unknown, + matcher: SecretMatcher, + safeLargeValues: WeakSet, + state: SanitizationTraversalState, + depth = 0 +): unknown { + visitNode(state, depth) + if (typeof value === 'string') { + const sanitized = sanitizeString(value, matcher, state.maxBytes - state.outputBytes) + state.outputBytes += Buffer.byteLength(sanitized, 'utf8') + return sanitized + } + if (value === null || typeof value === 'number' || typeof value === 'boolean') { + const rendered = String(value) + if (!containsSecret(rendered, matcher)) return value + const sanitized = sanitizeString(rendered, matcher, state.maxBytes - state.outputBytes) + state.outputBytes += Buffer.byteLength(sanitized, 'utf8') + return sanitized + } + if (value === undefined) return value + if (typeof value !== 'object') { + throw new TraceSecretProjectionError('Unsupported trace content value') + } + const largeValue = getLargeValueCandidate(value) + if (largeValue) { + if (!safeLargeValues.has(value as object)) { + throw new TraceSecretProjectionError('Trace content contains an unverified large value') + } + return value + } + if (!Array.isArray(value) && !isPlainRecord(value)) { + throw new TraceSecretProjectionError('Unsupported trace content object') + } + + enterObject(value, state) + try { + if (Array.isArray(value)) { + assertArrayFitsTraversal(value, state) + const sanitized = new Array(value.length) + for (const [index, item] of arrayDataEntries(value)) { + sanitized[index] = sanitizeInlineValue(item, matcher, safeLargeValues, state, depth + 1) + } + return sanitized + } + + const prototype = Object.getPrototypeOf(value) + const sanitized = Object.create(prototype) as Record + const sanitizedKeys = new Set() + for (const [key, item] of enumerableDataEntries(value)) { + const sanitizedKey = sanitizeString(key, matcher, state.maxBytes - state.outputBytes) + state.outputBytes += Buffer.byteLength(sanitizedKey, 'utf8') + if (sanitizedKeys.has(sanitizedKey)) { + throw new TraceSecretProjectionError('Secret replacement caused an object-key collision') + } + sanitizedKeys.add(sanitizedKey) + Object.defineProperty(sanitized, sanitizedKey, { + value: sanitizeInlineValue(item, matcher, safeLargeValues, state, depth + 1), + enumerable: true, + configurable: true, + writable: true, + }) + } + return sanitized + } finally { + leaveObject(value, state) + } +} + +function collectLargeValues( + value: unknown, + refs: object[], + seen: WeakSet, + state: TraversalState, + depth = 0 +): void { + visitNode(state, depth) + const largeValue = getLargeValueCandidate(value) + if (largeValue) { + refs.push(largeValue) + return + } + if (value === null || typeof value !== 'object') return + if (!Array.isArray(value) && !isPlainRecord(value)) { + throw new TraceSecretProjectionError('Unsupported trace content object') + } + if (seen.has(value)) return + seen.add(value) + enterObject(value, state) + try { + if (Array.isArray(value)) { + assertArrayFitsTraversal(value, state) + for (const [, item] of arrayDataEntries(value)) { + collectLargeValues(item, refs, seen, state, depth + 1) + } + return + } + for (const [, item] of enumerableDataEntries(value)) { + collectLargeValues(item, refs, seen, state, depth + 1) + } + } finally { + leaveObject(value, state) + } +} + +function substituteLargeValues( + value: unknown, + replacements: Map, + state: TraversalState, + depth = 0 +): unknown { + visitNode(state, depth) + if (getLargeValueCandidate(value)) { + if (!replacements.has(value as object)) { + throw new TraceSecretProjectionError('Large trace value was not safely replaced') + } + return replacements.get(value as object) + } + if (value === null || typeof value !== 'object') return value + if (!Array.isArray(value) && !isPlainRecord(value)) { + throw new TraceSecretProjectionError('Unsupported trace content object') + } + + enterObject(value, state) + try { + if (Array.isArray(value)) { + assertArrayFitsTraversal(value, state) + const substituted = new Array(value.length) + for (const [index, item] of arrayDataEntries(value)) { + substituted[index] = substituteLargeValues(item, replacements, state, depth + 1) + } + return substituted + } + const substituted = Object.create(Object.getPrototypeOf(value)) as Record + for (const [key, item] of enumerableDataEntries(value)) { + Object.defineProperty(substituted, key, { + value: substituteLargeValues(item, replacements, state, depth + 1), + enumerable: true, + configurable: true, + writable: true, + }) + } + return substituted + } finally { + leaveObject(value, state) + } +} + +async function runSerially(gate: OversizedHydrationGate, run: () => Promise): Promise { + const result = gate.chain.then(run) + gate.chain = result.then( + () => undefined, + () => undefined + ) + return result +} + +async function runWithSemaphore(semaphore: AsyncSemaphore, run: () => Promise): Promise { + if (semaphore.active >= REF_CONCURRENCY) { + await new Promise((resolve) => semaphore.waiters.push(resolve)) + } else { + semaphore.active += 1 + } + + try { + return await run() + } finally { + const next = semaphore.waiters.shift() + if (next) next() + else semaphore.active -= 1 + } +} + +function getLargeValueKey(value: LargeValueCandidate, context: ProjectionContext): string { + if (isLargeValueRef(value)) { + return `ref:${value.executionId ?? ''}:${value.key ?? ''}:${value.id}` + } + let id = context.manifestIds.get(value) + if (!id) { + id = `manifest:${context.nextManifestId}` + context.nextManifestId += 1 + context.manifestIds.set(value, id) + } + return id +} + +function registerSourceLargeValue( + value: LargeValueCandidate, + key: string, + context: ProjectionContext, + countBytes = true +): void { + if (context.seenLargeValues.has(key)) return + const size = countBytes ? (isLargeValueRef(value) ? value.size : value.byteSize) : 0 + if ( + context.largeValueCount + 1 > MAX_LARGE_VALUES || + context.sourceLargeValueBytes + size > MAX_DURABLE_LARGE_VALUE_BYTES + ) { + throw new TraceSecretProjectionError('Trace large-value projection budget exceeded') + } + context.seenLargeValues.add(key) + context.largeValueCount += 1 + context.sourceLargeValueBytes += size +} + +function extendLargeValuePath(path: ReadonlySet, key: string): Set { + if (path.has(key) || path.size >= MAX_LARGE_VALUE_CHAIN_DEPTH) { + throw new TraceSecretProjectionError( + 'Cyclic or deeply nested trace large values are unsupported' + ) + } + return new Set([...path, key]) +} + +async function materializeSourceRef( + ref: LargeValueRef, + context: ProjectionContext +): Promise { + const materialize = () => + runWithSemaphore(context.refIoSemaphore, () => + materializeLargeValueRef(ref, { + ...context.store, + trackReference: false, + maxBytes: MAX_DURABLE_LARGE_VALUE_BYTES, + }) + ) + const materialized = + ref.size > MAX_INLINE_MATERIALIZATION_BYTES + ? await runSerially(context.oversizedGate, materialize) + : await materialize() + if (materialized === undefined) { + throw new TraceSecretProjectionError('Large trace value could not be materialized') + } + return materialized +} + +async function sanitizeMaterializedValue( + value: unknown, + context: ProjectionContext, + maxBytes = MAX_INLINE_MATERIALIZATION_BYTES, + path: ReadonlySet = new Set(), + withinRefWorker = false +): Promise { + const withSafeRefs = await replaceLargeValues(value, context, path, withinRefWorker) + return sanitizeInlineValue(withSafeRefs, context.matcher, context.safeLargeValues, { + nodes: 0, + ancestors: new WeakSet(), + outputBytes: 0, + maxBytes, + }) +} + +async function storeSanitizedLargeValue( + value: unknown, + context: ProjectionContext +): Promise { + const json = JSON.stringify(value) + if (json === undefined) { + throw new TraceSecretProjectionError('Sanitized large value is not JSON serializable') + } + const size = Buffer.byteLength(json, 'utf8') + if (size > MAX_DURABLE_LARGE_VALUE_BYTES) { + throw new TraceSecretProjectionError('Sanitized large value exceeds the durable size limit') + } + if ( + context.storedLargeValueCount + 1 > MAX_LARGE_VALUES || + context.storedLargeValueBytes + size > MAX_DURABLE_LARGE_VALUE_BYTES + ) { + throw new TraceSecretProjectionError('Sanitized trace storage budget exceeded') + } + + context.storedLargeValueCount += 1 + context.storedLargeValueBytes += size + try { + const stored = await runWithSemaphore(context.refIoSemaphore, () => + storeLargeValue(value, json, size, { + ...context.store, + requireDurable: true, + }) + ) + if (!isLargeValueRef(stored)) { + throw new TraceSecretProjectionError('Trace storage returned invalid large-value metadata') + } + context.safeLargeValues.add(stored) + return stored + } catch (error) { + context.storedLargeValueCount -= 1 + context.storedLargeValueBytes -= size + throw error + } +} + +async function replaceLargeValueRef( + ref: LargeValueRef, + context: ProjectionContext, + path: ReadonlySet +): Promise { + const materialized = await materializeSourceRef(ref, context) + const sanitized = await sanitizeMaterializedValue( + materialized, + context, + MAX_DURABLE_LARGE_VALUE_BYTES, + path, + true + ) + const stored = await storeSanitizedLargeValue(sanitized, context) + if (getLargeValueKey(stored, context) === getLargeValueKey(ref, context)) { + throw new TraceSecretProjectionError('Trace storage reused the source large-value reference') + } + return stored +} + +async function replaceLargeArrayManifest( + manifest: LargeArrayManifest, + context: ProjectionContext, + path: ReadonlySet +): Promise { + const preview = await sanitizeMaterializedValue( + manifest.preview, + context, + LARGE_ARRAY_MANIFEST_PREVIEW_MAX_BYTES, + path, + true + ) + if (!Array.isArray(preview) || preview.length > 3) { + throw new TraceSecretProjectionError('Sanitized trace manifest preview is invalid') + } + + const chunks: LargeArrayManifest['chunks'] = [] + const sourceChunkKeys = new Set( + manifest.chunks.map((chunk) => getLargeValueKey(chunk.ref, context)) + ) + const storedChunkKeys = new Set() + let totalCount = 0 + for (const chunk of manifest.chunks) { + const chunkKey = getLargeValueKey(chunk.ref, context) + const chunkPath = extendLargeValuePath(path, chunkKey) + registerSourceLargeValue(chunk.ref, chunkKey, context, false) + const materialized = await materializeSourceRef(chunk.ref, context) + if (!Array.isArray(materialized) || materialized.length !== chunk.count) { + throw new TraceSecretProjectionError('Large trace manifest chunk is invalid') + } + const remainingOutputBytes = MAX_DURABLE_LARGE_VALUE_BYTES - context.storedLargeValueBytes + if (remainingOutputBytes <= 0) { + throw new TraceSecretProjectionError('Sanitized trace storage budget exceeded') + } + const sanitized = await sanitizeMaterializedValue( + materialized, + context, + remainingOutputBytes, + chunkPath, + true + ) + if (!Array.isArray(sanitized)) { + throw new TraceSecretProjectionError('Sanitized trace manifest chunk is invalid') + } + totalCount += sanitized.length + const ref = await storeSanitizedLargeValue(sanitized, context) + const storedChunkKey = getLargeValueKey(ref, context) + if (sourceChunkKeys.has(storedChunkKey) || storedChunkKeys.has(storedChunkKey)) { + throw new TraceSecretProjectionError('Trace storage returned a reused manifest chunk ref') + } + storedChunkKeys.add(storedChunkKey) + chunks.push({ ref, count: sanitized.length, byteSize: ref.size }) + } + if (totalCount !== manifest.totalCount) { + throw new TraceSecretProjectionError('Sanitized trace manifest item count changed') + } + + const result: LargeArrayManifest = { + __simLargeArrayManifest: true, + version: 2, + kind: 'array', + totalCount, + chunkCount: chunks.length, + byteSize: chunks.reduce((sum, chunk) => sum + chunk.byteSize, 0), + chunks, + preview, + } + context.safeLargeValues.add(result) + return result +} + +async function replaceLargeValue( + value: LargeValueCandidate, + context: ProjectionContext, + path: ReadonlySet +): Promise { + if (!context.allowLargeValueWrites) { + throw new TraceSecretProjectionError('Large trace values are disabled for read-only projection') + } + const key = getLargeValueKey(value, context) + const nextPath = extendLargeValuePath(path, key) + const cached = context.largeValueCache.get(key) + if (cached) return cached + + registerSourceLargeValue(value, key, context) + const replacement = isLargeValueRef(value) + ? replaceLargeValueRef(value, context, nextPath) + : replaceLargeArrayManifest(value, context, nextPath) + context.largeValueCache.set(key, replacement) + return replacement +} + +async function resolveLargeValueReplacements( + refs: object[], + context: ProjectionContext, + path: ReadonlySet, + withinRefWorker: boolean +): Promise> { + const uniqueRefs = [...new Set(refs)] + const replacements = new Map() + let firstError: unknown + let cursor = 0 + + const worker = async (): Promise => { + while (firstError === undefined) { + const index = cursor + cursor += 1 + if (index >= uniqueRefs.length) return + const ref = uniqueRefs[index] + try { + const candidate = getLargeValueCandidate(ref) + if (!candidate) { + throw new TraceSecretProjectionError( + 'Trace large-value metadata changed during projection' + ) + } + replacements.set(ref, await replaceLargeValue(candidate, context, path)) + } catch (error) { + if (firstError === undefined) firstError = error + return + } + } + } + + const workerCount = withinRefWorker ? 1 : Math.min(REF_CONCURRENCY, uniqueRefs.length) + await Promise.all(Array.from({ length: workerCount }, worker)) + + if (firstError !== undefined) throw firstError + return replacements +} + +async function replaceLargeValues( + value: unknown, + context: ProjectionContext, + path: ReadonlySet, + withinRefWorker: boolean +): Promise { + const refs: object[] = [] + collectLargeValues(value, refs, new WeakSet(), { + nodes: 0, + ancestors: new WeakSet(), + }) + if (refs.length === 0) return value + const replacements = await resolveLargeValueReplacements(refs, context, path, withinRefWorker) + return substituteLargeValues(value, replacements, { + nodes: 0, + ancestors: new WeakSet(), + }) +} + +async function sanitizeContentField( + value: unknown, + context: ProjectionContext +): Promise { + try { + return await sanitizeMaterializedValue(value, context) + } catch { + logger.warn('Omitting trace content that could not be sanitized') + return OMIT + } +} + +function assertTraceStructureArrayFits( + values: readonly unknown[], + state: BoundedTraceStructureState +): void { + if (values.length > MAX_CONTENT_NODES - state.nodes) { + state.truncated = true + throw new TraceSecretProjectionError('Trace structure array exceeds the projection limit') + } +} + +async function sanitizeIterationToolCalls( + calls: IterationToolCall[], + context: ProjectionContext, + state: BoundedTraceStructureState, + depth: number +): Promise { + assertTraceStructureArrayFits(calls, state) + const sanitized: IterationToolCall[] = [] + for (const call of calls) { + if (!takeTraceStructureNode(state, depth)) { + throw new TraceSecretProjectionError('Trace tool-call structure limit exceeded') + } + const args = await sanitizeContentField(call.arguments, context) + sanitized.push( + args === OMIT + ? (omit(call, ['arguments']) as IterationToolCall) + : { ...call, arguments: args as IterationToolCall['arguments'] } + ) + } + return sanitized +} + +async function sanitizeLegacyToolCalls( + calls: ToolCall[], + context: ProjectionContext, + state: BoundedTraceStructureState, + depth: number +): Promise { + assertTraceStructureArrayFits(calls, state) + const sanitized: ToolCall[] = [] + for (const call of calls) { + if (!takeTraceStructureNode(state, depth)) { + throw new TraceSecretProjectionError('Trace tool-call structure limit exceeded') + } + let projected: ToolCall = { ...call } + if (call.input !== undefined) { + const input = await sanitizeContentField(call.input, context) + if (input === OMIT) projected = omit(projected, ['input']) as ToolCall + else projected.input = input as Record + } + if (call.output !== undefined) { + const output = await sanitizeContentField(call.output, context) + if (output === OMIT) projected = omit(projected, ['output']) as ToolCall + else projected.output = output as Record + } + if (call.error !== undefined) { + const error = await sanitizeContentField(call.error, context) + if (error === OMIT) projected = omit(projected, ['error']) as ToolCall + else projected.error = error as string + } + sanitized.push(projected) + } + return sanitized +} + +async function sanitizeProviderSegment( + segment: ProviderTimingSegment, + context: ProjectionContext, + state: BoundedTraceStructureState, + depth: number +): Promise { + if (!takeTraceStructureNode(state, depth)) { + throw new TraceSecretProjectionError('Trace provider structure limit exceeded') + } + let projected: ProviderTimingSegment = { ...segment } + + for (const key of ['assistantContent', 'thinkingContent', 'errorMessage'] as const) { + const value = segment[key] + if (value === undefined) continue + const sanitized = await sanitizeContentField(value, context) + if (sanitized === OMIT) { + projected = omit(projected, [key]) as ProviderTimingSegment + } else { + projected[key] = sanitized as string + } + } + + if (segment.toolCalls !== undefined) { + projected.toolCalls = await sanitizeIterationToolCalls( + segment.toolCalls, + context, + state, + depth + 1 + ) + } + + return projected +} + +async function sanitizeTraceSpan( + span: TraceSpan, + context: ProjectionContext, + depth: number, + state: BoundedTraceStructureState +): Promise { + if (!takeTraceStructureNode(state, depth)) { + throw new TraceSecretProjectionError('Trace span structure limit exceeded') + } + + let projected: TraceSpan = { ...span } + + for (const key of ['input', 'output', 'thinking', 'errorMessage'] as const) { + const value = span[key] + if (value === undefined) continue + const sanitized = await sanitizeContentField(value, context) + if (sanitized === OMIT) { + projected = omit(projected, [key]) as TraceSpan + } else { + projected[key] = sanitized as never + } + } + + if (span.modelToolCalls !== undefined) { + projected.modelToolCalls = await sanitizeIterationToolCalls( + span.modelToolCalls, + context, + state, + depth + 1 + ) + } + + if (span.toolCalls !== undefined) { + projected.toolCalls = await sanitizeLegacyToolCalls(span.toolCalls, context, state, depth + 1) + } + + if (span.providerTiming !== undefined) { + assertTraceStructureArrayFits(span.providerTiming.segments, state) + const segments: ProviderTimingSegment[] = [] + for (const segment of span.providerTiming.segments) { + segments.push(await sanitizeProviderSegment(segment, context, state, depth + 1)) + } + projected.providerTiming = { ...span.providerTiming, segments } + } + + if (span.children !== undefined) { + assertTraceStructureArrayFits(span.children, state) + const children: TraceSpan[] = [] + for (const child of span.children) { + children.push(await sanitizeTraceSpan(child, context, depth + 1, state)) + } + projected.children = children + } + + return projected +} + +function stripLegacyToolCallContent(call: ToolCall): ToolCall { + return omit(call, ['input', 'output', 'error']) as ToolCall +} + +function stripIterationToolCallContent(call: IterationToolCall): IterationToolCall { + return omit(call, ['arguments']) as IterationToolCall +} + +interface BoundedTraceStructureState { + nodes: number + truncated: boolean +} + +function takeTraceStructureNode(state: BoundedTraceStructureState, depth: number): boolean { + if (state.nodes >= MAX_CONTENT_NODES || depth > MAX_CONTENT_DEPTH) { + state.truncated = true + return false + } + state.nodes += 1 + return true +} + +function projectBoundedArray( + values: readonly T[], + project: (value: T) => U | undefined +): U[] { + const projected: U[] = [] + for (const value of values) { + const item = project(value) + if (item === undefined) break + projected.push(item) + } + return projected +} + +function structuralOnlyIterationToolCall( + call: IterationToolCall, + state: BoundedTraceStructureState, + depth: number +): IterationToolCall | undefined { + if (!takeTraceStructureNode(state, depth)) return undefined + return stripIterationToolCallContent(call) +} + +function structuralOnlyLegacyToolCall( + call: ToolCall, + state: BoundedTraceStructureState, + depth: number +): ToolCall | undefined { + if (!takeTraceStructureNode(state, depth)) return undefined + return stripLegacyToolCallContent(call) +} + +function structuralOnlyProviderSegment( + segment: ProviderTimingSegment, + state: BoundedTraceStructureState, + depth: number +): ProviderTimingSegment | undefined { + if (!takeTraceStructureNode(state, depth)) return undefined + const { + assistantContent: _assistantContent, + thinkingContent: _thinkingContent, + toolCalls, + errorMessage: _errorMessage, + ...structural + } = segment + return { + ...structural, + ...(toolCalls + ? { + toolCalls: projectBoundedArray(toolCalls, (call) => + structuralOnlyIterationToolCall(call, state, depth + 1) + ), + } + : {}), + } +} + +function structuralOnlySpan( + span: TraceSpan, + state: BoundedTraceStructureState, + depth = 0 +): TraceSpan | undefined { + if (!takeTraceStructureNode(state, depth)) return undefined + const { + input: _input, + output: _output, + thinking: _thinking, + modelToolCalls, + toolCalls, + errorMessage: _errorMessage, + children, + providerTiming, + ...structural + } = span + + return { + ...structural, + ...(modelToolCalls + ? { + modelToolCalls: projectBoundedArray(modelToolCalls, (call) => + structuralOnlyIterationToolCall(call, state, depth + 1) + ), + } + : {}), + ...(toolCalls + ? { + toolCalls: projectBoundedArray(toolCalls, (call) => + structuralOnlyLegacyToolCall(call, state, depth + 1) + ), + } + : {}), + ...(providerTiming + ? { + providerTiming: { + ...providerTiming, + segments: projectBoundedArray(providerTiming.segments, (segment) => + structuralOnlyProviderSegment(segment, state, depth + 1) + ), + }, + } + : {}), + ...(children + ? { + children: projectBoundedArray(children, (child) => + structuralOnlySpan(child, state, depth + 1) + ), + } + : {}), + } +} + +function cloneIterationToolCall( + call: IterationToolCall, + state: BoundedTraceStructureState, + depth: number +): IterationToolCall | undefined { + if (!takeTraceStructureNode(state, depth)) return undefined + return { ...call } +} + +function cloneLegacyToolCall( + call: ToolCall, + state: BoundedTraceStructureState, + depth: number +): ToolCall | undefined { + if (!takeTraceStructureNode(state, depth)) return undefined + return { ...call } +} + +function cloneProviderSegment( + segment: ProviderTimingSegment, + state: BoundedTraceStructureState, + depth: number +): ProviderTimingSegment | undefined { + if (!takeTraceStructureNode(state, depth)) return undefined + return { + ...segment, + ...(segment.toolCalls + ? { + toolCalls: projectBoundedArray(segment.toolCalls, (call) => + cloneIterationToolCall(call, state, depth + 1) + ), + } + : {}), + } +} + +function cloneTraceSpanForProjection( + span: TraceSpan, + state: BoundedTraceStructureState, + depth = 0 +): TraceSpan | undefined { + if (!takeTraceStructureNode(state, depth)) return undefined + return { + ...span, + ...(span.modelToolCalls + ? { + modelToolCalls: projectBoundedArray(span.modelToolCalls, (call) => + cloneIterationToolCall(call, state, depth + 1) + ), + } + : {}), + ...(span.toolCalls + ? { + toolCalls: projectBoundedArray(span.toolCalls, (call) => + cloneLegacyToolCall(call, state, depth + 1) + ), + } + : {}), + ...(span.providerTiming + ? { + providerTiming: { + ...span.providerTiming, + segments: projectBoundedArray(span.providerTiming.segments, (segment) => + cloneProviderSegment(segment, state, depth + 1) + ), + }, + } + : {}), + ...(span.children + ? { + children: projectBoundedArray(span.children, (child) => + cloneTraceSpanForProjection(child, state, depth + 1) + ), + } + : {}), + } +} + +function projectBoundedTraceSpans( + traceSpans: TraceSpan[], + project: ( + span: TraceSpan, + state: BoundedTraceStructureState, + depth: number + ) => TraceSpan | undefined +): TraceSpan[] { + const state: BoundedTraceStructureState = { nodes: 0, truncated: false } + const projected = projectBoundedArray(traceSpans, (span) => project(span, state, 0)) + if (state.truncated) { + logger.warn('Trace structure exceeded projection limits; truncating projected structure') + } + return projected +} + +function structuralOnlyTraceSpans(traceSpans: TraceSpan[]): TraceSpan[] { + try { + return projectBoundedTraceSpans(traceSpans, structuralOnlySpan) + } catch { + logger.warn('Trace structure could not be safely traversed; omitting projected spans') + return [] + } +} + +function cloneTraceSpansForProjection(traceSpans: TraceSpan[]): TraceSpan[] { + return projectBoundedTraceSpans(traceSpans, cloneTraceSpanForProjection) +} + +function assertNoPlaintext( + value: unknown, + context: ProjectionContext, + state: TraversalState, + depth = 0 +): void { + visitNode(state, depth) + if (typeof value === 'string') { + if (containsSecret(value, context.matcher)) { + throw new TraceSecretProjectionError('Sanitized trace content still contains a secret') + } + return + } + if (value === null || typeof value === 'number' || typeof value === 'boolean') { + if (containsSecret(String(value), context.matcher)) { + throw new TraceSecretProjectionError('Sanitized trace primitive still contains a secret') + } + return + } + if (getLargeValueCandidate(value)) { + if (!context.safeLargeValues.has(value as object)) { + throw new TraceSecretProjectionError('Sanitized trace content contains an unverified ref') + } + return + } + if (value === null || typeof value !== 'object') return + if (!Array.isArray(value) && !isPlainRecord(value)) { + throw new TraceSecretProjectionError('Sanitized trace content contains an unsupported object') + } + + enterObject(value, state) + try { + if (Array.isArray(value)) { + assertArrayFitsTraversal(value, state) + for (const [, item] of arrayDataEntries(value)) { + assertNoPlaintext(item, context, state, depth + 1) + } + return + } + for (const [key, item] of enumerableDataEntries(value)) { + if (containsSecret(key, context.matcher)) { + throw new TraceSecretProjectionError('Sanitized trace key still contains a secret') + } + assertNoPlaintext(item, context, state, depth + 1) + } + } finally { + leaveObject(value, state) + } +} + +function assertTraceSpanContentIsSafe(span: TraceSpan, context: ProjectionContext): void { + const assertField = (value: unknown): void => { + if (value === undefined) return + assertNoPlaintext(value, context, { nodes: 0, ancestors: new WeakSet() }) + } + + assertField(span.input) + assertField(span.output) + assertField(span.thinking) + assertField(span.errorMessage) + if (span.modelToolCalls) { + for (const call of span.modelToolCalls) { + if (Object.hasOwn(call, 'arguments')) assertField(call.arguments) + } + } + if (span.toolCalls) { + for (const call of span.toolCalls) { + assertField(call.input) + assertField(call.output) + assertField(call.error) + } + } + if (span.providerTiming) { + for (const segment of span.providerTiming.segments) { + assertField(segment.assistantContent) + assertField(segment.thinkingContent) + assertField(segment.errorMessage) + for (const call of segment.toolCalls ?? []) { + if (Object.hasOwn(call, 'arguments')) assertField(call.arguments) + } + } + } + for (const child of span.children ?? []) assertTraceSpanContentIsSafe(child, context) +} + +/** + * Produces the only Secrets-feature-safe representation of execution TraceSpans. + * Runtime logs and outputs remain untouched; only schema-defined trace content is copied. + */ +export async function projectTraceSpansForSecrets( + traceSpans: TraceSpan[], + options: ProjectTraceSpansForSecretsOptions +): Promise { + if (!options.registry?.isComplete()) { + return structuralOnlyTraceSpans(traceSpans) + } + + try { + const replacements = normalizeReplacements(options.registry.getActiveMatches()) + if (replacements.length === 0) return cloneTraceSpansForProjection(traceSpans) + + const context: ProjectionContext = { + matcher: createSecretMatcher(replacements), + store: options.store, + allowLargeValueWrites: options.allowLargeValueWrites !== false, + safeLargeValues: new WeakSet(), + oversizedGate: { chain: Promise.resolve() }, + refIoSemaphore: { active: 0, waiters: [] }, + seenLargeValues: new Set(), + largeValueCache: new Map>(), + manifestIds: new WeakMap(), + nextManifestId: 0, + largeValueCount: 0, + sourceLargeValueBytes: 0, + storedLargeValueBytes: 0, + storedLargeValueCount: 0, + } + + const projected: TraceSpan[] = [] + const state: BoundedTraceStructureState = { nodes: 0, truncated: false } + assertTraceStructureArrayFits(traceSpans, state) + for (const span of traceSpans) { + projected.push(await sanitizeTraceSpan(span, context, 0, state)) + } + for (const span of projected) assertTraceSpanContentIsSafe(span, context) + return projected + } catch { + logger.warn('Trace secret projection failed; retaining structural spans only') + return structuralOnlyTraceSpans(traceSpans) + } +} diff --git a/apps/sim/lib/logs/execution/trace-store.test.ts b/apps/sim/lib/logs/execution/trace-store.test.ts new file mode 100644 index 00000000000..b1341811878 --- /dev/null +++ b/apps/sim/lib/logs/execution/trace-store.test.ts @@ -0,0 +1,140 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { decryptSecretMock } = vi.hoisted(() => ({ + decryptSecretMock: vi.fn(), +})) + +vi.mock('@/lib/core/security/encryption', () => ({ + decryptSecret: decryptSecretMock, +})) + +import { projectExecutionDataForDisplay } from '@/lib/logs/execution/trace-store' + +const CONTEXT = { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + userId: 'user-1', +} + +beforeEach(() => { + vi.clearAllMocks() + decryptSecretMock.mockResolvedValue({ decrypted: '1234' }) +}) + +describe('projectExecutionDataForDisplay', () => { + it('projects persisted output, input, errors, and spans from trusted provenance', async () => { + const executionData = { + finalOutput: { result: 1234, derived: 1239 }, + workflowInput: { nested: { token: 'prefix-1234-suffix' } }, + completionFailure: 'Function failed with 1234', + errorDetails: { blockId: 'function-1', error: 'Invalid token 1234' }, + traceSpans: [ + { + id: 'span-1', + name: 'Function 1', + type: 'function', + duration: 1, + startTime: '2026-07-31T00:00:00.000Z', + endTime: '2026-07-31T00:00:00.001Z', + output: { result: 1234 }, + }, + ], + executionState: { + blockStates: { 'function-1': { output: { result: 1234 } } }, + resolvedSecretTraceProvenance: { + version: 1 as const, + complete: true, + entries: [{ name: 'OPENAI_API_KEY', encryptedValue: 'ciphertext' }], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + }, + }, + } + + const displayData = await projectExecutionDataForDisplay(executionData, CONTEXT) + + expect(displayData.finalOutput).toEqual({ + result: '{{OPENAI_API_KEY}}', + derived: 1239, + }) + expect(displayData.workflowInput).toEqual({ + nested: { token: 'prefix-{{OPENAI_API_KEY}}-suffix' }, + }) + expect(displayData.completionFailure).toBe('Function failed with {{OPENAI_API_KEY}}') + expect(displayData.errorDetails).toEqual({ + blockId: 'function-1', + error: 'Invalid token {{OPENAI_API_KEY}}', + }) + expect(displayData.traceSpans).toEqual([ + expect.objectContaining({ output: { result: '{{OPENAI_API_KEY}}' } }), + ]) + expect(displayData).not.toHaveProperty('executionState') + expect(executionData.finalOutput).toEqual({ result: 1234, derived: 1239 }) + expect(executionData.executionState.resolvedSecretTraceProvenance.entries).toEqual([ + { name: 'OPENAI_API_KEY', encryptedValue: 'ciphertext' }, + ]) + expect(JSON.stringify(displayData)).not.toContain('1234') + }) + + it('omits log content and keeps only structural traces without trusted provenance', async () => { + const displayData = await projectExecutionDataForDisplay( + { + finalOutput: { result: 'unknown-secret' }, + workflowInput: { token: 'unknown-secret' }, + completionFailure: 'unknown-secret', + traceSpans: [ + { + id: 'span-1', + name: 'Function 1', + type: 'function', + duration: 1, + startTime: '2026-07-31T00:00:00.000Z', + endTime: '2026-07-31T00:00:00.001Z', + output: { result: 'unknown-secret' }, + }, + ], + }, + CONTEXT + ) + + expect(displayData).not.toHaveProperty('finalOutput') + expect(displayData).not.toHaveProperty('workflowInput') + expect(displayData).not.toHaveProperty('completionFailure') + expect(displayData.traceSpans).toEqual([ + expect.not.objectContaining({ output: expect.anything() }), + ]) + }) + + it('preserves direct literals when trusted provenance has no activated secrets', async () => { + const displayData = await projectExecutionDataForDisplay( + { + finalOutput: { result: 'direct-literal' }, + executionState: { + resolvedSecretTraceProvenance: { + version: 1, + complete: true, + entries: [], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + }, + }, + }, + CONTEXT + ) + + expect(displayData.finalOutput).toEqual({ result: 'direct-literal' }) + }) + + it('omits malformed trace content even when it was present on the stored row', async () => { + const displayData = await projectExecutionDataForDisplay( + { + traceSpans: { output: 'unsafe' }, + }, + CONTEXT + ) + + expect(displayData).not.toHaveProperty('traceSpans') + }) +}) diff --git a/apps/sim/lib/logs/execution/trace-store.ts b/apps/sim/lib/logs/execution/trace-store.ts index 83312eb1ec2..16bf3b6cd75 100644 --- a/apps/sim/lib/logs/execution/trace-store.ts +++ b/apps/sim/lib/logs/execution/trace-store.ts @@ -1,7 +1,14 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' +import { omit } from '@sim/utils/object' import { isLargeValueRef } from '@/lib/execution/payloads/large-value-ref' import { materializeLargeValueRef, storeLargeValue } from '@/lib/execution/payloads/store' +import { projectTraceSpansForSecrets } from '@/lib/logs/execution/trace-secret-projection' +import type { TraceSpan } from '@/lib/logs/types' +import { + isResolvedSecretTraceProvenanceV1, + ResolvedSecretTraceRegistry, +} from '@/executor/utils/resolved-secret-trace-registry' const logger = createLogger('TraceStore') @@ -26,10 +33,11 @@ const INLINE_MARKER_KEYS = ['hasTraceSpans', 'traceSpanCount'] as const * Read-path context. Resolves an externalized payload by storage key, authorized * via the (already-authorized) workspace — no owner needed. */ -interface TraceStoreReadContext { +export interface TraceStoreReadContext { workspaceId: string | null workflowId: string | null executionId: string + userId?: string } /** @@ -69,6 +77,14 @@ export function stripSpanCosts(spans: unknown): void { } } +/** Creates a persistence-owned span tree with per-span cost fields removed. */ +export function copyTraceSpansWithoutCosts(spans?: TraceSpan[]): TraceSpan[] | undefined { + return spans?.map(({ cost: _cost, children, ...span }) => ({ + ...span, + ...(children ? { children: copyTraceSpansWithoutCosts(children) } : {}), + })) +} + /** * Externalizes heavy `execution_data` to object storage as a single large value * (reusing the execution-context large-value store + its reference/dependency/GC @@ -177,3 +193,105 @@ export async function materializeExecutionData( return markers } } + +const LOG_DISPLAY_CONTENT_KEYS = [ + 'finalOutput', + 'workflowInput', + 'blockInput', + 'blockExecutions', + 'error', + 'errorDetails', + 'completionFailure', + 'message', +] as const + +const LOG_DISPLAY_PROJECTION_SPAN_ID = 'secret-safe-log-display-projection' + +/** + * Materializes trusted execution data and returns its log-facing projection. + * Functional readers must continue using {@link materializeExecutionData}. + */ +export async function materializeExecutionDataForDisplay( + executionData: Record | null | undefined, + context: TraceStoreReadContext +): Promise> { + const materialized = await materializeExecutionData(executionData, context) + return projectExecutionDataForDisplay(materialized, context) +} + +/** + * Projects execution-log content with the encrypted provenance saved by the + * trusted executor. Missing or malformed provenance deliberately yields a + * structural-only log instead of returning content that cannot be proven safe. + */ +export async function projectExecutionDataForDisplay( + executionData: Record, + context: TraceStoreReadContext +): Promise> { + const executionState = + executionData.executionState && + typeof executionData.executionState === 'object' && + !Array.isArray(executionData.executionState) + ? (executionData.executionState as Record) + : undefined + const provenance = executionState?.resolvedSecretTraceProvenance + let registry: ResolvedSecretTraceRegistry | undefined + + if (isResolvedSecretTraceProvenanceV1(provenance)) { + registry = new ResolvedSecretTraceRegistry([], provenance.scope) + await registry.importProvenance(provenance, { trusted: true }) + } + + const envelope: Record = {} + for (const key of LOG_DISPLAY_CONTENT_KEYS) { + if (Object.hasOwn(executionData, key)) envelope[key] = executionData[key] + } + + const now = new Date().toISOString() + const syntheticSpan: TraceSpan = { + id: LOG_DISPLAY_PROJECTION_SPAN_ID, + name: 'Log Display Projection', + type: 'display', + duration: 0, + startTime: now, + endTime: now, + output: envelope, + } + const sourceTraceSpans = Array.isArray(executionData.traceSpans) + ? (executionData.traceSpans as TraceSpan[]) + : [] + const projectedSpans = await projectTraceSpansForSecrets([syntheticSpan, ...sourceTraceSpans], { + registry, + allowLargeValueWrites: false, + store: { + workspaceId: context.workspaceId ?? undefined, + workflowId: context.workflowId ?? undefined, + executionId: context.executionId, + userId: context.userId, + trackReference: false, + }, + }) + + const displayData = omit(executionData, [ + ...LOG_DISPLAY_CONTENT_KEYS, + 'executionState', + 'traceSpans', + ]) as Record + + const projectedEnvelope = projectedSpans.find( + (span) => span.id === LOG_DISPLAY_PROJECTION_SPAN_ID + )?.output + if (projectedEnvelope) { + for (const key of LOG_DISPLAY_CONTENT_KEYS) { + if (Object.hasOwn(projectedEnvelope, key)) displayData[key] = projectedEnvelope[key] + } + } + + if (Array.isArray(executionData.traceSpans)) { + displayData.traceSpans = projectedSpans.filter( + (span) => span.id !== LOG_DISPLAY_PROJECTION_SPAN_ID + ) + } + + return displayData +} diff --git a/apps/sim/lib/logs/fetch-log-detail.ts b/apps/sim/lib/logs/fetch-log-detail.ts index 8a4b329a328..2ad0c43ffa2 100644 --- a/apps/sim/lib/logs/fetch-log-detail.ts +++ b/apps/sim/lib/logs/fetch-log-detail.ts @@ -15,7 +15,7 @@ import { pickLatestCompletedMarker, pickLatestStartedMarker, } from '@/lib/logs/execution/progress-markers' -import { materializeExecutionData } from '@/lib/logs/execution/trace-store' +import { materializeExecutionDataForDisplay } from '@/lib/logs/execution/trace-store' import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils' type LookupColumn = 'id' | 'executionId' @@ -170,9 +170,14 @@ export async function fetchLogDetail({ // Trace spans / heavy execution data may live in object storage; resolve the // pointer here (no-op for inline / pre-externalization rows). - const executionData = await materializeExecutionData( + const executionData = await materializeExecutionDataForDisplay( log.executionData as Record | null, - { workspaceId, workflowId: log.workflowId, executionId: log.executionId } + { + workspaceId, + workflowId: log.workflowId, + executionId: log.executionId, + userId, + } ) const liveMarkers = @@ -248,7 +253,15 @@ export async function fetchLogDetail({ const jobLog = jobRows[0] if (!jobLog) return null - const execData = (jobLog.executionData as Record | null) ?? {} + const execData = await materializeExecutionDataForDisplay( + jobLog.executionData as Record | null, + { + workspaceId, + workflowId: null, + executionId: jobLog.executionId, + userId, + } + ) return { id: jobLog.id, workflowId: null, diff --git a/apps/sim/lib/logs/types.ts b/apps/sim/lib/logs/types.ts index 4c0e303eb0c..29c6e60792a 100644 --- a/apps/sim/lib/logs/types.ts +++ b/apps/sim/lib/logs/types.ts @@ -433,6 +433,15 @@ export interface SnapshotCreationResult { } export interface ExecutionLoggerService { + prepareTraceSpansForProjection(params: { + executionId: string + workflowId: string + workspaceId: string | null + userId?: string | null + traceSpans: TraceSpan[] + isResume?: boolean + }): Promise + startWorkflowExecution(params: { workflowId: string workspaceId: string diff --git a/apps/sim/lib/mcp/client.ts b/apps/sim/lib/mcp/client.ts index 194755c9a05..85269355106 100644 --- a/apps/sim/lib/mcp/client.ts +++ b/apps/sim/lib/mcp/client.ts @@ -74,6 +74,7 @@ export class McpClient { private authProvider?: McpClientOptions['authProvider'] private isConnected = false private closeGuardedTransport?: () => Promise + private readonly resolvedSecretTraceProvenance?: McpClientOptions['resolvedSecretTraceProvenance'] constructor(options: McpClientOptions) { this.config = options.config @@ -84,6 +85,7 @@ export class McpClient { } this.onToolsChanged = options.onToolsChanged this.authProvider = options.authProvider + this.resolvedSecretTraceProvenance = options.resolvedSecretTraceProvenance const resolvedIP = options.resolvedIP this.connectionStatus = { connected: false } @@ -133,6 +135,12 @@ export class McpClient { } } + getResolvedSecretTraceProvenance(): McpClientOptions['resolvedSecretTraceProvenance'] { + return this.resolvedSecretTraceProvenance + ? structuredClone(this.resolvedSecretTraceProvenance) + : undefined + } + /** * Initialize connection to MCP server. * If an `onToolsChanged` callback was provided, registers a notification handler diff --git a/apps/sim/lib/mcp/resolve-config.test.ts b/apps/sim/lib/mcp/resolve-config.test.ts new file mode 100644 index 00000000000..4b7e0f25473 --- /dev/null +++ b/apps/sim/lib/mcp/resolve-config.test.ts @@ -0,0 +1,158 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockGetPersonalAndWorkspaceEnv } = vi.hoisted(() => ({ + mockGetPersonalAndWorkspaceEnv: vi.fn(), +})) + +vi.mock('@/lib/environment/utils', () => ({ + getPersonalAndWorkspaceEnv: mockGetPersonalAndWorkspaceEnv, +})) + +import { resolveMcpConfigEnvVars } from '@/lib/mcp/resolve-config' + +const BASE_CONFIG = { + id: 'server-1', + name: 'Server', + transport: 'streamable-http' as const, + url: 'https://{{MCP_HOST}}/mcp', + headers: { + Authorization: 'Bearer {{MCP_TOKEN}}', + 'X-Direct': 'literal-secret', + }, +} + +describe('resolveMcpConfigEnvVars secret provenance', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetPersonalAndWorkspaceEnv.mockResolvedValue({ + personalEncrypted: { + MCP_HOST: 'personal-host-encrypted', + MCP_TOKEN: 'personal-token-encrypted', + UNUSED: 'unused-encrypted', + }, + workspaceEncrypted: { MCP_TOKEN: 'workspace-token-encrypted' }, + personalDecrypted: { + MCP_HOST: 'api.example.com', + MCP_TOKEN: 'personal-token', + UNUSED: 'literal-secret', + }, + workspaceDecrypted: { MCP_TOKEN: 'workspace-token' }, + conflicts: ['MCP_TOKEN'], + decryptionFailures: [], + }) + }) + + it('returns encrypted provenance only for successful {{NAME}} substitutions', async () => { + const result = await resolveMcpConfigEnvVars(BASE_CONFIG, 'user-1', 'workspace-1') + + expect(result.config.url).toBe('https://api.example.com/mcp') + expect(result.config.headers).toEqual({ + Authorization: 'Bearer workspace-token', + 'X-Direct': 'literal-secret', + }) + expect(result.resolvedSecretTraceProvenance).toEqual({ + version: 1, + complete: true, + entries: [ + { name: 'MCP_HOST', encryptedValue: 'personal-host-encrypted' }, + { name: 'MCP_TOKEN', encryptedValue: 'workspace-token-encrypted' }, + ], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + }) + expect(JSON.stringify(result.resolvedSecretTraceProvenance)).not.toContain('workspace-token"') + expect(JSON.stringify(result.resolvedSecretTraceProvenance)).not.toContain('literal-secret') + }) + + it('marks provenance incomplete when a resolved value has no encrypted catalog entry', async () => { + mockGetPersonalAndWorkspaceEnv.mockResolvedValue({ + personalEncrypted: {}, + workspaceEncrypted: {}, + personalDecrypted: { MCP_HOST: 'api.example.com', MCP_TOKEN: 'token' }, + workspaceDecrypted: {}, + conflicts: [], + decryptionFailures: [], + }) + + const result = await resolveMcpConfigEnvVars(BASE_CONFIG, 'user-1', 'workspace-1') + + expect(result.resolvedSecretTraceProvenance).toEqual({ + version: 1, + complete: false, + entries: [], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + }) + }) + + it('does not let an unused configured-secret failure affect invocation provenance', async () => { + mockGetPersonalAndWorkspaceEnv.mockResolvedValue({ + personalEncrypted: { + MCP_HOST: 'host-encrypted', + MCP_TOKEN: 'token-encrypted', + UNUSED: 'broken-encrypted', + }, + workspaceEncrypted: {}, + personalDecrypted: { + MCP_HOST: 'api.example.com', + MCP_TOKEN: 'token', + UNUSED: '', + }, + workspaceDecrypted: {}, + conflicts: [], + decryptionFailures: ['UNUSED'], + }) + + const result = await resolveMcpConfigEnvVars(BASE_CONFIG, 'user-1', 'workspace-1') + + expect(result.resolvedSecretTraceProvenance.complete).toBe(true) + expect(result.resolvedSecretTraceProvenance.entries).toHaveLength(2) + }) + + it('reports successful substitutions before strict missing-reference failure', async () => { + const onProvenance = vi.fn() + const config = { + ...BASE_CONFIG, + headers: { + Authorization: 'Bearer {{MCP_TOKEN}}', + 'X-Missing': '{{NOT_CONFIGURED}}', + }, + } + + await expect( + resolveMcpConfigEnvVars(config, 'user-1', 'workspace-1', { + onResolvedSecretTraceProvenance: onProvenance, + }) + ).rejects.toThrow('NOT_CONFIGURED') + + expect(onProvenance).toHaveBeenCalledWith({ + version: 1, + complete: true, + entries: [ + { name: 'MCP_HOST', encryptedValue: 'personal-host-encrypted' }, + { name: 'MCP_TOKEN', encryptedValue: 'workspace-token-encrypted' }, + ], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + }) + }) + + it('reports incomplete provenance when the Secrets catalog cannot be loaded', async () => { + const onProvenance = vi.fn() + mockGetPersonalAndWorkspaceEnv.mockRejectedValueOnce(new Error('database unavailable')) + + const result = await resolveMcpConfigEnvVars(BASE_CONFIG, 'user-1', 'workspace-1', { + onResolvedSecretTraceProvenance: onProvenance, + }) + + const expected = { + version: 1 as const, + complete: false, + entries: [], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + } + expect(result.resolvedSecretTraceProvenance).toEqual(expected) + expect(onProvenance).toHaveBeenCalledWith(expected) + expect(result.config).toEqual(BASE_CONFIG) + }) +}) diff --git a/apps/sim/lib/mcp/resolve-config.ts b/apps/sim/lib/mcp/resolve-config.ts index 41322cc0282..d15f642350a 100644 --- a/apps/sim/lib/mcp/resolve-config.ts +++ b/apps/sim/lib/mcp/resolve-config.ts @@ -5,15 +5,21 @@ */ import { createLogger } from '@sim/logger' -import { getEffectiveDecryptedEnv } from '@/lib/environment/utils' +import { getPersonalAndWorkspaceEnv } from '@/lib/environment/utils' import type { McpServerConfig } from '@/lib/mcp/types' import { resolveEnvVarReferences } from '@/executor/utils/reference-validation' +import { + createResolvedSecretTraceRegistry, + type ResolvedSecretTraceProvenanceV1, + ResolvedSecretTraceRegistry, +} from '@/executor/utils/resolved-secret-trace-registry' const logger = createLogger('McpResolveConfig') export interface ResolveMcpConfigOptions { /** If true, throws an error when env vars are missing. Default: true */ strict?: boolean + onResolvedSecretTraceProvenance?: (provenance: ResolvedSecretTraceProvenanceV1) => void } /** @@ -31,22 +37,53 @@ export async function resolveMcpConfigEnvVars( userId: string, workspaceId?: string, options: ResolveMcpConfigOptions = {} -): Promise<{ config: McpServerConfig; missingVars: string[] }> { +): Promise<{ + config: McpServerConfig + missingVars: string[] + resolvedSecretTraceProvenance: ResolvedSecretTraceProvenanceV1 +}> { const { strict = true } = options const allMissingVars: string[] = [] + const scope = { userId, ...(workspaceId ? { workspaceId } : {}) } let envVars: Record = {} + let resolvedSecretTraceRegistry: ResolvedSecretTraceRegistry try { - envVars = await getEffectiveDecryptedEnv(userId, workspaceId) + const env = await getPersonalAndWorkspaceEnv(userId, workspaceId) + envVars = { ...env.personalDecrypted, ...env.workspaceDecrypted } + resolvedSecretTraceRegistry = await createResolvedSecretTraceRegistry({ + personalEncrypted: env.personalEncrypted, + workspaceEncrypted: env.workspaceEncrypted, + personalDecrypted: env.personalDecrypted, + workspaceDecrypted: env.workspaceDecrypted, + decryptionFailures: env.decryptionFailures, + scope, + }) } catch (error) { logger.error('Failed to fetch environment variables for MCP config:', error) - return { config, missingVars: [] } + resolvedSecretTraceRegistry = new ResolvedSecretTraceRegistry([], scope) + resolvedSecretTraceRegistry.markIncomplete() + const provenance = resolvedSecretTraceRegistry.exportProvenance() + options.onResolvedSecretTraceProvenance?.(provenance) + return { + config, + missingVars: [], + resolvedSecretTraceProvenance: provenance, + } } const resolveValue = (value: string): string => { const missingVars: string[] = [] const resolved = resolveEnvVarReferences(value, envVars, { missingKeys: missingVars, + onResolved: (name, resolvedValue) => { + if ( + resolvedValue.length > 0 && + !resolvedSecretTraceRegistry.recordResolved(name, resolvedValue) + ) { + resolvedSecretTraceRegistry.markIncomplete() + } + }, }) as string allMissingVars.push(...missingVars) return resolved @@ -66,7 +103,9 @@ export async function resolveMcpConfigEnvVars( resolvedConfig.headers = resolvedHeaders } - // Handle missing vars based on strict mode + const resolvedSecretTraceProvenance = resolvedSecretTraceRegistry.exportProvenance() + options.onResolvedSecretTraceProvenance?.(resolvedSecretTraceProvenance) + if (allMissingVars.length > 0) { const uniqueMissing = Array.from(new Set(allMissingVars)) @@ -81,5 +120,9 @@ export async function resolveMcpConfigEnvVars( }) } - return { config: resolvedConfig, missingVars: allMissingVars } + return { + config: resolvedConfig, + missingVars: allMissingVars, + resolvedSecretTraceProvenance, + } } diff --git a/apps/sim/lib/mcp/service-pool.test.ts b/apps/sim/lib/mcp/service-pool.test.ts index f499ba99111..2903558d561 100644 --- a/apps/sim/lib/mcp/service-pool.test.ts +++ b/apps/sim/lib/mcp/service-pool.test.ts @@ -15,6 +15,7 @@ import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { MockMcpClient, mockCallTool, + mockListTools, mockConnect, mockDisconnect, mockAcquire, @@ -24,12 +25,24 @@ const { poolClient, } = vi.hoisted(() => { const mockCallTool = vi.fn() + const mockListTools = vi.fn() const mockConnect = vi.fn() const mockDisconnect = vi.fn() const mockRelease = vi.fn(async () => {}) - const poolClient = { callTool: mockCallTool, disconnect: vi.fn() } + const poolClient = { + callTool: mockCallTool, + listTools: mockListTools, + disconnect: vi.fn(), + getResolvedSecretTraceProvenance: vi.fn(() => ({ + version: 1 as const, + complete: true, + entries: [{ name: 'MCP_TOKEN', encryptedValue: 'encrypted-token' }], + scope: { userId: 'user-1', workspaceId: 'ws-1' }, + })), + } return { mockCallTool, + mockListTools, mockConnect, mockDisconnect, mockRelease, @@ -50,9 +63,10 @@ const { connect: mockConnect, disconnect: mockDisconnect, callTool: mockCallTool, - listTools: vi.fn(async () => []), + listTools: mockListTools, hasListChangedCapability: vi.fn(() => false), onClose: vi.fn(), + getResolvedSecretTraceProvenance: poolClient.getResolvedSecretTraceProvenance, }) } } @@ -117,6 +131,7 @@ describe('McpService connection reuse wiring', () => { mockResolveEnvVars.mockImplementation(async (config: unknown) => ({ config })) mockAcquire.mockResolvedValue({ client: poolClient, release: mockRelease }) mockCallTool.mockResolvedValue({ content: [] }) + mockListTools.mockResolvedValue([]) }) afterAll(() => { @@ -137,6 +152,161 @@ describe('McpService connection reuse wiring', () => { expect(mockResolveEnvVars).not.toHaveBeenCalled() }) + it('emits the pooled connection provenance on every invocation', async () => { + const onProvenance = vi.fn() + + await mcpService.executeTool( + USER_ID, + 'server-1', + { name: 'first', arguments: {} }, + WORKSPACE_ID, + undefined, + onProvenance + ) + await mcpService.executeTool( + USER_ID, + 'server-1', + { name: 'second', arguments: {} }, + WORKSPACE_ID, + undefined, + onProvenance + ) + + expect(onProvenance).toHaveBeenCalledTimes(2) + expect(onProvenance).toHaveBeenNthCalledWith(1, { + version: 1, + complete: true, + entries: [{ name: 'MCP_TOKEN', encryptedValue: 'encrypted-token' }], + scope: { userId: USER_ID, workspaceId: WORKSPACE_ID }, + }) + expect(onProvenance).toHaveBeenNthCalledWith(2, { + version: 1, + complete: true, + entries: [{ name: 'MCP_TOKEN', encryptedValue: 'encrypted-token' }], + scope: { userId: USER_ID, workspaceId: WORKSPACE_ID }, + }) + }) + + it('emits cold-connection provenance once even though the connected client retains it', async () => { + const onProvenance = vi.fn() + const provenance = { + version: 1 as const, + complete: true, + entries: [{ name: 'MCP_TOKEN', encryptedValue: 'encrypted-token' }], + scope: { userId: USER_ID, workspaceId: WORKSPACE_ID }, + } + mockResolveEnvVars.mockImplementationOnce( + async ( + config: unknown, + _userId: unknown, + _workspaceId: unknown, + options: { + onResolvedSecretTraceProvenance?: (value: typeof provenance) => void + } + ) => { + options.onResolvedSecretTraceProvenance?.(provenance) + return { config, resolvedSecretTraceProvenance: provenance } + } + ) + mockAcquire.mockImplementationOnce( + async ({ create }: { create: () => Promise }) => ({ + client: await create(), + release: mockRelease, + }) + ) + + await mcpService.executeTool( + USER_ID, + 'server-1', + { name: 'do', arguments: {} }, + WORKSPACE_ID, + undefined, + onProvenance + ) + + expect(mockResolveEnvVars).toHaveBeenCalledTimes(1) + expect(onProvenance).toHaveBeenCalledTimes(1) + expect(onProvenance).toHaveBeenCalledWith(provenance) + }) + + it('emits cold-connection provenance for tools/list', async () => { + const onProvenance = vi.fn() + const provenance = { + version: 1 as const, + complete: true, + entries: [{ name: 'MCP_TOKEN', encryptedValue: 'encrypted-token' }], + scope: { userId: USER_ID, workspaceId: WORKSPACE_ID }, + } + mockResolveEnvVars.mockImplementationOnce( + async ( + config: unknown, + _userId: unknown, + _workspaceId: unknown, + options: { + onResolvedSecretTraceProvenance?: (value: typeof provenance) => void + } + ) => { + options.onResolvedSecretTraceProvenance?.(provenance) + return { config, resolvedSecretTraceProvenance: provenance } + } + ) + mockAcquire.mockImplementationOnce( + async ({ create }: { create: () => Promise }) => ({ + client: await create(), + release: mockRelease, + }) + ) + + await mcpService.discoverServerTools(USER_ID, 'server-1', WORKSPACE_ID, true, onProvenance) + + expect(mockResolveEnvVars).toHaveBeenCalledTimes(1) + expect(mockListTools).toHaveBeenCalledTimes(1) + expect(onProvenance).toHaveBeenCalledTimes(1) + expect(onProvenance).toHaveBeenCalledWith(provenance) + }) + + it('emits retained provenance on every warm-pool tools/list invocation', async () => { + const onProvenance = vi.fn() + + await mcpService.discoverServerTools(USER_ID, 'server-1', WORKSPACE_ID, true, onProvenance) + await mcpService.discoverServerTools(USER_ID, 'server-1', WORKSPACE_ID, true, onProvenance) + + expect(mockResolveEnvVars).not.toHaveBeenCalled() + expect(mockListTools).toHaveBeenCalledTimes(2) + expect(onProvenance).toHaveBeenCalledTimes(2) + expect(onProvenance).toHaveBeenNthCalledWith(1, { + version: 1, + complete: true, + entries: [{ name: 'MCP_TOKEN', encryptedValue: 'encrypted-token' }], + scope: { userId: USER_ID, workspaceId: WORKSPACE_ID }, + }) + expect(onProvenance).toHaveBeenNthCalledWith(2, { + version: 1, + complete: true, + entries: [{ name: 'MCP_TOKEN', encryptedValue: 'encrypted-token' }], + scope: { userId: USER_ID, workspaceId: WORKSPACE_ID }, + }) + }) + + it('reports incomplete provenance when a pooled tools/list client has no retained report', async () => { + const onProvenance = vi.fn() + const legacyClient = { + ...poolClient, + getResolvedSecretTraceProvenance: undefined, + } + mockAcquire.mockResolvedValueOnce({ client: legacyClient, release: mockRelease }) + + await mcpService.discoverServerTools(USER_ID, 'server-1', WORKSPACE_ID, true, onProvenance) + + expect(mockListTools).toHaveBeenCalledTimes(1) + expect(onProvenance).toHaveBeenCalledWith({ + version: 1, + complete: false, + entries: [], + scope: { userId: USER_ID, workspaceId: WORKSPACE_ID }, + }) + }) + it('bypasses the pool for calls carrying per-request headers', async () => { await mcpService.executeTool(USER_ID, 'server-1', { name: 'do', arguments: {} }, WORKSPACE_ID, { Authorization: 'Bearer per-call', @@ -226,4 +396,48 @@ describe('McpService connection reuse wiring', () => { expect(mockRelease).toHaveBeenCalledWith(true, false) expect(mockAcquire).toHaveBeenCalledTimes(2) }) + + it('reports both retained and rotated provenance when an auth retry rebuilds the client', async () => { + const staleProvenance = { + version: 1 as const, + complete: true, + entries: [{ name: 'MCP_TOKEN', encryptedValue: 'encrypted-token-v1' }], + scope: { userId: USER_ID, workspaceId: WORKSPACE_ID }, + } + const rotatedProvenance = { + version: 1 as const, + complete: true, + entries: [{ name: 'MCP_TOKEN', encryptedValue: 'encrypted-token-v2' }], + scope: { userId: USER_ID, workspaceId: WORKSPACE_ID }, + } + const staleClient = { + ...poolClient, + getResolvedSecretTraceProvenance: vi.fn(() => staleProvenance), + } + const rotatedClient = { + ...poolClient, + getResolvedSecretTraceProvenance: vi.fn(() => rotatedProvenance), + } + mockAcquire + .mockResolvedValueOnce({ client: staleClient, release: mockRelease }) + .mockResolvedValueOnce({ client: rotatedClient, release: mockRelease }) + mockCallTool + .mockRejectedValueOnce(new UnauthorizedError('stale key')) + .mockResolvedValueOnce({ content: [] }) + const onProvenance = vi.fn() + + await mcpService.executeTool( + USER_ID, + 'server-1', + { name: 'do', arguments: {} }, + WORKSPACE_ID, + undefined, + onProvenance + ) + + expect(onProvenance.mock.calls.map(([provenance]) => provenance)).toEqual([ + staleProvenance, + rotatedProvenance, + ]) + }) }) diff --git a/apps/sim/lib/mcp/service.ts b/apps/sim/lib/mcp/service.ts index d4ef715b8e2..4a867b35382 100644 --- a/apps/sim/lib/mcp/service.ts +++ b/apps/sim/lib/mcp/service.ts @@ -42,6 +42,10 @@ import { type McpTransport, } from '@/lib/mcp/types' import { MCP_CLIENT_CONSTANTS, MCP_CONSTANTS } from '@/lib/mcp/utils' +import { + isResolvedSecretTraceProvenanceV1, + type ResolvedSecretTraceProvenanceV1, +} from '@/executor/utils/resolved-secret-trace-registry' const logger = createLogger('McpService') @@ -55,6 +59,59 @@ function failureCacheKey(workspaceId: string, serverId: string): string { const FAILURE_CACHE_SENTINEL: McpTool[] = [] +type ResolvedSecretTraceProvenanceCallback = (provenance: ResolvedSecretTraceProvenanceV1) => void + +function reportRetainedClientProvenance( + provenance: unknown, + userId: string, + workspaceId: string, + callback?: ResolvedSecretTraceProvenanceCallback +): void { + if (!callback) return + callback( + isResolvedSecretTraceProvenanceV1(provenance) + ? provenance + : { + version: 1, + complete: false, + entries: [], + scope: { userId, workspaceId }, + } + ) +} + +function isSameProvenance( + left: ResolvedSecretTraceProvenanceV1, + right: ResolvedSecretTraceProvenanceV1 +): boolean { + if ( + left.complete !== right.complete || + left.scope?.userId !== right.scope?.userId || + left.scope?.workspaceId !== right.scope?.workspaceId || + left.entries.length !== right.entries.length + ) { + return false + } + + return left.entries.every((entry, index) => { + const other = right.entries[index] + return entry.name === other.name && entry.encryptedValue === other.encryptedValue + }) +} + +function createInvocationProvenanceReporter( + callback?: ResolvedSecretTraceProvenanceCallback +): ResolvedSecretTraceProvenanceCallback | undefined { + if (!callback) return undefined + + let lastReported: ResolvedSecretTraceProvenanceV1 | undefined + return (provenance) => { + if (lastReported && isSameProvenance(lastReported, provenance)) return + lastReported = provenance + callback(provenance) + } +} + type DiscoveryOutcome = | { kind: 'cached'; tools: McpTool[] } | { kind: 'fetched'; tools: McpTool[] } @@ -212,14 +269,25 @@ class McpService { private async resolveConfigEnvVars( config: McpServerConfig, userId: string, - workspaceId?: string - ): Promise<{ config: McpServerConfig; resolvedIP: string | null }> { - const { config: resolvedConfig } = await resolveMcpConfigEnvVars(config, userId, workspaceId, { - strict: true, - }) + workspaceId?: string, + onResolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceCallback + ): Promise<{ + config: McpServerConfig + resolvedIP: string | null + resolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceV1 + }> { + const { config: resolvedConfig, resolvedSecretTraceProvenance } = await resolveMcpConfigEnvVars( + config, + userId, + workspaceId, + { + strict: true, + onResolvedSecretTraceProvenance, + } + ) validateMcpDomain(resolvedConfig.url) const resolvedIP = await validateMcpServerSsrf(resolvedConfig.url) - return { config: resolvedConfig, resolvedIP } + return { config: resolvedConfig, resolvedIP, resolvedSecretTraceProvenance } } private async getServerConfig( @@ -298,7 +366,8 @@ class McpService { private async createClient( config: McpServerConfig, resolvedIP: string | null, - userId?: string + userId?: string, + resolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceV1 ): Promise { const securityPolicy = { requireConsent: true, @@ -312,6 +381,7 @@ class McpService { config, securityPolicy, resolvedIP: resolvedIP ?? undefined, + resolvedSecretTraceProvenance, }) await client.connect() return client @@ -343,6 +413,7 @@ class McpService { securityPolicy, authProvider, resolvedIP: resolvedIP ?? undefined, + resolvedSecretTraceProvenance, }) await client.connect() return client @@ -367,18 +438,24 @@ class McpService { config: McpServerConfig, userId: string, workspaceId: string, - extraHeaders?: Record + extraHeaders?: Record, + onResolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceCallback ): () => Promise { return async () => { - const { config: resolvedConfig, resolvedIP } = await this.resolveConfigEnvVars( + const { + config: resolvedConfig, + resolvedIP, + resolvedSecretTraceProvenance, + } = await this.resolveConfigEnvVars( config, userId, - workspaceId + workspaceId, + onResolvedSecretTraceProvenance ) if (extraHeaders) { resolvedConfig.headers = { ...resolvedConfig.headers, ...extraHeaders } } - return this.createClient(resolvedConfig, resolvedIP, userId) + return this.createClient(resolvedConfig, resolvedIP, userId, resolvedSecretTraceProvenance) } } @@ -392,7 +469,8 @@ class McpService { private async fetchServerTools( config: McpServerConfig, userId: string, - workspaceId: string + workspaceId: string, + onResolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceCallback ): Promise { for (let attempt = 0; ; attempt++) { try { @@ -402,8 +480,16 @@ class McpService { serverId: config.id, allowPool: true, }, - this.buildClient(config, userId, workspaceId), - (client) => client.listTools() + this.buildClient(config, userId, workspaceId, undefined, onResolvedSecretTraceProvenance), + (client) => { + reportRetainedClientProvenance( + client.getResolvedSecretTraceProvenance?.(), + userId, + workspaceId, + onResolvedSecretTraceProvenance + ) + return client.listTools() + } ) } catch (error) { if (attempt === 0 && isAuthError(error) && config.authType !== 'oauth') continue @@ -462,10 +548,12 @@ class McpService { serverId: string, toolCall: McpToolCall, workspaceId: string, - extraHeaders?: Record + extraHeaders?: Record, + onResolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceCallback ): Promise { const requestId = generateRequestId() const maxRetries = 2 + const reportProvenance = createInvocationProvenanceReporter(onResolvedSecretTraceProvenance) for (let attempt = 0; attempt < maxRetries; attempt++) { try { @@ -485,8 +573,22 @@ class McpService { serverId, allowPool: !hasExtraHeaders, }, - this.buildClient(config, userId, workspaceId, hasExtraHeaders ? extraHeaders : undefined), - (client) => client.callTool(toolCall) + this.buildClient( + config, + userId, + workspaceId, + hasExtraHeaders ? extraHeaders : undefined, + reportProvenance + ), + (client) => { + reportRetainedClientProvenance( + client.getResolvedSecretTraceProvenance?.(), + userId, + workspaceId, + reportProvenance + ) + return client.callTool(toolCall) + } ) logger.info(`[${requestId}] Successfully executed tool ${toolCall.name}`) return result @@ -874,8 +976,19 @@ class McpService { userId: string, serverId: string, workspaceId: string, - forceRefresh = false + forceRefresh = false, + onResolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceCallback ): Promise { + if (onResolvedSecretTraceProvenance) { + return this.discoverServerToolsImpl( + userId, + serverId, + workspaceId, + forceRefresh, + createInvocationProvenanceReporter(onResolvedSecretTraceProvenance) + ) + } + const inflightKey = `${workspaceId}:${serverId}:${userId}:${forceRefresh ? 'force' : 'cache'}` const existing = this.inflightServerDiscovery.get(inflightKey) if (existing) return existing @@ -884,7 +997,8 @@ class McpService { userId, serverId, workspaceId, - forceRefresh + forceRefresh, + undefined ).finally(() => { this.inflightServerDiscovery.delete(inflightKey) }) @@ -896,7 +1010,8 @@ class McpService { userId: string, serverId: string, workspaceId: string, - forceRefresh: boolean + forceRefresh: boolean, + onResolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceCallback ): Promise { const requestId = generateRequestId() const discoveryStartedAt = new Date() @@ -934,7 +1049,12 @@ class McpService { } authType = config.authType - const tools = await this.fetchServerTools(config, userId, workspaceId) + const tools = await this.fetchServerTools( + config, + userId, + workspaceId, + onResolvedSecretTraceProvenance + ) logger.info(`[${requestId}] Discovered ${tools.length} tools from server ${config.name}`) await Promise.allSettled([ this.cacheAdapter diff --git a/apps/sim/lib/mcp/types.ts b/apps/sim/lib/mcp/types.ts index 575e86ea2f8..a6f7d1f9363 100644 --- a/apps/sim/lib/mcp/types.ts +++ b/apps/sim/lib/mcp/types.ts @@ -1,4 +1,5 @@ import type { Tool } from '@modelcontextprotocol/sdk/types.js' +import type { ResolvedSecretTraceProvenanceV1 } from '@/executor/utils/resolved-secret-trace-registry' export type McpTransport = 'streamable-http' @@ -188,6 +189,8 @@ export interface McpClientOptions { * server configs. */ authProvider?: import('@modelcontextprotocol/sdk/client/auth.js').OAuthClientProvider + /** Encrypted-only provenance for Secrets-tab references resolved into this connection. */ + resolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceV1 } export interface ToolsChangedEvent { diff --git a/apps/sim/lib/table/backfill-runner.ts b/apps/sim/lib/table/backfill-runner.ts index d3e9946cfee..ce9c8bc717a 100644 --- a/apps/sim/lib/table/backfill-runner.ts +++ b/apps/sim/lib/table/backfill-runner.ts @@ -7,6 +7,10 @@ import { and, asc, count, eq, gt, inArray } from 'drizzle-orm' import { isTriggerDevEnabled } from '@/lib/core/config/env-flags' import { runDetached } from '@/lib/core/utils/background' import { MATERIALIZE_CONCURRENCY, mapWithConcurrency } from '@/lib/core/utils/concurrency' +import { + type FunctionalExecutionDataSource, + getFunctionalBlockOutput, +} from '@/lib/logs/execution/functional-outputs' import { materializeExecutionData } from '@/lib/logs/execution/trace-store' import { appendTableEvent } from '@/lib/table/events' import { @@ -47,27 +51,6 @@ export interface TableBackfillPayload { actorUserId?: string | null } -/** Minimal shape of a trace span we care about for backfill. */ -interface BackfillTraceSpan { - blockId?: string - output?: Record - children?: BackfillTraceSpan[] -} - -/** DFS the trace tree for the first span matching `blockId`. */ -function findSpanByBlockId( - spans: BackfillTraceSpan[] | undefined, - blockId: string -): BackfillTraceSpan | undefined { - if (!spans) return undefined - for (const span of spans) { - if (span.blockId === blockId) return span - const child = findSpanByBlockId(span.children, blockId) - if (child) return child - } - return undefined -} - /** One keyset page of completed (rowId, executionId) pairs for the group, ordered by rowId. */ async function selectCompletedExecPage( tableId: string, @@ -94,8 +77,8 @@ async function selectCompletedExecPage( } /** - * Backfills one page of rows: pulls each target output's value out of the rows' saved trace - * spans (materialized from object storage with bounded concurrency) and writes it into row data. + * Backfills one page of rows: pulls each target output from the saved raw execution state + * (materialized from object storage with bounded concurrency) and writes it into row data. * Returns the number of rows updated. */ async function processBackfillPage(opts: { @@ -136,17 +119,14 @@ async function processBackfillPage(opts: { .from(workflowExecutionLogs) .where(inArray(workflowExecutionLogs.executionId, executionIds)) - const logByExecutionId = new Map() + const logByExecutionId = new Map() // Heavy execution data may live in object storage; resolve pointers (bounded concurrency). await mapWithConcurrency(logs, MATERIALIZE_CONCURRENCY, async (log) => { const executionData = await materializeExecutionData( log.executionData as Record | null, { workspaceId: log.workspaceId, workflowId: log.workflowId, executionId: log.executionId } ) - logByExecutionId.set( - log.executionId, - (executionData as { traceSpans?: BackfillTraceSpan[] }) ?? {} - ) + logByExecutionId.set(log.executionId, (executionData as FunctionalExecutionDataSource) ?? {}) }) const updates: Array<{ rowId: string; data: RowData }> = [] @@ -160,9 +140,9 @@ async function processBackfillPage(opts: { let mutated = false for (const out of outputs) { if (!overwrite && (r.data as RowData)[out.columnName] !== undefined) continue - const span = findSpanByBlockId(log.traceSpans, out.blockId) - if (!span?.output) continue - const picked = pluckByPath(span.output, out.path) + const functionalOutput = getFunctionalBlockOutput(log, out.blockId) + if (functionalOutput === undefined) continue + const picked = pluckByPath(functionalOutput, out.path) if (picked === undefined) continue dataPatch[out.columnName] = picked as RowData[string] mutated = true diff --git a/apps/sim/lib/tokenization/streaming.test.ts b/apps/sim/lib/tokenization/streaming.test.ts deleted file mode 100644 index bceb9e1923d..00000000000 --- a/apps/sim/lib/tokenization/streaming.test.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { processStreamingBlockLog } from '@/lib/tokenization/streaming' - -describe('processStreamingBlockLog', () => { - it('does not estimate usage from sanitized environment references', () => { - const log = { - blockId: 'agent-1', - blockType: 'agent', - input: { - prompt: 'Use {{OPENAI_API_KEY}} to answer', - model: 'gpt-4o', - }, - output: { - content: 'streamed answer', - model: 'gpt-4o', - }, - } - - expect(processStreamingBlockLog(log, 'streamed answer')).toBe(false) - expect(log.output).toEqual({ - content: 'streamed answer', - model: 'gpt-4o', - }) - }) -}) diff --git a/apps/sim/lib/tokenization/streaming.ts b/apps/sim/lib/tokenization/streaming.ts index 03448f0120b..ca552fa8292 100644 --- a/apps/sim/lib/tokenization/streaming.ts +++ b/apps/sim/lib/tokenization/streaming.ts @@ -16,17 +16,11 @@ import { import type { BlockLog } from '@/executor/types' const logger = createLogger('StreamingTokenization') -const ENVIRONMENT_REFERENCE_PATTERN = /\{\{[^}]+\}\}/ - -type StreamingTokenizationLog = Pick /** * Processes a block log and adds tokenization data if needed */ -export function processStreamingBlockLog( - log: StreamingTokenizationLog, - streamedContent: string -): boolean { +export function processStreamingBlockLog(log: BlockLog, streamedContent: string): boolean { // Check if this block should be tokenized if (!isTokenizableBlockType(log.blockType)) { return false @@ -53,12 +47,6 @@ export function processStreamingBlockLog( // Prepare input text from log const inputText = extractTextContent(log.input) - // Environment values are restored to references before logs leave the - // executor. Never use those shorter placeholders as a billing estimate: - // the executor performs this fallback once with the raw resolved input. - if (ENVIRONMENT_REFERENCE_PATTERN.test(inputText)) { - return false - } // Calculate streaming cost const systemPrompt = @@ -113,7 +101,7 @@ export function processStreamingBlockLog( /** * Determines the appropriate model for a block */ -function getModelForBlock(log: StreamingTokenizationLog): string { +function getModelForBlock(log: BlockLog): string { // Try to get model from output first if (log.output?.model?.trim()) { return log.output.model diff --git a/apps/sim/lib/webhooks/env-resolver.test.ts b/apps/sim/lib/webhooks/env-resolver.test.ts index fcbdb139daa..0f802ed5947 100644 --- a/apps/sim/lib/webhooks/env-resolver.test.ts +++ b/apps/sim/lib/webhooks/env-resolver.test.ts @@ -68,4 +68,38 @@ describe('webhook env resolver', () => { expect(result).not.toBe(webhookRecord) expect(result.providerConfig).not.toBe(webhookRecord.providerConfig) }) + + it('reports only successful substitutions when resolving with a prepared environment', async () => { + const onResolved = vi.fn() + + const result = await resolveWebhookProviderConfig( + { + exact: '{{SLACK_BOT_TOKEN}}', + embedded: 'https://{{ SLACK_HOST }}/files', + missing: '{{MISSING_SECRET}}', + directLiteral: 'xoxb-resolved', + }, + 'user-1', + 'workspace-1', + { + envVars: { + SLACK_BOT_TOKEN: 'xoxb-resolved', + SLACK_HOST: 'files.slack.com', + }, + onResolved, + } + ) + + expect(result).toEqual({ + exact: 'xoxb-resolved', + embedded: 'https://files.slack.com/files', + missing: '{{MISSING_SECRET}}', + directLiteral: 'xoxb-resolved', + }) + expect(onResolved.mock.calls).toEqual([ + ['SLACK_BOT_TOKEN', 'xoxb-resolved'], + ['SLACK_HOST', 'files.slack.com'], + ]) + expect(mockGetEffectiveDecryptedEnv).not.toHaveBeenCalled() + }) }) diff --git a/apps/sim/lib/webhooks/env-resolver.ts b/apps/sim/lib/webhooks/env-resolver.ts index 1537879e7d0..1a2fb413fbd 100644 --- a/apps/sim/lib/webhooks/env-resolver.ts +++ b/apps/sim/lib/webhooks/env-resolver.ts @@ -1,6 +1,11 @@ import { getEffectiveDecryptedEnv } from '@/lib/environment/utils' import { resolveEnvVarReferences } from '@/executor/utils/reference-validation' +export interface WebhookEnvResolutionOptions { + envVars?: Record + onResolved?: (name: string, value: string) => void +} + /** * Recursively resolves all environment variable references in a configuration object. * Supports both exact matches (`{{VAR_NAME}}`) and embedded patterns (`https://{{HOST}}/path`). @@ -15,10 +20,14 @@ import { resolveEnvVarReferences } from '@/executor/utils/reference-validation' export async function resolveEnvVarsInObject>( config: T, userId: string, - workspaceId?: string + workspaceId?: string, + options: WebhookEnvResolutionOptions = {} ): Promise { - const envVars = await getEffectiveDecryptedEnv(userId, workspaceId) - return resolveEnvVarReferences(config, envVars, { deep: true }) as T + const envVars = options.envVars ?? (await getEffectiveDecryptedEnv(userId, workspaceId)) + return resolveEnvVarReferences(config, envVars, { + deep: true, + onResolved: options.onResolved, + }) as T } /** @@ -38,9 +47,15 @@ export function normalizeWebhookProviderConfig(providerConfig: unknown): Record< export async function resolveWebhookProviderConfig( providerConfig: unknown, userId: string, - workspaceId?: string + workspaceId?: string, + options: WebhookEnvResolutionOptions = {} ): Promise> { - return resolveEnvVarsInObject(normalizeWebhookProviderConfig(providerConfig), userId, workspaceId) + return resolveEnvVarsInObject( + normalizeWebhookProviderConfig(providerConfig), + userId, + workspaceId, + options + ) } /** @@ -49,14 +64,16 @@ export async function resolveWebhookProviderConfig( export async function resolveWebhookRecordProviderConfig( webhookRecord: T, userId: string, - workspaceId?: string + workspaceId?: string, + options: WebhookEnvResolutionOptions = {} ): Promise }> { return { ...webhookRecord, providerConfig: await resolveWebhookProviderConfig( webhookRecord.providerConfig, userId, - workspaceId + workspaceId, + options ), } } diff --git a/apps/sim/lib/workflows/executor/execute-workflow.test.ts b/apps/sim/lib/workflows/executor/execute-workflow.test.ts index 8a5c99d8000..24cb260f3fd 100644 --- a/apps/sim/lib/workflows/executor/execute-workflow.test.ts +++ b/apps/sim/lib/workflows/executor/execute-workflow.test.ts @@ -167,4 +167,23 @@ describe('executeWorkflow billing attribution', () => { 'request-1' ) }) + + it('forwards trusted initial trace-secret provenance to the execution core', async () => { + const provenance = { + version: 1 as const, + complete: true, + entries: [{ name: 'API_KEY', encryptedValue: 'encrypted-secret' }], + scope: { userId: 'actor-1', workspaceId: 'workspace-1' }, + } + + await executeWorkflow(workflow, 'request-1', { prompt: 'hello' }, 'actor-1', { + enabled: true, + billingAttribution, + trustedInitialResolvedSecretTraceProvenance: provenance, + }) + + expect(executeWorkflowCoreMock).toHaveBeenCalledWith( + expect.objectContaining({ trustedInitialResolvedSecretTraceProvenance: provenance }) + ) + }) }) diff --git a/apps/sim/lib/workflows/executor/execute-workflow.ts b/apps/sim/lib/workflows/executor/execute-workflow.ts index 639ef051945..98bdadb0546 100644 --- a/apps/sim/lib/workflows/executor/execute-workflow.ts +++ b/apps/sim/lib/workflows/executor/execute-workflow.ts @@ -12,6 +12,7 @@ import { handlePostExecutionPauseState } from '@/lib/workflows/executor/pause-pe import { ExecutionSnapshot } from '@/executor/execution/snapshot' import type { ExecutionMetadata, SerializableExecutionState } from '@/executor/execution/types' import type { ExecutionResult, StreamingExecution } from '@/executor/types' +import type { ResolvedSecretTraceProvenanceV1 } from '@/executor/utils/resolved-secret-trace-registry' const logger = createLogger('WorkflowExecution') @@ -51,6 +52,8 @@ export interface ExecuteWorkflowOptions { sourceSnapshot: SerializableExecutionState sourceExecutionId?: string } + /** Trusted encrypted provenance supplied by a server-only caller before execution starts. */ + trustedInitialResolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceV1 executionMode?: 'sync' | 'stream' | 'async' /** Immutable actor/payer decision captured by preprocessing. */ billingAttribution?: BillingAttributionSnapshot @@ -163,6 +166,8 @@ export async function executeWorkflow( base64MaxBytes: streamConfig?.base64MaxBytes, abortSignal: streamConfig?.abortSignal, stopAfterBlockId: streamConfig?.stopAfterBlockId, + trustedInitialResolvedSecretTraceProvenance: + streamConfig?.trustedInitialResolvedSecretTraceProvenance, runFromBlock: streamConfig?.runFromBlock, }) diff --git a/apps/sim/lib/workflows/executor/execution-core.test.ts b/apps/sim/lib/workflows/executor/execution-core.test.ts index af94cbbfc6e..94a0ae9c789 100644 --- a/apps/sim/lib/workflows/executor/execution-core.test.ts +++ b/apps/sim/lib/workflows/executor/execution-core.test.ts @@ -23,7 +23,10 @@ const { onBlockStartPersistenceMock, executorConstructorMock, findStartBlockMock, - setEnvironmentSecretSanitizerMock, + setResolvedSecretTraceRegistryMock, + setTraceLargeValueAccessMock, + projectDisplayContentMock, + decryptSecretMock, } = vi.hoisted(() => ({ mergeSubblockStateWithValuesMock: vi.fn(), safeStartMock: vi.fn(), @@ -39,7 +42,10 @@ const { onBlockStartPersistenceMock: vi.fn(), executorConstructorMock: vi.fn(), findStartBlockMock: vi.fn(), - setEnvironmentSecretSanitizerMock: vi.fn(), + setResolvedSecretTraceRegistryMock: vi.fn(), + setTraceLargeValueAccessMock: vi.fn(), + projectDisplayContentMock: vi.fn(), + decryptSecretMock: vi.fn(), })) const getPersonalAndWorkspaceEnvMock = environmentUtilsMockFns.mockGetPersonalAndWorkspaceEnv @@ -55,6 +61,10 @@ vi.mock('@/lib/execution/cancellation', () => ({ clearExecutionCancellation: clearExecutionCancellationMock, })) +vi.mock('@/lib/core/security/encryption', () => ({ + decryptSecret: decryptSecretMock, +})) + vi.mock('@/lib/logs/execution/trace-spans/trace-spans', () => ({ buildTraceSpans: buildTraceSpansMock, })) @@ -112,7 +122,9 @@ describe('executeWorkflowCore terminal finalization sequencing', () => { hasCompleted: hasCompletedMock, onBlockStart: onBlockStartPersistenceMock, onBlockComplete: vi.fn(), - setEnvironmentSecretSanitizer: setEnvironmentSecretSanitizerMock, + projectDisplayContent: projectDisplayContentMock, + setResolvedSecretTraceRegistry: setResolvedSecretTraceRegistryMock, + setTraceLargeValueAccess: setTraceLargeValueAccessMock, setPostExecutionPromise: vi.fn(), waitForPostExecution: vi.fn().mockResolvedValue(undefined), } @@ -191,8 +203,17 @@ describe('executeWorkflowCore terminal finalization sequencing', () => { safeCompleteWithPauseMock.mockResolvedValue(undefined) hasCompletedMock.mockReturnValue(true) onBlockStartPersistenceMock.mockResolvedValue(undefined) + projectDisplayContentMock.mockImplementation(async (content) => content) updateWorkflowRunCountsMock.mockResolvedValue(undefined) clearExecutionCancellationMock.mockResolvedValue(undefined) + decryptSecretMock.mockImplementation(async (encryptedValue: string) => ({ + decrypted: + encryptedValue === 'webhook-ciphertext' + ? 'webhook-secret-value' + : encryptedValue === 'old-secret-ciphertext' + ? 'old-secret-value' + : encryptedValue, + })) }) it('loads workflow state and env vars concurrently, then starts logging before constructing the executor', async () => { @@ -503,7 +524,7 @@ describe('executeWorkflowCore terminal finalization sequencing', () => { ]) }) - it('registers a workflow-scoped log sanitizer while preserving raw runtime output', async () => { + it('registers an inert resolution-scoped registry while preserving raw runtime output', async () => { const secret = 'sk-demo-core-7f3a91' const runtimeOutput = { keyWasResolved: true, @@ -549,13 +570,12 @@ describe('executeWorkflowCore terminal finalization sequencing', () => { }) await loggingSession.setPostExecutionPromise.mock.calls[0][0] - const sanitizer = setEnvironmentSecretSanitizerMock.mock.calls[0]?.[0] - expect(sanitizer).toBeTypeOf('function') - expect(sanitizer(runtimeOutput)).toEqual({ - keyWasResolved: true, - echoedKey: '{{OPENAI_API_KEY}}', - ordinary: 'us-east-1', - }) + const registry = setResolvedSecretTraceRegistryMock.mock.calls[0]?.[0] + expect(registry.getActiveMatches()).toEqual([]) + expect(registry.recordResolved('OPENAI_API_KEY', secret)).toBe(true) + expect(registry.getActiveMatches()).toEqual([ + { plaintext: secret, replacement: '{{OPENAI_API_KEY}}' }, + ]) expect(result.output).toEqual(runtimeOutput) expect(result.output.echoedKey).toBe(secret) expect(safeCompleteMock).toHaveBeenCalledWith( @@ -563,61 +583,78 @@ describe('executeWorkflowCore terminal finalization sequencing', () => { ) }) - it('rebuilds the log sanitizer with current environment values for resumed runs', async () => { - const firstSecret = 'sk-demo-before-resume' - const resumedSecret = 'sk-demo-after-resume' - const workflowWithSecret = { - blocks: [ - { - id: 'function-1', - config: { - tool: 'function', - params: { code: 'return "{{OPENAI_API_KEY}}"' }, - }, - }, - ], - connections: [], - loops: {}, - parallels: {}, - } - - serializeWorkflowMock.mockReturnValue(workflowWithSecret) - getPersonalAndWorkspaceEnvMock - .mockResolvedValueOnce({ - personalEncrypted: {}, - workspaceEncrypted: { OPENAI_API_KEY: 'encrypted-before' }, - personalDecrypted: {}, - workspaceDecrypted: { OPENAI_API_KEY: firstSecret }, - }) - .mockResolvedValueOnce({ - personalEncrypted: {}, - workspaceEncrypted: { OPENAI_API_KEY: 'encrypted-after' }, - personalDecrypted: {}, - workspaceDecrypted: { OPENAI_API_KEY: resumedSecret }, - }) - executorExecuteMock - .mockResolvedValueOnce({ - success: true, - status: 'completed', - output: { echoedKey: firstSecret }, - logs: [], - metadata: { duration: 1, startTime: 'start', endTime: 'end' }, - }) - .mockResolvedValueOnce({ - success: true, - status: 'completed', - output: { echoedKey: resumedSecret }, - logs: [], - metadata: { duration: 1, startTime: 'start', endTime: 'end' }, - }) + it('activates trusted pre-execution provenance on the installed execution registry', async () => { + executorExecuteMock.mockResolvedValue({ + success: true, + status: 'completed', + output: {}, + logs: [], + metadata: { duration: 1, startTime: 'start', endTime: 'end' }, + }) await executeWorkflowCore({ snapshot: createSnapshot() as any, callbacks: {}, loggingSession: loggingSession as any, + trustedInitialResolvedSecretTraceProvenance: { + version: 1, + complete: true, + entries: [{ name: 'WEBHOOK_SECRET', encryptedValue: 'webhook-ciphertext' }], + scope: { userId: 'workflow-owner', workspaceId: 'workspace-1' }, + }, + }) + await loggingSession.setPostExecutionPromise.mock.calls[0][0] + + const registry = setResolvedSecretTraceRegistryMock.mock.calls[0]?.[0] + expect(registry.getActiveMatches()).toEqual([ + { plaintext: 'webhook-secret-value', replacement: '{{WEBHOOK_SECRET}}' }, + ]) + }) + + it('marks a trusted legacy resume with inherited state incomplete', async () => { + executorExecuteMock.mockResolvedValue({ + success: true, + status: 'completed', + output: { done: true }, + logs: [], + metadata: { duration: 1, startTime: 'start', endTime: 'end' }, + }) + const resumedSnapshot = createSnapshot() + resumedSnapshot.metadata = { + ...resumedSnapshot.metadata, + executionId: 'execution-resumed', + resumeFromSnapshot: true, + resumeTerminalNoop: true, + } as any + ;(resumedSnapshot as any).state = { + blockStates: { previous: { output: { value: 'cached' } } }, + executedBlocks: ['previous'], + blockLogs: [], + decisions: { router: {}, condition: {} }, + completedLoops: [], + activeExecutionPath: [], + } + + await executeWorkflowCore({ + snapshot: resumedSnapshot as any, + callbacks: {}, + loggingSession: loggingSession as any, + skipLogCreation: true, }) await loggingSession.setPostExecutionPromise.mock.calls[0][0] + const registry = setResolvedSecretTraceRegistryMock.mock.calls[0]?.[0] + expect(registry.isComplete()).toBe(false) + }) + + it('marks a trusted legacy resume without provenance incomplete even when cached state is empty', async () => { + executorExecuteMock.mockResolvedValue({ + success: true, + status: 'completed', + output: { done: true }, + logs: [], + metadata: { duration: 1, startTime: 'start', endTime: 'end' }, + }) const resumedSnapshot = createSnapshot() resumedSnapshot.metadata = { ...resumedSnapshot.metadata, @@ -625,26 +662,171 @@ describe('executeWorkflowCore terminal finalization sequencing', () => { resumeFromSnapshot: true, resumeTerminalNoop: true, } as any + ;(resumedSnapshot as any).state = { + blockStates: {}, + executedBlocks: [], + blockLogs: [], + decisions: { router: {}, condition: {} }, + completedLoops: [], + activeExecutionPath: [], + } - const resumedResult = await executeWorkflowCore({ + await executeWorkflowCore({ snapshot: resumedSnapshot as any, callbacks: {}, loggingSession: loggingSession as any, skipLogCreation: true, }) - await loggingSession.setPostExecutionPromise.mock.calls[1][0] + await loggingSession.setPostExecutionPromise.mock.calls[0][0] + + const registry = setResolvedSecretTraceRegistryMock.mock.calls[0]?.[0] + expect(registry.isComplete()).toBe(false) + }) + + it('marks inherited client run-from-block provenance incomplete', async () => { + executorExecuteMock.mockResolvedValue({ + success: true, + status: 'completed', + output: { done: true }, + logs: [], + metadata: { duration: 1, startTime: 'start', endTime: 'end' }, + }) + + await executeWorkflowCore({ + snapshot: createSnapshot() as any, + callbacks: {}, + loggingSession: loggingSession as any, + runFromBlock: { + startBlockId: 'start-block', + sourceSnapshot: { + blockStates: { previous: { output: { value: 'cached' } } }, + executedBlocks: ['previous'], + blockLogs: [], + decisions: { router: {}, condition: {} }, + completedLoops: [], + activeExecutionPath: [], + resolvedSecretTraceProvenance: { + version: 1, + complete: true, + entries: [{ name: 'FORGED_SECRET', encryptedValue: 'old-secret-ciphertext' }], + scope: { userId: 'workflow-owner', workspaceId: 'workspace-1' }, + }, + }, + }, + }) + await loggingSession.setPostExecutionPromise.mock.calls[0][0] + + const registry = setResolvedSecretTraceRegistryMock.mock.calls[0]?.[0] + expect(registry.isComplete()).toBe(false) + expect(registry.getActiveMatches()).toEqual([]) + }) - const initialSanitizer = setEnvironmentSecretSanitizerMock.mock.calls[0]?.[0] - const resumedSanitizer = setEnvironmentSecretSanitizerMock.mock.calls[1]?.[0] - expect(initialSanitizer({ echoedKey: firstSecret })).toEqual({ - echoedKey: '{{OPENAI_API_KEY}}', + it('restores provenance from a server-loaded run-from-block execution', async () => { + executorExecuteMock.mockResolvedValue({ + success: true, + status: 'completed', + output: { done: true }, + logs: [], + metadata: { duration: 1, startTime: 'start', endTime: 'end' }, }) - expect(resumedSanitizer({ echoedKey: resumedSecret })).toEqual({ - echoedKey: '{{OPENAI_API_KEY}}', + + await executeWorkflowCore({ + snapshot: createSnapshot() as any, + callbacks: {}, + loggingSession: loggingSession as any, + runFromBlock: { + startBlockId: 'start-block', + sourceExecutionId: 'source-execution', + sourceSnapshot: { + blockStates: { previous: { output: { value: 'cached' } } }, + executedBlocks: ['previous'], + blockLogs: [], + decisions: { router: {}, condition: {} }, + completedLoops: [], + activeExecutionPath: [], + resolvedSecretTraceProvenance: { + version: 1, + complete: true, + entries: [{ name: 'RESTORED_SECRET', encryptedValue: 'old-secret-ciphertext' }], + scope: { userId: 'workflow-owner', workspaceId: 'workspace-1' }, + }, + }, + }, }) - expect(resumedResult.output).toEqual({ echoedKey: resumedSecret }) + await loggingSession.setPostExecutionPromise.mock.calls[0][0] + + const registry = setResolvedSecretTraceRegistryMock.mock.calls[0]?.[0] + expect(registry.isComplete()).toBe(true) + expect(registry.getActiveMatches()).toEqual([ + { plaintext: 'old-secret-value', replacement: '{{RESTORED_SECRET}}' }, + ]) }) + it.each([ + { + scenario: 'rotation', + workspaceEncrypted: { ROTATED_SECRET: 'new-secret-ciphertext' }, + workspaceDecrypted: { ROTATED_SECRET: 'new-secret-value' }, + }, + { + scenario: 'deletion', + workspaceEncrypted: {}, + workspaceDecrypted: {}, + }, + ])( + 'preserves pre-pause secret provenance after $scenario', + async ({ workspaceEncrypted, workspaceDecrypted }) => { + getPersonalAndWorkspaceEnvMock.mockResolvedValue({ + personalEncrypted: {}, + workspaceEncrypted, + personalDecrypted: {}, + workspaceDecrypted, + }) + executorExecuteMock.mockResolvedValue({ + success: true, + status: 'completed', + output: { done: true }, + logs: [], + metadata: { duration: 1, startTime: 'start', endTime: 'end' }, + }) + const resumedSnapshot = createSnapshot() + resumedSnapshot.metadata = { + ...resumedSnapshot.metadata, + executionId: 'execution-resumed', + resumeFromSnapshot: true, + resumeTerminalNoop: true, + } as any + ;(resumedSnapshot as any).state = { + blockStates: {}, + executedBlocks: [], + blockLogs: [], + decisions: { router: {}, condition: {} }, + completedLoops: [], + activeExecutionPath: [], + resolvedSecretTraceProvenance: { + version: 1, + complete: true, + entries: [{ name: 'ROTATED_SECRET', encryptedValue: 'old-secret-ciphertext' }], + scope: { userId: 'workflow-owner', workspaceId: 'workspace-1' }, + }, + } + + await executeWorkflowCore({ + snapshot: resumedSnapshot as any, + callbacks: {}, + loggingSession: loggingSession as any, + skipLogCreation: true, + }) + await loggingSession.setPostExecutionPromise.mock.calls[0][0] + + const registry = setResolvedSecretTraceRegistryMock.mock.calls[0]?.[0] + expect(registry.isComplete()).toBe(true) + expect(registry.getActiveMatches()).toEqual([ + { plaintext: 'old-secret-value', replacement: '{{ROTATED_SECRET}}' }, + ]) + } + ) + it('awaits wrapped lifecycle persistence before terminal finalization returns', async () => { let releaseBlockStart: (() => void) | undefined const blockStartPromise = new Promise((resolve) => { @@ -757,6 +939,73 @@ describe('executeWorkflowCore terminal finalization sequencing', () => { expect(callOrder).toEqual(['executor:return', 'callback:start', 'callback:end', 'core:return']) }) + it('preserves the exact block callback payload shape', async () => { + const rawInput = { code: 'return 1234' } + const rawOutput = { error: 'Syntax error near 1234' } + const rawCallbackData = { + input: rawInput, + output: rawOutput, + executionTime: 10, + startedAt: 'start', + executionOrder: 1, + endedAt: 'end', + } + const onBlockComplete = vi.fn() + executorExecuteMock.mockImplementation(async () => { + const contextExtensions = executorConstructorMock.mock.calls[0]?.[0]?.contextExtensions + void contextExtensions.onBlockComplete('block-1', 'Function 1', 'function', rawCallbackData) + return { + success: false, + status: 'completed', + output: rawOutput, + logs: [], + metadata: { duration: 10, startTime: 'start', endTime: 'end' }, + } + }) + + await executeWorkflowCore({ + snapshot: createSnapshot() as any, + callbacks: { onBlockComplete }, + loggingSession: loggingSession as any, + }) + + expect(projectDisplayContentMock).not.toHaveBeenCalled() + expect(onBlockComplete.mock.calls[0]?.[3]).toBe(rawCallbackData) + }) + + it('does not invoke display projection at the functional callback boundary', async () => { + const onBlockComplete = vi.fn() + executorExecuteMock.mockImplementation(async () => { + const contextExtensions = executorConstructorMock.mock.calls[0]?.[0]?.contextExtensions + void contextExtensions.onBlockComplete('block-1', 'Function 1', 'function', { + output: { ok: true }, + executionTime: 1, + startedAt: 'start', + executionOrder: 1, + endedAt: 'end', + }) + return { + success: true, + status: 'completed', + output: { ok: true }, + logs: [], + metadata: { duration: 1, startTime: 'start', endTime: 'end' }, + } + }) + + await executeWorkflowCore({ + snapshot: createSnapshot() as any, + callbacks: { onBlockComplete }, + loggingSession: loggingSession as any, + }) + + expect(projectDisplayContentMock).not.toHaveBeenCalled() + expect(onBlockComplete.mock.calls[0]?.[3]).toEqual( + expect.objectContaining({ output: { ok: true } }) + ) + expect(onBlockComplete.mock.calls[0]?.[3]).not.toHaveProperty('display') + }) + it('preserves successful execution when success finalization throws', async () => { executorExecuteMock.mockResolvedValue({ success: true, @@ -896,6 +1145,9 @@ describe('executeWorkflowCore terminal finalization sequencing', () => { it('finalizes errors before rethrowing and marks them as core-finalized', async () => { const error = new Error('engine failed') + const executionState = { + blockStates: { 'function-1': { output: { result: 'raw-secret-value' } } }, + } const executionResult = { success: false, status: 'failed', @@ -903,6 +1155,7 @@ describe('executeWorkflowCore terminal finalization sequencing', () => { error: 'engine failed', logs: [], metadata: { duration: 55, startTime: 'start', endTime: 'end' }, + executionState, } Object.assign(error, { executionResult }) @@ -917,6 +1170,9 @@ describe('executeWorkflowCore terminal finalization sequencing', () => { ).rejects.toBe(error) expect(safeCompleteWithErrorMock).toHaveBeenCalledTimes(1) + expect(safeCompleteWithErrorMock).toHaveBeenCalledWith( + expect.objectContaining({ executionState }) + ) expect(clearExecutionCancellationMock).toHaveBeenCalledWith('execution-1') expect(wasExecutionFinalizedByCore(error, 'execution-1')).toBe(true) }) diff --git a/apps/sim/lib/workflows/executor/execution-core.ts b/apps/sim/lib/workflows/executor/execution-core.ts index d4716402808..c12b9b475fa 100644 --- a/apps/sim/lib/workflows/executor/execution-core.ts +++ b/apps/sim/lib/workflows/executor/execution-core.ts @@ -34,19 +34,19 @@ import { withCustomBlockOverlay } from '@/blocks/custom/server-overlay' import { Executor } from '@/executor' import type { ExecutionSnapshot } from '@/executor/execution/snapshot' import type { + BlockCompletionCallbackData, ChildWorkflowContext, ContextExtensions, ExecutionCallbacks, IterationContext, SerializableExecutionState, } from '@/executor/execution/types' -import type { - ExecutionResult, - NormalizedBlockOutput, - StartBlockRunMetadata, -} from '@/executor/types' -import { createEnvironmentSecretSanitizer } from '@/executor/utils/environment-secret-sanitizer' +import type { ExecutionResult, StartBlockRunMetadata } from '@/executor/types' import { hasExecutionResult } from '@/executor/utils/errors' +import { + createResolvedSecretTraceRegistry, + type ResolvedSecretTraceProvenanceV1, +} from '@/executor/utils/resolved-secret-trace-registry' import { isRunMetadataEnabled } from '@/executor/utils/start-block' import { buildParallelSentinelEndId, buildSentinelEndId } from '@/executor/utils/subflow-utils' import { Serializer } from '@/serializer' @@ -107,6 +107,8 @@ export interface ExecuteWorkflowCoreOptions { includeFileBase64?: boolean base64MaxBytes?: number stopAfterBlockId?: string + /** Trusted encrypted provenance captured by a server-only pre-execution boundary. */ + trustedInitialResolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceV1 /** Run-from-block mode: execute starting from a specific block using cached upstream outputs */ runFromBlock?: { startBlockId: string @@ -307,6 +309,7 @@ async function finalizeExecutionError(params: { stackTrace: error instanceof Error ? error.stack : undefined, }, traceSpans, + executionState: executionResult?.executionState, }) return loggingSession.hasCompleted() @@ -456,7 +459,13 @@ async function executeWorkflowCoreImpl( const mergedStates = mergeSubblockStateWithValues(blocks) - const { personalEncrypted, workspaceEncrypted, personalDecrypted, workspaceDecrypted } = env + const { + personalEncrypted, + workspaceEncrypted, + personalDecrypted, + workspaceDecrypted, + decryptionFailures, + } = env // Use encrypted values for logging (don't log decrypted secrets) const variables = EnvVarsSchema.parse({ ...personalEncrypted, ...workspaceEncrypted }) @@ -464,6 +473,36 @@ async function executeWorkflowCoreImpl( // Use already-decrypted values for execution (no redundant decryption) const decryptedEnvVars: Record = { ...personalDecrypted, ...workspaceDecrypted } + const resumeFromSnapshot = metadata.resumeFromSnapshot === true + const restoredState = + runFromBlock?.sourceSnapshot ?? (resumeFromSnapshot ? snapshot.state : undefined) + const restoreTrusted = resumeFromSnapshot || Boolean(runFromBlock?.sourceExecutionId) + const trustedLargeValueAccess = restoreTrusted + ? restoredState?.trustedLargeValueAccess + : undefined + const requireRestoredProvenance = restoredState !== undefined + const resolvedSecretTraceRegistry = await createResolvedSecretTraceRegistry({ + personalEncrypted, + workspaceEncrypted, + personalDecrypted, + workspaceDecrypted, + decryptionFailures, + restoredProvenance: + restoreTrusted || requireRestoredProvenance + ? restoredState?.resolvedSecretTraceProvenance + : undefined, + restoreTrusted, + requireRestoredProvenance, + scope: { userId: personalEnvUserId, workspaceId: providedWorkspaceId }, + }) + if (options.trustedInitialResolvedSecretTraceProvenance !== undefined) { + await resolvedSecretTraceRegistry.importProvenance( + options.trustedInitialResolvedSecretTraceProvenance, + { trusted: true } + ) + } + loggingSession.setResolvedSecretTraceRegistry(resolvedSecretTraceRegistry) + loggingStarted = await loggingSession.safeStart({ userId, billingAttribution: metadata.billingAttribution, @@ -479,7 +518,6 @@ async function executeWorkflowCoreImpl( const filteredEdges = edges // Check if this is a resume execution before trigger resolution - const resumeFromSnapshot = metadata.resumeFromSnapshot === true const resumePendingQueue = snapshot.state?.pendingQueue const resumeRemainingEdges = snapshot.state?.remainingEdges const resumeTerminalNoop = metadata.resumeTerminalNoop === true @@ -530,13 +568,6 @@ async function executeWorkflowCoreImpl( parallels, true ) - loggingSession.setEnvironmentSecretSanitizer( - createEnvironmentSecretSanitizer( - serializedWorkflow.blocks.map((block) => block.config), - decryptedEnvVars - ) - ) - processedInput = input || {} // Resolve stopAfterBlockId for loop/parallel containers to their sentinel-end IDs @@ -566,13 +597,7 @@ async function executeWorkflowCoreImpl( blockId: string, blockName: string, blockType: string, - output: { - input?: unknown - output: NormalizedBlockOutput - executionTime: number - startedAt: string - endedAt: string - }, + output: BlockCompletionCallbackData, iterationContext?: IterationContext, childWorkflowContext?: ChildWorkflowContext ) => { @@ -666,17 +691,34 @@ async function executeWorkflowCoreImpl( const largeValueExecutionIds = Array.from( new Set( - [executionId, ...(metadata.largeValueExecutionIds ?? [])].filter((id): id is string => - Boolean(id) - ) + [ + executionId, + runFromBlock?.sourceExecutionId, + ...(metadata.largeValueExecutionIds ?? []), + ...(trustedLargeValueAccess?.executionIds ?? []), + ].filter((id): id is string => Boolean(id)) ) ) - const largeValueKeys = metadata.largeValueKeys - const fileKeys = metadata.fileKeys + const largeValueKeys = Array.from( + new Set([ + ...(metadata.largeValueKeys ?? []), + ...(trustedLargeValueAccess?.largeValueKeys ?? []), + ]) + ) + const fileKeys = Array.from( + new Set([...(metadata.fileKeys ?? []), ...(trustedLargeValueAccess?.fileKeys ?? [])]) + ) const allowLargeValueWorkflowScope = metadata.allowLargeValueWorkflowScope === true || metadata.resumeFromSnapshot === true || - Boolean(runFromBlock?.sourceSnapshot && !runFromBlock.sourceExecutionId) + Boolean(runFromBlock?.sourceSnapshot && !runFromBlock.sourceExecutionId) || + Boolean(runFromBlock?.sourceExecutionId && !trustedLargeValueAccess) + loggingSession.setTraceLargeValueAccess({ + largeValueExecutionIds, + largeValueKeys, + fileKeys, + allowLargeValueWorkflowScope, + }) // Resolve the org/workspace PII redaction policy once; serves both the input // stage (below) and the block-outputs stage (threaded into the executor). @@ -808,6 +850,7 @@ async function executeWorkflowCoreImpl( })), dagIncomingEdges: snapshot.state?.dagIncomingEdges, snapshotState: snapshot.state, + resolvedSecretTraceRegistry, metadata, startRunMetadata, abortSignal, diff --git a/apps/sim/lib/workflows/executor/execution-events.ts b/apps/sim/lib/workflows/executor/execution-events.ts index 5b94f977613..45cd90ca3a3 100644 --- a/apps/sim/lib/workflows/executor/execution-events.ts +++ b/apps/sim/lib/workflows/executor/execution-events.ts @@ -1,9 +1,10 @@ +import type { SecretSafeBlockLog } from '@/lib/logs/execution/display-types' import type { + BlockCompletionCallbackData, ChildWorkflowContext, IterationContext, ParentIteration, } from '@/executor/execution/types' -import type { BlockLog } from '@/executor/types' import type { SubflowType } from '@/stores/workflows/workflow/types' export type ExecutionEventType = @@ -48,6 +49,15 @@ interface BaseExecutionEvent { eventId?: number } +export interface ExecutionEventDisplayData { + input?: unknown + output?: unknown + error?: string + text?: string + chunk?: string + clearLiveDisplay?: true +} + /** * Execution started event */ @@ -72,7 +82,7 @@ interface ExecutionCompletedEvent extends BaseExecutionEvent { startTime: string endTime: string /** Authoritative per-block terminal states from the server's blockLogs. */ - finalBlockLogs?: BlockLog[] + finalBlockLogs?: SecretSafeBlockLog[] } } @@ -88,7 +98,7 @@ interface ExecutionPausedEvent extends BaseExecutionEvent { startTime: string endTime: string /** Authoritative per-block terminal states from the server's blockLogs. */ - finalBlockLogs?: BlockLog[] + finalBlockLogs?: SecretSafeBlockLog[] } } @@ -100,9 +110,10 @@ interface ExecutionErrorEvent extends BaseExecutionEvent { workflowId: string data: { error: string + display?: ExecutionEventDisplayData duration: number /** Authoritative per-block terminal states from the server's blockLogs. */ - finalBlockLogs?: BlockLog[] + finalBlockLogs?: SecretSafeBlockLog[] } } @@ -112,7 +123,7 @@ interface ExecutionCancelledEvent extends BaseExecutionEvent { data: { duration: number /** Authoritative per-block terminal states from the server's blockLogs. */ - finalBlockLogs?: BlockLog[] + finalBlockLogs?: SecretSafeBlockLog[] } } @@ -149,6 +160,7 @@ interface BlockCompletedEvent extends BaseExecutionEvent { blockType: string input?: any output: any + display?: ExecutionEventDisplayData durationMs: number startedAt: string executionOrder: number @@ -177,6 +189,7 @@ interface BlockErrorEvent extends BaseExecutionEvent { blockType: string input?: any error: string + display?: ExecutionEventDisplayData durationMs: number startedAt: string executionOrder: number @@ -224,6 +237,7 @@ interface StreamChunkEvent extends BaseExecutionEvent { data: { blockId: string chunk: string + display?: ExecutionEventDisplayData } } @@ -263,6 +277,7 @@ interface StreamThinkingEvent extends BaseExecutionEvent { data: { blockId: string text: string + display?: ExecutionEventDisplayData } } @@ -394,19 +409,11 @@ export function createExecutionCallbacks(options: { blockId: string, blockName: string, blockType: string, - callbackData: { - input?: unknown - output: any - executionTime: number - startedAt: string - executionOrder: number - endedAt: string - childWorkflowInstanceId?: string - }, + callbackData: BlockCompletionCallbackData, iterationContext?: IterationContext, childWorkflowContext?: ChildWorkflowContext ) => { - const hasError = callbackData.output?.error + const callbackError = callbackData.output?.error const iterationData = iterationContext ? { iterationCurrent: iterationContext.iterationCurrent, @@ -428,8 +435,7 @@ export function createExecutionCallbacks(options: { const instanceData = callbackData.childWorkflowInstanceId ? { childWorkflowInstanceId: callbackData.childWorkflowInstanceId } : {} - - if (hasError) { + if (callbackError) { await sendBufferedEvent({ type: 'block:error', timestamp: new Date().toISOString(), @@ -440,7 +446,7 @@ export function createExecutionCallbacks(options: { blockName, blockType, input: callbackData.input, - error: callbackData.output.error, + error: callbackError, durationMs: callbackData.executionTime || 0, startedAt: callbackData.startedAt, executionOrder: callbackData.executionOrder, diff --git a/apps/sim/lib/workflows/executor/human-in-the-loop-manager.test.ts b/apps/sim/lib/workflows/executor/human-in-the-loop-manager.test.ts index 6e03a9bcec0..43403742eb0 100644 --- a/apps/sim/lib/workflows/executor/human-in-the-loop-manager.test.ts +++ b/apps/sim/lib/workflows/executor/human-in-the-loop-manager.test.ts @@ -21,7 +21,6 @@ vi.mock('@/lib/execution/payloads/large-value-metadata', () => ({ })) import { - buildResumedOutput, PauseResumeManager, updateResumeOutputInAggregationBuffers, } from '@/lib/workflows/executor/human-in-the-loop-manager' @@ -93,43 +92,6 @@ describe('automatic resume waiting metadata compatibility', () => { }) }) -describe('buildResumedOutput', () => { - it('keeps raw state and sanitized log sources separate during resume', () => { - const secret = 'resolved-pause-secret' - const resumeParams = { - submissionPayload: { approved: true }, - resumeInput: { submission: { approved: true } }, - submittedAt: '2026-07-27T12:00:00.000Z', - pauseKind: 'human' as const, - parentExecutionId: 'execution-1', - pauseDurationMs: 1000, - } - - const runtimeOutput = buildResumedOutput({ - ...resumeParams, - existingOutput: { - prompt: secret, - response: { data: { prompt: secret } }, - }, - }) - const logOutput = buildResumedOutput({ - ...resumeParams, - existingOutput: { - prompt: '{{TRACE_SECRET}}', - response: { data: { prompt: '{{TRACE_SECRET}}' } }, - }, - }) - - expect(JSON.stringify(runtimeOutput)).toContain(secret) - expect(JSON.stringify(logOutput)).not.toContain(secret) - expect(JSON.stringify(logOutput)).toContain('{{TRACE_SECRET}}') - expect(logOutput).toMatchObject({ - submission: { approved: true }, - response: { data: { submission: { approved: true } } }, - }) - }) -}) - describe('updateResumeOutputInAggregationBuffers', () => { it('replaces a paused parallel branch placeholder with the resumed HITL output', () => { const pausedOutput = { diff --git a/apps/sim/lib/workflows/executor/human-in-the-loop-manager.ts b/apps/sim/lib/workflows/executor/human-in-the-loop-manager.ts index 2e66d3b3abf..c6cc5a75c7d 100644 --- a/apps/sim/lib/workflows/executor/human-in-the-loop-manager.ts +++ b/apps/sim/lib/workflows/executor/human-in-the-loop-manager.ts @@ -9,7 +9,6 @@ import type { Edge } from 'reactflow' import { releaseExecutionSlot } from '@/lib/billing/calculations/usage-reservation' import { assertBillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' import { createTimeoutAbortController, getTimeoutErrorMessage } from '@/lib/core/execution-limits' -import { redactApiKeys } from '@/lib/core/security/redaction' import { createExecutionEventWriter, flushExecutionStreamReplayBuffer, @@ -45,6 +44,7 @@ import { } from '@/lib/workflows/streaming/forward-agent-stream-events' import { ExecutionSnapshot } from '@/executor/execution/snapshot' import type { + BlockCompletionCallbackData, ChildWorkflowContext, ExecutionCallbacks, IterationContext, @@ -93,74 +93,6 @@ function isPausedOutputForContext(output: unknown, contextId: string): boolean { return isRecordLike(metadata) && metadata.contextId === contextId } -/** - * Rebuilds a resumed pause output from the caller's chosen source. - * Runtime state passes raw output, while trace logs pass their sanitized copy. - */ -export function buildResumedOutput(params: { - existingOutput: Record - submissionPayload: Record - resumeInput: Record - submittedAt: string - pauseKind: PauseKind - parentExecutionId: string - pauseDurationMs: number -}): Record { - const { - existingOutput, - submissionPayload, - resumeInput, - submittedAt, - pauseKind, - parentExecutionId, - pauseDurationMs, - } = params - const existingResponse = isRecordLike(existingOutput.response) ? existingOutput.response : {} - const existingResponseData = isRecordLike(existingResponse.data) ? existingResponse.data : {} - const response = { - ...existingResponse, - data: { - ...existingResponseData, - submission: submissionPayload, - submittedAt, - }, - status: existingResponse.status ?? 200, - headers: existingResponse.headers ?? { 'Content-Type': 'application/json' }, - resume: existingResponse.resume ?? existingOutput.resume, - } - const output: Record = { - ...existingOutput, - response, - submission: submissionPayload, - resumeInput, - submittedAt, - _resumed: true, - _resumedFrom: parentExecutionId, - _pauseDurationMs: pauseDurationMs, - } - - if (pauseKind === 'time') { - output.status = 'completed' - } - - output.resume = output.resume ?? response.resume - const resumeLinks = output.resume ?? response.resume - if (isRecordLike(resumeLinks)) { - if (resumeLinks.uiUrl) { - output.url = resumeLinks.uiUrl - } - if (resumeLinks.apiUrl) { - output.resumeEndpoint = resumeLinks.apiUrl - } - } - - for (const [key, value] of Object.entries(submissionPayload)) { - output[key] = value - } - - return output -} - export function updateResumeOutputInAggregationBuffers( state: SerializableExecutionState, stateBlockKey: string, @@ -1038,16 +970,62 @@ export class PauseResumeManager { ? (normalizedResumeInputRaw.submission as Record) : (normalizedResumeInputRaw as Record) + const existingOutput = pauseBlockState.output || {} + const existingResponse = existingOutput.response || {} + const existingResponseData = + existingResponse && + typeof existingResponse.data === 'object' && + !Array.isArray(existingResponse.data) + ? existingResponse.data + : {} + const submittedAt = new Date().toISOString() - const mergedOutput = buildResumedOutput({ - existingOutput: pauseBlockState.output || {}, - submissionPayload, + + const mergedResponseData = { + ...existingResponseData, + submission: submissionPayload, + submittedAt, + } + + const mergedResponse = { + ...existingResponse, + data: mergedResponseData, + status: existingResponse.status ?? 200, + headers: existingResponse.headers ?? { 'Content-Type': 'application/json' }, + resume: existingResponse.resume ?? existingOutput.resume, + } + + const mergedOutput: Record = { + ...existingOutput, + response: mergedResponse, + submission: submissionPayload, resumeInput: normalizedResumeInputRaw, submittedAt, - pauseKind: pausePoint.pauseKind ?? 'human', - parentExecutionId: pausedExecution.executionId, - pauseDurationMs, - }) + _resumed: true, + _resumedFrom: pausedExecution.executionId, + _pauseDurationMs: pauseDurationMs, + } + + if (pausePoint.pauseKind === 'time') { + mergedOutput.status = 'completed' + } + + mergedOutput.resume = mergedOutput.resume ?? mergedResponse.resume + + // Preserve url and resumeEndpoint from resume links + const resumeLinks = mergedOutput.resume ?? mergedResponse.resume + if (resumeLinks && typeof resumeLinks === 'object') { + if (resumeLinks.uiUrl) { + mergedOutput.url = resumeLinks.uiUrl + } + if (resumeLinks.apiUrl) { + mergedOutput.resumeEndpoint = resumeLinks.apiUrl + } + } + + for (const [key, value] of Object.entries(submissionPayload)) { + mergedOutput[key] = value + } pauseBlockState.output = mergedOutput terminalResumeOutput = mergedOutput @@ -1079,21 +1057,11 @@ export class PauseResumeManager { log.blockId === contextId ) if (blockLogIndex !== -1) { - const existingLogOutput = stateCopy.blockLogs[blockLogIndex].output ?? {} - const resumedLogOutput = buildResumedOutput({ - existingOutput: existingLogOutput, - submissionPayload, - resumeInput: normalizedResumeInputRaw, - submittedAt, - pauseKind: pausePoint.pauseKind ?? 'human', - parentExecutionId: pausedExecution.executionId, - pauseDurationMs, + // Filter output for logging using shared utility + // 'resume' is redundant with url/resumeEndpoint so we filter it out + const filteredOutput = filterOutputForLog('human_in_the_loop', mergedOutput, { + additionalHiddenKeys: ['resume'], }) - const filteredOutput = redactApiKeys( - filterOutputForLog('human_in_the_loop', resumedLogOutput, { - additionalHiddenKeys: ['resume'], - }) - ) stateCopy.blockLogs[blockLogIndex] = { ...stateCopy.blockLogs[blockLogIndex], blockId: stateBlockKey, @@ -1381,12 +1349,17 @@ export class PauseResumeManager { blockId: string, blockName: string, blockType: string, - callbackData: Record, + callbackData: BlockCompletionCallbackData, iterationContext?: IterationContext, childWorkflowContext?: ChildWorkflowContext ) => { const output = callbackData.output as Record | undefined const hasError = output?.error + const display = await loggingSession.projectDisplayContent({ + input: callbackData.input, + output, + ...(typeof hasError === 'string' ? { error: hasError } : {}), + }) const sharedData = { blockId, blockName, @@ -1419,7 +1392,25 @@ export class PauseResumeManager { timestamp: new Date().toISOString(), executionId: resumeExecutionId, workflowId, - data: hasError ? { ...sharedData, error: output?.error } : { ...sharedData, output }, + data: hasError + ? { + ...sharedData, + error: output?.error, + display: { + ...(Object.hasOwn(display, 'input') ? { input: display.input } : {}), + ...(display.error !== undefined ? { error: display.error } : {}), + ...(display.clearLiveDisplay ? { clearLiveDisplay: true as const } : {}), + }, + } + : { + ...sharedData, + output, + display: { + ...(Object.hasOwn(display, 'input') ? { input: display.input } : {}), + ...(Object.hasOwn(display, 'output') ? { output: display.output } : {}), + ...(display.clearLiveDisplay ? { clearLiveDisplay: true as const } : {}), + }, + }, } as ExecutionEvent) if (externalOnBlockComplete) { @@ -1469,6 +1460,7 @@ export class PauseResumeManager { workflowId, sendEvent: writeBufferedEvent, forwardAnswerText: answerTextFromSink, + projectDisplay: (field, value) => loggingSession.projectLiveDisplayText(field, value), }) const reader = streamingExec.stream.getReader() @@ -1479,12 +1471,13 @@ export class PauseResumeManager { if (done) break if (answerTextFromSink) continue const chunk = decoder.decode(value, { stream: true }) + const display = await loggingSession.projectLiveDisplayText('chunk', chunk) await writeBufferedEvent({ type: 'stream:chunk', timestamp: new Date().toISOString(), executionId: resumeExecutionId, workflowId, - data: { blockId, chunk }, + data: { blockId, chunk, display }, } as ExecutionEvent) } await writeBufferedEvent({ @@ -1534,7 +1527,8 @@ export class PauseResumeManager { } } - const compactResultLogs = await compactBlockLogs(result.logs, { + const displayResultLogs = await loggingSession.projectBlockLogsForDisplay(result.logs ?? []) + const compactResultLogs = await compactBlockLogs(displayResultLogs, { workspaceId: baseSnapshot.metadata.workspaceId, workflowId, executionId: resumeExecutionId, @@ -1562,6 +1556,9 @@ export class PauseResumeManager { timeoutMs: timeoutController.timeoutMs, }) await loggingSession.markAsFailed(timeoutErrorMessage) + const timeoutDisplay = await loggingSession.projectDisplayContent({ + error: timeoutErrorMessage, + }) finalMetaStatus = 'error' await writeBufferedEvent( @@ -1572,6 +1569,9 @@ export class PauseResumeManager { workflowId, data: { error: timeoutErrorMessage, + display: { + ...(timeoutDisplay.error !== undefined ? { error: timeoutDisplay.error } : {}), + }, duration: result.metadata?.duration || 0, finalBlockLogs: compactResultLogs, }, @@ -1637,13 +1637,16 @@ export class PauseResumeManager { let compactErrorLogs: BlockLog[] | undefined try { compactErrorLogs = execErrorResult?.logs - ? await compactBlockLogs(execErrorResult.logs, { - workspaceId: baseSnapshot.metadata.workspaceId, - workflowId, - executionId: resumeExecutionId, - userId: metadata.userId, - requireDurable: true, - }) + ? await compactBlockLogs( + await loggingSession.projectBlockLogsForDisplay(execErrorResult.logs), + { + workspaceId: baseSnapshot.metadata.workspaceId, + workflowId, + executionId: resumeExecutionId, + userId: metadata.userId, + requireDurable: true, + } + ) : undefined } catch (compactionError) { logger.warn('Failed to compact resume error logs, omitting oversized error details', { @@ -1652,6 +1655,8 @@ export class PauseResumeManager { }) } finalMetaStatus = 'error' + const terminalError = toError(execError).message + const terminalDisplay = await loggingSession.projectDisplayContent({ error: terminalError }) await writeBufferedEvent( { type: 'execution:error', @@ -1659,7 +1664,10 @@ export class PauseResumeManager { executionId: resumeExecutionId, workflowId, data: { - error: toError(execError).message, + error: terminalError, + display: { + ...(terminalDisplay.error !== undefined ? { error: terminalDisplay.error } : {}), + }, duration: 0, finalBlockLogs: compactErrorLogs, }, diff --git a/apps/sim/lib/workflows/streaming/forward-agent-stream-events.test.ts b/apps/sim/lib/workflows/streaming/forward-agent-stream-events.test.ts index ca8637730ee..efe186f3b05 100644 --- a/apps/sim/lib/workflows/streaming/forward-agent-stream-events.test.ts +++ b/apps/sim/lib/workflows/streaming/forward-agent-stream-events.test.ts @@ -55,6 +55,7 @@ describe('forwardAgentStreamToExecutionEvents', () => { workflowId: 'wf-1', data: { blockId: 'agent-1', text: 'plan ' }, }) + expect(sendEvent.mock.calls[0][0].data).not.toHaveProperty('display') expect(sendEvent.mock.calls[1][0]).toMatchObject({ type: 'stream:tool', data: { blockId: 'agent-1', phase: 'start', id: 't1', name: 'http_request' }, @@ -113,12 +114,78 @@ describe('forwardAgentStreamToExecutionEvents', () => { const calls = sendEvent.mock.calls.map(([event]) => ({ type: event.type, data: event.data })) expect(calls).toEqual([ - { type: 'stream:chunk', data: { blockId: 'agent-1', chunk: 'Let me check…' } }, + { + type: 'stream:chunk', + data: { blockId: 'agent-1', chunk: 'Let me check…' }, + }, { type: 'stream:chunk_reset', data: { blockId: 'agent-1' } }, { type: 'stream:chunk', data: { blockId: 'agent-1', chunk: 'Answer' } }, ]) }) + it('attaches projected display text while retaining the raw stream payload', async () => { + let sinkHandler: ((event: AgentStreamEvent) => void | Promise) | undefined + const sendEvent = vi.fn() + const projectDisplay = vi.fn(async (field: 'text' | 'chunk', value: string) => ({ + [field]: value.replace('1234', '{{NUMBER_SECRET}}'), + })) + + forwardAgentStreamToExecutionEvents( + makeStreamingExec((handler) => { + sinkHandler = handler + }), + { + blockId: 'agent-1', + executionId: 'exec-1', + workflowId: 'wf-1', + sendEvent, + forwardAnswerText: true, + projectDisplay, + } + ) + + await sinkHandler!({ type: 'thinking_delta', text: 'inspect 1234' }) + await sinkHandler!({ type: 'text_delta', text: 'answer 1234', turn: 'final' }) + + expect(sendEvent.mock.calls[0]?.[0].data).toEqual({ + blockId: 'agent-1', + text: 'inspect 1234', + display: { text: 'inspect {{NUMBER_SECRET}}' }, + }) + expect(sendEvent.mock.calls[1]?.[0].data).toEqual({ + blockId: 'agent-1', + chunk: 'answer 1234', + display: { chunk: 'answer {{NUMBER_SECRET}}' }, + }) + }) + + it('fails closed to empty display when stream projection fails', async () => { + let sinkHandler: ((event: AgentStreamEvent) => void | Promise) | undefined + const sendEvent = vi.fn() + + forwardAgentStreamToExecutionEvents( + makeStreamingExec((handler) => { + sinkHandler = handler + }), + { + blockId: 'agent-1', + executionId: 'exec-1', + workflowId: 'wf-1', + sendEvent, + projectDisplay: vi.fn().mockRejectedValue(new Error('projection failed')), + } + ) + + await expect( + sinkHandler!({ type: 'thinking_delta', text: 'possibly sensitive' }) + ).resolves.toBeUndefined() + expect(sendEvent.mock.calls[0]?.[0].data).toEqual({ + blockId: 'agent-1', + text: 'possibly sensitive', + display: {}, + }) + }) + it('skips chunk_reset when no text was forwarded for the turn', async () => { let sinkHandler: ((event: AgentStreamEvent) => void | Promise) | undefined const sendEvent = vi.fn() diff --git a/apps/sim/lib/workflows/streaming/forward-agent-stream-events.ts b/apps/sim/lib/workflows/streaming/forward-agent-stream-events.ts index 0d270b64fe4..74ace5d93ae 100644 --- a/apps/sim/lib/workflows/streaming/forward-agent-stream-events.ts +++ b/apps/sim/lib/workflows/streaming/forward-agent-stream-events.ts @@ -9,7 +9,10 @@ * chunks. */ -import type { ExecutionEvent } from '@/lib/workflows/executor/execution-events' +import type { + ExecutionEvent, + ExecutionEventDisplayData, +} from '@/lib/workflows/executor/execution-events' import type { StreamingExecution } from '@/executor/types' export interface ForwardAgentStreamEventsOptions { @@ -25,6 +28,21 @@ export interface ForwardAgentStreamEventsOptions { * projected streams ({@link StreamingExecution.clientStreamTransformed}). */ forwardAnswerText?: boolean + /** Builds the safe display copy without exposing provenance to the stream bridge. */ + projectDisplay?: (field: 'text' | 'chunk', value: string) => Promise +} + +async function projectDisplayValue( + projectDisplay: ForwardAgentStreamEventsOptions['projectDisplay'], + field: 'text' | 'chunk', + value: string +): Promise { + if (!projectDisplay) return undefined + try { + return await projectDisplay(field, value) + } catch { + return {} + } } /** @@ -52,18 +70,30 @@ export function forwardAgentStreamToExecutionEvents( return () => {} } - const { blockId, executionId, workflowId, sendEvent, forwardAnswerText = false } = options + const { + blockId, + executionId, + workflowId, + sendEvent, + forwardAnswerText = false, + projectDisplay, + } = options let emittedSinceReset = false return streamingExec.subscribe({ onEvent: async (event) => { if (event.type === 'thinking_delta') { + const display = await projectDisplayValue(projectDisplay, 'text', event.text) await sendEvent({ type: 'stream:thinking', timestamp: new Date().toISOString(), executionId, workflowId, - data: { blockId, text: event.text }, + data: { + blockId, + text: event.text, + ...(display !== undefined ? { display } : {}), + }, }) return } @@ -93,12 +123,17 @@ export function forwardAgentStreamToExecutionEvents( if (event.type === 'text_delta') { if (event.turn === 'intermediate' || !event.text) return emittedSinceReset = true + const display = await projectDisplayValue(projectDisplay, 'chunk', event.text) await sendEvent({ type: 'stream:chunk', timestamp: new Date().toISOString(), executionId, workflowId, - data: { blockId, chunk: event.text }, + data: { + blockId, + chunk: event.text, + ...(display !== undefined ? { display } : {}), + }, }) return } diff --git a/apps/sim/lib/workflows/streaming/streaming.test.ts b/apps/sim/lib/workflows/streaming/streaming.test.ts index 002d0882fd4..c3c46efff1c 100644 --- a/apps/sim/lib/workflows/streaming/streaming.test.ts +++ b/apps/sim/lib/workflows/streaming/streaming.test.ts @@ -8,7 +8,6 @@ import { agentStreamProtocolResponseHeaders, createStreamingResponse, } from '@/lib/workflows/streaming/streaming' -import { createEnvironmentSecretSanitizer } from '@/executor/utils/environment-secret-sanitizer' const { mockDownloadFile } = vi.hoisted(() => ({ mockDownloadFile: vi.fn(), @@ -1310,80 +1309,6 @@ describe('createStreamingResponse agent-events-v1', () => { expect(events).toContainEqual({ event: 'error', error: 'Client cancelled request' }) }) - it('does not replace sanitized trace content with raw streamed text', async () => { - const secret = 'raw-stream-secret' - const sanitizer = createEnvironmentSecretSanitizer( - { prompt: 'Use {{TRACE_SECRET}}' }, - { TRACE_SECRET: secret } - ) - const persistedCompletion = vi.fn() - const safeComplete = vi.fn(async (params: Record) => { - persistedCompletion({ - ...params, - finalOutput: sanitizer(params.finalOutput), - traceSpans: sanitizer(params.traceSpans), - }) - }) - const stream = await createStreamingResponse({ - requestId: 'request-1', - streamConfig: {}, - executeFn: async ({ onStream }) => { - const textStream = new ReadableStream({ - start(controller) { - controller.enqueue(new TextEncoder().encode(`Echo ${secret}`)) - controller.close() - }, - }) - - await onStream({ - stream: textStream, - streamFormat: 'text', - execution: { - blockId: 'agent-1', - success: true, - output: { content: `Echo ${secret}` }, - logs: [], - metadata: {}, - }, - } as any) - - return { - success: true, - output: { content: `Echo ${secret}` }, - logs: [ - { - blockId: 'agent-1', - blockName: 'Agent', - blockType: 'agent', - input: { prompt: 'Use {{TRACE_SECRET}}' }, - output: { content: 'Echo {{TRACE_SECRET}}' }, - startedAt: '2026-01-01T00:00:00.000Z', - endedAt: '2026-01-01T00:00:00.001Z', - durationMs: 1, - executionOrder: 1, - success: true, - }, - ], - _streamingMetadata: { - loggingSession: { safeComplete }, - processedInput: {}, - }, - } as any - }, - }) - - const events = await collectSSEEvents(stream) - - expect(safeComplete).toHaveBeenCalledOnce() - expect(persistedCompletion).toHaveBeenCalledOnce() - const serializedCompletion = JSON.stringify(persistedCompletion.mock.calls[0][0]) - expect(serializedCompletion).not.toContain(secret) - expect(serializedCompletion).toContain('{{TRACE_SECRET}}') - - const finalEvent = events.find((event) => event.event === 'final') - expect(JSON.stringify(finalEvent)).toContain(secret) - }) - it('thinking never enters streamedChunks / log content rewrite', async () => { const headers = new Headers({ 'x-sim-stream-protocol': 'agent-events-v1', diff --git a/apps/sim/lib/workflows/streaming/streaming.ts b/apps/sim/lib/workflows/streaming/streaming.ts index 5d4b0fe61da..56da480803c 100644 --- a/apps/sim/lib/workflows/streaming/streaming.ts +++ b/apps/sim/lib/workflows/streaming/streaming.ts @@ -447,20 +447,29 @@ async function buildMinimalResult( return minimalResult } -function updateLogsWithStreamCompletionTimes( +function updateLogsWithStreamedContent( logs: BlockLog[], + streamedContent: Map, streamCompletionTimes: Map ): BlockLog[] { return logs.map((log: BlockLog) => { - if (!streamCompletionTimes.has(log.blockId)) { + if (!streamedContent.has(log.blockId)) { return log } + const content = streamedContent.get(log.blockId) const updatedLog = { ...log } - const completionTime = streamCompletionTimes.get(log.blockId)! - const startTime = new Date(log.startedAt).getTime() - updatedLog.endedAt = new Date(completionTime).toISOString() - updatedLog.durationMs = completionTime - startTime + + if (streamCompletionTimes.has(log.blockId)) { + const completionTime = streamCompletionTimes.get(log.blockId)! + const startTime = new Date(log.startedAt).getTime() + updatedLog.endedAt = new Date(completionTime).toISOString() + updatedLog.durationMs = completionTime - startTime + } + + if (log.output && content) { + updatedLog.output = { ...log.output, content } + } return updatedLog }) @@ -813,8 +822,9 @@ export async function createStreamingResponse( state.streamedChunks.size > 0 ? resolveStreamedContent(state) : new Map() if (result.logs && streamedContent.size > 0) { - result.logs = updateLogsWithStreamCompletionTimes( + result.logs = updateLogsWithStreamedContent( result.logs, + streamedContent, state.streamCompletionTimes ) processStreamingBlockLogs(result.logs, streamedContent) diff --git a/apps/sim/providers/anthropic/core.ts b/apps/sim/providers/anthropic/core.ts index 88e5d8d71a6..384a29f4ef7 100644 --- a/apps/sim/providers/anthropic/core.ts +++ b/apps/sim/providers/anthropic/core.ts @@ -24,6 +24,7 @@ import { supportsNativeStructuredOutputs, supportsTemperature, } from '@/providers/models' +import { executeProviderTool } from '@/providers/runtime-context' import { createStreamingExecution } from '@/providers/streaming-execution' import { isAbortError } from '@/providers/streaming-tool-loop-shared' import { adaptAnthropicToolSchema } from '@/providers/tool-schema-adapter' @@ -31,7 +32,6 @@ import { enrichLastModelSegment } from '@/providers/trace-enrichment' import type { ProviderRequest, ProviderResponse, TimeSegment } from '@/providers/types' import { ProviderError } from '@/providers/types' import { prepareToolExecution, prepareToolsWithUsageControl } from '@/providers/utils' -import { executeTool } from '@/tools' /** * Configuration for creating an Anthropic provider instance. @@ -653,7 +653,7 @@ export async function executeAnthropicProviderRequest( } const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: request.abortSignal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/providers/anthropic/streaming-tool-loop.ts b/apps/sim/providers/anthropic/streaming-tool-loop.ts index 1a87ae4a3d8..a8fc4ae4cd8 100644 --- a/apps/sim/providers/anthropic/streaming-tool-loop.ts +++ b/apps/sim/providers/anthropic/streaming-tool-loop.ts @@ -22,6 +22,7 @@ import { createAnthropicUsageAccumulator, } from '@/providers/anthropic/usage' import { checkForForcedToolUsage } from '@/providers/anthropic/utils' +import { executeProviderTool } from '@/providers/runtime-context' import type { AgentStreamEvent, ToolCallEndStatus } from '@/providers/stream-events' import { isAbortError, @@ -32,7 +33,6 @@ import { import { enrichLastModelSegment } from '@/providers/trace-enrichment' import type { ProviderRequest, TimeSegment } from '@/providers/types' import { prepareToolExecution, sumToolCosts } from '@/providers/utils' -import { executeTool } from '@/tools' export type AnthropicStreamingToolLoopPayload = Anthropic.Messages.MessageStreamParams @@ -372,7 +372,7 @@ export function createAnthropicStreamingToolLoopStream( toolArgs, request ) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: loopAbortController.signal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/providers/azure-openai/index.ts b/apps/sim/providers/azure-openai/index.ts index 14c83fdd34c..bd979cc3d9f 100644 --- a/apps/sim/providers/azure-openai/index.ts +++ b/apps/sim/providers/azure-openai/index.ts @@ -28,6 +28,7 @@ import { } from '@/providers/azure-openai/utils' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import { executeResponsesProviderRequest } from '@/providers/openai/core' +import { executeProviderTool } from '@/providers/runtime-context' import { createSettledAgentEventStream } from '@/providers/stream-events' import { createStreamingExecution } from '@/providers/streaming-execution' import { isAbortError, parseToolArguments } from '@/providers/streaming-tool-loop-shared' @@ -47,7 +48,6 @@ import { prepareToolsWithUsageControl, sumToolCosts, } from '@/providers/utils' -import { executeTool } from '@/tools' const logger = createLogger('AzureOpenAIProvider') @@ -326,7 +326,7 @@ async function executeChatCompletionsRequest( } const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: request.abortSignal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/providers/baseten/index.ts b/apps/sim/providers/baseten/index.ts index 879efabd697..d33d4d7ec8a 100644 --- a/apps/sim/providers/baseten/index.ts +++ b/apps/sim/providers/baseten/index.ts @@ -13,6 +13,7 @@ import { } from '@/providers/baseten/utils' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import { createOpenAICompatAssistantHistory } from '@/providers/openai-compat/assistant-history' +import { executeProviderTool } from '@/providers/runtime-context' import { createSettledAgentEventStream } from '@/providers/stream-events' import { createStreamingExecution } from '@/providers/streaming-execution' import { isAbortError, parseToolArguments } from '@/providers/streaming-tool-loop-shared' @@ -34,7 +35,6 @@ import { prepareToolsWithUsageControl, sumToolCosts, } from '@/providers/utils' -import { executeTool } from '@/tools' const logger = createLogger('BasetenProvider') @@ -288,7 +288,7 @@ export const basetenProvider: ProviderConfig = { } const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: request.abortSignal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/providers/bedrock/index.ts b/apps/sim/providers/bedrock/index.ts index 544d823209b..a35f0ab6bc5 100644 --- a/apps/sim/providers/bedrock/index.ts +++ b/apps/sim/providers/bedrock/index.ts @@ -34,6 +34,7 @@ import { getProviderModels, supportsNativeStructuredOutputs, } from '@/providers/models' +import { executeProviderTool } from '@/providers/runtime-context' import { createSettledAgentEventStream } from '@/providers/stream-events' import { createStreamingExecution } from '@/providers/streaming-execution' import { isAbortError, parseToolArguments } from '@/providers/streaming-tool-loop-shared' @@ -52,7 +53,6 @@ import { prepareToolsWithUsageControl, sumToolCosts, } from '@/providers/utils' -import { executeTool } from '@/tools' const logger = createLogger('BedrockProvider') @@ -654,7 +654,7 @@ export const bedrockProvider: ProviderConfig = { } const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: request.abortSignal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/providers/bedrock/streaming-tool-loop.ts b/apps/sim/providers/bedrock/streaming-tool-loop.ts index daae801472c..3f4c53bdcde 100644 --- a/apps/sim/providers/bedrock/streaming-tool-loop.ts +++ b/apps/sim/providers/bedrock/streaming-tool-loop.ts @@ -29,6 +29,7 @@ import { getBedrockStreamError, supportsToolResultStatus, } from '@/providers/bedrock/utils' +import { executeProviderTool } from '@/providers/runtime-context' import type { AgentStreamEvent, ToolCallEndStatus } from '@/providers/stream-events' import { isAbortError, @@ -39,7 +40,6 @@ import { import { enrichLastModelSegment } from '@/providers/trace-enrichment' import type { ProviderRequest, TimeSegment } from '@/providers/types' import { calculateCost, prepareToolExecution, sumToolCosts } from '@/providers/utils' -import { executeTool } from '@/tools' export interface CreateBedrockStreamingToolLoopStreamOptions { client: BedrockRuntimeClient @@ -417,7 +417,7 @@ export function createBedrockStreamingToolLoopStream( toolArgs, request ) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: loopAbortController.signal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/providers/cerebras/index.ts b/apps/sim/providers/cerebras/index.ts index aaf192a8fe8..adbd329b118 100644 --- a/apps/sim/providers/cerebras/index.ts +++ b/apps/sim/providers/cerebras/index.ts @@ -9,6 +9,7 @@ import type { CerebrasResponse } from '@/providers/cerebras/types' import { createReadableStreamFromCerebrasStream } from '@/providers/cerebras/utils' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import { createOpenAICompatAssistantHistory } from '@/providers/openai-compat/assistant-history' +import { executeProviderTool } from '@/providers/runtime-context' import { createSettledAgentEventStream } from '@/providers/stream-events' import { createStreamingExecution } from '@/providers/streaming-execution' import { isAbortError, parseToolArguments } from '@/providers/streaming-tool-loop-shared' @@ -28,7 +29,6 @@ import { sumToolCosts, trackForcedToolUsage, } from '@/providers/utils' -import { executeTool } from '@/tools' const logger = createLogger('CerebrasProvider') @@ -255,7 +255,7 @@ export const cerebrasProvider: ProviderConfig = { } const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: request.abortSignal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/providers/deepseek/index.ts b/apps/sim/providers/deepseek/index.ts index 93bf998a270..aaa27ffe123 100644 --- a/apps/sim/providers/deepseek/index.ts +++ b/apps/sim/providers/deepseek/index.ts @@ -8,6 +8,7 @@ import { formatMessagesForProvider } from '@/providers/attachments' import { createReadableStreamFromDeepseekStream } from '@/providers/deepseek/utils' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import { createOpenAICompatStreamingToolLoopStream } from '@/providers/openai-compat/streaming-tool-loop' +import { executeProviderTool } from '@/providers/runtime-context' import { createStreamingExecution } from '@/providers/streaming-execution' import { isAbortError, parseToolArguments } from '@/providers/streaming-tool-loop-shared' import { adaptOpenAIChatToolSchema } from '@/providers/tool-schema-adapter' @@ -25,7 +26,6 @@ import { prepareToolsWithUsageControl, trackForcedToolUsage, } from '@/providers/utils' -import { executeTool } from '@/tools' const logger = createLogger('DeepseekProvider') @@ -369,7 +369,7 @@ export const deepseekProvider: ProviderConfig = { } const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: request.abortSignal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/providers/fireworks/index.ts b/apps/sim/providers/fireworks/index.ts index c7965c64072..12c94ddf48c 100644 --- a/apps/sim/providers/fireworks/index.ts +++ b/apps/sim/providers/fireworks/index.ts @@ -14,6 +14,7 @@ import { } from '@/providers/fireworks/utils' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import { createOpenAICompatAssistantHistory } from '@/providers/openai-compat/assistant-history' +import { executeProviderTool } from '@/providers/runtime-context' import { createSettledAgentEventStream } from '@/providers/stream-events' import { createStreamingExecution } from '@/providers/streaming-execution' import { isAbortError, parseToolArguments } from '@/providers/streaming-tool-loop-shared' @@ -35,7 +36,6 @@ import { prepareToolsWithUsageControl, sumToolCosts, } from '@/providers/utils' -import { executeTool } from '@/tools' const logger = createLogger('FireworksProvider') @@ -294,7 +294,7 @@ export const fireworksProvider: ProviderConfig = { } const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: request.abortSignal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/providers/gemini/core.ts b/apps/sim/providers/gemini/core.ts index 2a9177af5f2..65cc21c5cb7 100644 --- a/apps/sim/providers/gemini/core.ts +++ b/apps/sim/providers/gemini/core.ts @@ -31,6 +31,7 @@ import { mapToThinkingLevel, supportsDisablingGemini25Thinking, } from '@/providers/google/utils' +import { executeProviderTool } from '@/providers/runtime-context' import { createSettledAgentEventStream } from '@/providers/stream-events' import { createStreamingExecution } from '@/providers/streaming-execution' import { isAbortError } from '@/providers/streaming-tool-loop-shared' @@ -49,7 +50,6 @@ import { prepareToolsWithUsageControl, sumToolCosts, } from '@/providers/utils' -import { executeTool } from '@/tools' import type { ExecutionState, GeminiProviderType, GeminiUsage } from './types' /** @@ -133,7 +133,7 @@ async function executeToolCallsBatch( } const { toolParams, executionParams } = prepareToolExecution(tool, args, request) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: request.abortSignal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/providers/gemini/streaming-tool-loop.ts b/apps/sim/providers/gemini/streaming-tool-loop.ts index f3270a5471e..a91ce5ba12d 100644 --- a/apps/sim/providers/gemini/streaming-tool-loop.ts +++ b/apps/sim/providers/gemini/streaming-tool-loop.ts @@ -33,6 +33,7 @@ import { convertUsageMetadata, ensureStructResponse, } from '@/providers/google/utils' +import { executeProviderTool } from '@/providers/runtime-context' import type { AgentStreamEvent, ToolCallEndStatus } from '@/providers/stream-events' import { isAbortError, @@ -44,7 +45,6 @@ import { ensureToolCallId } from '@/providers/tool-call-id' import { enrichLastModelSegment } from '@/providers/trace-enrichment' import type { ModelPricing, ProviderRequest, TimeSegment } from '@/providers/types' import { isGemini3Model, prepareToolExecution, sumToolCosts } from '@/providers/utils' -import { executeTool } from '@/tools' import type { GeminiUsage } from './types' import { priceGeminiTokens, splitGeminiUsage } from './usage' @@ -465,7 +465,7 @@ export function createGeminiStreamingToolLoopStream( toolArgs, request ) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: loopAbortController.signal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/providers/groq/index.ts b/apps/sim/providers/groq/index.ts index 8185c4b3de6..06e5f431be5 100644 --- a/apps/sim/providers/groq/index.ts +++ b/apps/sim/providers/groq/index.ts @@ -13,6 +13,7 @@ import { formatMessagesForProvider } from '@/providers/attachments' import { createReadableStreamFromGroqStream } from '@/providers/groq/utils' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import { createOpenAICompatStreamingToolLoopStream } from '@/providers/openai-compat/streaming-tool-loop' +import { executeProviderTool } from '@/providers/runtime-context' import { createStreamingExecution } from '@/providers/streaming-execution' import { isAbortError, parseToolArguments } from '@/providers/streaming-tool-loop-shared' import { adaptOpenAIChatToolSchema } from '@/providers/tool-schema-adapter' @@ -30,7 +31,6 @@ import { prepareToolsWithUsageControl, trackForcedToolUsage, } from '@/providers/utils' -import { executeTool } from '@/tools' const logger = createLogger('GroqProvider') @@ -343,7 +343,7 @@ export const groqProvider: ProviderConfig = { } const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: request.abortSignal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/providers/index.ts b/apps/sim/providers/index.ts index 9fc2c5593a2..afda36e3f27 100644 --- a/apps/sim/providers/index.ts +++ b/apps/sim/providers/index.ts @@ -16,6 +16,10 @@ import { uploadLargeFilesToProvider, } from '@/providers/file-attachments.server' import { getProviderExecutor } from '@/providers/registry' +import { + type ProviderRuntimeContext, + runWithProviderRuntimeContext, +} from '@/providers/runtime-context' import type { ProviderId, ProviderRequest, ProviderResponse } from '@/providers/types' import { generateStructuredOutputInstructions, @@ -95,7 +99,8 @@ function applyStreamingCostPolicy(response: StreamingExecution, policy: ModelCos export async function executeProviderRequest( providerId: string, - request: ProviderRequest + request: ProviderRequest, + runtimeContext?: ProviderRuntimeContext ): Promise { const provider = await getProviderExecutor(providerId as ProviderId) if (!provider) { @@ -163,7 +168,9 @@ export async function executeProviderRequest( await attachLargeFileRemoteUrls(sanitizedRequest, providerId) await uploadLargeFilesToProvider(sanitizedRequest, providerId) - const response = await provider.executeRequest(sanitizedRequest) + const response = await runWithProviderRuntimeContext(runtimeContext, () => + provider.executeRequest(sanitizedRequest) + ) if (isStreamingExecution(response)) { logger.info('Provider returned StreamingExecution', { isBYOK }) diff --git a/apps/sim/providers/kimi/index.ts b/apps/sim/providers/kimi/index.ts index a211c33a9bf..ead35a5ddfa 100644 --- a/apps/sim/providers/kimi/index.ts +++ b/apps/sim/providers/kimi/index.ts @@ -13,6 +13,7 @@ import { getProviderModels, } from '@/providers/models' import { createOpenAICompatAssistantHistory } from '@/providers/openai-compat/assistant-history' +import { executeProviderTool } from '@/providers/runtime-context' import { createSettledAgentEventStream } from '@/providers/stream-events' import { createStreamingExecution } from '@/providers/streaming-execution' import { isAbortError, parseToolArguments } from '@/providers/streaming-tool-loop-shared' @@ -33,7 +34,6 @@ import { sumToolCosts, trackForcedToolUsage, } from '@/providers/utils' -import { executeTool } from '@/tools' const logger = createLogger('KimiProvider') @@ -337,7 +337,7 @@ export const kimiProvider: ProviderConfig = { } const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: request.abortSignal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/providers/litellm/index.ts b/apps/sim/providers/litellm/index.ts index 0135aac64b2..a77e423b367 100644 --- a/apps/sim/providers/litellm/index.ts +++ b/apps/sim/providers/litellm/index.ts @@ -10,6 +10,7 @@ import { formatMessagesForProvider } from '@/providers/attachments' import { createReadableStreamFromLiteLLMStream } from '@/providers/litellm/utils' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import { createOpenAICompatAssistantHistory } from '@/providers/openai-compat/assistant-history' +import { executeProviderTool } from '@/providers/runtime-context' import { createSettledAgentEventStream } from '@/providers/stream-events' import { createStreamingExecution } from '@/providers/streaming-execution' import { isAbortError, parseToolArguments } from '@/providers/streaming-tool-loop-shared' @@ -32,7 +33,6 @@ import { trackForcedToolUsage, } from '@/providers/utils' import { useProvidersStore } from '@/stores/providers' -import { executeTool } from '@/tools' const logger = createLogger('LiteLLMProvider') const LITELLM_VERSION = '1.0.0' @@ -374,7 +374,7 @@ export const litellmProvider: ProviderConfig = { } const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: request.abortSignal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/providers/meta/index.ts b/apps/sim/providers/meta/index.ts index f4acdd81be0..fc76172a23e 100644 --- a/apps/sim/providers/meta/index.ts +++ b/apps/sim/providers/meta/index.ts @@ -8,6 +8,7 @@ import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' import { createReadableStreamFromMetaStream } from '@/providers/meta/utils' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' +import { executeProviderTool } from '@/providers/runtime-context' import { createSettledAgentEventStream } from '@/providers/stream-events' import { createStreamingExecution } from '@/providers/streaming-execution' import { isAbortError, parseToolArguments } from '@/providers/streaming-tool-loop-shared' @@ -26,7 +27,6 @@ import { prepareToolsWithUsageControl, sumToolCosts, } from '@/providers/utils' -import { executeTool } from '@/tools' const logger = createLogger('MetaProvider') @@ -288,7 +288,7 @@ export const metaProvider: ProviderConfig = { } const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: request.abortSignal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/providers/mistral/index.ts b/apps/sim/providers/mistral/index.ts index ddf625a4596..52453e72487 100644 --- a/apps/sim/providers/mistral/index.ts +++ b/apps/sim/providers/mistral/index.ts @@ -8,6 +8,7 @@ import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' import { createReadableStreamFromMistralStream } from '@/providers/mistral/utils' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' +import { executeProviderTool } from '@/providers/runtime-context' import { createSettledAgentEventStream } from '@/providers/stream-events' import { createStreamingExecution } from '@/providers/streaming-execution' import { isAbortError, parseToolArguments } from '@/providers/streaming-tool-loop-shared' @@ -27,7 +28,6 @@ import { sumToolCosts, trackForcedToolUsage, } from '@/providers/utils' -import { executeTool } from '@/tools' const logger = createLogger('MistralProvider') @@ -300,7 +300,7 @@ export const mistralProvider: ProviderConfig = { } const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: request.abortSignal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/providers/nvidia/index.ts b/apps/sim/providers/nvidia/index.ts index 17e5c631c74..885f6049e81 100644 --- a/apps/sim/providers/nvidia/index.ts +++ b/apps/sim/providers/nvidia/index.ts @@ -9,6 +9,7 @@ import { formatMessagesForProvider } from '@/providers/attachments' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import { createReadableStreamFromNvidiaStream } from '@/providers/nvidia/utils' import { createOpenAICompatAssistantHistory } from '@/providers/openai-compat/assistant-history' +import { executeProviderTool } from '@/providers/runtime-context' import { createSettledAgentEventStream } from '@/providers/stream-events' import { createStreamingExecution } from '@/providers/streaming-execution' import { isAbortError, parseToolArguments } from '@/providers/streaming-tool-loop-shared' @@ -28,7 +29,6 @@ import { sumToolCosts, trackForcedToolUsage, } from '@/providers/utils' -import { executeTool } from '@/tools' const logger = createLogger('NvidiaProvider') @@ -288,7 +288,7 @@ export const nvidiaProvider: ProviderConfig = { } const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: request.abortSignal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/providers/ollama/core.ts b/apps/sim/providers/ollama/core.ts index c3ba7a2a097..064caf79350 100644 --- a/apps/sim/providers/ollama/core.ts +++ b/apps/sim/providers/ollama/core.ts @@ -11,6 +11,7 @@ import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' import { createOpenAICompatAssistantHistory } from '@/providers/openai-compat/assistant-history' +import { executeProviderTool } from '@/providers/runtime-context' import type { AgentStreamEvent } from '@/providers/stream-events' import { createSettledAgentEventStream } from '@/providers/stream-events' import { createStreamingExecution } from '@/providers/streaming-execution' @@ -25,7 +26,6 @@ import { prepareToolExecution, sumToolCosts, } from '@/providers/utils' -import { executeTool } from '@/tools' /** * Ollama enforces JSON mode (`json_object`) but ignores `json_schema`, so @@ -313,7 +313,7 @@ export async function executeOllamaProviderRequest( } const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: request.abortSignal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/providers/openai-compat/streaming-tool-loop.ts b/apps/sim/providers/openai-compat/streaming-tool-loop.ts index 7f766317be7..e14c5d431f4 100644 --- a/apps/sim/providers/openai-compat/streaming-tool-loop.ts +++ b/apps/sim/providers/openai-compat/streaming-tool-loop.ts @@ -22,6 +22,7 @@ import { type OpenAICompatAssembledToolCall, } from '@/providers/openai-compat/stream-events' import type { OpenRouterReasoningDetail } from '@/providers/openrouter/reasoning' +import { executeProviderTool } from '@/providers/runtime-context' import type { AgentStreamEvent, ToolCallEndStatus } from '@/providers/stream-events' import { isAbortError, @@ -38,7 +39,6 @@ import { sumToolCosts, trackForcedToolUsage, } from '@/providers/utils' -import { executeTool } from '@/tools' export type OpenAICompatCreateCompletion = ( params: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming, @@ -428,7 +428,7 @@ export function createOpenAICompatStreamingToolLoopStream( toolArgs, request ) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: loopAbortController.signal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/providers/openai/core.ts b/apps/sim/providers/openai/core.ts index df4ca1b6cd0..47dd9e18b7c 100644 --- a/apps/sim/providers/openai/core.ts +++ b/apps/sim/providers/openai/core.ts @@ -13,6 +13,7 @@ import { buildOpenAIUsageTokens, createOpenAIUsageAccumulator, } from '@/providers/openai/usage' +import { executeProviderTool } from '@/providers/runtime-context' import { createStreamingExecution } from '@/providers/streaming-execution' import { isAbortError, parseToolArguments } from '@/providers/streaming-tool-loop-shared' import { adaptOpenAIChatToolSchema } from '@/providers/tool-schema-adapter' @@ -25,7 +26,6 @@ import { supportsReasoningEffort, trackForcedToolUsage, } from '@/providers/utils' -import { executeTool } from '@/tools' import { buildResponsesInputFromMessages, convertResponseOutputToInputItems, @@ -529,7 +529,7 @@ export async function executeResponsesProviderRequest( } const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: request.abortSignal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/providers/openai/streaming-tool-loop.ts b/apps/sim/providers/openai/streaming-tool-loop.ts index 7f9a75788d7..1715891cc63 100644 --- a/apps/sim/providers/openai/streaming-tool-loop.ts +++ b/apps/sim/providers/openai/streaming-tool-loop.ts @@ -22,6 +22,7 @@ import { type ResponsesToolChoice, responseContainsFunctionCall, } from '@/providers/openai/utils' +import { executeProviderTool } from '@/providers/runtime-context' import type { AgentStreamEvent, ToolCallEndStatus } from '@/providers/stream-events' import { isAbortError, @@ -32,7 +33,6 @@ import { } from '@/providers/streaming-tool-loop-shared' import type { ProviderRequest, TimeSegment } from '@/providers/types' import { prepareToolExecution, sumToolCosts } from '@/providers/utils' -import { executeTool } from '@/tools' export type CreateOpenAIResponsesStream = ( input: ResponsesInputItem[], @@ -245,7 +245,7 @@ async function executeOpenAIToolCall(options: { } const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) - const result = await executeTool(toolCall.name, executionParams, { + const result = await executeProviderTool(toolCall.name, executionParams, { signal: request.abortSignal, }) return completeToolExecution( diff --git a/apps/sim/providers/openrouter/index.ts b/apps/sim/providers/openrouter/index.ts index 41146d1599c..e2949fd9dfb 100644 --- a/apps/sim/providers/openrouter/index.ts +++ b/apps/sim/providers/openrouter/index.ts @@ -20,6 +20,7 @@ import { createReadableStreamFromOpenAIStream, supportsNativeStructuredOutputs, } from '@/providers/openrouter/utils' +import { executeProviderTool } from '@/providers/runtime-context' import { createSettledAgentEventStream } from '@/providers/stream-events' import { createStreamingExecution } from '@/providers/streaming-execution' import { isAbortError, parseToolArguments } from '@/providers/streaming-tool-loop-shared' @@ -41,7 +42,6 @@ import { prepareToolsWithUsageControl, sumToolCosts, } from '@/providers/utils' -import { executeTool } from '@/tools' const logger = createLogger('OpenRouterProvider') @@ -303,7 +303,7 @@ export const openRouterProvider: ProviderConfig = { } const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: request.abortSignal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/providers/runtime-context.test.ts b/apps/sim/providers/runtime-context.test.ts new file mode 100644 index 00000000000..47dbbb9ef54 --- /dev/null +++ b/apps/sim/providers/runtime-context.test.ts @@ -0,0 +1,76 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockExecuteTool } = vi.hoisted(() => ({ + mockExecuteTool: vi.fn(async () => ({ success: true, output: {} })), +})) + +vi.mock('@/tools', () => ({ + executeTool: mockExecuteTool, +})) + +import { executeProviderTool, runWithProviderRuntimeContext } from '@/providers/runtime-context' + +describe('provider runtime context', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('isolates concurrent tool executions without adding registry data to params', async () => { + const registryA = { id: 'a' } + const registryB = { id: 'b' } + + await Promise.all([ + runWithProviderRuntimeContext( + { resolvedSecretTraceRegistry: registryA as never }, + async () => { + await Promise.resolve() + await executeProviderTool('tool-a', { visible: 'a' }) + } + ), + runWithProviderRuntimeContext( + { resolvedSecretTraceRegistry: registryB as never }, + async () => { + await Promise.resolve() + await executeProviderTool('tool-b', { visible: 'b' }) + } + ), + ]) + + expect(mockExecuteTool).toHaveBeenCalledWith( + 'tool-a', + { visible: 'a' }, + { resolvedSecretTraceRegistry: registryA } + ) + expect(mockExecuteTool).toHaveBeenCalledWith( + 'tool-b', + { visible: 'b' }, + { resolvedSecretTraceRegistry: registryB } + ) + }) + + it('preserves runtime context in a stream consumed after the provider call returns', async () => { + const registry = { id: 'stream' } + const stream = runWithProviderRuntimeContext( + { resolvedSecretTraceRegistry: registry as never }, + () => + new ReadableStream({ + async pull(controller) { + await executeProviderTool('stream-tool', { visible: true }) + controller.enqueue(new TextEncoder().encode('done')) + controller.close() + }, + }) + ) + + await new Response(stream).text() + + expect(mockExecuteTool).toHaveBeenCalledWith( + 'stream-tool', + { visible: true }, + { resolvedSecretTraceRegistry: registry } + ) + }) +}) diff --git a/apps/sim/providers/runtime-context.ts b/apps/sim/providers/runtime-context.ts new file mode 100644 index 00000000000..330caa3e34b --- /dev/null +++ b/apps/sim/providers/runtime-context.ts @@ -0,0 +1,30 @@ +import { AsyncLocalStorage } from 'node:async_hooks' +import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' +import { type ExecuteToolOptions, executeTool } from '@/tools' +import type { ToolResponse } from '@/tools/types' + +export interface ProviderRuntimeContext { + resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry +} + +const providerRuntimeContext = new AsyncLocalStorage() + +export function runWithProviderRuntimeContext( + context: ProviderRuntimeContext | undefined, + callback: () => T +): T { + return providerRuntimeContext.run(context ?? {}, callback) +} + +export function executeProviderTool( + toolId: string, + params: Parameters[1], + options: ExecuteToolOptions = {} +): Promise { + return executeTool(toolId, params, { + ...options, + resolvedSecretTraceRegistry: + options.resolvedSecretTraceRegistry ?? + providerRuntimeContext.getStore()?.resolvedSecretTraceRegistry, + }) +} diff --git a/apps/sim/providers/sakana/index.ts b/apps/sim/providers/sakana/index.ts index 7e9cf12da79..d005e4d0fef 100644 --- a/apps/sim/providers/sakana/index.ts +++ b/apps/sim/providers/sakana/index.ts @@ -7,6 +7,7 @@ import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' +import { executeProviderTool } from '@/providers/runtime-context' import { createReadableStreamFromSakanaStream } from '@/providers/sakana/utils' import { createSettledAgentEventStream } from '@/providers/stream-events' import { createStreamingExecution } from '@/providers/streaming-execution' @@ -27,7 +28,6 @@ import { sumToolCosts, trackForcedToolUsage, } from '@/providers/utils' -import { executeTool } from '@/tools' const logger = createLogger('SakanaProvider') @@ -288,7 +288,7 @@ export const sakanaProvider: ProviderConfig = { } const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: request.abortSignal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/providers/together/index.ts b/apps/sim/providers/together/index.ts index d8f67bb1435..615a66f967e 100644 --- a/apps/sim/providers/together/index.ts +++ b/apps/sim/providers/together/index.ts @@ -8,6 +8,7 @@ import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import { createOpenAICompatAssistantHistory } from '@/providers/openai-compat/assistant-history' +import { executeProviderTool } from '@/providers/runtime-context' import { createSettledAgentEventStream } from '@/providers/stream-events' import { createStreamingExecution } from '@/providers/streaming-execution' import { isAbortError, parseToolArguments } from '@/providers/streaming-tool-loop-shared' @@ -34,7 +35,6 @@ import { prepareToolsWithUsageControl, sumToolCosts, } from '@/providers/utils' -import { executeTool } from '@/tools' const logger = createLogger('TogetherProvider') @@ -288,7 +288,7 @@ export const togetherProvider: ProviderConfig = { } const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: request.abortSignal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/providers/vllm/index.ts b/apps/sim/providers/vllm/index.ts index 0f258b8bb22..4d2928266f8 100644 --- a/apps/sim/providers/vllm/index.ts +++ b/apps/sim/providers/vllm/index.ts @@ -11,6 +11,7 @@ import { formatMessagesForProvider } from '@/providers/attachments' import { getCachedProviderClient } from '@/providers/client-cache' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import { createOpenAICompatAssistantHistory } from '@/providers/openai-compat/assistant-history' +import { executeProviderTool } from '@/providers/runtime-context' import { createSettledAgentEventStream } from '@/providers/stream-events' import { createStreamingExecution } from '@/providers/streaming-execution' import { isAbortError, parseToolArguments } from '@/providers/streaming-tool-loop-shared' @@ -32,7 +33,6 @@ import { } from '@/providers/utils' import { checkForForcedToolUsage, createReadableStreamFromVLLMStream } from '@/providers/vllm/utils' import { useProvidersStore } from '@/stores/providers' -import { executeTool } from '@/tools' const logger = createLogger('VLLMProvider') const VLLM_VERSION = '1.0.0' @@ -384,7 +384,7 @@ export const vllmProvider: ProviderConfig = { } const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: request.abortSignal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/providers/xai/index.ts b/apps/sim/providers/xai/index.ts index 55c6ecdaecb..292f6111c4a 100644 --- a/apps/sim/providers/xai/index.ts +++ b/apps/sim/providers/xai/index.ts @@ -8,6 +8,7 @@ import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import { createOpenAICompatAssistantHistory } from '@/providers/openai-compat/assistant-history' +import { executeProviderTool } from '@/providers/runtime-context' import { createSettledAgentEventStream } from '@/providers/stream-events' import { createStreamingExecution } from '@/providers/streaming-execution' import { isAbortError, parseToolArguments } from '@/providers/streaming-tool-loop-shared' @@ -32,7 +33,6 @@ import { createReadableStreamFromXAIStream, createResponseFormatPayload, } from '@/providers/xai/utils' -import { executeTool } from '@/tools' const logger = createLogger('XAIProvider') @@ -271,7 +271,7 @@ export const xAIProvider: ProviderConfig = { } const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: request.abortSignal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/providers/zai/index.ts b/apps/sim/providers/zai/index.ts index 39358ef774a..acc5bb4ec36 100644 --- a/apps/sim/providers/zai/index.ts +++ b/apps/sim/providers/zai/index.ts @@ -8,6 +8,7 @@ import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import { createOpenAICompatAssistantHistory } from '@/providers/openai-compat/assistant-history' +import { executeProviderTool } from '@/providers/runtime-context' import { createSettledAgentEventStream } from '@/providers/stream-events' import { createStreamingExecution } from '@/providers/streaming-execution' import { isAbortError, parseToolArguments } from '@/providers/streaming-tool-loop-shared' @@ -27,7 +28,6 @@ import { sumToolCosts, } from '@/providers/utils' import { createReadableStreamFromZaiStream } from '@/providers/zai/utils' -import { executeTool } from '@/tools' const logger = createLogger('ZaiProvider') @@ -297,7 +297,7 @@ export const zaiProvider: ProviderConfig = { } const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) - const result = await executeTool(toolName, executionParams, { + const result = await executeProviderTool(toolName, executionParams, { signal: request.abortSignal, }) const toolCallEndTime = Date.now() diff --git a/apps/sim/stores/terminal/console/storage.test.ts b/apps/sim/stores/terminal/console/storage.test.ts new file mode 100644 index 00000000000..b047430366c --- /dev/null +++ b/apps/sim/stores/terminal/console/storage.test.ts @@ -0,0 +1,87 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + CONSOLE_STORAGE_VERSION, + migratePersistedConsoleData, +} from '@/stores/terminal/console/storage' + +const legacyEntry = { + id: 'entry-1', + timestamp: '2026-07-31T00:00:00.000Z', + workflowId: 'workflow-1', + blockId: 'function-1', + blockName: 'Function 1', + blockType: 'function', + executionId: 'execution-1', + executionOrder: 1, + success: false, + input: { secret: 'resolved-secret' }, + output: { value: 'resolved-secret' }, + error: 'ReferenceError: resolved-secret', + warning: 'resolved-secret', + agentStreamThinking: 'resolved-secret', + agentStreamToolCalls: [ + { + key: 'function-1:tool-1', + id: 'tool-1', + name: 'database_query', + displayName: 'Database Query', + status: 'error' as const, + }, + ], +} + +describe('terminal console storage migration', () => { + it('strips content-bearing fields from unversioned persisted entries', () => { + const result = migratePersistedConsoleData({ + workflowEntries: { 'workflow-1': [legacyEntry] }, + isOpen: true, + }) + + expect(result?.migrated).toBe(true) + expect(result?.data.storageVersion).toBe(CONSOLE_STORAGE_VERSION) + expect(result?.data.workflowEntries['workflow-1'][0]).toEqual({ + id: 'entry-1', + timestamp: '2026-07-31T00:00:00.000Z', + workflowId: 'workflow-1', + blockId: 'function-1', + blockName: 'Function 1', + blockType: 'function', + executionId: 'execution-1', + executionOrder: 1, + success: false, + agentStreamToolCalls: legacyEntry.agentStreamToolCalls, + }) + }) + + it('strips content from the original flat Zustand storage format', () => { + const result = migratePersistedConsoleData({ + state: { entries: [legacyEntry], isOpen: false }, + version: 0, + }) + + expect(result?.migrated).toBe(true) + expect(result?.data.workflowEntries['workflow-1'][0]).not.toHaveProperty('error') + expect(result?.data.workflowEntries['workflow-1'][0]).not.toHaveProperty('agentStreamThinking') + }) + + it('retains content already written under the current projection version', () => { + const projectedEntry = { + ...legacyEntry, + input: { secret: '{{OPENAI_API_KEY}}' }, + output: { value: '{{OPENAI_API_KEY}}' }, + error: 'ReferenceError: {{OPENAI_API_KEY}}', + agentStreamThinking: '{{OPENAI_API_KEY}}', + } + const result = migratePersistedConsoleData({ + storageVersion: CONSOLE_STORAGE_VERSION, + workflowEntries: { 'workflow-1': [projectedEntry] }, + isOpen: false, + }) + + expect(result?.migrated).toBe(false) + expect(result?.data.workflowEntries['workflow-1'][0]).toEqual(projectedEntry) + }) +}) diff --git a/apps/sim/stores/terminal/console/storage.ts b/apps/sim/stores/terminal/console/storage.ts index 4ee136db81b..df5ab1b46fc 100644 --- a/apps/sim/stores/terminal/console/storage.ts +++ b/apps/sim/stores/terminal/console/storage.ts @@ -1,11 +1,13 @@ import { createLogger } from '@sim/logger' +import { omit } from '@sim/utils/object' import { get, set } from 'idb-keyval' -import type { ConsoleEntry } from './types' +import type { ConsoleEntry } from '@/stores/terminal/console/types' const logger = createLogger('ConsoleStorage') const STORE_KEY = 'terminal-console-store' const MIGRATION_KEY = 'terminal-console-store-migrated' +export const CONSOLE_STORAGE_VERSION = 1 /** * Interval for persisting terminal state during active executions. @@ -18,10 +20,16 @@ const EXECUTION_PERSIST_INTERVAL_MS = 5_000 * Shape of terminal console data persisted to IndexedDB. */ export interface PersistedConsoleData { + storageVersion: typeof CONSOLE_STORAGE_VERSION workflowEntries: Record isOpen: boolean } +export interface ConsoleStorageMigrationResult { + data: PersistedConsoleData + migrated: boolean +} + let migrationPromise: Promise | null = null async function migrateFromLocalStorage(): Promise { @@ -50,12 +58,63 @@ if (typeof window !== 'undefined') { }) } +function stripPersistedContent(entry: ConsoleEntry): ConsoleEntry { + return omit(entry, ['input', 'output', 'error', 'warning', 'agentStreamThinking']) +} + +function stripPersistedWorkflowContent( + workflowEntries: Record +): Record { + return Object.fromEntries( + Object.entries(workflowEntries).map(([workflowId, entries]) => [ + workflowId, + entries.map(stripPersistedContent), + ]) + ) +} + +/** + * Normalizes all historical console formats and removes content from unversioned data. + * Historical rows have no trusted Secrets-resolution provenance, so their structure is + * retained while input, output, errors, warnings, and live thinking are discarded. + */ +export function migratePersistedConsoleData(parsed: unknown): ConsoleStorageMigrationResult | null { + if (!parsed || typeof parsed !== 'object') return null + + const parsedRecord = parsed as Record + const wrappedState = parsedRecord.state + const data = + wrappedState && typeof wrappedState === 'object' + ? (wrappedState as Record) + : parsedRecord + + let workflowEntries: Record = {} + if (Array.isArray(data.entries) && !data.workflowEntries) { + for (const rawEntry of data.entries) { + if (!rawEntry || typeof rawEntry !== 'object') continue + const entry = rawEntry as ConsoleEntry + if (!entry.workflowId) continue + if (!workflowEntries[entry.workflowId]) workflowEntries[entry.workflowId] = [] + workflowEntries[entry.workflowId].push(entry) + } + } else if (data.workflowEntries && typeof data.workflowEntries === 'object') { + workflowEntries = data.workflowEntries as Record + } + + const migrated = data.storageVersion !== CONSOLE_STORAGE_VERSION + return { + data: { + storageVersion: CONSOLE_STORAGE_VERSION, + workflowEntries: migrated ? stripPersistedWorkflowContent(workflowEntries) : workflowEntries, + isOpen: Boolean(data.isOpen), + }, + migrated, + } +} + /** * Loads persisted console data from IndexedDB. - * Handles three historical storage formats: - * 1. Zustand persist wrapper: `{ state: { entries: [...] }, version }` (original flat format) - * 2. Zustand persist wrapper: `{ state: { workflowEntries: {...} }, version }` (refactored format) - * 3. Raw data: `{ workflowEntries: {...}, isOpen }` (current format) + * Handles historical Zustand wrappers, the original flat entry array, and raw data. */ export async function loadConsoleData(): Promise { if (typeof window === 'undefined') return null @@ -69,25 +128,14 @@ export async function loadConsoleData(): Promise { if (!raw) return null const parsed = typeof raw === 'string' ? JSON.parse(raw) : raw - if (!parsed || typeof parsed !== 'object') return null - - const data = parsed.state ?? parsed - - if (Array.isArray(data.entries) && !data.workflowEntries) { - const workflowEntries: Record = {} - for (const entry of data.entries) { - if (!entry?.workflowId) continue - const wfId = entry.workflowId - if (!workflowEntries[wfId]) workflowEntries[wfId] = [] - workflowEntries[wfId].push(entry) - } - return { workflowEntries, isOpen: Boolean(data.isOpen) } - } + const result = migratePersistedConsoleData(parsed) + if (!result) return null - return { - workflowEntries: data.workflowEntries ?? {}, - isOpen: Boolean(data.isOpen), + if (result.migrated) { + await set(STORE_KEY, JSON.stringify(result.data)) } + + return result.data } catch (error) { logger.warn('Failed to load console data from IndexedDB', { error }) return null @@ -159,6 +207,7 @@ function mergePersistedConsoleData( } return { + storageVersion: CONSOLE_STORAGE_VERSION, workflowEntries, isOpen: incoming.isOpen, } diff --git a/apps/sim/stores/terminal/console/store.test.ts b/apps/sim/stores/terminal/console/store.test.ts index 4ce6ced4a54..c20ed28b17c 100644 --- a/apps/sim/stores/terminal/console/store.test.ts +++ b/apps/sim/stores/terminal/console/store.test.ts @@ -163,6 +163,7 @@ describe('terminal console store', () => { const [entry] = useTerminalConsoleStore.getState().getWorkflowEntries('wf-1') expect(entry.agentStreamActive).toBe(false) + expect(entry.agentStreamThinking).toBe('drafting…') expect(entry.agentStreamToolCalls?.[0]?.status).toBe('cancelled') }) @@ -284,7 +285,31 @@ describe('terminal console store', () => { const [entry] = useTerminalConsoleStore.getState().getWorkflowEntries('wf-1') expect(entry.agentStreamActive).toBe(false) + expect(entry.agentStreamThinking).toBe('working…') expect(entry.agentStreamToolCalls?.[0]?.status).toBe('error') }) + + it('clears thinking without changing an active block when projection is unavailable', () => { + useTerminalConsoleStore.getState().addConsole({ + workflowId: 'wf-1', + blockId: 'block-1', + blockName: 'Agent', + blockType: 'agent', + executionId: 'exec-1', + executionOrder: 1, + isRunning: true, + agentStreamActive: true, + agentStreamThinking: 'projected thinking', + }) + + useTerminalConsoleStore + .getState() + .updateConsole('block-1', { clearAgentStreamThinking: true }, 'exec-1') + + const [entry] = useTerminalConsoleStore.getState().getWorkflowEntries('wf-1') + expect(entry.isRunning).toBe(true) + expect(entry.agentStreamActive).toBe(true) + expect(entry.agentStreamThinking).toBeUndefined() + }) }) }) diff --git a/apps/sim/stores/terminal/console/store.ts b/apps/sim/stores/terminal/console/store.ts index 101fc8cc9d5..5fce1cec952 100644 --- a/apps/sim/stores/terminal/console/store.ts +++ b/apps/sim/stores/terminal/console/store.ts @@ -15,7 +15,11 @@ import { getQueryClient } from '@/app/_shell/providers/query-provider' import type { NormalizedBlockOutput } from '@/executor/types' import { type GeneralSettings, generalSettingsKeys } from '@/hooks/queries/general-settings' import { useExecutionStore } from '@/stores/execution' -import { consolePersistence, loadConsoleData } from '@/stores/terminal/console/storage' +import { + CONSOLE_STORAGE_VERSION, + consolePersistence, + loadConsoleData, +} from '@/stores/terminal/console/storage' import type { ConsoleEntry, ConsoleEntryLocation, @@ -644,6 +648,10 @@ export const useTerminalConsoleStore = create()( updatedEntry.agentStreamThinking = update.agentStreamThinking } + if (update.clearAgentStreamThinking) { + updatedEntry.agentStreamThinking = undefined + } + if (update.agentStreamToolCalls !== undefined) { updatedEntry.agentStreamToolCalls = update.agentStreamToolCalls } @@ -852,6 +860,7 @@ if (typeof window !== 'undefined') { consolePersistence.bind(() => { const state = useTerminalConsoleStore.getState() return { + storageVersion: CONSOLE_STORAGE_VERSION, workflowEntries: state.workflowEntries, isOpen: state.isOpen, } diff --git a/apps/sim/stores/terminal/console/types.ts b/apps/sim/stores/terminal/console/types.ts index 08bdafb514b..e3daa33f0e6 100644 --- a/apps/sim/stores/terminal/console/types.ts +++ b/apps/sim/stores/terminal/console/types.ts @@ -66,6 +66,7 @@ export interface ConsoleUpdate { childWorkflowName?: string childWorkflowInstanceId?: string agentStreamThinking?: string + clearAgentStreamThinking?: boolean agentStreamToolCalls?: AgentStreamToolCall[] agentStreamActive?: boolean } diff --git a/apps/sim/tools/index.test.ts b/apps/sim/tools/index.test.ts index b50f66cfc81..d7578e62c5e 100644 --- a/apps/sim/tools/index.test.ts +++ b/apps/sim/tools/index.test.ts @@ -11,6 +11,7 @@ import { createExecutionContext, createMockFetch, type ExecutionContext, + encryptionMockFns, environmentUtilsMockFns, inputValidationMock, inputValidationMockFns, @@ -25,6 +26,11 @@ import { import { sleep } from '@sim/utils/helpers' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' +import { + ANONYMOUS_SECRET_TRACE_REPLACEMENT, + ResolvedSecretTraceRegistry, +} from '@/executor/utils/resolved-secret-trace-registry' +import { workflowExecutorTool } from '@/tools/workflow/executor' // Hoisted mock state - these are available to vi.mock factories const { @@ -64,6 +70,11 @@ vi.mock('@/lib/auth/internal', () => ({ generateInternalToken: (...args: unknown[]) => mockGenerateInternalToken(...args), })) +vi.mock('@/lib/core/security/encryption', () => ({ + decryptSecret: encryptionMockFns.mockDecryptSecret, + encryptSecret: encryptionMockFns.mockEncryptSecret, +})) + vi.mock('@/ee/access-control/utils/permission-check', () => ({ assertPermissionsAllowed: vi.fn().mockResolvedValue(undefined), validateBlockType: vi.fn().mockResolvedValue(undefined), @@ -98,6 +109,7 @@ vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ // Mock the tools registry to avoid loading the full 4500+ line registry file. // Only the tools actually exercised in tests are provided. const mockRegistryTools: Record = { + workflow_executor: workflowExecutorTool, http_request: { id: 'http_request', name: 'HTTP Request', @@ -647,6 +659,215 @@ describe('executeTool Function', () => { ) }) + it('consumes Function secret provenance without exposing private transport metadata', async () => { + const registry = new ResolvedSecretTraceRegistry([ + { + name: 'API_KEY', + plaintext: 'secret-value', + encryptedValue: 'encrypted-value', + }, + ]) + global.fetch = Object.assign( + vi.fn().mockResolvedValue( + new Response( + JSON.stringify({ + success: true, + output: { result: 'secret-value', stdout: '' }, + __resolvedSecretNames: ['API_KEY'], + }), + { + status: 200, + headers: { + 'content-type': 'application/json', + 'x-sim-private-tool-metadata': 'resolved-secret-names-v1', + }, + } + ) + ), + { preconnect: vi.fn() } + ) as typeof fetch + + const result = await executeTool( + 'function_execute', + { + code: 'return {{API_KEY}}', + envVars: { API_KEY: 'secret-value' }, + }, + { resolvedSecretTraceRegistry: registry } + ) + + const [, requestInit] = vi.mocked(global.fetch).mock.calls[0] + expect(new Headers(requestInit?.headers).get('x-sim-request-private-tool-metadata')).toBe( + 'resolved-secret-names-v1' + ) + expect(result.output).not.toHaveProperty('__resolvedSecretNames') + expect(registry.getActiveMatches()).toEqual([ + { plaintext: 'secret-value', replacement: '{{API_KEY}}' }, + ]) + }) + + it('keeps the Function result unchanged when requested provenance is missing', async () => { + const registry = new ResolvedSecretTraceRegistry() + global.fetch = Object.assign( + vi + .fn() + .mockResolvedValue( + new Response( + JSON.stringify({ success: true, output: { result: 'unchanged', stdout: '' } }), + { status: 200, headers: { 'content-type': 'application/json' } } + ) + ), + { preconnect: vi.fn() } + ) as typeof fetch + + const result = await executeTool( + 'function_execute', + { code: 'return "unchanged"', envVars: {} }, + { resolvedSecretTraceRegistry: registry } + ) + + expect(result.success).toBe(true) + expect(result.output).toEqual({ + success: true, + output: { result: 'unchanged', stdout: '' }, + }) + expect(registry.isComplete()).toBe(false) + }) + + it('filters cross-scope workflow provenance to literals present in the unchanged result', async () => { + const registry = new ResolvedSecretTraceRegistry([], { + userId: 'parent-user', + workspaceId: 'workspace-456', + }) + encryptionMockFns.mockDecryptSecret.mockImplementation(async (encryptedValue: string) => ({ + decrypted: encryptedValue === 'crossed-encrypted' ? 'crossed-secret' : 'unrelated-secret', + })) + const childOutput = { + answer: 'The child returned crossed-secret verbatim', + publicValue: 'unchanged', + } + global.fetch = Object.assign( + vi.fn().mockResolvedValue( + new Response( + JSON.stringify({ + success: true, + workflowId: 'child-workflow', + workflowName: 'Child Workflow', + output: childOutput, + metadata: { duration: 17 }, + __resolvedSecretTraceProvenance: { + version: 1, + complete: true, + entries: [ + { name: 'CROSSED', encryptedValue: 'crossed-encrypted' }, + { name: 'UNRELATED', encryptedValue: 'unrelated-encrypted' }, + ], + scope: { userId: 'parent-user', workspaceId: 'child-workspace' }, + }, + }), + { + status: 200, + headers: { + 'content-type': 'application/json', + 'x-sim-private-tool-metadata': 'resolved-secret-provenance-v1', + }, + } + ) + ), + { preconnect: vi.fn() } + ) as typeof fetch + + const result = await executeTool( + 'workflow_executor_child-workflow', + { workflowId: 'child-workflow', inputMapping: {} }, + { + executionContext: createToolExecutionContext({ userId: 'parent-user' }), + resolvedSecretTraceRegistry: registry, + } + ) + + expect(result).toEqual({ + success: true, + duration: 17, + childWorkflowId: 'child-workflow', + childWorkflowName: 'Child Workflow', + output: childOutput, + result: childOutput, + error: undefined, + timing: { + startTime: expect.any(String), + endTime: expect.any(String), + duration: expect.any(Number), + }, + }) + expect(JSON.stringify(result)).not.toContain('__resolvedSecretTraceProvenance') + expect(registry.getActiveMatches()).toEqual([ + { + plaintext: 'crossed-secret', + replacement: ANONYMOUS_SECRET_TRACE_REPLACEMENT, + }, + ]) + expect(registry.isComplete()).toBe(true) + }) + + it('strips private metadata even when an internal endpoint returns the wrong marker', async () => { + const registry = new ResolvedSecretTraceRegistry() + global.fetch = Object.assign( + vi.fn().mockResolvedValue( + new Response( + JSON.stringify({ + success: true, + output: { result: 'unchanged' }, + __resolvedSecretTraceProvenance: { version: 1, complete: true, entries: [] }, + }), + { + status: 200, + headers: { + 'content-type': 'application/json', + 'x-sim-private-tool-metadata': 'resolved-secret-provenance-v1', + }, + } + ) + ), + { preconnect: vi.fn() } + ) as typeof fetch + + const result = await executeTool( + 'function_execute', + { code: 'return "unchanged"', envVars: {} }, + { resolvedSecretTraceRegistry: registry } + ) + + expect(JSON.stringify(result)).not.toContain('__resolvedSecretTraceProvenance') + expect(registry.isComplete()).toBe(false) + }) + + it('fails closed when a marked private response envelope is malformed', async () => { + const registry = new ResolvedSecretTraceRegistry() + global.fetch = Object.assign( + vi.fn().mockResolvedValue( + new Response('{"__resolvedSecretNames":["API_KEY"],"value":"secret-value"', { + status: 500, + headers: { + 'content-type': 'application/json', + 'x-sim-private-tool-metadata': 'resolved-secret-names-v1', + }, + }) + ), + { preconnect: vi.fn() } + ) as typeof fetch + + const result = await executeTool( + 'function_execute', + { code: 'return "unchanged"', envVars: { API_KEY: 'secret-value' } }, + { resolvedSecretTraceRegistry: registry } + ) + + expect(JSON.stringify(result)).not.toContain('secret-value') + expect(JSON.stringify(result)).not.toContain('__resolvedSecretNames') + expect(registry.isComplete()).toBe(false) + }) + it('should handle non-existent tool', async () => { vi.spyOn(console, 'error').mockImplementation(() => {}) @@ -1914,6 +2135,180 @@ describe('MCP Tool Execution', () => { expect(result.timing).toBeDefined() }) + it('consumes marker-gated MCP provenance while leaving the tool result unchanged', async () => { + const registry = new ResolvedSecretTraceRegistry([], { + userId: 'test-user', + workspaceId: 'workspace-456', + }) + const importProvenance = vi.spyOn(registry, 'importProvenance') + encryptionMockFns.mockDecryptSecret.mockResolvedValueOnce({ decrypted: 'secret-value' }) + global.fetch = Object.assign( + vi.fn().mockResolvedValue( + new Response( + JSON.stringify({ + success: true, + data: { output: { content: [{ type: 'text', text: 'secret-value' }] } }, + __resolvedSecretTraceProvenance: { + version: 1, + complete: true, + entries: [{ name: 'MCP_TOKEN', encryptedValue: 'encrypted-token' }], + scope: { userId: 'test-user', workspaceId: 'workspace-456' }, + }, + }), + { + status: 200, + headers: { + 'content-type': 'application/json', + 'x-sim-private-tool-metadata': 'resolved-secret-provenance-v1', + }, + } + ) + ), + { preconnect: vi.fn() } + ) as typeof fetch + + const result = await executeTool( + 'mcp-123-list_files', + { path: '/test' }, + { + executionContext: createToolExecutionContext(), + resolvedSecretTraceRegistry: registry, + } + ) + + const [, requestInit] = vi.mocked(global.fetch).mock.calls[0] + expect(new Headers(requestInit?.headers).get('x-sim-request-private-tool-metadata')).toBe( + 'resolved-secret-provenance-v1' + ) + expect(result.output).toEqual({ content: [{ type: 'text', text: 'secret-value' }] }) + expect(JSON.stringify(result)).not.toContain('__resolvedSecretTraceProvenance') + expect(importProvenance).toHaveBeenCalledTimes(1) + expect(importProvenance).toHaveBeenCalledWith( + { + version: 1, + complete: true, + entries: [{ name: 'MCP_TOKEN', encryptedValue: 'encrypted-token' }], + scope: { userId: 'test-user', workspaceId: 'workspace-456' }, + }, + { trusted: true } + ) + expect(encryptionMockFns.mockDecryptSecret).toHaveBeenCalledWith('encrypted-token') + expect(registry.isComplete()).toBe(true) + expect(registry.getActiveMatches()).toEqual([ + { plaintext: 'secret-value', replacement: '{{MCP_TOKEN}}' }, + ]) + }) + + it('rejects unmarked MCP provenance instead of trusting a response body field', async () => { + const registry = new ResolvedSecretTraceRegistry() + global.fetch = Object.assign( + vi.fn().mockResolvedValue( + new Response( + JSON.stringify({ + success: true, + data: { output: { content: [{ type: 'text', text: 'unchanged' }] } }, + __resolvedSecretTraceProvenance: { + version: 1, + complete: true, + entries: [{ name: 'MCP_TOKEN', encryptedValue: 'untrusted-encrypted-token' }], + }, + }), + { status: 200, headers: { 'content-type': 'application/json' } } + ) + ), + { preconnect: vi.fn() } + ) as typeof fetch + + const result = await executeTool( + 'mcp-123-list_files', + { path: '/test' }, + { + executionContext: createToolExecutionContext(), + resolvedSecretTraceRegistry: registry, + } + ) + + expect(result.success).toBe(true) + expect(JSON.stringify(result)).not.toContain('__resolvedSecretTraceProvenance') + expect(registry.getActiveMatches()).toEqual([]) + expect(registry.isComplete()).toBe(false) + }) + + it('preserves MCP error semantics while stripping marked private provenance', async () => { + const registry = new ResolvedSecretTraceRegistry() + global.fetch = Object.assign( + vi.fn().mockResolvedValue( + new Response( + JSON.stringify({ + error: 'provider error detail', + __resolvedSecretTraceProvenance: { version: 1, complete: true, entries: [] }, + }), + { + status: 500, + headers: { + 'content-type': 'application/json', + 'x-sim-private-tool-metadata': 'resolved-secret-provenance-v1', + }, + } + ) + ), + { preconnect: vi.fn() } + ) as typeof fetch + + const result = await executeTool( + 'mcp-123-list_files', + { path: '/test' }, + { + executionContext: createToolExecutionContext(), + resolvedSecretTraceRegistry: registry, + } + ) + + expect(result).toMatchObject({ success: false, error: 'provider error detail' }) + expect(JSON.stringify(result)).not.toContain('__resolvedSecretTraceProvenance') + expect(registry.isComplete()).toBe(true) + }) + + it('accepts MCP responses above the generic tool cap while stripping private metadata', async () => { + const registry = new ResolvedSecretTraceRegistry() + global.fetch = Object.assign( + vi.fn().mockResolvedValue( + new Response( + JSON.stringify({ + success: true, + data: { output: { content: [{ type: 'text', text: 'unchanged' }] } }, + __resolvedSecretTraceProvenance: { version: 1, complete: true, entries: [] }, + }), + { + status: 200, + headers: { + 'content-length': String(10 * 1024 * 1024 + 1), + 'content-type': 'application/json', + 'x-sim-private-tool-metadata': 'resolved-secret-provenance-v1', + }, + } + ) + ), + { preconnect: vi.fn() } + ) as typeof fetch + + const result = await executeTool( + 'mcp-123-list_files', + { path: '/test' }, + { + executionContext: createToolExecutionContext(), + resolvedSecretTraceRegistry: registry, + } + ) + + expect(result).toMatchObject({ + success: true, + output: { content: [{ type: 'text', text: 'unchanged' }] }, + }) + expect(JSON.stringify(result)).not.toContain('__resolvedSecretTraceProvenance') + expect(registry.isComplete()).toBe(true) + }) + it('should handle MCP tool ID parsing correctly', async () => { global.fetch = Object.assign( vi.fn().mockImplementation(async (url, options) => { diff --git a/apps/sim/tools/index.ts b/apps/sim/tools/index.ts index 887bbc21416..552841e6c9a 100644 --- a/apps/sim/tools/index.ts +++ b/apps/sim/tools/index.ts @@ -28,6 +28,17 @@ import { getBaseUrl, getInternalApiBaseUrl } from '@/lib/core/utils/urls' import { isUserFile } from '@/lib/core/utils/user-file' import { isSameOrigin } from '@/lib/core/utils/validation' import { SIM_VIA_HEADER, serializeCallChain } from '@/lib/execution/call-chain' +import { + getPrivateToolMetadataField, + isPrivateToolMetadataType, + PRIVATE_TOOL_METADATA_REQUEST_HEADER, + PRIVATE_TOOL_METADATA_RESPONSE_HEADER, + type PrivateToolMetadataType, + RESOLVED_SECRET_NAMES_FIELD, + RESOLVED_SECRET_NAMES_METADATA_V1, + RESOLVED_SECRET_PROVENANCE_FIELD, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, +} from '@/lib/execution/private-tool-metadata' import { parseMcpToolId } from '@/lib/mcp/utils' import { hostedKeyMetrics } from '@/lib/monitoring/metrics' import { resolveWorkspaceFileReference } from '@/lib/uploads/contexts/workspace/workspace-file-manager' @@ -36,6 +47,7 @@ import { isCustomTool, isMcpTool } from '@/executor/constants' import { resolveSkillContent } from '@/executor/handlers/agent/skills-resolver' import type { ExecutionContext, UserFile } from '@/executor/types' import { resolveEnvVarReferences } from '@/executor/utils/reference-validation' +import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' import type { ErrorInfo } from '@/tools/error-extractors' import { extractErrorMessage } from '@/tools/error-extractors' import { HostedKeyRateLimitedError, HostedKeyUnavailableError } from '@/tools/errors' @@ -205,7 +217,8 @@ async function normalizeCopilotFileParams( async function resolveCopilotEnvReferences( tool: ToolConfig, params: Record, - scope: ToolExecutionScope + scope: ToolExecutionScope, + resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry ): Promise { if (!scope.copilotToolExecution) { return @@ -238,6 +251,9 @@ async function resolveCopilotEnvReferences( const resolved = resolveEnvVarReferences(value, envVars, { allowEmbedded: false, missingKeys, + onResolved: (name, resolvedValue) => { + resolvedSecretTraceRegistry?.recordResolved(name, resolvedValue) + }, }) if (missingKeys.length > 0) { const scopeHint = scope.workspaceId @@ -1014,6 +1030,185 @@ export interface ExecuteToolOptions { skipPostProcess?: boolean executionContext?: ExecutionContext signal?: AbortSignal + resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry +} + +interface PrivateToolResponseMetadataResult { + response: Response + consumed: boolean +} + +function consumeResolvedSecretNames( + payload: unknown, + params: Record, + registry?: ResolvedSecretTraceRegistry +): void { + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) return + + const response = payload as Record + if (!Object.hasOwn(response, RESOLVED_SECRET_NAMES_FIELD)) return + + const names = response[RESOLVED_SECRET_NAMES_FIELD] + response[RESOLVED_SECRET_NAMES_FIELD] = undefined + if (!registry) return + + if (!Array.isArray(names) || !names.every((name) => typeof name === 'string')) { + registry.markIncomplete() + return + } + + const envVars = params.envVars + if (!envVars || typeof envVars !== 'object' || Array.isArray(envVars)) { + registry.markIncomplete() + return + } + + for (const name of names) { + const value = (envVars as Record)[name] + if (typeof value !== 'string') { + registry.markIncomplete() + continue + } + registry.recordResolved(name, value) + } +} + +async function consumeResolvedSecretProvenance( + payload: unknown, + registry?: ResolvedSecretTraceRegistry +): Promise { + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) return false + + const response = payload as Record + if (!Object.hasOwn(response, RESOLVED_SECRET_PROVENANCE_FIELD)) return false + + const provenance = response[RESOLVED_SECRET_PROVENANCE_FIELD] + response[RESOLVED_SECRET_PROVENANCE_FIELD] = undefined + if (registry) { + await registry.importCrossingProvenance(provenance, response, { trusted: true }) + } + return true +} + +function rebuildResponseWithoutPrivateToolMetadata( + response: Response, + payload: Record +): Response { + payload[RESOLVED_SECRET_NAMES_FIELD] = undefined + payload[RESOLVED_SECRET_PROVENANCE_FIELD] = undefined + const headers = new Headers(response.headers) + headers.delete('content-length') + headers.delete(PRIVATE_TOOL_METADATA_RESPONSE_HEADER) + return new Response(JSON.stringify(payload), { + status: response.status, + statusText: response.statusText, + headers, + }) +} + +function rebuildSafePrivateToolResponse(response: Response): Response { + const headers = new Headers(response.headers) + headers.delete('content-length') + headers.delete(PRIVATE_TOOL_METADATA_RESPONSE_HEADER) + headers.set('content-type', 'application/json') + return new Response( + JSON.stringify({ + success: false, + error: 'Internal tool response metadata could not be verified', + }), + { + status: response.status, + statusText: response.statusText, + headers, + } + ) +} + +async function consumePrivateToolPayloadMetadata( + payload: unknown, + headers: Headers, + requestedType: PrivateToolMetadataType | undefined, + params: Record, + registry?: ResolvedSecretTraceRegistry +): Promise { + if (!requestedType) return true + + const metadataType = headers.get(PRIVATE_TOOL_METADATA_RESPONSE_HEADER) + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) { + registry?.markIncomplete() + return false + } + + const record = payload as Record + const markerMatches = metadataType === requestedType + if (!markerMatches) { + registry?.markIncomplete() + } else { + const metadataKey = getPrivateToolMetadataField(metadataType) + if (!Object.hasOwn(record, metadataKey)) { + registry?.markIncomplete() + } else { + try { + if (metadataType === RESOLVED_SECRET_NAMES_METADATA_V1) { + consumeResolvedSecretNames(record, params, registry) + } else { + await consumeResolvedSecretProvenance(record, registry) + } + } catch { + registry?.markIncomplete() + } + } + } + + record[RESOLVED_SECRET_NAMES_FIELD] = undefined + record[RESOLVED_SECRET_PROVENANCE_FIELD] = undefined + return markerMatches +} + +async function consumePrivateToolResponseMetadata( + response: Response, + requestedType: PrivateToolMetadataType | undefined, + params: Record, + registry?: ResolvedSecretTraceRegistry +): Promise { + if (!requestedType) return { response, consumed: true } + const metadataType = response.headers.get(PRIVATE_TOOL_METADATA_RESPONSE_HEADER) + + let payload: unknown + try { + payload = await response.clone().json() + } catch { + registry?.markIncomplete() + return isPrivateToolMetadataType(metadataType) + ? { response: rebuildSafePrivateToolResponse(response), consumed: false } + : { response, consumed: false } + } + + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) { + registry?.markIncomplete() + return isPrivateToolMetadataType(metadataType) + ? { response: rebuildSafePrivateToolResponse(response), consumed: false } + : { response, consumed: false } + } + + const record = payload as Record + const hadPrivateMetadata = + Object.hasOwn(record, RESOLVED_SECRET_NAMES_FIELD) || + Object.hasOwn(record, RESOLVED_SECRET_PROVENANCE_FIELD) + const consumed = await consumePrivateToolPayloadMetadata( + record, + response.headers, + requestedType, + params, + registry + ) + + if (!consumed && !hadPrivateMetadata) return { response, consumed: false } + + return { + response: rebuildResponseWithoutPrivateToolMetadata(response, record), + consumed, + } } /** @@ -1025,7 +1220,20 @@ export async function executeTool( params: Record, options: ExecuteToolOptions = {} ): Promise { - const { skipPostProcess = false, executionContext, signal } = options + const { + skipPostProcess = false, + executionContext, + signal, + resolvedSecretTraceRegistry: explicitResolvedSecretTraceRegistry, + } = options + const resolvedSecretTraceRegistry = + explicitResolvedSecretTraceRegistry ?? executionContext?.resolvedSecretTraceRegistry + const executeNestedTool: typeof executeTool = (nestedToolId, nestedParams, nestedOptions = {}) => + executeTool(nestedToolId, nestedParams, { + ...nestedOptions, + resolvedSecretTraceRegistry: + nestedOptions.resolvedSecretTraceRegistry ?? resolvedSecretTraceRegistry, + }) // Fall back to the workflow execution's abort signal so plan-based execution timeouts // and cancellation propagate to tool fetches when the caller passes no explicit signal. const effectiveSignal = signal ?? executionContext?.abortSignal @@ -1053,6 +1261,13 @@ export async function executeTool( : isMcpTool(normalizedToolId) ? 'mcp' : undefined + const privateToolMetadataType: PrivateToolMetadataType | undefined = + resolvedSecretTraceRegistry && normalizedToolId === 'workflow_executor' + ? RESOLVED_SECRET_PROVENANCE_METADATA_V1 + : resolvedSecretTraceRegistry && + (normalizedToolId === 'function_execute' || toolKind === 'custom') + ? RESOLVED_SECRET_NAMES_METADATA_V1 + : undefined // Runs for ALL tools (not just kinded ones) so the per-tool `deniedTools` // denylist is enforced alongside the existing mcp/custom/skill gates. @@ -1105,7 +1320,8 @@ export async function executeTool( executionContext, requestId, startTimeISO, - effectiveSignal + effectiveSignal, + resolvedSecretTraceRegistry ) } else { // For built-in tools, use the synchronous version @@ -1135,7 +1351,7 @@ export async function executeTool( await normalizeCopilotFileParams(tool, contextParams, scope) normalizeCopilotCredentialParams(contextParams) enforceCopilotCredentialSelection(toolId, tool, contextParams, scope) - await resolveCopilotEnvReferences(tool, contextParams, scope) + await resolveCopilotEnvReferences(tool, contextParams, scope, resolvedSecretTraceRegistry) // Inject hosted API key if tool supports it and user didn't provide one const hostedKeyInfo = await injectHostedKeyIfNeeded( @@ -1290,7 +1506,10 @@ export async function executeTool( requestId, }, }, - { abortSignal: effectiveSignal } + { + abortSignal: effectiveSignal, + resolvedSecretTraceRegistry, + } ) const endTime = new Date() return { @@ -1315,7 +1534,7 @@ export async function executeTool( let finalResult = result if (tool.postProcess && result.success && !skipPostProcess) { try { - finalResult = await tool.postProcess(result, contextParams, executeTool) + finalResult = await tool.postProcess(result, contextParams, executeNestedTool) } catch (error) { logger.error(`[${requestId}] Post-processing error for ${toolId}:`, { error: toError(error).message, @@ -1362,7 +1581,15 @@ export async function executeTool( // Wrap with retry logic for hosted keys to handle rate limiting due to higher usage const result = hostedKeyInfo.isUsingHostedKey ? await executeWithRetry( - () => executeToolRequest(toolId, tool, contextParams, effectiveSignal), + () => + executeToolRequest( + toolId, + tool, + contextParams, + effectiveSignal, + privateToolMetadataType, + resolvedSecretTraceRegistry + ), { requestId, toolId, @@ -1380,17 +1607,32 @@ export async function executeTool( // Re-point metric labels at the freshly acquired key. hostedKeyInfo.envVarName = reacquiredEnvVar if (hostedKeyForMetrics) hostedKeyForMetrics.key = reacquiredEnvVar - return () => executeToolRequest(toolId, tool, contextParams, effectiveSignal) + return () => + executeToolRequest( + toolId, + tool, + contextParams, + effectiveSignal, + privateToolMetadataType, + resolvedSecretTraceRegistry + ) }, } ) - : await executeToolRequest(toolId, tool, contextParams, effectiveSignal) + : await executeToolRequest( + toolId, + tool, + contextParams, + effectiveSignal, + privateToolMetadataType, + resolvedSecretTraceRegistry + ) // Apply post-processing if available and not skipped let finalResult = result if (tool.postProcess && result.success && !skipPostProcess) { try { - finalResult = await tool.postProcess(result, contextParams, executeTool) + finalResult = await tool.postProcess(result, contextParams, executeNestedTool) } catch (error) { logger.error(`[${requestId}] Post-processing error for ${toolId}:`, { error: toError(error).message, @@ -1680,11 +1922,14 @@ async function executeToolRequest( toolId: string, tool: ToolConfig, params: Record, - signal?: AbortSignal + signal?: AbortSignal, + privateToolMetadataType?: PrivateToolMetadataType, + resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry ): Promise { const requestId = generateRequestId() const requestParams = formatRequestParams(tool, params) + let privateMetadataConsumed = privateToolMetadataType === undefined try { const endpointUrl = @@ -1750,6 +1995,9 @@ async function executeToolRequest( serializeBillingAttributionHeader(params._context.billingAttribution) ) } + if (isInternalRoute && privateToolMetadataType) { + headers.set(PRIVATE_TOOL_METADATA_REQUEST_HEADER, privateToolMetadataType) + } const shouldPropagateCallChain = isInternalRoute || isSelfOriginUrl(fullUrl) if (shouldPropagateCallChain) { @@ -1965,6 +2213,15 @@ async function executeToolRequest( throw lastError ?? new Error(`Request failed for ${toolId}`) } + const privateMetadata = await consumePrivateToolResponseMetadata( + response, + privateToolMetadataType, + params, + resolvedSecretTraceRegistry + ) + response = privateMetadata.response + privateMetadataConsumed = privateMetadata.consumed + if (!response.ok) { let errorData: any try { @@ -2079,6 +2336,9 @@ async function executeToolRequest( error: undefined, } } catch (error: any) { + if (privateToolMetadataType && !privateMetadataConsumed) { + resolvedSecretTraceRegistry?.markIncomplete() + } handleResponseSizeLimitError(error, requestId, toolId) // Check if this is a body size limit error and throw user-friendly message @@ -2175,7 +2435,8 @@ async function executeMcpTool( executionContext?: ExecutionContext, requestId?: string, startTimeISO?: string, - signal?: AbortSignal + signal?: AbortSignal, + resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry ): Promise { const actualRequestId = requestId || generateRequestId() const actualStartTime = startTimeISO || new Date().toISOString() @@ -2190,6 +2451,9 @@ async function executeMcpTool( const mcpScope = resolveToolScope(params, executionContext) const headers: Record = { 'Content-Type': 'application/json' } + if (resolvedSecretTraceRegistry) { + headers[PRIVATE_TOOL_METADATA_REQUEST_HEADER] = RESOLVED_SECRET_PROVENANCE_METADATA_V1 + } if (typeof window === 'undefined') { try { @@ -2312,10 +2576,18 @@ async function executeMcpTool( try { const errorData = await response.json() + await consumePrivateToolPayloadMetadata( + errorData, + response.headers, + resolvedSecretTraceRegistry ? RESOLVED_SECRET_PROVENANCE_METADATA_V1 : undefined, + params, + resolvedSecretTraceRegistry + ) if (errorData.error) { errorMessage = errorData.error } } catch { + resolvedSecretTraceRegistry?.markIncomplete() // Failed to parse error response, use default message } @@ -2332,6 +2604,13 @@ async function executeMcpTool( } const result = await response.json() + await consumePrivateToolPayloadMetadata( + result, + response.headers, + resolvedSecretTraceRegistry ? RESOLVED_SECRET_PROVENANCE_METADATA_V1 : undefined, + params, + resolvedSecretTraceRegistry + ) if (!result.success) { return { @@ -2358,6 +2637,7 @@ async function executeMcpTool( }, } } catch (error) { + resolvedSecretTraceRegistry?.markIncomplete() const endTime = new Date() const endTimeISO = endTime.toISOString() const duration = endTime.getTime() - new Date(actualStartTime).getTime() From 2eb706a833595728d2bab49de227ae1dd728e76b Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Sat, 1 Aug 2026 10:21:47 -0700 Subject: [PATCH 6/8] fix(execution): address review regressions --- .../app/api/mcp/tools/execute/route.test.ts | 2 + apps/sim/app/api/mcp/tools/execute/route.ts | 4 +- .../[id]/execute/route.async.test.ts | 43 +++++++++++++ .../executor/execution-state.test.ts | 60 +++++++++++++++++++ .../lib/workflows/executor/execution-state.ts | 26 +++++++- 5 files changed, 133 insertions(+), 2 deletions(-) diff --git a/apps/sim/app/api/mcp/tools/execute/route.test.ts b/apps/sim/app/api/mcp/tools/execute/route.test.ts index 29d78f43e9e..e6d1a29ae27 100644 --- a/apps/sim/app/api/mcp/tools/execute/route.test.ts +++ b/apps/sim/app/api/mcp/tools/execute/route.test.ts @@ -171,6 +171,8 @@ describe('MCP tool execution private secret provenance', () => { const response = await POST(request, {}) const body = (await response.json()) as Record + expect(response.status).toBe(500) + expect(response.ok).toBe(false) expect(body).toEqual({ success: false, error: 'Internal MCP response could not be verified', diff --git a/apps/sim/app/api/mcp/tools/execute/route.ts b/apps/sim/app/api/mcp/tools/execute/route.ts index 5986ea7bf30..48801c82053 100644 --- a/apps/sim/app/api/mcp/tools/execute/route.ts +++ b/apps/sim/app/api/mcp/tools/execute/route.ts @@ -72,6 +72,7 @@ async function attachPrivateProvenance( provenance: ResolvedSecretTraceProvenanceAccumulator ): Promise { let payload: Record + let status = response.status try { const body = await readResponseToBufferWithLimit(response, { maxBytes: MAX_PRIVATE_MCP_RESPONSE_BYTES, @@ -85,6 +86,7 @@ async function attachPrivateProvenance( payload = parsed as Record } catch { payload = { success: false, error: 'Internal MCP response could not be verified' } + status = 500 provenance.markIncomplete({ discardEntries: true }) } @@ -93,7 +95,7 @@ async function attachPrivateProvenance( headers.set(PRIVATE_TOOL_METADATA_RESPONSE_HEADER, RESOLVED_SECRET_PROVENANCE_METADATA_V1) return NextResponse.json( { ...payload, [RESOLVED_SECRET_PROVENANCE_FIELD]: provenance.exportProvenance() }, - { status: response.status, headers } + { status, headers } ) } diff --git a/apps/sim/app/api/workflows/[id]/execute/route.async.test.ts b/apps/sim/app/api/workflows/[id]/execute/route.async.test.ts index 0ccf344ab5a..c0620434fe7 100644 --- a/apps/sim/app/api/workflows/[id]/execute/route.async.test.ts +++ b/apps/sim/app/api/workflows/[id]/execute/route.async.test.ts @@ -374,6 +374,49 @@ describe('workflow execute async route', () => { ) }) + it('recovers legacy starter input by execution ID without returning it to the client', async () => { + const sourceInput = { token: 'legacy-retry-input', nested: { value: 42 } } + queueTableRows(schemaMock.workflowExecutionLogs, [ + { + executionId: 'source-execution', + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + executionData: { + executionState: { + blockStates: { + start: { + output: sourceInput, + executed: false, + executionTime: 0, + }, + }, + }, + }, + }, + ]) + const request = createMockRequest( + 'POST', + { inputFromExecutionId: 'source-execution' }, + { + 'Content-Type': 'application/json', + 'X-Execution-Mode': 'async', + Cookie: 'session=value', + } + ) + + const response = await POST(request, { params: Promise.resolve({ id: 'workflow-1' }) }) + const responseBody = await response.json() + + expect(response.status).toBe(202) + expect(responseBody).not.toHaveProperty('input') + expect(JSON.stringify(responseBody)).not.toContain('legacy-retry-input') + expect(mockEnqueue).toHaveBeenCalledWith( + 'workflow-execution', + expect.objectContaining({ input: sourceInput }), + expect.any(Object) + ) + }) + it('rejects client input alongside a stored execution input reference', async () => { const response = await POST( createMockRequest( diff --git a/apps/sim/lib/workflows/executor/execution-state.test.ts b/apps/sim/lib/workflows/executor/execution-state.test.ts index 49101daa652..39aa68c066d 100644 --- a/apps/sim/lib/workflows/executor/execution-state.test.ts +++ b/apps/sim/lib/workflows/executor/execution-state.test.ts @@ -112,6 +112,66 @@ describe('execution state lookup', () => { }) }) + it('recovers legacy workflow input from the pre-populated starter block state', async () => { + const legacyInput = { leadId: 'legacy-lead' } + queueTableRows(schemaMock.workflowExecutionLogs, [ + { + executionId: 'execution-1', + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + executionData: {}, + }, + ]) + mockMaterializeExecutionData.mockResolvedValueOnce({ + executionState: { + blockStates: { + 'executed-block': { + output: { leadId: 'wrong-lead' }, + executed: true, + executionTime: 10, + }, + start: { + output: legacyInput, + executed: false, + executionTime: 0, + }, + }, + }, + }) + + const result = await getExecutionInputForWorkflow('execution-1', 'workflow-1') + + expect(result).toEqual({ found: true, input: legacyInput }) + }) + + it('prefers persisted workflow input over the legacy starter block state', async () => { + const workflowInput = { leadId: 'current-lead' } + queueTableRows(schemaMock.workflowExecutionLogs, [ + { + executionId: 'execution-1', + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + executionData: {}, + }, + ]) + mockMaterializeExecutionData.mockResolvedValueOnce({ + workflowInput, + executionState: { + blockStates: { + start: { + output: { leadId: 'legacy-lead' }, + executed: false, + executionTime: 0, + }, + }, + }, + }) + + const result = await getExecutionInputForWorkflow('execution-1', 'workflow-1') + + expect(result).toEqual({ found: true, input: workflowInput }) + }) + it('checks older pointer-backed candidates when the latest has no execution state', async () => { queueTableRows(schemaMock.workflowExecutionLogs, [ { diff --git a/apps/sim/lib/workflows/executor/execution-state.ts b/apps/sim/lib/workflows/executor/execution-state.ts index f482f8e9e75..c854890d71e 100644 --- a/apps/sim/lib/workflows/executor/execution-state.ts +++ b/apps/sim/lib/workflows/executor/execution-state.ts @@ -1,5 +1,6 @@ import { db } from '@sim/db' import { workflowExecutionLogs } from '@sim/db/schema' +import { isRecordLike } from '@sim/utils/object' import { and, desc, eq, or, sql } from 'drizzle-orm' import { materializeExecutionData, TRACE_STORE_REF_KEY } from '@/lib/logs/execution/trace-store' import type { SerializableExecutionState } from '@/executor/execution/types' @@ -30,6 +31,25 @@ function extractExecutionState(executionData: unknown): SerializableExecutionSta return isSerializableExecutionState(state) ? state : null } +function extractLegacyWorkflowInput(executionData: Record): unknown | undefined { + if (!isRecordLike(executionData.executionState)) return undefined + const { blockStates } = executionData.executionState + if (!isRecordLike(blockStates)) return undefined + + for (const state of Object.values(blockStates)) { + if ( + isRecordLike(state) && + state.executed === false && + state.executionTime === 0 && + state.output != null + ) { + return state.output + } + } + + return undefined +} + interface ExecutionStateRow { executionId: string workflowId: string | null @@ -109,7 +129,11 @@ export async function getExecutionInputForWorkflow( } const data = await materializeExecutionDataFromRow(row) - return { found: true, input: data?.workflowInput } + if (!data) return { found: true } + if (Object.hasOwn(data, 'workflowInput')) { + return { found: true, input: data.workflowInput } + } + return { found: true, input: extractLegacyWorkflowInput(data) } } export async function getLatestExecutionStateWithExecutionId( From ce1740447dc2174cbd81734228ed2d9ad1933c74 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Sat, 1 Aug 2026 11:48:14 -0700 Subject: [PATCH 7/8] fix(execution): harden secret trace provenance --- .../content/docs/en/agents/custom-tools.mdx | 2 +- apps/docs/content/docs/en/agents/mcp.mdx | 4 +- .../content/docs/en/logs-debugging/index.mdx | 2 +- .../docs/en/logs-debugging/logging.mdx | 4 +- .../content/docs/en/platform/credentials.mdx | 11 +- .../docs/en/workflows/blocks/function.mdx | 9 +- .../content/docs/en/workflows/blocks/logs.mdx | 5 +- .../content/docs/en/workflows/variables.mdx | 4 +- .../app/api/mcp/tools/execute/route.test.ts | 54 +- apps/sim/app/api/mcp/tools/execute/route.ts | 28 +- .../app/api/mothership/execute/route.test.ts | 72 +++ apps/sim/app/api/mothership/execute/route.ts | 51 +- .../[id]/execute/route.async.test.ts | 50 ++ .../hooks/use-workflow-execution.test.tsx | 37 +- .../hooks/use-workflow-execution.ts | 3 +- apps/sim/background/webhook-execution.test.ts | 9 +- apps/sim/background/webhook-execution.ts | 76 ++- .../resolved-secret-trace-registry.test.ts | 122 +++- .../utils/resolved-secret-trace-registry.ts | 256 ++++++-- apps/sim/lib/environment/utils.test.ts | 83 +++ apps/sim/lib/environment/utils.ts | 109 ++-- apps/sim/lib/logs/execution/logger.ts | 15 +- .../logs/execution/logging-session.test.ts | 142 +++++ .../sim/lib/logs/execution/logging-session.ts | 22 +- .../execution/trace-secret-projection.test.ts | 316 +++++++++- .../logs/execution/trace-secret-projection.ts | 581 ++++++++++++++++-- apps/sim/lib/logs/types.ts | 9 +- apps/sim/lib/mcp/resolve-config.test.ts | 38 +- apps/sim/lib/mcp/resolve-config.ts | 49 +- apps/sim/tools/index.test.ts | 52 ++ apps/sim/tools/index.ts | 23 +- .../src/mocks/environment-utils.mock.test.ts | 8 + .../src/mocks/environment-utils.mock.ts | 7 + 33 files changed, 1962 insertions(+), 291 deletions(-) create mode 100644 apps/sim/lib/environment/utils.test.ts diff --git a/apps/docs/content/docs/en/agents/custom-tools.mdx b/apps/docs/content/docs/en/agents/custom-tools.mdx index c551d942652..9648b00910e 100644 --- a/apps/docs/content/docs/en/agents/custom-tools.mdx +++ b/apps/docs/content/docs/en/agents/custom-tools.mdx @@ -156,7 +156,7 @@ From **Settings → Custom Tools** you can: \ No newline at end of file +]} /> diff --git a/apps/docs/content/docs/en/logs-debugging/index.mdx b/apps/docs/content/docs/en/logs-debugging/index.mdx index df544382d9a..fcf9b8c8f49 100644 --- a/apps/docs/content/docs/en/logs-debugging/index.mdx +++ b/apps/docs/content/docs/en/logs-debugging/index.mdx @@ -43,7 +43,7 @@ This is the level you debug at, because a run fails when one of its blocks fails ### Input and output -Each block in the sidebar has two tabs. The **Input** tab shows the resolved values the block actually ran with: the literal values you typed, and the earlier outputs it read by reference (with API keys redacted). The **Output** tab shows what the block produced, formatted as an object, with markdown rendered for agent-generated text. +Each block in the sidebar has two tabs. The **Input** tab shows the resolved values the block actually ran with: the literal values you typed, and the earlier outputs it read by reference. Exact values successfully substituted from Secrets through `{{KEY}}` are masked in this trace copy; see [Execution log protection](/platform/credentials#execution-log-protection). The **Output** tab shows what the block produced, formatted as an object, with markdown rendered for agent-generated text. The input tab is the important one. A block reads earlier outputs by name, written ``, and the input tab shows what those references resolved to at run time. If a reference pointed at a value that was not there, you see it here as missing or wrong, not as the tag you wrote. See [how blocks pass data](/workflows/data-flow) for how those references resolve. diff --git a/apps/docs/content/docs/en/logs-debugging/logging.mdx b/apps/docs/content/docs/en/logs-debugging/logging.mdx index 3c1b3970501..fdbcbdbd64f 100644 --- a/apps/docs/content/docs/en/logs-debugging/logging.mdx +++ b/apps/docs/content/docs/en/logs-debugging/logging.mdx @@ -57,7 +57,7 @@ Click any entry to open its sidebar: the run's timeline (start/end, total durati The block's result — JSON-formatted structured data, markdown rendering for AI content, and a copy button. - What the block received — resolved variable values, referenced outputs, and environment variables. API keys are automatically redacted. + What the block received — resolved variable values, referenced outputs, and environment variables. Exact secret values activated by a successful `{{KEY}}` substitution are masked in this trace view. See [Execution log protection](/platform/credentials#execution-log-protection). @@ -95,7 +95,7 @@ import { FAQ } from '@/components/ui/faq' +### Execution log protection + +When a saved secret is successfully substituted through a `{{KEY}}` reference, Sim masks exact, case-sensitive occurrences of its resolved value in log-facing content. This includes the editor's live block-log display, Logs Overview input and output, stored execution traces, log-read API responses, and the Logs block's **Get Run Details** output. Function and Agent span inputs, outputs, and errors, Agent thinking, and Agent tool-call arguments, results, and errors are protected. The replacement is normally shown as `{{KEY}}`. + +This is an observability projection only. Secret resolution and workflow behavior are unchanged: blocks, tools, models, and downstream steps receive the real runtime value. Stored functional execution data, workflow execution responses, streams, callbacks, block state, and snapshots are not rewritten. Log-facing views and read APIs receive a separate protected copy, so the Logs Overview **Workflow Input** and **Workflow Output** are masked without changing the underlying workflow result. + -Secret values are never exposed in the workflow editor or execution logs — they are only resolved during execution. +Masking is activated only when Sim successfully resolves a value from **Settings → Secrets** through `{{KEY}}`. A hardcoded literal, direct `environmentVariables['KEY']` read, or shell `$KEY` read does not activate it by itself. Once activated, every exact occurrence of that value in the run's log-facing content is masked. Encoded, hashed, or otherwise transformed versions are not matched. Do not deliberately return or print secrets. ## Secret Details @@ -115,7 +121,8 @@ When a workflow runs, secrets resolve in this order: - **Never hardcode secrets** in workflow input fields — always use `{{KEY}}` references The block always operates on the current workspace. Costs are denominated in credits, both for - the cost filter and the cost output. + the cost filter and the cost output. **Get Run Details** is a log-read path, so both `traceSpans` + and `finalOutput` come from the protected log-facing projection described under + [Execution log protection](/platform/credentials#execution-log-protection). The underlying runtime + result remains unchanged. ({ - mockDiscoverServerTools: vi.fn(), - mockExecuteTool: vi.fn(), +const { mockDiscoverServerTools, mockExecuteTool, mockReadResponseToBufferWithLimit } = vi.hoisted( + () => ({ + mockDiscoverServerTools: vi.fn(), + mockExecuteTool: vi.fn(), + mockReadResponseToBufferWithLimit: vi.fn(), + }) +) + +vi.mock('@/lib/core/utils/stream-limits', () => ({ + readResponseToBufferWithLimit: mockReadResponseToBufferWithLimit, })) vi.mock('@/lib/mcp/middleware', () => ({ @@ -88,9 +95,12 @@ describe('MCP tool execution private secret provenance', () => { vi.clearAllMocks() mockDiscoverServerTools.mockResolvedValue([{ name: 'example_tool', inputSchema: {} }]) mockExecuteTool.mockResolvedValue({ content: [{ type: 'text', text: 'ok' }] }) + mockReadResponseToBufferWithLimit.mockImplementation(async (response: Response) => + Buffer.from(await response.arrayBuffer()) + ) }) - it('returns scoped encrypted provenance only to an authenticated internal caller', async () => { + it('returns fail-closed scoped provenance only to an authenticated internal caller', async () => { mockDiscoverServerTools.mockImplementationOnce( async ( _userId: string, @@ -102,7 +112,7 @@ describe('MCP tool execution private secret provenance', () => { report({ version: 1, complete: false, - entries: [{ name: 'OLD_TOKEN', encryptedValue: 'encrypted-v1' }], + entries: [], scope: { userId: 'user-1', workspaceId: 'workspace-1' }, }) return [{ name: 'example_tool', inputSchema: {} }] @@ -139,10 +149,7 @@ describe('MCP tool execution private secret provenance', () => { expect(body.__resolvedSecretTraceProvenance).toEqual({ version: 1, complete: false, - entries: [ - { name: 'OLD_TOKEN', encryptedValue: 'encrypted-v1' }, - { name: 'NEW_TOKEN', encryptedValue: 'encrypted-v2' }, - ], + entries: [], scope: { userId: 'user-1', workspaceId: 'workspace-1' }, }) }) @@ -158,11 +165,14 @@ describe('MCP tool execution private secret provenance', () => { expect(response.headers.has('x-sim-private-tool-metadata')).toBe(false) expect(body).not.toHaveProperty('__resolvedSecretTraceProvenance') + expect(mockDiscoverServerTools.mock.calls[0]?.[4]).toBeUndefined() + expect(mockExecuteTool.mock.calls[0]?.[5]).toBeUndefined() }) - it('fails closed when attaching provenance would exceed the response budget', async () => { + it('preserves the functional response when private provenance cannot be attached', async () => { + mockReadResponseToBufferWithLimit.mockRejectedValueOnce(new Error('Response exceeds limit')) mockExecuteTool.mockResolvedValueOnce({ - content: [{ type: 'text', text: 'x'.repeat(10 * 1024 * 1024) }], + content: [{ type: 'text', text: 'unchanged' }], }) const request = createRequest({ 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1', @@ -171,17 +181,19 @@ describe('MCP tool execution private secret provenance', () => { const response = await POST(request, {}) const body = (await response.json()) as Record - expect(response.status).toBe(500) - expect(response.ok).toBe(false) - expect(body).toEqual({ - success: false, - error: 'Internal MCP response could not be verified', - __resolvedSecretTraceProvenance: { - version: 1, - complete: false, - entries: [], - scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + expect(response.status).toBe(200) + expect(response.ok).toBe(true) + expect(response.headers.has('x-sim-private-tool-metadata')).toBe(false) + expect(body).not.toHaveProperty('__resolvedSecretTraceProvenance') + expect(body).toMatchObject({ + success: true, + data: { + success: true, + output: { content: [{ type: 'text' }] }, }, }) + expect( + (body.data as { output: { content: Array<{ text?: unknown }> } }).output.content[0]?.text + ).toBe('unchanged') }) }) diff --git a/apps/sim/app/api/mcp/tools/execute/route.ts b/apps/sim/app/api/mcp/tools/execute/route.ts index 48801c82053..564fb0cf0aa 100644 --- a/apps/sim/app/api/mcp/tools/execute/route.ts +++ b/apps/sim/app/api/mcp/tools/execute/route.ts @@ -72,9 +72,8 @@ async function attachPrivateProvenance( provenance: ResolvedSecretTraceProvenanceAccumulator ): Promise { let payload: Record - let status = response.status try { - const body = await readResponseToBufferWithLimit(response, { + const body = await readResponseToBufferWithLimit(response.clone(), { maxBytes: MAX_PRIVATE_MCP_RESPONSE_BYTES, label: 'MCP private metadata response', allowNoBodyFallback: true, @@ -85,9 +84,7 @@ async function attachPrivateProvenance( } payload = parsed as Record } catch { - payload = { success: false, error: 'Internal MCP response could not be verified' } - status = 500 - provenance.markIncomplete({ discardEntries: true }) + return response } const headers = new Headers(response.headers) @@ -95,7 +92,7 @@ async function attachPrivateProvenance( headers.set(PRIVATE_TOOL_METADATA_RESPONSE_HEADER, RESOLVED_SECRET_PROVENANCE_METADATA_V1) return NextResponse.json( { ...payload, [RESOLVED_SECRET_PROVENANCE_FIELD]: provenance.exportProvenance() }, - { status, headers } + { status: response.status, headers } ) } @@ -106,16 +103,17 @@ export const POST = withRouteHandler( withMcpAuth('read')( async (request: NextRequest, { userId, workspaceId, requestId, authType }) => { let serverId: string | undefined - const resolvedSecretTraceProvenance = new ResolvedSecretTraceProvenanceAccumulator({ - userId, - workspaceId, - }) - const recordProvenance = (provenance: ResolvedSecretTraceProvenanceV1): void => { - resolvedSecretTraceProvenance.record(provenance) - } const includePrivateProvenance = authType === AuthType.INTERNAL_JWT && requestsPrivateToolMetadata(request.headers, RESOLVED_SECRET_PROVENANCE_METADATA_V1) + const resolvedSecretTraceProvenance = includePrivateProvenance + ? new ResolvedSecretTraceProvenanceAccumulator({ userId, workspaceId }) + : undefined + const recordProvenance = resolvedSecretTraceProvenance + ? (provenance: ResolvedSecretTraceProvenanceV1): void => { + resolvedSecretTraceProvenance.record(provenance) + } + : undefined const response = await (async (): Promise => { try { const rawBody = await readMcpJsonBodyWithLimit(request) @@ -332,7 +330,7 @@ export const POST = withRouteHandler( return createMcpSuccessResponse(transformedResult) } catch (error) { if (getErrorMessage(error) === 'Tool execution timeout') { - resolvedSecretTraceProvenance.markIncomplete() + resolvedSecretTraceProvenance?.markIncomplete() } const bodyErrorResponse = mcpBodyReadErrorResponse(error, request) if (bodyErrorResponse) return bodyErrorResponse @@ -364,7 +362,7 @@ export const POST = withRouteHandler( } })() - return includePrivateProvenance + return resolvedSecretTraceProvenance ? attachPrivateProvenance(response, resolvedSecretTraceProvenance) : response } diff --git a/apps/sim/app/api/mothership/execute/route.test.ts b/apps/sim/app/api/mothership/execute/route.test.ts index 4490dca0ed2..350e85bd8b1 100644 --- a/apps/sim/app/api/mothership/execute/route.test.ts +++ b/apps/sim/app/api/mothership/execute/route.test.ts @@ -215,6 +215,78 @@ describe('mothership private trace provenance transport', () => { expect(response.headers.get('x-sim-private-tool-metadata')).toBeNull() expect(body.content).toBe('secret-value') expect(body).not.toHaveProperty('__resolvedSecretTraceProvenance') + expect(mockGetPersonalAndWorkspaceEnv).not.toHaveBeenCalled() + expect(mockRunHeadlessCopilotLifecycle).toHaveBeenCalledWith( + expect.any(Object), + expect.objectContaining({ resolvedSecretTraceRegistry: undefined }) + ) + }) + + it('keeps execution functional and fails trace provenance closed when catalog setup fails', async () => { + mockGetPersonalAndWorkspaceEnv.mockRejectedValueOnce(new Error('catalog unavailable')) + mockRunHeadlessCopilotLifecycle.mockImplementation( + async (_payload: Record, options: CopilotLifecycleOptions) => { + expect(options.resolvedSecretTraceRegistry?.isComplete()).toBe(false) + return successResult() + } + ) + + const response = await POST( + createMockRequest( + 'POST', + requestBody, + { + Authorization: 'Bearer internal', + 'x-sim-billing-attribution': 'billing', + 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1', + }, + 'http://localhost:3000/api/mothership/execute' + ) + ) + const body = await response.json() + + expect(response.status).toBe(200) + expect(body.content).toBe('secret-value') + expect(body.__resolvedSecretTraceProvenance).toEqual({ + version: 1, + complete: false, + entries: [], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + }) + }) + + it('fails provenance closed without changing a runtime value that rotated after catalog load', async () => { + mockRunHeadlessCopilotLifecycle.mockImplementation( + async (_payload: Record, options: CopilotLifecycleOptions) => { + expect( + options.resolvedSecretTraceRegistry?.recordResolved('API_KEY', 'rotated-secret-value') + ).toBe(false) + return { ...successResult(), content: 'rotated-secret-value' } + } + ) + + const response = await POST( + createMockRequest( + 'POST', + requestBody, + { + Authorization: 'Bearer internal', + 'x-sim-billing-attribution': 'billing', + 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1', + }, + 'http://localhost:3000/api/mothership/execute' + ) + ) + const body = await response.json() + + expect(response.status).toBe(200) + expect(body.content).toBe('rotated-secret-value') + expect(body.__resolvedSecretTraceProvenance).toEqual({ + version: 1, + complete: false, + entries: [], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + }) }) it('returns encrypted provenance on a marker-gated successful request', async () => { diff --git a/apps/sim/app/api/mothership/execute/route.ts b/apps/sim/app/api/mothership/execute/route.ts index 768d4a5ccf0..1febf5e513e 100644 --- a/apps/sim/app/api/mothership/execute/route.ts +++ b/apps/sim/app/api/mothership/execute/route.ts @@ -32,6 +32,7 @@ import { isWorkspaceAccessDeniedError, } from '@/lib/workspaces/permissions/utils' import { + createIncompleteResolvedSecretTraceRegistry, createResolvedSecretTraceRegistry, ResolvedSecretTraceProvenanceAccumulator, type ResolvedSecretTraceRegistry, @@ -185,23 +186,39 @@ export const POST = withRouteHandler(async (req: NextRequest) => { actorUserId: userId, workspaceId, }) - const environment = await getPersonalAndWorkspaceEnv(userId, workspaceId) - resolvedSecretTraceRegistry = await createResolvedSecretTraceRegistry({ - personalEncrypted: environment.personalEncrypted, - workspaceEncrypted: environment.workspaceEncrypted, - personalDecrypted: environment.personalDecrypted, - workspaceDecrypted: environment.workspaceDecrypted, - decryptionFailures: environment.decryptionFailures, - scope: { userId, workspaceId }, - }) + if (includePrivateProvenance) { + const scope = { userId, workspaceId } + try { + const environment = await getPersonalAndWorkspaceEnv(userId, workspaceId, { + workspaceAccess, + }) + resolvedSecretTraceRegistry = await createResolvedSecretTraceRegistry({ + personalEncrypted: environment.personalEncrypted, + workspaceEncrypted: environment.workspaceEncrypted, + personalDecrypted: environment.personalDecrypted, + workspaceDecrypted: environment.workspaceDecrypted, + decryptionFailures: environment.decryptionFailures, + scope, + }) + } catch (error) { + logger.warn('Failed to build Mothership trace secret catalog', { + error: getErrorMessage(error), + userId, + workspaceId, + }) + resolvedSecretTraceRegistry = createIncompleteResolvedSecretTraceRegistry(scope) + } + } const activeResolvedSecretTraceRegistry = resolvedSecretTraceRegistry const mcpDiscoveryProvenance = new ResolvedSecretTraceProvenanceAccumulator({ userId, workspaceId, }) - const recordMcpDiscoveryProvenance = (provenance: unknown): void => { - mcpDiscoveryProvenance.record(provenance) - } + const recordMcpDiscoveryProvenance = includePrivateProvenance + ? (provenance: unknown): void => { + mcpDiscoveryProvenance.record(provenance) + } + : undefined const effectiveChatId = chatId || generateId() messageId = providedMessageId || generateId() @@ -234,10 +251,12 @@ export const POST = withRouteHandler(async (req: NextRequest) => { recordMcpDiscoveryProvenance ), ]).then(async (results) => { - await activeResolvedSecretTraceRegistry.importProvenance( - mcpDiscoveryProvenance.exportProvenance(), - { trusted: true } - ) + if (activeResolvedSecretTraceRegistry) { + await activeResolvedSecretTraceRegistry.importProvenance( + mcpDiscoveryProvenance.exportProvenance(), + { trusted: true } + ) + } const groups = results.map((result) => { if (result.status === 'rejected') throw result.reason return result.value diff --git a/apps/sim/app/api/workflows/[id]/execute/route.async.test.ts b/apps/sim/app/api/workflows/[id]/execute/route.async.test.ts index c0620434fe7..c55a9c1030d 100644 --- a/apps/sim/app/api/workflows/[id]/execute/route.async.test.ts +++ b/apps/sim/app/api/workflows/[id]/execute/route.async.test.ts @@ -923,6 +923,56 @@ describe('workflow execute async route', () => { ) }) + it('falls back to an untrusted client snapshot while stored run-from-block state is pending', async () => { + const sourceSnapshot = { + blockStates: { previous: { output: { value: 'cached' } } }, + executedBlocks: ['previous'], + blockLogs: [], + decisions: { router: {}, condition: {} }, + completedLoops: [], + activeExecutionPath: [], + resolvedSecretTraceProvenance: { + version: 1, + complete: true, + entries: [{ name: 'TOKEN', encryptedValue: 'untrusted-ciphertext' }], + }, + } + queueTableRows(schemaMock.workflowExecutionLogs, []) + const request = createMockRequest( + 'POST', + { + input: { hello: 'world' }, + runFromBlock: { + startBlockId: 'start-block', + executionId: 'source-execution', + sourceSnapshot, + }, + }, + { + 'Content-Type': 'application/json', + Cookie: 'session=value', + } + ) + + const response = await POST(request, { params: Promise.resolve({ id: 'workflow-1' }) }) + + expect(response.status).toBe(200) + expect(mockExecuteWorkflowCore).toHaveBeenCalledWith( + expect.objectContaining({ + runFromBlock: { + startBlockId: 'start-block', + sourceSnapshot: expect.objectContaining({ + blockStates: sourceSnapshot.blockStates, + executedBlocks: sourceSnapshot.executedBlocks, + }), + }, + }) + ) + const runFromBlock = mockExecuteWorkflowCore.mock.calls[0]?.[0]?.runFromBlock + expect(runFromBlock).not.toHaveProperty('sourceExecutionId') + expect(runFromBlock?.sourceSnapshot).not.toHaveProperty('resolvedSecretTraceProvenance') + }) + it('returns encrypted resolution provenance only to an authenticated internal tool caller', async () => { const caller = EXECUTION_CALLERS[4] configureExecutionCaller(caller) diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-workflow-execution.test.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-workflow-execution.test.tsx index bccb79120c9..3ed0c144334 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-workflow-execution.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-workflow-execution.test.tsx @@ -10,6 +10,7 @@ const { DirectUploadErrorMock, executionStoreState, mockExecute, + mockExecuteFromBlock, mockFetch, mockResolveStartCandidates, mockRunUploadStrategy, @@ -90,6 +91,7 @@ const { DirectUploadErrorMock, executionStoreState, mockExecute: vi.fn(), + mockExecuteFromBlock: vi.fn(), mockFetch: vi.fn(), mockResolveStartCandidates: vi.fn(), mockRunUploadStrategy: vi.fn(), @@ -210,7 +212,7 @@ vi.mock('@/hooks/use-execution-stream', () => { SSEStreamInterruptedError, useExecutionStream: () => ({ execute: mockExecute, - executeFromBlock: vi.fn(), + executeFromBlock: mockExecuteFromBlock, reconnect: vi.fn(), cancel: vi.fn(), cancelExecute: vi.fn(), @@ -362,6 +364,8 @@ describe('useWorkflowExecution attachment uploads', () => { }) ) mockExecute.mockResolvedValue(undefined) + mockExecuteFromBlock.mockResolvedValue(undefined) + workflowStoreState.edges.length = 0 }) afterEach(() => { @@ -616,4 +620,35 @@ describe('useWorkflowExecution attachment uploads', () => { unmount() }) + + it('sends the snapshot as a fallback with a trusted run-from-block execution ID', async () => { + const sourceSnapshot = { + blockStates: { start: { output: { value: 'ready' } } }, + executedBlocks: ['start'], + blockLogs: [], + decisions: { router: {}, condition: {} }, + completedLoops: [], + activeExecutionPath: ['start'], + sourceExecutionId: 'source-execution-1', + } + executionStoreState.getLastExecutionSnapshot.mockReturnValueOnce(sourceSnapshot) + workflowStoreState.edges.push({ source: 'start', target: 'function-1' } as never) + + const { result, unmount } = renderWorkflowExecutionHook() + + await act(async () => { + await result().handleRunFromBlock('function-1', 'workflow-1') + }) + + expect(mockExecuteFromBlock).toHaveBeenCalledWith( + expect.objectContaining({ + workflowId: 'workflow-1', + startBlockId: 'function-1', + sourceExecutionId: 'source-execution-1', + sourceSnapshot, + }) + ) + + unmount() + }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-workflow-execution.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-workflow-execution.ts index f1b1c129b5d..201e9b62207 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-workflow-execution.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-workflow-execution.ts @@ -2037,7 +2037,8 @@ export function useWorkflowExecution() { await executionStream.executeFromBlock({ workflowId, startBlockId: blockId, - ...(sourceExecutionId ? { sourceExecutionId } : { sourceSnapshot: effectiveSnapshot }), + sourceSnapshot: effectiveSnapshot, + ...(sourceExecutionId ? { sourceExecutionId } : {}), input: workflowInput, onExecutionId: (id) => { if (runFromBlockOwnerRef.current !== runOwnerId) return diff --git a/apps/sim/background/webhook-execution.test.ts b/apps/sim/background/webhook-execution.test.ts index a72f241d854..c765770fec2 100644 --- a/apps/sim/background/webhook-execution.test.ts +++ b/apps/sim/background/webhook-execution.test.ts @@ -42,7 +42,8 @@ const { ), })) -const mockGetPersonalAndWorkspaceEnv = environmentUtilsMockFns.mockGetPersonalAndWorkspaceEnv +const mockGetEffectiveEnvironmentSnapshot = + environmentUtilsMockFns.mockGetEffectiveEnvironmentSnapshot afterAll(resetEnvironmentUtilsMock) @@ -237,7 +238,7 @@ describe('executeWebhookJob fault vs error handling', () => { executionTimeout: { async: 120_000 }, }) mockResolveWebhookRecordProviderConfig.mockImplementation(async (record) => record) - mockGetPersonalAndWorkspaceEnv.mockResolvedValue({ + mockGetEffectiveEnvironmentSnapshot.mockResolvedValue({ personalEncrypted: {}, workspaceEncrypted: {}, personalDecrypted: {}, @@ -308,7 +309,7 @@ describe('executeWebhookJob fault vs error handling', () => { }) it('passes encrypted webhook resolution provenance into workflow execution', async () => { - mockGetPersonalAndWorkspaceEnv.mockResolvedValue({ + mockGetEffectiveEnvironmentSnapshot.mockResolvedValue({ personalEncrypted: { WEBHOOK_SECRET: 'personal-ciphertext' }, workspaceEncrypted: { WEBHOOK_SECRET: 'workspace-ciphertext' }, personalDecrypted: { WEBHOOK_SECRET: 'personal-value' }, @@ -364,7 +365,7 @@ describe('executeWebhookJob fault vs error handling', () => { it('installs provenance before a post-resolution webhook setup failure', async () => { const rawMessage = 'Webhook handler exposed activated-secret-value' const rawError = new Error(rawMessage) - mockGetPersonalAndWorkspaceEnv.mockResolvedValue({ + mockGetEffectiveEnvironmentSnapshot.mockResolvedValue({ personalEncrypted: {}, workspaceEncrypted: { WEBHOOK_SECRET: 'workspace-ciphertext' }, personalDecrypted: {}, diff --git a/apps/sim/background/webhook-execution.ts b/apps/sim/background/webhook-execution.ts index 82f1518a78c..5bea7691d3a 100644 --- a/apps/sim/background/webhook-execution.ts +++ b/apps/sim/background/webhook-execution.ts @@ -14,7 +14,10 @@ import { import type { AsyncExecutionCorrelation } from '@/lib/core/async-jobs/types' import { createTimeoutAbortController, getTimeoutErrorMessage } from '@/lib/core/execution-limits' import { IdempotencyService, webhookIdempotency } from '@/lib/core/idempotency' -import { getPersonalAndWorkspaceEnv } from '@/lib/environment/utils' +import { + type EnvironmentResolutionSnapshot, + getEffectiveEnvironmentSnapshot, +} from '@/lib/environment/utils' import { preprocessExecution } from '@/lib/execution/preprocessing' import { LoggingSession } from '@/lib/logs/execution/logging-session' import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans' @@ -43,7 +46,10 @@ import { ExecutionSnapshot } from '@/executor/execution/snapshot' import type { ExecutionMetadata } from '@/executor/execution/types' import type { ExecutionResult } from '@/executor/types' import { hasExecutionResult } from '@/executor/utils/errors' -import { createResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' +import { + createIncompleteResolvedSecretTraceRegistry, + createResolvedSecretTraceRegistry, +} from '@/executor/utils/resolved-secret-trace-registry' import { safeAssign } from '@/tools/safe-assign' import { getTrigger, isTriggerValid } from '@/triggers' @@ -321,12 +327,31 @@ export async function resolveWebhookExecutionProviderConfig< provider: string, userId: string, workspaceId?: string, - options?: WebhookEnvResolutionOptions + options?: WebhookEnvResolutionOptions & { + onEnvironmentSnapshot?: (snapshot: EnvironmentResolutionSnapshot) => void | Promise + } ): Promise }> { try { - return options - ? await resolveWebhookRecordProviderConfig(webhookRecord, userId, workspaceId, options) - : await resolveWebhookRecordProviderConfig(webhookRecord, userId, workspaceId) + if (!options) { + return await resolveWebhookRecordProviderConfig(webhookRecord, userId, workspaceId) + } + + const { onEnvironmentSnapshot, ...resolutionOptions } = options + if (onEnvironmentSnapshot && resolutionOptions.envVars === undefined) { + const snapshot = await getEffectiveEnvironmentSnapshot(userId, workspaceId) + await onEnvironmentSnapshot(snapshot) + resolutionOptions.envVars = { + ...snapshot.personalDecrypted, + ...snapshot.workspaceDecrypted, + } + } + + return await resolveWebhookRecordProviderConfig( + webhookRecord, + userId, + workspaceId, + resolutionOptions + ) } catch (error) { const errorMessage = toError(error).message throw new Error( @@ -496,28 +521,35 @@ async function executeWebhookJobInternal( throw new Error(`Webhook record not found: ${payload.webhookId}`) } - const secretEnvironment = await getPersonalAndWorkspaceEnv(workflowRecord.userId, workspaceId) - const resolvedSecretTraceRegistry = await createResolvedSecretTraceRegistry({ - personalEncrypted: secretEnvironment.personalEncrypted, - workspaceEncrypted: secretEnvironment.workspaceEncrypted, - personalDecrypted: secretEnvironment.personalDecrypted, - workspaceDecrypted: secretEnvironment.workspaceDecrypted, - decryptionFailures: secretEnvironment.decryptionFailures, - scope: { userId: workflowRecord.userId, workspaceId }, - }) - loggingSession.setResolvedSecretTraceRegistry(resolvedSecretTraceRegistry) - const secretEnvVars = { - ...secretEnvironment.personalDecrypted, - ...secretEnvironment.workspaceDecrypted, - } + const secretScope = { userId: workflowRecord.userId, workspaceId } + let resolvedSecretTraceRegistry = createIncompleteResolvedSecretTraceRegistry(secretScope) const resolvedWebhookRecord = await resolveWebhookExecutionProviderConfig( webhookRecord, payload.provider, workflowRecord.userId, workspaceId, { - envVars: secretEnvVars, - onResolved: (name, value) => resolvedSecretTraceRegistry.recordResolved(name, value), + onEnvironmentSnapshot: async (secretEnvironment) => { + try { + resolvedSecretTraceRegistry = await createResolvedSecretTraceRegistry({ + personalEncrypted: secretEnvironment.personalEncrypted, + workspaceEncrypted: secretEnvironment.workspaceEncrypted, + personalDecrypted: secretEnvironment.personalDecrypted, + workspaceDecrypted: secretEnvironment.workspaceDecrypted, + decryptionFailures: secretEnvironment.decryptionFailures, + scope: secretScope, + }) + } catch (error) { + logger.warn(`[${requestId}] Failed to build webhook trace secret catalog`, { + error: toError(error).message, + }) + resolvedSecretTraceRegistry = createIncompleteResolvedSecretTraceRegistry(secretScope) + } + loggingSession.setResolvedSecretTraceRegistry(resolvedSecretTraceRegistry) + }, + onResolved: (name, value) => { + resolvedSecretTraceRegistry.recordResolved(name, value) + }, } ) diff --git a/apps/sim/executor/utils/resolved-secret-trace-registry.test.ts b/apps/sim/executor/utils/resolved-secret-trace-registry.test.ts index 70487660b6d..6b435afff6d 100644 --- a/apps/sim/executor/utils/resolved-secret-trace-registry.test.ts +++ b/apps/sim/executor/utils/resolved-secret-trace-registry.test.ts @@ -11,6 +11,7 @@ vi.mock('@/lib/core/security/encryption', () => ({ import { ANONYMOUS_SECRET_TRACE_REPLACEMENT, createResolvedSecretTraceRegistry, + isResolvedSecretTraceProvenanceV1, ResolvedSecretTraceProvenanceAccumulator, type ResolvedSecretTraceProvenanceV1, ResolvedSecretTraceRegistry, @@ -19,13 +20,13 @@ import { describe('ResolvedSecretTraceProvenanceAccumulator', () => { const scope = { userId: 'user-1', workspaceId: 'workspace-1' } - it('unions cold, warm, and retry reports while keeping completeness sticky', () => { + it('unions cold, warm, and retry reports while complete', () => { const accumulator = new ResolvedSecretTraceProvenanceAccumulator(scope) expect( accumulator.record({ version: 1, - complete: false, + complete: true, entries: [{ name: 'OLD_TOKEN', encryptedValue: 'encrypted-v1' }], scope, }) @@ -47,7 +48,7 @@ describe('ResolvedSecretTraceProvenanceAccumulator', () => { expect(accumulator.exportProvenance()).toEqual({ version: 1, - complete: false, + complete: true, entries: [ { name: 'OLD_TOKEN', encryptedValue: 'encrypted-v1' }, { name: 'NEW_TOKEN', encryptedValue: 'encrypted-v2' }, @@ -56,7 +57,41 @@ describe('ResolvedSecretTraceProvenanceAccumulator', () => { }) }) - it('anonymizes mismatched-scope reports and marks them incomplete', () => { + it('discards accumulated and future entries once completeness is lost', () => { + const accumulator = new ResolvedSecretTraceProvenanceAccumulator(scope) + + accumulator.record({ + version: 1, + complete: true, + entries: [{ name: 'OLD_TOKEN', encryptedValue: 'encrypted-v1' }], + scope, + }) + expect( + accumulator.record({ + version: 1, + complete: false, + entries: [], + scope, + }) + ).toBe(true) + expect( + accumulator.record({ + version: 1, + complete: true, + entries: [{ name: 'NEW_TOKEN', encryptedValue: 'encrypted-v2' }], + scope, + }) + ).toBe(true) + + expect(accumulator.exportProvenance()).toEqual({ + version: 1, + complete: false, + entries: [], + scope, + }) + }) + + it('discards mismatched-scope reports and marks them incomplete', () => { const accumulator = new ResolvedSecretTraceProvenanceAccumulator(scope) expect( @@ -71,12 +106,12 @@ describe('ResolvedSecretTraceProvenanceAccumulator', () => { expect(accumulator.exportProvenance()).toEqual({ version: 1, complete: false, - entries: [{ encryptedValue: 'encrypted-value' }], + entries: [], scope, }) }) - it('fails closed for malformed reports and supports terminal entry discard', () => { + it('fails closed for malformed reports and terminal incompleteness', () => { const accumulator = new ResolvedSecretTraceProvenanceAccumulator(scope) accumulator.record({ version: 1, @@ -99,7 +134,7 @@ describe('ResolvedSecretTraceProvenanceAccumulator', () => { entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-value' }], scope, }) - accumulator.markIncomplete({ discardEntries: true }) + accumulator.markIncomplete() expect(accumulator.exportProvenance().entries).toEqual([]) }) }) @@ -121,6 +156,7 @@ describe('ResolvedSecretTraceRegistry', () => { expect(registry.recordResolved('API_KEY', 'wrong-value')).toBe(false) expect(registry.recordResolved('MISSING', 'secret-value')).toBe(false) expect(registry.getActiveMatches()).toEqual([]) + expect(registry.isComplete()).toBe(false) expect(registry.recordResolved('API_KEY', 'secret-value')).toBe(true) expect(registry.getActiveMatches()).toEqual([ @@ -136,7 +172,6 @@ describe('ResolvedSecretTraceRegistry', () => { workspaceDecrypted: { SHARED: 'workspace-secret' }, }) - expect(registry.recordResolved('SHARED', 'personal-secret')).toBe(false) expect(registry.recordResolved('SHARED', 'workspace-secret')).toBe(true) expect(registry.exportProvenance()).toEqual({ version: 1, @@ -145,7 +180,7 @@ describe('ResolvedSecretTraceRegistry', () => { }) }) - it('does not make current decryption failures or decrypted-only keys globally incomplete', async () => { + it('ignores empty decryption failures but fails closed for a resolved value outside the catalog', async () => { const registry = await createResolvedSecretTraceRegistry({ personalEncrypted: { FAILED: 'failed-ciphertext' }, workspaceEncrypted: {}, @@ -156,7 +191,9 @@ describe('ResolvedSecretTraceRegistry', () => { expect(registry.isComplete()).toBe(true) expect(registry.recordResolved('FAILED', '')).toBe(false) + expect(registry.isComplete()).toBe(true) expect(registry.recordResolved('DECRYPTED_ONLY', 'not-catalogued')).toBe(false) + expect(registry.isComplete()).toBe(false) }) it('keeps a successful workspace override when the shadowed personal value failed', async () => { @@ -514,6 +551,73 @@ describe('ResolvedSecretTraceRegistry', () => { expect(registry.exportProvenance().entries).toEqual([]) }) + it('bounds provenance by serialized JSON bytes including control-character escapes', () => { + const encryptedValue = '\u0000'.repeat(1_400_000) + const provenance: ResolvedSecretTraceProvenanceV1 = { + version: 1, + complete: true, + entries: [{ name: 'TOKEN', encryptedValue }], + } + + expect(Buffer.byteLength(encryptedValue, 'utf8')).toBeLessThan(8 * 1024 * 1024) + expect(Buffer.byteLength(JSON.stringify(provenance), 'utf8')).toBeGreaterThan(8 * 1024 * 1024) + expect(isResolvedSecretTraceProvenanceV1(provenance)).toBe(false) + + const registry = new ResolvedSecretTraceRegistry([ + { name: 'TOKEN', plaintext: 'secret', encryptedValue }, + ]) + expect(registry.recordResolved('TOKEN', 'secret')).toBe(true) + expect(registry.isComplete()).toBe(false) + expect(registry.exportProvenance().entries).toEqual([]) + }) + + it('rejects incomplete provenance that still carries entries', () => { + expect( + isResolvedSecretTraceProvenanceV1({ + version: 1, + complete: false, + entries: [{ name: 'TOKEN', encryptedValue: 'ciphertext' }], + }) + ).toBe(false) + }) + + it('rejects non-canonical provenance fields before applying the serialized-size bound', () => { + const entry = { name: 'TOKEN', encryptedValue: 'ciphertext' } + const scope = { userId: 'user-1', workspaceId: 'workspace-1' } + + expect( + isResolvedSecretTraceProvenanceV1({ + version: 1, + complete: true, + entries: [entry], + scope, + extra: 'not-transported', + }) + ).toBe(false) + expect( + isResolvedSecretTraceProvenanceV1({ + version: 1, + complete: true, + entries: [{ ...entry, extra: 'not-transported' }], + scope, + }) + ).toBe(false) + expect( + isResolvedSecretTraceProvenanceV1({ + version: 1, + complete: true, + entries: [entry], + scope: { ...scope, extra: 'not-transported' }, + }) + ).toBe(false) + + const entries = [entry] + Object.assign(entries, { extra: 'not-transported' }) + expect(isResolvedSecretTraceProvenanceV1({ version: 1, complete: true, entries, scope })).toBe( + false + ) + }) + it('stops consuming a dormant catalog when its entry cap is exceeded', () => { let yieldedEntries = 0 function* catalogEntries() { diff --git a/apps/sim/executor/utils/resolved-secret-trace-registry.ts b/apps/sim/executor/utils/resolved-secret-trace-registry.ts index 6d7b9605040..632a1a56872 100644 --- a/apps/sim/executor/utils/resolved-secret-trace-registry.ts +++ b/apps/sim/executor/utils/resolved-secret-trace-registry.ts @@ -5,13 +5,16 @@ import { isLargeValueRef } from '@/lib/execution/payloads/large-value-ref' export const ANONYMOUS_SECRET_TRACE_REPLACEMENT = '[REDACTED_SECRET]' const MAX_PROVENANCE_ENTRIES = 10_000 -const MAX_PROVENANCE_BYTES = 8 * 1024 * 1024 +const MAX_SERIALIZED_PROVENANCE_BYTES = 8 * 1024 * 1024 const MAX_TRACE_CATALOG_ENTRIES = MAX_PROVENANCE_ENTRIES -const MAX_TRACE_CATALOG_BYTES = MAX_PROVENANCE_BYTES +const MAX_TRACE_CATALOG_BYTES = 8 * 1024 * 1024 const MAX_PROVENANCE_FILTER_NODES = 50_000 const MAX_PROVENANCE_FILTER_CHARACTERS = 16 * 1024 * 1024 const MAX_PROVENANCE_FILTER_COMPARISONS = 1_000_000 const ERROR_CONTENT_PROPERTY_NAMES = ['name', 'message', 'stack', 'cause', 'errors'] as const +const PROVENANCE_PROPERTY_NAMES = new Set(['version', 'complete', 'entries', 'scope']) +const PROVENANCE_ENTRY_PROPERTY_NAMES = new Set(['encryptedValue', 'name']) +const PROVENANCE_SCOPE_PROPERTY_NAMES = new Set(['userId', 'workspaceId']) export interface ResolvedSecretTraceCatalogEntry { name: string @@ -72,10 +75,146 @@ function compareStrings(left: string, right: string): number { return 0 } +function cloneProvenanceScope(scope: ResolvedSecretTraceScopeV1): ResolvedSecretTraceScopeV1 { + return { + userId: scope.userId, + ...(scope.workspaceId ? { workspaceId: scope.workspaceId } : {}), + } +} + +function serializedJsonStringByteSize(value: string): number { + let byteSize = 2 + for (let index = 0; index < value.length; index++) { + const codeUnit = value.charCodeAt(index) + if (codeUnit === 0x22 || codeUnit === 0x5c) { + byteSize += 2 + } else if ( + codeUnit === 0x08 || + codeUnit === 0x09 || + codeUnit === 0x0a || + codeUnit === 0x0c || + codeUnit === 0x0d + ) { + byteSize += 2 + } else if (codeUnit <= 0x1f) { + byteSize += 6 + } else if (codeUnit >= 0xd800 && codeUnit <= 0xdbff) { + const nextCodeUnit = value.charCodeAt(index + 1) + if (nextCodeUnit >= 0xdc00 && nextCodeUnit <= 0xdfff) { + byteSize += 4 + index++ + } else { + byteSize += 6 + } + } else if (codeUnit >= 0xdc00 && codeUnit <= 0xdfff) { + byteSize += 6 + } else if (codeUnit <= 0x7f) { + byteSize++ + } else if (codeUnit <= 0x7ff) { + byteSize += 2 + } else { + byteSize += 3 + } + } + return byteSize +} + +function serializedProvenanceEntryByteSize(entry: ResolvedSecretTraceProvenanceEntryV1): number { + let byteSize = + Buffer.byteLength('{"encryptedValue":', 'utf8') + + serializedJsonStringByteSize(entry.encryptedValue) + if (entry.name !== undefined) { + byteSize += Buffer.byteLength(',"name":', 'utf8') + serializedJsonStringByteSize(entry.name) + } + return byteSize + 1 +} + +function serializedProvenanceEnvelopeByteSize( + complete: boolean, + scope: ResolvedSecretTraceScopeV1 | undefined +): number { + let byteSize = Buffer.byteLength( + `{"version":1,"complete":${complete ? 'true' : 'false'},"entries":[]`, + 'utf8' + ) + if (scope) { + byteSize += + Buffer.byteLength(',"scope":{"userId":', 'utf8') + serializedJsonStringByteSize(scope.userId) + if (scope.workspaceId !== undefined) { + byteSize += + Buffer.byteLength(',"workspaceId":', 'utf8') + + serializedJsonStringByteSize(scope.workspaceId) + } + byteSize++ + } + return byteSize + 1 +} + +function isSerializedProvenanceWithinLimit( + complete: boolean, + entries: readonly ResolvedSecretTraceProvenanceEntryV1[], + scope: ResolvedSecretTraceScopeV1 | undefined +): boolean { + let byteSize = serializedProvenanceEnvelopeByteSize(complete, scope) + for (let index = 0; index < entries.length; index++) { + byteSize += serializedProvenanceEntryByteSize(entries[index]) + (index === 0 ? 0 : 1) + if (byteSize > MAX_SERIALIZED_PROVENANCE_BYTES) return false + } + return byteSize <= MAX_SERIALIZED_PROVENANCE_BYTES +} + +function toProvenanceEntry(entry: ActiveSecretEntry): ResolvedSecretTraceProvenanceEntryV1 { + return { + encryptedValue: entry.encryptedValue, + ...(!entry.anonymous && entry.name ? { name: entry.name } : {}), + } +} + function hasOwn(record: Record, name: string): boolean { return Object.hasOwn(record, name) } +function isExactPlainDataRecord( + value: unknown, + allowedProperties: ReadonlySet, + requiredProperties: readonly string[] +): value is Record { + if (value === null || typeof value !== 'object' || Array.isArray(value)) return false + + try { + const prototype = Object.getPrototypeOf(value) + if (prototype !== Object.prototype && prototype !== null) return false + + const properties = Reflect.ownKeys(value) + for (const property of properties) { + if (typeof property !== 'string' || !allowedProperties.has(property)) return false + const descriptor = Object.getOwnPropertyDescriptor(value, property) + if (!descriptor?.enumerable || !('value' in descriptor)) return false + } + return requiredProperties.every((property) => Object.hasOwn(value, property)) + } catch { + return false + } +} + +function isExactProvenanceEntriesArray(value: unknown): value is unknown[] { + if (!Array.isArray(value)) return false + + try { + if (Object.getPrototypeOf(value) !== Array.prototype) return false + const properties = Reflect.ownKeys(value) + if (properties.length !== value.length + 1 || !properties.includes('length')) return false + + for (let index = 0; index < value.length; index++) { + const descriptor = Object.getOwnPropertyDescriptor(value, String(index)) + if (!descriptor?.enumerable || !('value' in descriptor)) return false + } + return true + } catch { + return false + } +} + function catalogEntryByteSize(entry: ResolvedSecretTraceCatalogEntry): number { return ( Buffer.byteLength(entry.name, 'utf8') + @@ -127,8 +266,10 @@ function* iterateEffectiveCatalogEntries( } function isProvenanceEntry(value: unknown): value is ResolvedSecretTraceProvenanceEntryV1 { - if (value === null || typeof value !== 'object') return false - const entry = value as Record + if (!isExactPlainDataRecord(value, PROVENANCE_ENTRY_PROPERTY_NAMES, ['encryptedValue'])) { + return false + } + const entry = value return ( typeof entry.encryptedValue === 'string' && entry.encryptedValue.length > 0 && @@ -137,8 +278,8 @@ function isProvenanceEntry(value: unknown): value is ResolvedSecretTraceProvenan } function isProvenanceScope(value: unknown): value is ResolvedSecretTraceScopeV1 { - if (value === null || typeof value !== 'object') return false - const scope = value as Record + if (!isExactPlainDataRecord(value, PROVENANCE_SCOPE_PROPERTY_NAMES, ['userId'])) return false + const scope = value return ( typeof scope.userId === 'string' && scope.userId.length > 0 && @@ -163,32 +304,29 @@ function scopesMatch( export function isResolvedSecretTraceProvenanceV1( value: unknown ): value is ResolvedSecretTraceProvenanceV1 { - if (value === null || typeof value !== 'object') return false - const provenance = value as Record + if ( + !isExactPlainDataRecord(value, PROVENANCE_PROPERTY_NAMES, ['version', 'complete', 'entries']) + ) { + return false + } + const provenance = value if ( provenance.version !== 1 || typeof provenance.complete !== 'boolean' || - !Array.isArray(provenance.entries) || + !isExactProvenanceEntriesArray(provenance.entries) || provenance.entries.length > MAX_PROVENANCE_ENTRIES || !provenance.entries.every(isProvenanceEntry) || + (!provenance.complete && provenance.entries.length > 0) || (provenance.scope !== undefined && !isProvenanceScope(provenance.scope)) ) { return false } - let byteSize = 0 - for (const entry of provenance.entries) { - byteSize += Buffer.byteLength(entry.encryptedValue, 'utf8') - if (entry.name) byteSize += Buffer.byteLength(entry.name, 'utf8') - if (byteSize > MAX_PROVENANCE_BYTES) return false - } - if (provenance.scope) { - byteSize += Buffer.byteLength(provenance.scope.userId, 'utf8') - if (provenance.scope.workspaceId) { - byteSize += Buffer.byteLength(provenance.scope.workspaceId, 'utf8') - } - } - return byteSize <= MAX_PROVENANCE_BYTES + return isSerializedProvenanceWithinLimit( + provenance.complete, + provenance.entries, + provenance.scope + ) } /** @@ -201,7 +339,7 @@ export class ResolvedSecretTraceProvenanceAccumulator { private provenance: ResolvedSecretTraceProvenanceV1 constructor(scope?: ResolvedSecretTraceScopeV1) { - this.scope = scope ? { ...scope } : undefined + this.scope = scope ? cloneProvenanceScope(scope) : undefined this.provenance = this.emptyProvenance(true) } @@ -213,6 +351,12 @@ export class ResolvedSecretTraceProvenanceAccumulator { } const sameScope = scopesMatch(provenance.scope, this.scope) + const complete = this.provenance.complete && provenance.complete && sameScope + if (!complete) { + this.provenance = this.emptyProvenance(false) + return true + } + const entries = new Map( this.provenance.entries.map((entry) => [ `${entry.name ?? ''}\u0000${entry.encryptedValue}`, @@ -220,15 +364,18 @@ export class ResolvedSecretTraceProvenanceAccumulator { ]) ) for (const entry of provenance.entries) { - const scopedEntry = sameScope ? entry : { encryptedValue: entry.encryptedValue } + const scopedEntry: ResolvedSecretTraceProvenanceEntryV1 = { + encryptedValue: entry.encryptedValue, + ...(entry.name ? { name: entry.name } : {}), + } entries.set(`${scopedEntry.name ?? ''}\u0000${scopedEntry.encryptedValue}`, scopedEntry) } const merged: ResolvedSecretTraceProvenanceV1 = { version: 1, - complete: this.provenance.complete && provenance.complete && sameScope, + complete: true, entries: [...entries.values()], - ...(this.scope ? { scope: { ...this.scope } } : {}), + ...(this.scope ? { scope: cloneProvenanceScope(this.scope) } : {}), } if (!isResolvedSecretTraceProvenanceV1(merged)) { this.provenance = this.emptyProvenance(false) @@ -239,13 +386,9 @@ export class ResolvedSecretTraceProvenanceAccumulator { return true } - /** Marks the invocation incomplete, optionally discarding entries on a fail-closed boundary. */ - markIncomplete(options: { discardEntries?: boolean } = {}): void { - this.provenance = { - ...this.provenance, - complete: false, - ...(options.discardEntries ? { entries: [] } : {}), - } + /** Marks the invocation incomplete and discards entries that can no longer be trusted. */ + markIncomplete(): void { + this.provenance = this.emptyProvenance(false) } exportProvenance(): ResolvedSecretTraceProvenanceV1 { @@ -257,7 +400,7 @@ export class ResolvedSecretTraceProvenanceAccumulator { version: 1, complete, entries: [], - ...(this.scope ? { scope: { ...this.scope } } : {}), + ...(this.scope ? { scope: cloneProvenanceScope(this.scope) } : {}), } } } @@ -270,15 +413,20 @@ export class ResolvedSecretTraceRegistry { private readonly catalog = new Map() private catalogBytes = 0 private readonly activeEntries = new Map() - private activeProvenanceBytes = 0 + private activeProvenanceEntryBytes = 0 private complete = true private readonly scope?: ResolvedSecretTraceScopeV1 + private readonly completeProvenanceEnvelopeBytes: number constructor( catalogEntries: Iterable = [], scope?: ResolvedSecretTraceScopeV1 ) { - this.scope = scope ? { ...scope } : undefined + this.scope = scope ? cloneProvenanceScope(scope) : undefined + this.completeProvenanceEnvelopeBytes = serializedProvenanceEnvelopeByteSize(true, this.scope) + if (this.completeProvenanceEnvelopeBytes > MAX_SERIALIZED_PROVENANCE_BYTES) { + this.markIncomplete() + } let catalogEntriesSeen = 0 for (const entry of catalogEntries) { catalogEntriesSeen++ @@ -295,6 +443,7 @@ export class ResolvedSecretTraceRegistry { if (resolvedValue.length === 0) return false const catalogEntry = this.catalog.get(name) if (!catalogEntry || catalogEntry.plaintext !== resolvedValue) { + this.markIncomplete() return false } @@ -408,7 +557,7 @@ export class ResolvedSecretTraceRegistry { version: 1, complete: this.complete, entries, - ...(this.scope ? { scope: { ...this.scope } } : {}), + ...(this.scope ? { scope: cloneProvenanceScope(this.scope) } : {}), } } @@ -535,12 +684,15 @@ export class ResolvedSecretTraceRegistry { } } - const entries = this.buildProvenanceEntries([...matchedEntries.values()], options.anonymous) + const complete = this.complete && scanComplete + const entries = complete + ? this.buildProvenanceEntries([...matchedEntries.values()], options.anonymous) + : [] return { version: 1, - complete: this.complete && scanComplete, + complete, entries, - ...(this.scope ? { scope: { ...this.scope } } : {}), + ...(this.scope ? { scope: cloneProvenanceScope(this.scope) } : {}), } } @@ -551,17 +703,21 @@ export class ResolvedSecretTraceRegistry { return } - const entryBytes = - Buffer.byteLength(entry.encryptedValue, 'utf8') + Buffer.byteLength(entry.name, 'utf8') + const entryBytes = serializedProvenanceEntryByteSize(toProvenanceEntry(entry)) + const separatorBytes = this.activeEntries.size === 0 ? 0 : 1 if ( this.activeEntries.size >= MAX_PROVENANCE_ENTRIES || - this.activeProvenanceBytes + entryBytes > MAX_PROVENANCE_BYTES + this.completeProvenanceEnvelopeBytes + + this.activeProvenanceEntryBytes + + separatorBytes + + entryBytes > + MAX_SERIALIZED_PROVENANCE_BYTES ) { this.markIncomplete() return } this.activeEntries.set(key, entry) - this.activeProvenanceBytes += entryBytes + this.activeProvenanceEntryBytes += separatorBytes + entryBytes } private addCatalogEntry(entry: ResolvedSecretTraceCatalogEntry): boolean { @@ -592,8 +748,7 @@ export class ResolvedSecretTraceRegistry { compareStrings(left.encryptedValue, right.encryptedValue) ) .map((entry) => ({ - encryptedValue: entry.encryptedValue, - ...(!forceAnonymous && !entry.anonymous && entry.name ? { name: entry.name } : {}), + ...toProvenanceEntry({ ...entry, anonymous: forceAnonymous || entry.anonymous }), })) } } @@ -618,3 +773,12 @@ export async function createResolvedSecretTraceRegistry( return registry } + +/** Creates a scoped fail-closed registry when trusted catalog provenance is unavailable. */ +export function createIncompleteResolvedSecretTraceRegistry( + scope?: ResolvedSecretTraceScopeV1 +): ResolvedSecretTraceRegistry { + const registry = new ResolvedSecretTraceRegistry([], scope) + registry.markIncomplete() + return registry +} diff --git a/apps/sim/lib/environment/utils.test.ts b/apps/sim/lib/environment/utils.test.ts new file mode 100644 index 00000000000..b8c0905bff7 --- /dev/null +++ b/apps/sim/lib/environment/utils.test.ts @@ -0,0 +1,83 @@ +/** + * @vitest-environment node + */ +import { dbChainMockFns, encryptionMock, encryptionMockFns, resetDbChainMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.unmock('@/lib/environment/utils') +vi.mock('@/lib/core/security/encryption', () => encryptionMock) + +import { + getEffectiveDecryptedEnv, + getEffectiveEnvironmentSnapshot, + invalidateEffectiveDecryptedEnvCache, +} from '@/lib/environment/utils' + +describe('effective environment resolution cache', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + encryptionMockFns.mockDecryptSecret.mockReset() + encryptionMockFns.mockEncryptSecret.mockReset() + invalidateEffectiveDecryptedEnvCache({ userId: 'user-1' }) + dbChainMockFns.limit.mockResolvedValue([{ variables: { API_KEY: 'encrypted-value' } }]) + encryptionMockFns.mockDecryptSecret.mockResolvedValue({ decrypted: 'runtime-value' }) + }) + + it('shares one atomic snapshot and returns defensive clones', async () => { + const [decrypted, snapshot] = await Promise.all([ + getEffectiveDecryptedEnv('user-1'), + getEffectiveEnvironmentSnapshot('user-1'), + ]) + + expect(decrypted).toEqual({ API_KEY: 'runtime-value' }) + expect(snapshot).toMatchObject({ + personalEncrypted: { API_KEY: 'encrypted-value' }, + personalDecrypted: { API_KEY: 'runtime-value' }, + }) + expect(encryptionMockFns.mockDecryptSecret).toHaveBeenCalledOnce() + + decrypted.API_KEY = 'mutated-runtime' + snapshot.personalEncrypted.API_KEY = 'mutated-ciphertext' + snapshot.personalDecrypted.API_KEY = 'mutated-snapshot' + snapshot.conflicts.push('MUTATED') + + await expect(getEffectiveDecryptedEnv('user-1')).resolves.toEqual({ + API_KEY: 'runtime-value', + }) + await expect(getEffectiveEnvironmentSnapshot('user-1')).resolves.toMatchObject({ + personalEncrypted: { API_KEY: 'encrypted-value' }, + personalDecrypted: { API_KEY: 'runtime-value' }, + conflicts: [], + }) + expect(encryptionMockFns.mockDecryptSecret).toHaveBeenCalledOnce() + }) + + it('evicts rejected loads and retries the canonical lookup', async () => { + dbChainMockFns.limit.mockRejectedValueOnce(new Error('database unavailable')) + + await expect(getEffectiveEnvironmentSnapshot('user-1')).rejects.toThrow('database unavailable') + + dbChainMockFns.limit.mockResolvedValue([{ variables: { API_KEY: 'encrypted-value' } }]) + await expect(getEffectiveDecryptedEnv('user-1')).resolves.toEqual({ + API_KEY: 'runtime-value', + }) + expect(encryptionMockFns.mockDecryptSecret).toHaveBeenCalledOnce() + }) + + it('reloads the full snapshot after invalidation', async () => { + await expect(getEffectiveDecryptedEnv('user-1')).resolves.toEqual({ + API_KEY: 'runtime-value', + }) + + invalidateEffectiveDecryptedEnvCache({ userId: 'user-1' }) + dbChainMockFns.limit.mockResolvedValue([{ variables: { API_KEY: 'rotated-ciphertext' } }]) + encryptionMockFns.mockDecryptSecret.mockResolvedValue({ decrypted: 'rotated-runtime' }) + + await expect(getEffectiveEnvironmentSnapshot('user-1')).resolves.toMatchObject({ + personalEncrypted: { API_KEY: 'rotated-ciphertext' }, + personalDecrypted: { API_KEY: 'rotated-runtime' }, + }) + expect(encryptionMockFns.mockDecryptSecret).toHaveBeenCalledTimes(2) + }) +}) diff --git a/apps/sim/lib/environment/utils.ts b/apps/sim/lib/environment/utils.ts index f85989cecb0..f6527b1764a 100644 --- a/apps/sim/lib/environment/utils.ts +++ b/apps/sim/lib/environment/utils.ts @@ -14,26 +14,44 @@ import { import { checkWorkspaceAccess, type WorkspaceAccess } from '@/lib/workspaces/permissions/utils' const logger = createLogger('EnvironmentUtils') -const EFFECTIVE_DECRYPTED_ENV_CACHE_TTL_MS = 2_000 -const EFFECTIVE_DECRYPTED_ENV_CACHE_MAX_ENTRIES = 1_000 +const EFFECTIVE_ENVIRONMENT_CACHE_TTL_MS = 2_000 +const EFFECTIVE_ENVIRONMENT_CACHE_MAX_ENTRIES = 1_000 -interface EffectiveDecryptedEnvCacheEntry { +export interface EnvironmentResolutionSnapshot { + personalEncrypted: Record + workspaceEncrypted: Record + personalDecrypted: Record + workspaceDecrypted: Record + conflicts: string[] + decryptionFailures: string[] +} + +interface EffectiveEnvironmentCacheEntry { userId: string workspaceId?: string - promise: Promise> + promise: Promise } -const effectiveDecryptedEnvCache = new LRUCache({ - max: EFFECTIVE_DECRYPTED_ENV_CACHE_MAX_ENTRIES, - ttl: EFFECTIVE_DECRYPTED_ENV_CACHE_TTL_MS, +const effectiveEnvironmentCache = new LRUCache({ + max: EFFECTIVE_ENVIRONMENT_CACHE_MAX_ENTRIES, + ttl: EFFECTIVE_ENVIRONMENT_CACHE_TTL_MS, }) -function getEffectiveDecryptedEnvCacheKey(userId: string, workspaceId?: string): string { +function getEffectiveEnvironmentCacheKey(userId: string, workspaceId?: string): string { return JSON.stringify([userId, workspaceId ?? null]) } -function cloneEnvVars(envVars: Record): Record { - return { ...envVars } +function cloneEnvironmentResolutionSnapshot( + snapshot: EnvironmentResolutionSnapshot +): EnvironmentResolutionSnapshot { + return { + personalEncrypted: { ...snapshot.personalEncrypted }, + workspaceEncrypted: { ...snapshot.workspaceEncrypted }, + personalDecrypted: { ...snapshot.personalDecrypted }, + workspaceDecrypted: { ...snapshot.workspaceDecrypted }, + conflicts: [...snapshot.conflicts], + decryptionFailures: [...snapshot.decryptionFailures], + } } export function invalidateEffectiveDecryptedEnvCache(input: { @@ -43,13 +61,13 @@ export function invalidateEffectiveDecryptedEnvCache(input: { const { userId, workspaceId } = input if (!userId && !workspaceId) return - effectiveDecryptedEnvCache.forEach((entry, cacheKey) => { + effectiveEnvironmentCache.forEach((entry, cacheKey) => { if (userId && entry.userId === userId) { - effectiveDecryptedEnvCache.delete(cacheKey) + effectiveEnvironmentCache.delete(cacheKey) return } if (workspaceId && entry.workspaceId === workspaceId) { - effectiveDecryptedEnvCache.delete(cacheKey) + effectiveEnvironmentCache.delete(cacheKey) } }) } @@ -94,14 +112,7 @@ export async function getPersonalAndWorkspaceEnv( userId: string, workspaceId?: string, options?: { workspaceAccess?: WorkspaceAccess } -): Promise<{ - personalEncrypted: Record - workspaceEncrypted: Record - personalDecrypted: Record - workspaceDecrypted: Record - conflicts: string[] - decryptionFailures: string[] -}> { +): Promise { let workspaceCanAdmin = false if (workspaceId) { const access = options?.workspaceAccess ?? (await checkWorkspaceAccess(workspaceId, userId)) @@ -357,34 +368,52 @@ export async function upsertWorkspaceEnvVars( return updatedKeys } -/** - * Returns a merged decrypted env map for webhook/copilot/MCP config resolution. - */ -export async function getEffectiveDecryptedEnv( +async function getCachedEnvironmentResolutionSnapshot( userId: string, workspaceId?: string -): Promise> { - const cacheKey = getEffectiveDecryptedEnvCacheKey(userId, workspaceId) - const cached = effectiveDecryptedEnvCache.get(cacheKey) +): Promise { + const cacheKey = getEffectiveEnvironmentCacheKey(userId, workspaceId) + const cached = effectiveEnvironmentCache.get(cacheKey) if (cached) { - return cloneEnvVars(await cached.promise) + return cached.promise } - const promise = getPersonalAndWorkspaceEnv(userId, workspaceId) - .then(({ personalDecrypted, workspaceDecrypted }) => ({ - ...personalDecrypted, - ...workspaceDecrypted, - })) - .catch((error) => { - effectiveDecryptedEnvCache.delete(cacheKey) - throw error - }) + const promise = getPersonalAndWorkspaceEnv(userId, workspaceId).catch((error) => { + effectiveEnvironmentCache.delete(cacheKey) + throw error + }) - effectiveDecryptedEnvCache.set(cacheKey, { + effectiveEnvironmentCache.set(cacheKey, { userId, workspaceId, promise, }) - return cloneEnvVars(await promise) + return promise +} + +/** + * Returns a defensive clone of the cached environment snapshot used for runtime resolution. + */ +export async function getEffectiveEnvironmentSnapshot( + userId: string, + workspaceId?: string +): Promise { + return cloneEnvironmentResolutionSnapshot( + await getCachedEnvironmentResolutionSnapshot(userId, workspaceId) + ) +} + +/** + * Returns a merged decrypted env map for webhook/copilot/MCP config resolution. + */ +export async function getEffectiveDecryptedEnv( + userId: string, + workspaceId?: string +): Promise> { + const { personalDecrypted, workspaceDecrypted } = await getCachedEnvironmentResolutionSnapshot( + userId, + workspaceId + ) + return { ...personalDecrypted, ...workspaceDecrypted } } diff --git a/apps/sim/lib/logs/execution/logger.ts b/apps/sim/lib/logs/execution/logger.ts index 1fb46351681..897275d78f8 100644 --- a/apps/sim/lib/logs/execution/logger.ts +++ b/apps/sim/lib/logs/execution/logger.ts @@ -801,11 +801,10 @@ export class ExecutionLogger implements IExecutionLoggerService { : redactedState } - async prepareTraceSpansForProjection(params: { + async loadTraceSpansForProjection(params: { executionId: string workflowId: string workspaceId: string | null - userId?: string | null traceSpans: TraceSpan[] isResume?: boolean }): Promise { @@ -829,7 +828,17 @@ export class ExecutionLogger implements IExecutionLoggerService { } } - const filtered = filterForDisplay(sourceSpans) + return sourceSpans + } + + async prepareTraceSpansForProjection(params: { + workflowId: string + executionId: string + workspaceId: string | null + userId?: string | null + traceSpans: TraceSpan[] + }): Promise { + const filtered = filterForDisplay(params.traceSpans) const redacted = redactApiKeys(filtered) const pii = await this.applyPiiRedaction( params.workspaceId, diff --git a/apps/sim/lib/logs/execution/logging-session.test.ts b/apps/sim/lib/logs/execution/logging-session.test.ts index 7fd9aa26b3b..daef03e2971 100644 --- a/apps/sim/lib/logs/execution/logging-session.test.ts +++ b/apps/sim/lib/logs/execution/logging-session.test.ts @@ -9,6 +9,7 @@ const dbMocks = vi.hoisted(() => ({ const { completeWorkflowExecutionMock, + loadTraceSpansForProjectionMock, prepareTraceSpansForProjectionMock, startWorkflowExecutionMock, loadWorkflowStateForExecutionMock, @@ -17,6 +18,7 @@ const { workflowExecutedMock, } = vi.hoisted(() => ({ completeWorkflowExecutionMock: vi.fn(), + loadTraceSpansForProjectionMock: vi.fn(), prepareTraceSpansForProjectionMock: vi.fn(), startWorkflowExecutionMock: vi.fn(), loadWorkflowStateForExecutionMock: vi.fn(), @@ -25,6 +27,11 @@ const { workflowExecutedMock: vi.fn(), })) +const { materializeLargeValueRefMock, storeLargeValueMock } = vi.hoisted(() => ({ + materializeLargeValueRefMock: vi.fn(), + storeLargeValueMock: vi.fn(), +})) + vi.mock('drizzle-orm', () => ({ eq: dbMocks.eq, and: dbMocks.and, @@ -35,6 +42,7 @@ vi.mock('@/lib/logs/execution/logger', () => ({ executionLogger: { startWorkflowExecution: startWorkflowExecutionMock, completeWorkflowExecution: completeWorkflowExecutionMock, + loadTraceSpansForProjection: loadTraceSpansForProjectionMock, prepareTraceSpansForProjection: prepareTraceSpansForProjectionMock, }, })) @@ -48,6 +56,11 @@ vi.mock('@/lib/core/telemetry', () => ({ PlatformEvents: { workflowExecuted: workflowExecutedMock }, })) +vi.mock('@/lib/execution/payloads/store', () => ({ + materializeLargeValueRef: materializeLargeValueRefMock, + storeLargeValue: storeLargeValueMock, +})) + const { setLastStartedBlockMock, setLastCompletedBlockMock, @@ -141,6 +154,9 @@ describe('LoggingSession terminal provenance', () => { }) beforeEach(() => { + loadTraceSpansForProjectionMock.mockImplementation( + async ({ traceSpans }: { traceSpans: unknown[] }) => traceSpans + ) prepareTraceSpansForProjectionMock.mockImplementation( async ({ traceSpans }: { traceSpans: unknown[] }) => traceSpans ) @@ -400,6 +416,132 @@ describe('LoggingSession completion retries', () => { expect(createOTelSpansMock.mock.calls[0]?.[0].traceSpans).toBe(persistedSpans) }) + it('projects secrets before display filters can truncate a long active literal', async () => { + const session = new LoggingSession('workflow-1', 'execution-long-secret', 'api', 'req-1') + const secret = `secret-${'x'.repeat(16_000)}` + const sourceTraceSpans = [ + { + id: 'span-long-secret', + name: 'Function', + type: 'function', + duration: 1, + startTime: '2026-07-01T00:00:00.000Z', + endTime: '2026-07-01T00:00:00.001Z', + output: { result: secret }, + }, + ] + session.setResolvedSecretTraceRegistry( + createSecretRegistry([{ plaintext: secret, replacement: '{{LONG_SECRET}}' }]) + ) + prepareTraceSpansForProjectionMock.mockImplementationOnce( + async ({ traceSpans }: { traceSpans: Array<{ output?: { result?: string } }> }) => { + expect(traceSpans[0]?.output?.result).toBe('{{LONG_SECRET}}') + return traceSpans + } + ) + + await session.safeComplete({ traceSpans: sourceTraceSpans as any }) + + expect(completeWorkflowExecutionMock).toHaveBeenCalledWith( + expect.objectContaining({ + traceSpans: [expect.objectContaining({ output: { result: '{{LONG_SECRET}}' } })], + }) + ) + expect(sourceTraceSpans[0].output.result).toBe(secret) + }) + + it('fails closed when generic log transforms reintroduce a secret literal for persistence and OTel', async () => { + const session = new LoggingSession('workflow-1', 'execution-invariant', 'api', 'req-1') + const sourceTraceSpans = [ + { + id: 'span-invariant', + name: 'Function', + type: 'function', + duration: 1, + startTime: '2026-07-01T00:00:00.000Z', + endTime: '2026-07-01T00:00:00.001Z', + status: 'success', + output: { apiKey: 'ordinary-value' }, + }, + ] + session.setResolvedSecretTraceRegistry( + createSecretRegistry([{ plaintext: 'E', replacement: '{{X}}' }]) + ) + prepareTraceSpansForProjectionMock.mockImplementationOnce( + async ({ traceSpans }: { traceSpans: Array> }) => + traceSpans.map((span) => ({ ...span, output: { apiKey: '[REDACTED]' } })) + ) + + await session.safeComplete({ traceSpans: sourceTraceSpans as any }) + + const persistedSpans = completeWorkflowExecutionMock.mock.calls[0]?.[0].traceSpans + expect(persistedSpans).toEqual([ + expect.objectContaining({ + id: 'span-invariant', + status: 'success', + }), + ]) + expect(persistedSpans[0]).not.toHaveProperty('output') + expect(createOTelSpansMock).toHaveBeenCalledWith( + expect.objectContaining({ traceSpans: persistedSpans }) + ) + expect(createOTelSpansMock.mock.calls[0]?.[0].traceSpans).toBe(persistedSpans) + expect(sourceTraceSpans[0].output).toEqual({ apiKey: 'ordinary-value' }) + }) + + it('hydrates post-transform refs before sharing structural-only spans with persistence and OTel', async () => { + const session = new LoggingSession('workflow-1', 'execution-ref-invariant', 'api', 'req-1') + const sourceTraceSpans = [ + { + id: 'span-ref-invariant', + name: 'Function', + type: 'function', + duration: 1, + startTime: '2026-07-01T00:00:00.000Z', + endTime: '2026-07-01T00:00:00.001Z', + status: 'success', + output: { apiKey: 'ordinary-value' }, + }, + ] + const ref = { + __simLargeValueRef: true, + version: 1, + id: 'lv_bbbbbbbbbbbb', + kind: 'string', + size: 32, + preview: '{{X}}', + } as const + session.setResolvedSecretTraceRegistry( + createSecretRegistry([{ plaintext: 'E', replacement: '{{X}}' }]) + ) + prepareTraceSpansForProjectionMock.mockImplementationOnce( + async ({ traceSpans }: { traceSpans: Array> }) => + traceSpans.map((span) => ({ ...span, output: { payload: ref } })) + ) + materializeLargeValueRefMock.mockResolvedValue({ value: 'hidden-E' }) + + await session.safeComplete({ traceSpans: sourceTraceSpans as any }) + + const persistedSpans = completeWorkflowExecutionMock.mock.calls[0]?.[0].traceSpans + expect(materializeLargeValueRefMock).toHaveBeenCalledWith( + ref, + expect.objectContaining({ + executionId: 'execution-ref-invariant', + trackReference: false, + }) + ) + expect(storeLargeValueMock).not.toHaveBeenCalled() + expect(persistedSpans).toEqual([ + expect.objectContaining({ + id: 'span-ref-invariant', + status: 'success', + }), + ]) + expect(persistedSpans[0]).not.toHaveProperty('output') + expect(createOTelSpansMock.mock.calls[0]?.[0].traceSpans).toBe(persistedSpans) + expect(sourceTraceSpans[0].output).toEqual({ apiKey: 'ordinary-value' }) + }) + it('projects synthetic error spans without copying the raw error into OTel metadata', async () => { const session = new LoggingSession('workflow-1', 'execution-error-safe', 'api', 'req-1') const secret = 'sk-demo-error-7f3a91' diff --git a/apps/sim/lib/logs/execution/logging-session.ts b/apps/sim/lib/logs/execution/logging-session.ts index 5af39bfe4f9..a14e23f2907 100644 --- a/apps/sim/lib/logs/execution/logging-session.ts +++ b/apps/sim/lib/logs/execution/logging-session.ts @@ -23,7 +23,10 @@ import { setLastCompletedBlock, setLastStartedBlock, } from '@/lib/logs/execution/progress-markers' -import { projectTraceSpansForSecrets } from '@/lib/logs/execution/trace-secret-projection' +import { + enforceTraceSpanSecretInvariant, + projectTraceSpansForSecrets, +} from '@/lib/logs/execution/trace-secret-projection' import { traceSpansIndicateFailure } from '@/lib/logs/execution/trace-spans/trace-spans' import type { ExecutionEnvironment, @@ -374,15 +377,26 @@ export class LoggingSession { } private async projectTraceSpans(traceSpans: TraceSpan[]): Promise { - const preparedTraceSpans = await executionLogger.prepareTraceSpansForProjection({ + const sourceTraceSpans = await executionLogger.loadTraceSpansForProjection({ executionId: this.executionId, workflowId: this.workflowId, workspaceId: this.environment?.workspaceId ?? null, - userId: this.actorUserId ?? this.environment?.userId, traceSpans, isResume: this.isResume, }) - return this.projectRawTraceSpans(preparedTraceSpans) + const secretSafeTraceSpans = await this.projectRawTraceSpans(sourceTraceSpans) + const preparedTraceSpans = await executionLogger.prepareTraceSpansForProjection({ + executionId: this.executionId, + workflowId: this.workflowId, + workspaceId: this.environment?.workspaceId ?? null, + userId: this.actorUserId ?? this.environment?.userId, + traceSpans: secretSafeTraceSpans, + }) + const invariantSafeTraceSpans = await enforceTraceSpanSecretInvariant(preparedTraceSpans, { + registry: this.resolvedSecretTraceRegistry, + store: this.getSecretProjectionStore(), + }) + return invariantSafeTraceSpans } async onBlockStart( diff --git a/apps/sim/lib/logs/execution/trace-secret-projection.test.ts b/apps/sim/lib/logs/execution/trace-secret-projection.test.ts index 9353dfd8b78..835896e36d0 100644 --- a/apps/sim/lib/logs/execution/trace-secret-projection.test.ts +++ b/apps/sim/lib/logs/execution/trace-secret-projection.test.ts @@ -14,7 +14,10 @@ vi.mock('@/lib/execution/payloads/store', () => ({ storeLargeValue: storeLargeValueMock, })) -import { projectTraceSpansForSecrets } from '@/lib/logs/execution/trace-secret-projection' +import { + enforceTraceSpanSecretInvariant, + projectTraceSpansForSecrets, +} from '@/lib/logs/execution/trace-secret-projection' import type { TraceSpan } from '@/lib/logs/types' import { type ResolvedSecretTraceMatch, @@ -358,6 +361,317 @@ describe('projectTraceSpansForSecrets', () => { expect(span.children?.[0]).not.toHaveProperty('output') }) + it('fails closed when a later log transform reintroduces an active literal', async () => { + const source = [createSpan({ output: { apiKey: '[REDACTED]' } })] + + const result = await enforceTraceSpanSecretInvariant(source, { + registry: createRegistry([{ plaintext: 'E', replacement: '{{X}}' }]), + store: STORE, + }) + + expect(result[0]).not.toHaveProperty('output') + expect(source[0].output).toEqual({ apiKey: '[REDACTED]' }) + }) + + it('fails the final invariant closed when provenance is incomplete', async () => { + const source = [createSpan({ output: { value: 'ordinary' } })] + + const result = await enforceTraceSpanSecretInvariant(source, { + registry: createRegistry([], false), + store: STORE, + }) + + expect(result[0]).not.toHaveProperty('output') + }) + + it('preserves the exact span array after hydrated trace-owned refs pass the invariant', async () => { + const safeRef = { + __simLargeValueRef: true, + version: 1, + id: 'lv_bbbbbbbbbbbb', + kind: 'string', + size: 7, + preview: '{{X}}', + } as const + const source = [createSpan({ output: { payload: safeRef } })] + materializeLargeValueRefMock.mockResolvedValue({ value: '{{X}}' }) + + const result = await enforceTraceSpanSecretInvariant(source, { + registry: createRegistry([{ plaintext: 'E', replacement: '{{X}}' }]), + store: STORE, + }) + + expect(result).toBe(source) + expect(result[0].output).toEqual({ payload: safeRef }) + expect(materializeLargeValueRefMock).toHaveBeenCalledTimes(1) + expect(storeLargeValueMock).not.toHaveBeenCalled() + }) + + it('checks every ref preview before deduplicating shared underlying values', async () => { + const safeRef = { + __simLargeValueRef: true, + version: 1, + id: 'lv_bbbbbbbbbbbb', + kind: 'string', + size: 32, + preview: '{{X}}', + } as const + const unsafePreviewRef = { + ...safeRef, + preview: '[REDACTED]', + } as const + const source = [createSpan({ output: { first: safeRef, second: unsafePreviewRef } })] + materializeLargeValueRefMock.mockResolvedValue({ value: '{{X}}' }) + + const result = await enforceTraceSpanSecretInvariant(source, { + registry: createRegistry([{ plaintext: 'E', replacement: '{{X}}' }]), + store: STORE, + }) + + expect(result[0]).not.toHaveProperty('output') + expect(materializeLargeValueRefMock).not.toHaveBeenCalled() + expect(storeLargeValueMock).not.toHaveBeenCalled() + }) + + it('rejects secret-bearing metadata on a duplicate ref before storage-key deduplication', async () => { + const safeRef = { + __simLargeValueRef: true, + version: 1, + id: 'lv_bbbbbbbbbbbb', + kind: 'string', + size: 32, + preview: '{{X}}', + } as const + const refWithExtraMetadata = { + ...safeRef, + leaked: 'hidden-E', + } + const source = [createSpan({ output: { first: safeRef, duplicate: refWithExtraMetadata } })] + materializeLargeValueRefMock.mockResolvedValue({ value: '{{X}}' }) + + const result = await enforceTraceSpanSecretInvariant(source, { + registry: createRegistry([{ plaintext: 'E', replacement: '{{X}}' }]), + store: STORE, + }) + + expect(result[0]).not.toHaveProperty('output') + expect(materializeLargeValueRefMock).not.toHaveBeenCalled() + expect(storeLargeValueMock).not.toHaveBeenCalled() + }) + + it('hydrates nested refs found inside a ref preview', async () => { + const nestedRef = { + __simLargeValueRef: true, + version: 1, + id: 'lv_cccccccccccc', + kind: 'string', + size: 32, + preview: '{{X}}', + } as const + const outerRef = { + __simLargeValueRef: true, + version: 1, + id: 'lv_bbbbbbbbbbbb', + kind: 'object', + size: 64, + preview: { nested: nestedRef }, + } as const + const source = [createSpan({ output: { payload: outerRef } })] + materializeLargeValueRefMock.mockImplementation(async (ref: { id: string }) => + ref.id === nestedRef.id ? { value: 'hidden-E' } : { value: '{{X}}' } + ) + + const result = await enforceTraceSpanSecretInvariant(source, { + registry: createRegistry([{ plaintext: 'E', replacement: '{{X}}' }]), + store: STORE, + }) + + expect(result[0]).not.toHaveProperty('output') + expect(materializeLargeValueRefMock).toHaveBeenCalledWith( + nestedRef, + expect.objectContaining({ trackReference: false }) + ) + expect(storeLargeValueMock).not.toHaveBeenCalled() + }) + + it('verifies nested preview refs before deduplicating equal storage keys', async () => { + const nestedRef = { + __simLargeValueRef: true, + version: 1, + id: 'lv_cccccccccccc', + kind: 'string', + size: 32, + preview: '{{X}}', + } as const + const outerRefWithNestedPreview = { + __simLargeValueRef: true, + version: 1, + id: 'lv_bbbbbbbbbbbb', + kind: 'object', + size: 64, + preview: { nested: nestedRef }, + } as const + const outerRefWithPlainPreview = { + ...outerRefWithNestedPreview, + preview: { visible: '{{X}}' }, + } as const + const source = [ + createSpan({ + output: { + first: outerRefWithNestedPreview, + second: outerRefWithPlainPreview, + }, + }), + ] + materializeLargeValueRefMock.mockImplementation(async (ref: { id: string }) => + ref.id === nestedRef.id ? { value: 'hidden-E' } : { value: '{{X}}' } + ) + + const result = await enforceTraceSpanSecretInvariant(source, { + registry: createRegistry([{ plaintext: 'E', replacement: '{{X}}' }]), + store: STORE, + }) + + expect(result[0]).not.toHaveProperty('output') + expect(materializeLargeValueRefMock).toHaveBeenCalledWith( + nestedRef, + expect.objectContaining({ trackReference: false }) + ) + expect(storeLargeValueMock).not.toHaveBeenCalled() + }) + + it('fails closed when a trace-owned ref contains a secret beyond its preview', async () => { + const ref = { + __simLargeValueRef: true, + version: 1, + id: 'lv_bbbbbbbbbbbb', + kind: 'string', + size: 32, + preview: '{{X}}', + } as const + const source = [createSpan({ output: { payload: ref } })] + materializeLargeValueRefMock.mockResolvedValue({ visibleAfterPreview: 'value-E' }) + + const result = await enforceTraceSpanSecretInvariant(source, { + registry: createRegistry([{ plaintext: 'E', replacement: '{{X}}' }]), + store: STORE, + }) + + expect(result[0]).not.toHaveProperty('output') + expect(materializeLargeValueRefMock).toHaveBeenCalledTimes(1) + expect(storeLargeValueMock).not.toHaveBeenCalled() + }) + + it('does not double-count manifest chunks against the hydration byte budget', async () => { + const byteSize = 40 * 1024 * 1024 + const chunkRef = { + __simLargeValueRef: true, + version: 1, + id: 'lv_cccccccccccc', + kind: 'array', + size: byteSize, + preview: [{ token: '{{X}}' }], + } as const + const manifest = { + __simLargeArrayManifest: true, + version: 2, + kind: 'array', + totalCount: 1, + chunkCount: 1, + byteSize, + chunks: [{ ref: chunkRef, count: 1, byteSize }], + preview: [{ token: '{{X}}' }], + } as const + const source = [createSpan({ output: { items: manifest } })] + materializeLargeValueRefMock.mockResolvedValue([{ token: '{{X}}' }]) + + const result = await enforceTraceSpanSecretInvariant(source, { + registry: createRegistry([{ plaintext: 'E', replacement: '{{X}}' }]), + store: STORE, + }) + + expect(result).toBe(source) + expect(materializeLargeValueRefMock).toHaveBeenCalledTimes(1) + expect(storeLargeValueMock).not.toHaveBeenCalled() + }) + + it.each(['ref-first', 'manifest-first'] as const)( + 'charges shared manifest payload refs once with %s property order', + async (order) => { + const byteSize = 40 * 1024 * 1024 + const chunkRef = { + __simLargeValueRef: true, + version: 1, + id: 'lv_cccccccccccc', + kind: 'array', + size: byteSize, + preview: [{ token: '{{X}}' }], + } as const + const manifest = { + __simLargeArrayManifest: true, + version: 2, + kind: 'array', + totalCount: 1, + chunkCount: 1, + byteSize, + chunks: [{ ref: chunkRef, count: 1, byteSize }], + preview: [{ token: '{{X}}' }], + } as const + const clonedManifest = { + ...manifest, + chunks: [{ ...manifest.chunks[0] }], + preview: [...manifest.preview], + } + const output = + order === 'ref-first' + ? { direct: chunkRef, manifest, clonedManifest } + : { manifest, clonedManifest, direct: chunkRef } + const source = [createSpan({ output })] + materializeLargeValueRefMock.mockResolvedValue([{ token: '{{X}}' }]) + + const result = await enforceTraceSpanSecretInvariant(source, { + registry: createRegistry([{ plaintext: 'E', replacement: '{{X}}' }]), + store: STORE, + }) + + expect(result).toBe(source) + expect(materializeLargeValueRefMock).toHaveBeenCalledTimes(1) + expect(storeLargeValueMock).not.toHaveBeenCalled() + } + ) + + it('fails closed when a trace-owned manifest chunk contains a secret', async () => { + const chunkRef = { + __simLargeValueRef: true, + version: 1, + id: 'lv_cccccccccccc', + kind: 'array', + size: 32, + preview: [{ token: '{{X}}' }], + } as const + const manifest = { + __simLargeArrayManifest: true, + version: 2, + kind: 'array', + totalCount: 1, + chunkCount: 1, + byteSize: 32, + chunks: [{ ref: chunkRef, count: 1, byteSize: 32 }], + preview: [{ token: '{{X}}' }], + } as const + const source = [createSpan({ output: { items: manifest } })] + materializeLargeValueRefMock.mockResolvedValue([{ token: 'hidden-E' }]) + + const result = await enforceTraceSpanSecretInvariant(source, { + registry: createRegistry([{ plaintext: 'E', replacement: '{{X}}' }]), + store: STORE, + }) + + expect(result[0]).not.toHaveProperty('output') + expect(materializeLargeValueRefMock).toHaveBeenCalledTimes(1) + expect(storeLargeValueMock).not.toHaveBeenCalled() + }) + it('hydrates and re-stores large values without retaining the source ref', async () => { const sourceRef = { __simLargeValueRef: true, diff --git a/apps/sim/lib/logs/execution/trace-secret-projection.ts b/apps/sim/lib/logs/execution/trace-secret-projection.ts index 16d40182b96..0632dc231c7 100644 --- a/apps/sim/lib/logs/execution/trace-secret-projection.ts +++ b/apps/sim/lib/logs/execution/trace-secret-projection.ts @@ -36,6 +36,34 @@ const MAX_LARGE_VALUE_CHAIN_DEPTH = 32 const MAX_LARGE_MANIFEST_CHUNKS = MAX_LARGE_VALUES const MAX_LARGE_MANIFEST_ITEMS = MAX_CONTENT_NODES const OMIT = Symbol('omit-trace-content') +const LARGE_VALUE_REF_KEYS = new Set([ + LARGE_VALUE_REF_MARKER, + 'version', + 'id', + 'kind', + 'size', + 'key', + 'executionId', + 'preview', +]) +const LARGE_VALUE_REF_REQUIRED_KEYS = new Set([ + LARGE_VALUE_REF_MARKER, + 'version', + 'id', + 'kind', + 'size', +]) +const LARGE_ARRAY_MANIFEST_KEYS = new Set([ + LARGE_ARRAY_MANIFEST_MARKER, + 'version', + 'kind', + 'totalCount', + 'chunkCount', + 'byteSize', + 'chunks', + 'preview', +]) +const LARGE_ARRAY_MANIFEST_CHUNK_KEYS = new Set(['ref', 'count', 'byteSize']) interface SecretReplacement { plaintext: string @@ -61,7 +89,7 @@ interface ProjectionContext { safeLargeValues: WeakSet oversizedGate: OversizedHydrationGate refIoSemaphore: AsyncSemaphore - seenLargeValues: Set + sourceLargeValueSizes: Map largeValueCache: Map> manifestIds: WeakMap nextManifestId: number @@ -90,6 +118,24 @@ interface SanitizationTraversalState extends TraversalState { maxBytes: number } +interface PlaintextInvariantContext { + matcher: SecretMatcher + safeLargeValues: WeakSet + /** Valid refs are trusted only after the full projector has already rewritten them. */ + allowVerifiedLargeValues?: boolean + /** Full post-transform verification hydrates refs instead of trusting their previews. */ + inspectLargeValuePreviews?: boolean +} + +interface PostTransformInvariantContext { + projection: ProjectionContext + plaintext: PlaintextInvariantContext + contentState: TraversalState + collectionState: TraversalState + collectionSeen: WeakSet + verificationCache: Map> +} + export interface ProjectTraceSpansForSecretsOptions { registry?: ResolvedSecretTraceRegistry store: LargeValueStoreContext @@ -191,6 +237,29 @@ function createSecretMatcher(replacements: readonly SecretReplacement[]): Secret return { root, maxPatternLength } } +function createProjectionContext( + matcher: SecretMatcher, + store: LargeValueStoreContext, + allowLargeValueWrites: boolean +): ProjectionContext { + return { + matcher, + store, + allowLargeValueWrites, + safeLargeValues: new WeakSet(), + oversizedGate: { chain: Promise.resolve() }, + refIoSemaphore: { active: 0, waiters: [] }, + sourceLargeValueSizes: new Map(), + largeValueCache: new Map>(), + manifestIds: new WeakMap(), + nextManifestId: 0, + largeValueCount: 0, + sourceLargeValueBytes: 0, + storedLargeValueBytes: 0, + storedLargeValueCount: 0, + } +} + function advanceMatcher( matcher: SecretMatcher, node: SecretTrieNode, @@ -309,8 +378,25 @@ function assertArrayFitsTraversal(value: readonly unknown[], state: TraversalSta } } +function readArrayLength(value: readonly unknown[]): number { + const lengthDescriptor = Object.getOwnPropertyDescriptor(value, 'length') + if ( + !lengthDescriptor || + !('value' in lengthDescriptor) || + lengthDescriptor.enumerable || + typeof lengthDescriptor.value !== 'number' || + !Number.isSafeInteger(lengthDescriptor.value) || + lengthDescriptor.value < 0 + ) { + throw new TraceSecretProjectionError('Trace array length metadata is invalid') + } + return lengthDescriptor.value +} + function* arrayDataEntries(value: readonly unknown[]): Generator<[number, unknown]> { - for (let index = 0; index < value.length; index += 1) { + const length = readArrayLength(value) + + for (let index = 0; index < length; index += 1) { const descriptor = Object.getOwnPropertyDescriptor(value, index) if (!descriptor) continue if (!('value' in descriptor)) { @@ -320,6 +406,41 @@ function* arrayDataEntries(value: readonly unknown[]): Generator<[number, unknow } } +function readCanonicalDataArray(value: unknown, label: string): unknown[] { + if (!Array.isArray(value)) { + throw new TraceSecretProjectionError(`${label} must be an array`) + } + + const length = readArrayLength(value) + const entries: unknown[] = new Array(length) + let entryCount = 0 + + for (const key of Reflect.ownKeys(value)) { + if (key === 'length') continue + if (typeof key !== 'string') { + throw new TraceSecretProjectionError('Trace arrays cannot contain symbol properties') + } + const index = Number(key) + if (!Number.isInteger(index) || index < 0 || index >= length || String(index) !== key) { + throw new TraceSecretProjectionError('Trace arrays cannot contain custom properties') + } + const descriptor = Object.getOwnPropertyDescriptor(value, key) + if (!descriptor?.enumerable || !('value' in descriptor)) { + throw new TraceSecretProjectionError('Trace array entries must be enumerable data properties') + } + entries[index] = descriptor.value + entryCount += 1 + } + + if (entryCount !== length) { + throw new TraceSecretProjectionError(`${label} cannot contain sparse entries`) + } + if (Object.getPrototypeOf(value) !== Array.prototype) { + throw new TraceSecretProjectionError(`${label} must use the canonical array prototype`) + } + return entries +} + function enterObject(value: object, state: TraversalState): void { if (state.ancestors.has(value)) { throw new TraceSecretProjectionError('Cyclic trace content is not supported') @@ -332,11 +453,12 @@ function leaveObject(value: object, state: TraversalState): void { } function* enumerableDataEntries(value: object): Generator<[string, unknown]> { - for (const key in value) { - if (!Object.hasOwn(value, key)) continue + for (const key of Reflect.ownKeys(value)) { + if (typeof key !== 'string') { + throw new TraceSecretProjectionError('Trace objects cannot contain symbol properties') + } const descriptor = Object.getOwnPropertyDescriptor(value, key) - if (!descriptor?.enumerable) continue - if (!('value' in descriptor)) { + if (!descriptor?.enumerable || !('value' in descriptor)) { throw new TraceSecretProjectionError('Trace content accessors are not supported') } yield [key, descriptor.value] @@ -354,6 +476,47 @@ function readOwnDataProperty(value: object, key: string): unknown { return descriptor.value } +function assertExactPlainDataRecord( + value: object, + allowedKeys: ReadonlySet, + requiredKeys: ReadonlySet, + label: string +): void { + const presentKeys = new Set() + for (const [key] of enumerableDataEntries(value)) { + if (!allowedKeys.has(key)) { + throw new TraceSecretProjectionError(`${label} contains unsupported metadata`) + } + presentKeys.add(key) + } + for (const key of requiredKeys) { + if (!presentKeys.has(key)) { + throw new TraceSecretProjectionError(`${label} is missing required metadata`) + } + } + const prototype = Object.getPrototypeOf(value) + if (prototype !== Object.prototype && prototype !== null) { + throw new TraceSecretProjectionError(`${label} must be a plain object`) + } +} + +function readCanonicalLargeValueRef(value: object, marker: unknown): LargeValueRef { + const canonicalRef = { + [LARGE_VALUE_REF_MARKER]: marker, + version: readOwnDataProperty(value, 'version'), + id: readOwnDataProperty(value, 'id'), + kind: readOwnDataProperty(value, 'kind'), + size: readOwnDataProperty(value, 'size'), + key: readOwnDataProperty(value, 'key'), + executionId: readOwnDataProperty(value, 'executionId'), + preview: readOwnDataProperty(value, 'preview'), + } + if (!isLargeValueRef(canonicalRef) || canonicalRef.size > MAX_DURABLE_LARGE_VALUE_BYTES) { + throw new TraceSecretProjectionError('Trace content has invalid large-value metadata') + } + return canonicalRef +} + function getLargeValueCandidate(value: unknown): LargeValueCandidate | undefined { if (!value || typeof value !== 'object') return undefined @@ -363,30 +526,69 @@ function getLargeValueCandidate(value: unknown): LargeValueCandidate | undefined throw new TraceSecretProjectionError('Trace content has ambiguous large-value metadata') } if (largeValueMarker === true) { - if (!isLargeValueRef(value) || value.size > MAX_DURABLE_LARGE_VALUE_BYTES) { - throw new TraceSecretProjectionError('Trace content has invalid large-value metadata') - } - return value + assertExactPlainDataRecord( + value, + LARGE_VALUE_REF_KEYS, + LARGE_VALUE_REF_REQUIRED_KEYS, + 'Trace large-value ref' + ) + readCanonicalLargeValueRef(value, largeValueMarker) + return value as LargeValueRef } if (manifestMarker !== true) return undefined + assertExactPlainDataRecord( + value, + LARGE_ARRAY_MANIFEST_KEYS, + LARGE_ARRAY_MANIFEST_KEYS, + 'Trace large-array manifest' + ) + const chunks = readOwnDataProperty(value, 'chunks') - const chunkCount = readOwnDataProperty(value, 'chunkCount') - const totalCount = readOwnDataProperty(value, 'totalCount') - const byteSize = readOwnDataProperty(value, 'byteSize') + const preview = readOwnDataProperty(value, 'preview') + const chunkValues = readCanonicalDataArray(chunks, 'Trace manifest chunks') + const previewValues = readCanonicalDataArray(preview, 'Trace manifest preview') + const canonicalChunks: Array<{ ref: LargeValueRef; count: unknown; byteSize: unknown }> = [] + for (const chunk of chunkValues) { + if (!chunk || typeof chunk !== 'object') { + throw new TraceSecretProjectionError('Trace manifest chunk metadata is invalid') + } + assertExactPlainDataRecord( + chunk, + LARGE_ARRAY_MANIFEST_CHUNK_KEYS, + LARGE_ARRAY_MANIFEST_CHUNK_KEYS, + 'Trace manifest chunk' + ) + const chunkRef = getLargeValueCandidate(readOwnDataProperty(chunk, 'ref')) + if (!chunkRef || readOwnDataProperty(chunkRef, LARGE_VALUE_REF_MARKER) !== true) { + throw new TraceSecretProjectionError('Trace manifest chunk ref is invalid') + } + canonicalChunks.push({ + ref: readCanonicalLargeValueRef(chunkRef, true), + count: readOwnDataProperty(chunk, 'count'), + byteSize: readOwnDataProperty(chunk, 'byteSize'), + }) + } + + const canonicalManifest = { + [LARGE_ARRAY_MANIFEST_MARKER]: manifestMarker, + version: readOwnDataProperty(value, 'version'), + kind: readOwnDataProperty(value, 'kind'), + totalCount: readOwnDataProperty(value, 'totalCount'), + chunkCount: readOwnDataProperty(value, 'chunkCount'), + byteSize: readOwnDataProperty(value, 'byteSize'), + chunks: canonicalChunks, + preview: previewValues, + } if ( - !Array.isArray(chunks) || - chunks.length > MAX_LARGE_MANIFEST_CHUNKS || - chunkCount !== chunks.length || - typeof totalCount !== 'number' || - totalCount > MAX_LARGE_MANIFEST_ITEMS || - typeof byteSize !== 'number' || - byteSize > MAX_DURABLE_LARGE_VALUE_BYTES || - !isLargeArrayManifest(value) + !isLargeArrayManifest(canonicalManifest) || + canonicalManifest.chunks.length > MAX_LARGE_MANIFEST_CHUNKS || + canonicalManifest.totalCount > MAX_LARGE_MANIFEST_ITEMS || + canonicalManifest.byteSize > MAX_DURABLE_LARGE_VALUE_BYTES ) { throw new TraceSecretProjectionError('Trace content has invalid large-array metadata') } - return value + return value as LargeArrayManifest } function sanitizeInlineValue( @@ -578,18 +780,23 @@ function getLargeValueKey(value: LargeValueCandidate, context: ProjectionContext function registerSourceLargeValue( value: LargeValueCandidate, key: string, - context: ProjectionContext, - countBytes = true + context: ProjectionContext ): void { - if (context.seenLargeValues.has(key)) return - const size = countBytes ? (isLargeValueRef(value) ? value.size : value.byteSize) : 0 + const size = isLargeValueRef(value) ? value.size : 0 + const registeredSize = context.sourceLargeValueSizes.get(key) + if (registeredSize !== undefined) { + if (registeredSize !== size) { + throw new TraceSecretProjectionError('Trace large-value metadata is inconsistent') + } + return + } if ( context.largeValueCount + 1 > MAX_LARGE_VALUES || context.sourceLargeValueBytes + size > MAX_DURABLE_LARGE_VALUE_BYTES ) { throw new TraceSecretProjectionError('Trace large-value projection budget exceeded') } - context.seenLargeValues.add(key) + context.sourceLargeValueSizes.set(key, size) context.largeValueCount += 1 context.sourceLargeValueBytes += size } @@ -726,7 +933,7 @@ async function replaceLargeArrayManifest( for (const chunk of manifest.chunks) { const chunkKey = getLargeValueKey(chunk.ref, context) const chunkPath = extendLargeValuePath(path, chunkKey) - registerSourceLargeValue(chunk.ref, chunkKey, context, false) + registerSourceLargeValue(chunk.ref, chunkKey, context) const materialized = await materializeSourceRef(chunk.ref, context) if (!Array.isArray(materialized) || materialized.length !== chunk.count) { throw new TraceSecretProjectionError('Large trace manifest chunk is invalid') @@ -782,10 +989,10 @@ async function replaceLargeValue( } const key = getLargeValueKey(value, context) const nextPath = extendLargeValuePath(path, key) + registerSourceLargeValue(value, key, context) const cached = context.largeValueCache.get(key) if (cached) return cached - registerSourceLargeValue(value, key, context) const replacement = isLargeValueRef(value) ? replaceLargeValueRef(value, context, nextPath) : replaceLargeArrayManifest(value, context, nextPath) @@ -1260,7 +1467,7 @@ function cloneTraceSpansForProjection(traceSpans: TraceSpan[]): TraceSpan[] { function assertNoPlaintext( value: unknown, - context: ProjectionContext, + context: PlaintextInvariantContext, state: TraversalState, depth = 0 ): void { @@ -1277,10 +1484,24 @@ function assertNoPlaintext( } return } - if (getLargeValueCandidate(value)) { - if (!context.safeLargeValues.has(value as object)) { + const largeValue = getLargeValueCandidate(value) + if (largeValue) { + if (!context.safeLargeValues.has(value as object) && !context.allowVerifiedLargeValues) { throw new TraceSecretProjectionError('Sanitized trace content contains an unverified ref') } + if (context.inspectLargeValuePreviews !== false) { + if (isLargeValueRef(largeValue)) { + const preview = readOwnDataProperty(largeValue, 'preview') + if (preview !== undefined) { + assertNoPlaintext(preview, context, state, depth + 1) + } + } else { + assertNoPlaintext(largeValue.preview, context, state, depth + 1) + for (const chunk of largeValue.chunks) { + assertNoPlaintext(chunk.ref, context, state, depth + 1) + } + } + } return } if (value === null || typeof value !== 'object') return @@ -1308,39 +1529,278 @@ function assertNoPlaintext( } } -function assertTraceSpanContentIsSafe(span: TraceSpan, context: ProjectionContext): void { - const assertField = (value: unknown): void => { - if (value === undefined) return - assertNoPlaintext(value, context, { nodes: 0, ancestors: new WeakSet() }) +type TraceContentVisitor = (value: unknown) => void + +function visitTraceSpanContent( + span: TraceSpan, + visit: TraceContentVisitor, + structureState: BoundedTraceStructureState, + depth = 0 +): void { + if (!takeTraceStructureNode(structureState, depth)) { + throw new TraceSecretProjectionError('Trace invariant structure limit exceeded') } - assertField(span.input) - assertField(span.output) - assertField(span.thinking) - assertField(span.errorMessage) + const visitField = (value: unknown): void => { + if (value !== undefined) visit(value) + } + + visitField(span.input) + visitField(span.output) + visitField(span.thinking) + visitField(span.errorMessage) if (span.modelToolCalls) { + assertTraceStructureArrayFits(span.modelToolCalls, structureState) for (const call of span.modelToolCalls) { - if (Object.hasOwn(call, 'arguments')) assertField(call.arguments) + if (!takeTraceStructureNode(structureState, depth + 1)) { + throw new TraceSecretProjectionError('Trace invariant tool-call limit exceeded') + } + if (Object.hasOwn(call, 'arguments')) visitField(call.arguments) } } if (span.toolCalls) { + assertTraceStructureArrayFits(span.toolCalls, structureState) for (const call of span.toolCalls) { - assertField(call.input) - assertField(call.output) - assertField(call.error) + if (!takeTraceStructureNode(structureState, depth + 1)) { + throw new TraceSecretProjectionError('Trace invariant legacy tool-call limit exceeded') + } + visitField(call.input) + visitField(call.output) + visitField(call.error) } } if (span.providerTiming) { + assertTraceStructureArrayFits(span.providerTiming.segments, structureState) for (const segment of span.providerTiming.segments) { - assertField(segment.assistantContent) - assertField(segment.thinkingContent) - assertField(segment.errorMessage) + if (!takeTraceStructureNode(structureState, depth + 1)) { + throw new TraceSecretProjectionError('Trace invariant provider-segment limit exceeded') + } + visitField(segment.assistantContent) + visitField(segment.thinkingContent) + visitField(segment.errorMessage) + assertTraceStructureArrayFits(segment.toolCalls ?? [], structureState) for (const call of segment.toolCalls ?? []) { - if (Object.hasOwn(call, 'arguments')) assertField(call.arguments) + if (!takeTraceStructureNode(structureState, depth + 2)) { + throw new TraceSecretProjectionError('Trace invariant provider tool-call limit exceeded') + } + if (Object.hasOwn(call, 'arguments')) visitField(call.arguments) + } + } + } + assertTraceStructureArrayFits(span.children ?? [], structureState) + for (const child of span.children ?? []) { + visitTraceSpanContent(child, visit, structureState, depth + 1) + } +} + +function visitTraceSpansContent(traceSpans: TraceSpan[], visit: TraceContentVisitor): void { + const structureState: BoundedTraceStructureState = { nodes: 0, truncated: false } + assertTraceStructureArrayFits(traceSpans, structureState) + for (const span of traceSpans) { + visitTraceSpanContent(span, visit, structureState) + } +} + +function assertTraceSpansContentIsSafe( + traceSpans: TraceSpan[], + context: PlaintextInvariantContext +): void { + const contentState: TraversalState = { nodes: 0, ancestors: new WeakSet() } + visitTraceSpansContent(traceSpans, (value) => assertNoPlaintext(value, context, contentState)) +} + +function collectPostTransformLargeValues( + value: unknown, + context: PostTransformInvariantContext, + refs: object[], + depth = 0 +): void { + if (depth === 0) { + assertNoPlaintext(value, context.plaintext, context.contentState) + } + + visitNode(context.collectionState, depth) + const largeValue = getLargeValueCandidate(value) + if (largeValue) { + const objectValue = value as object + if (context.collectionSeen.has(objectValue)) return + context.collectionSeen.add(objectValue) + refs.push(objectValue) + + if (isLargeValueRef(largeValue)) { + const preview = readOwnDataProperty(largeValue, 'preview') + if (preview !== undefined) { + collectPostTransformLargeValues(preview, context, refs, depth + 1) + } + } else { + collectPostTransformLargeValues(largeValue.preview, context, refs, depth + 1) + for (const chunk of largeValue.chunks) { + collectPostTransformLargeValues(chunk.ref, context, refs, depth + 1) } } + return + } + + if (value === null || typeof value !== 'object') return + if (!Array.isArray(value) && !isPlainRecord(value)) { + throw new TraceSecretProjectionError('Trace invariant contains an unsupported object') + } + if (context.collectionSeen.has(value)) return + context.collectionSeen.add(value) + + enterObject(value, context.collectionState) + try { + if (Array.isArray(value)) { + assertArrayFitsTraversal(value, context.collectionState) + for (const [, item] of arrayDataEntries(value)) { + collectPostTransformLargeValues(item, context, refs, depth + 1) + } + return + } + for (const [, item] of enumerableDataEntries(value)) { + collectPostTransformLargeValues(item, context, refs, depth + 1) + } + } finally { + leaveObject(value, context.collectionState) + } +} + +async function verifyPostTransformValue( + value: unknown, + context: PostTransformInvariantContext, + path: ReadonlySet +): Promise { + const refs: object[] = [] + collectPostTransformLargeValues(value, context, refs) + await verifyPostTransformLargeValues(refs, context, path) +} + +async function verifyPostTransformLargeValue( + value: LargeValueCandidate, + context: PostTransformInvariantContext, + path: ReadonlySet +): Promise { + const key = getLargeValueKey(value, context.projection) + const nextPath = extendLargeValuePath(path, key) + registerSourceLargeValue(value, key, context.projection) + const cached = context.verificationCache.get(key) + if (cached) return cached + + const verification = (async (): Promise => { + if (isLargeValueRef(value)) { + const preview = readOwnDataProperty(value, 'preview') + if (preview !== undefined) { + await verifyPostTransformValue(preview, context, nextPath) + } + const materialized = await materializeSourceRef(value, context.projection) + await verifyPostTransformValue(materialized, context, nextPath) + return materialized + } + + await verifyPostTransformValue(value.preview, context, nextPath) + for (const chunk of value.chunks) { + const materialized = await verifyPostTransformLargeValue(chunk.ref, context, nextPath) + if (!Array.isArray(materialized) || materialized.length !== chunk.count) { + throw new TraceSecretProjectionError('Trace invariant manifest chunk is invalid') + } + } + return value + })() + context.verificationCache.set(key, verification) + return verification +} + +async function verifyPostTransformLargeValues( + refs: object[], + context: PostTransformInvariantContext, + path: ReadonlySet +): Promise { + const candidates = new Map() + for (const ref of refs) { + const candidate = getLargeValueCandidate(ref) + if (!candidate) { + throw new TraceSecretProjectionError('Trace invariant large-value metadata changed') + } + const key = getLargeValueKey(candidate, context.projection) + const current = candidates.get(key) + if ( + current && + isLargeValueRef(current) && + isLargeValueRef(candidate) && + current.size !== candidate.size + ) { + throw new TraceSecretProjectionError('Trace invariant ref metadata is inconsistent') + } + if (!current) candidates.set(key, candidate) + if (candidates.size > MAX_LARGE_VALUES) { + throw new TraceSecretProjectionError('Trace invariant large-value count exceeded') + } + } + + await Promise.all( + [...candidates.values()].map((candidate) => + verifyPostTransformLargeValue(candidate, context, path) + ) + ) +} + +async function assertPostTransformTraceSpansAreSafe( + traceSpans: TraceSpan[], + matcher: SecretMatcher, + store: LargeValueStoreContext +): Promise { + const projection = createProjectionContext(matcher, store, false) + const context: PostTransformInvariantContext = { + projection, + plaintext: { + matcher, + safeLargeValues: new WeakSet(), + allowVerifiedLargeValues: true, + inspectLargeValuePreviews: true, + }, + contentState: { nodes: 0, ancestors: new WeakSet() }, + collectionState: { nodes: 0, ancestors: new WeakSet() }, + collectionSeen: new WeakSet(), + verificationCache: new Map>(), + } + const refs: object[] = [] + visitTraceSpansContent(traceSpans, (value) => + collectPostTransformLargeValues(value, context, refs) + ) + await verifyPostTransformLargeValues(refs, context, new Set()) +} + +export interface EnforceTraceSpanSecretInvariantOptions { + registry?: ResolvedSecretTraceRegistry + store: LargeValueStoreContext +} + +/** + * Re-checks an already-projected TraceSpan tree after generic log transforms. + * Inline content and hydrated trace-owned refs are inspected without performing + * replacement or storage work. The caller must pass the output of + * {@link projectTraceSpansForSecrets} through directly. + */ +export async function enforceTraceSpanSecretInvariant( + traceSpans: TraceSpan[], + options: EnforceTraceSpanSecretInvariantOptions +): Promise { + try { + if (!options.registry?.isComplete()) return structuralOnlyTraceSpans(traceSpans) + + const replacements = normalizeReplacements(options.registry.getActiveMatches()) + if (replacements.length === 0) return traceSpans + + await assertPostTransformTraceSpansAreSafe( + traceSpans, + createSecretMatcher(replacements), + options.store + ) + return traceSpans + } catch { + logger.warn('Trace secret invariant failed; retaining structural spans only') + return structuralOnlyTraceSpans(traceSpans) } - for (const child of span.children ?? []) assertTraceSpanContentIsSafe(child, context) } /** @@ -1359,22 +1819,11 @@ export async function projectTraceSpansForSecrets( const replacements = normalizeReplacements(options.registry.getActiveMatches()) if (replacements.length === 0) return cloneTraceSpansForProjection(traceSpans) - const context: ProjectionContext = { - matcher: createSecretMatcher(replacements), - store: options.store, - allowLargeValueWrites: options.allowLargeValueWrites !== false, - safeLargeValues: new WeakSet(), - oversizedGate: { chain: Promise.resolve() }, - refIoSemaphore: { active: 0, waiters: [] }, - seenLargeValues: new Set(), - largeValueCache: new Map>(), - manifestIds: new WeakMap(), - nextManifestId: 0, - largeValueCount: 0, - sourceLargeValueBytes: 0, - storedLargeValueBytes: 0, - storedLargeValueCount: 0, - } + const context = createProjectionContext( + createSecretMatcher(replacements), + options.store, + options.allowLargeValueWrites !== false + ) const projected: TraceSpan[] = [] const state: BoundedTraceStructureState = { nodes: 0, truncated: false } @@ -1382,7 +1831,7 @@ export async function projectTraceSpansForSecrets( for (const span of traceSpans) { projected.push(await sanitizeTraceSpan(span, context, 0, state)) } - for (const span of projected) assertTraceSpanContentIsSafe(span, context) + assertTraceSpansContentIsSafe(projected, context) return projected } catch { logger.warn('Trace secret projection failed; retaining structural spans only') diff --git a/apps/sim/lib/logs/types.ts b/apps/sim/lib/logs/types.ts index 29c6e60792a..be1a960c7c8 100644 --- a/apps/sim/lib/logs/types.ts +++ b/apps/sim/lib/logs/types.ts @@ -433,13 +433,20 @@ export interface SnapshotCreationResult { } export interface ExecutionLoggerService { + loadTraceSpansForProjection(params: { + executionId: string + workflowId: string + workspaceId: string | null + traceSpans: TraceSpan[] + isResume?: boolean + }): Promise + prepareTraceSpansForProjection(params: { executionId: string workflowId: string workspaceId: string | null userId?: string | null traceSpans: TraceSpan[] - isResume?: boolean }): Promise startWorkflowExecution(params: { diff --git a/apps/sim/lib/mcp/resolve-config.test.ts b/apps/sim/lib/mcp/resolve-config.test.ts index 4b7e0f25473..9c18f6e3c5d 100644 --- a/apps/sim/lib/mcp/resolve-config.test.ts +++ b/apps/sim/lib/mcp/resolve-config.test.ts @@ -3,12 +3,12 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockGetPersonalAndWorkspaceEnv } = vi.hoisted(() => ({ - mockGetPersonalAndWorkspaceEnv: vi.fn(), +const { mockGetEffectiveEnvironmentSnapshot } = vi.hoisted(() => ({ + mockGetEffectiveEnvironmentSnapshot: vi.fn(), })) vi.mock('@/lib/environment/utils', () => ({ - getPersonalAndWorkspaceEnv: mockGetPersonalAndWorkspaceEnv, + getEffectiveEnvironmentSnapshot: mockGetEffectiveEnvironmentSnapshot, })) import { resolveMcpConfigEnvVars } from '@/lib/mcp/resolve-config' @@ -27,7 +27,7 @@ const BASE_CONFIG = { describe('resolveMcpConfigEnvVars secret provenance', () => { beforeEach(() => { vi.clearAllMocks() - mockGetPersonalAndWorkspaceEnv.mockResolvedValue({ + mockGetEffectiveEnvironmentSnapshot.mockResolvedValue({ personalEncrypted: { MCP_HOST: 'personal-host-encrypted', MCP_TOKEN: 'personal-token-encrypted', @@ -67,7 +67,7 @@ describe('resolveMcpConfigEnvVars secret provenance', () => { }) it('marks provenance incomplete when a resolved value has no encrypted catalog entry', async () => { - mockGetPersonalAndWorkspaceEnv.mockResolvedValue({ + mockGetEffectiveEnvironmentSnapshot.mockResolvedValue({ personalEncrypted: {}, workspaceEncrypted: {}, personalDecrypted: { MCP_HOST: 'api.example.com', MCP_TOKEN: 'token' }, @@ -87,7 +87,7 @@ describe('resolveMcpConfigEnvVars secret provenance', () => { }) it('does not let an unused configured-secret failure affect invocation provenance', async () => { - mockGetPersonalAndWorkspaceEnv.mockResolvedValue({ + mockGetEffectiveEnvironmentSnapshot.mockResolvedValue({ personalEncrypted: { MCP_HOST: 'host-encrypted', MCP_TOKEN: 'token-encrypted', @@ -139,7 +139,7 @@ describe('resolveMcpConfigEnvVars secret provenance', () => { it('reports incomplete provenance when the Secrets catalog cannot be loaded', async () => { const onProvenance = vi.fn() - mockGetPersonalAndWorkspaceEnv.mockRejectedValueOnce(new Error('database unavailable')) + mockGetEffectiveEnvironmentSnapshot.mockRejectedValueOnce(new Error('database unavailable')) const result = await resolveMcpConfigEnvVars(BASE_CONFIG, 'user-1', 'workspace-1', { onResolvedSecretTraceProvenance: onProvenance, @@ -155,4 +155,28 @@ describe('resolveMcpConfigEnvVars secret provenance', () => { expect(onProvenance).toHaveBeenCalledWith(expected) expect(result.config).toEqual(BASE_CONFIG) }) + + it('uses one atomic cached snapshot for runtime values and encrypted provenance', async () => { + mockGetEffectiveEnvironmentSnapshot.mockResolvedValueOnce({ + personalEncrypted: { MCP_HOST: 'old-host-ciphertext', MCP_TOKEN: 'old-token-ciphertext' }, + workspaceEncrypted: {}, + personalDecrypted: { MCP_HOST: 'old.example.com', MCP_TOKEN: 'old-token' }, + workspaceDecrypted: {}, + conflicts: [], + decryptionFailures: [], + }) + + const result = await resolveMcpConfigEnvVars(BASE_CONFIG, 'user-1', 'workspace-1') + + expect(mockGetEffectiveEnvironmentSnapshot).toHaveBeenCalledOnce() + expect(mockGetEffectiveEnvironmentSnapshot).toHaveBeenCalledWith('user-1', 'workspace-1') + expect(result.config).toMatchObject({ + url: 'https://old.example.com/mcp', + headers: expect.objectContaining({ Authorization: 'Bearer old-token' }), + }) + expect(result.resolvedSecretTraceProvenance.entries).toEqual([ + { name: 'MCP_HOST', encryptedValue: 'old-host-ciphertext' }, + { name: 'MCP_TOKEN', encryptedValue: 'old-token-ciphertext' }, + ]) + }) }) diff --git a/apps/sim/lib/mcp/resolve-config.ts b/apps/sim/lib/mcp/resolve-config.ts index d15f642350a..c3c21a104d0 100644 --- a/apps/sim/lib/mcp/resolve-config.ts +++ b/apps/sim/lib/mcp/resolve-config.ts @@ -5,13 +5,18 @@ */ import { createLogger } from '@sim/logger' -import { getPersonalAndWorkspaceEnv } from '@/lib/environment/utils' +import { getErrorMessage } from '@sim/utils/errors' +import { + type EnvironmentResolutionSnapshot, + getEffectiveEnvironmentSnapshot, +} from '@/lib/environment/utils' import type { McpServerConfig } from '@/lib/mcp/types' import { resolveEnvVarReferences } from '@/executor/utils/reference-validation' import { + createIncompleteResolvedSecretTraceRegistry, createResolvedSecretTraceRegistry, type ResolvedSecretTraceProvenanceV1, - ResolvedSecretTraceRegistry, + type ResolvedSecretTraceRegistry, } from '@/executor/utils/resolved-secret-trace-registry' const logger = createLogger('McpResolveConfig') @@ -46,11 +51,24 @@ export async function resolveMcpConfigEnvVars( const allMissingVars: string[] = [] const scope = { userId, ...(workspaceId ? { workspaceId } : {}) } - let envVars: Record = {} + let env: EnvironmentResolutionSnapshot + try { + env = await getEffectiveEnvironmentSnapshot(userId, workspaceId) + } catch (error) { + logger.error('Failed to fetch environment variables for MCP config:', error) + const resolvedSecretTraceRegistry = createIncompleteResolvedSecretTraceRegistry(scope) + const provenance = resolvedSecretTraceRegistry.exportProvenance() + options.onResolvedSecretTraceProvenance?.(provenance) + return { + config, + missingVars: [], + resolvedSecretTraceProvenance: provenance, + } + } + const envVars = { ...env.personalDecrypted, ...env.workspaceDecrypted } + let resolvedSecretTraceRegistry: ResolvedSecretTraceRegistry try { - const env = await getPersonalAndWorkspaceEnv(userId, workspaceId) - envVars = { ...env.personalDecrypted, ...env.workspaceDecrypted } resolvedSecretTraceRegistry = await createResolvedSecretTraceRegistry({ personalEncrypted: env.personalEncrypted, workspaceEncrypted: env.workspaceEncrypted, @@ -60,16 +78,10 @@ export async function resolveMcpConfigEnvVars( scope, }) } catch (error) { - logger.error('Failed to fetch environment variables for MCP config:', error) - resolvedSecretTraceRegistry = new ResolvedSecretTraceRegistry([], scope) - resolvedSecretTraceRegistry.markIncomplete() - const provenance = resolvedSecretTraceRegistry.exportProvenance() - options.onResolvedSecretTraceProvenance?.(provenance) - return { - config, - missingVars: [], - resolvedSecretTraceProvenance: provenance, - } + logger.warn('Failed to build MCP trace secret catalog; provenance will be incomplete', { + error: getErrorMessage(error), + }) + resolvedSecretTraceRegistry = createIncompleteResolvedSecretTraceRegistry(scope) } const resolveValue = (value: string): string => { @@ -77,12 +89,7 @@ export async function resolveMcpConfigEnvVars( const resolved = resolveEnvVarReferences(value, envVars, { missingKeys: missingVars, onResolved: (name, resolvedValue) => { - if ( - resolvedValue.length > 0 && - !resolvedSecretTraceRegistry.recordResolved(name, resolvedValue) - ) { - resolvedSecretTraceRegistry.markIncomplete() - } + resolvedSecretTraceRegistry.recordResolved(name, resolvedValue) }, }) as string allMissingVars.push(...missingVars) diff --git a/apps/sim/tools/index.test.ts b/apps/sim/tools/index.test.ts index d7578e62c5e..d3895b9c367 100644 --- a/apps/sim/tools/index.test.ts +++ b/apps/sim/tools/index.test.ts @@ -810,6 +810,58 @@ describe('executeTool Function', () => { expect(registry.isComplete()).toBe(true) }) + it('does not charge private provenance against the functional response limit', async () => { + const registry = new ResolvedSecretTraceRegistry([], { + userId: 'parent-user', + workspaceId: 'workspace-456', + }) + const functionalValue = 'f'.repeat(9 * 1024 * 1024) + const encryptedValue = 'e'.repeat(2 * 1024 * 1024) + encryptionMockFns.mockDecryptSecret.mockResolvedValueOnce({ decrypted: 'secret-value' }) + global.fetch = Object.assign( + vi.fn().mockResolvedValue( + new Response( + JSON.stringify({ + success: true, + workflowId: 'child-workflow', + workflowName: 'Child Workflow', + output: { value: functionalValue }, + metadata: { duration: 17 }, + __resolvedSecretTraceProvenance: { + version: 1, + complete: true, + entries: [{ name: 'TOKEN', encryptedValue }], + scope: { userId: 'parent-user', workspaceId: 'workspace-456' }, + }, + }), + { + status: 200, + headers: { + 'content-type': 'application/json', + 'x-sim-private-tool-metadata': 'resolved-secret-provenance-v1', + }, + } + ) + ), + { preconnect: vi.fn() } + ) as typeof fetch + + const result = await executeTool( + 'workflow_executor_child-workflow', + { workflowId: 'child-workflow', inputMapping: {} }, + { + executionContext: createToolExecutionContext({ userId: 'parent-user' }), + resolvedSecretTraceRegistry: registry, + } + ) + + expect(result.success).toBe(true) + expect((result.output as { value: string }).value).toHaveLength(functionalValue.length) + expect(result).not.toHaveProperty('__resolvedSecretTraceProvenance') + expect(result.output).not.toHaveProperty('__resolvedSecretTraceProvenance') + expect(registry.isComplete()).toBe(true) + }) + it('strips private metadata even when an internal endpoint returns the wrong marker', async () => { const registry = new ResolvedSecretTraceRegistry() global.fetch = Object.assign( diff --git a/apps/sim/tools/index.ts b/apps/sim/tools/index.ts index 552841e6c9a..9944a58a474 100644 --- a/apps/sim/tools/index.ts +++ b/apps/sim/tools/index.ts @@ -812,6 +812,7 @@ import { normalizeToolId } from '@/tools/normalize' */ const MAX_REQUEST_BODY_SIZE_BYTES = 10 * 1024 * 1024 // 10MB const MAX_TOOL_RESPONSE_BODY_BYTES = 10 * 1024 * 1024 // 10MB +const MAX_PRIVATE_TOOL_METADATA_OVERHEAD_BYTES = 10 * 1024 * 1024 // 10MB /** * User-friendly error message for body size limit exceeded @@ -925,11 +926,12 @@ async function readToolResponseBody( requestId: string toolId: string signal?: AbortSignal + maxBytes?: number } ): Promise { try { return await readResponseToBufferWithLimit(response, { - maxBytes: MAX_TOOL_RESPONSE_BODY_BYTES, + maxBytes: options.maxBytes ?? MAX_TOOL_RESPONSE_BODY_BYTES, label: `${options.toolId} response body`, signal: options.signal, allowNoBodyFallback: true, @@ -2076,6 +2078,12 @@ async function executeToolRequest( requestId, toolId, signal: controller.signal, + ...(privateToolMetadataType + ? { + maxBytes: + MAX_TOOL_RESPONSE_BODY_BYTES + MAX_PRIVATE_TOOL_METADATA_OVERHEAD_BYTES, + } + : {}), }) response = new Response(new Uint8Array(bodyBuffer), { status: internalResponse.status, @@ -2222,6 +2230,19 @@ async function executeToolRequest( response = privateMetadata.response privateMetadataConsumed = privateMetadata.consumed + if (privateToolMetadataType) { + const functionalBody = await readToolResponseBody(response, { + requestId, + toolId, + signal, + }) + response = new Response(new Uint8Array(functionalBody), { + status: response.status, + statusText: response.statusText, + headers: cloneResponseHeaders(response.headers), + }) + } + if (!response.ok) { let errorData: any try { diff --git a/packages/testing/src/mocks/environment-utils.mock.test.ts b/packages/testing/src/mocks/environment-utils.mock.test.ts index 389a89fa3f6..c36272aad58 100644 --- a/packages/testing/src/mocks/environment-utils.mock.test.ts +++ b/packages/testing/src/mocks/environment-utils.mock.test.ts @@ -24,6 +24,14 @@ describe('environment-utils mock', () => { conflicts: [], decryptionFailures: [], }) + await expect(environmentUtilsMock.getEffectiveEnvironmentSnapshot('user-1')).resolves.toEqual({ + personalEncrypted: {}, + workspaceEncrypted: {}, + personalDecrypted: {}, + workspaceDecrypted: {}, + conflicts: [], + decryptionFailures: [], + }) await expect(environmentUtilsMock.upsertPersonalEnvVars('user-1', {})).resolves.toEqual({ added: [], updated: [], diff --git a/packages/testing/src/mocks/environment-utils.mock.ts b/packages/testing/src/mocks/environment-utils.mock.ts index 40c07de5391..69b12c37b32 100644 --- a/packages/testing/src/mocks/environment-utils.mock.ts +++ b/packages/testing/src/mocks/environment-utils.mock.ts @@ -36,6 +36,9 @@ export const environmentUtilsMockFns = { mockGetPersonalAndWorkspaceEnv: vi .fn() .mockImplementation(async () => emptyPersonalAndWorkspaceEnv()), + mockGetEffectiveEnvironmentSnapshot: vi + .fn() + .mockImplementation(async () => emptyPersonalAndWorkspaceEnv()), mockUpsertPersonalEnvVars: vi.fn().mockResolvedValue({ added: [], updated: [] }), mockUpsertWorkspaceEnvVars: vi.fn().mockResolvedValue([]), mockGetEffectiveDecryptedEnv: vi.fn().mockResolvedValue({}), @@ -52,6 +55,9 @@ export function resetEnvironmentUtilsMock(): void { environmentUtilsMockFns.mockGetPersonalAndWorkspaceEnv .mockReset() .mockImplementation(async () => emptyPersonalAndWorkspaceEnv()) + environmentUtilsMockFns.mockGetEffectiveEnvironmentSnapshot + .mockReset() + .mockImplementation(async () => emptyPersonalAndWorkspaceEnv()) environmentUtilsMockFns.mockUpsertPersonalEnvVars .mockReset() .mockResolvedValue({ added: [], updated: [] }) @@ -73,6 +79,7 @@ export const environmentUtilsMock = { environmentUtilsMockFns.mockInvalidateEffectiveDecryptedEnvCache, getEnvironmentVariableKeys: environmentUtilsMockFns.mockGetEnvironmentVariableKeys, getPersonalAndWorkspaceEnv: environmentUtilsMockFns.mockGetPersonalAndWorkspaceEnv, + getEffectiveEnvironmentSnapshot: environmentUtilsMockFns.mockGetEffectiveEnvironmentSnapshot, upsertPersonalEnvVars: environmentUtilsMockFns.mockUpsertPersonalEnvVars, upsertWorkspaceEnvVars: environmentUtilsMockFns.mockUpsertWorkspaceEnvVars, getEffectiveDecryptedEnv: environmentUtilsMockFns.mockGetEffectiveDecryptedEnv, From c9302a55a93ae8e115ec7c5877d7ec245f295e0d Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Sat, 1 Aug 2026 12:02:51 -0700 Subject: [PATCH 8/8] fix(execution): preserve functional state during trace projection --- .../app/api/workflows/[id]/log/route.test.ts | 51 +++++++++++++++- apps/sim/app/api/workflows/[id]/log/route.ts | 13 ++-- .../workflow/workflow-handler.test.ts | 17 ++++++ .../handlers/workflow/workflow-handler.ts | 8 ++- .../logs/execution/logging-session.test.ts | 60 +++++++++++++++++++ .../sim/lib/logs/execution/logging-session.ts | 7 +++ .../workflows/executor/execution-core.test.ts | 10 ++++ .../lib/workflows/executor/execution-core.ts | 2 + .../lib/workflows/streaming/streaming.test.ts | 38 ++++++++++++ apps/sim/lib/workflows/streaming/streaming.ts | 1 + 10 files changed, 199 insertions(+), 8 deletions(-) diff --git a/apps/sim/app/api/workflows/[id]/log/route.test.ts b/apps/sim/app/api/workflows/[id]/log/route.test.ts index 8611a1302ba..98dc6ac9631 100644 --- a/apps/sim/app/api/workflows/[id]/log/route.test.ts +++ b/apps/sim/app/api/workflows/[id]/log/route.test.ts @@ -312,15 +312,22 @@ describe('POST /api/workflows/[id]/log completion attribution', () => { }) it('restores trusted Secrets provenance before projecting legacy completion traces', async () => { + const trustedExecutionState = { + blockStates: { 'function-1': { output: { result: 'raw-secret-value' } } }, + executedBlocks: ['function-1'], + blockLogs: [], + decisions: { router: {}, condition: {} }, + completedLoops: [], + activeExecutionPath: ['function-1'], + resolvedSecretTraceProvenance: { version: 1, complete: true, entries: [] }, + } dbChainMockFns.limit.mockResolvedValueOnce([ { workflowId: OWNER_WORKFLOW_ID, workspaceId: 'workspace-1', executionData: { billingAttribution: storedBillingAttribution, - executionState: { - resolvedSecretTraceProvenance: { version: 1, complete: true, entries: [] }, - }, + executionState: trustedExecutionState, }, }, ]) @@ -342,6 +349,44 @@ describe('POST /api/workflows/[id]/log completion attribution', () => { entries: [], scope: { userId: 'user-1', workspaceId: 'workspace-1' }, }) + expect(mockSafeComplete).toHaveBeenCalledWith( + expect.objectContaining({ executionState: trustedExecutionState }) + ) + }) + + it('forwards trusted execution state through legacy error completion', async () => { + const trustedExecutionState = { + blockStates: { 'function-1': { output: { error: 'raw-secret-value' } } }, + executedBlocks: ['function-1'], + blockLogs: [], + decisions: { router: {}, condition: {} }, + completedLoops: [], + activeExecutionPath: ['function-1'], + resolvedSecretTraceProvenance: { version: 1, complete: true, entries: [] }, + } + dbChainMockFns.limit.mockResolvedValueOnce([ + { + workflowId: OWNER_WORKFLOW_ID, + workspaceId: 'workspace-1', + executionData: { + billingAttribution: storedBillingAttribution, + executionState: trustedExecutionState, + }, + }, + ]) + + const res = await POST( + makeRequest(OWNER_WORKFLOW_ID, { + executionId: 'trusted-error-execution-id', + result: { success: false, error: 'failed' }, + }), + { params: Promise.resolve({ id: OWNER_WORKFLOW_ID }) } + ) + + expect(res.status).toBe(200) + expect(mockSafeCompleteWithError).toHaveBeenCalledWith( + expect.objectContaining({ executionState: trustedExecutionState }) + ) }) it('forces structural-only traces when trusted stored provenance is unavailable', async () => { diff --git a/apps/sim/app/api/workflows/[id]/log/route.ts b/apps/sim/app/api/workflows/[id]/log/route.ts index 271ab98bed1..8e5123504ed 100644 --- a/apps/sim/app/api/workflows/[id]/log/route.ts +++ b/apps/sim/app/api/workflows/[id]/log/route.ts @@ -16,6 +16,7 @@ import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans' import { materializeExecutionData } from '@/lib/logs/execution/trace-store' import { validateWorkflowAccess } from '@/app/api/workflows/middleware' import { createErrorResponse, createSuccessResponse } from '@/app/api/workflows/utils' +import type { SerializableExecutionState } from '@/executor/execution/types' import type { ExecutionResult } from '@/executor/types' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' @@ -133,11 +134,13 @@ export const POST = withRouteHandler( executionId, } ) - const trustedExecutionState = trustedExecutionData.executionState - const trustedProvenance = - trustedExecutionState && typeof trustedExecutionState === 'object' - ? (trustedExecutionState as Record).resolvedSecretTraceProvenance + const trustedExecutionState = + trustedExecutionData.executionState && + typeof trustedExecutionData.executionState === 'object' && + !Array.isArray(trustedExecutionData.executionState) + ? (trustedExecutionData.executionState as SerializableExecutionState) : undefined + const trustedProvenance = trustedExecutionState?.resolvedSecretTraceProvenance if (trustedProvenance === undefined) { resolvedSecretTraceRegistry.markIncomplete() } else { @@ -168,6 +171,7 @@ export const POST = withRouteHandler( totalDurationMs: totalDuration || result.metadata?.duration || 0, error: { message }, traceSpans, + executionState: trustedExecutionState, }) } else { await loggingSession.safeComplete({ @@ -175,6 +179,7 @@ export const POST = withRouteHandler( totalDurationMs: totalDuration || result.metadata?.duration || 0, finalOutput: result.output || {}, traceSpans, + executionState: trustedExecutionState, }) } diff --git a/apps/sim/executor/handlers/workflow/workflow-handler.test.ts b/apps/sim/executor/handlers/workflow/workflow-handler.test.ts index cb8a60737dc..ac8e0c3458e 100644 --- a/apps/sim/executor/handlers/workflow/workflow-handler.test.ts +++ b/apps/sim/executor/handlers/workflow/workflow-handler.test.ts @@ -1242,9 +1242,19 @@ describe('WorkflowBlockHandler', () => { }) it('completes the child session and disposes the cancellation bridge', async () => { + const executionState = { + blockStates: { 'function-1': { output: { result: 'raw-secret-value' } } }, + } + mockExecutorExecute.mockResolvedValue({ + success: true, + output: { data: 'ok' }, + executionState, + }) + await handler.execute(customBlockContext(), customBlock(), {}) expect(mockSafeComplete).toHaveBeenCalledTimes(1) + expect(mockSafeComplete).toHaveBeenCalledWith(expect.objectContaining({ executionState })) expect(mockSafeCompleteWithError).not.toHaveBeenCalled() expect(mockDispose).toHaveBeenCalledTimes(1) }) @@ -1252,15 +1262,22 @@ describe('WorkflowBlockHandler', () => { it('records a cancelled child through the cancellation path', async () => { // Production shape: the engine reports cancellation as `success: false` // plus `status: 'cancelled'` on the ExecutionResult (never on metadata). + const executionState = { + blockStates: { 'function-1': { output: { result: 'raw-secret-value' } } }, + } mockExecutorExecute.mockResolvedValue({ success: false, output: {}, status: 'cancelled', + executionState, }) await handler.execute(customBlockContext(), customBlock(), {}).catch(() => {}) expect(mockSafeCompleteWithCancellation).toHaveBeenCalledTimes(1) + expect(mockSafeCompleteWithCancellation).toHaveBeenCalledWith( + expect.objectContaining({ executionState }) + ) expect(mockSafeComplete).not.toHaveBeenCalled() // Already finalized as cancelled — must not be re-completed as an error. expect(mockSafeCompleteWithError).not.toHaveBeenCalled() diff --git a/apps/sim/executor/handlers/workflow/workflow-handler.ts b/apps/sim/executor/handlers/workflow/workflow-handler.ts index 2ae125b9499..97ded3b267e 100644 --- a/apps/sim/executor/handlers/workflow/workflow-handler.ts +++ b/apps/sim/executor/handlers/workflow/workflow-handler.ts @@ -805,7 +805,12 @@ export class WorkflowBlockHandler implements BlockHandler { // Cancellation lives on `ExecutionResult.status` — `ExecutionMetadata.status` // has no 'cancelled' member, so reading it there never matches. if (executionResult.status === 'cancelled') { - await session.safeCompleteWithCancellation({ endedAt, totalDurationMs, traceSpans }) + await session.safeCompleteWithCancellation({ + endedAt, + totalDurationMs, + traceSpans, + executionState: executionResult.executionState, + }) return } @@ -815,6 +820,7 @@ export class WorkflowBlockHandler implements BlockHandler { finalOutput: executionResult.output ?? {}, traceSpans, workflowInput, + executionState: executionResult.executionState, }) } diff --git a/apps/sim/lib/logs/execution/logging-session.test.ts b/apps/sim/lib/logs/execution/logging-session.test.ts index daef03e2971..b4e79338534 100644 --- a/apps/sim/lib/logs/execution/logging-session.test.ts +++ b/apps/sim/lib/logs/execution/logging-session.test.ts @@ -151,6 +151,41 @@ describe('LoggingSession terminal provenance', () => { }) ) }) + + it.each(['cancellation', 'pause'] as const)( + 'preserves raw execution state on %s finalization', + async (finalization) => { + const session = new LoggingSession('workflow-1', `execution-state-${finalization}`, 'manual') + session.setResolvedSecretTraceRegistry(createSecretRegistry([])) + const executionState = { + blockStates: { 'function-1': { output: { result: 'raw-secret-value' } } }, + executedBlocks: ['function-1'], + blockLogs: [], + decisions: { router: {}, condition: {} }, + completedLoops: [], + activeExecutionPath: ['function-1'], + } + + if (finalization === 'cancellation') { + await session.completeWithCancellation({ executionState }) + } else { + await session.completeWithPause({ executionState }) + } + + expect(completeWorkflowExecutionMock).toHaveBeenCalledWith( + expect.objectContaining({ + executionState: expect.objectContaining({ + blockStates: executionState.blockStates, + resolvedSecretTraceProvenance: { + version: 1, + complete: true, + entries: [], + }, + }), + }) + ) + } + ) }) beforeEach(() => { @@ -596,15 +631,25 @@ describe('LoggingSession completion retries', () => { completeWorkflowExecutionMock .mockRejectedValueOnce(new Error('primary persistence failed')) .mockResolvedValueOnce({}) + const executionState = { + blockStates: { 'function-1': { output: { result: secret } } }, + executedBlocks: ['function-1'], + blockLogs: [], + decisions: { router: {}, condition: {} }, + completedLoops: [], + activeExecutionPath: ['function-1'], + } await session.safeComplete({ finalOutput: { echoed: secret }, + executionState, }) expect(completeWorkflowExecutionMock).toHaveBeenLastCalledWith( expect.objectContaining({ finalOutput: { echoed: secret }, finalizationPath: 'fallback_completed', + executionState: expect.objectContaining({ blockStates: executionState.blockStates }), }) ) }) @@ -944,6 +989,14 @@ describe('LoggingSession completion retries', () => { completeWorkflowExecutionMock .mockRejectedValueOnce(new Error('pause finalize failed')) .mockResolvedValueOnce({}) + const executionState = { + blockStates: { 'function-1': { output: { result: 'raw-secret-value' } } }, + executedBlocks: ['function-1'], + blockLogs: [], + decisions: { router: {}, condition: {} }, + completedLoops: [], + activeExecutionPath: ['function-1'], + } await expect( session.safeCompleteWithPause({ @@ -951,11 +1004,18 @@ describe('LoggingSession completion retries', () => { totalDurationMs: 10, traceSpans: [], workflowInput: { hello: 'world' }, + executionState, }) ).resolves.toBeUndefined() expect(session.hasCompleted()).toBe(true) expect(completeWorkflowExecutionMock).toHaveBeenCalledTimes(2) + expect(completeWorkflowExecutionMock).toHaveBeenLastCalledWith( + expect.objectContaining({ + finalizationPath: 'paused', + executionState: expect.objectContaining({ blockStates: executionState.blockStates }), + }) + ) }) it('persists last started block independently from cost accumulation', async () => { diff --git a/apps/sim/lib/logs/execution/logging-session.ts b/apps/sim/lib/logs/execution/logging-session.ts index a14e23f2907..9703d5b57bd 100644 --- a/apps/sim/lib/logs/execution/logging-session.ts +++ b/apps/sim/lib/logs/execution/logging-session.ts @@ -156,6 +156,7 @@ export interface SessionCancelledParams { endedAt?: string totalDurationMs?: number traceSpans?: TraceSpan[] + executionState?: SerializableExecutionState } export interface SessionPausedParams { @@ -163,6 +164,7 @@ export interface SessionPausedParams { totalDurationMs?: number traceSpans?: TraceSpan[] workflowInput?: any + executionState?: SerializableExecutionState } export interface LoggingSessionOptions { @@ -902,6 +904,7 @@ export class LoggingSession { costSummary, finalOutput: { cancelled: true }, traceSpans, + executionState: params.executionState, finalizationPath: 'cancelled', status: 'cancelled', }) @@ -999,6 +1002,7 @@ export class LoggingSession { finalOutput: { paused: true }, traceSpans, workflowInput, + executionState: params.executionState, finalizationPath: 'paused', status: 'pending', }) @@ -1197,6 +1201,7 @@ export class LoggingSession { isError: false, finalizationPath: 'fallback_completed', finalOutput: params.finalOutput || {}, + executionState: params.executionState, }) } } @@ -1256,6 +1261,7 @@ export class LoggingSession { isError: false, finalizationPath: 'cancelled', finalOutput: { cancelled: true }, + executionState: params?.executionState, status: 'cancelled', }) } @@ -1283,6 +1289,7 @@ export class LoggingSession { isError: false, finalizationPath: 'paused', finalOutput: { paused: true }, + executionState: params?.executionState, status: 'pending', }) } diff --git a/apps/sim/lib/workflows/executor/execution-core.test.ts b/apps/sim/lib/workflows/executor/execution-core.test.ts index 94a0ae9c789..ee6ecc3543b 100644 --- a/apps/sim/lib/workflows/executor/execution-core.test.ts +++ b/apps/sim/lib/workflows/executor/execution-core.test.ts @@ -1032,12 +1032,16 @@ describe('executeWorkflowCore terminal finalization sequencing', () => { }) it('routes cancelled executions through safeCompleteWithCancellation', async () => { + const executionState = { + blockStates: { 'function-1': { output: { result: 'raw-secret-value' } } }, + } executorExecuteMock.mockResolvedValue({ success: false, status: 'cancelled', output: {}, logs: [], metadata: { duration: 123, startTime: 'start', endTime: 'end' }, + executionState, }) const result = await executeWorkflowCore({ @@ -1052,6 +1056,7 @@ describe('executeWorkflowCore terminal finalization sequencing', () => { expect.objectContaining({ totalDurationMs: 123, traceSpans: [{ id: 'span-1' }], + executionState, }) ) expect(safeCompleteMock).not.toHaveBeenCalled() @@ -1060,12 +1065,16 @@ describe('executeWorkflowCore terminal finalization sequencing', () => { }) it('routes paused executions through safeCompleteWithPause', async () => { + const executionState = { + blockStates: { 'function-1': { output: { result: 'raw-secret-value' } } }, + } executorExecuteMock.mockResolvedValue({ success: true, status: 'paused', output: {}, logs: [], metadata: { duration: 123, startTime: 'start', endTime: 'end' }, + executionState, }) const result = await executeWorkflowCore({ @@ -1081,6 +1090,7 @@ describe('executeWorkflowCore terminal finalization sequencing', () => { totalDurationMs: 123, traceSpans: [{ id: 'span-1' }], workflowInput: { hello: 'world' }, + executionState, }) ) expect(safeCompleteMock).not.toHaveBeenCalled() diff --git a/apps/sim/lib/workflows/executor/execution-core.ts b/apps/sim/lib/workflows/executor/execution-core.ts index c12b9b475fa..160985884e7 100644 --- a/apps/sim/lib/workflows/executor/execution-core.ts +++ b/apps/sim/lib/workflows/executor/execution-core.ts @@ -256,6 +256,7 @@ async function finalizeExecutionOutcome(params: { endedAt, totalDurationMs: totalDuration || 0, traceSpans: traceSpans || [], + executionState: result.executionState, }) return } @@ -266,6 +267,7 @@ async function finalizeExecutionOutcome(params: { totalDurationMs: totalDuration || 0, traceSpans: traceSpans || [], workflowInput, + executionState: result.executionState, }) return } diff --git a/apps/sim/lib/workflows/streaming/streaming.test.ts b/apps/sim/lib/workflows/streaming/streaming.test.ts index c3c46efff1c..3c7b4d140d0 100644 --- a/apps/sim/lib/workflows/streaming/streaming.test.ts +++ b/apps/sim/lib/workflows/streaming/streaming.test.ts @@ -91,6 +91,44 @@ describe('createStreamingResponse', () => { clearLargeValueCacheForTests() }) + it('forwards raw execution state to terminal logging', async () => { + const safeComplete = vi.fn().mockResolvedValue(undefined) + const executionState = { + blockStates: { 'function-1': { output: { result: 'raw-secret-value' } } }, + } + const stream = await createStreamingResponse({ + requestId: 'request-1', + executionId: 'execution-1', + streamConfig: { + selectedOutputs: [], + includeFileBase64: false, + }, + executeFn: async () => + ({ + success: true, + status: 'completed', + output: { result: 'raw-secret-value' }, + logs: [], + metadata: { duration: 1 }, + executionState, + _streamingMetadata: { + loggingSession: { safeComplete }, + processedInput: { input: 'raw-runtime-value' }, + }, + }) as any, + }) + + await readSSEStream(stream) + + expect(safeComplete).toHaveBeenCalledWith( + expect.objectContaining({ + finalOutput: { result: 'raw-secret-value' }, + workflowInput: { input: 'raw-runtime-value' }, + executionState, + }) + ) + }) + it('extracts block-level selected outputs from JSON content payloads', async () => { const output = { content: JSON.stringify({ answer: 'ok' }) } const stream = await createStreamingResponse({ diff --git a/apps/sim/lib/workflows/streaming/streaming.ts b/apps/sim/lib/workflows/streaming/streaming.ts index 56da480803c..8ed8278a2e8 100644 --- a/apps/sim/lib/workflows/streaming/streaming.ts +++ b/apps/sim/lib/workflows/streaming/streaming.ts @@ -488,6 +488,7 @@ async function completeLoggingSession(result: ExecutionResult): Promise { finalOutput: result.output || {}, traceSpans: (traceSpans || []) as any, workflowInput: result._streamingMetadata.processedInput, + executionState: result.executionState, }) result._streamingMetadata = undefined