improvement(agent): report a model level the sanitizer discards

A model bound to a variable or block reference only resolves at execution time,
so a run whose reference landed on a model outside Sim's catalogue had its
requested level cleared with no signal and quietly fell back to that model's
default. Dropping stays the safe default — a provider with no such parameter
rejects the whole request — but it is now reported.

- log the field, model, and value whenever an unsupported-field level is cleared
- cover both diagnostics, including that they stay quiet for a declared level
  and for the `auto` / `none` sentinels

Author: waleedlatif1

apps/sim/providers/index.test.ts

@@ -4,11 +4,32 @@
 import { envFlagsMockFns, resetEnvFlagsMock } from '@sim/testing'
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 
-const { mockGetApiKeyWithBYOK, mockExecuteRequest } = vi.hoisted(() => ({
+const { mockGetApiKeyWithBYOK, mockExecuteRequest, mockLoggerWarn } = vi.hoisted(() => ({
   mockGetApiKeyWithBYOK: vi.fn(),
   mockExecuteRequest: vi.fn(),
+  mockLoggerWarn: vi.fn(),
 }))
 
+/** Overrides the global logger mock so the sanitizer's warnings are assertable. */
+vi.mock('@sim/logger', () => {
+  const createLogger = () => ({
+    info: vi.fn(),
+    warn: mockLoggerWarn,
+    error: vi.fn(),
+    debug: vi.fn(),
+    trace: vi.fn(),
+    fatal: vi.fn(),
+    child: () => createLogger(),
+    withMetadata: () => createLogger(),
+  })
+  return {
+    createLogger,
+    logger: createLogger(),
+    runWithRequestContext: <T>(_ctx: unknown, fn: () => T): T => fn(),
+    getRequestContext: () => undefined,
+  }
+})
+
 vi.mock('@/lib/api-key/byok', () => ({
   getApiKeyWithBYOK: (...args: unknown[]) => mockGetApiKeyWithBYOK(...args),
 }))
@@ -528,4 +549,59 @@ describe('executeProviderRequest — model level normalization', () => {
     expect(sentRequest().reasoningEffort).toBeUndefined()
     expect(sentRequest().verbosity).toBeUndefined()
   })
+
+  /**
+   * The model can itself be a reference, so it is only known at execution time. A run whose
+   * reference resolved to a model outside Sim's catalogue must not fall back to that model's
+   * default in silence.
+   */
+  it('reports the level it drops when the resolved model does not support the field', async () => {
+    await executeProviderRequest('anthropic', {
+      model: 'claude-opus-4-6',
+      workspaceId: 'ws-1',
+      reasoningEffort: 'high',
+    })
+
+    expect(mockLoggerWarn).toHaveBeenCalledWith(
+      'Model does not support this level; dropping it from the request',
+      expect.objectContaining({
+        field: 'reasoningEffort',
+        model: 'claude-opus-4-6',
+        value: 'high',
+      })
+    )
+  })
+
+  it('stays quiet when an unsupported model was never given a level', async () => {
+    await executeProviderRequest('anthropic', {
+      model: 'claude-opus-4-6',
+      workspaceId: 'ws-1',
+    })
+
+    expect(mockLoggerWarn).not.toHaveBeenCalled()
+  })
+
+  it('reports a level the model accepts but does not declare', async () => {
+    await executeProviderRequest('openai', {
+      model: 'gpt-5',
+      workspaceId: 'ws-1',
+      reasoningEffort: 'xhigh',
+    })
+
+    expect(mockLoggerWarn).toHaveBeenCalledWith(
+      'Model level is not one this model declares; forwarding to the provider',
+      expect.objectContaining({ field: 'reasoningEffort', model: 'gpt-5', value: 'xhigh' })
+    )
+  })
+
+  it('stays quiet for a declared level and for the auto and none sentinels', async () => {
+    await executeProviderRequest('openai', {
+      model: 'gpt-5',
+      workspaceId: 'ws-1',
+      reasoningEffort: 'auto',
+      verbosity: 'high',
+    })
+
+    expect(mockLoggerWarn).not.toHaveBeenCalled()
+  })
 })

apps/sim/providers/index.ts

@@ -64,8 +64,34 @@ function normalizeModelLevel(value: string | undefined): string | undefined {
  */
 const MODEL_LEVEL_SENTINELS = new Set(['auto', 'none'])
 
+type ModelLevelField = 'reasoningEffort' | 'verbosity' | 'thinkingLevel'
+
+/**
+ * Clears a level whose resolved model does not accept the field at all.
+ *
+ * Dropping is the safe default — a provider that has no such parameter rejects the whole
+ * request — but the discard is reported because the model can be bound to a variable or block
+ * reference and is therefore only known at execution time. Without this, a run whose reference
+ * resolved to a model outside Sim's catalogue would quietly fall back to that model's default
+ * while the caller believed the level applied.
+ */
+function dropUnsupportedLevel(
+  field: ModelLevelField,
+  model: string,
+  value: string | undefined
+): undefined {
+  if (value) {
+    logger.warn('Model does not support this level; dropping it from the request', {
+      field,
+      model,
+      value,
+    })
+  }
+  return undefined
+}
+
 /**
- * Logs a level that is not one the model declares.
+ * Logs a level that the model accepts as a field but does not list as a value.
  *
  * Deliberately does not drop the value. Sim's per-model level lists exist to populate the
  * pickers and can lag a provider that has started accepting a new level, so rejecting on them
@@ -74,7 +100,7 @@ const MODEL_LEVEL_SENTINELS = new Set(['auto', 'none'])
  * levels needs, where silently substituting the model default would corrupt the results.
  */
 function warnOnUnrecognizedLevel(
-  field: 'reasoningEffort' | 'verbosity' | 'thinkingLevel',
+  field: ModelLevelField,
   model: string | undefined,
   value: string | undefined,
   declaredValues: string[] | null
@@ -103,15 +129,27 @@ function sanitizeRequest(request: ProviderRequest): ProviderRequest {
   }
 
   if (model && !supportsReasoningEffort(model)) {
-    sanitizedRequest.reasoningEffort = undefined
+    sanitizedRequest.reasoningEffort = dropUnsupportedLevel(
+      'reasoningEffort',
+      model,
+      sanitizedRequest.reasoningEffort
+    )
   }
 
   if (model && !supportsVerbosity(model)) {
-    sanitizedRequest.verbosity = undefined
+    sanitizedRequest.verbosity = dropUnsupportedLevel(
+      'verbosity',
+      model,
+      sanitizedRequest.verbosity
+    )
   }
 
   if (model && !supportsThinking(model)) {
-    sanitizedRequest.thinkingLevel = undefined
+    sanitizedRequest.thinkingLevel = dropUnsupportedLevel(
+      'thinkingLevel',
+      model,
+      sanitizedRequest.thinkingLevel
+    )
   }
 
   if (model && !supportsPromptCaching(model)) {