Add per-model response length overrides

application/single_app/config.py

@@ -95,7 +95,7 @@
 EXECUTOR_TYPE = 'thread'
 EXECUTOR_MAX_WORKERS = 30
 SESSION_TYPE = 'filesystem'
-VERSION = "0.250.108"
+VERSION = "0.250.109"
 IS_DEVELOPMENT = is_development_env_enabled()
 
 SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax')

application/single_app/functions_settings.py

@@ -1938,6 +1938,42 @@
     return changed
 
 
+MODEL_RESPONSE_LENGTH_FIELDS = (
+    "responseLength",
+    "response_length",
+    "maxTokens",
+    "max_tokens",
+    "maxCompletionTokens",
+    "max_completion_tokens",
+)
+
+
+def normalize_model_response_length(value):
+    """Return a positive integer response length, or None when unset/invalid."""
+    if value in (None, "") or isinstance(value, bool):
+        return None
+    if isinstance(value, int):
+        return value if value > 0 else None
+
+    value_text = str(value).strip()
+    if not value_text or not value_text.isdigit():
+        return None
+
+    response_length = int(value_text)
+    return response_length if response_length > 0 else None
+
+
+def normalize_model_response_length_from_model(model):
+    """Resolve the canonical response length from a model endpoint row."""
+    if not isinstance(model, dict):
+        return None
+
+    for field_name in MODEL_RESPONSE_LENGTH_FIELDS:
+        if field_name in model:
+            return normalize_model_response_length(model.get(field_name))
+    return None
+
+
 def normalize_model_endpoints(endpoints):
     """Normalize model endpoints with stable IDs and enabled flags."""
     if not isinstance(endpoints, list):
@@ -1986,6 +2022,18 @@
             if model_copy.get("enabled") is None:
                 model_copy["enabled"] = True
                 changed = True
+            response_length = normalize_model_response_length_from_model(model_copy)
+            for response_length_field in MODEL_RESPONSE_LENGTH_FIELDS:
+                if response_length_field != "responseLength" and response_length_field in model_copy:
+                    model_copy.pop(response_length_field, None)
+                    changed = True
+            if response_length is None:
+                if "responseLength" in model_copy:
+                    model_copy.pop("responseLength", None)
+                    changed = True
+            elif model_copy.get("responseLength") != response_length:
+                model_copy["responseLength"] = response_length
+                changed = True
             try:
                 normalized_icon = normalize_icon_payload(model_copy.get("icon"), field_name="model.icon")
             except ValueError:

application/single_app/model_endpoint_clients.py

@@ -54,7 +54,15 @@
 
     @property
     def is_openai_reasoning_model(self) -> bool:
-        return self.normalized_deployment_name.startswith(OPENAI_REASONING_MODEL_PREFIXES)
+        normalized_model_name = (
+            self.normalized_deployment_name
+            .replace("_", "-")
+            .replace(" ", "-")
+        )
+        return (
+            normalized_model_name.startswith(OPENAI_REASONING_MODEL_PREFIXES)
+            or "gpt-5" in normalized_model_name
+        )
 
     @property
     def is_foundry_non_openai_model(self) -> bool:
@@ -76,6 +84,10 @@
             return ""
         return normalized_reasoning_effort if self.is_openai_reasoning_model else ""
 
+    @property
+    def response_length_parameter(self) -> str:
+        return "max_completion_tokens" if self.is_openai_reasoning_model else "max_tokens"
+
 
 def normalize_endpoint_text(endpoint: Any) -> str:
     """Return a trimmed endpoint URL without a trailing slash."""

application/single_app/route_backend_chats.py

@@ -290,6 +290,19 @@
     return resolved_reasoning_effort
 
 
+def _apply_response_length_for_model(api_params, response_length, model_name, provider=None, response_length_parameter=None):
+    normalized_response_length = normalize_model_response_length(response_length)
+    if not normalized_response_length:
+        return None
+
+    response_length_parameter = response_length_parameter or ModelEndpointBehavior(provider, model_name).response_length_parameter
+    api_params[response_length_parameter] = normalized_response_length
+    debug_print(
+        f"[ModelEndpoint] Applying response length: {response_length_parameter}={normalized_response_length} for {model_name}"
+    )
+    return response_length_parameter
+
+
 def _is_foundry_non_openai_model(provider, model_name):
     return ModelEndpointBehavior(provider, model_name).is_foundry_non_openai_model
 
@@ -298,6 +311,18 @@
     return ModelEndpointBehavior(provider, model_name).context_mode != MODEL_CONTEXT_MODE_FOLD_LATEST_USER
 
 
+def _build_model_endpoint_behavior_name(model_cfg, deployment):
+    if not isinstance(model_cfg, dict):
+        return deployment
+    model_names = [
+        deployment,
+        model_cfg.get('modelName'),
+        model_cfg.get('displayName'),
+        model_cfg.get('name'),
+    ]
+    return ' '.join(str(model_name or '').strip() for model_name in model_names if str(model_name or '').strip())
+
+
 def _build_plain_fact_memory_background_notes(prompt_payload):
     note_values = []
     for payload_key in ('instruction_payload', 'recall_payload'):
@@ -10876,6 +10901,13 @@
     api_version = str(connection.get('openai_api_version') or connection.get('api_version') or '').strip()
     runtime_protocol = infer_model_endpoint_protocol(provider, endpoint, deployment)
     model_icon = _normalize_model_icon_payload(model_cfg.get('icon'))
+    model_response_length = normalize_model_response_length_from_model(model_cfg)
+    model_behavior_name = _build_model_endpoint_behavior_name(model_cfg, deployment)
+    model_response_length_parameter = (
+        ModelEndpointBehavior(provider, model_behavior_name).response_length_parameter
+        if model_response_length
+        else None
+    )
 
     if requested_provider and requested_provider != provider:
         debug_print(
@@ -10905,7 +10937,9 @@
     debug_print(
         f"[Streaming][Model Resolution] Resolved {selection_source} multi-endpoint model | "
         f"provider={provider} | endpoint_id={requested_endpoint_id} | model_id={model_cfg.get('id')} | "
-        f"deployment={deployment} | api_version={api_version} | protocol={runtime_protocol}"
+        f"deployment={deployment} | api_version={api_version} | protocol={runtime_protocol} | "
+        f"response_length={model_response_length or ''} | "
+        f"response_length_parameter={model_response_length_parameter or ''}"
     )
     return (
         gpt_client,
@@ -10917,6 +10951,8 @@
         requested_endpoint_id,
         str(model_cfg.get('id') or '').strip(),
         model_icon,
+        model_response_length,
+        model_response_length_parameter,
     )
 
 
@@ -12889,6 +12925,8 @@
             gpt_endpoint_id = None
             gpt_model_id = None
             gpt_model_icon = None
+            gpt_response_length = None
+            gpt_response_length_parameter = None
             tabular_model_context = None
             enable_gpt_apim = settings.get('enable_gpt_apim', False)
             enable_image_gen_apim = settings.get('enable_image_gen_apim', False)
@@ -12921,6 +12959,8 @@
                         gpt_endpoint_id,
                         gpt_model_id,
                         gpt_model_icon,
+                        gpt_response_length,
+                        gpt_response_length_parameter,
                     ) = multi_endpoint_config
                 elif enable_gpt_apim:
                     # read raw comma-delimited deployments
@@ -13342,6 +13382,7 @@
                     'model_id': gpt_model_id or data.get('model_id'),
                     'model_provider': gpt_provider or data.get('model_provider'),
                     'model_icon': gpt_model_icon,
+                    'response_length': gpt_response_length,
                     'reasoning_effort': reasoning_effort if reasoning_effort and reasoning_effort != 'none' else None,
                     'streaming': 'Disabled'
                 }
@@ -15597,6 +15638,13 @@
                     'model': gpt_model,
                     'messages': conversation_history_for_api,
                 }
+                _apply_response_length_for_model(
+                    api_params,
+                    gpt_response_length,
+                    gpt_model,
+                    provider=gpt_provider,
+                    response_length_parameter=gpt_response_length_parameter,
+                )
 
                 request_reasoning_effort = _resolve_reasoning_effort_for_model(
                     reasoning_effort,
@@ -15869,6 +15917,7 @@
                         'model_id': gpt_model_id,
                         'model_provider': gpt_provider,
                         'model_icon': gpt_model_icon,
+                        'response_length': gpt_response_length,
                         'streaming': 'Disabled',
                     },
                     'history_context': history_debug_info,
@@ -16592,6 +16641,8 @@
                 gpt_endpoint_id = None
                 gpt_model_id = None
                 gpt_model_icon = None
+                gpt_response_length = None
+                gpt_response_length_parameter = None
                 tabular_model_context = None
                 enable_gpt_apim = settings.get('enable_gpt_apim', False)
                 should_use_default_model = (
@@ -16625,6 +16676,8 @@
                             gpt_endpoint_id,
                             gpt_model_id,
                             gpt_model_icon,
+                            gpt_response_length,
+                            gpt_response_length_parameter,
                         ) = streaming_multi_endpoint_config
                     elif enable_gpt_apim:
                         raw = settings.get('azure_apim_gpt_deployment', '')
@@ -16993,6 +17046,7 @@
                         'model_id': gpt_model_id or data.get('model_id'),
                         'model_provider': gpt_provider or data.get('model_provider'),
                         'model_icon': gpt_model_icon,
+                        'response_length': gpt_response_length,
                         'reasoning_effort': reasoning_effort if reasoning_effort and reasoning_effort != 'none' else None,
                         'streaming': 'Enabled'
                     }
@@ -18432,6 +18486,7 @@
                                     'model_id': gpt_model_id,
                                     'model_provider': gpt_provider,
                                     'model_icon': gpt_model_icon,
+                                    'response_length': gpt_response_length,
                                     'streaming': 'Enabled',
                                 },
                                 'history_context': history_debug_info,
@@ -18836,6 +18891,13 @@
                             'stream': True,
                             'stream_options': {'include_usage': True}  # Request token usage in final chunk
                         }
+                        _apply_response_length_for_model(
+                            stream_params,
+                            gpt_response_length,
+                            gpt_model,
+                            provider=gpt_provider,
+                            response_length_parameter=gpt_response_length_parameter,
+                        )
 
                         request_reasoning_effort = _resolve_reasoning_effort_for_model(
                             reasoning_effort,
@@ -19014,6 +19076,7 @@
                                 'model_id': gpt_model_id,
                                 'model_provider': gpt_provider,
                                 'model_icon': gpt_model_icon,
+                                'response_length': gpt_response_length,
                                 'streaming': 'Enabled',
                             },
                             'history_context': history_debug_info,
@@ -19102,6 +19165,7 @@
                         if 'metadata' in user_message_doc and 'model_selection' in user_message_doc['metadata']:
                             user_message_doc['metadata']['model_selection']['selected_model'] = final_model_used if use_agent_streaming else gpt_model
                             user_message_doc['metadata']['model_selection']['model_icon'] = gpt_model_icon
+                            user_message_doc['metadata']['model_selection']['response_length'] = gpt_response_length
                         if selected_agent_metadata:
                             user_message_doc.setdefault('metadata', {})['agent_selection'] = selected_agent_metadata
                         cosmos_messages_container.upsert_item(user_message_doc)

application/single_app/static/js/admin/admin_model_endpoints.js

@@ -998,6 +998,44 @@
     return input;
 }
 
+function normalizeModelResponseLength(value) {
+    const valueText = String(value ?? "").trim();
+    if (!valueText) {
+        return "";
+    }
+    if (!/^\d+$/.test(valueText)) {
+        return null;
+    }
+
+    const parsedValue = Number.parseInt(valueText, 10);
+    return parsedValue > 0 ? parsedValue : null;
+}
+
+function getModelResponseLength(model) {
+    return normalizeModelResponseLength(
+        model.responseLength
+        ?? model.response_length
+        ?? model.maxTokens
+        ?? model.max_tokens
+        ?? model.maxCompletionTokens
+        ?? model.max_completion_tokens
+    );
+}
+
+function createModelResponseLengthInput(modelId, value) {
+    const input = document.createElement("input");
+    input.type = "number";
+    input.className = "form-control form-control-sm";
+    input.min = "1";
+    input.step = "1";
+    input.placeholder = "Optional";
+    input.dataset.responseLengthFor = modelId;
+    input.id = getModelIconDomId(modelId, "response-length");
+    input.value = value || "";
+    input.setAttribute("aria-describedby", getModelIconDomId(modelId, "response-length-help"));
+    return input;
+}
+
 function getModelIconDomId(modelId, suffix) {
     const safeModelId = String(modelId || "model").replace(/[^A-Za-z0-9_-]/g, "-");
     return `model-${safeModelId}-${suffix}`;
@@ -1148,6 +1186,7 @@
         const modelName = model.modelName || "";
         const displayName = model.displayName || deploymentName;
         const description = model.description || "";
+        const responseLength = getModelResponseLength(model);
         const deploymentReadonly = model.isDiscovered ? "readonly" : "";
         const modelId = model.id || generateId();
         model.id = modelId;
@@ -1179,12 +1218,22 @@
         const iconCol = createElement("div", "col-md-4");
         iconCol.appendChild(createSmallLabel("Icon"));
         iconCol.appendChild(createModelIconEditor(model, modelId));
+        const responseLengthCol = createElement("div", "col-md-4");
+        const responseLengthLabel = createSmallLabel("Response Length");
+        responseLengthLabel.htmlFor = getModelIconDomId(modelId, "response-length");
+        responseLengthCol.appendChild(responseLengthLabel);
+        responseLengthCol.appendChild(createModelResponseLengthInput(modelId, responseLength));
+        const responseLengthHelp = createElement("div", "form-text");
+        responseLengthHelp.id = getModelIconDomId(modelId, "response-length-help");
+        responseLengthHelp.textContent = "Optional output token ceiling for standard chat responses.";
+        responseLengthCol.appendChild(responseLengthHelp);
         const descriptionCol = createElement("div", "col-md-8");
         descriptionCol.appendChild(createSmallLabel("Description (optional)"));
         descriptionCol.appendChild(createModelTextInput(modelId, "descriptionFor", description));
         fieldsRow.appendChild(deploymentCol);
         fieldsRow.appendChild(displayCol);
         fieldsRow.appendChild(iconCol);
+        fieldsRow.appendChild(responseLengthCol);
         fieldsRow.appendChild(descriptionCol);
 
         const actions = createElement("div", "d-flex gap-2 mt-2");
@@ -1224,12 +1273,22 @@
         const deploymentInput = modelsListEl.querySelector(`input[data-deployment-name-for="${model.id}"]`);
         const displayInput = modelsListEl.querySelector(`input[data-display-name-for="${model.id}"]`);
         const descriptionInput = modelsListEl.querySelector(`input[data-description-for="${model.id}"]`);
+        const responseLengthInput = modelsListEl.querySelector(`input[data-response-length-for="${model.id}"]`);
         const iconEditor = findModelEditor(model.id);
+        const responseLength = responseLengthInput ? normalizeModelResponseLength(responseLengthInput.value) : "";
+        if (responseLength === null) {
+            throw new Error("Response length must be a positive whole number.");
+        }
         model.enabled = checkbox ? checkbox.checked : model.enabled;
         model.deploymentName = deploymentInput ? deploymentInput.value.trim() : model.deploymentName;
         model.displayName = displayInput ? displayInput.value.trim() : model.displayName;
         model.icon = iconEditor ? getIconPayload(iconEditor, MODEL_ICON_CONTROL_CONFIG) : model.icon || {};
         model.description = descriptionInput ? descriptionInput.value.trim() : model.description;
+        if (responseLength) {
+            model.responseLength = responseLength;
+        } else {
+            delete model.responseLength;
+        }
     });
     return updated;
 }

application/single_app/templates/_multiendpoint_modal.html

@@ -153,6 +153,9 @@ <h6 class="alert-heading mb-2"><i class="bi bi-info-circle me-2"></i>Identity se
 
                 <div class="mt-3">
                     <h6>Available Models</h6>
+                    <div class="form-text mb-2">
+                        Set an optional response length on each model to cap standard chat output tokens for that model.
+                    </div>
                     <button type="button" class="btn btn-sm btn-outline-primary mt-2 d-none" id="model-endpoint-add-model-btn">
                         <i class="bi bi-plus-circle me-1"></i>Add Model
                     </button>