feat(anthropic): update model YAMLs [bot] - #2087
Conversation
|
/test-models |
Gateway test results
Failures (36)
ErrorCode snippetfrom openai import OpenAI
client = OpenAI(api_key="***", base_url="https://internal.devtest.truefoundry.tech/api/llm")
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather for a location.",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city name, e.g. London",
},
},
"required": ["location"],
"additionalProperties": False,
},
"strict": True,
},
},
]
response = client.chat.completions.create(
model="test-v2-anthropic/claude-opus-4-8",
messages=[
{"role": "system", "content": "You are a helpful assistant with access to tools. You MUST strictly use the provided tools to answer. Never respond with plain text when a tool is available."},
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "Use the get_weather tool to check the weather in London. You must call the tool, do not respond with plain text."},
],
tools=tools,
tool_choice="auto",
stream=False,
)
_message = response.choices[0].message
if _message.tool_calls:
for _tc in _message.tool_calls:
print(f"Function: {_tc.function.name}")
print(f"Arguments: {_tc.function.arguments}")
else:
print(_message.content)
if not _message.tool_calls or len(_message.tool_calls) == 0:
raise Exception("VALIDATION FAILED: tool-call - no tool calls in response")
print("VALIDATION: tool-call SUCCESS")
ErrorCode snippetfrom openai import OpenAI
client = OpenAI(api_key="***", base_url="https://internal.devtest.truefoundry.tech/api/llm")
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather for a location.",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city name, e.g. London",
},
},
"required": ["location"],
"additionalProperties": False,
},
"strict": True,
},
},
]
response = client.chat.completions.create(
model="test-v2-anthropic/claude-opus-4-8",
messages=[
{"role": "system", "content": "You are a helpful assistant with access to tools. You MUST strictly use the provided tools to answer. Never respond with plain text when a tool is available."},
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "Use the get_weather tool to check the weather in London. You must call the tool, do not respond with plain text."},
],
tools=tools,
tool_choice="auto",
stream=True,
)
_tool_calls_made = False
for chunk in response:
if chunk.choices and len(chunk.choices) > 0:
delta = chunk.choices[0].delta
if delta.content is not None:
print(delta.content, end="", flush=True)
if delta.tool_calls:
_tool_calls_made = True
for _tc in delta.tool_calls:
if _tc.function:
print(_tc.function.arguments or "", end="", flush=True)
if not _tool_calls_made:
raise Exception("VALIDATION FAILED: tool-call stream - no tool calls received")
print("\nVALIDATION: tool-call stream SUCCESS")
ErrorCode snippetfrom openai import OpenAI
client = OpenAI(api_key="***", base_url="https://internal.devtest.truefoundry.tech/api/llm")
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather for a location.",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city name, e.g. London",
},
},
"required": ["location"],
"additionalProperties": False,
},
"strict": True,
},
},
]
response = client.chat.completions.create(
model="test-v2-anthropic/claude-opus-4-8",
messages=[
{"role": "system", "content": "You are a helpful assistant with access to tools. You MUST strictly call multiple tools in parallel whenever possible. Never call them sequentially."},
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "Use the get_weather tool to check the weather in London and Paris. You MUST make both tool calls strictly in parallel, not sequentially."},
],
tools=tools,
tool_choice="auto",
parallel_tool_calls=True,
stream=False,
)
_message = response.choices[0].message
if _message.tool_calls:
print(f"Number of parallel tool calls: {len(_message.tool_calls)}")
for _tc in _message.tool_calls:
print(f"Function: {_tc.function.name}")
print(f"Arguments: {_tc.function.arguments}")
else:
print(_message.content)
if not _message.tool_calls or len(_message.tool_calls) < 1:
raise Exception(
f"VALIDATION FAILED: parallel-tool-call - expected at least 1 tool call, "
f"got {len(_message.tool_calls) if _message.tool_calls else 0}"
)
print("VALIDATION: parallel-tool-call SUCCESS")
ErrorCode snippetfrom openai import OpenAI
client = OpenAI(api_key="***", base_url="https://internal.devtest.truefoundry.tech/api/llm")
response = client.chat.completions.create(
model="test-v2-anthropic/claude-opus-4-8",
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "What is the capital of France?"},
],
max_tokens=1000,
stream=False,
)
print(response.choices[0].message.content)
ErrorCode snippetfrom openai import OpenAI
client = OpenAI(api_key="***", base_url="https://internal.devtest.truefoundry.tech/api/llm")
response = client.chat.completions.create(
model="test-v2-anthropic/claude-opus-4-8",
messages=[
{"role": "system", "content": "You are a helpful assistant. You MUST think step by step and show your reasoning. Never skip reasoning steps."},
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "How to calculate 3^3^3^3? Think step by step and show all reasoning."},
],
reasoning_effort="medium",
stream=True,
)
_reasoning_detected = False
for chunk in response:
if chunk.choices and len(chunk.choices) > 0:
delta = chunk.choices[0].delta
if delta.content is not None:
print(delta.content, end="", flush=True)
if getattr(delta, "reasoning_content", None) is not None:
_reasoning_detected = True
if getattr(delta, "reasoning", None) is not None:
_reasoning_detected = True
_usage = getattr(chunk, "usage", None)
if _usage is not None:
_details = getattr(_usage, "completion_tokens_details", None)
if _details and getattr(_details, "reasoning_tokens", 0) > 0:
_reasoning_detected = True
if not _reasoning_detected:
raise Exception("VALIDATION FAILED: reasoning stream - no reasoning information in stream")
print("\nVALIDATION: reasoning stream SUCCESS")
ErrorCode snippetimport anthropic
_api_key = "***"
_model = "test-v2-anthropic/claude-opus-4-8"
client = anthropic.Anthropic(
api_key=_api_key,
base_url="https://internal.devtest.truefoundry.tech/api/llm",
)
tools = [
{
"name": "get_weather",
"description": "Get the current weather for a location.",
"input_schema": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city name, e.g. London",
},
},
"required": ["location"],
},
},
]
messages = [
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "Use the get_weather tool to check the weather in London. You must call the tool, do not respond with plain text."},
]
with client.messages.stream(
model=_model,
system="You are a helpful assistant with access to tools. You MUST strictly use the provided tools to answer. Never respond with plain text when a tool is available.",
messages=messages,
tools=tools,
tool_choice={"type": "auto"},
max_tokens=1024,
) as stream:
response = stream.get_final_message()
for block in response.content:
if block.type == "tool_use":
print(f"Tool: {block.name}")
print(f"Input: {block.input}")
elif block.type == "text":
print(block.text)
_tool_uses = [block for block in response.content if block.type == "tool_use"]
if not _tool_uses:
raise Exception("VALIDATION FAILED: tool-call stream - no tool uses in Anthropic stream response")
print("\nVALIDATION: tool-call stream SUCCESS")
ErrorCode snippetimport anthropic
_api_key = "***"
_model = "test-v2-anthropic/claude-opus-4-8"
client = anthropic.Anthropic(
api_key=_api_key,
base_url="https://internal.devtest.truefoundry.tech/api/llm",
)
tools = [
{
"name": "get_weather",
"description": "Get the current weather for a location.",
"input_schema": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city name, e.g. London",
},
},
"required": ["location"],
},
},
]
messages = [
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "Use the get_weather tool to check the weather in London. You must call the tool, do not respond with plain text."},
]
response = client.messages.create(
model=_model,
system="You are a helpful assistant with access to tools. You MUST strictly use the provided tools to answer. Never respond with plain text when a tool is available.",
messages=messages,
tools=tools,
tool_choice={"type": "auto"},
max_tokens=1024,
)
for block in response.content:
if block.type == "tool_use":
print(f"Tool: {block.name}")
print(f"Input: {block.input}")
elif block.type == "text":
print(block.text)
_tool_uses = [block for block in response.content if block.type == "tool_use"]
if not _tool_uses:
raise Exception("VALIDATION FAILED: tool-call - no tool uses in Anthropic response")
print("VALIDATION: tool-call SUCCESS")
ErrorCode snippetimport anthropic
_api_key = "***"
_model = "test-v2-anthropic/claude-opus-4-8"
client = anthropic.Anthropic(
api_key=_api_key,
base_url="https://internal.devtest.truefoundry.tech/api/llm",
)
messages = [
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "What is the capital of France?"},
]
response = client.messages.create(
model=_model,
system="You are a helpful assistant.",
messages=messages,
max_tokens=1000,
)
for block in response.content:
if block.type == "text":
print(block.text)
ErrorCode snippetfrom openai import OpenAI
client = OpenAI(api_key="***", base_url="https://internal.devtest.truefoundry.tech/api/llm")
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather for a location.",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city name, e.g. London",
},
},
"required": ["location"],
"additionalProperties": False,
},
"strict": True,
},
},
]
response = client.chat.completions.create(
model="test-v2-anthropic/claude-opus-4-8",
messages=[
{"role": "system", "content": "You are a helpful assistant with access to tools. You MUST strictly call multiple tools in parallel whenever possible. Never call them sequentially."},
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "Use the get_weather tool to check the weather in London and Paris. You MUST make both tool calls strictly in parallel, not sequentially."},
],
tools=tools,
tool_choice="auto",
parallel_tool_calls=True,
stream=True,
)
_tool_call_indices = set()
for chunk in response:
if chunk.choices and len(chunk.choices) > 0:
delta = chunk.choices[0].delta
if delta.content is not None:
print(delta.content, end="", flush=True)
if delta.tool_calls:
for _tc in delta.tool_calls:
_tool_call_indices.add(_tc.index)
if _tc.function:
print(_tc.function.arguments or "", end="", flush=True)
if len(_tool_call_indices) < 1:
raise Exception(
f"VALIDATION FAILED: parallel-tool-call stream - expected at least 1 tool call, "
f"got {len(_tool_call_indices)}"
)
print(f"\nNumber of parallel tool calls: {len(_tool_call_indices)}")
print("VALIDATION: parallel-tool-call stream SUCCESS")
ErrorCode snippetimport anthropic
_api_key = "***"
_model = "test-v2-anthropic/claude-opus-4-8"
client = anthropic.Anthropic(
api_key=_api_key,
base_url="https://internal.devtest.truefoundry.tech/api/llm",
)
messages = [
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "How to calculate 3^3^3^3? Think step by step and show all reasoning."},
]
response = client.messages.create(
model=_model,
system="You are a helpful assistant. You MUST think step by step and show your reasoning. Never skip reasoning steps.",
messages=messages,
max_tokens=16000,
thinking={"type":"adaptive"},
)
for block in response.content:
if block.type == "thinking":
print(f"Thinking: {block.thinking[:200]}...")
elif block.type == "text":
print(block.text)
_reasoning_detected = False
for block in response.content:
if block.type == "thinking":
_reasoning_detected = True
if hasattr(response, "usage") and response.usage:
if getattr(response.usage, "cache_creation_input_tokens", 0) or getattr(response.usage, "cache_read_input_tokens", 0):
pass
if not _reasoning_detected:
print("Response content types: ", [block.type for block in response.content])
raise Exception("VALIDATION FAILED: reasoning - no thinking blocks in Anthropic response")
print("VALIDATION: reasoning SUCCESS")
ErrorCode snippetfrom openai import OpenAI
client = OpenAI(api_key="***", base_url="https://internal.devtest.truefoundry.tech/api/llm")
response = client.chat.completions.create(
model="test-v2-anthropic/claude-opus-4-8",
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "What is the capital of France?"},
],
max_tokens=1000,
stream=True,
)
for chunk in response:
if chunk.choices and len(chunk.choices) > 0:
delta = chunk.choices[0].delta
if delta.content is not None:
print(delta.content, end="", flush=True)
ErrorCode snippetfrom openai import OpenAI
client = OpenAI(api_key="***", base_url="https://internal.devtest.truefoundry.tech/api/llm")
response = client.chat.completions.create(
model="test-v2-anthropic/claude-opus-4-8",
messages=[
{"role": "system", "content": "You are a helpful assistant. You MUST think step by step and show your reasoning. Never skip reasoning steps."},
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "How to calculate 3^3^3^3? Think step by step and show all reasoning."},
],
reasoning_effort="medium",
stream=False,
)
_usage = getattr(response, "usage", None)
_reasoning_detected = False
_choices = getattr(response, "choices", None)
if _choices and len(_choices) > 0:
_message = getattr(_choices[0], "message", None)
else:
_message = None
if _message and getattr(_message, "content", None) is not None:
print(_message.content)
if _usage is not None:
_output_token_details = getattr(_usage, "completion_tokens_details", None)
if _output_token_details and getattr(_output_token_details, "reasoning_tokens", 0) > 0:
_reasoning_detected = True
elif getattr(_usage, "reasoning", None) is not None:
_reasoning_detected = True
if getattr(_message, "reasoning_content", None) is not None:
_reasoning_detected = True
elif getattr(_message, "reasoning", None) is not None:
_reasoning_detected = True
if not _reasoning_detected:
print("Response: ", response)
raise Exception("VALIDATION FAILED: reasoning - no reasoning information in response")
print("VALIDATION: reasoning SUCCESS")
ErrorCode snippetimport anthropic
_api_key = "***"
_model = "test-v2-anthropic/claude-opus-4-8"
client = anthropic.Anthropic(
api_key=_api_key,
base_url="https://internal.devtest.truefoundry.tech/api/llm",
)
tools = [
{
"name": "get_weather",
"description": "Get the current weather for a location.",
"input_schema": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city name, e.g. London",
},
},
"required": ["location"],
},
},
]
messages = [
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "Use the get_weather tool to check the weather in London and Paris. You MUST make both tool calls strictly in parallel, not sequentially."},
]
response = client.messages.create(
model=_model,
system="You are a helpful assistant with access to tools. You MUST strictly call multiple tools in parallel whenever possible. Never call them sequentially.",
messages=messages,
tools=tools,
tool_choice={"type": "auto"},
max_tokens=1024,
)
for block in response.content:
if block.type == "tool_use":
print(f"Tool: {block.name}")
print(f"Input: {block.input}")
elif block.type == "text":
print(block.text)
_tool_uses = [block for block in response.content if block.type == "tool_use"]
if _tool_uses:
print(f"Number of parallel tool calls: {len(_tool_uses)}")
if not _tool_uses or len(_tool_uses) < 1:
raise Exception(
f"VALIDATION FAILED: parallel-tool-call - expected at least 1 tool call, "
f"got {len(_tool_uses) if _tool_uses else 0}"
)
print("VALIDATION: parallel-tool-call SUCCESS")
ErrorCode snippetimport anthropic
_api_key = "***"
_model = "test-v2-anthropic/claude-opus-4-8"
client = anthropic.Anthropic(
api_key=_api_key,
base_url="https://internal.devtest.truefoundry.tech/api/llm",
)
tools = [
{
"name": "get_weather",
"description": "Get the current weather for a location.",
"input_schema": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city name, e.g. London",
},
},
"required": ["location"],
},
},
]
messages = [
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "Use the get_weather tool to check the weather in London and Paris. You MUST make both tool calls strictly in parallel, not sequentially."},
]
with client.messages.stream(
model=_model,
system="You are a helpful assistant with access to tools. You MUST strictly call multiple tools in parallel whenever possible. Never call them sequentially.",
messages=messages,
tools=tools,
tool_choice={"type": "auto"},
max_tokens=1024,
) as stream:
response = stream.get_final_message()
for block in response.content:
if block.type == "tool_use":
print(f"Tool: {block.name}")
print(f"Input: {block.input}")
elif block.type == "text":
print(block.text)
_tool_uses = [block for block in response.content if block.type == "tool_use"]
if _tool_uses:
print(f"Number of parallel tool calls: {len(_tool_uses)}")
if not _tool_uses or len(_tool_uses) < 1:
raise Exception(
f"VALIDATION FAILED: parallel-tool-call stream - expected at least 1 tool call, "
f"got {len(_tool_uses) if _tool_uses else 0}"
)
print(f"\nVALIDATION: parallel-tool-call stream SUCCESS")
ErrorCode snippetimport anthropic
_api_key = "***"
_model = "test-v2-anthropic/claude-opus-4-8"
client = anthropic.Anthropic(
api_key=_api_key,
base_url="https://internal.devtest.truefoundry.tech/api/llm",
)
messages = [
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "What is the capital of France?"},
]
with client.messages.stream(
model=_model,
system="You are a helpful assistant.",
messages=messages,
max_tokens=1000,
) as stream:
for text in stream.text_stream:
print(text, end="", flush=True)
ErrorCode snippetimport anthropic
_api_key = "***"
_model = "test-v2-anthropic/claude-opus-4-8"
client = anthropic.Anthropic(
api_key=_api_key,
base_url="https://internal.devtest.truefoundry.tech/api/llm",
)
messages = [
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "How to calculate 3^3^3^3? Think step by step and show all reasoning."},
]
with client.messages.stream(
model=_model,
system="You are a helpful assistant. You MUST think step by step and show your reasoning. Never skip reasoning steps.",
messages=messages,
max_tokens=16000,
thinking={"type":"adaptive"},
) as stream:
response = stream.get_final_message()
for block in response.content:
if block.type == "thinking":
print(f"Thinking: {block.thinking[:200]}...")
elif block.type == "text":
print(block.text)
_reasoning_detected = False
for block in response.content:
if block.type == "thinking":
_reasoning_detected = True
if not _reasoning_detected:
print("Response content types: ", [block.type for block in response.content])
raise Exception("VALIDATION FAILED: reasoning stream - no thinking blocks in Anthropic stream response")
print("\nVALIDATION: reasoning stream SUCCESS")
ErrorCode snippetfrom openai import OpenAI
client = OpenAI(api_key="***", base_url="https://internal.devtest.truefoundry.tech/api/llm")
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather for a location.",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city name, e.g. London",
},
},
"required": ["location"],
"additionalProperties": False,
},
"strict": True,
},
},
]
response = client.chat.completions.create(
model="test-v2-anthropic/claude-opus-5",
messages=[
{"role": "system", "content": "You are a helpful assistant with access to tools. You MUST strictly use the provided tools to answer. Never respond with plain text when a tool is available."},
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "Use the get_weather tool to check the weather in London. You must call the tool, do not respond with plain text."},
],
tools=tools,
tool_choice="auto",
stream=True,
)
_tool_calls_made = False
for chunk in response:
if chunk.choices and len(chunk.choices) > 0:
delta = chunk.choices[0].delta
if delta.content is not None:
print(delta.content, end="", flush=True)
if delta.tool_calls:
_tool_calls_made = True
for _tc in delta.tool_calls:
if _tc.function:
print(_tc.function.arguments or "", end="", flush=True)
if not _tool_calls_made:
raise Exception("VALIDATION FAILED: tool-call stream - no tool calls received")
print("\nVALIDATION: tool-call stream SUCCESS")
ErrorCode snippetfrom openai import OpenAI
import json
client = OpenAI(api_key="***", base_url="https://internal.devtest.truefoundry.tech/api/llm")
response_schema = json.loads('''{
"title": "CalendarEvent",
"type": "object",
"properties": {
"name": { "type": "string" },
"date": { "type": "string" },
"participants": {
"type": "array",
"items": { "type": "string" }
}
},
"required": ["name", "date", "participants"],
"additionalProperties": false
}''')
response = client.chat.completions.create(
model="test-v2-anthropic/claude-opus-5",
messages=[
{"role": "system", "content": "Extract the event information as JSON."},
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "Alice and Bob are going to a science fair on Friday. Extract the event details as JSON."},
],
response_format={"type": "json_schema", "json_schema": {"name": "CalendarEvent", "schema": response_schema}},
stream=True,
)
import json as _json
_accumulated = ""
for chunk in response:
if chunk.choices and len(chunk.choices) > 0:
delta = chunk.choices[0].delta
if delta.content is not None:
_accumulated += delta.content
print(delta.content, end="", flush=True)
if not _accumulated:
raise Exception("VALIDATION FAILED: structured-output stream - no content received")
_parsed = _json.loads(_accumulated)
if "name" not in _parsed or "date" not in _parsed or "participants" not in _parsed:
raise Exception("VALIDATION FAILED: structured-output stream - missing expected fields (name, date, participants)")
if not isinstance(_parsed.get("participants"), list):
raise Exception("VALIDATION FAILED: structured-output stream - 'participants' is not a list, schema not enforced")
if set(_parsed.keys()) != {"name", "date", "participants"}:
raise Exception(
f"VALIDATION FAILED: structured-output stream - unexpected keys present: {set(_parsed.keys())}"
)
print("\nVALIDATION: structured-output stream SUCCESS")
ErrorCode snippetfrom openai import OpenAI
client = OpenAI(api_key="***", base_url="https://internal.devtest.truefoundry.tech/api/llm")
response = client.chat.completions.create(
model="test-v2-anthropic/claude-opus-5",
messages=[
{"role": "system", "content": "You are a helpful assistant. You MUST think step by step and show your reasoning. Never skip reasoning steps."},
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "How to calculate 3^3^3^3? Think step by step and show all reasoning."},
],
reasoning_effort="medium",
stream=True,
)
_reasoning_detected = False
for chunk in response:
if chunk.choices and len(chunk.choices) > 0:
delta = chunk.choices[0].delta
if delta.content is not None:
print(delta.content, end="", flush=True)
if getattr(delta, "reasoning_content", None) is not None:
_reasoning_detected = True
if getattr(delta, "reasoning", None) is not None:
_reasoning_detected = True
_usage = getattr(chunk, "usage", None)
if _usage is not None:
_details = getattr(_usage, "completion_tokens_details", None)
if _details and getattr(_details, "reasoning_tokens", 0) > 0:
_reasoning_detected = True
if not _reasoning_detected:
raise Exception("VALIDATION FAILED: reasoning stream - no reasoning information in stream")
print("\nVALIDATION: reasoning stream SUCCESS")
ErrorCode snippetfrom openai import OpenAI
client = OpenAI(api_key="***", base_url="https://internal.devtest.truefoundry.tech/api/llm")
response = client.chat.completions.create(
model="test-v2-anthropic/claude-opus-5",
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "What is the capital of France?"},
],
max_tokens=256,
stream=False,
)
print(response.choices[0].message.content)
ErrorCode snippetfrom openai import OpenAI
client = OpenAI(api_key="***", base_url="https://internal.devtest.truefoundry.tech/api/llm")
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather for a location.",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city name, e.g. London",
},
},
"required": ["location"],
"additionalProperties": False,
},
"strict": True,
},
},
]
response = client.chat.completions.create(
model="test-v2-anthropic/claude-opus-5",
messages=[
{"role": "system", "content": "You are a helpful assistant with access to tools. You MUST strictly call multiple tools in parallel whenever possible. Never call them sequentially."},
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "Use the get_weather tool to check the weather in London and Paris. You MUST make both tool calls strictly in parallel, not sequentially."},
],
tools=tools,
tool_choice="auto",
parallel_tool_calls=True,
stream=False,
)
_message = response.choices[0].message
if _message.tool_calls:
print(f"Number of parallel tool calls: {len(_message.tool_calls)}")
for _tc in _message.tool_calls:
print(f"Function: {_tc.function.name}")
print(f"Arguments: {_tc.function.arguments}")
else:
print(_message.content)
if not _message.tool_calls or len(_message.tool_calls) < 1:
raise Exception(
f"VALIDATION FAILED: parallel-tool-call - expected at least 1 tool call, "
f"got {len(_message.tool_calls) if _message.tool_calls else 0}"
)
print("VALIDATION: parallel-tool-call SUCCESS")
ErrorCode snippetfrom openai import OpenAI
client = OpenAI(api_key="***", base_url="https://internal.devtest.truefoundry.tech/api/llm")
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather for a location.",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city name, e.g. London",
},
},
"required": ["location"],
"additionalProperties": False,
},
"strict": True,
},
},
]
response = client.chat.completions.create(
model="test-v2-anthropic/claude-opus-5",
messages=[
{"role": "system", "content": "You are a helpful assistant with access to tools. You MUST strictly call multiple tools in parallel whenever possible. Never call them sequentially."},
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "Use the get_weather tool to check the weather in London and Paris. You MUST make both tool calls strictly in parallel, not sequentially."},
],
tools=tools,
tool_choice="auto",
parallel_tool_calls=True,
stream=True,
)
_tool_call_indices = set()
for chunk in response:
if chunk.choices and len(chunk.choices) > 0:
delta = chunk.choices[0].delta
if delta.content is not None:
print(delta.content, end="", flush=True)
if delta.tool_calls:
for _tc in delta.tool_calls:
_tool_call_indices.add(_tc.index)
if _tc.function:
print(_tc.function.arguments or "", end="", flush=True)
if len(_tool_call_indices) < 1:
raise Exception(
f"VALIDATION FAILED: parallel-tool-call stream - expected at least 1 tool call, "
f"got {len(_tool_call_indices)}"
)
print(f"\nNumber of parallel tool calls: {len(_tool_call_indices)}")
print("VALIDATION: parallel-tool-call stream SUCCESS")
ErrorCode snippetfrom openai import OpenAI
client = OpenAI(api_key="***", base_url="https://internal.devtest.truefoundry.tech/api/llm")
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather for a location.",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city name, e.g. London",
},
},
"required": ["location"],
"additionalProperties": False,
},
"strict": True,
},
},
]
response = client.chat.completions.create(
model="test-v2-anthropic/claude-opus-5",
messages=[
{"role": "system", "content": "You are a helpful assistant with access to tools. You MUST strictly use the provided tools to answer. Never respond with plain text when a tool is available."},
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "Use the get_weather tool to check the weather in London. You must call the tool, do not respond with plain text."},
],
tools=tools,
tool_choice="auto",
stream=False,
)
_message = response.choices[0].message
if _message.tool_calls:
for _tc in _message.tool_calls:
print(f"Function: {_tc.function.name}")
print(f"Arguments: {_tc.function.arguments}")
else:
print(_message.content)
if not _message.tool_calls or len(_message.tool_calls) == 0:
raise Exception("VALIDATION FAILED: tool-call - no tool calls in response")
print("VALIDATION: tool-call SUCCESS")
ErrorCode snippetimport anthropic
_api_key = "***"
_model = "test-v2-anthropic/claude-opus-5"
client = anthropic.Anthropic(
api_key=_api_key,
base_url="https://internal.devtest.truefoundry.tech/api/llm",
)
tools = [
{
"name": "get_weather",
"description": "Get the current weather for a location.",
"input_schema": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city name, e.g. London",
},
},
"required": ["location"],
},
},
]
messages = [
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "Use the get_weather tool to check the weather in London. You must call the tool, do not respond with plain text."},
]
response = client.messages.create(
model=_model,
system="You are a helpful assistant with access to tools. You MUST strictly use the provided tools to answer. Never respond with plain text when a tool is available.",
messages=messages,
tools=tools,
tool_choice={"type": "auto"},
max_tokens=1024,
)
for block in response.content:
if block.type == "tool_use":
print(f"Tool: {block.name}")
print(f"Input: {block.input}")
elif block.type == "text":
print(block.text)
_tool_uses = [block for block in response.content if block.type == "tool_use"]
if not _tool_uses:
raise Exception("VALIDATION FAILED: tool-call - no tool uses in Anthropic response")
print("VALIDATION: tool-call SUCCESS")
ErrorCode snippetfrom openai import OpenAI
client = OpenAI(api_key="***", base_url="https://internal.devtest.truefoundry.tech/api/llm")
response = client.chat.completions.create(
model="test-v2-anthropic/claude-opus-5",
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "What is the capital of France?"},
],
max_tokens=256,
stream=True,
)
for chunk in response:
if chunk.choices and len(chunk.choices) > 0:
delta = chunk.choices[0].delta
if delta.content is not None:
print(delta.content, end="", flush=True)
ErrorCode snippetimport anthropic
_api_key = "***"
_model = "test-v2-anthropic/claude-opus-5"
client = anthropic.Anthropic(
api_key=_api_key,
base_url="https://internal.devtest.truefoundry.tech/api/llm",
)
tools = [
{
"name": "get_weather",
"description": "Get the current weather for a location.",
"input_schema": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city name, e.g. London",
},
},
"required": ["location"],
},
},
]
messages = [
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "Use the get_weather tool to check the weather in London. You must call the tool, do not respond with plain text."},
]
with client.messages.stream(
model=_model,
system="You are a helpful assistant with access to tools. You MUST strictly use the provided tools to answer. Never respond with plain text when a tool is available.",
messages=messages,
tools=tools,
tool_choice={"type": "auto"},
max_tokens=1024,
) as stream:
response = stream.get_final_message()
for block in response.content:
if block.type == "tool_use":
print(f"Tool: {block.name}")
print(f"Input: {block.input}")
elif block.type == "text":
print(block.text)
_tool_uses = [block for block in response.content if block.type == "tool_use"]
if not _tool_uses:
raise Exception("VALIDATION FAILED: tool-call stream - no tool uses in Anthropic stream response")
print("\nVALIDATION: tool-call stream SUCCESS")
ErrorCode snippetimport anthropic
import json
_api_key = "***"
_model = "test-v2-anthropic/claude-opus-5"
client = anthropic.Anthropic(
api_key=_api_key,
base_url="https://internal.devtest.truefoundry.tech/api/llm",
)
response_schema = {
"type": "object",
"properties": {
"name": {"type": "string"},
"date": {"type": "string"},
"participants": {
"type": "array",
"items": {"type": "string"},
},
},
"required": ["name", "date", "participants"],
}
tools = [
{
"name": "CalendarEvent",
"description": "Extract event information as a structured CalendarEvent.",
"input_schema": response_schema,
},
]
messages = [
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "Alice and Bob are going to a science fair on Friday."},
]
with client.messages.stream(
model=_model,
system="Extract the event information using the CalendarEvent tool.",
messages=messages,
tools=tools,
tool_choice={"type": "tool", "name": "CalendarEvent"},
max_tokens=1024,
) as stream:
response = stream.get_final_message()
for block in response.content:
if block.type == "tool_use":
print(json.dumps(block.input, indent=2))
elif block.type == "text":
print(block.text)
import json as _json
_tool_uses = [block for block in response.content if block.type == "tool_use"]
if _tool_uses:
_parsed = _tool_uses[0].input
else:
_text_blocks = [block.text for block in response.content if block.type == "text"]
_text = "".join(_text_blocks)
if not _text:
raise Exception("VALIDATION FAILED: structured-output stream - no content received from Anthropic stream")
_parsed = _json.loads(_text)
if "name" not in _parsed or "date" not in _parsed or "participants" not in _parsed:
raise Exception("VALIDATION FAILED: structured-output stream - missing expected fields (name, date, participants)")
if not isinstance(_parsed.get("participants"), list):
raise Exception("VALIDATION FAILED: structured-output stream - 'participants' is not a list, schema not enforced")
print("\nVALIDATION: structured-output stream SUCCESS")
ErrorCode snippetimport anthropic
_api_key = "***"
_model = "test-v2-anthropic/claude-opus-5"
client = anthropic.Anthropic(
api_key=_api_key,
base_url="https://internal.devtest.truefoundry.tech/api/llm",
)
tools = [
{
"name": "get_weather",
"description": "Get the current weather for a location.",
"input_schema": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city name, e.g. London",
},
},
"required": ["location"],
},
},
]
messages = [
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "Use the get_weather tool to check the weather in London and Paris. You MUST make both tool calls strictly in parallel, not sequentially."},
]
response = client.messages.create(
model=_model,
system="You are a helpful assistant with access to tools. You MUST strictly call multiple tools in parallel whenever possible. Never call them sequentially.",
messages=messages,
tools=tools,
tool_choice={"type": "auto"},
max_tokens=1024,
)
for block in response.content:
if block.type == "tool_use":
print(f"Tool: {block.name}")
print(f"Input: {block.input}")
elif block.type == "text":
print(block.text)
_tool_uses = [block for block in response.content if block.type == "tool_use"]
if _tool_uses:
print(f"Number of parallel tool calls: {len(_tool_uses)}")
if not _tool_uses or len(_tool_uses) < 1:
raise Exception(
f"VALIDATION FAILED: parallel-tool-call - expected at least 1 tool call, "
f"got {len(_tool_uses) if _tool_uses else 0}"
)
print("VALIDATION: parallel-tool-call SUCCESS")
ErrorCode snippetimport anthropic
_api_key = "***"
_model = "test-v2-anthropic/claude-opus-5"
client = anthropic.Anthropic(
api_key=_api_key,
base_url="https://internal.devtest.truefoundry.tech/api/llm",
)
messages = [
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "How to calculate 3^3^3^3? Think step by step and show all reasoning."},
]
with client.messages.stream(
model=_model,
system="You are a helpful assistant. You MUST think step by step and show your reasoning. Never skip reasoning steps.",
messages=messages,
max_tokens=16000,
thinking={"type":"adaptive"},
) as stream:
response = stream.get_final_message()
for block in response.content:
if block.type == "thinking":
print(f"Thinking: {block.thinking[:200]}...")
elif block.type == "text":
print(block.text)
_reasoning_detected = False
for block in response.content:
if block.type == "thinking":
_reasoning_detected = True
if not _reasoning_detected:
print("Response content types: ", [block.type for block in response.content])
raise Exception("VALIDATION FAILED: reasoning stream - no thinking blocks in Anthropic stream response")
print("\nVALIDATION: reasoning stream SUCCESS")
ErrorCode snippetfrom openai import OpenAI
import json
client = OpenAI(api_key="***", base_url="https://internal.devtest.truefoundry.tech/api/llm")
response_schema = json.loads('''{
"title": "CalendarEvent",
"type": "object",
"properties": {
"name": { "type": "string" },
"date": { "type": "string" },
"participants": {
"type": "array",
"items": { "type": "string" }
}
},
"required": ["name", "date", "participants"],
"additionalProperties": false
}''')
response = client.chat.completions.create(
model="test-v2-anthropic/claude-opus-5",
messages=[
{"role": "system", "content": "Extract the event information as JSON."},
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "Alice and Bob are going to a science fair on Friday. Extract the event details as JSON."},
],
response_format={"type": "json_schema", "json_schema": {"name": "CalendarEvent", "schema": response_schema}},
stream=False,
)
import json as _json
_content = response.choices[0].message.content
print(_content)
if not _content:
raise Exception("VALIDATION FAILED: structured-output - response content is empty")
_parsed = _json.loads(_content)
if "name" not in _parsed or "date" not in _parsed or "participants" not in _parsed:
raise Exception("VALIDATION FAILED: structured-output - missing expected fields (name, date, participants)")
if not isinstance(_parsed.get("participants"), list):
raise Exception("VALIDATION FAILED: structured-output - 'participants' is not a list, schema not enforced")
if set(_parsed.keys()) != {"name", "date", "participants"}:
raise Exception(
f"VALIDATION FAILED: structured-output - unexpected keys present: {set(_parsed.keys())}"
)
print("VALIDATION: structured-output SUCCESS")
ErrorCode snippetimport anthropic
_api_key = "***"
_model = "test-v2-anthropic/claude-opus-5"
client = anthropic.Anthropic(
api_key=_api_key,
base_url="https://internal.devtest.truefoundry.tech/api/llm",
)
messages = [
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "What is the capital of France?"},
]
with client.messages.stream(
model=_model,
system="You are a helpful assistant.",
messages=messages,
max_tokens=256,
) as stream:
for text in stream.text_stream:
print(text, end="", flush=True)
ErrorCode snippetimport anthropic
_api_key = "***"
_model = "test-v2-anthropic/claude-opus-5"
client = anthropic.Anthropic(
api_key=_api_key,
base_url="https://internal.devtest.truefoundry.tech/api/llm",
)
tools = [
{
"name": "get_weather",
"description": "Get the current weather for a location.",
"input_schema": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city name, e.g. London",
},
},
"required": ["location"],
},
},
]
messages = [
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "Use the get_weather tool to check the weather in London and Paris. You MUST make both tool calls strictly in parallel, not sequentially."},
]
with client.messages.stream(
model=_model,
system="You are a helpful assistant with access to tools. You MUST strictly call multiple tools in parallel whenever possible. Never call them sequentially.",
messages=messages,
tools=tools,
tool_choice={"type": "auto"},
max_tokens=1024,
) as stream:
response = stream.get_final_message()
for block in response.content:
if block.type == "tool_use":
print(f"Tool: {block.name}")
print(f"Input: {block.input}")
elif block.type == "text":
print(block.text)
_tool_uses = [block for block in response.content if block.type == "tool_use"]
if _tool_uses:
print(f"Number of parallel tool calls: {len(_tool_uses)}")
if not _tool_uses or len(_tool_uses) < 1:
raise Exception(
f"VALIDATION FAILED: parallel-tool-call stream - expected at least 1 tool call, "
f"got {len(_tool_uses) if _tool_uses else 0}"
)
print(f"\nVALIDATION: parallel-tool-call stream SUCCESS")
ErrorCode snippetfrom openai import OpenAI
client = OpenAI(api_key="***", base_url="https://internal.devtest.truefoundry.tech/api/llm")
response = client.chat.completions.create(
model="test-v2-anthropic/claude-opus-5",
messages=[
{"role": "system", "content": "You are a helpful assistant. You MUST think step by step and show your reasoning. Never skip reasoning steps."},
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "How to calculate 3^3^3^3? Think step by step and show all reasoning."},
],
reasoning_effort="medium",
stream=False,
)
_usage = getattr(response, "usage", None)
_reasoning_detected = False
_choices = getattr(response, "choices", None)
if _choices and len(_choices) > 0:
_message = getattr(_choices[0], "message", None)
else:
_message = None
if _message and getattr(_message, "content", None) is not None:
print(_message.content)
if _usage is not None:
_output_token_details = getattr(_usage, "completion_tokens_details", None)
if _output_token_details and getattr(_output_token_details, "reasoning_tokens", 0) > 0:
_reasoning_detected = True
elif getattr(_usage, "reasoning", None) is not None:
_reasoning_detected = True
if getattr(_message, "reasoning_content", None) is not None:
_reasoning_detected = True
elif getattr(_message, "reasoning", None) is not None:
_reasoning_detected = True
if not _reasoning_detected:
print("Response: ", response)
raise Exception("VALIDATION FAILED: reasoning - no reasoning information in response")
print("VALIDATION: reasoning SUCCESS")
ErrorCode snippetimport anthropic
import json
_api_key = "***"
_model = "test-v2-anthropic/claude-opus-5"
client = anthropic.Anthropic(
api_key=_api_key,
base_url="https://internal.devtest.truefoundry.tech/api/llm",
)
response_schema = {
"type": "object",
"properties": {
"name": {"type": "string"},
"date": {"type": "string"},
"participants": {
"type": "array",
"items": {"type": "string"},
},
},
"required": ["name", "date", "participants"],
}
tools = [
{
"name": "CalendarEvent",
"description": "Extract event information as a structured CalendarEvent.",
"input_schema": response_schema,
},
]
messages = [
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "Alice and Bob are going to a science fair on Friday."},
]
response = client.messages.create(
model=_model,
system="Extract the event information using the CalendarEvent tool.",
messages=messages,
tools=tools,
tool_choice={"type": "tool", "name": "CalendarEvent"},
max_tokens=1024,
)
for block in response.content:
if block.type == "tool_use":
print(json.dumps(block.input, indent=2))
elif block.type == "text":
print(block.text)
import json as _json
_tool_uses = [block for block in response.content if block.type == "tool_use"]
if _tool_uses:
_parsed = _tool_uses[0].input
else:
_text_blocks = [block.text for block in response.content if block.type == "text"]
_text = "".join(_text_blocks)
if not _text:
raise Exception("VALIDATION FAILED: structured-output - no content received from Anthropic response")
_parsed = _json.loads(_text)
if "name" not in _parsed or "date" not in _parsed or "participants" not in _parsed:
raise Exception("VALIDATION FAILED: structured-output - missing expected fields (name, date, participants)")
if not isinstance(_parsed.get("participants"), list):
raise Exception("VALIDATION FAILED: structured-output - 'participants' is not a list, schema not enforced")
print("VALIDATION: structured-output SUCCESS")
ErrorCode snippetimport anthropic
_api_key = "***"
_model = "test-v2-anthropic/claude-opus-5"
client = anthropic.Anthropic(
api_key=_api_key,
base_url="https://internal.devtest.truefoundry.tech/api/llm",
)
messages = [
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "What is the capital of France?"},
]
response = client.messages.create(
model=_model,
system="You are a helpful assistant.",
messages=messages,
max_tokens=256,
)
for block in response.content:
if block.type == "text":
print(block.text)
ErrorCode snippetimport anthropic
_api_key = "***"
_model = "test-v2-anthropic/claude-opus-5"
client = anthropic.Anthropic(
api_key=_api_key,
base_url="https://internal.devtest.truefoundry.tech/api/llm",
)
messages = [
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hi, how can I help you"},
{"role": "user", "content": "How to calculate 3^3^3^3? Think step by step and show all reasoning."},
]
response = client.messages.create(
model=_model,
system="You are a helpful assistant. You MUST think step by step and show your reasoning. Never skip reasoning steps.",
messages=messages,
max_tokens=16000,
thinking={"type":"adaptive"},
)
for block in response.content:
if block.type == "thinking":
print(f"Thinking: {block.thinking[:200]}...")
elif block.type == "text":
print(block.text)
_reasoning_detected = False
for block in response.content:
if block.type == "thinking":
_reasoning_detected = True
if hasattr(response, "usage") and response.usage:
if getattr(response.usage, "cache_creation_input_tokens", 0) or getattr(response.usage, "cache_read_input_tokens", 0):
pass
if not _reasoning_detected:
print("Response content types: ", [block.type for block in response.content])
raise Exception("VALIDATION FAILED: reasoning - no thinking blocks in Anthropic response")
print("VALIDATION: reasoning SUCCESS") |
Auto-generated by poc-agent for provider
anthropic.Note
Low Risk
Metadata-only YAML updates with no runtime or routing logic changes.
Overview
Adds
retirementDatemetadata to two Anthropic model definitions so consumers can plan around end-of-life.claude-opus-4-8.yamlis set to retire on 2027-05-28;claude-opus-5.yamlon 2027-07-24. No pricing, limits, features, or status fields are changed.Reviewed by Cursor Bugbot for commit ace3343. Bugbot is set up for automated code reviews on this repo. Configure here.