[CI] Reorganize entrypoints CI (#44947)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-06-09 00:46:11 -07:00
committed by GitHub
parent e6fc848d4f
commit 996222f4bf
20 changed files with 166 additions and 165 deletions
@@ -1,42 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from tests.utils import VLLM_PATH, RemoteOpenAIServer
chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
assert chatml_jinja_path.exists()
def run_and_test_dummy_opt_api_server(model, tp=1):
# the model is registered through the plugin
server_args = [
"--gpu-memory-utilization",
"0.10",
"--dtype",
"float32",
"--chat-template",
str(chatml_jinja_path),
"--load-format",
"dummy",
"-tp",
f"{tp}",
]
with RemoteOpenAIServer(model, server_args) as server:
client = server.get_client()
completion = client.chat.completions.create(
model=model,
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Hello!"},
],
temperature=0,
)
generated_text = completion.choices[0].message.content
assert generated_text is not None
# make sure only the first token is generated
rest = generated_text.replace("<s>", "")
assert rest == ""
def test_oot_registration_for_api_server(dummy_opt_path: str):
run_and_test_dummy_opt_api_server(dummy_opt_path)
@@ -1,220 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import openai
import pytest
from ....utils import RemoteOpenAIServer
MODEL = "ibm-granite/granite-4.0-h-tiny"
@pytest.fixture(scope="module")
def server():
model = MODEL
args_for_model = [
"--enforce-eager",
"--enable-auto-tool-choice",
"--tool-call-parser",
"granite4",
"--tokenizer",
"ibm-granite/granite-4.0-h-tiny",
"--max-model-len",
"4096",
"--max-num-seqs",
"2",
]
with RemoteOpenAIServer(model, args_for_model, max_wait_seconds=480) as server:
yield server
tools = [
{
"type": "function",
"function": {
"name": "get_acme_region_name_for_transaction_id",
"description": "Returns ACME transaction/transaction ID information"
" including ACME regions\n\nArgs:\n start_time "
"(str): Start date and time in datetime format "
'"%Y-%m-%dT%H:%M:%S.%f"\n end_time (str): End '
"date and time in datetime format "
'"%Y-%m-%dT%H:%M:%S.%f"\n size (int, optional): '
"Number of ACME Transaction IDs to return\n "
"order (str, optional): Sort by most run "
"transaction IDs. The value can be 'asc' for "
"ascending or 'desc' for descending\n "
"transaction_id (str, optional): ACME Transaction "
"ID to filter on\n acme_region (str, optional): "
"ACME Region to filter on\nReturns:\n - A "
"dictionary containing a list of ACME transaction "
"ids and the ACME regions they run in:\n {\n"
' "Number of transaction IDs" : int,\n'
' "Total transaction IDs available": int'
',\n "ACME Transaction IDs": [\n '
' {\n "Transaction ID": '
'str,\n "Number of runs": int,\n'
' "ACME Regions": [str],\n '
" },\n ...\n ],"
'\n "Start time" : datetime,\n '
' "End time" : datetime,\n '
' "Order" : str\n }\n '
" - If no ACME region found for transaction id, "
'returns:\n {"Success": "No ACME region '
'found for transaction id."}\n - If an error '
'occurs, returns:\n {"Error": "{exception'
' message}"}',
"parameters": {
"properties": {
"start_time": {},
"end_time": {},
"size": {"default": 500},
"order": {"default": "desc"},
"transaction_id": {"default": None},
"acme_region": {"default": None},
},
"required": ["start_time", "end_time"],
"type": "object",
},
},
}
]
tools2 = [
{
"type": "function",
"function": {
"name": "get_current_weather",
"description": "Get the current weather",
"parameters": {
"type": "object",
"properties": {
"location": {
"description": "The city and state, e.g. San Francisco, CA",
"type": "string",
}
},
"required": ["location"],
},
},
},
{
"type": "function",
"function": {
"name": "get_stock_price",
"description": "Retrieves the current stock price for a given "
"ticker symbol. The ticker symbol must be a valid "
"symbol for a publicly traded company on a major US"
" stock exchange like NYSE or NASDAQ. The tool will"
" return the latest trade price in USD. It should "
"be used when the user asks about the current or "
"most recent price of a specific stock. It will not"
" provide any other information about the stock or"
" company.",
"parameters": {
"type": "object",
"properties": {
"ticker": {
"description": "The stock ticker symbol, e.g."
" AAPL for Apple Inc.",
"type": "string",
}
},
},
},
},
]
messages = [
{
"content": "\n\nSystem: You are a helpful, precise, and methodical AI"
" assistant that uses tool outputs provided inline.\nAlways"
" assume the current datetime is 2026-01-29T13:59:09.238901"
"+00:00.\n\nIf you receive a ToolMessage with `tool_call_id"
'` equal to "get_time_range" (or "time_range_tool"), you '
"MUST:\n 1. Parse that JSON and use the values `start` and"
" `end` directly when calling other tools.\n 2. Do not "
"re-call or re-compute the time range.\n 3. Pass resolved "
"values (ISO strings) as arguments to any subsequent tool "
"(do not pass function metadata or placeholders).\n 4. If "
"a tool requires datetime objects rather than strings, "
"convert the ISO strings into language-native datetime "
"objects before invoking.\n\nAlways return fully resolved "
"arguments in correct types (e.g., ISO datetime strings or"
" datetime objects) and never include placeholders like "
'"<start>".\n\n',
"role": "system",
},
{
"content": "What are the transaction IDs that ran in the"
" ACME region A9345 over the last two months?",
"role": "user",
},
{
"content": '["2026-01-26T09: 51: 55.467722Z", "2026-01-27T09: 51: 55.467722Z"]',
"role": "tool",
"tool_call_id": "time_range_tool",
},
]
messages2 = [{"role": "user", "content": "What's stock price for IBM?"}]
messages3 = [{"role": "user", "content": "What's the current weather in New York?"}]
def get_args(client: openai.OpenAI, _tools, _messages, _stop):
response = client.chat.completions.create(
model=MODEL,
messages=_messages,
temperature=0,
tools=_tools,
max_tokens=200,
stop=_stop,
tool_choice="auto",
)
return response.choices[0].message.tool_calls[0].function.arguments
async def get_args_streaming(
async_client: openai.AsyncOpenAI, _tools, _messages, _stop
):
stream = await async_client.chat.completions.create(
model=MODEL,
messages=_messages,
temperature=0,
tools=_tools,
max_tokens=200,
stop=_stop,
tool_choice="auto",
stream=True,
)
full_call = []
async for chunk in stream:
tc = chunk.choices[0].delta.tool_calls
if tc and tc[0].function.arguments:
full_call.append(tc[0].function.arguments)
return "".join(full_call)
async def run_scenario(server: RemoteOpenAIServer, _tools, _messages, _stop):
non_streaming = get_args(server.get_client(), _tools, _messages, _stop)
json.loads(non_streaming) # verify that it is json loadable
streaming = await get_args_streaming(
server.get_async_client(), _tools, _messages, _stop
)
json.loads(streaming)
assert non_streaming == streaming, f"{non_streaming=}, {streaming=}"
@pytest.mark.asyncio
async def test_stop_sequence_interference(server: RemoteOpenAIServer):
print("Testing scenario 1")
await run_scenario(server, tools, messages, "veroniqueprattyushveroniqueprattyush")
print("Testing scenario 2")
await run_scenario(
server, tools2, messages2, "veroniqueprattyushveroniqueprattyush"
)
print("Testing scenario 3")
await run_scenario(server, tools2, messages3, "prattyush")
@@ -1,325 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import openai
import pytest
import pytest_asyncio
from huggingface_hub import snapshot_download
from typing_extensions import TypedDict
from vllm.tool_parsers.abstract_tool_parser import ToolParser
from vllm.tool_parsers.granite4_tool_parser import Granite4ToolParser
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
from ....utils import RemoteOpenAIServer
LORA_MODEL = "minpeter/LoRA-Llama-3.2-1B-tool-vllm-ci"
TOOLS = [
{
"type": "function",
"function": {
"name": "get_current_weather",
"description": "Get the current weather in a given location",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city and state, e.g. San Francisco, CA",
},
"unit": {
"type": "string",
"enum": ["celsius", "fahrenheit"],
},
},
"required": ["location"],
},
},
}
]
class ServerConfig(TypedDict, total=False):
model: str
arguments: list[str]
model_arg: str
tool_parser: ToolParser
CONFIGS: dict[str, ServerConfig] = {
"llama": {
"model": "meta-llama/Llama-3.2-1B-Instruct",
"arguments": [
"--enforce-eager",
"--enable-auto-tool-choice",
"--tool-call-parser",
"hermes",
"--enable-lora",
"--lora-modules",
f"{LORA_MODEL}={LORA_MODEL}",
"--tokenizer",
f"{LORA_MODEL}",
],
"model_arg": LORA_MODEL,
"tool_parser": Hermes2ProToolParser,
},
"granite4": {
"model": "ibm-granite/granite-4.0-h-tiny",
"arguments": [
"--enforce-eager",
"--enable-auto-tool-choice",
"--tool-call-parser",
"granite4",
"--tokenizer",
"ibm-granite/granite-4.0-h-tiny",
"--max-model-len",
"4096",
"--max-num-seqs",
"2",
],
"model_arg": "ibm-granite/granite-4.0-h-tiny",
"tool_parser": Granite4ToolParser,
},
}
# for each server config, download the model and return the config
@pytest.fixture(scope="session", params=CONFIGS.keys())
def server_config(request):
config = CONFIGS[request.param]
# download model and tokenizer using transformers
snapshot_download(config["model"])
yield CONFIGS[request.param]
@pytest.fixture(scope="module")
def server(request, server_config: ServerConfig):
model = server_config["model"]
args_for_model = server_config["arguments"]
with RemoteOpenAIServer(model, args_for_model, max_wait_seconds=480) as server:
yield server
@pytest_asyncio.fixture
async def client(server: RemoteOpenAIServer):
async with server.get_async_client() as async_client:
yield async_client
PRODUCT_TOOLS = [
{
"type": "function",
"function": {
"name": "get_product_info",
"description": "Get detailed information of a product based on its "
"product ID.",
"parameters": {
"type": "object",
"properties": {
"inserted": {
"type": "boolean",
"description": "inserted.",
},
"product_id": {
"type": "integer",
"description": "The product ID of the product.",
},
},
"required": ["product_id", "inserted"],
},
},
}
]
MESSAGES = [{"role": "user", "content": "What's the weather like in Boston?"}]
PRODUCT_MESSAGES = [
{
"role": "user",
"content": "Hi! Do you have any detailed information about the product id "
"7355608 and inserted true?",
}
]
@pytest.mark.asyncio
async def test_non_streaming_tool_call(
client: openai.AsyncOpenAI, server_config: ServerConfig
):
"""Test tool call in non-streaming mode."""
response = await client.chat.completions.create(
model=server_config["model_arg"],
messages=MESSAGES,
tools=TOOLS,
tool_choice="auto",
temperature=0.0,
)
assert response.choices
choice = response.choices[0]
message = choice.message
assert choice.finish_reason == "tool_calls"
assert message.tool_calls is not None
tool_call = message.tool_calls[0]
assert tool_call.type == "function"
assert tool_call.function.name == "get_current_weather"
arguments = json.loads(tool_call.function.arguments)
assert "location" in arguments
assert "Boston" in arguments["location"]
print("\n[Non-Streaming Test Passed]")
print(f"Tool Call: {tool_call.function.name}")
print(f"Arguments: {arguments}")
@pytest.mark.asyncio
async def test_streaming_tool_call(
client: openai.AsyncOpenAI, server_config: ServerConfig
):
"""Test tool call in streaming mode."""
stream = await client.chat.completions.create(
model=server_config["model_arg"],
messages=MESSAGES,
tools=TOOLS,
tool_choice="auto",
temperature=0.0,
stream=True,
)
tool_call_chunks = {}
async for chunk in stream:
if not chunk.choices:
continue
delta = chunk.choices[0].delta
if not delta or not delta.tool_calls:
continue
for tool_chunk in delta.tool_calls:
index = tool_chunk.index
if index not in tool_call_chunks:
tool_call_chunks[index] = {"name": "", "arguments": ""}
if tool_chunk.function.name:
tool_call_chunks[index]["name"] += tool_chunk.function.name
if tool_chunk.function.arguments:
tool_call_chunks[index]["arguments"] += tool_chunk.function.arguments
assert len(tool_call_chunks) == 1
reconstructed_tool_call = tool_call_chunks[0]
assert reconstructed_tool_call["name"] == "get_current_weather"
arguments = json.loads(reconstructed_tool_call["arguments"])
assert "location" in arguments
assert "Boston" in arguments["location"]
print("\n[Streaming Test Passed]")
print(f"Reconstructed Tool Call: {reconstructed_tool_call['name']}")
print(f"Reconstructed Arguments: {arguments}")
@pytest.mark.asyncio
async def test_non_streaming_product_tool_call(
client: openai.AsyncOpenAI, server_config: ServerConfig
):
"""Test tool call integer and boolean parameters in non-streaming mode."""
response = await client.chat.completions.create(
model=server_config["model_arg"],
messages=PRODUCT_MESSAGES,
tools=PRODUCT_TOOLS,
tool_choice="auto",
temperature=0.66,
)
assert response.choices
choice = response.choices[0]
message = choice.message
assert choice.finish_reason == "tool_calls"
assert message.tool_calls is not None
tool_call = message.tool_calls[0]
assert tool_call.type == "function"
assert tool_call.function.name == "get_product_info"
arguments = json.loads(tool_call.function.arguments)
assert "product_id" in arguments
assert "inserted" in arguments
product_id = arguments.get("product_id")
inserted = arguments.get("inserted")
assert isinstance(product_id, int)
assert product_id == 7355608
assert isinstance(inserted, bool)
assert inserted is True
print("\n[Non-Streaming Product Test Passed]")
print(f"Tool Call: {tool_call.function.name}")
print(f"Arguments: {arguments}")
@pytest.mark.asyncio
async def test_streaming_product_tool_call(
client: openai.AsyncOpenAI, server_config: ServerConfig
):
"""Test tool call integer and boolean parameters in streaming mode."""
stream = await client.chat.completions.create(
model=server_config["model_arg"],
messages=PRODUCT_MESSAGES,
tools=PRODUCT_TOOLS,
tool_choice="auto",
temperature=0.66,
stream=True,
)
tool_call_chunks = {}
async for chunk in stream:
if not chunk.choices:
continue
delta = chunk.choices[0].delta
if not delta or not delta.tool_calls:
continue
for tool_chunk in delta.tool_calls:
index = tool_chunk.index
if index not in tool_call_chunks:
tool_call_chunks[index] = {"name": "", "arguments": ""}
if tool_chunk.function.name:
tool_call_chunks[index]["name"] += tool_chunk.function.name
if tool_chunk.function.arguments:
tool_call_chunks[index]["arguments"] += tool_chunk.function.arguments
assert len(tool_call_chunks) == 1
reconstructed_tool_call = tool_call_chunks[0]
assert reconstructed_tool_call["name"] == "get_product_info"
arguments = json.loads(reconstructed_tool_call["arguments"])
assert "product_id" in arguments
assert "inserted" in arguments
# Handle type coercion for streaming test as well
product_id = arguments.get("product_id")
inserted = arguments.get("inserted")
assert isinstance(product_id, int)
assert product_id == 7355608
assert isinstance(inserted, bool)
assert inserted is True
print("\n[Streaming Product Test Passed]")
print(f"Reconstructed Tool Call: {reconstructed_tool_call['name']}")
print(f"Reconstructed Arguments: {arguments}")
@@ -1,359 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import jsonschema
import openai
import pytest
import pytest_asyncio
from rapidfuzz import fuzz
from ....utils import RemoteOpenAIServer
MODEL_NAME = "openai/gpt-oss-20b"
@pytest.fixture(scope="module")
def server():
args = [
"--max-model-len",
"8192",
"--enforce-eager",
"--enable-auto-tool-choice",
"--tool-call-parser",
"openai",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server):
"""Async fixture providing an OpenAI-compatible vLLM client."""
async with server.get_async_client() as async_client:
yield async_client
# ==========================================================
# Tool Definitions
# ==========================================================
TOOLS = [
{
"type": "function",
"function": {
"name": "calculator",
"description": "Performs basic arithmetic calculations.",
"parameters": {
"type": "object",
"properties": {
"expression": {
"type": "string",
"description": (
"Arithmetic expression to evaluate, e.g. '123 + 456'."
),
}
},
"required": ["expression"],
},
},
},
{
"type": "function",
"function": {
"name": "get_time",
"description": "Retrieves the current local time for a given city.",
"parameters": {
"type": "object",
"properties": {
"city": {
"type": "string",
"description": "City name, e.g. 'New York'.",
}
},
"required": ["city"],
},
},
},
]
# ==========================================================
# Message Examples
# ==========================================================
MESSAGES_CALC = [
{"role": "user", "content": "Calculate 123 + 456 using the calculator."}
]
MESSAGES_GET_TIME = [
{"role": "user", "content": "What is the current time in New York?"}
]
MESSAGES_MULTIPLE_CALLS = [
{
"role": "system",
"content": (
"You can call multiple tools. "
"When using more than one, return single JSON object with tool_calls array"
"containing each tool call with its function name and arguments. "
"Do not output multiple JSON objects separately."
),
},
{
"role": "user",
"content": "First, calculate 7 * 8 using the calculator. "
"Then, use get_time to tell me the current time in New York.",
},
]
MESSAGES_INVALID_CALL = [
{
"role": "user",
"content": "Can you help with something, "
"but don’t actually perform any calculation?",
}
]
# Expected outputs
FUNC_CALC = "calculator"
FUNC_ARGS_CALC = '{"expression":"123 + 456"}'
FUNC_TIME = "get_time"
FUNC_ARGS_TIME = '{"city": "New York"}'
# ==========================================================
# Utility to extract reasoning and tool calls
# ==========================================================
def extract_reasoning_and_calls(chunks: list) -> tuple[str, list[str], list[str]]:
"""
Extract accumulated reasoning text and tool call arguments
from streaming chunks.
"""
reasoning: str = ""
tool_calls: dict[int, dict[str, str]] = {}
for chunk in chunks:
choice = getattr(chunk.choices[0], "delta", None)
if not choice:
continue
if hasattr(choice, "reasoning") and choice.reasoning:
reasoning += choice.reasoning
for tc in getattr(choice, "tool_calls", []) or []:
idx = getattr(tc, "index", 0)
tool_entry = tool_calls.setdefault(idx, {"name": "", "arguments": ""})
if getattr(tc, "function", None):
func = tc.function
if getattr(func, "name", None):
tool_entry["name"] = func.name
if getattr(func, "arguments", None):
tool_entry["arguments"] += func.arguments
function_names: list[str] = [v["name"] for _, v in sorted(tool_calls.items())]
arguments: list[str] = [v["arguments"] for _, v in sorted(tool_calls.items())]
return reasoning, arguments, function_names
# ==========================================================
# Test Scenarios
# ==========================================================
@pytest.mark.asyncio
async def test_calculator_tool_call_and_argument_accuracy(client: openai.AsyncOpenAI):
"""Verify calculator tool call is made and arguments are accurate."""
response = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES_CALC,
tools=TOOLS,
temperature=0.0,
stream=False,
)
message = response.choices[0].message
tool_calls = getattr(message, "tool_calls", [])
assert tool_calls, "No tool calls detected"
calc_call = next((c for c in tool_calls if c.function.name == FUNC_CALC), None)
assert calc_call, "Calculator function not called"
raw_args = calc_call.function.arguments
assert raw_args, "Calculator arguments missing"
assert "123" in raw_args and "456" in raw_args, (
f"Expected values not in raw arguments: {raw_args}"
)
try:
parsed_args = json.loads(raw_args)
except json.JSONDecodeError:
pytest.fail(f"Invalid JSON in calculator arguments: {raw_args}")
expected_expr = "123 + 456"
actual_expr = parsed_args.get("expression", "")
similarity = fuzz.ratio(actual_expr, expected_expr)
assert similarity > 90, (
f"Expression mismatch: expected '{expected_expr}' "
f"got '{actual_expr}' (similarity={similarity}%)"
)
@pytest.mark.asyncio
async def test_streaming_tool_call_get_time_with_reasoning(client: openai.AsyncOpenAI):
"""Verify streamed reasoning and tool call behavior for get_time."""
stream = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES_GET_TIME,
tools=TOOLS,
temperature=0.0,
stream=True,
)
chunks = [chunk async for chunk in stream]
reasoning, arguments, function_names = extract_reasoning_and_calls(chunks)
assert FUNC_TIME in function_names, "get_time function not called"
assert any("New York" in arg for arg in arguments), (
f"Expected get_time arguments for New York not found in {arguments}"
)
assert len(reasoning) > 0, "Expected reasoning content missing"
assert any(keyword in reasoning for keyword in ["New York", "time", "current"]), (
f"Reasoning is not relevant to the request: {reasoning}"
)
@pytest.mark.asyncio
async def test_streaming_multiple_tools(client: openai.AsyncOpenAI):
"""Test streamed multi-tool response with reasoning."""
stream = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES_MULTIPLE_CALLS,
tools=TOOLS,
temperature=0.0,
stream=True,
)
chunks = [chunk async for chunk in stream]
reasoning, arguments, function_names = extract_reasoning_and_calls(chunks)
try:
assert FUNC_CALC in function_names, (
f"Calculator tool missing — found {function_names}"
)
assert FUNC_TIME in function_names, (
f"Time tool missing — found {function_names}"
)
assert len(reasoning) > 0, "Expected reasoning content in streamed response"
except AssertionError as e:
print(f"ERROR: {e}")
@pytest.mark.asyncio
async def test_invalid_tool_call(client: openai.AsyncOpenAI):
"""
Verify that ambiguous instructions that should not trigger a tool
do not produce any tool calls.
"""
response = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES_INVALID_CALL,
tools=TOOLS,
temperature=0.0,
stream=False,
)
message = response.choices[0].message
assert message is not None, "Expected message in response"
assert hasattr(message, "content"), "Expected 'content' field in message"
tool_calls = getattr(message, "tool_calls", [])
assert not tool_calls, (
f"Model unexpectedly attempted a tool call on invalid input: {tool_calls}"
)
@pytest.mark.asyncio
async def test_tool_call_with_temperature(client: openai.AsyncOpenAI):
"""
Verify model produces valid tool or text output
under non-deterministic sampling.
"""
response = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES_CALC,
tools=TOOLS,
temperature=0.7,
stream=False,
)
message = response.choices[0].message
assert message is not None, "Expected non-empty message in response"
assert message.tool_calls or message.content, (
"Response missing both text and tool calls"
)
print(f"\nTool calls: {message.tool_calls}")
print(f"Text: {message.content}")
@pytest.mark.asyncio
async def test_tool_response_schema_accuracy(client: openai.AsyncOpenAI):
"""Validate that tool call arguments adhere to their declared JSON schema."""
response = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES_MULTIPLE_CALLS,
tools=TOOLS,
temperature=0.0,
)
calls = response.choices[0].message.tool_calls
assert calls, "No tool calls produced"
for call in calls:
func_name = call.function.name
args = json.loads(call.function.arguments)
schema: dict[str, object] | None = None
for tool_entry in TOOLS:
function_def = tool_entry.get("function")
if (
function_def
and isinstance(function_def, dict)
and function_def.get("name") == func_name
):
schema = function_def.get("parameters")
break
assert schema is not None, f"No matching tool schema found for {func_name}"
jsonschema.validate(instance=args, schema=schema)
@pytest.mark.asyncio
async def test_semantic_consistency_with_temperature(client: openai.AsyncOpenAI):
"""Test that temperature variation doesn't cause contradictory reasoning."""
responses = []
for temp in [0.0, 0.5, 1.0]:
resp = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES_CALC,
tools=TOOLS,
temperature=temp,
)
text = (resp.choices[0].message.content or "").strip()
responses.append(text)
# Compare fuzzy similarity between low- and mid-temperature outputs
low_mid_sim = fuzz.ratio(responses[0], responses[1])
assert low_mid_sim > 60, (
f"Semantic drift too large between T=0.0 and T=0.5 ({low_mid_sim}%)"
)