[Frontend] Fix Kimi K2 tool call IDs for required tool choice (#46344)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
Chauncey
2026-06-24 19:59:40 +00:00
committed by GitHub
parent 49f2104c53
commit 84c2f9f0fb
11 changed files with 406 additions and 178 deletions
@@ -24,6 +24,7 @@ tools = [
"description": "Get the current weather in a given location",
"parameters": {
"type": "object",
"strict": True,
"properties": {
"city": {
"type": "string",
@@ -215,80 +216,6 @@ async def test_function_tool_use(
assert len(reasoning) > 0
@pytest.fixture(scope="module")
def k2_server():
args = [
# use half precision for speed and memory savings in CI environment
"--dtype",
"half",
"--enable-auto-tool-choice",
"--structured-outputs-config.backend",
"xgrammar",
"--tool-call-parser",
"hermes",
"--reasoning-parser",
"qwen3",
"--gpu-memory-utilization",
"0.4",
] + ROCM_EXTRA_ARGS
# Test kimi_k2 tool use tool_id format by overriding model_type.
# is_deepseek_mla safely returns False via getattr when kv_lora_rank
# is absent from the underlying config.
with RemoteOpenAIServer(
MODEL_NAME,
args,
env_dict=ROCM_ENV_OVERRIDES,
override_hf_configs={"model_type": "kimi_k2"},
) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def k2_client(k2_server):
async with k2_server.get_async_client() as async_client:
yield async_client
@pytest.mark.asyncio
@pytest.mark.skip(reason="Skipping Kimi K2 tool ID test")
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("stream", [True, False])
@pytest.mark.parametrize("tool_choice", ["required"])
async def test_tool_id_kimi_k2(
k2_client: openai.AsyncOpenAI, model_name: str, stream: bool, tool_choice: str
):
if not stream:
# Non-streaming test
chat_completion = await k2_client.chat.completions.create(
messages=messages, model=model_name, tools=tools, tool_choice=tool_choice
)
assert chat_completion.choices[0].message.tool_calls is not None
assert len(chat_completion.choices[0].message.tool_calls) > 0
assert chat_completion.choices[0].message.tool_calls[0].id in [
"functions.get_current_weather:0",
"functions.get_forecast:1",
]
else:
# Streaming test
output_stream = await k2_client.chat.completions.create(
messages=messages,
model=model_name,
tools=tools,
tool_choice=tool_choice,
stream=True,
)
output = []
async for chunk in output_stream:
if chunk.choices and chunk.choices[0].delta.tool_calls:
output.extend(chunk.choices[0].delta.tool_calls)
for o in output:
assert o.id is None or o.id in [
"functions.get_current_weather:0",
"functions.get_forecast:1",
]
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("arguments", ["{}", ""])
@@ -183,11 +183,19 @@ def _make_request_output(
def _make_context(parser_cls, **overrides):
# ParsableContext no longer lazily builds a parser from ``parser_cls``;
# the caller (here, the serving layer in production) must supply one.
request = overrides.get("request", _make_request())
response_parser = overrides.pop("response_parser", None)
if response_parser is None and parser_cls is not None:
response_parser = parser_cls(MagicMock(), request.tools)
defaults = dict(
tokenizer=MagicMock(),
parser_cls=parser_cls,
response_parser=response_parser,
response_messages=[],
request=_make_request(),
request=request,
available_tools=None,
chat_template=None,
chat_template_content_format="auto",