[Frontend] Fix Kimi K2 tool call IDs for required tool choice (#46344)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -24,6 +24,7 @@ tools = [
|
||||
"description": "Get the current weather in a given location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"strict": True,
|
||||
"properties": {
|
||||
"city": {
|
||||
"type": "string",
|
||||
@@ -215,80 +216,6 @@ async def test_function_tool_use(
|
||||
assert len(reasoning) > 0
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def k2_server():
|
||||
args = [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
"half",
|
||||
"--enable-auto-tool-choice",
|
||||
"--structured-outputs-config.backend",
|
||||
"xgrammar",
|
||||
"--tool-call-parser",
|
||||
"hermes",
|
||||
"--reasoning-parser",
|
||||
"qwen3",
|
||||
"--gpu-memory-utilization",
|
||||
"0.4",
|
||||
] + ROCM_EXTRA_ARGS
|
||||
# Test kimi_k2 tool use tool_id format by overriding model_type.
|
||||
# is_deepseek_mla safely returns False via getattr when kv_lora_rank
|
||||
# is absent from the underlying config.
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
env_dict=ROCM_ENV_OVERRIDES,
|
||||
override_hf_configs={"model_type": "kimi_k2"},
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def k2_client(k2_server):
|
||||
async with k2_server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip(reason="Skipping Kimi K2 tool ID test")
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize("stream", [True, False])
|
||||
@pytest.mark.parametrize("tool_choice", ["required"])
|
||||
async def test_tool_id_kimi_k2(
|
||||
k2_client: openai.AsyncOpenAI, model_name: str, stream: bool, tool_choice: str
|
||||
):
|
||||
if not stream:
|
||||
# Non-streaming test
|
||||
chat_completion = await k2_client.chat.completions.create(
|
||||
messages=messages, model=model_name, tools=tools, tool_choice=tool_choice
|
||||
)
|
||||
assert chat_completion.choices[0].message.tool_calls is not None
|
||||
assert len(chat_completion.choices[0].message.tool_calls) > 0
|
||||
assert chat_completion.choices[0].message.tool_calls[0].id in [
|
||||
"functions.get_current_weather:0",
|
||||
"functions.get_forecast:1",
|
||||
]
|
||||
else:
|
||||
# Streaming test
|
||||
output_stream = await k2_client.chat.completions.create(
|
||||
messages=messages,
|
||||
model=model_name,
|
||||
tools=tools,
|
||||
tool_choice=tool_choice,
|
||||
stream=True,
|
||||
)
|
||||
|
||||
output = []
|
||||
async for chunk in output_stream:
|
||||
if chunk.choices and chunk.choices[0].delta.tool_calls:
|
||||
output.extend(chunk.choices[0].delta.tool_calls)
|
||||
for o in output:
|
||||
assert o.id is None or o.id in [
|
||||
"functions.get_current_weather:0",
|
||||
"functions.get_forecast:1",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize("arguments", ["{}", ""])
|
||||
|
||||
@@ -183,11 +183,19 @@ def _make_request_output(
|
||||
|
||||
|
||||
def _make_context(parser_cls, **overrides):
|
||||
# ParsableContext no longer lazily builds a parser from ``parser_cls``;
|
||||
# the caller (here, the serving layer in production) must supply one.
|
||||
request = overrides.get("request", _make_request())
|
||||
response_parser = overrides.pop("response_parser", None)
|
||||
if response_parser is None and parser_cls is not None:
|
||||
response_parser = parser_cls(MagicMock(), request.tools)
|
||||
|
||||
defaults = dict(
|
||||
tokenizer=MagicMock(),
|
||||
parser_cls=parser_cls,
|
||||
response_parser=response_parser,
|
||||
response_messages=[],
|
||||
request=_make_request(),
|
||||
request=request,
|
||||
available_tools=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
|
||||
+149
-2
@@ -3,6 +3,7 @@
|
||||
|
||||
import json
|
||||
import os
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -13,7 +14,9 @@ os.environ[_STRICT_TOOL_CALLING_ENV] = "0"
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ( # noqa: E402
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest # noqa: E402
|
||||
from vllm.parser.abstract_parser import DelegatingParser # noqa: E402
|
||||
from vllm.parser.utils import count_history_tool_calls # noqa: E402
|
||||
from vllm.reasoning.basic_parsers import ( # noqa: E402
|
||||
BaseThinkingReasoningParser,
|
||||
)
|
||||
@@ -82,12 +85,55 @@ TOOLS = [
|
||||
]
|
||||
|
||||
|
||||
def make_parser(tokenizer, reasoning=False, tool=False):
|
||||
KIMI_K2_MODEL_CONFIG = SimpleNamespace(
|
||||
hf_text_config=SimpleNamespace(model_type="kimi_k2"),
|
||||
hf_overrides=None,
|
||||
)
|
||||
|
||||
HISTORY_MESSAGES = [
|
||||
{"role": "user", "content": "first"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "functions.get_current_weather:0",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_current_weather",
|
||||
"arguments": "{}",
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "functions.get_forecast:1",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_forecast",
|
||||
"arguments": "{}",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "functions.get_current_weather:0",
|
||||
"content": "{}",
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "functions.get_forecast:1",
|
||||
"content": "{}",
|
||||
},
|
||||
{"role": "user", "content": "again"},
|
||||
]
|
||||
|
||||
|
||||
def make_parser(tokenizer, reasoning=False, tool=False, **kwargs):
|
||||
class TestParser(DelegatingParser):
|
||||
reasoning_parser_cls = ThinkReasoningParser if reasoning else None
|
||||
tool_parser_cls = Hermes2ProToolParser if tool else None
|
||||
|
||||
return TestParser(tokenizer)
|
||||
return TestParser(tokenizer, **kwargs)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -232,6 +278,107 @@ def test_parse_required_tool_choice(tokenizer):
|
||||
assert json.loads(tool_calls[1].arguments) == {"timezone": "UTC"}
|
||||
|
||||
|
||||
def test_parse_required_tool_choice_kimi_k2_ids(tokenizer):
|
||||
parser = make_parser(
|
||||
tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG
|
||||
)
|
||||
functions_json = json.dumps(
|
||||
[
|
||||
{"name": "get_current_weather", "parameters": {"city": "Dallas"}},
|
||||
{"name": "get_forecast", "parameters": {"city": "Dallas", "days": 2}},
|
||||
]
|
||||
)
|
||||
request = make_request(tools=TOOLS, tool_choice="required")
|
||||
_, content, tool_calls = parser.parse(
|
||||
functions_json, request, enable_auto_tools=True
|
||||
)
|
||||
|
||||
assert content is None
|
||||
assert tool_calls is not None
|
||||
assert [tc.id for tc in tool_calls] == [
|
||||
"functions.get_current_weather:0",
|
||||
"functions.get_forecast:1",
|
||||
]
|
||||
|
||||
|
||||
def test_parse_required_tool_choice_kimi_k2_ids_after_history(tokenizer):
|
||||
parser = make_parser(
|
||||
tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG
|
||||
)
|
||||
functions_json = json.dumps(
|
||||
[{"name": "get_current_weather", "parameters": {"city": "Dallas"}}]
|
||||
)
|
||||
request = make_request(
|
||||
messages=HISTORY_MESSAGES,
|
||||
tools=TOOLS,
|
||||
tool_choice="required",
|
||||
)
|
||||
_, _, tool_calls = parser.parse(functions_json, request, enable_auto_tools=True)
|
||||
|
||||
assert tool_calls is not None
|
||||
assert tool_calls[0].id == "functions.get_current_weather:2"
|
||||
|
||||
|
||||
def test_count_history_tool_calls_responses_request():
|
||||
request = ResponsesRequest.model_validate(
|
||||
{
|
||||
"model": "test-model",
|
||||
"input": [
|
||||
{
|
||||
"type": "function_call",
|
||||
"call_id": "call_0",
|
||||
"name": "get_current_weather",
|
||||
"arguments": "{}",
|
||||
},
|
||||
{
|
||||
"type": "function_call",
|
||||
"call_id": "call_1",
|
||||
"name": "get_forecast",
|
||||
"arguments": "{}",
|
||||
},
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
assert count_history_tool_calls(request) == 2
|
||||
|
||||
|
||||
def test_parse_required_tool_choice_random_ids_deferred(tokenizer):
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=True)
|
||||
functions_json = json.dumps(
|
||||
[{"name": "get_current_weather", "parameters": {"city": "Dallas"}}]
|
||||
)
|
||||
request = make_request(
|
||||
messages=HISTORY_MESSAGES,
|
||||
tools=TOOLS,
|
||||
tool_choice="required",
|
||||
)
|
||||
_, _, tool_calls = parser.parse(functions_json, request, enable_auto_tools=True)
|
||||
|
||||
assert tool_calls is not None
|
||||
assert tool_calls[0].id is None
|
||||
|
||||
|
||||
def test_parse_named_tool_choice_kimi_k2_id(tokenizer):
|
||||
parser = make_parser(
|
||||
tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG
|
||||
)
|
||||
request = make_request(
|
||||
tools=TOOLS,
|
||||
tool_choice={
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather"},
|
||||
},
|
||||
)
|
||||
_, content, tool_calls = parser.parse(
|
||||
TOOL_ARGUMENTS, request, enable_auto_tools=True
|
||||
)
|
||||
|
||||
assert content is None
|
||||
assert tool_calls is not None
|
||||
assert tool_calls[0].id == "functions.get_weather:0"
|
||||
|
||||
|
||||
def test_parse_named_tool_choice_content_none(tokenizer):
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=True)
|
||||
request = make_request(
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -47,6 +48,36 @@ TOOLS = [
|
||||
]
|
||||
|
||||
|
||||
KIMI_K2_MODEL_CONFIG = SimpleNamespace(
|
||||
hf_text_config=SimpleNamespace(model_type="kimi_k2"),
|
||||
hf_overrides=None,
|
||||
)
|
||||
|
||||
HISTORY_MESSAGES = [
|
||||
{"role": "user", "content": "first"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "functions.get_current_weather:0",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_current_weather",
|
||||
"arguments": "{}",
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "functions.get_current_weather:0",
|
||||
"content": "{}",
|
||||
},
|
||||
{"role": "user", "content": "again"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def request_obj():
|
||||
return ChatCompletionRequest(
|
||||
@@ -57,12 +88,12 @@ def request_obj():
|
||||
)
|
||||
|
||||
|
||||
def make_parser(tokenizer, reasoning=False, tool=False):
|
||||
def make_parser(tokenizer, reasoning=False, tool=False, **kwargs):
|
||||
class TestParser(DelegatingParser):
|
||||
reasoning_parser_cls = ThinkReasoningParser if reasoning else None
|
||||
tool_parser_cls = Hermes2ProToolParser if tool else None
|
||||
|
||||
return TestParser(tokenizer)
|
||||
return TestParser(tokenizer, **kwargs)
|
||||
|
||||
|
||||
def stream_text(parser, tokenizer, text, request, prompt_token_ids=None):
|
||||
@@ -365,3 +396,76 @@ def test_parse_delta_tool_choice_none_with_reasoning(tokenizer, request_obj):
|
||||
assert len(tool_calls) == 0
|
||||
assert "<tool_call>" in content
|
||||
assert "get_weather" in content
|
||||
|
||||
|
||||
def test_parse_delta_required_tool_choice_kimi_k2_ids(tokenizer, request_obj):
|
||||
parser = make_parser(
|
||||
tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG
|
||||
)
|
||||
request = request_obj.model_copy(update={"tool_choice": "required"})
|
||||
output = json.dumps(
|
||||
[
|
||||
{
|
||||
"name": "get_current_weather",
|
||||
"parameters": {"city": "Dallas"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
results: list[DeltaMessage | None] = []
|
||||
prompt_token_ids: list[int] | None = []
|
||||
for i in range(0, len(output), 3):
|
||||
chunk = output[i : i + 3]
|
||||
results.append(
|
||||
parser.parse_delta(
|
||||
chunk,
|
||||
[],
|
||||
request,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
finished=False,
|
||||
)
|
||||
)
|
||||
prompt_token_ids = None
|
||||
|
||||
_, content, tool_calls = collect_fields(results)
|
||||
assert content == ""
|
||||
assert any(tc.id == "functions.get_current_weather:0" for tc in tool_calls)
|
||||
assert all(tc.id in (None, "functions.get_current_weather:0") for tc in tool_calls)
|
||||
|
||||
|
||||
def test_parse_delta_required_tool_choice_kimi_k2_ids_after_history(
|
||||
tokenizer, request_obj
|
||||
):
|
||||
parser = make_parser(
|
||||
tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG
|
||||
)
|
||||
request = request_obj.model_copy(
|
||||
update={"messages": HISTORY_MESSAGES, "tool_choice": "required"}
|
||||
)
|
||||
output = json.dumps(
|
||||
[
|
||||
{
|
||||
"name": "get_current_weather",
|
||||
"parameters": {"city": "Dallas"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
results: list[DeltaMessage | None] = []
|
||||
prompt_token_ids: list[int] | None = []
|
||||
for i in range(0, len(output), 3):
|
||||
chunk = output[i : i + 3]
|
||||
results.append(
|
||||
parser.parse_delta(
|
||||
chunk,
|
||||
[],
|
||||
request,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
finished=False,
|
||||
)
|
||||
)
|
||||
prompt_token_ids = None
|
||||
|
||||
_, _, tool_calls = collect_fields(results)
|
||||
assert any(tc.id == "functions.get_current_weather:1" for tc in tool_calls)
|
||||
assert all(tc.id in (None, "functions.get_current_weather:1") for tc in tool_calls)
|
||||
|
||||
Reference in New Issue
Block a user