[Frontend] Fix Kimi K2 tool call IDs for required tool choice (#46344)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
Chauncey
2026-06-24 19:59:40 +00:00
committed by GitHub
parent 49f2104c53
commit 84c2f9f0fb
11 changed files with 406 additions and 178 deletions
+46 -2
View File
@@ -11,6 +11,10 @@ from functools import cached_property
from openai.types.responses import ToolChoiceFunction
from pydantic import TypeAdapter, ValidationError
from vllm.entrypoints.chat_utils import (
get_tool_call_id_type,
make_tool_call_id,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionNamedToolChoiceParam,
ChatCompletionRequest,
@@ -24,6 +28,7 @@ from vllm.entrypoints.openai.engine.protocol import (
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.logger import init_logger
from vllm.parser.metrics import record_tool_parser_invocation
from vllm.parser.utils import count_history_tool_calls
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
from vllm.sampling_params import StructuredOutputsParams
from vllm.tokenizers import TokenizerLike
@@ -46,6 +51,7 @@ class StreamState:
previous_text: str = ""
previous_token_ids: list[int] = field(default_factory=list)
history_tool_call_cnt: int = 0
history_tool_call_cnt_initialized: bool = False
tool_call_id_type: str = "random"
# only used for "required" and "named tool" choices,
# tracks whether function name has been fully returned in the stream yet
@@ -108,6 +114,7 @@ class Parser:
tokenizer: TokenizerLike,
tools: list[Tool] | None = None,
*args,
model_config=None,
**kwargs,
):
self.model_tokenizer = tokenizer
@@ -124,7 +131,14 @@ class Parser:
self._reasoning_parser is None
or self._reasoning_parser.engine_based_streaming
) and (self._tool_parser is None or self._tool_parser.engine_based_streaming)
self._stream_state = StreamState(engine_based=self._engine_based)
self._stream_state = StreamState(
tool_call_id_type=(
get_tool_call_id_type(model_config)
if model_config is not None
else "random"
),
engine_based=self._engine_based,
)
@cached_property
def vocab(self) -> dict[str, int]:
@@ -149,6 +163,19 @@ class Parser:
def tool_parser(self, parser: ToolParser | None) -> None:
self._tool_parser = parser
def _initialize_history_tool_call_cnt(
self,
request: ChatCompletionRequest | ResponsesRequest,
) -> None:
state = self._stream_state
if state.history_tool_call_cnt_initialized:
return
if state.tool_call_id_type != "kimi_k2":
state.history_tool_call_cnt_initialized = True
return
state.history_tool_call_cnt = count_history_tool_calls(request)
state.history_tool_call_cnt_initialized = True
# ========== Reasoning Parser Methods ==========
@abstractmethod
@@ -375,6 +402,18 @@ class DelegatingParser(Parser):
return request.tool_choice.function.name
raise ValueError("Invalid tool_choice for function name extraction.")
def _make_tool_call_id(self, function_name: str) -> str | None:
state = self._stream_state
if state.tool_call_id_type != "kimi_k2":
return None
tool_call_id = make_tool_call_id(
id_type=state.tool_call_id_type,
func_name=function_name,
idx=state.history_tool_call_cnt,
)
state.history_tool_call_cnt += 1
return tool_call_id
def _extract_tool_calls(
self,
content: str | None,
@@ -404,9 +443,11 @@ class DelegatingParser(Parser):
if is_named_tool_choice and supports_required_and_named:
if content is None:
return [], None
function_name = self._get_function_name(request)
tool_calls.append(
FunctionCall(
name=self._get_function_name(request),
id=self._make_tool_call_id(function_name),
name=function_name,
arguments=content,
)
)
@@ -422,6 +463,7 @@ class DelegatingParser(Parser):
for tc in parsed_calls:
tool_calls.append(
FunctionCall(
id=self._make_tool_call_id(tc.name),
name=tc.name,
arguments=json.dumps(tc.parameters, ensure_ascii=False),
)
@@ -733,6 +775,7 @@ class DelegatingParser(Parser):
enable_auto_tools: bool = False,
model_output_token_ids: Sequence[int] = (),
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
self._initialize_history_tool_call_cnt(request)
reasoning, content = self.extract_reasoning(model_output, request)
tool_calls, content = self._extract_tool_calls(
content=content,
@@ -750,6 +793,7 @@ class DelegatingParser(Parser):
*,
finished: bool,
) -> DeltaMessage | None:
self._initialize_history_tool_call_cnt(request)
state = self._stream_state
if not state.prompt_reasoning_checked and prompt_token_ids is not None:
+11 -2
View File
@@ -13,7 +13,7 @@ from typing import TYPE_CHECKING
import regex as re
from vllm.entrypoints.chat_utils import make_tool_call_id
from vllm.entrypoints.chat_utils import get_tool_call_id_type, make_tool_call_id
from vllm.entrypoints.openai.engine.protocol import (
DeltaFunctionCall,
DeltaMessage,
@@ -89,11 +89,18 @@ class ParserEngine(Parser):
tools: list[Tool] | None = None,
*,
parser_engine_config: ParserEngineConfig,
model_config=None,
**kwargs,
) -> None:
self.model_tokenizer = tokenizer
self._tools = tools
self._stream_state = StreamState()
self._stream_state = StreamState(
tool_call_id_type=(
get_tool_call_id_type(model_config)
if model_config is not None
else "random"
),
)
self._reasoning_parser = None
self._tool_parser = None
self.parser_engine_config = parser_engine_config
@@ -419,6 +426,7 @@ class ParserEngine(Parser):
*,
finished: bool,
) -> DeltaMessage | None:
self._initialize_history_tool_call_cnt(request)
if not self._prompt_streaming_prepared and prompt_token_ids is not None:
# NOTE: call the hook BEFORE setting the flag, because the hook
# may invoke ``_reset`` (e.g. via ``initialize_streaming``) which
@@ -658,6 +666,7 @@ class ParserEngine(Parser):
enable_auto_tools: bool = False,
model_output_token_ids: Sequence[int] = (),
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
self._initialize_history_tool_call_cnt(request)
self._check_skip_tool_parsing(request)
reasoning, content, tool_call_info = self._single_pass_parse(
model_output,
+65
View File
@@ -0,0 +1,65 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Sequence
from openai.types.responses import ResponseFunctionToolCall
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.responses.protocol import (
ResponseInputOutputItem,
ResponsesRequest,
)
def count_tool_calls(tool_calls: object) -> int:
if tool_calls is None:
return 0
if isinstance(tool_calls, (str, bytes, dict)):
return 1
if isinstance(tool_calls, Iterable):
return sum(1 for _ in tool_calls)
return 1
def count_chat_history_tool_calls(
messages: Sequence[ChatCompletionMessageParam],
) -> int:
return sum(
count_tool_calls(msg.get("tool_calls"))
for msg in messages
if isinstance(msg, dict) and msg.get("role") == "assistant"
)
def count_response_history_tool_calls(
response_items: Sequence[ResponseInputOutputItem],
) -> int:
count = 0
for item in response_items:
if isinstance(item, ResponseFunctionToolCall):
count += 1
continue
if isinstance(item, dict):
item_type = item.get("type")
if item_type == "function_call":
count += 1
elif item.get("role") == "assistant":
count += count_tool_calls(item.get("tool_calls"))
return count
def count_history_tool_calls(
request: ChatCompletionRequest | ResponsesRequest,
) -> int:
if isinstance(request, ChatCompletionRequest):
return count_chat_history_tool_calls(request.messages)
request_input = request.input
if isinstance(request_input, str):
return 0
return count_response_history_tool_calls(request_input)