[Frontend] Fix Kimi K2 tool call IDs for required tool choice (#46344)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -17,8 +17,6 @@ from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
ChatTemplateContentFormatOption,
|
||||
ConversationMessage,
|
||||
get_history_tool_calls_cnt,
|
||||
get_tool_call_id_type,
|
||||
make_tool_call_id,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
@@ -170,8 +168,6 @@ class OpenAIServingChat(OpenAIServing):
|
||||
if mc.generation_config not in ("auto", "vllm")
|
||||
else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
|
||||
)
|
||||
self.tool_call_id_type = get_tool_call_id_type(self.model_config)
|
||||
|
||||
# NOTE(woosuk): While OpenAI's chat completion API supports browsing
|
||||
# for some models, currently vLLM doesn't support it. Please use the
|
||||
# Responses API instead.
|
||||
@@ -261,6 +257,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
tokenizer,
|
||||
request.tools,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
model_config=self.model_config,
|
||||
)
|
||||
result = await self.render_chat_request(request)
|
||||
if isinstance(result, ErrorResponse):
|
||||
@@ -433,11 +430,6 @@ class OpenAIServingChat(OpenAIServing):
|
||||
else:
|
||||
tool_choice_function_name = None
|
||||
|
||||
if self.tool_call_id_type == "kimi_k2":
|
||||
history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
|
||||
else:
|
||||
history_tool_call_cnt = 0
|
||||
|
||||
previous_texts = [""] * num_choices
|
||||
|
||||
try:
|
||||
@@ -451,14 +443,10 @@ class OpenAIServingChat(OpenAIServing):
|
||||
tokenizer,
|
||||
request.tools,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
model_config=self.model_config,
|
||||
)
|
||||
for _ in range(num_choices)
|
||||
]
|
||||
for p in parsers:
|
||||
if p is not None:
|
||||
# NOTE: HarmonyParser ignores _stream_state (uses its own FSM).
|
||||
p._stream_state.tool_call_id_type = self.tool_call_id_type
|
||||
p._stream_state.history_tool_call_cnt = history_tool_call_cnt
|
||||
else:
|
||||
parsers = [None] * num_choices
|
||||
except Exception as e:
|
||||
@@ -842,10 +830,6 @@ class OpenAIServingChat(OpenAIServing):
|
||||
)
|
||||
|
||||
choices: list[ChatCompletionResponseChoice] = []
|
||||
if self.tool_call_id_type == "kimi_k2":
|
||||
history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
|
||||
else:
|
||||
history_tool_call_cnt = 0
|
||||
|
||||
role = self.get_chat_request_role(request)
|
||||
tool_parser_cls = (
|
||||
@@ -885,54 +869,26 @@ class OpenAIServingChat(OpenAIServing):
|
||||
tool_calls = []
|
||||
|
||||
auto_tools_called = False
|
||||
is_named_tool_choice = (
|
||||
request.tool_choice is not None
|
||||
and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
|
||||
)
|
||||
is_required_tool_choice = request.tool_choice == "required"
|
||||
|
||||
if (not self.enable_auto_tools or not tool_parser_cls) and (
|
||||
not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
|
||||
and request.tool_choice != "required"
|
||||
not is_named_tool_choice and not is_required_tool_choice
|
||||
):
|
||||
message = ChatMessage(role=role, reasoning=reasoning, content=content)
|
||||
|
||||
elif (
|
||||
request.tool_choice
|
||||
and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
|
||||
):
|
||||
tool_call_items = []
|
||||
tool_calls = tool_calls or []
|
||||
for tc in tool_calls:
|
||||
if not tc.id:
|
||||
tc.id = make_tool_call_id(
|
||||
id_type=self.tool_call_id_type,
|
||||
func_name=tc.name,
|
||||
idx=history_tool_call_cnt,
|
||||
)
|
||||
tool_call_items.append(ToolCall(id=tc.id, function=tc))
|
||||
history_tool_call_cnt += 1
|
||||
elif is_named_tool_choice or is_required_tool_choice:
|
||||
message = ChatMessage(
|
||||
role=role,
|
||||
reasoning=reasoning,
|
||||
content=content or "",
|
||||
tool_calls=tool_call_items,
|
||||
)
|
||||
|
||||
elif request.tool_choice and request.tool_choice == "required":
|
||||
tool_call_items = []
|
||||
tool_calls = tool_calls or []
|
||||
for tool_call in tool_calls:
|
||||
if not tool_call.id:
|
||||
tool_call.id = make_tool_call_id(
|
||||
id_type=self.tool_call_id_type,
|
||||
func_name=tool_call.name,
|
||||
idx=history_tool_call_cnt,
|
||||
)
|
||||
tool_call_items.append(
|
||||
ToolCall(id=tool_call.id, function=tool_call)
|
||||
)
|
||||
history_tool_call_cnt += 1
|
||||
message = ChatMessage(
|
||||
role=role,
|
||||
content=content or "",
|
||||
tool_calls=tool_call_items,
|
||||
reasoning=reasoning,
|
||||
tool_calls=[
|
||||
ToolCall(id=tc.id or make_tool_call_id(), function=tc)
|
||||
for tc in (tool_calls or [])
|
||||
],
|
||||
)
|
||||
|
||||
# if the request doesn't use tool choice
|
||||
@@ -949,21 +905,14 @@ class OpenAIServingChat(OpenAIServing):
|
||||
):
|
||||
auto_tools_called = tool_calls is not None and len(tool_calls) > 0
|
||||
if tool_calls:
|
||||
tool_call_items = []
|
||||
for tc in tool_calls:
|
||||
if not tc.id:
|
||||
tc.id = make_tool_call_id(
|
||||
id_type=self.tool_call_id_type,
|
||||
func_name=tc.name,
|
||||
idx=history_tool_call_cnt,
|
||||
)
|
||||
tool_call_items.append(ToolCall(id=tc.id, function=tc))
|
||||
history_tool_call_cnt += 1
|
||||
message = ChatMessage(
|
||||
role=role,
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
tool_calls=tool_call_items,
|
||||
tool_calls=[
|
||||
ToolCall(id=tc.id or make_tool_call_id(), function=tc)
|
||||
for tc in tool_calls
|
||||
],
|
||||
)
|
||||
|
||||
else:
|
||||
|
||||
@@ -301,7 +301,6 @@ class ParsableContext(ConversationContext):
|
||||
chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
response_parser: Parser | None = None,
|
||||
enable_auto_tools: bool = False,
|
||||
tool_call_id_type: str = "random",
|
||||
):
|
||||
self.num_prompt_tokens = 0
|
||||
self.num_output_tokens = 0
|
||||
@@ -314,20 +313,8 @@ class ParsableContext(ConversationContext):
|
||||
self.num_init_messages = len(response_messages)
|
||||
self.finish_reason: str | None = None
|
||||
self.enable_auto_tools = enable_auto_tools
|
||||
self.tool_call_id_type = tool_call_id_type
|
||||
|
||||
self.response_parser = response_parser
|
||||
if self.response_parser is None and parser_cls is not None:
|
||||
chat_template_kwargs = request.build_chat_params(
|
||||
default_template=chat_template,
|
||||
default_template_content_format=chat_template_content_format,
|
||||
).chat_template_kwargs
|
||||
self.response_parser = parser_cls(
|
||||
tokenizer,
|
||||
tools=request.tools,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
|
||||
self.parser_cls = parser_cls
|
||||
self.request = request
|
||||
|
||||
@@ -365,7 +352,6 @@ class ParsableContext(ConversationContext):
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
tool_calls=tool_calls,
|
||||
tool_call_id_type=self.tool_call_id_type,
|
||||
)
|
||||
)
|
||||
elif completion.text:
|
||||
|
||||
@@ -30,7 +30,6 @@ from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateContentFormatOption,
|
||||
get_tool_call_id_type,
|
||||
)
|
||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
@@ -222,9 +221,6 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
"For gpt-oss, we ignore --enable-auto-tool-choice "
|
||||
"and always enable tool use."
|
||||
)
|
||||
|
||||
self.tool_call_id_type = get_tool_call_id_type(self.model_config)
|
||||
|
||||
self.enable_auto_tools = enable_auto_tools
|
||||
# HACK(woosuk): This is a hack. We should use a better store.
|
||||
# FIXME: If enable_store=True, this may cause a memory leak since we
|
||||
@@ -272,6 +268,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
tokenizer,
|
||||
request.tools,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
model_config=self.model_config,
|
||||
)
|
||||
|
||||
def _validate_generator_input(
|
||||
@@ -493,7 +490,6 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
chat_template=self.chat_template,
|
||||
chat_template_content_format=self.chat_template_content_format,
|
||||
enable_auto_tools=self.enable_auto_tools,
|
||||
tool_call_id_type=self.tool_call_id_type,
|
||||
)
|
||||
else:
|
||||
context = SimpleContext(
|
||||
@@ -1073,7 +1069,6 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
content=content,
|
||||
tool_calls=tool_calls,
|
||||
logprobs=logprobs,
|
||||
tool_call_id_type=self.tool_call_id_type,
|
||||
)
|
||||
|
||||
# Fallback when no parser is configured
|
||||
|
||||
@@ -45,7 +45,6 @@ def build_response_output_items(
|
||||
content: str | None,
|
||||
tool_calls: list[FunctionCall] | None,
|
||||
logprobs: list[Logprob] | None = None,
|
||||
tool_call_id_type: str = "random",
|
||||
) -> list[ResponseOutputItem]:
|
||||
outputs: list[ResponseOutputItem] = []
|
||||
|
||||
@@ -86,12 +85,7 @@ def build_response_output_items(
|
||||
ResponseFunctionToolCall(
|
||||
id=f"fc_{random_uuid()}",
|
||||
call_id=tool_call.id
|
||||
if tool_call.id
|
||||
else make_tool_call_id(
|
||||
id_type=tool_call_id_type,
|
||||
func_name=tool_call.name,
|
||||
idx=idx,
|
||||
),
|
||||
or make_tool_call_id(func_name=tool_call.name, idx=idx),
|
||||
type="function_call",
|
||||
status="completed",
|
||||
name=tool_call.name,
|
||||
|
||||
@@ -11,6 +11,10 @@ from functools import cached_property
|
||||
from openai.types.responses import ToolChoiceFunction
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
get_tool_call_id_type,
|
||||
make_tool_call_id,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
ChatCompletionRequest,
|
||||
@@ -24,6 +28,7 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser.metrics import record_tool_parser_invocation
|
||||
from vllm.parser.utils import count_history_tool_calls
|
||||
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
@@ -46,6 +51,7 @@ class StreamState:
|
||||
previous_text: str = ""
|
||||
previous_token_ids: list[int] = field(default_factory=list)
|
||||
history_tool_call_cnt: int = 0
|
||||
history_tool_call_cnt_initialized: bool = False
|
||||
tool_call_id_type: str = "random"
|
||||
# only used for "required" and "named tool" choices,
|
||||
# tracks whether function name has been fully returned in the stream yet
|
||||
@@ -108,6 +114,7 @@ class Parser:
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
*args,
|
||||
model_config=None,
|
||||
**kwargs,
|
||||
):
|
||||
self.model_tokenizer = tokenizer
|
||||
@@ -124,7 +131,14 @@ class Parser:
|
||||
self._reasoning_parser is None
|
||||
or self._reasoning_parser.engine_based_streaming
|
||||
) and (self._tool_parser is None or self._tool_parser.engine_based_streaming)
|
||||
self._stream_state = StreamState(engine_based=self._engine_based)
|
||||
self._stream_state = StreamState(
|
||||
tool_call_id_type=(
|
||||
get_tool_call_id_type(model_config)
|
||||
if model_config is not None
|
||||
else "random"
|
||||
),
|
||||
engine_based=self._engine_based,
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def vocab(self) -> dict[str, int]:
|
||||
@@ -149,6 +163,19 @@ class Parser:
|
||||
def tool_parser(self, parser: ToolParser | None) -> None:
|
||||
self._tool_parser = parser
|
||||
|
||||
def _initialize_history_tool_call_cnt(
|
||||
self,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> None:
|
||||
state = self._stream_state
|
||||
if state.history_tool_call_cnt_initialized:
|
||||
return
|
||||
if state.tool_call_id_type != "kimi_k2":
|
||||
state.history_tool_call_cnt_initialized = True
|
||||
return
|
||||
state.history_tool_call_cnt = count_history_tool_calls(request)
|
||||
state.history_tool_call_cnt_initialized = True
|
||||
|
||||
# ========== Reasoning Parser Methods ==========
|
||||
|
||||
@abstractmethod
|
||||
@@ -375,6 +402,18 @@ class DelegatingParser(Parser):
|
||||
return request.tool_choice.function.name
|
||||
raise ValueError("Invalid tool_choice for function name extraction.")
|
||||
|
||||
def _make_tool_call_id(self, function_name: str) -> str | None:
|
||||
state = self._stream_state
|
||||
if state.tool_call_id_type != "kimi_k2":
|
||||
return None
|
||||
tool_call_id = make_tool_call_id(
|
||||
id_type=state.tool_call_id_type,
|
||||
func_name=function_name,
|
||||
idx=state.history_tool_call_cnt,
|
||||
)
|
||||
state.history_tool_call_cnt += 1
|
||||
return tool_call_id
|
||||
|
||||
def _extract_tool_calls(
|
||||
self,
|
||||
content: str | None,
|
||||
@@ -404,9 +443,11 @@ class DelegatingParser(Parser):
|
||||
if is_named_tool_choice and supports_required_and_named:
|
||||
if content is None:
|
||||
return [], None
|
||||
function_name = self._get_function_name(request)
|
||||
tool_calls.append(
|
||||
FunctionCall(
|
||||
name=self._get_function_name(request),
|
||||
id=self._make_tool_call_id(function_name),
|
||||
name=function_name,
|
||||
arguments=content,
|
||||
)
|
||||
)
|
||||
@@ -422,6 +463,7 @@ class DelegatingParser(Parser):
|
||||
for tc in parsed_calls:
|
||||
tool_calls.append(
|
||||
FunctionCall(
|
||||
id=self._make_tool_call_id(tc.name),
|
||||
name=tc.name,
|
||||
arguments=json.dumps(tc.parameters, ensure_ascii=False),
|
||||
)
|
||||
@@ -733,6 +775,7 @@ class DelegatingParser(Parser):
|
||||
enable_auto_tools: bool = False,
|
||||
model_output_token_ids: Sequence[int] = (),
|
||||
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
|
||||
self._initialize_history_tool_call_cnt(request)
|
||||
reasoning, content = self.extract_reasoning(model_output, request)
|
||||
tool_calls, content = self._extract_tool_calls(
|
||||
content=content,
|
||||
@@ -750,6 +793,7 @@ class DelegatingParser(Parser):
|
||||
*,
|
||||
finished: bool,
|
||||
) -> DeltaMessage | None:
|
||||
self._initialize_history_tool_call_cnt(request)
|
||||
state = self._stream_state
|
||||
|
||||
if not state.prompt_reasoning_checked and prompt_token_ids is not None:
|
||||
|
||||
@@ -13,7 +13,7 @@ from typing import TYPE_CHECKING
|
||||
|
||||
import regex as re
|
||||
|
||||
from vllm.entrypoints.chat_utils import make_tool_call_id
|
||||
from vllm.entrypoints.chat_utils import get_tool_call_id_type, make_tool_call_id
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
@@ -89,11 +89,18 @@ class ParserEngine(Parser):
|
||||
tools: list[Tool] | None = None,
|
||||
*,
|
||||
parser_engine_config: ParserEngineConfig,
|
||||
model_config=None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
self.model_tokenizer = tokenizer
|
||||
self._tools = tools
|
||||
self._stream_state = StreamState()
|
||||
self._stream_state = StreamState(
|
||||
tool_call_id_type=(
|
||||
get_tool_call_id_type(model_config)
|
||||
if model_config is not None
|
||||
else "random"
|
||||
),
|
||||
)
|
||||
self._reasoning_parser = None
|
||||
self._tool_parser = None
|
||||
self.parser_engine_config = parser_engine_config
|
||||
@@ -419,6 +426,7 @@ class ParserEngine(Parser):
|
||||
*,
|
||||
finished: bool,
|
||||
) -> DeltaMessage | None:
|
||||
self._initialize_history_tool_call_cnt(request)
|
||||
if not self._prompt_streaming_prepared and prompt_token_ids is not None:
|
||||
# NOTE: call the hook BEFORE setting the flag, because the hook
|
||||
# may invoke ``_reset`` (e.g. via ``initialize_streaming``) which
|
||||
@@ -658,6 +666,7 @@ class ParserEngine(Parser):
|
||||
enable_auto_tools: bool = False,
|
||||
model_output_token_ids: Sequence[int] = (),
|
||||
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
|
||||
self._initialize_history_tool_call_cnt(request)
|
||||
self._check_skip_tool_parsing(request)
|
||||
reasoning, content, tool_call_info = self._single_pass_parse(
|
||||
model_output,
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Iterable, Sequence
|
||||
|
||||
from openai.types.responses import ResponseFunctionToolCall
|
||||
|
||||
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import (
|
||||
ResponseInputOutputItem,
|
||||
ResponsesRequest,
|
||||
)
|
||||
|
||||
|
||||
def count_tool_calls(tool_calls: object) -> int:
|
||||
if tool_calls is None:
|
||||
return 0
|
||||
if isinstance(tool_calls, (str, bytes, dict)):
|
||||
return 1
|
||||
if isinstance(tool_calls, Iterable):
|
||||
return sum(1 for _ in tool_calls)
|
||||
return 1
|
||||
|
||||
|
||||
def count_chat_history_tool_calls(
|
||||
messages: Sequence[ChatCompletionMessageParam],
|
||||
) -> int:
|
||||
return sum(
|
||||
count_tool_calls(msg.get("tool_calls"))
|
||||
for msg in messages
|
||||
if isinstance(msg, dict) and msg.get("role") == "assistant"
|
||||
)
|
||||
|
||||
|
||||
def count_response_history_tool_calls(
|
||||
response_items: Sequence[ResponseInputOutputItem],
|
||||
) -> int:
|
||||
count = 0
|
||||
for item in response_items:
|
||||
if isinstance(item, ResponseFunctionToolCall):
|
||||
count += 1
|
||||
continue
|
||||
|
||||
if isinstance(item, dict):
|
||||
item_type = item.get("type")
|
||||
if item_type == "function_call":
|
||||
count += 1
|
||||
elif item.get("role") == "assistant":
|
||||
count += count_tool_calls(item.get("tool_calls"))
|
||||
|
||||
return count
|
||||
|
||||
|
||||
def count_history_tool_calls(
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> int:
|
||||
if isinstance(request, ChatCompletionRequest):
|
||||
return count_chat_history_tool_calls(request.messages)
|
||||
|
||||
request_input = request.input
|
||||
if isinstance(request_input, str):
|
||||
return 0
|
||||
|
||||
return count_response_history_tool_calls(request_input)
|
||||
Reference in New Issue
Block a user