diff --git a/tests/entrypoints/openai/chat_completion/test_completion_with_function_calling.py b/tests/entrypoints/openai/chat_completion/test_completion_with_function_calling.py index 62e4965b8e..33cb576f35 100644 --- a/tests/entrypoints/openai/chat_completion/test_completion_with_function_calling.py +++ b/tests/entrypoints/openai/chat_completion/test_completion_with_function_calling.py @@ -24,6 +24,7 @@ tools = [ "description": "Get the current weather in a given location", "parameters": { "type": "object", + "strict": True, "properties": { "city": { "type": "string", @@ -215,80 +216,6 @@ async def test_function_tool_use( assert len(reasoning) > 0 -@pytest.fixture(scope="module") -def k2_server(): - args = [ - # use half precision for speed and memory savings in CI environment - "--dtype", - "half", - "--enable-auto-tool-choice", - "--structured-outputs-config.backend", - "xgrammar", - "--tool-call-parser", - "hermes", - "--reasoning-parser", - "qwen3", - "--gpu-memory-utilization", - "0.4", - ] + ROCM_EXTRA_ARGS - # Test kimi_k2 tool use tool_id format by overriding model_type. - # is_deepseek_mla safely returns False via getattr when kv_lora_rank - # is absent from the underlying config. - with RemoteOpenAIServer( - MODEL_NAME, - args, - env_dict=ROCM_ENV_OVERRIDES, - override_hf_configs={"model_type": "kimi_k2"}, - ) as remote_server: - yield remote_server - - -@pytest_asyncio.fixture -async def k2_client(k2_server): - async with k2_server.get_async_client() as async_client: - yield async_client - - -@pytest.mark.asyncio -@pytest.mark.skip(reason="Skipping Kimi K2 tool ID test") -@pytest.mark.parametrize("model_name", [MODEL_NAME]) -@pytest.mark.parametrize("stream", [True, False]) -@pytest.mark.parametrize("tool_choice", ["required"]) -async def test_tool_id_kimi_k2( - k2_client: openai.AsyncOpenAI, model_name: str, stream: bool, tool_choice: str -): - if not stream: - # Non-streaming test - chat_completion = await k2_client.chat.completions.create( - messages=messages, model=model_name, tools=tools, tool_choice=tool_choice - ) - assert chat_completion.choices[0].message.tool_calls is not None - assert len(chat_completion.choices[0].message.tool_calls) > 0 - assert chat_completion.choices[0].message.tool_calls[0].id in [ - "functions.get_current_weather:0", - "functions.get_forecast:1", - ] - else: - # Streaming test - output_stream = await k2_client.chat.completions.create( - messages=messages, - model=model_name, - tools=tools, - tool_choice=tool_choice, - stream=True, - ) - - output = [] - async for chunk in output_stream: - if chunk.choices and chunk.choices[0].delta.tool_calls: - output.extend(chunk.choices[0].delta.tool_calls) - for o in output: - assert o.id is None or o.id in [ - "functions.get_current_weather:0", - "functions.get_forecast:1", - ] - - @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) @pytest.mark.parametrize("arguments", ["{}", ""]) diff --git a/tests/entrypoints/openai/responses/test_parsable_context_unit.py b/tests/entrypoints/openai/responses/test_parsable_context_unit.py index 0aadfbe99d..2bad3032c4 100644 --- a/tests/entrypoints/openai/responses/test_parsable_context_unit.py +++ b/tests/entrypoints/openai/responses/test_parsable_context_unit.py @@ -183,11 +183,19 @@ def _make_request_output( def _make_context(parser_cls, **overrides): + # ParsableContext no longer lazily builds a parser from ``parser_cls``; + # the caller (here, the serving layer in production) must supply one. + request = overrides.get("request", _make_request()) + response_parser = overrides.pop("response_parser", None) + if response_parser is None and parser_cls is not None: + response_parser = parser_cls(MagicMock(), request.tools) + defaults = dict( tokenizer=MagicMock(), parser_cls=parser_cls, + response_parser=response_parser, response_messages=[], - request=_make_request(), + request=request, available_tools=None, chat_template=None, chat_template_content_format="auto", diff --git a/tests/parser/test_parse.py b/tests/parser/test_parse.py index 39c5c2e3d5..2a34ac7eea 100644 --- a/tests/parser/test_parse.py +++ b/tests/parser/test_parse.py @@ -3,6 +3,7 @@ import json import os +from types import SimpleNamespace import pytest @@ -13,7 +14,9 @@ os.environ[_STRICT_TOOL_CALLING_ENV] = "0" from vllm.entrypoints.openai.chat_completion.protocol import ( # noqa: E402 ChatCompletionRequest, ) +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest # noqa: E402 from vllm.parser.abstract_parser import DelegatingParser # noqa: E402 +from vllm.parser.utils import count_history_tool_calls # noqa: E402 from vllm.reasoning.basic_parsers import ( # noqa: E402 BaseThinkingReasoningParser, ) @@ -82,12 +85,55 @@ TOOLS = [ ] -def make_parser(tokenizer, reasoning=False, tool=False): +KIMI_K2_MODEL_CONFIG = SimpleNamespace( + hf_text_config=SimpleNamespace(model_type="kimi_k2"), + hf_overrides=None, +) + +HISTORY_MESSAGES = [ + {"role": "user", "content": "first"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "functions.get_current_weather:0", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": "{}", + }, + }, + { + "id": "functions.get_forecast:1", + "type": "function", + "function": { + "name": "get_forecast", + "arguments": "{}", + }, + }, + ], + }, + { + "role": "tool", + "tool_call_id": "functions.get_current_weather:0", + "content": "{}", + }, + { + "role": "tool", + "tool_call_id": "functions.get_forecast:1", + "content": "{}", + }, + {"role": "user", "content": "again"}, +] + + +def make_parser(tokenizer, reasoning=False, tool=False, **kwargs): class TestParser(DelegatingParser): reasoning_parser_cls = ThinkReasoningParser if reasoning else None tool_parser_cls = Hermes2ProToolParser if tool else None - return TestParser(tokenizer) + return TestParser(tokenizer, **kwargs) @pytest.mark.parametrize( @@ -232,6 +278,107 @@ def test_parse_required_tool_choice(tokenizer): assert json.loads(tool_calls[1].arguments) == {"timezone": "UTC"} +def test_parse_required_tool_choice_kimi_k2_ids(tokenizer): + parser = make_parser( + tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG + ) + functions_json = json.dumps( + [ + {"name": "get_current_weather", "parameters": {"city": "Dallas"}}, + {"name": "get_forecast", "parameters": {"city": "Dallas", "days": 2}}, + ] + ) + request = make_request(tools=TOOLS, tool_choice="required") + _, content, tool_calls = parser.parse( + functions_json, request, enable_auto_tools=True + ) + + assert content is None + assert tool_calls is not None + assert [tc.id for tc in tool_calls] == [ + "functions.get_current_weather:0", + "functions.get_forecast:1", + ] + + +def test_parse_required_tool_choice_kimi_k2_ids_after_history(tokenizer): + parser = make_parser( + tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG + ) + functions_json = json.dumps( + [{"name": "get_current_weather", "parameters": {"city": "Dallas"}}] + ) + request = make_request( + messages=HISTORY_MESSAGES, + tools=TOOLS, + tool_choice="required", + ) + _, _, tool_calls = parser.parse(functions_json, request, enable_auto_tools=True) + + assert tool_calls is not None + assert tool_calls[0].id == "functions.get_current_weather:2" + + +def test_count_history_tool_calls_responses_request(): + request = ResponsesRequest.model_validate( + { + "model": "test-model", + "input": [ + { + "type": "function_call", + "call_id": "call_0", + "name": "get_current_weather", + "arguments": "{}", + }, + { + "type": "function_call", + "call_id": "call_1", + "name": "get_forecast", + "arguments": "{}", + }, + ], + } + ) + + assert count_history_tool_calls(request) == 2 + + +def test_parse_required_tool_choice_random_ids_deferred(tokenizer): + parser = make_parser(tokenizer, reasoning=False, tool=True) + functions_json = json.dumps( + [{"name": "get_current_weather", "parameters": {"city": "Dallas"}}] + ) + request = make_request( + messages=HISTORY_MESSAGES, + tools=TOOLS, + tool_choice="required", + ) + _, _, tool_calls = parser.parse(functions_json, request, enable_auto_tools=True) + + assert tool_calls is not None + assert tool_calls[0].id is None + + +def test_parse_named_tool_choice_kimi_k2_id(tokenizer): + parser = make_parser( + tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG + ) + request = make_request( + tools=TOOLS, + tool_choice={ + "type": "function", + "function": {"name": "get_weather"}, + }, + ) + _, content, tool_calls = parser.parse( + TOOL_ARGUMENTS, request, enable_auto_tools=True + ) + + assert content is None + assert tool_calls is not None + assert tool_calls[0].id == "functions.get_weather:0" + + def test_parse_named_tool_choice_content_none(tokenizer): parser = make_parser(tokenizer, reasoning=False, tool=True) request = make_request( diff --git a/tests/parser/test_streaming.py b/tests/parser/test_streaming.py index dbc64e7559..e6ec273b25 100644 --- a/tests/parser/test_streaming.py +++ b/tests/parser/test_streaming.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import json +from types import SimpleNamespace import pytest @@ -47,6 +48,36 @@ TOOLS = [ ] +KIMI_K2_MODEL_CONFIG = SimpleNamespace( + hf_text_config=SimpleNamespace(model_type="kimi_k2"), + hf_overrides=None, +) + +HISTORY_MESSAGES = [ + {"role": "user", "content": "first"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "functions.get_current_weather:0", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": "{}", + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "functions.get_current_weather:0", + "content": "{}", + }, + {"role": "user", "content": "again"}, +] + + @pytest.fixture def request_obj(): return ChatCompletionRequest( @@ -57,12 +88,12 @@ def request_obj(): ) -def make_parser(tokenizer, reasoning=False, tool=False): +def make_parser(tokenizer, reasoning=False, tool=False, **kwargs): class TestParser(DelegatingParser): reasoning_parser_cls = ThinkReasoningParser if reasoning else None tool_parser_cls = Hermes2ProToolParser if tool else None - return TestParser(tokenizer) + return TestParser(tokenizer, **kwargs) def stream_text(parser, tokenizer, text, request, prompt_token_ids=None): @@ -365,3 +396,76 @@ def test_parse_delta_tool_choice_none_with_reasoning(tokenizer, request_obj): assert len(tool_calls) == 0 assert "" in content assert "get_weather" in content + + +def test_parse_delta_required_tool_choice_kimi_k2_ids(tokenizer, request_obj): + parser = make_parser( + tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG + ) + request = request_obj.model_copy(update={"tool_choice": "required"}) + output = json.dumps( + [ + { + "name": "get_current_weather", + "parameters": {"city": "Dallas"}, + } + ] + ) + + results: list[DeltaMessage | None] = [] + prompt_token_ids: list[int] | None = [] + for i in range(0, len(output), 3): + chunk = output[i : i + 3] + results.append( + parser.parse_delta( + chunk, + [], + request, + prompt_token_ids=prompt_token_ids, + finished=False, + ) + ) + prompt_token_ids = None + + _, content, tool_calls = collect_fields(results) + assert content == "" + assert any(tc.id == "functions.get_current_weather:0" for tc in tool_calls) + assert all(tc.id in (None, "functions.get_current_weather:0") for tc in tool_calls) + + +def test_parse_delta_required_tool_choice_kimi_k2_ids_after_history( + tokenizer, request_obj +): + parser = make_parser( + tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG + ) + request = request_obj.model_copy( + update={"messages": HISTORY_MESSAGES, "tool_choice": "required"} + ) + output = json.dumps( + [ + { + "name": "get_current_weather", + "parameters": {"city": "Dallas"}, + } + ] + ) + + results: list[DeltaMessage | None] = [] + prompt_token_ids: list[int] | None = [] + for i in range(0, len(output), 3): + chunk = output[i : i + 3] + results.append( + parser.parse_delta( + chunk, + [], + request, + prompt_token_ids=prompt_token_ids, + finished=False, + ) + ) + prompt_token_ids = None + + _, _, tool_calls = collect_fields(results) + assert any(tc.id == "functions.get_current_weather:1" for tc in tool_calls) + assert all(tc.id in (None, "functions.get_current_weather:1") for tc in tool_calls) diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index 0b41c4d7fa..284b2511db 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -17,8 +17,6 @@ from vllm.engine.protocol import EngineClient from vllm.entrypoints.chat_utils import ( ChatTemplateContentFormatOption, ConversationMessage, - get_history_tool_calls_cnt, - get_tool_call_id_type, make_tool_call_id, ) from vllm.entrypoints.openai.chat_completion.protocol import ( @@ -170,8 +168,6 @@ class OpenAIServingChat(OpenAIServing): if mc.generation_config not in ("auto", "vllm") else getattr(mc, "override_generation_config", {}).get("max_new_tokens") ) - self.tool_call_id_type = get_tool_call_id_type(self.model_config) - # NOTE(woosuk): While OpenAI's chat completion API supports browsing # for some models, currently vLLM doesn't support it. Please use the # Responses API instead. @@ -261,6 +257,7 @@ class OpenAIServingChat(OpenAIServing): tokenizer, request.tools, chat_template_kwargs=chat_template_kwargs, + model_config=self.model_config, ) result = await self.render_chat_request(request) if isinstance(result, ErrorResponse): @@ -433,11 +430,6 @@ class OpenAIServingChat(OpenAIServing): else: tool_choice_function_name = None - if self.tool_call_id_type == "kimi_k2": - history_tool_call_cnt = get_history_tool_calls_cnt(conversation) - else: - history_tool_call_cnt = 0 - previous_texts = [""] * num_choices try: @@ -451,14 +443,10 @@ class OpenAIServingChat(OpenAIServing): tokenizer, request.tools, chat_template_kwargs=chat_template_kwargs, + model_config=self.model_config, ) for _ in range(num_choices) ] - for p in parsers: - if p is not None: - # NOTE: HarmonyParser ignores _stream_state (uses its own FSM). - p._stream_state.tool_call_id_type = self.tool_call_id_type - p._stream_state.history_tool_call_cnt = history_tool_call_cnt else: parsers = [None] * num_choices except Exception as e: @@ -842,10 +830,6 @@ class OpenAIServingChat(OpenAIServing): ) choices: list[ChatCompletionResponseChoice] = [] - if self.tool_call_id_type == "kimi_k2": - history_tool_call_cnt = get_history_tool_calls_cnt(conversation) - else: - history_tool_call_cnt = 0 role = self.get_chat_request_role(request) tool_parser_cls = ( @@ -885,54 +869,26 @@ class OpenAIServingChat(OpenAIServing): tool_calls = [] auto_tools_called = False + is_named_tool_choice = ( + request.tool_choice is not None + and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam + ) + is_required_tool_choice = request.tool_choice == "required" if (not self.enable_auto_tools or not tool_parser_cls) and ( - not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam) - and request.tool_choice != "required" + not is_named_tool_choice and not is_required_tool_choice ): message = ChatMessage(role=role, reasoning=reasoning, content=content) - elif ( - request.tool_choice - and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam - ): - tool_call_items = [] - tool_calls = tool_calls or [] - for tc in tool_calls: - if not tc.id: - tc.id = make_tool_call_id( - id_type=self.tool_call_id_type, - func_name=tc.name, - idx=history_tool_call_cnt, - ) - tool_call_items.append(ToolCall(id=tc.id, function=tc)) - history_tool_call_cnt += 1 + elif is_named_tool_choice or is_required_tool_choice: message = ChatMessage( role=role, reasoning=reasoning, content=content or "", - tool_calls=tool_call_items, - ) - - elif request.tool_choice and request.tool_choice == "required": - tool_call_items = [] - tool_calls = tool_calls or [] - for tool_call in tool_calls: - if not tool_call.id: - tool_call.id = make_tool_call_id( - id_type=self.tool_call_id_type, - func_name=tool_call.name, - idx=history_tool_call_cnt, - ) - tool_call_items.append( - ToolCall(id=tool_call.id, function=tool_call) - ) - history_tool_call_cnt += 1 - message = ChatMessage( - role=role, - content=content or "", - tool_calls=tool_call_items, - reasoning=reasoning, + tool_calls=[ + ToolCall(id=tc.id or make_tool_call_id(), function=tc) + for tc in (tool_calls or []) + ], ) # if the request doesn't use tool choice @@ -949,21 +905,14 @@ class OpenAIServingChat(OpenAIServing): ): auto_tools_called = tool_calls is not None and len(tool_calls) > 0 if tool_calls: - tool_call_items = [] - for tc in tool_calls: - if not tc.id: - tc.id = make_tool_call_id( - id_type=self.tool_call_id_type, - func_name=tc.name, - idx=history_tool_call_cnt, - ) - tool_call_items.append(ToolCall(id=tc.id, function=tc)) - history_tool_call_cnt += 1 message = ChatMessage( role=role, reasoning=reasoning, content=content, - tool_calls=tool_call_items, + tool_calls=[ + ToolCall(id=tc.id or make_tool_call_id(), function=tc) + for tc in tool_calls + ], ) else: diff --git a/vllm/entrypoints/openai/responses/context.py b/vllm/entrypoints/openai/responses/context.py index 6b987f449d..3c9a31a141 100644 --- a/vllm/entrypoints/openai/responses/context.py +++ b/vllm/entrypoints/openai/responses/context.py @@ -301,7 +301,6 @@ class ParsableContext(ConversationContext): chat_template_content_format: ChatTemplateContentFormatOption, response_parser: Parser | None = None, enable_auto_tools: bool = False, - tool_call_id_type: str = "random", ): self.num_prompt_tokens = 0 self.num_output_tokens = 0 @@ -314,20 +313,8 @@ class ParsableContext(ConversationContext): self.num_init_messages = len(response_messages) self.finish_reason: str | None = None self.enable_auto_tools = enable_auto_tools - self.tool_call_id_type = tool_call_id_type self.response_parser = response_parser - if self.response_parser is None and parser_cls is not None: - chat_template_kwargs = request.build_chat_params( - default_template=chat_template, - default_template_content_format=chat_template_content_format, - ).chat_template_kwargs - self.response_parser = parser_cls( - tokenizer, - tools=request.tools, - chat_template_kwargs=chat_template_kwargs, - ) - self.parser_cls = parser_cls self.request = request @@ -365,7 +352,6 @@ class ParsableContext(ConversationContext): reasoning=reasoning, content=content, tool_calls=tool_calls, - tool_call_id_type=self.tool_call_id_type, ) ) elif completion.text: diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index 62af1953dd..f2e1f8e5d8 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -30,7 +30,6 @@ from vllm.engine.protocol import EngineClient from vllm.entrypoints.chat_utils import ( ChatCompletionMessageParam, ChatTemplateContentFormatOption, - get_tool_call_id_type, ) from vllm.entrypoints.mcp.tool_server import ToolServer from vllm.entrypoints.openai.engine.protocol import ( @@ -222,9 +221,6 @@ class OpenAIServingResponses(OpenAIServing): "For gpt-oss, we ignore --enable-auto-tool-choice " "and always enable tool use." ) - - self.tool_call_id_type = get_tool_call_id_type(self.model_config) - self.enable_auto_tools = enable_auto_tools # HACK(woosuk): This is a hack. We should use a better store. # FIXME: If enable_store=True, this may cause a memory leak since we @@ -272,6 +268,7 @@ class OpenAIServingResponses(OpenAIServing): tokenizer, request.tools, chat_template_kwargs=chat_template_kwargs, + model_config=self.model_config, ) def _validate_generator_input( @@ -493,7 +490,6 @@ class OpenAIServingResponses(OpenAIServing): chat_template=self.chat_template, chat_template_content_format=self.chat_template_content_format, enable_auto_tools=self.enable_auto_tools, - tool_call_id_type=self.tool_call_id_type, ) else: context = SimpleContext( @@ -1073,7 +1069,6 @@ class OpenAIServingResponses(OpenAIServing): content=content, tool_calls=tool_calls, logprobs=logprobs, - tool_call_id_type=self.tool_call_id_type, ) # Fallback when no parser is configured diff --git a/vllm/entrypoints/openai/responses/utils.py b/vllm/entrypoints/openai/responses/utils.py index 15b6fa88ab..a2f35dca23 100644 --- a/vllm/entrypoints/openai/responses/utils.py +++ b/vllm/entrypoints/openai/responses/utils.py @@ -45,7 +45,6 @@ def build_response_output_items( content: str | None, tool_calls: list[FunctionCall] | None, logprobs: list[Logprob] | None = None, - tool_call_id_type: str = "random", ) -> list[ResponseOutputItem]: outputs: list[ResponseOutputItem] = [] @@ -86,12 +85,7 @@ def build_response_output_items( ResponseFunctionToolCall( id=f"fc_{random_uuid()}", call_id=tool_call.id - if tool_call.id - else make_tool_call_id( - id_type=tool_call_id_type, - func_name=tool_call.name, - idx=idx, - ), + or make_tool_call_id(func_name=tool_call.name, idx=idx), type="function_call", status="completed", name=tool_call.name, diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py index 11fca8e43a..62275ff228 100644 --- a/vllm/parser/abstract_parser.py +++ b/vllm/parser/abstract_parser.py @@ -11,6 +11,10 @@ from functools import cached_property from openai.types.responses import ToolChoiceFunction from pydantic import TypeAdapter, ValidationError +from vllm.entrypoints.chat_utils import ( + get_tool_call_id_type, + make_tool_call_id, +) from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionNamedToolChoiceParam, ChatCompletionRequest, @@ -24,6 +28,7 @@ from vllm.entrypoints.openai.engine.protocol import ( from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.logger import init_logger from vllm.parser.metrics import record_tool_parser_invocation +from vllm.parser.utils import count_history_tool_calls from vllm.reasoning.abs_reasoning_parsers import ReasoningParser from vllm.sampling_params import StructuredOutputsParams from vllm.tokenizers import TokenizerLike @@ -46,6 +51,7 @@ class StreamState: previous_text: str = "" previous_token_ids: list[int] = field(default_factory=list) history_tool_call_cnt: int = 0 + history_tool_call_cnt_initialized: bool = False tool_call_id_type: str = "random" # only used for "required" and "named tool" choices, # tracks whether function name has been fully returned in the stream yet @@ -108,6 +114,7 @@ class Parser: tokenizer: TokenizerLike, tools: list[Tool] | None = None, *args, + model_config=None, **kwargs, ): self.model_tokenizer = tokenizer @@ -124,7 +131,14 @@ class Parser: self._reasoning_parser is None or self._reasoning_parser.engine_based_streaming ) and (self._tool_parser is None or self._tool_parser.engine_based_streaming) - self._stream_state = StreamState(engine_based=self._engine_based) + self._stream_state = StreamState( + tool_call_id_type=( + get_tool_call_id_type(model_config) + if model_config is not None + else "random" + ), + engine_based=self._engine_based, + ) @cached_property def vocab(self) -> dict[str, int]: @@ -149,6 +163,19 @@ class Parser: def tool_parser(self, parser: ToolParser | None) -> None: self._tool_parser = parser + def _initialize_history_tool_call_cnt( + self, + request: ChatCompletionRequest | ResponsesRequest, + ) -> None: + state = self._stream_state + if state.history_tool_call_cnt_initialized: + return + if state.tool_call_id_type != "kimi_k2": + state.history_tool_call_cnt_initialized = True + return + state.history_tool_call_cnt = count_history_tool_calls(request) + state.history_tool_call_cnt_initialized = True + # ========== Reasoning Parser Methods ========== @abstractmethod @@ -375,6 +402,18 @@ class DelegatingParser(Parser): return request.tool_choice.function.name raise ValueError("Invalid tool_choice for function name extraction.") + def _make_tool_call_id(self, function_name: str) -> str | None: + state = self._stream_state + if state.tool_call_id_type != "kimi_k2": + return None + tool_call_id = make_tool_call_id( + id_type=state.tool_call_id_type, + func_name=function_name, + idx=state.history_tool_call_cnt, + ) + state.history_tool_call_cnt += 1 + return tool_call_id + def _extract_tool_calls( self, content: str | None, @@ -404,9 +443,11 @@ class DelegatingParser(Parser): if is_named_tool_choice and supports_required_and_named: if content is None: return [], None + function_name = self._get_function_name(request) tool_calls.append( FunctionCall( - name=self._get_function_name(request), + id=self._make_tool_call_id(function_name), + name=function_name, arguments=content, ) ) @@ -422,6 +463,7 @@ class DelegatingParser(Parser): for tc in parsed_calls: tool_calls.append( FunctionCall( + id=self._make_tool_call_id(tc.name), name=tc.name, arguments=json.dumps(tc.parameters, ensure_ascii=False), ) @@ -733,6 +775,7 @@ class DelegatingParser(Parser): enable_auto_tools: bool = False, model_output_token_ids: Sequence[int] = (), ) -> tuple[str | None, str | None, list[FunctionCall] | None]: + self._initialize_history_tool_call_cnt(request) reasoning, content = self.extract_reasoning(model_output, request) tool_calls, content = self._extract_tool_calls( content=content, @@ -750,6 +793,7 @@ class DelegatingParser(Parser): *, finished: bool, ) -> DeltaMessage | None: + self._initialize_history_tool_call_cnt(request) state = self._stream_state if not state.prompt_reasoning_checked and prompt_token_ids is not None: diff --git a/vllm/parser/engine/parser_engine.py b/vllm/parser/engine/parser_engine.py index 6848a90514..497eb9039b 100644 --- a/vllm/parser/engine/parser_engine.py +++ b/vllm/parser/engine/parser_engine.py @@ -13,7 +13,7 @@ from typing import TYPE_CHECKING import regex as re -from vllm.entrypoints.chat_utils import make_tool_call_id +from vllm.entrypoints.chat_utils import get_tool_call_id_type, make_tool_call_id from vllm.entrypoints.openai.engine.protocol import ( DeltaFunctionCall, DeltaMessage, @@ -89,11 +89,18 @@ class ParserEngine(Parser): tools: list[Tool] | None = None, *, parser_engine_config: ParserEngineConfig, + model_config=None, **kwargs, ) -> None: self.model_tokenizer = tokenizer self._tools = tools - self._stream_state = StreamState() + self._stream_state = StreamState( + tool_call_id_type=( + get_tool_call_id_type(model_config) + if model_config is not None + else "random" + ), + ) self._reasoning_parser = None self._tool_parser = None self.parser_engine_config = parser_engine_config @@ -419,6 +426,7 @@ class ParserEngine(Parser): *, finished: bool, ) -> DeltaMessage | None: + self._initialize_history_tool_call_cnt(request) if not self._prompt_streaming_prepared and prompt_token_ids is not None: # NOTE: call the hook BEFORE setting the flag, because the hook # may invoke ``_reset`` (e.g. via ``initialize_streaming``) which @@ -658,6 +666,7 @@ class ParserEngine(Parser): enable_auto_tools: bool = False, model_output_token_ids: Sequence[int] = (), ) -> tuple[str | None, str | None, list[FunctionCall] | None]: + self._initialize_history_tool_call_cnt(request) self._check_skip_tool_parsing(request) reasoning, content, tool_call_info = self._single_pass_parse( model_output, diff --git a/vllm/parser/utils.py b/vllm/parser/utils.py new file mode 100644 index 0000000000..51382cd290 --- /dev/null +++ b/vllm/parser/utils.py @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from collections.abc import Iterable, Sequence + +from openai.types.responses import ResponseFunctionToolCall + +from vllm.entrypoints.chat_utils import ChatCompletionMessageParam +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest +from vllm.entrypoints.openai.responses.protocol import ( + ResponseInputOutputItem, + ResponsesRequest, +) + + +def count_tool_calls(tool_calls: object) -> int: + if tool_calls is None: + return 0 + if isinstance(tool_calls, (str, bytes, dict)): + return 1 + if isinstance(tool_calls, Iterable): + return sum(1 for _ in tool_calls) + return 1 + + +def count_chat_history_tool_calls( + messages: Sequence[ChatCompletionMessageParam], +) -> int: + return sum( + count_tool_calls(msg.get("tool_calls")) + for msg in messages + if isinstance(msg, dict) and msg.get("role") == "assistant" + ) + + +def count_response_history_tool_calls( + response_items: Sequence[ResponseInputOutputItem], +) -> int: + count = 0 + for item in response_items: + if isinstance(item, ResponseFunctionToolCall): + count += 1 + continue + + if isinstance(item, dict): + item_type = item.get("type") + if item_type == "function_call": + count += 1 + elif item.get("role") == "assistant": + count += count_tool_calls(item.get("tool_calls")) + + return count + + +def count_history_tool_calls( + request: ChatCompletionRequest | ResponsesRequest, +) -> int: + if isinstance(request, ChatCompletionRequest): + return count_chat_history_tool_calls(request.messages) + + request_input = request.input + if isinstance(request_input, str): + return 0 + + return count_response_history_tool_calls(request_input)