diff --git a/tests/tool_use/test_gemma4_responses_adjust_request.py b/tests/tool_use/test_gemma4_responses_adjust_request.py index 64c12ee661..27f7209f3e 100644 --- a/tests/tool_use/test_gemma4_responses_adjust_request.py +++ b/tests/tool_use/test_gemma4_responses_adjust_request.py @@ -20,6 +20,13 @@ calling for parsers relying on special-token delimiters (Gemma4): tracked in ``__fields_set__``, which can drop the nested config from ``model_dump``. It also passed a ``description`` kwarg carrying the wrong-purpose string ``"Response format for tool calling"``. + +3. :class:`Gemma4EngineToolParser` (the engine-based parser, #45588) sets + ``supports_required_and_named=False`` but did not skip the forced + ``structured_outputs`` JSON for ``required``/named tool choice. The model + was constrained to JSON the native parser cannot read, so the call leaked + as content with empty ``tool_calls``. ``adjust_request`` now skips that + constraint so Gemma4 emits its native ``<|tool_call>`` syntax. """ from __future__ import annotations @@ -28,6 +35,7 @@ from typing import Any from openai.types.responses.tool_param import FunctionToolParam +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.tool_parsers.abstract_tool_parser import ToolParser from vllm.tool_parsers.gemma4_engine_tool_parser import ( @@ -49,7 +57,7 @@ def _get_weather_tool() -> FunctionToolParam: ) -def _build_responses_request(*, tool_choice: str) -> ResponsesRequest: +def _build_responses_request(*, tool_choice: str | dict[str, Any]) -> ResponsesRequest: return ResponsesRequest( model="gemma4-test", input=[{"role": "user", "content": "What is the weather in Hanoi?"}], @@ -60,6 +68,30 @@ def _build_responses_request(*, tool_choice: str) -> ResponsesRequest: ) +def _build_chat_request(*, tool_choice: str | dict[str, Any]) -> ChatCompletionRequest: + return ChatCompletionRequest.model_validate( + { + "model": "gemma4-test", + "messages": [{"role": "user", "content": "What is the weather in Hanoi?"}], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get current weather for a city", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + ], + "tool_choice": tool_choice, + } + ) + + class _StubTokenizer: """Minimal tokenizer stub to satisfy ``Gemma4EngineToolParser.__init__``.""" @@ -121,3 +153,62 @@ def test_tool_parser_adjust_request_builds_valid_response_text_config() -> None: # The old code passed a wrong-purpose string; valid field should now # either be absent or None (the openai-python default). assert fmt.get("description") in (None, "") + + +def test_gemma4_required_skips_structured_outputs_chatcompletion() -> None: + """required + ChatCompletion: ``Gemma4EngineToolParser`` must skip the + forced JSON ``structured_outputs`` so the model emits its native + ``<|tool_call>`` syntax. The base parser constrained output to JSON the + native parser cannot read, leaking it as content with empty + ``tool_calls`` (regression after #45588). + """ + parser = Gemma4ToolParser(_StubTokenizer()) + request = _build_chat_request(tool_choice="required") + + parser.adjust_request(request) + + assert request.structured_outputs is None + assert request.skip_special_tokens is False + + +def test_gemma4_named_skips_structured_outputs_chatcompletion() -> None: + """named + ChatCompletion: the forced single-function JSON schema must be + skipped, same as ``required``. + """ + parser = Gemma4ToolParser(_StubTokenizer()) + request = _build_chat_request( + tool_choice={"type": "function", "function": {"name": "get_weather"}} + ) + + parser.adjust_request(request) + + assert request.structured_outputs is None + assert request.skip_special_tokens is False + + +def test_gemma4_required_skips_structured_outputs_responses() -> None: + """required + Responses: the forced JSON schema (``request.text``) must be + skipped so the native delimiters reach the extractor. + """ + parser = Gemma4ToolParser(_StubTokenizer()) + request = _build_responses_request(tool_choice="required") + + parser.adjust_request(request) + + assert request.text is None + assert request.skip_special_tokens is False + + +def test_gemma4_named_skips_structured_outputs_responses() -> None: + """named (``ToolChoiceFunction``) + Responses: the forced single-function + JSON schema must be skipped. + """ + parser = Gemma4ToolParser(_StubTokenizer()) + request = _build_responses_request( + tool_choice={"type": "function", "name": "get_weather"} + ) + + parser.adjust_request(request) + + assert request.text is None + assert request.skip_special_tokens is False diff --git a/vllm/tool_parsers/gemma4_engine_tool_parser.py b/vllm/tool_parsers/gemma4_engine_tool_parser.py index 72c3b2e552..04c03ecaa2 100644 --- a/vllm/tool_parsers/gemma4_engine_tool_parser.py +++ b/vllm/tool_parsers/gemma4_engine_tool_parser.py @@ -1,8 +1,36 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from openai.types.responses import ToolChoiceFunction + +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionNamedToolChoiceParam, + ChatCompletionRequest, +) +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.parser.engine.registered_adapters import Gemma4ParserToolAdapter class Gemma4EngineToolParser(Gemma4ParserToolAdapter): # type: ignore[valid-type, misc] supports_required_and_named = False + + def adjust_request( + self, request: ChatCompletionRequest | ResponsesRequest + ) -> ChatCompletionRequest | ResponsesRequest: + """Skip structured-output JSON for required/named tool choice. + + Gemma4 emits its native ``<|tool_call>call:...`` syntax, which the + parser extracts directly. The base ``ToolParser.adjust_request`` would + set ``structured_outputs`` for required/named and force JSON via guided + decoding, conflicting with that native syntax (it leaks as content and + crashes EngineCore under speculative decoding). Skip it so the model + emits its native format (mirrors the GLM4 parser). + """ + if request.tools: + tc = request.tool_choice + if tc == "required" or isinstance( + tc, (ChatCompletionNamedToolChoiceParam, ToolChoiceFunction) + ): + request.skip_special_tokens = False + return request + return super().adjust_request(request)