[Bugfix] Gemma4: skip forced JSON for required/named tool choice (#45795)

Signed-off-by: Federico Iezzi <fiezzi@google.com>
This commit is contained in:
Federico
2026-06-16 20:38:29 +00:00
committed by GitHub
parent 4fadf9c92c
commit b9684d99e9
2 changed files with 120 additions and 1 deletions
@@ -20,6 +20,13 @@ calling for parsers relying on special-token delimiters (Gemma4):
tracked in ``__fields_set__``, which can drop the nested config from
``model_dump``. It also passed a ``description`` kwarg carrying the
wrong-purpose string ``"Response format for tool calling"``.
3. :class:`Gemma4EngineToolParser` (the engine-based parser, #45588) sets
``supports_required_and_named=False`` but did not skip the forced
``structured_outputs`` JSON for ``required``/named tool choice. The model
was constrained to JSON the native parser cannot read, so the call leaked
as content with empty ``tool_calls``. ``adjust_request`` now skips that
constraint so Gemma4 emits its native ``<|tool_call>`` syntax.
"""
from __future__ import annotations
@@ -28,6 +35,7 @@ from typing import Any
from openai.types.responses.tool_param import FunctionToolParam
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.tool_parsers.abstract_tool_parser import ToolParser
from vllm.tool_parsers.gemma4_engine_tool_parser import (
@@ -49,7 +57,7 @@ def _get_weather_tool() -> FunctionToolParam:
)
def _build_responses_request(*, tool_choice: str) -> ResponsesRequest:
def _build_responses_request(*, tool_choice: str | dict[str, Any]) -> ResponsesRequest:
return ResponsesRequest(
model="gemma4-test",
input=[{"role": "user", "content": "What is the weather in Hanoi?"}],
@@ -60,6 +68,30 @@ def _build_responses_request(*, tool_choice: str) -> ResponsesRequest:
)
def _build_chat_request(*, tool_choice: str | dict[str, Any]) -> ChatCompletionRequest:
return ChatCompletionRequest.model_validate(
{
"model": "gemma4-test",
"messages": [{"role": "user", "content": "What is the weather in Hanoi?"}],
"tools": [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get current weather for a city",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
},
}
],
"tool_choice": tool_choice,
}
)
class _StubTokenizer:
"""Minimal tokenizer stub to satisfy ``Gemma4EngineToolParser.__init__``."""
@@ -121,3 +153,62 @@ def test_tool_parser_adjust_request_builds_valid_response_text_config() -> None:
# The old code passed a wrong-purpose string; valid field should now
# either be absent or None (the openai-python default).
assert fmt.get("description") in (None, "")
def test_gemma4_required_skips_structured_outputs_chatcompletion() -> None:
"""required + ChatCompletion: ``Gemma4EngineToolParser`` must skip the
forced JSON ``structured_outputs`` so the model emits its native
``<|tool_call>`` syntax. The base parser constrained output to JSON the
native parser cannot read, leaking it as content with empty
``tool_calls`` (regression after #45588).
"""
parser = Gemma4ToolParser(_StubTokenizer())
request = _build_chat_request(tool_choice="required")
parser.adjust_request(request)
assert request.structured_outputs is None
assert request.skip_special_tokens is False
def test_gemma4_named_skips_structured_outputs_chatcompletion() -> None:
"""named + ChatCompletion: the forced single-function JSON schema must be
skipped, same as ``required``.
"""
parser = Gemma4ToolParser(_StubTokenizer())
request = _build_chat_request(
tool_choice={"type": "function", "function": {"name": "get_weather"}}
)
parser.adjust_request(request)
assert request.structured_outputs is None
assert request.skip_special_tokens is False
def test_gemma4_required_skips_structured_outputs_responses() -> None:
"""required + Responses: the forced JSON schema (``request.text``) must be
skipped so the native delimiters reach the extractor.
"""
parser = Gemma4ToolParser(_StubTokenizer())
request = _build_responses_request(tool_choice="required")
parser.adjust_request(request)
assert request.text is None
assert request.skip_special_tokens is False
def test_gemma4_named_skips_structured_outputs_responses() -> None:
"""named (``ToolChoiceFunction``) + Responses: the forced single-function
JSON schema must be skipped.
"""
parser = Gemma4ToolParser(_StubTokenizer())
request = _build_responses_request(
tool_choice={"type": "function", "name": "get_weather"}
)
parser.adjust_request(request)
assert request.text is None
assert request.skip_special_tokens is False
@@ -1,8 +1,36 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from openai.types.responses import ToolChoiceFunction
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionNamedToolChoiceParam,
ChatCompletionRequest,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.parser.engine.registered_adapters import Gemma4ParserToolAdapter
class Gemma4EngineToolParser(Gemma4ParserToolAdapter): # type: ignore[valid-type, misc]
supports_required_and_named = False
def adjust_request(
self, request: ChatCompletionRequest | ResponsesRequest
) -> ChatCompletionRequest | ResponsesRequest:
"""Skip structured-output JSON for required/named tool choice.
Gemma4 emits its native ``<|tool_call>call:...`` syntax, which the
parser extracts directly. The base ``ToolParser.adjust_request`` would
set ``structured_outputs`` for required/named and force JSON via guided
decoding, conflicting with that native syntax (it leaks as content and
crashes EngineCore under speculative decoding). Skip it so the model
emits its native format (mirrors the GLM4 parser).
"""
if request.tools:
tc = request.tool_choice
if tc == "required" or isinstance(
tc, (ChatCompletionNamedToolChoiceParam, ToolChoiceFunction)
):
request.skip_special_tokens = False
return request
return super().adjust_request(request)