[Bugfix] Gemma4: skip forced JSON for required/named tool choice (#45795)
Signed-off-by: Federico Iezzi <fiezzi@google.com>
This commit is contained in:
@@ -1,8 +1,36 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from openai.types.responses import ToolChoiceFunction
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.parser.engine.registered_adapters import Gemma4ParserToolAdapter
|
||||
|
||||
|
||||
class Gemma4EngineToolParser(Gemma4ParserToolAdapter): # type: ignore[valid-type, misc]
|
||||
supports_required_and_named = False
|
||||
|
||||
def adjust_request(
|
||||
self, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
"""Skip structured-output JSON for required/named tool choice.
|
||||
|
||||
Gemma4 emits its native ``<|tool_call>call:...`` syntax, which the
|
||||
parser extracts directly. The base ``ToolParser.adjust_request`` would
|
||||
set ``structured_outputs`` for required/named and force JSON via guided
|
||||
decoding, conflicting with that native syntax (it leaks as content and
|
||||
crashes EngineCore under speculative decoding). Skip it so the model
|
||||
emits its native format (mirrors the GLM4 parser).
|
||||
"""
|
||||
if request.tools:
|
||||
tc = request.tool_choice
|
||||
if tc == "required" or isinstance(
|
||||
tc, (ChatCompletionNamedToolChoiceParam, ToolChoiceFunction)
|
||||
):
|
||||
request.skip_special_tokens = False
|
||||
return request
|
||||
return super().adjust_request(request)
|
||||
|
||||
Reference in New Issue
Block a user