[Tool] adjust_request to reasoning parser, and Gemma4 fixes (#39027)
Signed-off-by: Ben Browning <bbrownin@redhat.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
Cursor
parent
e24e0a43a4
commit
8477fe427d
@@ -170,7 +170,8 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
else:
|
||||
cls._convert_message_content(msg, openai_msg, openai_messages)
|
||||
|
||||
openai_messages.append(openai_msg)
|
||||
if not (msg.role == "user" and "content" not in openai_msg):
|
||||
openai_messages.append(openai_msg)
|
||||
|
||||
@classmethod
|
||||
def _convert_message_content(
|
||||
|
||||
@@ -379,6 +379,7 @@ async def init_app_state(
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
@@ -481,6 +482,7 @@ async def init_render_app_state(
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
|
||||
@@ -594,6 +594,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
default_template_kwargs=None,
|
||||
tool_dicts=tool_dicts,
|
||||
tool_parser=self.parser.tool_parser_cls if self.parser else None,
|
||||
reasoning_parser=self.parser.reasoning_parser_cls if self.parser else None,
|
||||
)
|
||||
return messages, engine_inputs
|
||||
|
||||
@@ -618,6 +619,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
default_template_kwargs=None,
|
||||
tool_dicts=tool_dicts,
|
||||
tool_parser=tool_parser,
|
||||
reasoning_parser=self.parser.reasoning_parser_cls if self.parser else None,
|
||||
)
|
||||
return engine_inputs
|
||||
|
||||
|
||||
@@ -44,6 +44,7 @@ from vllm.inputs import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser import ParserManager
|
||||
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
|
||||
from vllm.renderers import BaseRenderer, merge_kwargs
|
||||
from vllm.renderers.inputs.preprocess import (
|
||||
extract_prompt_components,
|
||||
@@ -74,6 +75,7 @@ class OpenAIServingRender:
|
||||
enable_auto_tools: bool = False,
|
||||
exclude_tools_when_tool_choice_none: bool = False,
|
||||
tool_parser: str | None = None,
|
||||
reasoning_parser: str | None = None,
|
||||
default_chat_template_kwargs: dict[str, Any] | None = None,
|
||||
log_error_stack: bool = False,
|
||||
) -> None:
|
||||
@@ -94,6 +96,11 @@ class OpenAIServingRender:
|
||||
enable_auto_tools=enable_auto_tools,
|
||||
model_name=model_config.model,
|
||||
)
|
||||
self.reasoning_parser: type[ReasoningParser] | None = (
|
||||
ParserManager.get_reasoning_parser(
|
||||
reasoning_parser_name=reasoning_parser,
|
||||
)
|
||||
)
|
||||
self.default_chat_template_kwargs: dict[str, Any] = (
|
||||
default_chat_template_kwargs or {}
|
||||
)
|
||||
@@ -245,6 +252,7 @@ class OpenAIServingRender:
|
||||
default_template_kwargs=self.default_chat_template_kwargs,
|
||||
tool_dicts=tool_dicts,
|
||||
tool_parser=tool_parser,
|
||||
reasoning_parser=self.reasoning_parser,
|
||||
)
|
||||
else:
|
||||
# For GPT-OSS.
|
||||
@@ -503,6 +511,7 @@ class OpenAIServingRender:
|
||||
default_template_kwargs: dict[str, Any] | None,
|
||||
tool_dicts: list[dict[str, Any]] | None = None,
|
||||
tool_parser: type[ToolParser] | None = None,
|
||||
reasoning_parser: type[ReasoningParser] | None = None,
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> tuple[list[ConversationMessage], list[EngineInput]]:
|
||||
@@ -539,6 +548,10 @@ class OpenAIServingRender:
|
||||
skip_mm_cache=skip_mm_cache,
|
||||
)
|
||||
|
||||
if reasoning_parser is not None:
|
||||
tokenizer = renderer.get_tokenizer()
|
||||
request = reasoning_parser(tokenizer).adjust_request(request=request)
|
||||
|
||||
# tool parsing is done only if a tool_parser has been set and if
|
||||
# tool_choice is not "none" (if tool_choice is "none" but a tool_parser
|
||||
# is set, we want to prevent parsing a tool_call hallucinated by the LLM
|
||||
|
||||
@@ -496,6 +496,15 @@ class DelegatingParser(Parser):
|
||||
# No tool calls
|
||||
return [], content
|
||||
|
||||
def adjust_request(
|
||||
self, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
if self._reasoning_parser is not None:
|
||||
request = self._reasoning_parser.adjust_request(request)
|
||||
if self._tool_parser is not None:
|
||||
request = self._tool_parser.adjust_request(request)
|
||||
return request
|
||||
|
||||
def extract_reasoning_streaming(
|
||||
self,
|
||||
previous_text: str,
|
||||
|
||||
@@ -6,7 +6,7 @@ import os
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Callable, Iterable, Sequence
|
||||
from functools import cached_property
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||
from vllm.logger import init_logger
|
||||
@@ -150,6 +150,12 @@ class ReasoningParser:
|
||||
previously been parsed and extracted (see constructor)
|
||||
"""
|
||||
|
||||
def adjust_request(
|
||||
self, request: "ChatCompletionRequest | ResponsesRequest"
|
||||
) -> "ChatCompletionRequest | ResponsesRequest":
|
||||
"""Adjust request parameters; override in subclasses as needed."""
|
||||
return request
|
||||
|
||||
def prepare_structured_tag(
|
||||
self,
|
||||
original_tag: str | None,
|
||||
@@ -298,7 +304,7 @@ class ReasoningParserManager:
|
||||
if isinstance(name, str):
|
||||
names = [name]
|
||||
elif is_list_of(name, str):
|
||||
names = name
|
||||
names = cast(list[str], name)
|
||||
else:
|
||||
names = [class_name]
|
||||
|
||||
|
||||
@@ -52,6 +52,16 @@ class Gemma4ReasoningParser(BaseThinkingReasoningParser):
|
||||
# skip_special_tokens=True).
|
||||
self._reasoning_text: str = ""
|
||||
self._prefix_stripped: bool = False
|
||||
self.new_turn_token_id = self.vocab["<|turn>"]
|
||||
self.tool_call_token_id = self.vocab["<|tool_call>"]
|
||||
self.tool_response_token_id = self.vocab["<|tool_response>"]
|
||||
|
||||
def adjust_request(
|
||||
self, request: "ChatCompletionRequest | ResponsesRequest"
|
||||
) -> "ChatCompletionRequest | ResponsesRequest":
|
||||
"""Disable special-token stripping to preserve boundary tokens."""
|
||||
request.skip_special_tokens = False
|
||||
return request
|
||||
|
||||
@property
|
||||
def start_token(self) -> str:
|
||||
@@ -63,6 +73,29 @@ class Gemma4ReasoningParser(BaseThinkingReasoningParser):
|
||||
"""The token that ends reasoning content."""
|
||||
return "<channel|>"
|
||||
|
||||
def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
|
||||
start_token_id = self.start_token_id
|
||||
end_token_id = self.end_token_id
|
||||
new_turn_token_id = self.new_turn_token_id
|
||||
tool_call_token_id = self.tool_call_token_id
|
||||
tool_response_token_id = self.tool_response_token_id
|
||||
|
||||
# Search from the end of input_ids to find the last match.
|
||||
for i in range(len(input_ids) - 1, -1, -1):
|
||||
if input_ids[i] == start_token_id:
|
||||
return False
|
||||
if input_ids[i] == tool_call_token_id:
|
||||
# We're generating a tool call, so reasoning must be ended.
|
||||
return True
|
||||
if input_ids[i] in (new_turn_token_id, tool_response_token_id):
|
||||
# We found a new turn or tool response token so don't consider
|
||||
# reasoning ended yet, since the model starts new reasoning
|
||||
# after these tokens.
|
||||
return False
|
||||
if input_ids[i] == end_token_id:
|
||||
return True
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Non-streaming path
|
||||
# ------------------------------------------------------------------
|
||||
@@ -159,11 +192,10 @@ class Gemma4ReasoningParser(BaseThinkingReasoningParser):
|
||||
result.reasoning = stripped
|
||||
return result
|
||||
else:
|
||||
# This entire delta was prefix — suppress it.
|
||||
# Don't set _prefix_stripped yet; there may be more
|
||||
# prefix chars to consume in the next delta.
|
||||
if len(self._reasoning_text) >= prefix_len:
|
||||
self._prefix_stripped = True
|
||||
result.reasoning = ""
|
||||
return result
|
||||
return None
|
||||
|
||||
# Case 2: Accumulated text is a strict prefix of
|
||||
|
||||
@@ -122,14 +122,16 @@ def _parse_gemma4_args(args_str: str, *, partial: bool = False) -> dict:
|
||||
|
||||
# Parse value
|
||||
if i >= n:
|
||||
result[key] = ""
|
||||
if not partial:
|
||||
result[key] = ""
|
||||
break
|
||||
|
||||
# Skip whitespace after ':'
|
||||
while i < n and args_str[i] in (" ", "\n", "\t"):
|
||||
i += 1
|
||||
if i >= n:
|
||||
result[key] = ""
|
||||
if not partial:
|
||||
result[key] = ""
|
||||
break
|
||||
|
||||
# String value: <|"|>...<|"|>
|
||||
|
||||
Reference in New Issue
Block a user