[Refactor] Move unstreamed tool-arg flush from serving layer to parser (#44017)

Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
Flora Feng
2026-06-02 10:37:43 +08:00
committed by GitHub
parent 816cc73a9b
commit 9affc17a05
5 changed files with 138 additions and 264 deletions
@@ -3,7 +3,6 @@
import asyncio
import io
import json
import time
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
@@ -40,9 +39,7 @@ from vllm.entrypoints.openai.chat_completion.stream_harmony import (
extract_harmony_streaming_delta,
)
from vllm.entrypoints.openai.engine.protocol import (
DeltaFunctionCall,
DeltaMessage,
DeltaToolCall,
ErrorResponse,
FunctionCall,
PromptTokenUsageInfo,
@@ -65,7 +62,7 @@ from vllm.entrypoints.utils import get_max_tokens, should_include_usage
from vllm.inputs import EngineInput
from vllm.logger import init_logger
from vllm.logprobs import Logprob
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.outputs import RequestOutput
from vllm.parser import ParserManager
from vllm.parser.abstract_parser import Parser
from vllm.reasoning import ReasoningParser
@@ -715,6 +712,7 @@ class OpenAIServingChat(OpenAIServing):
delta_token_ids=as_list(output.token_ids),
request=request,
prompt_token_ids=res.prompt_token_ids,
finished=output.finish_reason is not None,
)
if delta_message and delta_message.tool_calls:
tools_streamed[i] = True
@@ -805,81 +803,13 @@ class OpenAIServingChat(OpenAIServing):
# finish_reason='error' indicates a retryable error
self._raise_if_error(output.finish_reason, request_id)
# check to make sure we haven't "forgotten" to stream
# any tokens that were generated but previously
# matched by partial json parsing
# only happens if we are NOT using structured outputs
index = 0
auto_tools_called = False
if tool_parser:
auto_tools_called = len(tool_parser.prev_tool_call_arr) > 0
index = (
len(tool_parser.prev_tool_call_arr) - 1
if auto_tools_called
else 0
)
should_check = (
self._should_check_for_unstreamed_tool_arg_tokens(
delta_message, output
)
)
# only check if there are any tool calls
# detected by partial parsing
if should_check and tool_parser and auto_tools_called:
latest_delta_len = 0
if (
isinstance(
delta_message.tool_calls[0].function,
DeltaFunctionCall,
)
) and isinstance(
delta_message.tool_calls[0].function.arguments, str
):
latest_delta_len = len(
delta_message.tool_calls[0].function.arguments
)
# get the expected call based on partial JSON
# parsing which "autocompletes" the JSON.
# Tool parsers (e.g. Qwen3Coder) store
# arguments as a JSON string in
# prev_tool_call_arr. Calling json.dumps()
# on an already-serialized string would
# double-serialize it (e.g. '{"k":1}' becomes
# '"{\\"k\\":1}"'), which then causes the
# replace() below to fail and append the
# entire double-serialized string as a
# spurious final delta.
args = tool_parser.prev_tool_call_arr[index].get(
"arguments", {}
)
if isinstance(args, str):
expected_call = args
else:
expected_call = json.dumps(args, ensure_ascii=False)
# get what we've streamed so far for arguments
# for the current tool
actual_call = tool_parser.streamed_args_for_tool[index]
if latest_delta_len > 0:
actual_call = actual_call[:-latest_delta_len]
# check to see if there's anything left to stream
remaining_call = expected_call.replace(actual_call, "", 1)
# set that as a delta message
delta_message = self._create_remaining_args_delta(
delta_message, remaining_call, index
)
# Send the finish response for each request.n only once
# In OpenAI's API, when a tool is called, the
# finish_reason is:
# "tool_calls" for "auto" or "required" tool calls,
# and "stop" for named tool calls.
if (
auto_tools_called
or (tools_streamed[i] and not tool_choice_function_name)
or (self.use_harmony and harmony_tools_streamed[i])
if (tools_streamed[i] and not tool_choice_function_name) or (
self.use_harmony and harmony_tools_streamed[i]
):
finish_reason_ = "tool_calls"
else:
@@ -1535,56 +1465,3 @@ class OpenAIServingChat(OpenAIServing):
and self.enable_auto_tools
and request.tool_choice in ["auto", None]
)
def _should_check_for_unstreamed_tool_arg_tokens(
self,
delta_message: DeltaMessage | None,
output: CompletionOutput,
) -> bool:
"""
Check to see if we should check for unstreamed tool arguments tokens.
This is only applicable when auto tool parsing is enabled, the delta
is a tool call with arguments.
"""
return bool(
# if there is a delta message that includes tool calls which
# include a function that has arguments
output.finish_reason is not None
and self.enable_auto_tools
and self.tool_parser
and delta_message
and delta_message.tool_calls
and delta_message.tool_calls[0]
and delta_message.tool_calls[0].function
and delta_message.tool_calls[0].function.arguments is not None
)
@staticmethod
def _create_remaining_args_delta(
delta_message: DeltaMessage,
remaining_call: str,
index: int,
) -> DeltaMessage:
"""
Create a delta message for remaining tool arguments, preserving
id/type/name from the original delta.
"""
original_tc = next(
(tc for tc in delta_message.tool_calls if tc.index == index),
None,
)
original_fn = original_tc.function if original_tc else None
return DeltaMessage(
tool_calls=[
DeltaToolCall(
index=index,
id=original_tc.id if original_tc else None,
type=original_tc.type if original_tc else None,
function=DeltaFunctionCall(
name=original_fn.name if original_fn else None,
arguments=remaining_call,
),
)
]
)
+21
View File
@@ -320,6 +320,7 @@ class Parser:
delta_token_ids: list[int],
request: ChatCompletionRequest | ResponsesRequest,
prompt_token_ids: list[int] | None = None,
finished: bool = False,
) -> DeltaMessage | None:
"""Parse a single streaming delta, orchestrating reasoning then
tool call extraction via internal stream state.
@@ -656,12 +657,28 @@ class DelegatingParser(Parser):
return False
return state.reasoning_ended
def _append_unstreamed_tool_args(
self,
delta_message: DeltaMessage | None,
) -> None:
"""Append parsed-but-unstreamed tool-call arguments to *delta_message*."""
if (
self._tool_parser is not None
and delta_message
and delta_message.tool_calls
and (last_tc := delta_message.tool_calls[-1]).function
):
last_tc.function.arguments = (
last_tc.function.arguments or ""
) + self._tool_parser.get_remaining_unstreamed_args()
def parse_delta(
self,
delta_text: str,
delta_token_ids: list[int],
request: ChatCompletionRequest | ResponsesRequest,
prompt_token_ids: list[int] | None = None,
finished: bool = False,
) -> DeltaMessage | None:
state = self._stream_state
@@ -745,6 +762,10 @@ class DelegatingParser(Parser):
state.previous_text = current_text
state.previous_token_ids = current_token_ids
if finished:
self._append_unstreamed_tool_args(delta_message)
return delta_message
+19
View File
@@ -79,6 +79,25 @@ class ToolParser:
else:
self.tools = []
def get_remaining_unstreamed_args(self) -> str:
"""Return tool call arguments parsed but not yet streamed."""
if not self.prev_tool_call_arr:
return ""
index = len(self.prev_tool_call_arr) - 1
args = self.prev_tool_call_arr[index].get("arguments", {})
if isinstance(args, str):
expected = args
else:
expected = json.dumps(args, ensure_ascii=False)
actual = (
self.streamed_args_for_tool[index]
if index < len(self.streamed_args_for_tool)
else ""
)
if expected.startswith(actual):
return expected[len(actual) :]
return ""
@cached_property
def vocab(self) -> dict[str, int]:
# NOTE: Only PreTrainedTokenizerFast is guaranteed to have .vocab