[Refactor] Move unstreamed tool-arg flush from serving layer to parser (#44017)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -3,7 +3,6 @@
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import json
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, AsyncIterator
|
||||
from collections.abc import Sequence as GenericSequence
|
||||
@@ -40,9 +39,7 @@ from vllm.entrypoints.openai.chat_completion.stream_harmony import (
|
||||
extract_harmony_streaming_delta,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
DeltaToolCall,
|
||||
ErrorResponse,
|
||||
FunctionCall,
|
||||
PromptTokenUsageInfo,
|
||||
@@ -65,7 +62,7 @@ from vllm.entrypoints.utils import get_max_tokens, should_include_usage
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.parser import ParserManager
|
||||
from vllm.parser.abstract_parser import Parser
|
||||
from vllm.reasoning import ReasoningParser
|
||||
@@ -715,6 +712,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
delta_token_ids=as_list(output.token_ids),
|
||||
request=request,
|
||||
prompt_token_ids=res.prompt_token_ids,
|
||||
finished=output.finish_reason is not None,
|
||||
)
|
||||
if delta_message and delta_message.tool_calls:
|
||||
tools_streamed[i] = True
|
||||
@@ -805,81 +803,13 @@ class OpenAIServingChat(OpenAIServing):
|
||||
# finish_reason='error' indicates a retryable error
|
||||
self._raise_if_error(output.finish_reason, request_id)
|
||||
|
||||
# check to make sure we haven't "forgotten" to stream
|
||||
# any tokens that were generated but previously
|
||||
# matched by partial json parsing
|
||||
# only happens if we are NOT using structured outputs
|
||||
index = 0
|
||||
auto_tools_called = False
|
||||
if tool_parser:
|
||||
auto_tools_called = len(tool_parser.prev_tool_call_arr) > 0
|
||||
index = (
|
||||
len(tool_parser.prev_tool_call_arr) - 1
|
||||
if auto_tools_called
|
||||
else 0
|
||||
)
|
||||
should_check = (
|
||||
self._should_check_for_unstreamed_tool_arg_tokens(
|
||||
delta_message, output
|
||||
)
|
||||
)
|
||||
# only check if there are any tool calls
|
||||
# detected by partial parsing
|
||||
if should_check and tool_parser and auto_tools_called:
|
||||
latest_delta_len = 0
|
||||
if (
|
||||
isinstance(
|
||||
delta_message.tool_calls[0].function,
|
||||
DeltaFunctionCall,
|
||||
)
|
||||
) and isinstance(
|
||||
delta_message.tool_calls[0].function.arguments, str
|
||||
):
|
||||
latest_delta_len = len(
|
||||
delta_message.tool_calls[0].function.arguments
|
||||
)
|
||||
|
||||
# get the expected call based on partial JSON
|
||||
# parsing which "autocompletes" the JSON.
|
||||
# Tool parsers (e.g. Qwen3Coder) store
|
||||
# arguments as a JSON string in
|
||||
# prev_tool_call_arr. Calling json.dumps()
|
||||
# on an already-serialized string would
|
||||
# double-serialize it (e.g. '{"k":1}' becomes
|
||||
# '"{\\"k\\":1}"'), which then causes the
|
||||
# replace() below to fail and append the
|
||||
# entire double-serialized string as a
|
||||
# spurious final delta.
|
||||
args = tool_parser.prev_tool_call_arr[index].get(
|
||||
"arguments", {}
|
||||
)
|
||||
if isinstance(args, str):
|
||||
expected_call = args
|
||||
else:
|
||||
expected_call = json.dumps(args, ensure_ascii=False)
|
||||
|
||||
# get what we've streamed so far for arguments
|
||||
# for the current tool
|
||||
actual_call = tool_parser.streamed_args_for_tool[index]
|
||||
if latest_delta_len > 0:
|
||||
actual_call = actual_call[:-latest_delta_len]
|
||||
|
||||
# check to see if there's anything left to stream
|
||||
remaining_call = expected_call.replace(actual_call, "", 1)
|
||||
# set that as a delta message
|
||||
delta_message = self._create_remaining_args_delta(
|
||||
delta_message, remaining_call, index
|
||||
)
|
||||
|
||||
# Send the finish response for each request.n only once
|
||||
# In OpenAI's API, when a tool is called, the
|
||||
# finish_reason is:
|
||||
# "tool_calls" for "auto" or "required" tool calls,
|
||||
# and "stop" for named tool calls.
|
||||
if (
|
||||
auto_tools_called
|
||||
or (tools_streamed[i] and not tool_choice_function_name)
|
||||
or (self.use_harmony and harmony_tools_streamed[i])
|
||||
if (tools_streamed[i] and not tool_choice_function_name) or (
|
||||
self.use_harmony and harmony_tools_streamed[i]
|
||||
):
|
||||
finish_reason_ = "tool_calls"
|
||||
else:
|
||||
@@ -1535,56 +1465,3 @@ class OpenAIServingChat(OpenAIServing):
|
||||
and self.enable_auto_tools
|
||||
and request.tool_choice in ["auto", None]
|
||||
)
|
||||
|
||||
def _should_check_for_unstreamed_tool_arg_tokens(
|
||||
self,
|
||||
delta_message: DeltaMessage | None,
|
||||
output: CompletionOutput,
|
||||
) -> bool:
|
||||
"""
|
||||
Check to see if we should check for unstreamed tool arguments tokens.
|
||||
This is only applicable when auto tool parsing is enabled, the delta
|
||||
is a tool call with arguments.
|
||||
"""
|
||||
|
||||
return bool(
|
||||
# if there is a delta message that includes tool calls which
|
||||
# include a function that has arguments
|
||||
output.finish_reason is not None
|
||||
and self.enable_auto_tools
|
||||
and self.tool_parser
|
||||
and delta_message
|
||||
and delta_message.tool_calls
|
||||
and delta_message.tool_calls[0]
|
||||
and delta_message.tool_calls[0].function
|
||||
and delta_message.tool_calls[0].function.arguments is not None
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _create_remaining_args_delta(
|
||||
delta_message: DeltaMessage,
|
||||
remaining_call: str,
|
||||
index: int,
|
||||
) -> DeltaMessage:
|
||||
"""
|
||||
Create a delta message for remaining tool arguments, preserving
|
||||
id/type/name from the original delta.
|
||||
"""
|
||||
original_tc = next(
|
||||
(tc for tc in delta_message.tool_calls if tc.index == index),
|
||||
None,
|
||||
)
|
||||
original_fn = original_tc.function if original_tc else None
|
||||
return DeltaMessage(
|
||||
tool_calls=[
|
||||
DeltaToolCall(
|
||||
index=index,
|
||||
id=original_tc.id if original_tc else None,
|
||||
type=original_tc.type if original_tc else None,
|
||||
function=DeltaFunctionCall(
|
||||
name=original_fn.name if original_fn else None,
|
||||
arguments=remaining_call,
|
||||
),
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
@@ -320,6 +320,7 @@ class Parser:
|
||||
delta_token_ids: list[int],
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
finished: bool = False,
|
||||
) -> DeltaMessage | None:
|
||||
"""Parse a single streaming delta, orchestrating reasoning then
|
||||
tool call extraction via internal stream state.
|
||||
@@ -656,12 +657,28 @@ class DelegatingParser(Parser):
|
||||
return False
|
||||
return state.reasoning_ended
|
||||
|
||||
def _append_unstreamed_tool_args(
|
||||
self,
|
||||
delta_message: DeltaMessage | None,
|
||||
) -> None:
|
||||
"""Append parsed-but-unstreamed tool-call arguments to *delta_message*."""
|
||||
if (
|
||||
self._tool_parser is not None
|
||||
and delta_message
|
||||
and delta_message.tool_calls
|
||||
and (last_tc := delta_message.tool_calls[-1]).function
|
||||
):
|
||||
last_tc.function.arguments = (
|
||||
last_tc.function.arguments or ""
|
||||
) + self._tool_parser.get_remaining_unstreamed_args()
|
||||
|
||||
def parse_delta(
|
||||
self,
|
||||
delta_text: str,
|
||||
delta_token_ids: list[int],
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
finished: bool = False,
|
||||
) -> DeltaMessage | None:
|
||||
state = self._stream_state
|
||||
|
||||
@@ -745,6 +762,10 @@ class DelegatingParser(Parser):
|
||||
|
||||
state.previous_text = current_text
|
||||
state.previous_token_ids = current_token_ids
|
||||
|
||||
if finished:
|
||||
self._append_unstreamed_tool_args(delta_message)
|
||||
|
||||
return delta_message
|
||||
|
||||
|
||||
|
||||
@@ -79,6 +79,25 @@ class ToolParser:
|
||||
else:
|
||||
self.tools = []
|
||||
|
||||
def get_remaining_unstreamed_args(self) -> str:
|
||||
"""Return tool call arguments parsed but not yet streamed."""
|
||||
if not self.prev_tool_call_arr:
|
||||
return ""
|
||||
index = len(self.prev_tool_call_arr) - 1
|
||||
args = self.prev_tool_call_arr[index].get("arguments", {})
|
||||
if isinstance(args, str):
|
||||
expected = args
|
||||
else:
|
||||
expected = json.dumps(args, ensure_ascii=False)
|
||||
actual = (
|
||||
self.streamed_args_for_tool[index]
|
||||
if index < len(self.streamed_args_for_tool)
|
||||
else ""
|
||||
)
|
||||
if expected.startswith(actual):
|
||||
return expected[len(actual) :]
|
||||
return ""
|
||||
|
||||
@cached_property
|
||||
def vocab(self) -> dict[str, int]:
|
||||
# NOTE: Only PreTrainedTokenizerFast is guaranteed to have .vocab
|
||||
|
||||
Reference in New Issue
Block a user