[Parser] Migrate response api streaming to unified parser (#38755)

Signed-off-by: sfeng33 <4florafeng@gmail.com>
Signed-off-by: Andrew Xia <axia@meta.com>
This commit is contained in:
Flora Feng
2026-04-08 10:09:00 +08:00
committed by GitHub
parent 9ea7d670d8
commit 927975ead8
3 changed files with 153 additions and 119 deletions
+119
View File
@@ -5,6 +5,7 @@ import contextlib
import json
from abc import abstractmethod
from collections.abc import Sequence
from dataclasses import dataclass, field
from functools import cached_property
from openai.types.responses import (
@@ -43,6 +44,17 @@ from vllm.utils import random_uuid
logger = init_logger(__name__)
@dataclass
class StreamState:
"""Mutable state for ``Parser.parse_delta()``. One per stream."""
reasoning_ended: bool = False
tool_call_text_started: bool = False
prompt_reasoning_checked: bool = False
previous_text: str = ""
previous_token_ids: list[int] = field(default_factory=list)
class Parser:
"""
Abstract Parser class that unifies ReasoningParser and ToolParser into
@@ -80,6 +92,7 @@ class Parser:
self.model_tokenizer = tokenizer
self._reasoning_parser: ReasoningParser | None = None
self._tool_parser: ToolParser | None = None
self._stream_state = StreamState()
@cached_property
def vocab(self) -> dict[str, int]:
@@ -291,6 +304,18 @@ class Parser:
A DeltaMessage with tool_calls field, or None.
"""
@abstractmethod
def parse_delta(
self,
delta_text: str,
delta_token_ids: list[int],
request: ChatCompletionRequest | ResponsesRequest,
prompt_token_ids: list[int] | None = None,
) -> DeltaMessage | None:
"""Parse a single streaming delta, orchestrating reasoning then
tool call extraction via internal stream state.
"""
class DelegatingParser(Parser):
"""
@@ -524,6 +549,100 @@ class DelegatingParser(Parser):
request,
)
def is_reasoning_end(self, input_ids: list[int]) -> bool:
if self._reasoning_parser is None:
return False
return self._reasoning_parser.is_reasoning_end(input_ids)
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
if self._reasoning_parser is None:
return input_ids
return self._reasoning_parser.extract_content_ids(input_ids)
def parse_delta(
self,
delta_text: str,
delta_token_ids: list[int],
request: ChatCompletionRequest | ResponsesRequest,
prompt_token_ids: list[int] | None = None,
) -> DeltaMessage | None:
state = self._stream_state
if not state.prompt_reasoning_checked and prompt_token_ids is not None:
state.prompt_reasoning_checked = True
if self.is_reasoning_end(prompt_token_ids):
state.reasoning_ended = True
current_text = state.previous_text + delta_text
current_token_ids = state.previous_token_ids + delta_token_ids
delta_message: DeltaMessage | None = None
if self._reasoning_parser and self._tool_parser:
if not state.reasoning_ended:
delta_message = self.extract_reasoning_streaming(
previous_text=state.previous_text,
current_text=current_text,
delta_text=delta_text,
previous_token_ids=state.previous_token_ids,
current_token_ids=current_token_ids,
delta_token_ids=delta_token_ids,
)
if self.is_reasoning_end(delta_token_ids):
state.reasoning_ended = True
current_token_ids = self.extract_content_ids(delta_token_ids)
if delta_message and delta_message.content:
current_text = delta_message.content
delta_message.content = None
else:
current_text = ""
if state.reasoning_ended:
if not state.tool_call_text_started:
state.tool_call_text_started = True
state.previous_text = ""
state.previous_token_ids = []
delta_text = current_text
delta_token_ids = current_token_ids
delta_message = self.extract_tool_calls_streaming(
previous_text=state.previous_text,
current_text=current_text,
delta_text=delta_text,
previous_token_ids=state.previous_token_ids,
current_token_ids=current_token_ids,
delta_token_ids=delta_token_ids,
request=request, # type: ignore[arg-type]
)
elif self._reasoning_parser:
delta_message = self.extract_reasoning_streaming(
previous_text=state.previous_text,
current_text=current_text,
delta_text=delta_text,
previous_token_ids=state.previous_token_ids,
current_token_ids=current_token_ids,
delta_token_ids=delta_token_ids,
)
elif self._tool_parser:
delta_message = self.extract_tool_calls_streaming(
previous_text=state.previous_text,
current_text=current_text,
delta_text=delta_text,
previous_token_ids=state.previous_token_ids,
current_token_ids=current_token_ids,
delta_token_ids=delta_token_ids,
request=request, # type: ignore[arg-type]
)
else:
delta_message = DeltaMessage(content=delta_text)
state.previous_text = current_text
state.previous_token_ids = current_token_ids
return delta_message
class _WrappedParser(DelegatingParser):
"""