[Parser] Migrate response api streaming to unified parser (#38755)
Signed-off-by: sfeng33 <4florafeng@gmail.com> Signed-off-by: Andrew Xia <axia@meta.com>
This commit is contained in:
@@ -5,6 +5,7 @@ import contextlib
|
||||
import json
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from functools import cached_property
|
||||
|
||||
from openai.types.responses import (
|
||||
@@ -43,6 +44,17 @@ from vllm.utils import random_uuid
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class StreamState:
|
||||
"""Mutable state for ``Parser.parse_delta()``. One per stream."""
|
||||
|
||||
reasoning_ended: bool = False
|
||||
tool_call_text_started: bool = False
|
||||
prompt_reasoning_checked: bool = False
|
||||
previous_text: str = ""
|
||||
previous_token_ids: list[int] = field(default_factory=list)
|
||||
|
||||
|
||||
class Parser:
|
||||
"""
|
||||
Abstract Parser class that unifies ReasoningParser and ToolParser into
|
||||
@@ -80,6 +92,7 @@ class Parser:
|
||||
self.model_tokenizer = tokenizer
|
||||
self._reasoning_parser: ReasoningParser | None = None
|
||||
self._tool_parser: ToolParser | None = None
|
||||
self._stream_state = StreamState()
|
||||
|
||||
@cached_property
|
||||
def vocab(self) -> dict[str, int]:
|
||||
@@ -291,6 +304,18 @@ class Parser:
|
||||
A DeltaMessage with tool_calls field, or None.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def parse_delta(
|
||||
self,
|
||||
delta_text: str,
|
||||
delta_token_ids: list[int],
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
) -> DeltaMessage | None:
|
||||
"""Parse a single streaming delta, orchestrating reasoning then
|
||||
tool call extraction via internal stream state.
|
||||
"""
|
||||
|
||||
|
||||
class DelegatingParser(Parser):
|
||||
"""
|
||||
@@ -524,6 +549,100 @@ class DelegatingParser(Parser):
|
||||
request,
|
||||
)
|
||||
|
||||
def is_reasoning_end(self, input_ids: list[int]) -> bool:
|
||||
if self._reasoning_parser is None:
|
||||
return False
|
||||
return self._reasoning_parser.is_reasoning_end(input_ids)
|
||||
|
||||
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
|
||||
if self._reasoning_parser is None:
|
||||
return input_ids
|
||||
return self._reasoning_parser.extract_content_ids(input_ids)
|
||||
|
||||
def parse_delta(
|
||||
self,
|
||||
delta_text: str,
|
||||
delta_token_ids: list[int],
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
) -> DeltaMessage | None:
|
||||
state = self._stream_state
|
||||
|
||||
if not state.prompt_reasoning_checked and prompt_token_ids is not None:
|
||||
state.prompt_reasoning_checked = True
|
||||
if self.is_reasoning_end(prompt_token_ids):
|
||||
state.reasoning_ended = True
|
||||
|
||||
current_text = state.previous_text + delta_text
|
||||
current_token_ids = state.previous_token_ids + delta_token_ids
|
||||
|
||||
delta_message: DeltaMessage | None = None
|
||||
|
||||
if self._reasoning_parser and self._tool_parser:
|
||||
if not state.reasoning_ended:
|
||||
delta_message = self.extract_reasoning_streaming(
|
||||
previous_text=state.previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=delta_text,
|
||||
previous_token_ids=state.previous_token_ids,
|
||||
current_token_ids=current_token_ids,
|
||||
delta_token_ids=delta_token_ids,
|
||||
)
|
||||
if self.is_reasoning_end(delta_token_ids):
|
||||
state.reasoning_ended = True
|
||||
current_token_ids = self.extract_content_ids(delta_token_ids)
|
||||
if delta_message and delta_message.content:
|
||||
current_text = delta_message.content
|
||||
delta_message.content = None
|
||||
else:
|
||||
current_text = ""
|
||||
|
||||
if state.reasoning_ended:
|
||||
if not state.tool_call_text_started:
|
||||
state.tool_call_text_started = True
|
||||
state.previous_text = ""
|
||||
state.previous_token_ids = []
|
||||
delta_text = current_text
|
||||
delta_token_ids = current_token_ids
|
||||
|
||||
delta_message = self.extract_tool_calls_streaming(
|
||||
previous_text=state.previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=delta_text,
|
||||
previous_token_ids=state.previous_token_ids,
|
||||
current_token_ids=current_token_ids,
|
||||
delta_token_ids=delta_token_ids,
|
||||
request=request, # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
elif self._reasoning_parser:
|
||||
delta_message = self.extract_reasoning_streaming(
|
||||
previous_text=state.previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=delta_text,
|
||||
previous_token_ids=state.previous_token_ids,
|
||||
current_token_ids=current_token_ids,
|
||||
delta_token_ids=delta_token_ids,
|
||||
)
|
||||
|
||||
elif self._tool_parser:
|
||||
delta_message = self.extract_tool_calls_streaming(
|
||||
previous_text=state.previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=delta_text,
|
||||
previous_token_ids=state.previous_token_ids,
|
||||
current_token_ids=current_token_ids,
|
||||
delta_token_ids=delta_token_ids,
|
||||
request=request, # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
else:
|
||||
delta_message = DeltaMessage(content=delta_text)
|
||||
|
||||
state.previous_text = current_text
|
||||
state.previous_token_ids = current_token_ids
|
||||
return delta_message
|
||||
|
||||
|
||||
class _WrappedParser(DelegatingParser):
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user