[Frontend] Add Streaming Parser Engine and new Qwen3 Parser (#45413)

Signed-off-by: Ben Browning <bbrownin@redhat.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Ben Browning
2026-06-15 11:59:05 +08:00
committed by GitHub
co-authored by Flora Feng
parent e3e3cd5458
commit c4a3f9d137
33 changed files with 8494 additions and 904 deletions
+135 -26
View File
@@ -50,6 +50,31 @@ class StreamState:
# only used for "required" and "named tool" choices,
# tracks whether function name has been fully returned in the stream yet
function_name_returned: bool = False
engine_based: bool = False
def advance(
self,
delta_text: str,
delta_token_ids: list[int],
) -> tuple[str, list[int]]:
if self.engine_based:
return delta_text, delta_token_ids
return (
self.previous_text + delta_text,
self.previous_token_ids + delta_token_ids,
)
def commit(
self,
current_text: str,
current_token_ids: list[int],
) -> None:
if self.engine_based:
self.previous_text = ""
self.previous_token_ids = []
else:
self.previous_text = current_text
self.previous_token_ids = current_token_ids
class Parser:
@@ -88,8 +113,6 @@ class Parser:
self.model_tokenizer = tokenizer
self._reasoning_parser: ReasoningParser | None = None
self._tool_parser: ToolParser | None = None
self._stream_state = StreamState()
if self.__class__.reasoning_parser_cls is not None:
self._reasoning_parser = self.__class__.reasoning_parser_cls(
tokenizer, *args, **kwargs
@@ -97,6 +120,12 @@ class Parser:
if self.__class__.tool_parser_cls is not None:
self._tool_parser = self.__class__.tool_parser_cls(tokenizer, tools)
self._engine_based = (
self._reasoning_parser is None
or self._reasoning_parser.engine_based_streaming
) and (self._tool_parser is None or self._tool_parser.engine_based_streaming)
self._stream_state = StreamState(engine_based=self._engine_based)
@cached_property
def vocab(self) -> dict[str, int]:
"""Get the vocabulary mapping from tokens to IDs."""
@@ -571,11 +600,28 @@ class DelegatingParser(Parser):
tool_call_id_type: str = "random",
function_name_returned: bool = False,
) -> tuple[DeltaMessage | None, bool]:
if request.tool_choice == "none":
return (DeltaMessage(content=delta_text) if delta_text else None), False
assert self._tool_parser is not None
supports_required_and_named = self._tool_parser.supports_required_and_named
if request.tool_choice == "none":
if self._engine_based:
# Engine-backed parsers route content extraction through
# extract_tool_calls_streaming, so run the full pipeline
# and strip tool_calls after.
delta_message = self.extract_tool_calls_streaming(
previous_text,
current_text,
delta_text,
previous_token_ids,
current_token_ids,
delta_token_ids,
request, # type: ignore[arg-type]
)
if delta_message:
delta_message.tool_calls = []
return delta_message, False
return (DeltaMessage(content=delta_text) if delta_text else None), False
if (
supports_required_and_named
and request.tool_choice
@@ -713,9 +759,9 @@ class DelegatingParser(Parser):
):
state.reasoning_ended = True
current_text = state.previous_text + delta_text
current_token_ids = state.previous_token_ids + delta_token_ids
current_text, current_token_ids = state.advance(delta_text, delta_token_ids)
delta_message: DeltaMessage | None = None
reasoning_transitioned = False
# Reasoning extraction
if self._in_reasoning_phase(state):
@@ -727,16 +773,34 @@ class DelegatingParser(Parser):
current_token_ids=current_token_ids,
delta_token_ids=delta_token_ids,
)
if self.is_reasoning_end_streaming(current_token_ids, delta_token_ids):
state.reasoning_ended = True
current_token_ids = self.extract_content_ids(delta_token_ids)
current_text = (
delta_message.content
if delta_message and delta_message.content
else ""
reasoning_parser = self._reasoning_parser
if reasoning_parser is not None and reasoning_parser.engine_based_streaming:
should_transition = (
reasoning_parser.has_engine_confirmed_reasoning_end()
)
delta_text = current_text
delta_token_ids = current_token_ids
else:
should_transition = self.is_reasoning_end_streaming(
current_token_ids, delta_token_ids
)
if should_transition:
state.reasoning_ended = True
reasoning_transitioned = True
current_token_ids = self.extract_content_ids(delta_token_ids)
if self._engine_based:
current_text = (
self.model_tokenizer.decode(current_token_ids)
if current_token_ids
else ""
)
if delta_message and self._tool_parser is not None:
delta_message.content = None
else:
current_text = (
delta_message.content
if delta_message and delta_message.content
else ""
)
delta_text = current_text
# Tool call extraction
if self._in_tool_call_phase(state):
@@ -747,9 +811,10 @@ class DelegatingParser(Parser):
delta_text = current_text
delta_token_ids = current_token_ids
# A boundary delta may carry both reasoning and tool call,
# save it before the tool parser overwrites delta_message.
reasoning = delta_message.reasoning if delta_message else None
reasoning_from_this_batch = (
delta_message.reasoning if delta_message else None
)
delta_message, state.function_name_returned = (
self._extract_tool_calls_streaming(
previous_text=state.previous_text,
@@ -764,10 +829,12 @@ class DelegatingParser(Parser):
function_name_returned=state.function_name_returned,
)
)
if reasoning:
if not delta_message:
delta_message = DeltaMessage()
delta_message.reasoning = reasoning
if reasoning_from_this_batch:
if delta_message is None:
delta_message = DeltaMessage(reasoning=reasoning_from_this_batch)
elif not delta_message.reasoning:
delta_message.reasoning = reasoning_from_this_batch
if (
delta_message
@@ -776,18 +843,60 @@ class DelegatingParser(Parser):
):
state.history_tool_call_cnt += 1
# No phase active: pass through as content
# No phase active: pass through as content.
# Skip when reasoning just ended in this delta — the engine already
# consumed the end-of-reasoning marker (e.g. </think>) and
# delta_text still contains the raw marker text.
if (
delta_message is None
and not reasoning_transitioned
and not self._in_reasoning_phase(state)
and not self._in_tool_call_phase(state)
):
delta_message = DeltaMessage(content=delta_text)
state.previous_text = current_text
state.previous_token_ids = current_token_ids
state.commit(current_text, current_token_ids)
if finished:
delta_message = self.finalize_generation(delta_message, request, state)
delta_message = self._flush_engine_parsers(delta_message)
return delta_message
def _flush_engine_parsers(
self, delta_message: DeltaMessage | None
) -> DeltaMessage | None:
"""Flush buffered state from engine-based parsers at stream end."""
reasoning_ended = self._stream_state.reasoning_ended
for parser in (self._reasoning_parser, self._tool_parser):
if not getattr(parser, "engine_based_streaming", False):
continue
# When reasoning has ended and we transitioned to the tool
# phase, the reasoning parser's engine may still have buffered
# characters from tool-call markup it saw with
# skip_tool_parsing=True. Flushing that would leak spurious
# content (e.g. a stray '"'), so skip it.
if parser is self._reasoning_parser and reasoning_ended:
continue
finish = getattr(parser, "finish_streaming", None)
if finish is None:
continue
flush_delta = finish()
if flush_delta is None:
continue
if delta_message is None:
delta_message = flush_delta
else:
if flush_delta.content:
delta_message.content = (
delta_message.content or ""
) + flush_delta.content
if flush_delta.reasoning:
delta_message.reasoning = (
delta_message.reasoning or ""
) + flush_delta.reasoning
if flush_delta.tool_calls:
delta_message.tool_calls = (
delta_message.tool_calls or []
) + flush_delta.tool_calls
return delta_message