[Frontend] Add Streaming Parser Engine and new Qwen3 Parser (#45413)

Signed-off-by: Ben Browning <bbrownin@redhat.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Ben Browning
2026-06-15 11:59:05 +08:00
committed by GitHub
co-authored by Flora Feng
parent e3e3cd5458
commit c4a3f9d137
33 changed files with 8494 additions and 904 deletions
+17
View File
@@ -0,0 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Streaming parser engine framework for tool call and reasoning extraction.
Instead of hand-rolling a parser for every model's tool-call / reasoning
format, each format is declared as a ParserEngineConfig (terminals,
states, and transitions) and a shared incremental engine handles
streaming, ambiguity buffering, token-ID mapping, and delta computation.
"""
from vllm.parser.engine.events import EventType, SemanticEvent
__all__ = [
"EventType",
"SemanticEvent",
]
+199
View File
@@ -0,0 +1,199 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Adapters that expose :class:`ParserEngine` through the legacy
:class:`ReasoningParser` and :class:`ToolParser` interfaces.
This lets parser engines flow through the existing serving-layer code
paths that expect separate reasoning and tool parser instances, without
any changes to the serving layer itself.
"""
from __future__ import annotations
from collections.abc import Iterator, Sequence
from contextlib import contextmanager
from typing import TYPE_CHECKING
from vllm.parser.engine.parser_engine_config import ParserState
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
from vllm.tool_parsers.abstract_tool_parser import ToolParser
if TYPE_CHECKING:
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.entrypoints.openai.engine.protocol import (
DeltaMessage,
ExtractedToolCallInformation,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.parser.engine.parser_engine import ParserEngine
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.utils import Tool
class ParserEngineReasoningAdapter(ReasoningParser):
"""Adapts a :class:`ParserEngine` to the :class:`ReasoningParser`
interface so parser engines can be used as reasoning parsers in the
existing serving code.
Subclasses set :attr:`_parser_engine_cls` to the concrete
:class:`ParserEngine` class.
"""
_parser_engine_cls: type[ParserEngine]
engine_based_streaming: bool = True
def __init__(self, tokenizer: TokenizerLike, *args, **kwargs) -> None:
super().__init__(tokenizer, *args, **kwargs)
self._parser_engine = self._parser_engine_cls(tokenizer, **kwargs) # type: ignore[call-arg]
@contextmanager
def _skip_tool_parsing(self) -> Iterator[None]:
saved = self._parser_engine.skip_tool_parsing
self._parser_engine.skip_tool_parsing = True
try:
yield
finally:
self._parser_engine.skip_tool_parsing = saved
def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
return self._parser_engine.is_reasoning_end(list(input_ids))
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
return self._parser_engine.extract_content_ids(input_ids)
def extract_reasoning(
self,
model_output: str,
request: ChatCompletionRequest | ResponsesRequest,
) -> tuple[str | None, str | None]:
with self._skip_tool_parsing():
return self._parser_engine.extract_reasoning(model_output, request)
def extract_reasoning_streaming(
self,
previous_text: str,
current_text: str,
delta_text: str,
previous_token_ids: Sequence[int],
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
) -> DeltaMessage | None:
with self._skip_tool_parsing():
return self._parser_engine.extract_reasoning_streaming(
previous_text,
current_text,
delta_text,
previous_token_ids,
current_token_ids,
delta_token_ids,
)
@property
def reasoning_start_str(self) -> str | None:
return self._parser_engine.reasoning_start_str
@property
def reasoning_end_str(self) -> str | None:
return self._parser_engine.reasoning_end_str
def adjust_request(
self,
request: ChatCompletionRequest | ResponsesRequest,
) -> ChatCompletionRequest | ResponsesRequest:
return self._parser_engine.adjust_request(request)
def has_engine_confirmed_reasoning_end(self) -> bool:
return self._parser_engine.reasoning_ended
def finish_streaming(self) -> DeltaMessage | None:
return self._parser_engine.finish_streaming()
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
return self._parser_engine.count_reasoning_tokens(token_ids)
class ParserEngineToolAdapter(ToolParser):
"""Adapts a :class:`ParserEngine` to the :class:`ToolParser` interface.
:meth:`extract_tool_calls` starts the parser engine in ``CONTENT``
state so it can parse reasoning-stripped content (i.e. the output of
:meth:`ReasoningParser.extract_reasoning`).
Subclasses set :attr:`_parser_engine_cls` to the concrete
:class:`ParserEngine` class.
"""
_parser_engine_cls: type[ParserEngine]
engine_based_streaming: bool = True
def __init__(
self,
tokenizer: TokenizerLike,
tools: list[Tool] | None = None,
**kwargs,
) -> None:
super().__init__(tokenizer, tools)
self._parser_engine = self._parser_engine_cls(tokenizer, tools, **kwargs) # type: ignore[call-arg]
def adjust_request(
self,
request: ChatCompletionRequest | ResponsesRequest,
) -> ChatCompletionRequest | ResponsesRequest:
request = super().adjust_request(request)
return self._parser_engine.adjust_request(request)
def extract_tool_calls(
self,
model_output: str,
request: ChatCompletionRequest,
) -> ExtractedToolCallInformation:
return self._parser_engine.extract_tool_calls_from_content(
model_output, request
)
def extract_tool_calls_streaming(
self,
previous_text: str,
current_text: str,
delta_text: str,
previous_token_ids: Sequence[int],
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> DeltaMessage | None:
engine = self._parser_engine
engine.initialize_streaming(initial_state=ParserState.CONTENT)
return engine.extract_tool_calls_streaming(
previous_text,
current_text,
delta_text,
previous_token_ids,
current_token_ids,
delta_token_ids,
request,
)
def finish_streaming(self) -> DeltaMessage | None:
return self._parser_engine.finish_streaming()
def make_adapters(
parser_engine_cls: type[ParserEngine],
) -> tuple[type[ParserEngineReasoningAdapter], type[ParserEngineToolAdapter]]:
reasoning_adapter = type(
f"{parser_engine_cls.__name__}ReasoningAdapter",
(ParserEngineReasoningAdapter,),
{"_parser_engine_cls": parser_engine_cls},
)
tool_adapter = type(
f"{parser_engine_cls.__name__}ToolAdapter",
(ParserEngineToolAdapter,),
{"_parser_engine_cls": parser_engine_cls},
)
# Let the serving layer find the adapters and call adjust_request(),
# which sets skip_special_tokens=False for the detokenizer.
parser_engine_cls.reasoning_parser_cls = reasoning_adapter # type: ignore[attr-defined]
parser_engine_cls.tool_parser_cls = tool_adapter # type: ignore[attr-defined]
return reasoning_adapter, tool_adapter
+26
View File
@@ -0,0 +1,26 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Semantic event types emitted by the streaming parser engine."""
from __future__ import annotations
from dataclasses import dataclass
from enum import Enum, auto
class EventType(Enum):
TEXT_CHUNK = auto()
REASONING_START = auto()
REASONING_CHUNK = auto()
REASONING_END = auto()
TOOL_CALL_START = auto()
TOOL_NAME = auto()
ARG_VALUE_CHUNK = auto()
TOOL_CALL_END = auto()
@dataclass(slots=True)
class SemanticEvent:
type: EventType
value: str = ""
tool_index: int = -1
+210
View File
@@ -0,0 +1,210 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Incremental text lexer that converts text chunks into terminal
tokens, with prefix-match buffering for ambiguous boundaries."""
from __future__ import annotations
from dataclasses import dataclass
import regex as re
CONTENT_TERMINAL = "__CONTENT__"
@dataclass(slots=True)
class TerminalDef:
name: str
pattern: re.Pattern[str]
is_literal: bool = False
literal: str = ""
@dataclass(slots=True)
class LexToken:
terminal: str
value: str
class LexerShape:
"""Immutable pre-computed data derived from terminal definitions.
Created once per :class:`ParserEngineConfig` and shared across all
:class:`IncrementalLexer` instances that use the same config.
"""
__slots__ = (
"terminals",
"literal_strings",
"max_literal_len",
"literal_first_chars",
"has_only_literals",
"prefix_set",
"literals_by_first",
)
def __init__(self, terminals: list[TerminalDef]) -> None:
self.terminals = sorted(
terminals,
key=lambda t: (not t.is_literal, -len(t.pattern.pattern)),
)
literal_strings: list[tuple[str, str]] = []
for t in self.terminals:
if t.is_literal:
literal_strings.append((t.literal, t.name))
self.literal_strings = literal_strings
max_len = 0
for lit, _ in literal_strings:
if len(lit) > max_len:
max_len = len(lit)
self.max_literal_len = max_len
self.literal_first_chars = frozenset(
lit[0] for lit, _ in literal_strings if lit
)
self.has_only_literals = all(t.is_literal for t in terminals)
prefix_set: set[str] = set()
for lit, _ in literal_strings:
for i in range(1, len(lit)):
prefix_set.add(lit[:i])
self.prefix_set = frozenset(prefix_set)
by_first: dict[str, list[tuple[str, str]]] = {}
for lit, name in literal_strings:
if lit:
by_first.setdefault(lit[0], []).append((lit, name))
self.literals_by_first = by_first
class IncrementalLexer:
"""Converts streaming text into terminal tokens.
The key feature is **prefix-match buffering**: when the text in the
buffer could be the start of a multi-character terminal (e.g.
``"<tool_"`` that could become ``"<tool_call>"``), the lexer holds
the text rather than emitting it. When the next chunk arrives, it
either completes the terminal or flushes the buffered text as
content.
Terminals are tried in priority order (literals first, then by
descending priority, then by pattern length).
"""
def __init__(
self,
terminals: list[TerminalDef] | LexerShape,
content_terminal: str = CONTENT_TERMINAL,
) -> None:
if isinstance(terminals, LexerShape):
shape = terminals
else:
shape = LexerShape(terminals)
self._shape = shape
self.terminals = shape.terminals
self.content_terminal = content_terminal
self.buffer = ""
self._literal_strings = shape.literal_strings
self._max_literal_len = shape.max_literal_len
self._literal_first_chars = shape.literal_first_chars
self._has_only_literals = shape.has_only_literals
self._prefix_set = shape.prefix_set
self._literals_by_first = shape.literals_by_first
def reset(self) -> None:
self.buffer = ""
def feed(self, text: str) -> list[LexToken]:
if not self.buffer and self._has_only_literals and self._literal_first_chars:
for ch in text:
if ch in self._literal_first_chars:
break
else:
return [LexToken(self.content_terminal, text)]
self.buffer += text
return self._drain()
def flush(self) -> list[LexToken]:
tokens: list[LexToken] = []
if self.buffer:
tokens.append(LexToken(self.content_terminal, self.buffer))
self.buffer = ""
return tokens
def _drain(self) -> list[LexToken]:
tokens: list[LexToken] = []
first_chars = self._literal_first_chars
content_terminal = self.content_terminal
has_only_literals = self._has_only_literals
literals_by_first = self._literals_by_first
prefix_set = self._prefix_set
while self.buffer:
if has_only_literals and first_chars:
has_potential = False
for ch in self.buffer:
if ch in first_chars:
has_potential = True
break
if not has_potential:
tokens.append(LexToken(content_terminal, self.buffer))
self.buffer = ""
break
best_match: tuple[str, str, int] | None = None
first = self.buffer[0]
for lit, name in literals_by_first.get(first, ()):
if self.buffer.startswith(lit) and (
best_match is None or len(lit) > best_match[2]
):
best_match = (name, lit, len(lit))
if self.buffer in prefix_set:
if best_match is not None:
tokens.append(LexToken(best_match[0], best_match[1]))
self.buffer = self.buffer[best_match[2] :]
continue
else:
break
if best_match is not None:
tokens.append(LexToken(best_match[0], best_match[1]))
self.buffer = self.buffer[best_match[2] :]
else:
content_end = self._find_content_boundary()
if content_end > 0:
tokens.append(LexToken(content_terminal, self.buffer[:content_end]))
self.buffer = self.buffer[content_end:]
else:
tokens.append(LexToken(content_terminal, self.buffer[0]))
self.buffer = self.buffer[1:]
return tokens
def _find_content_boundary(self) -> int:
buf = self.buffer
n = len(buf)
first_chars = self._literal_first_chars
for i in range(1, n):
if buf[i] not in first_chars:
continue
remaining = n - i
for lit, _ in self._literal_strings:
check_len = min(remaining, len(lit))
if buf[i : i + check_len] == lit[:check_len]:
return i
return n
def terminals_from_literals(literals: dict[str, str]) -> list[TerminalDef]:
return [
TerminalDef(
name=name,
pattern=re.compile(re.escape(lit)),
is_literal=True,
literal=lit,
)
for name, lit in literals.items()
]
+969
View File
@@ -0,0 +1,969 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Parser engine base that handles both reasoning and tool call
extraction with a single :class:`StreamingParserEngine`.
"""
from __future__ import annotations
import json
from collections.abc import Sequence
from functools import cached_property
from typing import TYPE_CHECKING
import regex as re
from vllm.entrypoints.chat_utils import make_tool_call_id
from vllm.entrypoints.openai.engine.protocol import (
DeltaFunctionCall,
DeltaMessage,
DeltaToolCall,
ExtractedToolCallInformation,
FunctionCall,
ToolCall,
)
from vllm.logger import init_logger
from vllm.parser.abstract_parser import Parser, StreamState
from vllm.parser.engine.events import EventType, SemanticEvent
from vllm.parser.engine.parser_engine_config import ParserEngineConfig, ParserState
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
from vllm.tool_parsers.utils import (
coerce_to_schema_type,
extract_types_from_schema,
find_tool_properties,
)
if TYPE_CHECKING:
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.abstract_tool_parser import Tool
logger = init_logger(__name__)
class ToolCallSlot:
__slots__ = (
"id",
"name",
"_args_parts",
"_args_joined",
"name_sent",
"streamed_json",
)
def __init__(self) -> None:
self.id: str = ""
self.name: str = ""
self._args_parts: list[str] = []
self._args_joined: str | None = ""
self.name_sent: bool = False
self.streamed_json: str = ""
@property
def args(self) -> str:
if self._args_joined is None:
self._args_joined = "".join(self._args_parts)
return self._args_joined
def append_args(self, value: str) -> None:
self._args_parts.append(value)
self._args_joined = None
class ParserEngine(Parser):
"""A :class:`Parser` backed by a single declarative engine config.
Subclasses set the ``ParserEngineConfig`` in ``__init__`` to define the
complete output format for a model (reasoning + tool calls).
"""
def __init__(
self,
tokenizer: TokenizerLike,
tools: list[Tool] | None = None,
*,
parser_engine_config: ParserEngineConfig,
**kwargs,
) -> None:
self.model_tokenizer = tokenizer
self._tools = tools
self._stream_state = StreamState()
self._reasoning_parser = None
self._tool_parser = None
self.parser_engine_config = parser_engine_config
self._engine = StreamingParserEngine(
parser_engine_config, tokenizer, vocab=self.vocab
)
self._reasoning_ended: bool = False
self._streaming_initialized: bool = False
self._tool_slots: list[ToolCallSlot] = []
self._deferred_content: str = ""
self._deferred_reasoning: str = ""
self._content_has_nonws: bool = False
self._arg_converter = parser_engine_config.arg_converter
self._arg_structural_chars = parser_engine_config.arg_structural_chars
self._stream_arg_deltas = parser_engine_config.stream_arg_deltas
self._strip_trailing_reasoning_ws = (
parser_engine_config.strip_trailing_reasoning_whitespace
)
self._drop_ws_only_content_before_tools = (
parser_engine_config.drop_whitespace_only_content_before_tools
)
self._strip_content_ws_with_tools = (
parser_engine_config.strip_content_whitespace_with_tools
)
vocab = self.vocab
self._reasoning_start_token_id: int | None = None
self._reasoning_end_token_id: int | None = None
start_text = parser_engine_config.token_id_terminals.get("THINK_START")
end_text = parser_engine_config.token_id_terminals.get("THINK_END")
if start_text:
self._reasoning_start_token_id = vocab.get(start_text)
if end_text:
self._reasoning_end_token_id = vocab.get(end_text)
@property
def reasoning_start_str(self) -> str | None:
return self.parser_engine_config.terminals.get("THINK_START")
@property
def reasoning_end_str(self) -> str | None:
return self.parser_engine_config.terminals.get("THINK_END")
@cached_property
def vocab(self) -> dict[str, int]:
return self.model_tokenizer.get_vocab()
# ── Engine lifecycle ──────────────────────────────────────────────
@property
def skip_tool_parsing(self) -> bool:
return self._engine.skip_tool_parsing
@skip_tool_parsing.setter
def skip_tool_parsing(self, value: bool) -> None:
self._engine.skip_tool_parsing = value
@property
def reasoning_ended(self) -> bool:
return self._reasoning_ended
def initialize_streaming(
self,
initial_state: ParserState | None = None,
) -> None:
if not self._streaming_initialized:
self._streaming_initialized = True
self._reset(initial_state=initial_state)
def finish_streaming(self) -> DeltaMessage | None:
events = self._engine.finish()
return self._events_to_delta(events) if events else None
def _reset(self, initial_state: ParserState | None = None) -> None:
self._engine.reset(initial_state=initial_state)
self._reasoning_ended = False
self._tool_slots.clear()
self._deferred_content = ""
self._deferred_reasoning = ""
self._content_has_nonws = False
def adjust_request(
self, request: ChatCompletionRequest | ResponsesRequest
) -> ChatCompletionRequest | ResponsesRequest:
request.skip_special_tokens = False
return request
# ── Schema-aware type correction ─────────────────────────────────
@staticmethod
def _coerce_value(value: object, schema: dict) -> tuple[object, bool]:
"""Coerce a single value according to its schema.
Returns ``(coerced_value, changed)``.
"""
if isinstance(value, str):
types = extract_types_from_schema(schema)
coerced = coerce_to_schema_type(value, types)
if coerced is not value:
return coerced, True
return value, False
if isinstance(value, dict):
nested_props = schema.get("properties")
if isinstance(nested_props, dict):
_, changed = ParserEngine._coerce_dict(value, nested_props)
return value, changed
return value, False
if isinstance(value, list):
items_schema = schema.get("items")
if isinstance(items_schema, dict):
changed = False
for i, item in enumerate(value):
coerced, item_changed = ParserEngine._coerce_value(
item, items_schema
)
if item_changed:
value[i] = coerced
changed = True
return value, changed
return value, False
types = extract_types_from_schema(schema)
as_str = json.dumps(value, ensure_ascii=False)
coerced = coerce_to_schema_type(as_str, types)
if coerced != value:
return coerced, True
return value, False
@staticmethod
def _coerce_dict(args: dict, properties: dict) -> tuple[dict, bool]:
"""Coerce all values in *args* using *properties* schemas."""
changed = False
for key, value in args.items():
prop = properties.get(key)
if not isinstance(prop, dict):
continue
coerced, val_changed = ParserEngine._coerce_value(value, prop)
if val_changed:
args[key] = coerced
changed = True
return args, changed
@staticmethod
def _safe_arg_prefix(json_str: str) -> str:
"""Return the prefix of *json_str* up to the last top-level value.
Middle values (followed by a comma) are stable across streaming
ticks and included. The trailing value is excluded because type
coercion may change its serialised form between ticks, which
would violate the ``startswith(prev)`` prefix invariant.
"""
last_colon = -1
in_string = False
escape = False
depth = 0
for i, c in enumerate(json_str):
if escape:
escape = False
continue
if in_string:
if c == "\\":
escape = True
elif c == '"':
in_string = False
continue
if c == '"':
in_string = True
elif c in ("{", "["):
depth += 1
elif c in ("}", "]"):
depth -= 1
elif c == ":" and depth == 1:
last_colon = i
if last_colon < 0:
return ""
end = last_colon + 1
while end < len(json_str) and json_str[end] in (" ", "\t", "\n", "\r"):
end += 1
return json_str[:end]
def _fix_arg_types(self, args_json: str, func_name: str) -> str:
"""Correct parameter types using the tool schema.
String values are coerced via :func:`coerce_to_schema_type`.
Nested objects and arrays are recursed into when the schema
defines ``properties`` or ``items``. Without a schema, values
stay as strings.
"""
if not self._tools or not func_name:
return args_json
try:
args = json.loads(args_json)
except (json.JSONDecodeError, ValueError):
return args_json
if not isinstance(args, dict):
return args_json
properties = find_tool_properties(self._tools, func_name)
if not properties:
return args_json
_, changed = self._coerce_dict(args, properties)
if changed:
return json.dumps(args, ensure_ascii=False)
return args_json
# ── Private helpers ─────────────────────────────────────────────
def _check_skip_tool_parsing(
self,
request: ChatCompletionRequest | ResponsesRequest,
) -> None:
if not self.skip_tool_parsing:
tool_choice = getattr(request, "tool_choice", None)
tools = getattr(request, "tools", None)
if tool_choice == "none" and tools:
self.skip_tool_parsing = True
def _strip_content_whitespace(
self,
content: str,
tools_called: bool,
) -> str | None:
if tools_called:
if self._strip_content_ws_with_tools:
content = content.strip()
elif self._drop_ws_only_content_before_tools and not content.strip():
content = ""
return content or None
# ── Streaming: parse_delta ────────────────────────────────────────
def parse_delta(
self,
delta_text: str,
delta_token_ids: list[int],
request: ChatCompletionRequest | ResponsesRequest,
prompt_token_ids: list[int] | None = None,
*,
finished: bool,
) -> DeltaMessage | None:
self._check_skip_tool_parsing(request)
events = self._engine.feed(delta_text, delta_token_ids)
if finished:
events.extend(self._engine.finish())
result = self._events_to_delta(events, finished=finished)
return self._strip_trailing_reasoning(result)
def _strip_trailing_reasoning(
self,
delta: DeltaMessage | None,
) -> DeltaMessage | None:
"""Strip trailing whitespace from reasoning, deferring it until we
know whether more reasoning follows or reasoning has ended.
Runs in ``parse_delta`` *after* ``_events_to_delta`` (and any
subclass overrides) so that overrides see the raw reasoning text.
Gated by ``strip_trailing_reasoning_whitespace``; when disabled,
passes through unchanged.
"""
if not self._strip_trailing_reasoning_ws:
return delta
if delta is not None and delta.reasoning is not None:
combined = self._deferred_reasoning + delta.reasoning
trimmed = combined.rstrip()
self._deferred_reasoning = combined[len(trimmed) :]
delta.reasoning = trimmed or None
if (
delta.reasoning is None
and delta.content is None
and not delta.tool_calls
):
return None
elif self._deferred_reasoning and self._reasoning_ended:
self._deferred_reasoning = ""
return delta
# ── Non-streaming: extract_reasoning ──────────────────────────────
def extract_reasoning(
self,
model_output: str,
request: ChatCompletionRequest | ResponsesRequest,
) -> tuple[str | None, str | None]:
self._reset()
events = self._engine.feed(model_output, [])
events.extend(self._engine.finish())
reasoning_parts: list[str] = []
content_parts: list[str] = []
for event in events:
if event.type == EventType.REASONING_CHUNK:
reasoning_parts.append(event.value)
elif event.type == EventType.TEXT_CHUNK:
content_parts.append(event.value)
elif event.type == EventType.REASONING_END:
self._reasoning_ended = True
raw_reasoning = "".join(reasoning_parts)
if self._strip_trailing_reasoning_ws:
raw_reasoning = raw_reasoning.rstrip()
reasoning = raw_reasoning or None
content = "".join(content_parts) or None
return reasoning, content
# ── Non-streaming: extract_reasoning_streaming ────────────────────
def extract_reasoning_streaming(
self,
previous_text: str,
current_text: str,
delta_text: str,
previous_token_ids: Sequence[int],
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
) -> DeltaMessage | None:
self.initialize_streaming()
events = self._engine.feed(delta_text, delta_token_ids)
return self._strip_trailing_reasoning(self._events_to_delta(events))
# ── Non-streaming: extract_tool_calls ─────────────────────────────
def extract_tool_calls(
self,
model_output: str,
request: ChatCompletionRequest | ResponsesRequest,
) -> ExtractedToolCallInformation:
self._reset()
self._streaming_initialized = True
result = self.extract_tool_calls_streaming(
previous_text="",
current_text=model_output,
delta_text=model_output,
previous_token_ids=[],
current_token_ids=[],
delta_token_ids=[],
request=request,
)
finish_delta = self.finish_streaming()
return self._build_extracted_result(result, finish_delta)
def extract_tool_calls_from_content(
self,
content: str,
request: ChatCompletionRequest,
) -> ExtractedToolCallInformation:
"""Extract tool calls from reasoning-stripped content.
Unlike :meth:`extract_tool_calls` which re-parses the full model
output, this method starts the parser engine in ``CONTENT`` state
so it can parse content that has already had reasoning stripped.
"""
_, parsed_content, tool_call_info = self._single_pass_parse(
content,
[],
initial_state=ParserState.CONTENT,
)
if parsed_content is not None and tool_call_info.content is None:
tool_call_info = ExtractedToolCallInformation(
tools_called=tool_call_info.tools_called,
tool_calls=tool_call_info.tool_calls,
content=parsed_content,
)
return tool_call_info
def extract_tool_calls_streaming(
self,
previous_text: str,
current_text: str,
delta_text: str,
previous_token_ids: Sequence[int],
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest | ResponsesRequest,
) -> DeltaMessage | None:
self.initialize_streaming()
self._check_skip_tool_parsing(request)
events = self._engine.feed(delta_text, delta_token_ids)
return self._strip_trailing_reasoning(self._events_to_delta(events))
# ── Reasoning state queries ───────────────────────────────────────
def is_reasoning_end(self, input_ids: list[int]) -> bool:
end_id = self._reasoning_end_token_id
start_id = self._reasoning_start_token_id
if end_id is not None:
if not input_ids:
return self.parser_engine_config.initial_state != ParserState.REASONING
for i in range(len(input_ids) - 1, -1, -1):
if input_ids[i] == end_id:
return True
if start_id is not None and input_ids[i] == start_id:
return False
return False
return self._reasoning_ended
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
end_id = self._reasoning_end_token_id
if end_id is not None:
for i in range(len(input_ids) - 1, -1, -1):
if input_ids[i] == end_id:
return input_ids[i + 1 :]
return input_ids
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
start_id = self._reasoning_start_token_id
end_id = self._reasoning_end_token_id
if start_id is None or end_id is None:
return 0
count = 0
depth = 0
for token_id in token_ids:
if token_id == start_id:
depth += 1
continue
if token_id == end_id:
if depth > 0:
depth -= 1
continue
if depth > 0:
count += 1
return count
# ── Single-pass parse helper ────────────────────────────────────────
def _single_pass_parse(
self,
text: str,
token_ids: Sequence[int],
initial_state: ParserState | None = None,
) -> tuple[str | None, str | None, ExtractedToolCallInformation]:
"""Reset, feed, finish, and extract results in one pass.
Must be called as a unit — ``_events_to_delta`` populates tool
state that ``_build_extracted_result`` reads.
"""
self._reset(initial_state=initial_state)
events = self._engine.feed(text, token_ids)
events.extend(self._engine.finish())
delta = self._events_to_delta(events)
tool_call_info = self._build_extracted_result()
reasoning = delta.reasoning if delta else None
if reasoning and self._strip_trailing_reasoning_ws:
reasoning = reasoning.rstrip() or None
content = delta.content if delta else None
if content:
content = self._strip_content_whitespace(
content, tool_call_info.tools_called
)
return reasoning, content, tool_call_info
# ── Non-streaming: parse ───────────────────────────────────────────
def parse(
self,
model_output: str,
request: ChatCompletionRequest | ResponsesRequest,
enable_auto_tools: bool = False,
model_output_token_ids: Sequence[int] = (),
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
reasoning, content, tool_call_info = self._single_pass_parse(
model_output,
model_output_token_ids,
)
tool_calls: list[FunctionCall] | None = None
if tool_call_info.tools_called:
tool_calls = [
FunctionCall(
id=tc.id,
name=tc.function.name,
arguments=tc.function.arguments,
)
for tc in tool_call_info.tool_calls
]
return reasoning, content, tool_calls
# ── Event-to-delta conversion ─────────────────────────────────────
def _events_to_delta(
self,
events: list[SemanticEvent],
finished: bool = False,
) -> DeltaMessage | None:
if not events and not self._deferred_content:
return None
tool_call_deltas: list[DeltaToolCall] = []
content_parts: list[str] = []
reasoning_parts: list[str] = []
seen_tool_event = False
for event in events:
match event.type:
case EventType.TEXT_CHUNK:
if seen_tool_event:
self._deferred_content += event.value
else:
content_parts.append(event.value)
case EventType.REASONING_CHUNK:
reasoning_parts.append(event.value)
case EventType.REASONING_END:
self._reasoning_ended = True
case EventType.TOOL_CALL_START:
seen_tool_event = True
self._ensure_slot(event.tool_index)
case EventType.TOOL_NAME:
seen_tool_event = True
self._handle_tool_name(event)
case EventType.ARG_VALUE_CHUNK:
seen_tool_event = True
self._handle_arg_chunk(event, tool_call_deltas)
case EventType.TOOL_CALL_END:
seen_tool_event = True
self._handle_tool_end(event, tool_call_deltas)
case EventType.REASONING_START:
pass # no delta-level effect
if len(tool_call_deltas) > 1:
tool_call_deltas = self._coalesce_tool_call_deltas(tool_call_deltas)
if self._deferred_content and not seen_tool_event:
content_parts.insert(0, self._deferred_content)
self._deferred_content = ""
content_str = "".join(content_parts)
if self._content_has_nonws:
pass
elif content_str:
stripped = content_str.strip()
if stripped:
self._content_has_nonws = True
elif self._tool_slots:
if self._drop_ws_only_content_before_tools:
content_str = ""
elif not finished:
self._deferred_content = content_str
content_str = ""
content = content_str or None
reasoning = "".join(reasoning_parts) or None
if content or tool_call_deltas or reasoning:
kwargs: dict[str, object] = {}
if content is not None:
kwargs["content"] = content
if reasoning is not None:
kwargs["reasoning"] = reasoning
if tool_call_deltas:
kwargs["tool_calls"] = tool_call_deltas
return DeltaMessage(**kwargs)
return None
def _ensure_slot(self, idx: int) -> None:
while len(self._tool_slots) <= idx:
self._tool_slots.append(ToolCallSlot())
def _ensure_tool_id(self, slot: ToolCallSlot, name: str) -> None:
if not slot.id:
state = self._stream_state
slot.id = make_tool_call_id(
id_type=state.tool_call_id_type,
func_name=name,
idx=state.history_tool_call_cnt,
)
state.history_tool_call_cnt += 1
def _handle_tool_name(self, event: SemanticEvent) -> None:
idx = event.tool_index
self._tool_slots[idx].name += event.value
def _emit_name_delta(
self,
idx: int,
deltas: list[DeltaToolCall],
name: str | None,
) -> None:
if not name:
return
slot = self._tool_slots[idx]
slot.name = name
slot.name_sent = True
self._ensure_tool_id(slot, name)
deltas.append(
DeltaToolCall(
index=idx,
id=slot.id,
type="function",
function=DeltaFunctionCall(name=name),
)
)
def _handle_arg_chunk(
self,
event: SemanticEvent,
deltas: list[DeltaToolCall],
) -> None:
idx = event.tool_index
slot = self._tool_slots[idx]
if event.value:
slot.append_args(event.value)
if not slot.name_sent:
if slot.name:
self._emit_name_delta(idx, deltas, slot.name)
elif event.value:
# Name not yet known — try to extract from accumulated args
name = self._try_extract_name(idx)
self._emit_name_delta(idx, deltas, name)
elif event.value:
# Name already sent — emit arg delta
arg_delta = self._compute_arg_delta(idx, event.value)
if arg_delta:
deltas.append(
DeltaToolCall(
index=idx,
function=DeltaFunctionCall(arguments=arg_delta),
)
)
def _handle_tool_end(
self,
event: SemanticEvent,
deltas: list[DeltaToolCall],
) -> None:
idx = event.tool_index
if idx >= len(self._tool_slots):
return
remaining = self._flush_arg_converter(idx)
slot = self._tool_slots[idx]
if not slot.name_sent:
name = slot.name or self._try_extract_name(idx)
if name:
slot.name = name
slot.name_sent = True
self._ensure_tool_id(slot, name)
deltas.append(
DeltaToolCall(
index=idx,
id=slot.id,
type="function",
function=DeltaFunctionCall(
name=name,
arguments=remaining or "",
),
)
)
remaining = None
if remaining and slot.name_sent:
deltas.append(
DeltaToolCall(
index=idx,
function=DeltaFunctionCall(arguments=remaining),
)
)
# ── Tool-call delta coalescing ──────────────────────────────────────
@staticmethod
def _coalesce_tool_call_deltas(
deltas: list[DeltaToolCall],
) -> list[DeltaToolCall]:
"""Merge entries that share the same index into one per index."""
merged: dict[int, DeltaToolCall] = {}
for tc in deltas:
existing = merged.get(tc.index)
if existing is None:
merged[tc.index] = tc
continue
if tc.id is not None and existing.id is None:
existing.id = tc.id
if tc.type is not None and existing.type is None:
existing.type = tc.type
if tc.function is not None:
if existing.function is None:
existing.function = tc.function
else:
if tc.function.name is not None and existing.function.name is None:
existing.function.name = tc.function.name
if tc.function.arguments is not None:
if existing.function.arguments is None:
existing.function.arguments = tc.function.arguments
else:
existing.function.arguments += tc.function.arguments
if len(merged) == len(deltas):
return deltas
return list(merged.values())
# ── Arg conversion helpers ─────────────────────────────────────────
def _compute_arg_delta(self, idx: int, raw_delta: str) -> str | None:
converter = self._arg_converter
if converter is None:
return raw_delta
if not self._stream_arg_deltas:
return None
structural = self._arg_structural_chars
if structural is not None and structural.isdisjoint(raw_delta):
return None
slot = self._tool_slots[idx]
try:
current_json = converter(slot.args, True)
except (json.JSONDecodeError, ValueError, TypeError):
logger.debug("arg converter failed (streaming): %s", slot.args[:80])
return None
if not current_json:
return None
if slot.name:
current_json = self._fix_arg_types(current_json, slot.name)
prev = slot.streamed_json
safe_json = self._safe_arg_prefix(current_json)
if not safe_json or safe_json == prev:
return None
if prev:
if not safe_json.startswith(prev):
return None
diff = safe_json[len(prev) :]
else:
diff = safe_json
if diff:
slot.streamed_json = safe_json
return diff
return None
def _flush_arg_converter(self, idx: int) -> str | None:
converter = self._arg_converter
if converter is None:
return None
slot = self._tool_slots[idx]
try:
final_json = converter(slot.args, False)
except (json.JSONDecodeError, ValueError, TypeError):
logger.debug("arg converter failed (flush): %s", slot.args[:80])
return None
if final_json:
final_json = self._fix_arg_types(final_json, slot.name)
prev = slot.streamed_json
if final_json and len(final_json) > len(prev):
if prev and not final_json.startswith(prev):
return None
diff = final_json[len(prev) :]
slot.streamed_json = final_json
return diff
return None
_NAME_RE = re.compile(r'"name"\s*:\s*"([^"]*)"')
def _try_extract_name(self, idx: int) -> str | None:
m = self._NAME_RE.search(self._tool_slots[idx].args)
if m:
name = m.group(1)
if name:
return name
return None
# ── Build ExtractedToolCallInformation ─────────────────────────────
def _build_extracted_result(
self,
*deltas: DeltaMessage | None,
) -> ExtractedToolCallInformation:
content_parts: list[str] = []
for delta in deltas:
if delta is not None and delta.content:
content_parts.append(delta.content)
tool_calls: list[ToolCall] = []
for idx, slot in enumerate(self._tool_slots):
if not slot.name and not slot.args:
continue
name = slot.name.strip()
raw_body = slot.args
if not name and raw_body.strip():
name, args_json = self._extract_name_and_args(raw_body)
elif raw_body.strip():
converter = self._arg_converter
if converter is not None:
try:
args_json = converter(raw_body, False)
except (json.JSONDecodeError, ValueError, TypeError):
logger.debug(
"arg converter failed (extract): %s", raw_body[:80]
)
args_json = self._extract_args_json(raw_body, name)
else:
args_json = self._extract_args_json(raw_body, name)
else:
args_json = "{}"
if name:
self._ensure_tool_id(slot, name)
args_json = self._fix_arg_types(args_json, name)
tool_calls.append(
ToolCall(
id=slot.id,
function=FunctionCall(name=name, arguments=args_json),
)
)
content_str = "".join(content_parts)
content = self._strip_content_whitespace(content_str, len(tool_calls) > 0)
return ExtractedToolCallInformation(
tools_called=len(tool_calls) > 0,
tool_calls=tool_calls,
content=content,
)
@staticmethod
def _extract_args_value(parsed: dict) -> str | None:
for key in ("arguments", "parameters"):
if key in parsed:
val = parsed[key]
if isinstance(val, str):
return val
return json.dumps(val, ensure_ascii=False)
return None
def _extract_name_and_args(
self,
raw_body: str,
) -> tuple[str, str]:
raw_body = raw_body.strip()
try:
parsed = json.loads(raw_body)
except json.JSONDecodeError:
return "", raw_body
if not isinstance(parsed, dict):
return "", raw_body
name = parsed.get("name", "")
args = self._extract_args_value(parsed)
if args is not None:
return name, args
without_name = {k: v for k, v in parsed.items() if k != "name"}
return name, json.dumps(without_name, ensure_ascii=False)
def _extract_args_json(self, raw_args: str, func_name: str) -> str:
if not raw_args.strip():
return "{}"
_, args = self._extract_name_and_args(raw_args)
return args
+114
View File
@@ -0,0 +1,114 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Declarative configuration for model tool-call and reasoning formats.
Each model format is described by a :class:`ParserEngineConfig` that specifies:
* **terminals** – literal strings or regex patterns that delimit the format
(e.g. ``<tool_call>``, ``</think>``).
* **token_id_terminals** – terminals that should be matched by token ID
rather than (or in addition to) text.
* **transitions** – a state machine mapping
``(state, terminal) → (new_state, events_to_emit)`` that drives semantic
event generation during streaming.
* **content_events** – what :class:`EventType` to emit for plain content
(non-terminal text) in each state.
"""
from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass, field
from enum import Enum, auto
from functools import cached_property
from vllm.parser.engine.events import EventType
STRUCTURAL_DROP_TOKENS: frozenset[str] = frozenset(
{
"<eos>",
"<bos>",
"<pad>",
"<unk>",
"<mask>",
}
)
class ParserState(Enum):
CONTENT = auto()
REASONING = auto()
TOOL_PREAMBLE = auto()
TOOL_NAME = auto()
TOOL_ARGS = auto()
TOOL_BETWEEN = auto()
@dataclass(frozen=True, slots=True)
class Transition:
next_state: ParserState
events: tuple[EventType, ...] = field(default_factory=tuple)
skip_in_token_id_mode: bool = False
@dataclass(frozen=True)
class ParserEngineConfig:
"""Declarative description of a model's tool-call / reasoning format.
The engine feeds terminals from the incremental lexer into the
transition table and emits the corresponding semantic events.
Content tokens (text between terminals) are classified by the
current state via ``content_events``.
"""
name: str
terminals: dict[str, str] = field(default_factory=dict)
token_id_terminals: dict[str, str] = field(default_factory=dict)
transitions: dict[tuple[ParserState, str], Transition] = field(
default_factory=dict,
)
content_events: dict[ParserState, EventType] = field(
default_factory=lambda: {
ParserState.CONTENT: EventType.TEXT_CHUNK,
ParserState.REASONING: EventType.REASONING_CHUNK,
ParserState.TOOL_NAME: EventType.TOOL_NAME,
ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
},
)
initial_state: ParserState = ParserState.CONTENT
arg_converter: Callable[[str, bool], str] | None = None
stream_arg_deltas: bool = True
tool_args_json: bool = True
arg_structural_chars: frozenset[str] | None = None
# Prevents trailing-whitespace accumulation across multi-turn conversations.
strip_trailing_reasoning_whitespace: bool = True
# Drop content that is entirely whitespace when tool calls follow.
drop_whitespace_only_content_before_tools: bool = True
# .strip() content text when tool calls are present.
strip_content_whitespace_with_tools: bool = True
drop_tokens: frozenset[str] = field(default_factory=frozenset)
@cached_property
def terminal_defs(self):
from vllm.parser.engine.incremental_lexer import terminals_from_literals
return terminals_from_literals(self.terminals)
@cached_property
def lexer_shape(self):
from vllm.parser.engine.incremental_lexer import LexerShape
return LexerShape(self.terminal_defs)
+16
View File
@@ -0,0 +1,16 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Concrete adapter classes for each registered parser engine.
These are created via :func:`make_adapters` and exposed as module-level
names so that :class:`ReasoningParserManager` and
:class:`ToolParserManager` can load them lazily.
"""
from vllm.parser.engine.adapters import make_adapters
from vllm.parser.qwen3 import Qwen3Parser
(
Qwen3ParserReasoningAdapter,
Qwen3ParserToolAdapter,
) = make_adapters(Qwen3Parser)
@@ -0,0 +1,408 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Streaming parser engine that orchestrates token ID scanning,
incremental lexing, and state-machine-driven semantic event emission."""
from __future__ import annotations
from collections.abc import Sequence
from vllm.parser.engine.events import EventType, SemanticEvent
from vllm.parser.engine.incremental_lexer import (
CONTENT_TERMINAL,
IncrementalLexer,
LexToken,
)
from vllm.parser.engine.parser_engine_config import (
STRUCTURAL_DROP_TOKENS,
ParserEngineConfig,
ParserState,
Transition,
)
from vllm.parser.engine.token_id_scanner import (
LexerInput,
PreLexedTerminal,
TextChunk,
TokenIDScanner,
)
class StreamingParserEngine:
"""Consumes ``(delta_text, delta_token_ids)`` pairs and produces a
stream of :class:`SemanticEvent` instances.
This is the main entry point for streaming parsing.
Create one per request (it is stateful).
The pipeline is::
delta_text + delta_token_ids
→ TokenIDScanner (special token pre-lexing)
→ IncrementalLexer (text → terminal tokens with prefix buffering)
→ State Machine (terminal → semantic events)
→ list[SemanticEvent]
Usage::
engine = StreamingParserEngine(config, tokenizer)
for each streaming delta:
events = engine.feed(delta_text, delta_token_ids)
# convert events to DeltaMessage
"""
def __init__(
self,
config: ParserEngineConfig,
tokenizer,
initial_state: ParserState | None = None,
vocab: dict[str, int] | None = None,
) -> None:
self.config = config
resolved_token_ids: dict[int, str] = {}
drop_token_ids: set[int] = set()
if tokenizer is not None:
if vocab is None:
vocab = tokenizer.get_vocab()
if config.token_id_terminals:
for terminal_name, token_text in config.token_id_terminals.items():
tid = vocab.get(token_text)
if tid is not None:
resolved_token_ids[tid] = terminal_name
all_drop = config.drop_tokens | STRUCTURAL_DROP_TOKENS
for token_text in all_drop:
tid = vocab.get(token_text)
if tid is not None:
drop_token_ids.add(tid)
for attr in ("eos_token_id", "bos_token_id", "pad_token_id"):
tid = getattr(tokenizer, attr, None)
if tid is not None:
drop_token_ids.add(tid)
self._resolved_token_ids = resolved_token_ids
self._drop_token_ids = drop_token_ids
self._scanner = TokenIDScanner(
resolved_token_ids,
tokenizer,
drop_token_ids,
)
self._token_id_terminal_names: frozenset[str] = frozenset(
resolved_token_ids.values()
)
self._lexer = IncrementalLexer(
config.lexer_shape, content_terminal=CONTENT_TERMINAL
)
self._tool_terminals: frozenset[str] = frozenset(
terminal
for (state, terminal), tr in config.transitions.items()
if tr.next_state in self._TOOL_STATES or state in self._TOOL_STATES
)
self.skip_tool_parsing = False
self.reset(initial_state=initial_state)
def _reset_args_state(self) -> None:
self._args_buffer: str = ""
self._args_safe_end: int = 0
self._args_brace_depth: int = 0
self._args_in_string: bool = False
self._args_escape_next: bool = False
def reset(self, initial_state: ParserState | None = None) -> None:
"""Reset mutable state for reuse across requests.
Preserves cached immutable structures (compiled terminals,
resolved token IDs, lexer shape, token text cache) to avoid
redundant initialization work.
"""
self.state = (
initial_state if initial_state is not None else self.config.initial_state
)
self.tool_index = -1
self._ever_had_token_ids = False
# DO NOT reset skip_tool_parsing here — callers set it before
# calling methods that trigger reset() (e.g. extract_reasoning),
# and clearing it silently breaks non-streaming tool-call-as-
# implicit-reasoning-end (content returns None).
self._scanner.reset()
self._lexer.reset()
self._reset_args_state()
def feed(
self,
delta_text: str,
delta_token_ids: Sequence[int],
) -> list[SemanticEvent]:
if delta_token_ids:
self._ever_had_token_ids = True
# Fast path: skip scanner and lexer when the delta is plain
# content with no special tokens and no terminal-starting chars.
if (
delta_text
and not self._lexer.buffer
and not self._scanner._deferred_terminals
and self._lexer._literal_first_chars.isdisjoint(delta_text)
):
has_special = False
for tid in delta_token_ids:
if tid in self._resolved_token_ids or tid in self._drop_token_ids:
has_special = True
break
if not has_special:
return self._emit_for_state(delta_text)
scanner_items = self._scanner.scan(delta_text, delta_token_ids)
if len(scanner_items) == 1 and isinstance(scanner_items[0], TextChunk):
lex_tokens = self._lexer.feed(scanner_items[0].text)
if len(lex_tokens) == 1 and lex_tokens[0].terminal == CONTENT_TERMINAL:
text = lex_tokens[0].value
return self._emit_for_state(text)
return self._process_lex_tokens(lex_tokens)
return self._process_scanner_items(scanner_items)
def _process_scanner_items(
self, items: Sequence[LexerInput]
) -> list[SemanticEvent]:
events: list[SemanticEvent] = []
for item in items:
if isinstance(item, PreLexedTerminal):
events.extend(self._process_lex_tokens(self._lexer.flush()))
events.extend(self._on_terminal(item.terminal, item.text))
elif isinstance(item, TextChunk):
events.extend(self._process_lex_tokens(self._lexer.feed(item.text)))
return events
def finish(self) -> list[SemanticEvent]:
events = self._process_scanner_items(self._scanner.flush_pending())
events.extend(self._process_lex_tokens(self._lexer.flush()))
if self._args_buffer:
events.append(
SemanticEvent(
EventType.ARG_VALUE_CHUNK,
value=self._args_buffer,
tool_index=self.tool_index,
)
)
self._args_buffer = ""
self._args_safe_end = 0
if self.state in (
ParserState.TOOL_PREAMBLE,
ParserState.TOOL_ARGS,
ParserState.TOOL_NAME,
ParserState.TOOL_BETWEEN,
):
if self.tool_index >= 0:
events.append(
SemanticEvent(
EventType.TOOL_CALL_END,
tool_index=self.tool_index,
)
)
self.state = ParserState.CONTENT
elif self.state == ParserState.REASONING:
events.append(
SemanticEvent(EventType.REASONING_END, tool_index=self.tool_index)
)
self.state = ParserState.CONTENT
return events
def parse_complete(self, text: str) -> list[SemanticEvent]:
token_ids: list[int] = []
events = self.feed(text, token_ids)
events.extend(self.finish())
return events
def _process_lex_tokens(self, tokens: list[LexToken]) -> list[SemanticEvent]:
events: list[SemanticEvent] = []
strict = self._token_id_terminal_names if self._ever_had_token_ids else None
for tok in tokens:
if tok.terminal == CONTENT_TERMINAL or (strict and tok.terminal in strict):
events.extend(self._on_content(tok.value))
else:
events.extend(self._on_terminal(tok.terminal, tok.value))
return events
_TOOL_STATES = frozenset(
{
ParserState.TOOL_PREAMBLE,
ParserState.TOOL_NAME,
ParserState.TOOL_ARGS,
ParserState.TOOL_BETWEEN,
}
)
def _on_terminal(self, terminal: str, value: str) -> list[SemanticEvent]:
key = (self.state, terminal)
transition = self.config.transitions.get(key)
if transition is None:
return self._emit_for_state(value)
if self.skip_tool_parsing and terminal in self._tool_terminals:
if EventType.REASONING_END in transition.events:
self.state = ParserState.CONTENT
return [
SemanticEvent(
EventType.REASONING_END,
value=value,
tool_index=self.tool_index,
),
SemanticEvent(
EventType.TEXT_CHUNK,
value=value,
tool_index=self.tool_index,
),
]
content_type = self.config.content_events.get(self.state)
if content_type is not None:
return [
SemanticEvent(content_type, value=value, tool_index=self.tool_index)
]
return []
if transition.skip_in_token_id_mode and self._ever_had_token_ids:
return self._emit_for_state(value)
return self._apply_transition(transition, value)
def _emit_for_state(self, text: str) -> list[SemanticEvent]:
if self.state == ParserState.TOOL_ARGS:
if self.config.tool_args_json:
return self._feed_args_text(text)
return [
SemanticEvent(
EventType.ARG_VALUE_CHUNK,
value=text,
tool_index=self.tool_index,
)
]
content_type = self.config.content_events.get(self.state)
if content_type is not None:
return [SemanticEvent(content_type, value=text, tool_index=self.tool_index)]
return []
def _on_content(self, text: str) -> list[SemanticEvent]:
if not text:
return []
return self._emit_for_state(text)
def _apply_transition(
self,
transition: Transition,
value: str,
) -> list[SemanticEvent]:
events: list[SemanticEvent] = []
if (
self.state == ParserState.TOOL_ARGS
and transition.next_state != ParserState.TOOL_ARGS
and self._args_buffer
):
events.append(
SemanticEvent(
EventType.ARG_VALUE_CHUNK,
value=self._args_buffer,
tool_index=self.tool_index,
)
)
self._args_buffer = ""
self.state = transition.next_state
for event_type in transition.events:
if event_type == EventType.TOOL_CALL_START:
self.tool_index += 1
events.append(
SemanticEvent(
event_type,
value=value,
tool_index=self.tool_index,
)
)
if self.state == ParserState.TOOL_ARGS:
self._args_brace_depth = 0
self._args_in_string = False
self._args_escape_next = False
self._args_safe_end = 0
return events
def _feed_args_text(self, text: str) -> list[SemanticEvent]:
"""Feed text into the JSON argument streaming buffer.
Streams argument characters incrementally while holding back
closing braces/brackets that might change as more input arrives.
"""
events: list[SemanticEvent] = []
for ch in text:
result = self._feed_args_char(ch)
events.extend(result)
return events
def _feed_args_char(self, ch: str) -> list[SemanticEvent]:
self._args_buffer += ch
if self._args_escape_next:
self._args_escape_next = False
self._args_safe_end = len(self._args_buffer)
return self._flush_safe_args()
if self._args_in_string:
if ch == "\\":
self._args_escape_next = True
elif ch == '"':
self._args_in_string = False
self._args_safe_end = len(self._args_buffer)
return self._flush_safe_args()
if ch == '"':
self._args_in_string = True
self._args_safe_end = len(self._args_buffer)
return self._flush_safe_args()
if ch in ("{", "["):
self._args_brace_depth += 1
self._args_safe_end = len(self._args_buffer)
return self._flush_safe_args()
if ch in ("}", "]"):
if self._args_brace_depth > 0:
self._args_brace_depth -= 1
if self._args_brace_depth == 0:
return []
self._args_safe_end = len(self._args_buffer)
return self._flush_safe_args()
self._args_safe_end = len(self._args_buffer)
return self._flush_safe_args()
def _flush_safe_args(self) -> list[SemanticEvent]:
"""Emit buffered argument characters up to the safe-end watermark.
Top-level closing braces are held back (safe_end not advanced)
until confirmed safe by a subsequent character or finish().
"""
if self._args_safe_end == 0:
return []
to_emit = self._args_buffer[: self._args_safe_end]
self._args_buffer = self._args_buffer[self._args_safe_end :]
self._args_safe_end = 0
return [
SemanticEvent(
EventType.ARG_VALUE_CHUNK,
value=to_emit,
tool_index=self.tool_index,
)
]
+309
View File
@@ -0,0 +1,309 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Scan delta token IDs for special tokens and split the stream into
pre-lexed terminals and plain text chunks."""
from __future__ import annotations
from collections.abc import Sequence
from dataclasses import dataclass
@dataclass(slots=True)
class TextChunk:
text: str
@dataclass(slots=True)
class PreLexedTerminal:
terminal: str
token_id: int
text: str
LexerInput = TextChunk | PreLexedTerminal
class TokenIDScanner:
"""Maps special token IDs in the delta to terminals.
Before text-based lexing happens, the scanner checks each token ID
in the delta against a mapping of ``{token_id: terminal_name}``.
Matched tokens are emitted as :class:`PreLexedTerminal` items;
everything else is grouped into :class:`TextChunk` items for the
incremental lexer to process.
When a terminal's text is not yet in ``delta_text`` (held back by
the detokenizer), the terminal is deferred until the text arrives
in a subsequent delta.
"""
def __init__(
self,
token_id_to_terminal: dict[int, str],
tokenizer,
drop_token_ids: set[int] | None = None,
) -> None:
self.token_id_to_terminal = token_id_to_terminal
self.tokenizer = tokenizer
self._token_text_cache: dict[int, str] = {}
self._drop_token_ids = drop_token_ids or set()
self._deferred_terminals: list[PreLexedTerminal] = []
self._deferred_post_text: str = ""
def reset(self) -> None:
"""Clear mutable state for reuse. Preserves the token text cache."""
self._deferred_terminals.clear()
self._deferred_post_text = ""
def _decode_token(self, token_id: int) -> str:
if token_id not in self._token_text_cache:
self._token_text_cache[token_id] = self.tokenizer.decode([token_id])
return self._token_text_cache[token_id]
_EMPTY: tuple[LexerInput, ...] = ()
def scan(
self,
delta_text: str,
delta_token_ids: Sequence[int],
) -> Sequence[LexerInput]:
prefix_items: list[LexerInput] = []
effective_text = delta_text
if self._deferred_terminals:
prefix_items, effective_text = self._resolve_deferred(delta_text)
if not self.token_id_to_terminal and not self._drop_token_ids:
if effective_text:
prefix_items.append(TextChunk(effective_text))
return prefix_items
has_special = False
has_drop = False
token_id_to_terminal = self.token_id_to_terminal
drop_token_ids = self._drop_token_ids
for tid in delta_token_ids:
if tid in token_id_to_terminal:
has_special = True
if tid in drop_token_ids:
has_drop = True
if not has_special and not has_drop:
if effective_text:
if not prefix_items:
return [TextChunk(effective_text)]
prefix_items.append(TextChunk(effective_text))
return prefix_items or self._EMPTY
token_texts = [self._decode_token(tid) for tid in delta_token_ids]
results: list[LexerInput] = []
text_accum: list[str] = []
for idx, tid in enumerate(delta_token_ids):
if tid in self._drop_token_ids:
continue
terminal = self.token_id_to_terminal.get(tid)
if terminal is not None:
if text_accum:
joined = "".join(text_accum)
if joined:
results.append(TextChunk(joined))
text_accum.clear()
results.append(PreLexedTerminal(terminal, tid, token_texts[idx]))
else:
text_accum.append(token_texts[idx])
if text_accum:
joined = "".join(text_accum)
if joined:
results.append(TextChunk(joined))
if effective_text:
if has_drop:
clean_delta = effective_text
for idx, tid in enumerate(delta_token_ids):
if tid in self._drop_token_ids:
dropped = token_texts[idx]
pos = clean_delta.find(dropped)
if pos >= 0:
clean_delta = (
clean_delta[:pos] + clean_delta[pos + len(dropped) :]
)
if clean_delta:
if results:
results = self._recover_holdback_text(clean_delta, results)
else:
results = [TextChunk(clean_delta)]
else:
results = self._recover_holdback_text(effective_text, results)
else:
# No detokenizer text to validate against — individually-decoded
# TextChunks are unreliable (context-dependent decoding).
# Defer PreLexedTerminals so the state machine doesn't
# transition before the preceding text has arrived. The
# deferred terminals will be resolved against the actual
# delta_text in a subsequent scan() or flushed by finish().
for r in results:
if isinstance(r, PreLexedTerminal):
self._deferred_terminals.append(r)
results = []
return prefix_items + results
def flush_pending(self) -> list[LexerInput]:
if not self._deferred_terminals and not self._deferred_post_text:
return []
results: list[LexerInput] = []
if self._deferred_post_text:
results.append(TextChunk(self._deferred_post_text))
self._deferred_post_text = ""
results.extend(self._deferred_terminals)
self._deferred_terminals.clear()
return results
def _resolve_deferred(
self,
delta_text: str,
) -> tuple[list[LexerInput], str]:
"""Resolve deferred terminals against new delta_text.
When a previous ``scan()`` deferred a terminal (its text hadn't
arrived yet), the next delta's text should contain that terminal's
text. Split delta_text at the terminal boundary: text before
belongs to the previous parser state, the terminal triggers the
state transition, and text after belongs to the new state.
Returns ``(prefix_items, remaining_text)`` where prefix_items
are the resolved deferred terminals (with any preceding text)
and remaining_text is the unconsumed portion of delta_text that
should be scanned with the current delta's token IDs.
"""
deferred = self._deferred_terminals
self._deferred_terminals = []
results: list[LexerInput] = []
remaining = delta_text
if self._deferred_post_text:
remaining = self._deferred_post_text + remaining
self._deferred_post_text = ""
# Duplicate-text deferred terminals resolve left-to-right via
# find(); correct when each terminal text appears once in sequence.
for terminal in deferred:
pos = remaining.find(terminal.text)
if pos > 0:
results.append(TextChunk(remaining[:pos]))
results.append(terminal)
remaining = remaining[pos + len(terminal.text) :]
elif pos == 0:
results.append(terminal)
remaining = remaining[len(terminal.text) :]
else:
# Accumulate text until terminal text arrives —
# only the terminal provides a reliable split point.
if remaining:
self._deferred_post_text += remaining
remaining = ""
self._deferred_terminals.append(terminal)
return results, remaining
def _recover_holdback_text(
self,
delta_text: str,
results: list[LexerInput],
) -> list[LexerInput]:
"""Recover detokenizer hold-back text not in delta_token_ids.
The detokenizer may flush previously held-back text in
``delta_text`` that has no corresponding token ID in
``delta_token_ids``. This hold-back text always appears as a
prefix of ``delta_text``.
"""
if not results:
return [TextChunk(delta_text)]
reconstructed = self._join_decoded_text(results)
if not reconstructed:
return [TextChunk(delta_text)] + results
pos = delta_text.find(reconstructed)
if pos > 0:
return [TextChunk(delta_text[:pos])] + results
if pos == 0:
return results
# Fallback: SentencePiece context-dependent decoding mismatch.
# Rebuild from delta_text using PreLexedTerminals as split anchors.
return self._rebuild_from_anchors(delta_text, results)
def _join_decoded_text(self, results: list[LexerInput]) -> str:
"""Join TextChunk and PreLexedTerminal text into one string."""
parts: list[str] = []
for item in results:
if isinstance(item, (TextChunk, PreLexedTerminal)):
parts.append(item.text)
return "".join(parts)
def _rebuild_from_anchors(
self,
delta_text: str,
results: list[LexerInput],
) -> list[LexerInput]:
"""Rebuild results from delta_text using terminals as anchors.
When context-dependent decoding creates a mismatch between
individually-decoded tokens and delta_text, use
PreLexedTerminals as split points and reallocate text from
delta_text. If a terminal's text is not found in delta_text,
it is deferred to the next scan() call.
Anchors are resolved right-to-left with ``rfind`` so that each
anchor binds to the *rightmost* available occurrence of its
text. This prevents earlier literal lookalikes (e.g. a user
mentioning ``<tool_call>`` in prose) from stealing the position
of a real special-token anchor that appears later.
If the same anchor text appears multiple times as real special
tokens (not prose), the rightmost-first binding could misalign.
In practice this doesn't happen: each special token ID maps to
a distinct PreLexedTerminal, and duplicates in prose are resolved
by the token-ID filtering layer above.
"""
anchors = [item for item in results if isinstance(item, PreLexedTerminal)]
if not anchors:
return [TextChunk(delta_text)]
# Resolve positions right-to-left: each anchor gets the
# rightmost occurrence that is still before the next anchor.
positions: list[int] = [-1] * len(anchors)
search_end = len(delta_text)
for i in range(len(anchors) - 1, -1, -1):
pos = delta_text.rfind(anchors[i].text, 0, search_end)
if pos >= 0:
positions[i] = pos
search_end = pos
# Build results left-to-right using the resolved positions.
new_results: list[LexerInput] = []
consumed = 0
for i, anchor in enumerate(anchors):
pos = positions[i]
if pos >= consumed:
if pos > consumed:
new_results.append(TextChunk(delta_text[consumed:pos]))
new_results.append(anchor)
consumed = pos + len(anchor.text)
else:
has_later_valid = any(p >= 0 for p in positions[i + 1 :])
if not has_later_valid and consumed < len(delta_text):
self._deferred_post_text += delta_text[consumed:]
consumed = len(delta_text)
self._deferred_terminals.append(anchor)
if consumed < len(delta_text):
new_results.append(TextChunk(delta_text[consumed:]))
return new_results