[Frontend] Add Streaming Parser Engine and new Qwen3 Parser (#45413)
Signed-off-by: Ben Browning <bbrownin@redhat.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
co-authored by
Flora Feng
parent
e3e3cd5458
commit
c4a3f9d137
@@ -0,0 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Streaming parser engine framework for tool call and reasoning extraction.
|
||||
|
||||
Instead of hand-rolling a parser for every model's tool-call / reasoning
|
||||
format, each format is declared as a ParserEngineConfig (terminals,
|
||||
states, and transitions) and a shared incremental engine handles
|
||||
streaming, ambiguity buffering, token-ID mapping, and delta computation.
|
||||
"""
|
||||
|
||||
from vllm.parser.engine.events import EventType, SemanticEvent
|
||||
|
||||
__all__ = [
|
||||
"EventType",
|
||||
"SemanticEvent",
|
||||
]
|
||||
@@ -0,0 +1,199 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Adapters that expose :class:`ParserEngine` through the legacy
|
||||
:class:`ReasoningParser` and :class:`ToolParser` interfaces.
|
||||
|
||||
This lets parser engines flow through the existing serving-layer code
|
||||
paths that expect separate reasoning and tool parser instances, without
|
||||
any changes to the serving layer itself.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator, Sequence
|
||||
from contextlib import contextmanager
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.parser.engine.parser_engine_config import ParserState
|
||||
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
|
||||
from vllm.tool_parsers.abstract_tool_parser import ToolParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
DeltaMessage,
|
||||
ExtractedToolCallInformation,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.utils import Tool
|
||||
|
||||
|
||||
class ParserEngineReasoningAdapter(ReasoningParser):
|
||||
"""Adapts a :class:`ParserEngine` to the :class:`ReasoningParser`
|
||||
interface so parser engines can be used as reasoning parsers in the
|
||||
existing serving code.
|
||||
|
||||
Subclasses set :attr:`_parser_engine_cls` to the concrete
|
||||
:class:`ParserEngine` class.
|
||||
"""
|
||||
|
||||
_parser_engine_cls: type[ParserEngine]
|
||||
engine_based_streaming: bool = True
|
||||
|
||||
def __init__(self, tokenizer: TokenizerLike, *args, **kwargs) -> None:
|
||||
super().__init__(tokenizer, *args, **kwargs)
|
||||
self._parser_engine = self._parser_engine_cls(tokenizer, **kwargs) # type: ignore[call-arg]
|
||||
|
||||
@contextmanager
|
||||
def _skip_tool_parsing(self) -> Iterator[None]:
|
||||
saved = self._parser_engine.skip_tool_parsing
|
||||
self._parser_engine.skip_tool_parsing = True
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self._parser_engine.skip_tool_parsing = saved
|
||||
|
||||
def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
|
||||
return self._parser_engine.is_reasoning_end(list(input_ids))
|
||||
|
||||
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
|
||||
return self._parser_engine.extract_content_ids(input_ids)
|
||||
|
||||
def extract_reasoning(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> tuple[str | None, str | None]:
|
||||
with self._skip_tool_parsing():
|
||||
return self._parser_engine.extract_reasoning(model_output, request)
|
||||
|
||||
def extract_reasoning_streaming(
|
||||
self,
|
||||
previous_text: str,
|
||||
current_text: str,
|
||||
delta_text: str,
|
||||
previous_token_ids: Sequence[int],
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
) -> DeltaMessage | None:
|
||||
with self._skip_tool_parsing():
|
||||
return self._parser_engine.extract_reasoning_streaming(
|
||||
previous_text,
|
||||
current_text,
|
||||
delta_text,
|
||||
previous_token_ids,
|
||||
current_token_ids,
|
||||
delta_token_ids,
|
||||
)
|
||||
|
||||
@property
|
||||
def reasoning_start_str(self) -> str | None:
|
||||
return self._parser_engine.reasoning_start_str
|
||||
|
||||
@property
|
||||
def reasoning_end_str(self) -> str | None:
|
||||
return self._parser_engine.reasoning_end_str
|
||||
|
||||
def adjust_request(
|
||||
self,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
return self._parser_engine.adjust_request(request)
|
||||
|
||||
def has_engine_confirmed_reasoning_end(self) -> bool:
|
||||
return self._parser_engine.reasoning_ended
|
||||
|
||||
def finish_streaming(self) -> DeltaMessage | None:
|
||||
return self._parser_engine.finish_streaming()
|
||||
|
||||
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
|
||||
return self._parser_engine.count_reasoning_tokens(token_ids)
|
||||
|
||||
|
||||
class ParserEngineToolAdapter(ToolParser):
|
||||
"""Adapts a :class:`ParserEngine` to the :class:`ToolParser` interface.
|
||||
|
||||
:meth:`extract_tool_calls` starts the parser engine in ``CONTENT``
|
||||
state so it can parse reasoning-stripped content (i.e. the output of
|
||||
:meth:`ReasoningParser.extract_reasoning`).
|
||||
|
||||
Subclasses set :attr:`_parser_engine_cls` to the concrete
|
||||
:class:`ParserEngine` class.
|
||||
"""
|
||||
|
||||
_parser_engine_cls: type[ParserEngine]
|
||||
engine_based_streaming: bool = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(tokenizer, tools)
|
||||
self._parser_engine = self._parser_engine_cls(tokenizer, tools, **kwargs) # type: ignore[call-arg]
|
||||
|
||||
def adjust_request(
|
||||
self,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
request = super().adjust_request(request)
|
||||
return self._parser_engine.adjust_request(request)
|
||||
|
||||
def extract_tool_calls(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest,
|
||||
) -> ExtractedToolCallInformation:
|
||||
return self._parser_engine.extract_tool_calls_from_content(
|
||||
model_output, request
|
||||
)
|
||||
|
||||
def extract_tool_calls_streaming(
|
||||
self,
|
||||
previous_text: str,
|
||||
current_text: str,
|
||||
delta_text: str,
|
||||
previous_token_ids: Sequence[int],
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
) -> DeltaMessage | None:
|
||||
engine = self._parser_engine
|
||||
engine.initialize_streaming(initial_state=ParserState.CONTENT)
|
||||
return engine.extract_tool_calls_streaming(
|
||||
previous_text,
|
||||
current_text,
|
||||
delta_text,
|
||||
previous_token_ids,
|
||||
current_token_ids,
|
||||
delta_token_ids,
|
||||
request,
|
||||
)
|
||||
|
||||
def finish_streaming(self) -> DeltaMessage | None:
|
||||
return self._parser_engine.finish_streaming()
|
||||
|
||||
|
||||
def make_adapters(
|
||||
parser_engine_cls: type[ParserEngine],
|
||||
) -> tuple[type[ParserEngineReasoningAdapter], type[ParserEngineToolAdapter]]:
|
||||
reasoning_adapter = type(
|
||||
f"{parser_engine_cls.__name__}ReasoningAdapter",
|
||||
(ParserEngineReasoningAdapter,),
|
||||
{"_parser_engine_cls": parser_engine_cls},
|
||||
)
|
||||
tool_adapter = type(
|
||||
f"{parser_engine_cls.__name__}ToolAdapter",
|
||||
(ParserEngineToolAdapter,),
|
||||
{"_parser_engine_cls": parser_engine_cls},
|
||||
)
|
||||
# Let the serving layer find the adapters and call adjust_request(),
|
||||
# which sets skip_special_tokens=False for the detokenizer.
|
||||
parser_engine_cls.reasoning_parser_cls = reasoning_adapter # type: ignore[attr-defined]
|
||||
parser_engine_cls.tool_parser_cls = tool_adapter # type: ignore[attr-defined]
|
||||
return reasoning_adapter, tool_adapter
|
||||
@@ -0,0 +1,26 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Semantic event types emitted by the streaming parser engine."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
|
||||
|
||||
class EventType(Enum):
|
||||
TEXT_CHUNK = auto()
|
||||
REASONING_START = auto()
|
||||
REASONING_CHUNK = auto()
|
||||
REASONING_END = auto()
|
||||
TOOL_CALL_START = auto()
|
||||
TOOL_NAME = auto()
|
||||
ARG_VALUE_CHUNK = auto()
|
||||
TOOL_CALL_END = auto()
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class SemanticEvent:
|
||||
type: EventType
|
||||
value: str = ""
|
||||
tool_index: int = -1
|
||||
@@ -0,0 +1,210 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Incremental text lexer that converts text chunks into terminal
|
||||
tokens, with prefix-match buffering for ambiguous boundaries."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import regex as re
|
||||
|
||||
CONTENT_TERMINAL = "__CONTENT__"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class TerminalDef:
|
||||
name: str
|
||||
pattern: re.Pattern[str]
|
||||
is_literal: bool = False
|
||||
literal: str = ""
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class LexToken:
|
||||
terminal: str
|
||||
value: str
|
||||
|
||||
|
||||
class LexerShape:
|
||||
"""Immutable pre-computed data derived from terminal definitions.
|
||||
|
||||
Created once per :class:`ParserEngineConfig` and shared across all
|
||||
:class:`IncrementalLexer` instances that use the same config.
|
||||
"""
|
||||
|
||||
__slots__ = (
|
||||
"terminals",
|
||||
"literal_strings",
|
||||
"max_literal_len",
|
||||
"literal_first_chars",
|
||||
"has_only_literals",
|
||||
"prefix_set",
|
||||
"literals_by_first",
|
||||
)
|
||||
|
||||
def __init__(self, terminals: list[TerminalDef]) -> None:
|
||||
self.terminals = sorted(
|
||||
terminals,
|
||||
key=lambda t: (not t.is_literal, -len(t.pattern.pattern)),
|
||||
)
|
||||
literal_strings: list[tuple[str, str]] = []
|
||||
for t in self.terminals:
|
||||
if t.is_literal:
|
||||
literal_strings.append((t.literal, t.name))
|
||||
|
||||
self.literal_strings = literal_strings
|
||||
max_len = 0
|
||||
for lit, _ in literal_strings:
|
||||
if len(lit) > max_len:
|
||||
max_len = len(lit)
|
||||
self.max_literal_len = max_len
|
||||
self.literal_first_chars = frozenset(
|
||||
lit[0] for lit, _ in literal_strings if lit
|
||||
)
|
||||
self.has_only_literals = all(t.is_literal for t in terminals)
|
||||
|
||||
prefix_set: set[str] = set()
|
||||
for lit, _ in literal_strings:
|
||||
for i in range(1, len(lit)):
|
||||
prefix_set.add(lit[:i])
|
||||
self.prefix_set = frozenset(prefix_set)
|
||||
|
||||
by_first: dict[str, list[tuple[str, str]]] = {}
|
||||
for lit, name in literal_strings:
|
||||
if lit:
|
||||
by_first.setdefault(lit[0], []).append((lit, name))
|
||||
self.literals_by_first = by_first
|
||||
|
||||
|
||||
class IncrementalLexer:
|
||||
"""Converts streaming text into terminal tokens.
|
||||
|
||||
The key feature is **prefix-match buffering**: when the text in the
|
||||
buffer could be the start of a multi-character terminal (e.g.
|
||||
``"<tool_"`` that could become ``"<tool_call>"``), the lexer holds
|
||||
the text rather than emitting it. When the next chunk arrives, it
|
||||
either completes the terminal or flushes the buffered text as
|
||||
content.
|
||||
|
||||
Terminals are tried in priority order (literals first, then by
|
||||
descending priority, then by pattern length).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
terminals: list[TerminalDef] | LexerShape,
|
||||
content_terminal: str = CONTENT_TERMINAL,
|
||||
) -> None:
|
||||
if isinstance(terminals, LexerShape):
|
||||
shape = terminals
|
||||
else:
|
||||
shape = LexerShape(terminals)
|
||||
self._shape = shape
|
||||
self.terminals = shape.terminals
|
||||
self.content_terminal = content_terminal
|
||||
self.buffer = ""
|
||||
|
||||
self._literal_strings = shape.literal_strings
|
||||
self._max_literal_len = shape.max_literal_len
|
||||
self._literal_first_chars = shape.literal_first_chars
|
||||
self._has_only_literals = shape.has_only_literals
|
||||
self._prefix_set = shape.prefix_set
|
||||
self._literals_by_first = shape.literals_by_first
|
||||
|
||||
def reset(self) -> None:
|
||||
self.buffer = ""
|
||||
|
||||
def feed(self, text: str) -> list[LexToken]:
|
||||
if not self.buffer and self._has_only_literals and self._literal_first_chars:
|
||||
for ch in text:
|
||||
if ch in self._literal_first_chars:
|
||||
break
|
||||
else:
|
||||
return [LexToken(self.content_terminal, text)]
|
||||
self.buffer += text
|
||||
return self._drain()
|
||||
|
||||
def flush(self) -> list[LexToken]:
|
||||
tokens: list[LexToken] = []
|
||||
if self.buffer:
|
||||
tokens.append(LexToken(self.content_terminal, self.buffer))
|
||||
self.buffer = ""
|
||||
return tokens
|
||||
|
||||
def _drain(self) -> list[LexToken]:
|
||||
tokens: list[LexToken] = []
|
||||
first_chars = self._literal_first_chars
|
||||
content_terminal = self.content_terminal
|
||||
has_only_literals = self._has_only_literals
|
||||
literals_by_first = self._literals_by_first
|
||||
prefix_set = self._prefix_set
|
||||
|
||||
while self.buffer:
|
||||
if has_only_literals and first_chars:
|
||||
has_potential = False
|
||||
for ch in self.buffer:
|
||||
if ch in first_chars:
|
||||
has_potential = True
|
||||
break
|
||||
if not has_potential:
|
||||
tokens.append(LexToken(content_terminal, self.buffer))
|
||||
self.buffer = ""
|
||||
break
|
||||
|
||||
best_match: tuple[str, str, int] | None = None
|
||||
|
||||
first = self.buffer[0]
|
||||
for lit, name in literals_by_first.get(first, ()):
|
||||
if self.buffer.startswith(lit) and (
|
||||
best_match is None or len(lit) > best_match[2]
|
||||
):
|
||||
best_match = (name, lit, len(lit))
|
||||
|
||||
if self.buffer in prefix_set:
|
||||
if best_match is not None:
|
||||
tokens.append(LexToken(best_match[0], best_match[1]))
|
||||
self.buffer = self.buffer[best_match[2] :]
|
||||
continue
|
||||
else:
|
||||
break
|
||||
|
||||
if best_match is not None:
|
||||
tokens.append(LexToken(best_match[0], best_match[1]))
|
||||
self.buffer = self.buffer[best_match[2] :]
|
||||
else:
|
||||
content_end = self._find_content_boundary()
|
||||
if content_end > 0:
|
||||
tokens.append(LexToken(content_terminal, self.buffer[:content_end]))
|
||||
self.buffer = self.buffer[content_end:]
|
||||
else:
|
||||
tokens.append(LexToken(content_terminal, self.buffer[0]))
|
||||
self.buffer = self.buffer[1:]
|
||||
|
||||
return tokens
|
||||
|
||||
def _find_content_boundary(self) -> int:
|
||||
buf = self.buffer
|
||||
n = len(buf)
|
||||
first_chars = self._literal_first_chars
|
||||
for i in range(1, n):
|
||||
if buf[i] not in first_chars:
|
||||
continue
|
||||
remaining = n - i
|
||||
for lit, _ in self._literal_strings:
|
||||
check_len = min(remaining, len(lit))
|
||||
if buf[i : i + check_len] == lit[:check_len]:
|
||||
return i
|
||||
return n
|
||||
|
||||
|
||||
def terminals_from_literals(literals: dict[str, str]) -> list[TerminalDef]:
|
||||
return [
|
||||
TerminalDef(
|
||||
name=name,
|
||||
pattern=re.compile(re.escape(lit)),
|
||||
is_literal=True,
|
||||
literal=lit,
|
||||
)
|
||||
for name, lit in literals.items()
|
||||
]
|
||||
@@ -0,0 +1,969 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Parser engine base that handles both reasoning and tool call
|
||||
extraction with a single :class:`StreamingParserEngine`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from functools import cached_property
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import regex as re
|
||||
|
||||
from vllm.entrypoints.chat_utils import make_tool_call_id
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
DeltaToolCall,
|
||||
ExtractedToolCallInformation,
|
||||
FunctionCall,
|
||||
ToolCall,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser.abstract_parser import Parser, StreamState
|
||||
from vllm.parser.engine.events import EventType, SemanticEvent
|
||||
from vllm.parser.engine.parser_engine_config import ParserEngineConfig, ParserState
|
||||
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
|
||||
from vllm.tool_parsers.utils import (
|
||||
coerce_to_schema_type,
|
||||
extract_types_from_schema,
|
||||
find_tool_properties,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ToolCallSlot:
|
||||
__slots__ = (
|
||||
"id",
|
||||
"name",
|
||||
"_args_parts",
|
||||
"_args_joined",
|
||||
"name_sent",
|
||||
"streamed_json",
|
||||
)
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.id: str = ""
|
||||
self.name: str = ""
|
||||
self._args_parts: list[str] = []
|
||||
self._args_joined: str | None = ""
|
||||
self.name_sent: bool = False
|
||||
self.streamed_json: str = ""
|
||||
|
||||
@property
|
||||
def args(self) -> str:
|
||||
if self._args_joined is None:
|
||||
self._args_joined = "".join(self._args_parts)
|
||||
return self._args_joined
|
||||
|
||||
def append_args(self, value: str) -> None:
|
||||
self._args_parts.append(value)
|
||||
self._args_joined = None
|
||||
|
||||
|
||||
class ParserEngine(Parser):
|
||||
"""A :class:`Parser` backed by a single declarative engine config.
|
||||
|
||||
Subclasses set the ``ParserEngineConfig`` in ``__init__`` to define the
|
||||
complete output format for a model (reasoning + tool calls).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
*,
|
||||
parser_engine_config: ParserEngineConfig,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
self.model_tokenizer = tokenizer
|
||||
self._tools = tools
|
||||
self._stream_state = StreamState()
|
||||
self._reasoning_parser = None
|
||||
self._tool_parser = None
|
||||
self.parser_engine_config = parser_engine_config
|
||||
self._engine = StreamingParserEngine(
|
||||
parser_engine_config, tokenizer, vocab=self.vocab
|
||||
)
|
||||
|
||||
self._reasoning_ended: bool = False
|
||||
self._streaming_initialized: bool = False
|
||||
|
||||
self._tool_slots: list[ToolCallSlot] = []
|
||||
self._deferred_content: str = ""
|
||||
self._deferred_reasoning: str = ""
|
||||
self._content_has_nonws: bool = False
|
||||
|
||||
self._arg_converter = parser_engine_config.arg_converter
|
||||
self._arg_structural_chars = parser_engine_config.arg_structural_chars
|
||||
self._stream_arg_deltas = parser_engine_config.stream_arg_deltas
|
||||
self._strip_trailing_reasoning_ws = (
|
||||
parser_engine_config.strip_trailing_reasoning_whitespace
|
||||
)
|
||||
self._drop_ws_only_content_before_tools = (
|
||||
parser_engine_config.drop_whitespace_only_content_before_tools
|
||||
)
|
||||
self._strip_content_ws_with_tools = (
|
||||
parser_engine_config.strip_content_whitespace_with_tools
|
||||
)
|
||||
|
||||
vocab = self.vocab
|
||||
self._reasoning_start_token_id: int | None = None
|
||||
self._reasoning_end_token_id: int | None = None
|
||||
|
||||
start_text = parser_engine_config.token_id_terminals.get("THINK_START")
|
||||
end_text = parser_engine_config.token_id_terminals.get("THINK_END")
|
||||
if start_text:
|
||||
self._reasoning_start_token_id = vocab.get(start_text)
|
||||
if end_text:
|
||||
self._reasoning_end_token_id = vocab.get(end_text)
|
||||
|
||||
@property
|
||||
def reasoning_start_str(self) -> str | None:
|
||||
return self.parser_engine_config.terminals.get("THINK_START")
|
||||
|
||||
@property
|
||||
def reasoning_end_str(self) -> str | None:
|
||||
return self.parser_engine_config.terminals.get("THINK_END")
|
||||
|
||||
@cached_property
|
||||
def vocab(self) -> dict[str, int]:
|
||||
return self.model_tokenizer.get_vocab()
|
||||
|
||||
# ── Engine lifecycle ──────────────────────────────────────────────
|
||||
|
||||
@property
|
||||
def skip_tool_parsing(self) -> bool:
|
||||
return self._engine.skip_tool_parsing
|
||||
|
||||
@skip_tool_parsing.setter
|
||||
def skip_tool_parsing(self, value: bool) -> None:
|
||||
self._engine.skip_tool_parsing = value
|
||||
|
||||
@property
|
||||
def reasoning_ended(self) -> bool:
|
||||
return self._reasoning_ended
|
||||
|
||||
def initialize_streaming(
|
||||
self,
|
||||
initial_state: ParserState | None = None,
|
||||
) -> None:
|
||||
if not self._streaming_initialized:
|
||||
self._streaming_initialized = True
|
||||
self._reset(initial_state=initial_state)
|
||||
|
||||
def finish_streaming(self) -> DeltaMessage | None:
|
||||
events = self._engine.finish()
|
||||
return self._events_to_delta(events) if events else None
|
||||
|
||||
def _reset(self, initial_state: ParserState | None = None) -> None:
|
||||
self._engine.reset(initial_state=initial_state)
|
||||
self._reasoning_ended = False
|
||||
self._tool_slots.clear()
|
||||
self._deferred_content = ""
|
||||
self._deferred_reasoning = ""
|
||||
self._content_has_nonws = False
|
||||
|
||||
def adjust_request(
|
||||
self, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
request.skip_special_tokens = False
|
||||
return request
|
||||
|
||||
# ── Schema-aware type correction ─────────────────────────────────
|
||||
|
||||
@staticmethod
|
||||
def _coerce_value(value: object, schema: dict) -> tuple[object, bool]:
|
||||
"""Coerce a single value according to its schema.
|
||||
|
||||
Returns ``(coerced_value, changed)``.
|
||||
"""
|
||||
if isinstance(value, str):
|
||||
types = extract_types_from_schema(schema)
|
||||
coerced = coerce_to_schema_type(value, types)
|
||||
if coerced is not value:
|
||||
return coerced, True
|
||||
return value, False
|
||||
|
||||
if isinstance(value, dict):
|
||||
nested_props = schema.get("properties")
|
||||
if isinstance(nested_props, dict):
|
||||
_, changed = ParserEngine._coerce_dict(value, nested_props)
|
||||
return value, changed
|
||||
return value, False
|
||||
|
||||
if isinstance(value, list):
|
||||
items_schema = schema.get("items")
|
||||
if isinstance(items_schema, dict):
|
||||
changed = False
|
||||
for i, item in enumerate(value):
|
||||
coerced, item_changed = ParserEngine._coerce_value(
|
||||
item, items_schema
|
||||
)
|
||||
if item_changed:
|
||||
value[i] = coerced
|
||||
changed = True
|
||||
return value, changed
|
||||
return value, False
|
||||
|
||||
types = extract_types_from_schema(schema)
|
||||
as_str = json.dumps(value, ensure_ascii=False)
|
||||
coerced = coerce_to_schema_type(as_str, types)
|
||||
if coerced != value:
|
||||
return coerced, True
|
||||
return value, False
|
||||
|
||||
@staticmethod
|
||||
def _coerce_dict(args: dict, properties: dict) -> tuple[dict, bool]:
|
||||
"""Coerce all values in *args* using *properties* schemas."""
|
||||
changed = False
|
||||
for key, value in args.items():
|
||||
prop = properties.get(key)
|
||||
if not isinstance(prop, dict):
|
||||
continue
|
||||
coerced, val_changed = ParserEngine._coerce_value(value, prop)
|
||||
if val_changed:
|
||||
args[key] = coerced
|
||||
changed = True
|
||||
return args, changed
|
||||
|
||||
@staticmethod
|
||||
def _safe_arg_prefix(json_str: str) -> str:
|
||||
"""Return the prefix of *json_str* up to the last top-level value.
|
||||
|
||||
Middle values (followed by a comma) are stable across streaming
|
||||
ticks and included. The trailing value is excluded because type
|
||||
coercion may change its serialised form between ticks, which
|
||||
would violate the ``startswith(prev)`` prefix invariant.
|
||||
"""
|
||||
last_colon = -1
|
||||
in_string = False
|
||||
escape = False
|
||||
depth = 0
|
||||
for i, c in enumerate(json_str):
|
||||
if escape:
|
||||
escape = False
|
||||
continue
|
||||
if in_string:
|
||||
if c == "\\":
|
||||
escape = True
|
||||
elif c == '"':
|
||||
in_string = False
|
||||
continue
|
||||
if c == '"':
|
||||
in_string = True
|
||||
elif c in ("{", "["):
|
||||
depth += 1
|
||||
elif c in ("}", "]"):
|
||||
depth -= 1
|
||||
elif c == ":" and depth == 1:
|
||||
last_colon = i
|
||||
if last_colon < 0:
|
||||
return ""
|
||||
end = last_colon + 1
|
||||
while end < len(json_str) and json_str[end] in (" ", "\t", "\n", "\r"):
|
||||
end += 1
|
||||
return json_str[:end]
|
||||
|
||||
def _fix_arg_types(self, args_json: str, func_name: str) -> str:
|
||||
"""Correct parameter types using the tool schema.
|
||||
|
||||
String values are coerced via :func:`coerce_to_schema_type`.
|
||||
Nested objects and arrays are recursed into when the schema
|
||||
defines ``properties`` or ``items``. Without a schema, values
|
||||
stay as strings.
|
||||
"""
|
||||
if not self._tools or not func_name:
|
||||
return args_json
|
||||
try:
|
||||
args = json.loads(args_json)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return args_json
|
||||
if not isinstance(args, dict):
|
||||
return args_json
|
||||
|
||||
properties = find_tool_properties(self._tools, func_name)
|
||||
if not properties:
|
||||
return args_json
|
||||
|
||||
_, changed = self._coerce_dict(args, properties)
|
||||
|
||||
if changed:
|
||||
return json.dumps(args, ensure_ascii=False)
|
||||
return args_json
|
||||
|
||||
# ── Private helpers ─────────────────────────────────────────────
|
||||
|
||||
def _check_skip_tool_parsing(
|
||||
self,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> None:
|
||||
if not self.skip_tool_parsing:
|
||||
tool_choice = getattr(request, "tool_choice", None)
|
||||
tools = getattr(request, "tools", None)
|
||||
if tool_choice == "none" and tools:
|
||||
self.skip_tool_parsing = True
|
||||
|
||||
def _strip_content_whitespace(
|
||||
self,
|
||||
content: str,
|
||||
tools_called: bool,
|
||||
) -> str | None:
|
||||
if tools_called:
|
||||
if self._strip_content_ws_with_tools:
|
||||
content = content.strip()
|
||||
elif self._drop_ws_only_content_before_tools and not content.strip():
|
||||
content = ""
|
||||
return content or None
|
||||
|
||||
# ── Streaming: parse_delta ────────────────────────────────────────
|
||||
|
||||
def parse_delta(
|
||||
self,
|
||||
delta_text: str,
|
||||
delta_token_ids: list[int],
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
*,
|
||||
finished: bool,
|
||||
) -> DeltaMessage | None:
|
||||
self._check_skip_tool_parsing(request)
|
||||
events = self._engine.feed(delta_text, delta_token_ids)
|
||||
if finished:
|
||||
events.extend(self._engine.finish())
|
||||
result = self._events_to_delta(events, finished=finished)
|
||||
return self._strip_trailing_reasoning(result)
|
||||
|
||||
def _strip_trailing_reasoning(
|
||||
self,
|
||||
delta: DeltaMessage | None,
|
||||
) -> DeltaMessage | None:
|
||||
"""Strip trailing whitespace from reasoning, deferring it until we
|
||||
know whether more reasoning follows or reasoning has ended.
|
||||
|
||||
Runs in ``parse_delta`` *after* ``_events_to_delta`` (and any
|
||||
subclass overrides) so that overrides see the raw reasoning text.
|
||||
|
||||
Gated by ``strip_trailing_reasoning_whitespace``; when disabled,
|
||||
passes through unchanged.
|
||||
"""
|
||||
if not self._strip_trailing_reasoning_ws:
|
||||
return delta
|
||||
if delta is not None and delta.reasoning is not None:
|
||||
combined = self._deferred_reasoning + delta.reasoning
|
||||
trimmed = combined.rstrip()
|
||||
self._deferred_reasoning = combined[len(trimmed) :]
|
||||
delta.reasoning = trimmed or None
|
||||
if (
|
||||
delta.reasoning is None
|
||||
and delta.content is None
|
||||
and not delta.tool_calls
|
||||
):
|
||||
return None
|
||||
elif self._deferred_reasoning and self._reasoning_ended:
|
||||
self._deferred_reasoning = ""
|
||||
return delta
|
||||
|
||||
# ── Non-streaming: extract_reasoning ──────────────────────────────
|
||||
|
||||
def extract_reasoning(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> tuple[str | None, str | None]:
|
||||
self._reset()
|
||||
events = self._engine.feed(model_output, [])
|
||||
events.extend(self._engine.finish())
|
||||
|
||||
reasoning_parts: list[str] = []
|
||||
content_parts: list[str] = []
|
||||
|
||||
for event in events:
|
||||
if event.type == EventType.REASONING_CHUNK:
|
||||
reasoning_parts.append(event.value)
|
||||
elif event.type == EventType.TEXT_CHUNK:
|
||||
content_parts.append(event.value)
|
||||
elif event.type == EventType.REASONING_END:
|
||||
self._reasoning_ended = True
|
||||
|
||||
raw_reasoning = "".join(reasoning_parts)
|
||||
if self._strip_trailing_reasoning_ws:
|
||||
raw_reasoning = raw_reasoning.rstrip()
|
||||
reasoning = raw_reasoning or None
|
||||
content = "".join(content_parts) or None
|
||||
return reasoning, content
|
||||
|
||||
# ── Non-streaming: extract_reasoning_streaming ────────────────────
|
||||
|
||||
def extract_reasoning_streaming(
|
||||
self,
|
||||
previous_text: str,
|
||||
current_text: str,
|
||||
delta_text: str,
|
||||
previous_token_ids: Sequence[int],
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
) -> DeltaMessage | None:
|
||||
self.initialize_streaming()
|
||||
events = self._engine.feed(delta_text, delta_token_ids)
|
||||
return self._strip_trailing_reasoning(self._events_to_delta(events))
|
||||
|
||||
# ── Non-streaming: extract_tool_calls ─────────────────────────────
|
||||
|
||||
def extract_tool_calls(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> ExtractedToolCallInformation:
|
||||
self._reset()
|
||||
self._streaming_initialized = True
|
||||
result = self.extract_tool_calls_streaming(
|
||||
previous_text="",
|
||||
current_text=model_output,
|
||||
delta_text=model_output,
|
||||
previous_token_ids=[],
|
||||
current_token_ids=[],
|
||||
delta_token_ids=[],
|
||||
request=request,
|
||||
)
|
||||
finish_delta = self.finish_streaming()
|
||||
return self._build_extracted_result(result, finish_delta)
|
||||
|
||||
def extract_tool_calls_from_content(
|
||||
self,
|
||||
content: str,
|
||||
request: ChatCompletionRequest,
|
||||
) -> ExtractedToolCallInformation:
|
||||
"""Extract tool calls from reasoning-stripped content.
|
||||
|
||||
Unlike :meth:`extract_tool_calls` which re-parses the full model
|
||||
output, this method starts the parser engine in ``CONTENT`` state
|
||||
so it can parse content that has already had reasoning stripped.
|
||||
"""
|
||||
_, parsed_content, tool_call_info = self._single_pass_parse(
|
||||
content,
|
||||
[],
|
||||
initial_state=ParserState.CONTENT,
|
||||
)
|
||||
if parsed_content is not None and tool_call_info.content is None:
|
||||
tool_call_info = ExtractedToolCallInformation(
|
||||
tools_called=tool_call_info.tools_called,
|
||||
tool_calls=tool_call_info.tool_calls,
|
||||
content=parsed_content,
|
||||
)
|
||||
return tool_call_info
|
||||
|
||||
def extract_tool_calls_streaming(
|
||||
self,
|
||||
previous_text: str,
|
||||
current_text: str,
|
||||
delta_text: str,
|
||||
previous_token_ids: Sequence[int],
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> DeltaMessage | None:
|
||||
self.initialize_streaming()
|
||||
self._check_skip_tool_parsing(request)
|
||||
events = self._engine.feed(delta_text, delta_token_ids)
|
||||
return self._strip_trailing_reasoning(self._events_to_delta(events))
|
||||
|
||||
# ── Reasoning state queries ───────────────────────────────────────
|
||||
|
||||
def is_reasoning_end(self, input_ids: list[int]) -> bool:
|
||||
end_id = self._reasoning_end_token_id
|
||||
start_id = self._reasoning_start_token_id
|
||||
if end_id is not None:
|
||||
if not input_ids:
|
||||
return self.parser_engine_config.initial_state != ParserState.REASONING
|
||||
for i in range(len(input_ids) - 1, -1, -1):
|
||||
if input_ids[i] == end_id:
|
||||
return True
|
||||
if start_id is not None and input_ids[i] == start_id:
|
||||
return False
|
||||
return False
|
||||
return self._reasoning_ended
|
||||
|
||||
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
|
||||
end_id = self._reasoning_end_token_id
|
||||
if end_id is not None:
|
||||
for i in range(len(input_ids) - 1, -1, -1):
|
||||
if input_ids[i] == end_id:
|
||||
return input_ids[i + 1 :]
|
||||
return input_ids
|
||||
|
||||
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
|
||||
start_id = self._reasoning_start_token_id
|
||||
end_id = self._reasoning_end_token_id
|
||||
if start_id is None or end_id is None:
|
||||
return 0
|
||||
count = 0
|
||||
depth = 0
|
||||
for token_id in token_ids:
|
||||
if token_id == start_id:
|
||||
depth += 1
|
||||
continue
|
||||
if token_id == end_id:
|
||||
if depth > 0:
|
||||
depth -= 1
|
||||
continue
|
||||
if depth > 0:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
# ── Single-pass parse helper ────────────────────────────────────────
|
||||
|
||||
def _single_pass_parse(
|
||||
self,
|
||||
text: str,
|
||||
token_ids: Sequence[int],
|
||||
initial_state: ParserState | None = None,
|
||||
) -> tuple[str | None, str | None, ExtractedToolCallInformation]:
|
||||
"""Reset, feed, finish, and extract results in one pass.
|
||||
|
||||
Must be called as a unit — ``_events_to_delta`` populates tool
|
||||
state that ``_build_extracted_result`` reads.
|
||||
"""
|
||||
self._reset(initial_state=initial_state)
|
||||
events = self._engine.feed(text, token_ids)
|
||||
events.extend(self._engine.finish())
|
||||
|
||||
delta = self._events_to_delta(events)
|
||||
tool_call_info = self._build_extracted_result()
|
||||
|
||||
reasoning = delta.reasoning if delta else None
|
||||
if reasoning and self._strip_trailing_reasoning_ws:
|
||||
reasoning = reasoning.rstrip() or None
|
||||
|
||||
content = delta.content if delta else None
|
||||
if content:
|
||||
content = self._strip_content_whitespace(
|
||||
content, tool_call_info.tools_called
|
||||
)
|
||||
|
||||
return reasoning, content, tool_call_info
|
||||
|
||||
# ── Non-streaming: parse ───────────────────────────────────────────
|
||||
|
||||
def parse(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
enable_auto_tools: bool = False,
|
||||
model_output_token_ids: Sequence[int] = (),
|
||||
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
|
||||
reasoning, content, tool_call_info = self._single_pass_parse(
|
||||
model_output,
|
||||
model_output_token_ids,
|
||||
)
|
||||
|
||||
tool_calls: list[FunctionCall] | None = None
|
||||
if tool_call_info.tools_called:
|
||||
tool_calls = [
|
||||
FunctionCall(
|
||||
id=tc.id,
|
||||
name=tc.function.name,
|
||||
arguments=tc.function.arguments,
|
||||
)
|
||||
for tc in tool_call_info.tool_calls
|
||||
]
|
||||
|
||||
return reasoning, content, tool_calls
|
||||
|
||||
# ── Event-to-delta conversion ─────────────────────────────────────
|
||||
|
||||
def _events_to_delta(
|
||||
self,
|
||||
events: list[SemanticEvent],
|
||||
finished: bool = False,
|
||||
) -> DeltaMessage | None:
|
||||
if not events and not self._deferred_content:
|
||||
return None
|
||||
|
||||
tool_call_deltas: list[DeltaToolCall] = []
|
||||
content_parts: list[str] = []
|
||||
reasoning_parts: list[str] = []
|
||||
|
||||
seen_tool_event = False
|
||||
for event in events:
|
||||
match event.type:
|
||||
case EventType.TEXT_CHUNK:
|
||||
if seen_tool_event:
|
||||
self._deferred_content += event.value
|
||||
else:
|
||||
content_parts.append(event.value)
|
||||
case EventType.REASONING_CHUNK:
|
||||
reasoning_parts.append(event.value)
|
||||
case EventType.REASONING_END:
|
||||
self._reasoning_ended = True
|
||||
case EventType.TOOL_CALL_START:
|
||||
seen_tool_event = True
|
||||
self._ensure_slot(event.tool_index)
|
||||
case EventType.TOOL_NAME:
|
||||
seen_tool_event = True
|
||||
self._handle_tool_name(event)
|
||||
case EventType.ARG_VALUE_CHUNK:
|
||||
seen_tool_event = True
|
||||
self._handle_arg_chunk(event, tool_call_deltas)
|
||||
case EventType.TOOL_CALL_END:
|
||||
seen_tool_event = True
|
||||
self._handle_tool_end(event, tool_call_deltas)
|
||||
case EventType.REASONING_START:
|
||||
pass # no delta-level effect
|
||||
|
||||
if len(tool_call_deltas) > 1:
|
||||
tool_call_deltas = self._coalesce_tool_call_deltas(tool_call_deltas)
|
||||
|
||||
if self._deferred_content and not seen_tool_event:
|
||||
content_parts.insert(0, self._deferred_content)
|
||||
self._deferred_content = ""
|
||||
|
||||
content_str = "".join(content_parts)
|
||||
|
||||
if self._content_has_nonws:
|
||||
pass
|
||||
elif content_str:
|
||||
stripped = content_str.strip()
|
||||
if stripped:
|
||||
self._content_has_nonws = True
|
||||
elif self._tool_slots:
|
||||
if self._drop_ws_only_content_before_tools:
|
||||
content_str = ""
|
||||
elif not finished:
|
||||
self._deferred_content = content_str
|
||||
content_str = ""
|
||||
|
||||
content = content_str or None
|
||||
reasoning = "".join(reasoning_parts) or None
|
||||
|
||||
if content or tool_call_deltas or reasoning:
|
||||
kwargs: dict[str, object] = {}
|
||||
if content is not None:
|
||||
kwargs["content"] = content
|
||||
if reasoning is not None:
|
||||
kwargs["reasoning"] = reasoning
|
||||
if tool_call_deltas:
|
||||
kwargs["tool_calls"] = tool_call_deltas
|
||||
return DeltaMessage(**kwargs)
|
||||
return None
|
||||
|
||||
def _ensure_slot(self, idx: int) -> None:
|
||||
while len(self._tool_slots) <= idx:
|
||||
self._tool_slots.append(ToolCallSlot())
|
||||
|
||||
def _ensure_tool_id(self, slot: ToolCallSlot, name: str) -> None:
|
||||
if not slot.id:
|
||||
state = self._stream_state
|
||||
slot.id = make_tool_call_id(
|
||||
id_type=state.tool_call_id_type,
|
||||
func_name=name,
|
||||
idx=state.history_tool_call_cnt,
|
||||
)
|
||||
state.history_tool_call_cnt += 1
|
||||
|
||||
def _handle_tool_name(self, event: SemanticEvent) -> None:
|
||||
idx = event.tool_index
|
||||
self._tool_slots[idx].name += event.value
|
||||
|
||||
def _emit_name_delta(
|
||||
self,
|
||||
idx: int,
|
||||
deltas: list[DeltaToolCall],
|
||||
name: str | None,
|
||||
) -> None:
|
||||
if not name:
|
||||
return
|
||||
slot = self._tool_slots[idx]
|
||||
slot.name = name
|
||||
slot.name_sent = True
|
||||
self._ensure_tool_id(slot, name)
|
||||
deltas.append(
|
||||
DeltaToolCall(
|
||||
index=idx,
|
||||
id=slot.id,
|
||||
type="function",
|
||||
function=DeltaFunctionCall(name=name),
|
||||
)
|
||||
)
|
||||
|
||||
def _handle_arg_chunk(
|
||||
self,
|
||||
event: SemanticEvent,
|
||||
deltas: list[DeltaToolCall],
|
||||
) -> None:
|
||||
idx = event.tool_index
|
||||
slot = self._tool_slots[idx]
|
||||
if event.value:
|
||||
slot.append_args(event.value)
|
||||
|
||||
if not slot.name_sent:
|
||||
if slot.name:
|
||||
self._emit_name_delta(idx, deltas, slot.name)
|
||||
elif event.value:
|
||||
# Name not yet known — try to extract from accumulated args
|
||||
name = self._try_extract_name(idx)
|
||||
self._emit_name_delta(idx, deltas, name)
|
||||
elif event.value:
|
||||
# Name already sent — emit arg delta
|
||||
arg_delta = self._compute_arg_delta(idx, event.value)
|
||||
if arg_delta:
|
||||
deltas.append(
|
||||
DeltaToolCall(
|
||||
index=idx,
|
||||
function=DeltaFunctionCall(arguments=arg_delta),
|
||||
)
|
||||
)
|
||||
|
||||
def _handle_tool_end(
|
||||
self,
|
||||
event: SemanticEvent,
|
||||
deltas: list[DeltaToolCall],
|
||||
) -> None:
|
||||
idx = event.tool_index
|
||||
if idx >= len(self._tool_slots):
|
||||
return
|
||||
|
||||
remaining = self._flush_arg_converter(idx)
|
||||
slot = self._tool_slots[idx]
|
||||
|
||||
if not slot.name_sent:
|
||||
name = slot.name or self._try_extract_name(idx)
|
||||
if name:
|
||||
slot.name = name
|
||||
slot.name_sent = True
|
||||
self._ensure_tool_id(slot, name)
|
||||
deltas.append(
|
||||
DeltaToolCall(
|
||||
index=idx,
|
||||
id=slot.id,
|
||||
type="function",
|
||||
function=DeltaFunctionCall(
|
||||
name=name,
|
||||
arguments=remaining or "",
|
||||
),
|
||||
)
|
||||
)
|
||||
remaining = None
|
||||
|
||||
if remaining and slot.name_sent:
|
||||
deltas.append(
|
||||
DeltaToolCall(
|
||||
index=idx,
|
||||
function=DeltaFunctionCall(arguments=remaining),
|
||||
)
|
||||
)
|
||||
|
||||
# ── Tool-call delta coalescing ──────────────────────────────────────
|
||||
|
||||
@staticmethod
|
||||
def _coalesce_tool_call_deltas(
|
||||
deltas: list[DeltaToolCall],
|
||||
) -> list[DeltaToolCall]:
|
||||
"""Merge entries that share the same index into one per index."""
|
||||
merged: dict[int, DeltaToolCall] = {}
|
||||
for tc in deltas:
|
||||
existing = merged.get(tc.index)
|
||||
if existing is None:
|
||||
merged[tc.index] = tc
|
||||
continue
|
||||
if tc.id is not None and existing.id is None:
|
||||
existing.id = tc.id
|
||||
if tc.type is not None and existing.type is None:
|
||||
existing.type = tc.type
|
||||
if tc.function is not None:
|
||||
if existing.function is None:
|
||||
existing.function = tc.function
|
||||
else:
|
||||
if tc.function.name is not None and existing.function.name is None:
|
||||
existing.function.name = tc.function.name
|
||||
if tc.function.arguments is not None:
|
||||
if existing.function.arguments is None:
|
||||
existing.function.arguments = tc.function.arguments
|
||||
else:
|
||||
existing.function.arguments += tc.function.arguments
|
||||
if len(merged) == len(deltas):
|
||||
return deltas
|
||||
return list(merged.values())
|
||||
|
||||
# ── Arg conversion helpers ─────────────────────────────────────────
|
||||
|
||||
def _compute_arg_delta(self, idx: int, raw_delta: str) -> str | None:
|
||||
converter = self._arg_converter
|
||||
if converter is None:
|
||||
return raw_delta
|
||||
|
||||
if not self._stream_arg_deltas:
|
||||
return None
|
||||
|
||||
structural = self._arg_structural_chars
|
||||
if structural is not None and structural.isdisjoint(raw_delta):
|
||||
return None
|
||||
|
||||
slot = self._tool_slots[idx]
|
||||
try:
|
||||
current_json = converter(slot.args, True)
|
||||
except (json.JSONDecodeError, ValueError, TypeError):
|
||||
logger.debug("arg converter failed (streaming): %s", slot.args[:80])
|
||||
return None
|
||||
|
||||
if not current_json:
|
||||
return None
|
||||
|
||||
if slot.name:
|
||||
current_json = self._fix_arg_types(current_json, slot.name)
|
||||
|
||||
prev = slot.streamed_json
|
||||
safe_json = self._safe_arg_prefix(current_json)
|
||||
|
||||
if not safe_json or safe_json == prev:
|
||||
return None
|
||||
|
||||
if prev:
|
||||
if not safe_json.startswith(prev):
|
||||
return None
|
||||
diff = safe_json[len(prev) :]
|
||||
else:
|
||||
diff = safe_json
|
||||
|
||||
if diff:
|
||||
slot.streamed_json = safe_json
|
||||
return diff
|
||||
return None
|
||||
|
||||
def _flush_arg_converter(self, idx: int) -> str | None:
|
||||
converter = self._arg_converter
|
||||
if converter is None:
|
||||
return None
|
||||
|
||||
slot = self._tool_slots[idx]
|
||||
try:
|
||||
final_json = converter(slot.args, False)
|
||||
except (json.JSONDecodeError, ValueError, TypeError):
|
||||
logger.debug("arg converter failed (flush): %s", slot.args[:80])
|
||||
return None
|
||||
|
||||
if final_json:
|
||||
final_json = self._fix_arg_types(final_json, slot.name)
|
||||
|
||||
prev = slot.streamed_json
|
||||
if final_json and len(final_json) > len(prev):
|
||||
if prev and not final_json.startswith(prev):
|
||||
return None
|
||||
diff = final_json[len(prev) :]
|
||||
slot.streamed_json = final_json
|
||||
return diff
|
||||
return None
|
||||
|
||||
_NAME_RE = re.compile(r'"name"\s*:\s*"([^"]*)"')
|
||||
|
||||
def _try_extract_name(self, idx: int) -> str | None:
|
||||
m = self._NAME_RE.search(self._tool_slots[idx].args)
|
||||
if m:
|
||||
name = m.group(1)
|
||||
if name:
|
||||
return name
|
||||
return None
|
||||
|
||||
# ── Build ExtractedToolCallInformation ─────────────────────────────
|
||||
|
||||
def _build_extracted_result(
|
||||
self,
|
||||
*deltas: DeltaMessage | None,
|
||||
) -> ExtractedToolCallInformation:
|
||||
content_parts: list[str] = []
|
||||
for delta in deltas:
|
||||
if delta is not None and delta.content:
|
||||
content_parts.append(delta.content)
|
||||
|
||||
tool_calls: list[ToolCall] = []
|
||||
for idx, slot in enumerate(self._tool_slots):
|
||||
if not slot.name and not slot.args:
|
||||
continue
|
||||
|
||||
name = slot.name.strip()
|
||||
raw_body = slot.args
|
||||
|
||||
if not name and raw_body.strip():
|
||||
name, args_json = self._extract_name_and_args(raw_body)
|
||||
elif raw_body.strip():
|
||||
converter = self._arg_converter
|
||||
if converter is not None:
|
||||
try:
|
||||
args_json = converter(raw_body, False)
|
||||
except (json.JSONDecodeError, ValueError, TypeError):
|
||||
logger.debug(
|
||||
"arg converter failed (extract): %s", raw_body[:80]
|
||||
)
|
||||
args_json = self._extract_args_json(raw_body, name)
|
||||
else:
|
||||
args_json = self._extract_args_json(raw_body, name)
|
||||
else:
|
||||
args_json = "{}"
|
||||
|
||||
if name:
|
||||
self._ensure_tool_id(slot, name)
|
||||
args_json = self._fix_arg_types(args_json, name)
|
||||
tool_calls.append(
|
||||
ToolCall(
|
||||
id=slot.id,
|
||||
function=FunctionCall(name=name, arguments=args_json),
|
||||
)
|
||||
)
|
||||
|
||||
content_str = "".join(content_parts)
|
||||
content = self._strip_content_whitespace(content_str, len(tool_calls) > 0)
|
||||
|
||||
return ExtractedToolCallInformation(
|
||||
tools_called=len(tool_calls) > 0,
|
||||
tool_calls=tool_calls,
|
||||
content=content,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _extract_args_value(parsed: dict) -> str | None:
|
||||
for key in ("arguments", "parameters"):
|
||||
if key in parsed:
|
||||
val = parsed[key]
|
||||
if isinstance(val, str):
|
||||
return val
|
||||
return json.dumps(val, ensure_ascii=False)
|
||||
return None
|
||||
|
||||
def _extract_name_and_args(
|
||||
self,
|
||||
raw_body: str,
|
||||
) -> tuple[str, str]:
|
||||
raw_body = raw_body.strip()
|
||||
try:
|
||||
parsed = json.loads(raw_body)
|
||||
except json.JSONDecodeError:
|
||||
return "", raw_body
|
||||
|
||||
if not isinstance(parsed, dict):
|
||||
return "", raw_body
|
||||
|
||||
name = parsed.get("name", "")
|
||||
args = self._extract_args_value(parsed)
|
||||
if args is not None:
|
||||
return name, args
|
||||
|
||||
without_name = {k: v for k, v in parsed.items() if k != "name"}
|
||||
return name, json.dumps(without_name, ensure_ascii=False)
|
||||
|
||||
def _extract_args_json(self, raw_args: str, func_name: str) -> str:
|
||||
if not raw_args.strip():
|
||||
return "{}"
|
||||
_, args = self._extract_name_and_args(raw_args)
|
||||
return args
|
||||
@@ -0,0 +1,114 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Declarative configuration for model tool-call and reasoning formats.
|
||||
|
||||
Each model format is described by a :class:`ParserEngineConfig` that specifies:
|
||||
|
||||
* **terminals** – literal strings or regex patterns that delimit the format
|
||||
(e.g. ``<tool_call>``, ``</think>``).
|
||||
* **token_id_terminals** – terminals that should be matched by token ID
|
||||
rather than (or in addition to) text.
|
||||
* **transitions** – a state machine mapping
|
||||
``(state, terminal) → (new_state, events_to_emit)`` that drives semantic
|
||||
event generation during streaming.
|
||||
* **content_events** – what :class:`EventType` to emit for plain content
|
||||
(non-terminal text) in each state.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum, auto
|
||||
from functools import cached_property
|
||||
|
||||
from vllm.parser.engine.events import EventType
|
||||
|
||||
STRUCTURAL_DROP_TOKENS: frozenset[str] = frozenset(
|
||||
{
|
||||
"<eos>",
|
||||
"<bos>",
|
||||
"<pad>",
|
||||
"<unk>",
|
||||
"<mask>",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class ParserState(Enum):
|
||||
CONTENT = auto()
|
||||
REASONING = auto()
|
||||
TOOL_PREAMBLE = auto()
|
||||
TOOL_NAME = auto()
|
||||
TOOL_ARGS = auto()
|
||||
TOOL_BETWEEN = auto()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Transition:
|
||||
next_state: ParserState
|
||||
events: tuple[EventType, ...] = field(default_factory=tuple)
|
||||
skip_in_token_id_mode: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParserEngineConfig:
|
||||
"""Declarative description of a model's tool-call / reasoning format.
|
||||
|
||||
The engine feeds terminals from the incremental lexer into the
|
||||
transition table and emits the corresponding semantic events.
|
||||
Content tokens (text between terminals) are classified by the
|
||||
current state via ``content_events``.
|
||||
"""
|
||||
|
||||
name: str
|
||||
|
||||
terminals: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
token_id_terminals: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
transitions: dict[tuple[ParserState, str], Transition] = field(
|
||||
default_factory=dict,
|
||||
)
|
||||
|
||||
content_events: dict[ParserState, EventType] = field(
|
||||
default_factory=lambda: {
|
||||
ParserState.CONTENT: EventType.TEXT_CHUNK,
|
||||
ParserState.REASONING: EventType.REASONING_CHUNK,
|
||||
ParserState.TOOL_NAME: EventType.TOOL_NAME,
|
||||
ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
|
||||
},
|
||||
)
|
||||
|
||||
initial_state: ParserState = ParserState.CONTENT
|
||||
|
||||
arg_converter: Callable[[str, bool], str] | None = None
|
||||
|
||||
stream_arg_deltas: bool = True
|
||||
|
||||
tool_args_json: bool = True
|
||||
|
||||
arg_structural_chars: frozenset[str] | None = None
|
||||
|
||||
# Prevents trailing-whitespace accumulation across multi-turn conversations.
|
||||
strip_trailing_reasoning_whitespace: bool = True
|
||||
|
||||
# Drop content that is entirely whitespace when tool calls follow.
|
||||
drop_whitespace_only_content_before_tools: bool = True
|
||||
|
||||
# .strip() content text when tool calls are present.
|
||||
strip_content_whitespace_with_tools: bool = True
|
||||
|
||||
drop_tokens: frozenset[str] = field(default_factory=frozenset)
|
||||
|
||||
@cached_property
|
||||
def terminal_defs(self):
|
||||
from vllm.parser.engine.incremental_lexer import terminals_from_literals
|
||||
|
||||
return terminals_from_literals(self.terminals)
|
||||
|
||||
@cached_property
|
||||
def lexer_shape(self):
|
||||
from vllm.parser.engine.incremental_lexer import LexerShape
|
||||
|
||||
return LexerShape(self.terminal_defs)
|
||||
@@ -0,0 +1,16 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Concrete adapter classes for each registered parser engine.
|
||||
|
||||
These are created via :func:`make_adapters` and exposed as module-level
|
||||
names so that :class:`ReasoningParserManager` and
|
||||
:class:`ToolParserManager` can load them lazily.
|
||||
"""
|
||||
|
||||
from vllm.parser.engine.adapters import make_adapters
|
||||
from vllm.parser.qwen3 import Qwen3Parser
|
||||
|
||||
(
|
||||
Qwen3ParserReasoningAdapter,
|
||||
Qwen3ParserToolAdapter,
|
||||
) = make_adapters(Qwen3Parser)
|
||||
@@ -0,0 +1,408 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Streaming parser engine that orchestrates token ID scanning,
|
||||
incremental lexing, and state-machine-driven semantic event emission."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from vllm.parser.engine.events import EventType, SemanticEvent
|
||||
from vllm.parser.engine.incremental_lexer import (
|
||||
CONTENT_TERMINAL,
|
||||
IncrementalLexer,
|
||||
LexToken,
|
||||
)
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
STRUCTURAL_DROP_TOKENS,
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
from vllm.parser.engine.token_id_scanner import (
|
||||
LexerInput,
|
||||
PreLexedTerminal,
|
||||
TextChunk,
|
||||
TokenIDScanner,
|
||||
)
|
||||
|
||||
|
||||
class StreamingParserEngine:
|
||||
"""Consumes ``(delta_text, delta_token_ids)`` pairs and produces a
|
||||
stream of :class:`SemanticEvent` instances.
|
||||
|
||||
This is the main entry point for streaming parsing.
|
||||
Create one per request (it is stateful).
|
||||
|
||||
The pipeline is::
|
||||
|
||||
delta_text + delta_token_ids
|
||||
→ TokenIDScanner (special token pre-lexing)
|
||||
→ IncrementalLexer (text → terminal tokens with prefix buffering)
|
||||
→ State Machine (terminal → semantic events)
|
||||
→ list[SemanticEvent]
|
||||
|
||||
Usage::
|
||||
|
||||
engine = StreamingParserEngine(config, tokenizer)
|
||||
for each streaming delta:
|
||||
events = engine.feed(delta_text, delta_token_ids)
|
||||
# convert events to DeltaMessage
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: ParserEngineConfig,
|
||||
tokenizer,
|
||||
initial_state: ParserState | None = None,
|
||||
vocab: dict[str, int] | None = None,
|
||||
) -> None:
|
||||
self.config = config
|
||||
|
||||
resolved_token_ids: dict[int, str] = {}
|
||||
drop_token_ids: set[int] = set()
|
||||
if tokenizer is not None:
|
||||
if vocab is None:
|
||||
vocab = tokenizer.get_vocab()
|
||||
if config.token_id_terminals:
|
||||
for terminal_name, token_text in config.token_id_terminals.items():
|
||||
tid = vocab.get(token_text)
|
||||
if tid is not None:
|
||||
resolved_token_ids[tid] = terminal_name
|
||||
all_drop = config.drop_tokens | STRUCTURAL_DROP_TOKENS
|
||||
for token_text in all_drop:
|
||||
tid = vocab.get(token_text)
|
||||
if tid is not None:
|
||||
drop_token_ids.add(tid)
|
||||
for attr in ("eos_token_id", "bos_token_id", "pad_token_id"):
|
||||
tid = getattr(tokenizer, attr, None)
|
||||
if tid is not None:
|
||||
drop_token_ids.add(tid)
|
||||
|
||||
self._resolved_token_ids = resolved_token_ids
|
||||
self._drop_token_ids = drop_token_ids
|
||||
|
||||
self._scanner = TokenIDScanner(
|
||||
resolved_token_ids,
|
||||
tokenizer,
|
||||
drop_token_ids,
|
||||
)
|
||||
|
||||
self._token_id_terminal_names: frozenset[str] = frozenset(
|
||||
resolved_token_ids.values()
|
||||
)
|
||||
|
||||
self._lexer = IncrementalLexer(
|
||||
config.lexer_shape, content_terminal=CONTENT_TERMINAL
|
||||
)
|
||||
|
||||
self._tool_terminals: frozenset[str] = frozenset(
|
||||
terminal
|
||||
for (state, terminal), tr in config.transitions.items()
|
||||
if tr.next_state in self._TOOL_STATES or state in self._TOOL_STATES
|
||||
)
|
||||
|
||||
self.skip_tool_parsing = False
|
||||
self.reset(initial_state=initial_state)
|
||||
|
||||
def _reset_args_state(self) -> None:
|
||||
self._args_buffer: str = ""
|
||||
self._args_safe_end: int = 0
|
||||
self._args_brace_depth: int = 0
|
||||
self._args_in_string: bool = False
|
||||
self._args_escape_next: bool = False
|
||||
|
||||
def reset(self, initial_state: ParserState | None = None) -> None:
|
||||
"""Reset mutable state for reuse across requests.
|
||||
|
||||
Preserves cached immutable structures (compiled terminals,
|
||||
resolved token IDs, lexer shape, token text cache) to avoid
|
||||
redundant initialization work.
|
||||
"""
|
||||
self.state = (
|
||||
initial_state if initial_state is not None else self.config.initial_state
|
||||
)
|
||||
self.tool_index = -1
|
||||
self._ever_had_token_ids = False
|
||||
# DO NOT reset skip_tool_parsing here — callers set it before
|
||||
# calling methods that trigger reset() (e.g. extract_reasoning),
|
||||
# and clearing it silently breaks non-streaming tool-call-as-
|
||||
# implicit-reasoning-end (content returns None).
|
||||
self._scanner.reset()
|
||||
self._lexer.reset()
|
||||
self._reset_args_state()
|
||||
|
||||
def feed(
|
||||
self,
|
||||
delta_text: str,
|
||||
delta_token_ids: Sequence[int],
|
||||
) -> list[SemanticEvent]:
|
||||
if delta_token_ids:
|
||||
self._ever_had_token_ids = True
|
||||
|
||||
# Fast path: skip scanner and lexer when the delta is plain
|
||||
# content with no special tokens and no terminal-starting chars.
|
||||
if (
|
||||
delta_text
|
||||
and not self._lexer.buffer
|
||||
and not self._scanner._deferred_terminals
|
||||
and self._lexer._literal_first_chars.isdisjoint(delta_text)
|
||||
):
|
||||
has_special = False
|
||||
for tid in delta_token_ids:
|
||||
if tid in self._resolved_token_ids or tid in self._drop_token_ids:
|
||||
has_special = True
|
||||
break
|
||||
if not has_special:
|
||||
return self._emit_for_state(delta_text)
|
||||
|
||||
scanner_items = self._scanner.scan(delta_text, delta_token_ids)
|
||||
|
||||
if len(scanner_items) == 1 and isinstance(scanner_items[0], TextChunk):
|
||||
lex_tokens = self._lexer.feed(scanner_items[0].text)
|
||||
if len(lex_tokens) == 1 and lex_tokens[0].terminal == CONTENT_TERMINAL:
|
||||
text = lex_tokens[0].value
|
||||
return self._emit_for_state(text)
|
||||
return self._process_lex_tokens(lex_tokens)
|
||||
|
||||
return self._process_scanner_items(scanner_items)
|
||||
|
||||
def _process_scanner_items(
|
||||
self, items: Sequence[LexerInput]
|
||||
) -> list[SemanticEvent]:
|
||||
events: list[SemanticEvent] = []
|
||||
for item in items:
|
||||
if isinstance(item, PreLexedTerminal):
|
||||
events.extend(self._process_lex_tokens(self._lexer.flush()))
|
||||
events.extend(self._on_terminal(item.terminal, item.text))
|
||||
elif isinstance(item, TextChunk):
|
||||
events.extend(self._process_lex_tokens(self._lexer.feed(item.text)))
|
||||
return events
|
||||
|
||||
def finish(self) -> list[SemanticEvent]:
|
||||
events = self._process_scanner_items(self._scanner.flush_pending())
|
||||
|
||||
events.extend(self._process_lex_tokens(self._lexer.flush()))
|
||||
|
||||
if self._args_buffer:
|
||||
events.append(
|
||||
SemanticEvent(
|
||||
EventType.ARG_VALUE_CHUNK,
|
||||
value=self._args_buffer,
|
||||
tool_index=self.tool_index,
|
||||
)
|
||||
)
|
||||
self._args_buffer = ""
|
||||
self._args_safe_end = 0
|
||||
|
||||
if self.state in (
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
ParserState.TOOL_ARGS,
|
||||
ParserState.TOOL_NAME,
|
||||
ParserState.TOOL_BETWEEN,
|
||||
):
|
||||
if self.tool_index >= 0:
|
||||
events.append(
|
||||
SemanticEvent(
|
||||
EventType.TOOL_CALL_END,
|
||||
tool_index=self.tool_index,
|
||||
)
|
||||
)
|
||||
self.state = ParserState.CONTENT
|
||||
elif self.state == ParserState.REASONING:
|
||||
events.append(
|
||||
SemanticEvent(EventType.REASONING_END, tool_index=self.tool_index)
|
||||
)
|
||||
self.state = ParserState.CONTENT
|
||||
|
||||
return events
|
||||
|
||||
def parse_complete(self, text: str) -> list[SemanticEvent]:
|
||||
token_ids: list[int] = []
|
||||
events = self.feed(text, token_ids)
|
||||
events.extend(self.finish())
|
||||
return events
|
||||
|
||||
def _process_lex_tokens(self, tokens: list[LexToken]) -> list[SemanticEvent]:
|
||||
events: list[SemanticEvent] = []
|
||||
strict = self._token_id_terminal_names if self._ever_had_token_ids else None
|
||||
for tok in tokens:
|
||||
if tok.terminal == CONTENT_TERMINAL or (strict and tok.terminal in strict):
|
||||
events.extend(self._on_content(tok.value))
|
||||
else:
|
||||
events.extend(self._on_terminal(tok.terminal, tok.value))
|
||||
return events
|
||||
|
||||
_TOOL_STATES = frozenset(
|
||||
{
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
ParserState.TOOL_NAME,
|
||||
ParserState.TOOL_ARGS,
|
||||
ParserState.TOOL_BETWEEN,
|
||||
}
|
||||
)
|
||||
|
||||
def _on_terminal(self, terminal: str, value: str) -> list[SemanticEvent]:
|
||||
key = (self.state, terminal)
|
||||
transition = self.config.transitions.get(key)
|
||||
|
||||
if transition is None:
|
||||
return self._emit_for_state(value)
|
||||
|
||||
if self.skip_tool_parsing and terminal in self._tool_terminals:
|
||||
if EventType.REASONING_END in transition.events:
|
||||
self.state = ParserState.CONTENT
|
||||
return [
|
||||
SemanticEvent(
|
||||
EventType.REASONING_END,
|
||||
value=value,
|
||||
tool_index=self.tool_index,
|
||||
),
|
||||
SemanticEvent(
|
||||
EventType.TEXT_CHUNK,
|
||||
value=value,
|
||||
tool_index=self.tool_index,
|
||||
),
|
||||
]
|
||||
content_type = self.config.content_events.get(self.state)
|
||||
if content_type is not None:
|
||||
return [
|
||||
SemanticEvent(content_type, value=value, tool_index=self.tool_index)
|
||||
]
|
||||
return []
|
||||
|
||||
if transition.skip_in_token_id_mode and self._ever_had_token_ids:
|
||||
return self._emit_for_state(value)
|
||||
|
||||
return self._apply_transition(transition, value)
|
||||
|
||||
def _emit_for_state(self, text: str) -> list[SemanticEvent]:
|
||||
if self.state == ParserState.TOOL_ARGS:
|
||||
if self.config.tool_args_json:
|
||||
return self._feed_args_text(text)
|
||||
return [
|
||||
SemanticEvent(
|
||||
EventType.ARG_VALUE_CHUNK,
|
||||
value=text,
|
||||
tool_index=self.tool_index,
|
||||
)
|
||||
]
|
||||
content_type = self.config.content_events.get(self.state)
|
||||
if content_type is not None:
|
||||
return [SemanticEvent(content_type, value=text, tool_index=self.tool_index)]
|
||||
return []
|
||||
|
||||
def _on_content(self, text: str) -> list[SemanticEvent]:
|
||||
if not text:
|
||||
return []
|
||||
return self._emit_for_state(text)
|
||||
|
||||
def _apply_transition(
|
||||
self,
|
||||
transition: Transition,
|
||||
value: str,
|
||||
) -> list[SemanticEvent]:
|
||||
events: list[SemanticEvent] = []
|
||||
|
||||
if (
|
||||
self.state == ParserState.TOOL_ARGS
|
||||
and transition.next_state != ParserState.TOOL_ARGS
|
||||
and self._args_buffer
|
||||
):
|
||||
events.append(
|
||||
SemanticEvent(
|
||||
EventType.ARG_VALUE_CHUNK,
|
||||
value=self._args_buffer,
|
||||
tool_index=self.tool_index,
|
||||
)
|
||||
)
|
||||
self._args_buffer = ""
|
||||
|
||||
self.state = transition.next_state
|
||||
|
||||
for event_type in transition.events:
|
||||
if event_type == EventType.TOOL_CALL_START:
|
||||
self.tool_index += 1
|
||||
events.append(
|
||||
SemanticEvent(
|
||||
event_type,
|
||||
value=value,
|
||||
tool_index=self.tool_index,
|
||||
)
|
||||
)
|
||||
|
||||
if self.state == ParserState.TOOL_ARGS:
|
||||
self._args_brace_depth = 0
|
||||
self._args_in_string = False
|
||||
self._args_escape_next = False
|
||||
self._args_safe_end = 0
|
||||
|
||||
return events
|
||||
|
||||
def _feed_args_text(self, text: str) -> list[SemanticEvent]:
|
||||
"""Feed text into the JSON argument streaming buffer.
|
||||
|
||||
Streams argument characters incrementally while holding back
|
||||
closing braces/brackets that might change as more input arrives.
|
||||
"""
|
||||
events: list[SemanticEvent] = []
|
||||
for ch in text:
|
||||
result = self._feed_args_char(ch)
|
||||
events.extend(result)
|
||||
return events
|
||||
|
||||
def _feed_args_char(self, ch: str) -> list[SemanticEvent]:
|
||||
self._args_buffer += ch
|
||||
|
||||
if self._args_escape_next:
|
||||
self._args_escape_next = False
|
||||
self._args_safe_end = len(self._args_buffer)
|
||||
return self._flush_safe_args()
|
||||
|
||||
if self._args_in_string:
|
||||
if ch == "\\":
|
||||
self._args_escape_next = True
|
||||
elif ch == '"':
|
||||
self._args_in_string = False
|
||||
self._args_safe_end = len(self._args_buffer)
|
||||
return self._flush_safe_args()
|
||||
|
||||
if ch == '"':
|
||||
self._args_in_string = True
|
||||
self._args_safe_end = len(self._args_buffer)
|
||||
return self._flush_safe_args()
|
||||
|
||||
if ch in ("{", "["):
|
||||
self._args_brace_depth += 1
|
||||
self._args_safe_end = len(self._args_buffer)
|
||||
return self._flush_safe_args()
|
||||
|
||||
if ch in ("}", "]"):
|
||||
if self._args_brace_depth > 0:
|
||||
self._args_brace_depth -= 1
|
||||
if self._args_brace_depth == 0:
|
||||
return []
|
||||
self._args_safe_end = len(self._args_buffer)
|
||||
return self._flush_safe_args()
|
||||
|
||||
self._args_safe_end = len(self._args_buffer)
|
||||
return self._flush_safe_args()
|
||||
|
||||
def _flush_safe_args(self) -> list[SemanticEvent]:
|
||||
"""Emit buffered argument characters up to the safe-end watermark.
|
||||
|
||||
Top-level closing braces are held back (safe_end not advanced)
|
||||
until confirmed safe by a subsequent character or finish().
|
||||
"""
|
||||
if self._args_safe_end == 0:
|
||||
return []
|
||||
to_emit = self._args_buffer[: self._args_safe_end]
|
||||
self._args_buffer = self._args_buffer[self._args_safe_end :]
|
||||
self._args_safe_end = 0
|
||||
return [
|
||||
SemanticEvent(
|
||||
EventType.ARG_VALUE_CHUNK,
|
||||
value=to_emit,
|
||||
tool_index=self.tool_index,
|
||||
)
|
||||
]
|
||||
@@ -0,0 +1,309 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Scan delta token IDs for special tokens and split the stream into
|
||||
pre-lexed terminals and plain text chunks."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class TextChunk:
|
||||
text: str
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class PreLexedTerminal:
|
||||
terminal: str
|
||||
token_id: int
|
||||
text: str
|
||||
|
||||
|
||||
LexerInput = TextChunk | PreLexedTerminal
|
||||
|
||||
|
||||
class TokenIDScanner:
|
||||
"""Maps special token IDs in the delta to terminals.
|
||||
|
||||
Before text-based lexing happens, the scanner checks each token ID
|
||||
in the delta against a mapping of ``{token_id: terminal_name}``.
|
||||
Matched tokens are emitted as :class:`PreLexedTerminal` items;
|
||||
everything else is grouped into :class:`TextChunk` items for the
|
||||
incremental lexer to process.
|
||||
|
||||
When a terminal's text is not yet in ``delta_text`` (held back by
|
||||
the detokenizer), the terminal is deferred until the text arrives
|
||||
in a subsequent delta.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
token_id_to_terminal: dict[int, str],
|
||||
tokenizer,
|
||||
drop_token_ids: set[int] | None = None,
|
||||
) -> None:
|
||||
self.token_id_to_terminal = token_id_to_terminal
|
||||
self.tokenizer = tokenizer
|
||||
self._token_text_cache: dict[int, str] = {}
|
||||
self._drop_token_ids = drop_token_ids or set()
|
||||
self._deferred_terminals: list[PreLexedTerminal] = []
|
||||
self._deferred_post_text: str = ""
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Clear mutable state for reuse. Preserves the token text cache."""
|
||||
self._deferred_terminals.clear()
|
||||
self._deferred_post_text = ""
|
||||
|
||||
def _decode_token(self, token_id: int) -> str:
|
||||
if token_id not in self._token_text_cache:
|
||||
self._token_text_cache[token_id] = self.tokenizer.decode([token_id])
|
||||
return self._token_text_cache[token_id]
|
||||
|
||||
_EMPTY: tuple[LexerInput, ...] = ()
|
||||
|
||||
def scan(
|
||||
self,
|
||||
delta_text: str,
|
||||
delta_token_ids: Sequence[int],
|
||||
) -> Sequence[LexerInput]:
|
||||
prefix_items: list[LexerInput] = []
|
||||
effective_text = delta_text
|
||||
|
||||
if self._deferred_terminals:
|
||||
prefix_items, effective_text = self._resolve_deferred(delta_text)
|
||||
|
||||
if not self.token_id_to_terminal and not self._drop_token_ids:
|
||||
if effective_text:
|
||||
prefix_items.append(TextChunk(effective_text))
|
||||
return prefix_items
|
||||
|
||||
has_special = False
|
||||
has_drop = False
|
||||
token_id_to_terminal = self.token_id_to_terminal
|
||||
drop_token_ids = self._drop_token_ids
|
||||
for tid in delta_token_ids:
|
||||
if tid in token_id_to_terminal:
|
||||
has_special = True
|
||||
if tid in drop_token_ids:
|
||||
has_drop = True
|
||||
|
||||
if not has_special and not has_drop:
|
||||
if effective_text:
|
||||
if not prefix_items:
|
||||
return [TextChunk(effective_text)]
|
||||
prefix_items.append(TextChunk(effective_text))
|
||||
return prefix_items or self._EMPTY
|
||||
|
||||
token_texts = [self._decode_token(tid) for tid in delta_token_ids]
|
||||
|
||||
results: list[LexerInput] = []
|
||||
text_accum: list[str] = []
|
||||
|
||||
for idx, tid in enumerate(delta_token_ids):
|
||||
if tid in self._drop_token_ids:
|
||||
continue
|
||||
terminal = self.token_id_to_terminal.get(tid)
|
||||
if terminal is not None:
|
||||
if text_accum:
|
||||
joined = "".join(text_accum)
|
||||
if joined:
|
||||
results.append(TextChunk(joined))
|
||||
text_accum.clear()
|
||||
results.append(PreLexedTerminal(terminal, tid, token_texts[idx]))
|
||||
else:
|
||||
text_accum.append(token_texts[idx])
|
||||
|
||||
if text_accum:
|
||||
joined = "".join(text_accum)
|
||||
if joined:
|
||||
results.append(TextChunk(joined))
|
||||
|
||||
if effective_text:
|
||||
if has_drop:
|
||||
clean_delta = effective_text
|
||||
for idx, tid in enumerate(delta_token_ids):
|
||||
if tid in self._drop_token_ids:
|
||||
dropped = token_texts[idx]
|
||||
pos = clean_delta.find(dropped)
|
||||
if pos >= 0:
|
||||
clean_delta = (
|
||||
clean_delta[:pos] + clean_delta[pos + len(dropped) :]
|
||||
)
|
||||
if clean_delta:
|
||||
if results:
|
||||
results = self._recover_holdback_text(clean_delta, results)
|
||||
else:
|
||||
results = [TextChunk(clean_delta)]
|
||||
else:
|
||||
results = self._recover_holdback_text(effective_text, results)
|
||||
else:
|
||||
# No detokenizer text to validate against — individually-decoded
|
||||
# TextChunks are unreliable (context-dependent decoding).
|
||||
# Defer PreLexedTerminals so the state machine doesn't
|
||||
# transition before the preceding text has arrived. The
|
||||
# deferred terminals will be resolved against the actual
|
||||
# delta_text in a subsequent scan() or flushed by finish().
|
||||
for r in results:
|
||||
if isinstance(r, PreLexedTerminal):
|
||||
self._deferred_terminals.append(r)
|
||||
results = []
|
||||
|
||||
return prefix_items + results
|
||||
|
||||
def flush_pending(self) -> list[LexerInput]:
|
||||
if not self._deferred_terminals and not self._deferred_post_text:
|
||||
return []
|
||||
results: list[LexerInput] = []
|
||||
if self._deferred_post_text:
|
||||
results.append(TextChunk(self._deferred_post_text))
|
||||
self._deferred_post_text = ""
|
||||
results.extend(self._deferred_terminals)
|
||||
self._deferred_terminals.clear()
|
||||
return results
|
||||
|
||||
def _resolve_deferred(
|
||||
self,
|
||||
delta_text: str,
|
||||
) -> tuple[list[LexerInput], str]:
|
||||
"""Resolve deferred terminals against new delta_text.
|
||||
|
||||
When a previous ``scan()`` deferred a terminal (its text hadn't
|
||||
arrived yet), the next delta's text should contain that terminal's
|
||||
text. Split delta_text at the terminal boundary: text before
|
||||
belongs to the previous parser state, the terminal triggers the
|
||||
state transition, and text after belongs to the new state.
|
||||
|
||||
Returns ``(prefix_items, remaining_text)`` where prefix_items
|
||||
are the resolved deferred terminals (with any preceding text)
|
||||
and remaining_text is the unconsumed portion of delta_text that
|
||||
should be scanned with the current delta's token IDs.
|
||||
"""
|
||||
deferred = self._deferred_terminals
|
||||
self._deferred_terminals = []
|
||||
|
||||
results: list[LexerInput] = []
|
||||
remaining = delta_text
|
||||
|
||||
if self._deferred_post_text:
|
||||
remaining = self._deferred_post_text + remaining
|
||||
self._deferred_post_text = ""
|
||||
|
||||
# Duplicate-text deferred terminals resolve left-to-right via
|
||||
# find(); correct when each terminal text appears once in sequence.
|
||||
for terminal in deferred:
|
||||
pos = remaining.find(terminal.text)
|
||||
if pos > 0:
|
||||
results.append(TextChunk(remaining[:pos]))
|
||||
results.append(terminal)
|
||||
remaining = remaining[pos + len(terminal.text) :]
|
||||
elif pos == 0:
|
||||
results.append(terminal)
|
||||
remaining = remaining[len(terminal.text) :]
|
||||
else:
|
||||
# Accumulate text until terminal text arrives —
|
||||
# only the terminal provides a reliable split point.
|
||||
if remaining:
|
||||
self._deferred_post_text += remaining
|
||||
remaining = ""
|
||||
self._deferred_terminals.append(terminal)
|
||||
|
||||
return results, remaining
|
||||
|
||||
def _recover_holdback_text(
|
||||
self,
|
||||
delta_text: str,
|
||||
results: list[LexerInput],
|
||||
) -> list[LexerInput]:
|
||||
"""Recover detokenizer hold-back text not in delta_token_ids.
|
||||
|
||||
The detokenizer may flush previously held-back text in
|
||||
``delta_text`` that has no corresponding token ID in
|
||||
``delta_token_ids``. This hold-back text always appears as a
|
||||
prefix of ``delta_text``.
|
||||
"""
|
||||
if not results:
|
||||
return [TextChunk(delta_text)]
|
||||
|
||||
reconstructed = self._join_decoded_text(results)
|
||||
|
||||
if not reconstructed:
|
||||
return [TextChunk(delta_text)] + results
|
||||
|
||||
pos = delta_text.find(reconstructed)
|
||||
if pos > 0:
|
||||
return [TextChunk(delta_text[:pos])] + results
|
||||
if pos == 0:
|
||||
return results
|
||||
|
||||
# Fallback: SentencePiece context-dependent decoding mismatch.
|
||||
# Rebuild from delta_text using PreLexedTerminals as split anchors.
|
||||
return self._rebuild_from_anchors(delta_text, results)
|
||||
|
||||
def _join_decoded_text(self, results: list[LexerInput]) -> str:
|
||||
"""Join TextChunk and PreLexedTerminal text into one string."""
|
||||
parts: list[str] = []
|
||||
for item in results:
|
||||
if isinstance(item, (TextChunk, PreLexedTerminal)):
|
||||
parts.append(item.text)
|
||||
return "".join(parts)
|
||||
|
||||
def _rebuild_from_anchors(
|
||||
self,
|
||||
delta_text: str,
|
||||
results: list[LexerInput],
|
||||
) -> list[LexerInput]:
|
||||
"""Rebuild results from delta_text using terminals as anchors.
|
||||
|
||||
When context-dependent decoding creates a mismatch between
|
||||
individually-decoded tokens and delta_text, use
|
||||
PreLexedTerminals as split points and reallocate text from
|
||||
delta_text. If a terminal's text is not found in delta_text,
|
||||
it is deferred to the next scan() call.
|
||||
|
||||
Anchors are resolved right-to-left with ``rfind`` so that each
|
||||
anchor binds to the *rightmost* available occurrence of its
|
||||
text. This prevents earlier literal lookalikes (e.g. a user
|
||||
mentioning ``<tool_call>`` in prose) from stealing the position
|
||||
of a real special-token anchor that appears later.
|
||||
|
||||
If the same anchor text appears multiple times as real special
|
||||
tokens (not prose), the rightmost-first binding could misalign.
|
||||
In practice this doesn't happen: each special token ID maps to
|
||||
a distinct PreLexedTerminal, and duplicates in prose are resolved
|
||||
by the token-ID filtering layer above.
|
||||
"""
|
||||
anchors = [item for item in results if isinstance(item, PreLexedTerminal)]
|
||||
if not anchors:
|
||||
return [TextChunk(delta_text)]
|
||||
|
||||
# Resolve positions right-to-left: each anchor gets the
|
||||
# rightmost occurrence that is still before the next anchor.
|
||||
positions: list[int] = [-1] * len(anchors)
|
||||
search_end = len(delta_text)
|
||||
for i in range(len(anchors) - 1, -1, -1):
|
||||
pos = delta_text.rfind(anchors[i].text, 0, search_end)
|
||||
if pos >= 0:
|
||||
positions[i] = pos
|
||||
search_end = pos
|
||||
|
||||
# Build results left-to-right using the resolved positions.
|
||||
new_results: list[LexerInput] = []
|
||||
consumed = 0
|
||||
for i, anchor in enumerate(anchors):
|
||||
pos = positions[i]
|
||||
if pos >= consumed:
|
||||
if pos > consumed:
|
||||
new_results.append(TextChunk(delta_text[consumed:pos]))
|
||||
new_results.append(anchor)
|
||||
consumed = pos + len(anchor.text)
|
||||
else:
|
||||
has_later_valid = any(p >= 0 for p in positions[i + 1 :])
|
||||
if not has_later_valid and consumed < len(delta_text):
|
||||
self._deferred_post_text += delta_text[consumed:]
|
||||
consumed = len(delta_text)
|
||||
self._deferred_terminals.append(anchor)
|
||||
if consumed < len(delta_text):
|
||||
new_results.append(TextChunk(delta_text[consumed:]))
|
||||
return new_results
|
||||
Reference in New Issue
Block a user