[Frontend] Add the Beam reasoning and tool parsers (#62052)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yoni Gozlan <74535834+yonigozlan@users.noreply.github.com> Co-authored-by: neilkg <33635204+neilkg@users.noreply.github.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Bugen Zhao
Yoni Gozlan
neilkg
Cursor
Claude Opus 5.5
parent
9731f30a03
commit
153a17166e
@@ -0,0 +1,262 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Parser for Beam checkpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
from openai.types.responses import ToolChoiceFunction
|
||||
|
||||
from vllm import envs
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser.abstract_parser import (
|
||||
_xgrammar_supports,
|
||||
structured_outputs_to_format,
|
||||
)
|
||||
from vllm.parser.chat_parsing import ResponseParser
|
||||
from vllm.parser.chat_parsing.response_templates import ResponseTemplate
|
||||
from vllm.parser.response_template import (
|
||||
CONTENT_FIELD,
|
||||
THINKING_FIELD,
|
||||
TOOL_FIELD,
|
||||
ResponseTemplateParser,
|
||||
ResponseTemplateReasoningParser,
|
||||
ResponseTemplateToolParser,
|
||||
_as_tool_dicts,
|
||||
_is_named_tool_choice,
|
||||
_tool_parsing_enabled,
|
||||
)
|
||||
from vllm.parser.response_template_grammar import (
|
||||
_openers,
|
||||
answer_rest_structural_tag,
|
||||
message_start_anchors,
|
||||
tool_call_structural_tag,
|
||||
)
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
from vllm.tool_parsers.structural_tag_registry import resolve_tool_strictness
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _named_tool(request: ChatCompletionRequest | ResponsesRequest) -> str:
|
||||
choice = request.tool_choice
|
||||
if isinstance(choice, ToolChoiceFunction):
|
||||
return choice.name
|
||||
return cast(ChatCompletionNamedToolChoiceParam, choice).function.name
|
||||
|
||||
|
||||
def _starts_message(template: ResponseTemplate, prefix: str) -> bool:
|
||||
"""Whether output after `prefix` starts the assistant message."""
|
||||
ends = [match.end() for match in template.start_anchor_re.finditer(prefix)]
|
||||
return bool(ends) and ends[-1] == len(prefix)
|
||||
|
||||
|
||||
def _open_field(template: ResponseTemplate, prefix: str) -> str | None:
|
||||
"""The field that `prefix` leaves open with its explicit opener, if any."""
|
||||
field = None
|
||||
for event in ResponseParser(template, prefix=prefix).initial_events:
|
||||
if event["type"] == "region_open" and event["end"] > event["start"]:
|
||||
field = event["field"]
|
||||
elif event["type"] in {"region_close", "region_malformed"}:
|
||||
field = None
|
||||
return field
|
||||
|
||||
|
||||
class BeamReasoningParser(ResponseTemplateReasoningParser):
|
||||
"""Selects `BeamParser` as the reasoning parser."""
|
||||
|
||||
@classmethod
|
||||
def unified_parser_cls(cls) -> type[ResponseTemplateParser]:
|
||||
return BeamParser
|
||||
|
||||
@property
|
||||
def reasoning_start_str(self) -> str | None:
|
||||
if literal := super().reasoning_start_str:
|
||||
return literal
|
||||
thinking = self.response_template.fields.get(THINKING_FIELD)
|
||||
if thinking is None:
|
||||
return None
|
||||
openers = _openers(thinking, message_start_anchors(self.response_template))
|
||||
if not openers:
|
||||
return None
|
||||
# Budget tracking needs one marker for both a prompt continuation
|
||||
# and a complete assistant message in the checkpoint's pattern.
|
||||
marker = min(openers, key=len)
|
||||
return marker if all(opener.endswith(marker) for opener in openers) else None
|
||||
|
||||
|
||||
class BeamToolParser(ResponseTemplateToolParser):
|
||||
"""Selects `BeamParser` as the tool parser."""
|
||||
|
||||
@classmethod
|
||||
def unified_parser_cls(cls) -> type[ResponseTemplateParser]:
|
||||
return BeamParser
|
||||
|
||||
|
||||
class BeamParser(ResponseTemplateParser):
|
||||
"""Parse Beam output with the checkpoint's `response_template` metadata,
|
||||
and constrain it with a grammar derived from that template."""
|
||||
|
||||
reasoning_parser_cls = BeamReasoningParser
|
||||
tool_parser_cls = BeamToolParser
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
chat_template_kwargs = kwargs.get("chat_template_kwargs") or {}
|
||||
# From the message start, the grammar opens the message with non-empty
|
||||
# reasoning, unless the request turns thinking off.
|
||||
self._requires_reasoning = bool(
|
||||
chat_template_kwargs.get("enable_thinking", True)
|
||||
)
|
||||
|
||||
def _constrain_request(
|
||||
self,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
message_start = _starts_message(self.response_template, self._prefix)
|
||||
tag = self._template_tag(request, message_start=message_start)
|
||||
if tag is not None:
|
||||
grammar = StructuredOutputsParams(structural_tag=json.dumps(tag))
|
||||
if isinstance(request, ChatCompletionRequest):
|
||||
request.structured_outputs = grammar
|
||||
request.response_format = None
|
||||
else:
|
||||
request._parser_structured_outputs = grammar
|
||||
# From the message start, the tag also covers the reasoning.
|
||||
request._grammar_from_parser = message_start
|
||||
return request
|
||||
|
||||
def _template_tag(
|
||||
self,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
*,
|
||||
message_start: bool,
|
||||
) -> dict[str, Any] | None:
|
||||
"""The template grammar for `request`, or `None` to leave its output
|
||||
to the request's own constraint, if any."""
|
||||
fields = self.response_template.fields
|
||||
named = _is_named_tool_choice(request)
|
||||
tools_enabled = (
|
||||
self._tool_parser is not None
|
||||
and bool(request.tools)
|
||||
and _tool_parsing_enabled(
|
||||
request, enable_auto_tools=self._enable_auto_tools
|
||||
)
|
||||
)
|
||||
forced = tools_enabled and (request.tool_choice == "required" or named)
|
||||
# The prompt may still list the tools, so only the grammar keeps the
|
||||
# model from calling them.
|
||||
tools_forbidden = (
|
||||
bool(request.tools)
|
||||
and request.tool_choice == "none"
|
||||
and TOOL_FIELD in fields
|
||||
)
|
||||
constrained_tools = None
|
||||
if tools_enabled and (forced or envs.VLLM_ENFORCE_STRICT_TOOL_CALLING):
|
||||
assert self._tool_parser is not None
|
||||
strict_level = self.tool_strict_level
|
||||
if strict_level is None:
|
||||
strict_level = self._tool_parser.default_tool_strict_level
|
||||
constrained_tools = resolve_tool_strictness(
|
||||
request.tools or [], request.tool_choice, strict_level
|
||||
)
|
||||
output = request.extract_structured_outputs()
|
||||
if forced and output is not None:
|
||||
logger.warning_once(
|
||||
"structured outputs are ignored because tool_choice forces tool call."
|
||||
)
|
||||
output = None
|
||||
answer = fields.get(CONTENT_FIELD)
|
||||
if output is not None and (
|
||||
output.structural_tag is not None
|
||||
or answer is None
|
||||
or answer.open_re is None
|
||||
or not _xgrammar_supports(output)
|
||||
):
|
||||
if constrained_tools is not None:
|
||||
logger.warning_once(
|
||||
"response_template parsing cannot enforce strict tools "
|
||||
"alongside this structured output."
|
||||
)
|
||||
return None
|
||||
# Without a tool or structured output constraint, the output still
|
||||
# follows the template.
|
||||
always_constrain = envs.VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT and not forced
|
||||
if (
|
||||
constrained_tools is None
|
||||
and output is None
|
||||
and not tools_forbidden
|
||||
and not always_constrain
|
||||
):
|
||||
return None
|
||||
|
||||
# Inside an open answer, only the always-on grammar applies, finishing a
|
||||
# free answer. Inside open reasoning, the grammar needs the reasoning
|
||||
# gate to defer it to the reasoning's end.
|
||||
open_field = _open_field(self.response_template, self._prefix)
|
||||
reasoning_gated = (
|
||||
self._reasoning_parser is not None and request.include_reasoning
|
||||
)
|
||||
tag = None
|
||||
if open_field == CONTENT_FIELD:
|
||||
if always_constrain and output is None:
|
||||
tag = answer_rest_structural_tag(fields[CONTENT_FIELD])
|
||||
elif open_field != THINKING_FIELD or reasoning_gated:
|
||||
functions = []
|
||||
if tools_enabled:
|
||||
tools = _as_tool_dicts(constrained_tools or request.tools)
|
||||
functions = [tool.get("function", tool) for tool in tools]
|
||||
if named:
|
||||
name = _named_tool(request)
|
||||
functions = [f for f in functions if f.get("name") == name]
|
||||
answer_format = (
|
||||
None if output is None else structured_outputs_to_format(output)
|
||||
)
|
||||
try:
|
||||
tag = tool_call_structural_tag(
|
||||
fields.get(TOOL_FIELD),
|
||||
functions,
|
||||
parallel_tool_calls=(
|
||||
not named and request.parallel_tool_calls is not False
|
||||
),
|
||||
thinking=fields.get(THINKING_FIELD),
|
||||
answer=None if forced else answer,
|
||||
answer_format=(
|
||||
None if answer_format is None else answer_format.model_dump()
|
||||
),
|
||||
message_start=(
|
||||
message_start_anchors(self.response_template)
|
||||
if message_start
|
||||
else None
|
||||
),
|
||||
requires_reasoning=self._requires_reasoning,
|
||||
# Beam drifts from schemas the grammar does not enforce,
|
||||
# and also writes arguments sorted by name.
|
||||
sorted_arguments=True,
|
||||
strict_arguments=True,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise VLLMValidationError(str(exc), parameter="tools") from exc
|
||||
if tag is None and forced:
|
||||
kind = "named" if named else "required"
|
||||
raise VLLMValidationError(
|
||||
f"response_template parsing cannot guarantee {kind} tool choice "
|
||||
"for this request",
|
||||
parameter="tool_choice",
|
||||
)
|
||||
if tag is None and constrained_tools is not None:
|
||||
logger.warning_once(
|
||||
"response_template parsing cannot enforce strict tools for this "
|
||||
"request."
|
||||
)
|
||||
return tag
|
||||
@@ -16,8 +16,6 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
HF_PARSER = "hf"
|
||||
|
||||
|
||||
class ParserManager:
|
||||
"""Provides a unified Parser by composing reasoning and tool parser adapters."""
|
||||
@@ -122,8 +120,10 @@ class ParserManager:
|
||||
tags (``--tool-strict-level``), or ``None`` for the tool
|
||||
parser's default level.
|
||||
tokenizer: Tokenizer the composed parser is validated against
|
||||
at startup (see :meth:`validate_parser_tokenizer`). ``None``
|
||||
skips the check (e.g. ``skip_tokenizer_init``).
|
||||
at startup (see :meth:`validate_parser_tokenizer`), including
|
||||
its `response_template` metadata when a response-template
|
||||
parser is selected. ``None`` skips the check (e.g.
|
||||
``skip_tokenizer_init``).
|
||||
|
||||
Returns:
|
||||
A Parser class, or None if neither parser is specified.
|
||||
@@ -184,39 +184,40 @@ class ParserManager:
|
||||
HarmonyParser.tool_strict_level = strict_level
|
||||
return HarmonyParser
|
||||
|
||||
if HF_PARSER in (reasoning_parser_name, tool_parser_name):
|
||||
if {reasoning_parser_name, tool_parser_name} - {
|
||||
HF_PARSER,
|
||||
None,
|
||||
"",
|
||||
}:
|
||||
raise TypeError(
|
||||
"The hf parser cannot be combined with other "
|
||||
"reasoning or tool call parsers"
|
||||
)
|
||||
from vllm.parser.response_template import (
|
||||
ResponseTemplateParser,
|
||||
validate_tokenizer_response_template,
|
||||
)
|
||||
from vllm.parser.response_template import (
|
||||
response_template_parser_cls,
|
||||
validate_tokenizer_response_template,
|
||||
)
|
||||
|
||||
template_parsers = {
|
||||
response_template_parser_cls(parser_cls)
|
||||
for parser_cls in (reasoning_parser_cls, tool_parser_cls)
|
||||
if parser_cls is not None
|
||||
}
|
||||
template_parser_cls = next(iter(template_parsers - {None}), None)
|
||||
if template_parser_cls is not None:
|
||||
if len(template_parsers) > 1:
|
||||
raise TypeError(
|
||||
f"--reasoning-parser {reasoning_parser_name} and "
|
||||
f"--tool-call-parser {tool_parser_name} cannot be combined"
|
||||
)
|
||||
if tokenizer is not None:
|
||||
validate_tokenizer_response_template(
|
||||
tokenizer,
|
||||
reasoning=reasoning_parser_cls is not None,
|
||||
tools=tool_parser_cls is not None,
|
||||
reasoning_parser=reasoning_parser_name,
|
||||
# Without auto tool choice, no tool parser is selected.
|
||||
tool_parser=tool_parser_name if tool_parser_cls else None,
|
||||
)
|
||||
|
||||
r_cls = reasoning_parser_cls
|
||||
t_cls = tool_parser_cls
|
||||
auto_tools = enable_auto_tools
|
||||
|
||||
class _ResponseTemplateParser(ResponseTemplateParser):
|
||||
reasoning_parser_cls = r_cls
|
||||
tool_parser_cls = t_cls
|
||||
tool_strict_level = strict_level
|
||||
_enable_auto_tools = auto_tools
|
||||
|
||||
return _ResponseTemplateParser
|
||||
return type(
|
||||
template_parser_cls.__name__,
|
||||
(template_parser_cls,),
|
||||
{
|
||||
"reasoning_parser_cls": reasoning_parser_cls,
|
||||
"tool_parser_cls": tool_parser_cls,
|
||||
"tool_strict_level": strict_level,
|
||||
"_enable_auto_tools": enable_auto_tools,
|
||||
},
|
||||
)
|
||||
|
||||
if reasoning_parser_name == "kimi_k3" or tool_parser_name == "kimi_k3":
|
||||
from vllm.parser.kimi_k3 import KimiK3Parser
|
||||
|
||||
@@ -7,6 +7,7 @@ from __future__ import annotations
|
||||
import copy
|
||||
import json
|
||||
from collections.abc import Callable, Iterable, Sequence
|
||||
from math import inf
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
from openai.types.responses import ToolChoiceCustom, ToolChoiceFunction
|
||||
@@ -28,6 +29,7 @@ from vllm.parser.chat_parsing.response_templates import (
|
||||
ResponseTemplate,
|
||||
load_response_template,
|
||||
)
|
||||
from vllm.parser.response_template_grammar import message_start_anchors
|
||||
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
|
||||
from vllm.renderers.chat_utils import make_tool_call_id
|
||||
from vllm.tool_parsers.abstract_tool_parser import ToolParser
|
||||
@@ -135,27 +137,38 @@ def validate_response_template_for_serving(
|
||||
def validate_tokenizer_response_template(
|
||||
tokenizer: Any,
|
||||
*,
|
||||
reasoning: bool,
|
||||
tools: bool,
|
||||
reasoning_parser: str | None,
|
||||
tool_parser: str | None,
|
||||
) -> None:
|
||||
"""Fail at startup if the tokenizer cannot serve the requested parsing."""
|
||||
"""Fail at startup if the tokenizer cannot serve the selected parsers,
|
||||
named as given on the command line."""
|
||||
template = resolve_response_template(tokenizer)
|
||||
if template is None:
|
||||
selected = " ".join(
|
||||
f"{flag} {name}"
|
||||
for flag, name in (
|
||||
("--reasoning-parser", reasoning_parser),
|
||||
("--tool-call-parser", tool_parser),
|
||||
)
|
||||
if name
|
||||
)
|
||||
raise TypeError(
|
||||
"The hf parser requires `response_template` metadata "
|
||||
f"{selected} requires `response_template` metadata "
|
||||
"in the tokenizer configuration"
|
||||
)
|
||||
try:
|
||||
fields = validate_response_template_for_serving(template).fields
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise TypeError(f"Invalid response_template metadata: {exc}") from exc
|
||||
if reasoning and THINKING_FIELD not in fields:
|
||||
if reasoning_parser and THINKING_FIELD not in fields:
|
||||
raise TypeError(
|
||||
"--reasoning-parser hf requires a response_template with a thinking field"
|
||||
f"--reasoning-parser {reasoning_parser} requires a response_template "
|
||||
"with a thinking field"
|
||||
)
|
||||
if tools and TOOL_FIELD not in fields:
|
||||
if tool_parser and TOOL_FIELD not in fields:
|
||||
raise TypeError(
|
||||
"--tool-call-parser hf requires a response_template with a tool_calls field"
|
||||
f"--tool-call-parser {tool_parser} requires a response_template "
|
||||
"with a tool_calls field"
|
||||
)
|
||||
|
||||
|
||||
@@ -292,7 +305,7 @@ def _serving_template(
|
||||
template = resolve_response_template(tokenizer, template)
|
||||
if template is None:
|
||||
raise ValueError(
|
||||
"The hf parser requires `response_template` "
|
||||
"Parsing with a response template requires `response_template` "
|
||||
"metadata in the tokenizer configuration"
|
||||
)
|
||||
validate_response_template_for_serving(template)
|
||||
@@ -319,12 +332,17 @@ def _to_tool_calls(value: Any) -> list[tuple[str, str]]:
|
||||
|
||||
|
||||
class ResponseTemplateReasoningParser(ReasoningParser):
|
||||
"""Reasoning end detection for `--reasoning-parser hf`.
|
||||
"""Reasoning end detection for a response-template parser.
|
||||
|
||||
`ResponseTemplateParser` parses the output; this parser tells it and the
|
||||
structured-output engine where reasoning ends.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def unified_parser_cls(cls) -> type[ResponseTemplateParser]:
|
||||
"""The unified parser that selecting this reasoning parser stands for."""
|
||||
return ResponseTemplateParser
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
@@ -400,8 +418,26 @@ class ResponseTemplateReasoningParser(ReasoningParser):
|
||||
|
||||
|
||||
class ResponseTemplateToolParser(ToolParser):
|
||||
"""Registers `--tool-call-parser hf`; `ResponseTemplateParser` extracts the
|
||||
tool calls."""
|
||||
"""Selects a response-template parser as the tool parser;
|
||||
`ResponseTemplateParser` extracts the tool calls."""
|
||||
|
||||
@classmethod
|
||||
def unified_parser_cls(cls) -> type[ResponseTemplateParser]:
|
||||
"""The unified parser that selecting this tool parser stands for."""
|
||||
return ResponseTemplateParser
|
||||
|
||||
|
||||
def response_template_parser_cls(
|
||||
split_parser_cls: type[ReasoningParser] | type[ToolParser],
|
||||
) -> type[ResponseTemplateParser] | None:
|
||||
"""The response-template parser that a registered reasoning or tool parser
|
||||
stands for, or `None` for any other parser."""
|
||||
if issubclass(
|
||||
split_parser_cls,
|
||||
(ResponseTemplateReasoningParser, ResponseTemplateToolParser),
|
||||
):
|
||||
return split_parser_cls.unified_parser_cls()
|
||||
return None
|
||||
|
||||
|
||||
class _DeltaBuilder:
|
||||
@@ -441,23 +477,50 @@ class _ResponseStream:
|
||||
self.include_reasoning = include_reasoning
|
||||
self.parse_tools = parse_tools
|
||||
self.new_tool_call_id = new_tool_call_id
|
||||
self.template = template
|
||||
self.parser = ResponseParser(template, prefix=prefix, tools=tools)
|
||||
self._prefix_end = len(self.parser.input_text)
|
||||
self._next_tool_index = 0
|
||||
self._finalized = False
|
||||
self._open_region: dict[str, Any] | None = None
|
||||
self._track(self.parser.initial_events)
|
||||
|
||||
def feed(self, text: str, *, finished: bool) -> DeltaMessage | None:
|
||||
if self._finalized:
|
||||
return None
|
||||
delta = _DeltaBuilder()
|
||||
self._route(self.parser.feed(text), delta)
|
||||
self._route(self._track(self.parser.feed(text)), delta)
|
||||
if finished:
|
||||
self._finalized = True
|
||||
if len(self.parser.input_text) > self._prefix_end:
|
||||
_, events = self.parser.finalize()
|
||||
self._route(events, delta)
|
||||
events = self.parser.feed(self._closer_rest())
|
||||
self._route(events + self.parser.finalize()[1], delta)
|
||||
return delta.build()
|
||||
|
||||
def _track(self, events: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
for event in events:
|
||||
if event["type"] == "region_open":
|
||||
self._open_region = event
|
||||
elif event["type"] in ("region_close", "region_malformed"):
|
||||
self._open_region = None
|
||||
return events
|
||||
|
||||
def _closer_rest(self) -> str:
|
||||
"""The rest of a closer the output stopped in after ending its region,
|
||||
when only the next message's start anchor is missing."""
|
||||
if self._open_region is None:
|
||||
return ""
|
||||
field = self.template.fields[self._open_region["field"]]
|
||||
text = self.parser.input_text[self._open_region["end"] :]
|
||||
for closer in field.close_literals or ():
|
||||
for anchor in message_start_anchors(self.template):
|
||||
if not anchor or not closer.endswith(anchor) or closer == anchor:
|
||||
continue
|
||||
for cut in range(len(closer) - 1, len(closer) - len(anchor) - 1, -1):
|
||||
if text.endswith(closer[:cut]):
|
||||
return closer[cut:]
|
||||
return ""
|
||||
|
||||
def _route(self, events: Sequence[dict[str, Any]], delta: _DeltaBuilder) -> None:
|
||||
for event in events:
|
||||
field = event.get("field")
|
||||
@@ -499,6 +562,54 @@ class _ResponseStream:
|
||||
self._next_tool_index += 1
|
||||
|
||||
|
||||
class _ReasoningTokenCounter:
|
||||
"""Counts the reasoning tokens of one growing generation, parsing each
|
||||
token once."""
|
||||
|
||||
def __init__(
|
||||
self, template: ResponseTemplate, prefix: str, tokenizer: TokenizerLike
|
||||
) -> None:
|
||||
self.tokenizer = tokenizer
|
||||
self.parser = ResponseParser(template, prefix=prefix)
|
||||
self.start = len(self.parser.input_text)
|
||||
self.token_ids: list[int] = []
|
||||
self.token_ends: list[int] = []
|
||||
# Input spans between the delimiters of thinking regions.
|
||||
self.spans: list[tuple[int, float]] = []
|
||||
self.open: int | None = None
|
||||
self._track(self.parser.initial_events)
|
||||
|
||||
def continues(self, token_ids: Sequence[int]) -> bool:
|
||||
return list(token_ids[: len(self.token_ids)]) == self.token_ids
|
||||
|
||||
def count(self, token_ids: Sequence[int]) -> int:
|
||||
new_ids = list(token_ids[len(self.token_ids) :])
|
||||
texts = [_decode(self.tokenizer, [token_id]) for token_id in new_ids]
|
||||
end = self.token_ends[-1] if self.token_ends else self.start
|
||||
for text in texts:
|
||||
end += len(text)
|
||||
self.token_ends.append(end)
|
||||
self.token_ids.extend(new_ids)
|
||||
self._track(self.parser.feed("".join(texts)))
|
||||
# Text held back as a possible closer counts until a closer arrives.
|
||||
spans = self.spans if self.open is None else [*self.spans, (self.open, inf)]
|
||||
count, start = 0, self.start
|
||||
for end in self.token_ends:
|
||||
count += any(begin < end and start < close for begin, close in spans)
|
||||
start = end
|
||||
return count
|
||||
|
||||
def _track(self, events: Iterable[dict[str, Any]]) -> None:
|
||||
for event in events:
|
||||
if event["field"] != THINKING_FIELD or event["type"] == "region_chunk":
|
||||
continue
|
||||
if event["type"] == "region_open":
|
||||
self.open = event["end"]
|
||||
elif self.open is not None:
|
||||
self.spans.append((self.open, event["start"]))
|
||||
self.open = None
|
||||
|
||||
|
||||
class ResponseTemplateParser(Parser):
|
||||
"""Parse reasoning, content, and tool calls with one response template."""
|
||||
|
||||
@@ -526,6 +637,7 @@ class ResponseTemplateParser(Parser):
|
||||
self._prompt_token_ids: list[int] | None = None
|
||||
self._prefix = ""
|
||||
self._stream: _ResponseStream | None = None
|
||||
self._reasoning_counter: _ReasoningTokenCounter | None = None
|
||||
|
||||
def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None:
|
||||
prompt_token_ids = list(prompt_token_ids)
|
||||
@@ -534,6 +646,7 @@ class ResponseTemplateParser(Parser):
|
||||
self._prompt_token_ids = prompt_token_ids
|
||||
self._prefix = _decode(self.model_tokenizer, prompt_token_ids)
|
||||
self._stream = None
|
||||
self._reasoning_counter = None
|
||||
|
||||
def adjust_request(
|
||||
self,
|
||||
@@ -547,6 +660,14 @@ class ResponseTemplateParser(Parser):
|
||||
request.skip_special_tokens = False
|
||||
if hasattr(request, "spaces_between_special_tokens"):
|
||||
request.spaces_between_special_tokens = False
|
||||
return self._constrain_request(request)
|
||||
|
||||
def _constrain_request(
|
||||
self,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
"""Constrain `request` to the tool guarantees it asks for, or reject
|
||||
those this parser cannot enforce."""
|
||||
if (
|
||||
self._tool_parser is not None
|
||||
and request.tools
|
||||
@@ -567,6 +688,21 @@ class ResponseTemplateParser(Parser):
|
||||
or self._reasoning_parser.is_reasoning_end(input_ids)
|
||||
)
|
||||
|
||||
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
|
||||
"""Count the generated tokens whose text the template parses as
|
||||
reasoning, continuing reasoning left open by the prompt."""
|
||||
if (
|
||||
self._reasoning_parser is None
|
||||
or THINKING_FIELD not in self.response_template.fields
|
||||
):
|
||||
return 0
|
||||
counter = self._reasoning_counter
|
||||
if counter is None or not counter.continues(token_ids):
|
||||
counter = self._reasoning_counter = _ReasoningTokenCounter(
|
||||
self.response_template, self._prefix, self.model_tokenizer
|
||||
)
|
||||
return counter.count(token_ids)
|
||||
|
||||
def parse(
|
||||
self,
|
||||
model_output: str,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user