[Bugfix][Structured Output] Feed the token that implicitly ends reasoning to the grammar (#59627)

Signed-off-by: Simon Veitner <sveitner@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
Co-authored-by: Yifan Zong <263738792+yzong-rh@users.noreply.github.com>
This commit is contained in:
simon-veitner-redhat
2026-10-08 19:38:33 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 Misha Goin Yifan Zong
parent bab0ac61f8
commit c23ca06b6d
7 changed files with 150 additions and 52 deletions
+2 -2
View File
@@ -643,8 +643,8 @@ class TestImplicitReasoningEnd:
# token-ID fast path; the implicit tool-call exit still goes through
# the text lexer.
assert thinking_parser.reasoning_end_token_ids == {_THINK_END_ID}
assert thinking_parser.find_reasoning_end_offset([7, _THINK_END_ID, 8]) == 1
assert thinking_parser.find_reasoning_end_offset([7, 8]) == 2
assert thinking_parser.find_reasoning_end([7, _THINK_END_ID, 8]) == (1, False)
assert thinking_parser.find_reasoning_end([7, 8]) == (2, False)
def test_streaming_reasoning_implicit_end(self, thinking_parser):
chunks = [
+36 -9
View File
@@ -28,12 +28,13 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
from vllm.parser.abstract_parser import DelegatingParser
from vllm.parser.engine.adapters import make_adapters
from vllm.parser.engine.events import EventType, SemanticEvent
from vllm.parser.engine.parser_engine import ParserEngine
from vllm.parser.engine.parser_engine import ParserEngine, ReasoningEnd
from vllm.parser.engine.parser_engine_config import (
ParserEngineConfig,
ParserState,
Transition,
)
from vllm.parser.glm47_moe import glm47_moe_config
from vllm.parser.parser_manager import ParserManager
# ── Shared test configs ──────────────────────────────────────────────
@@ -158,7 +159,10 @@ class TestReasoningEndTokenIds:
(EventType.REASONING_END, EventType.TOOL_CALL_START),
),
)
assert _make_engine(cfg).reasoning_end_token_ids == {201, 202}
engine = _make_engine(cfg)
assert engine.reasoning_end_token_ids == {201, 202}
assert engine.find_reasoning_end([5, 201]) == ReasoningEnd(1, False)
assert engine.find_reasoning_end([5, 202]) == ReasoningEnd(1, True)
def test_transition_staying_in_reasoning_is_ignored(self):
cfg = _with_reasoning_exits(("THINK_START", ParserState.REASONING, ()))
@@ -182,17 +186,40 @@ class TestReasoningEndTokenIds:
def test_config_without_reasoning_has_empty_set(self):
assert _make_engine(_hermes_config()).reasoning_end_token_ids == frozenset()
def test_find_reasoning_end_offset_returns_first_match(self):
def test_find_reasoning_end_returns_first_match(self):
engine = _make_engine()
assert engine.find_reasoning_end_offset([5, 201, 6, 201]) == 1
assert engine.find_reasoning_end_offset([5, 6]) == 2
assert engine.find_reasoning_end_offset([]) == 0
assert engine.find_reasoning_end([5, 201, 6, 201]) == ReasoningEnd(1, False)
assert engine.find_reasoning_end([5, 6]) == ReasoningEnd(2, False)
assert engine.find_reasoning_end([]) == ReasoningEnd(0, False)
# Rejected-draft placeholders never match.
assert engine.find_reasoning_end_offset([-1, 201]) == 1
assert engine.find_reasoning_end([-1, 201]) == ReasoningEnd(1, False)
def test_find_reasoning_end_offset_with_empty_set_returns_none(self):
def test_find_reasoning_end_with_empty_set_returns_none(self):
engine = _make_engine(_hermes_config())
assert engine.find_reasoning_end_offset([201]) is None
assert engine.find_reasoning_end([201]) is None
@pytest.mark.parametrize(
("config", "implicit_token"),
[
pytest.param(glm47_moe_config(), "<tool_call>", id="glm47_moe"),
],
)
def test_implicit_reasoning_end_of_parsers(
config: ParserEngineConfig, implicit_token: str
):
literals = sorted(set(config.token_id_terminals.values()))
vocab = {literal: 1000 + i for i, literal in enumerate(literals)}
engine = _make_engine(config, vocab=vocab)
implicit_ids = set()
for token_id in engine.reasoning_end_token_ids:
end = engine.find_reasoning_end([5, token_id, 6])
assert end is not None and end.offset == 1
kept = engine.extract_content_ids([5, token_id, 6]) == [token_id, 6]
assert end.implicit == kept
if end.implicit:
implicit_ids.add(token_id)
assert implicit_ids == {vocab[implicit_token]}
# ── TestEventsToDelta ────────────────────────────────────────────────
@@ -11,6 +11,7 @@ from vllm.config import StructuredOutputsConfig, VllmConfig
from vllm.config.model import ModelConfig
from vllm.config.speculative import SpeculativeConfig
from vllm.parser.engine.adapters import ParserEngineReasoningAdapter
from vllm.parser.engine.parser_engine import ReasoningEnd
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.v1.request import Request
from vllm.v1.structured_output import StructuredOutputManager
@@ -113,12 +114,12 @@ class _EngineReasonerStub(ParserEngineReasoningAdapter):
def reasoning_end_token_ids(self):
return self._end_token_ids
def find_reasoning_end_offset(self, token_ids):
def find_reasoning_end(self, token_ids):
self.windows.append(list(token_ids))
for offset, token in enumerate(token_ids):
if token in self._end_token_ids:
return offset
return len(token_ids)
return ReasoningEnd(offset, False)
return ReasoningEnd(len(token_ids), False)
def is_reasoning_end(self, input_ids):
return any(token in self._end_token_ids for token in input_ids)
@@ -11,6 +11,8 @@ from vllm.config import DeviceConfig, StructuredOutputsConfig, VllmConfig
from vllm.config.model import ModelConfig
from vllm.config.speculative import SpeculativeConfig
from vllm.config.structured_outputs import StructuredOutputsBackend
from vllm.parser.engine.adapters import ParserEngineReasoningAdapter
from vllm.parser.engine.parser_engine import ReasoningEnd
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.v1.request import Request
from vllm.v1.structured_output import StructuredOutputManager
@@ -18,6 +20,7 @@ from vllm.v1.structured_output.backend_outlines import OutlinesGrammar
TOKENIZER = "gpt2"
THINK_END = "\n" # reasoning-end marker (single GPT-2 token)
IMPLICIT_THINK_END = "z" # accepted in strings, rejected at the object start
EOS = "<|eos|>" # resolved to tokenizer.eos_token_id
JSON_SCHEMA = '{"type": "object"}'
BACKENDS = ("xgrammar", "guidance")
@@ -43,6 +46,7 @@ class FlowCase:
prefix: str = ""
expected_validated: tuple[str, ...] | None = None
reasoning_ended: bool | None = False
engine_reasoner: bool = False
xfail_guidance: str | None = None
@@ -59,6 +63,23 @@ class MockReasoner:
return self.is_reasoning_end(delta_ids)
class MockEngineReasoner(ParserEngineReasoningAdapter):
"""Ends reasoning implicitly on ``marker``, which is also content."""
def __init__(self, tokenizer, marker: int):
self.marker = marker
@property
def reasoning_end_token_ids(self):
return frozenset({self.marker})
def find_reasoning_end(self, token_ids):
ids = list(token_ids)
if self.marker in ids:
return ReasoningEnd(ids.index(self.marker), True)
return ReasoningEnd(len(ids), False)
def _single_token(tokenizer, text: str) -> int:
token_ids = tokenizer.encode(text)
assert len(token_ids) == 1, (text, token_ids)
@@ -331,6 +352,28 @@ FLOW_CASES = [
),
id="becomes_active_terminates",
),
pytest.param(
FlowCase(
prefix='{"a": "',
raw_drafts=(IMPLICIT_THINK_END,),
expected_row_pattern="UC",
expected_reasoning=True,
expect_terminated=False,
engine_reasoner=True,
),
id="implicit_end_accepted",
),
pytest.param(
FlowCase(
raw_drafts=(" ", IMPLICIT_THINK_END, "{", "}"),
expected_validated=(" ",),
expected_row_pattern="UUUUU",
expected_reasoning=False,
expect_terminated=False,
engine_reasoner=True,
),
id="implicit_end_rejected",
),
]
@@ -344,7 +387,8 @@ def test_real_flow(
if backend == "guidance" and case.xfail_guidance:
pytest.xfail(case.xfail_guidance)
reasoner_kwargs = {"marker": _single_token(tokenizer, THINK_END)}
marker = IMPLICIT_THINK_END if case.engine_reasoner else THINK_END
reasoner_kwargs = {"marker": _single_token(tokenizer, marker)}
manager, request = _build_harness(
tokenizer,
backend,
@@ -352,6 +396,8 @@ def test_real_flow(
reasoning_ended=case.reasoning_ended,
reasoning_parser_kwargs=reasoner_kwargs,
)
if case.engine_reasoner:
manager.reasoner_cls = MockEngineReasoner
raw_drafts = _to_token_ids(tokenizer, case.raw_drafts)
expected_texts = (
+3 -3
View File
@@ -27,7 +27,7 @@ if TYPE_CHECKING:
ChatCompletionRequest,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.parser.engine.parser_engine import ParserEngine
from vllm.parser.engine.parser_engine import ParserEngine, ReasoningEnd
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.utils import Tool
@@ -65,8 +65,8 @@ class ParserEngineReasoningAdapter(ReasoningParser):
def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
return self._parser_engine.is_reasoning_end(list(input_ids))
def find_reasoning_end_offset(self, token_ids: Sequence[int]) -> int | None:
return self._parser_engine.find_reasoning_end_offset(token_ids)
def find_reasoning_end(self, token_ids: Sequence[int]) -> ReasoningEnd | None:
return self._parser_engine.find_reasoning_end(token_ids)
def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None:
self._parser_engine.adjust_initial_state_from_prompt(prompt_token_ids)
+12 -4
View File
@@ -9,7 +9,7 @@ from __future__ import annotations
import json
from collections.abc import Sequence
from functools import cached_property
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, NamedTuple
import regex as re
@@ -45,6 +45,13 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
class ReasoningEnd(NamedTuple):
"""Where reasoning ends in a token sequence."""
offset: int # index of the first end token, or the sequence length
implicit: bool # the end token is also content, e.g. a tool-call start
class ToolCallSlot:
__slots__ = (
"id",
@@ -638,14 +645,15 @@ class ParserEngine(Parser):
def reasoning_end_token_ids(self) -> frozenset[int]:
return self._reasoning_end_token_ids
def find_reasoning_end_offset(self, token_ids: Sequence[int]) -> int | None:
def find_reasoning_end(self, token_ids: Sequence[int]) -> ReasoningEnd | None:
"""First reasoning end in `token_ids`, and if its token is content."""
end_ids = self._reasoning_end_token_ids
if not end_ids:
return None
for offset, token_id in enumerate(token_ids):
if token_id in end_ids:
return offset
return len(token_ids)
return ReasoningEnd(offset, token_id != self._reasoning_end_token_id)
return ReasoningEnd(len(token_ids), False)
def is_reasoning_end(self, input_ids: list[int]) -> bool:
config = self.parser_engine_config
+46 -30
View File
@@ -3,7 +3,7 @@
import multiprocessing
from collections.abc import Iterable, Sequence
from concurrent.futures import Future, ThreadPoolExecutor
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, NamedTuple
from vllm.config import VllmConfig
from vllm.logger import init_logger
@@ -33,6 +33,17 @@ else:
logger = init_logger(__name__)
class ConstraintBounds(NamedTuple):
"""Where tokens start being fed to the grammar and being masked."""
grammar_start: int
constraint_start: int
_CONSTRAINED_FROM_START = ConstraintBounds(0, 0)
_NO_SPEC_TOKENS = ConstraintBounds(1, 1)
class StructuredOutputManager:
"""Engine-level manager for structured output requests."""
@@ -217,32 +228,34 @@ class StructuredOutputManager:
) -> Future:
return self.executor_for_fillmask.submit(self._fill_bitmasks, batch)
def _get_constraint_start(
def _get_constraint_bounds(
self,
request: "Request",
spec_tokens: Sequence[int],
spec_tokens_committed: bool = False,
) -> int:
) -> ConstraintBounds:
"""Return the index into `spec_tokens` where tokens should start being
constrained by the grammar.
Assumes `spec_tokens` are stripped of -1 padding tokens.
Returns `len(spec_tokens) + 1` if no token should be constrained after
accepting all `spec_tokens`.
This is `constraint_start`; `grammar_start` is one less if reasoning ends
implicitly on a content token (e.g. a tool-call start).
`spec_tokens_committed` is True if spec_tokens is already in
`request.all_token_ids`.
"""
if self.enable_in_reasoning:
return 0
return _CONSTRAINED_FROM_START
structured_req = request.structured_output_request
assert structured_req is not None
if structured_req.reasoning_ended:
return 0
return _CONSTRAINED_FROM_START
reasoner = self._get_reasoner(request)
if reasoner is None:
return 0
return _CONSTRAINED_FROM_START
if structured_req.reasoning_ended is None:
# This should be removed here, but since `openai_gptoss`
@@ -250,30 +263,33 @@ class StructuredOutputManager:
# After unifying the `openai_gptoss` and non-`openai_gptoss` styles,
# it can be removed.
if reasoner.is_reasoning_end(request.prompt_token_ids or []):
return 0
return _CONSTRAINED_FROM_START
structured_req.reasoning_ended = False
num_spec_tokens = len(spec_tokens)
if num_spec_tokens <= 0:
return num_spec_tokens + 1
return _NO_SPEC_TOKENS
# Use `find_reasoning_end_offset` to find constraint start if supported
# Engine-based parsers also report whether the end token is content.
if (
isinstance(reasoner, ParserEngineReasoningAdapter)
and reasoner.reasoning_end_token_ids
):
offset = reasoner.find_reasoning_end_offset(spec_tokens)
if offset is not None:
return offset + 1
end = reasoner.find_reasoning_end(spec_tokens)
if end is not None:
constraint_start = end.offset + 1
if end.implicit:
return ConstraintBounds(end.offset, constraint_start)
return ConstraintBounds(constraint_start, constraint_start)
# Fallback to `find_reasoning_end_offset`
# Fallback: the grammar starts after the end token.
# TODO: Build a read-only Sequence view over all_token_ids
# instead of copying the entire all_token_ids into input_ids.
if spec_tokens_committed:
if not spec_tokens or not reasoner.is_reasoning_end_streaming(
request.all_token_ids, spec_tokens
):
return num_spec_tokens + 1
return ConstraintBounds(num_spec_tokens + 1, num_spec_tokens + 1)
input_ids = request.all_token_ids.copy()
delta_ids = list(spec_tokens)
@@ -281,15 +297,15 @@ class StructuredOutputManager:
input_ids = request.all_token_ids.copy()
input_ids.extend(spec_tokens)
if not reasoner.is_reasoning_end_streaming(input_ids, spec_tokens):
return num_spec_tokens + 1
return ConstraintBounds(num_spec_tokens + 1, num_spec_tokens + 1)
delta_ids = list(spec_tokens)
for i in range(num_spec_tokens - 1, 0, -1):
input_ids.pop()
delta_ids.pop()
if not reasoner.is_reasoning_end_streaming(input_ids, delta_ids):
return i + 1
return 1
return ConstraintBounds(i + 1, i + 1)
return ConstraintBounds(1, 1)
def validate_tokens(self, request: "Request", spec_tokens: list[int]) -> list[int]:
"""Return the longest unconstrained or grammar-valid prefix of `spec_tokens`."""
@@ -297,8 +313,8 @@ class StructuredOutputManager:
return spec_tokens
spec_tokens = strip_speculative_padding(spec_tokens)
constraint_start = self._get_constraint_start(request, spec_tokens)
if constraint_start >= len(spec_tokens):
grammar_start, _ = self._get_constraint_bounds(request, spec_tokens)
if grammar_start >= len(spec_tokens):
return spec_tokens
structured_req = request.structured_output_request
@@ -307,8 +323,8 @@ class StructuredOutputManager:
grammar = structured_req.grammar
if TYPE_CHECKING:
assert isinstance(grammar, StructuredOutputGrammar)
prefix = spec_tokens[:constraint_start]
validated = grammar.validate_tokens(spec_tokens[constraint_start:])
prefix = spec_tokens[:grammar_start]
validated = grammar.validate_tokens(spec_tokens[grammar_start:])
return prefix + validated
def grammar_bitmask(
@@ -358,7 +374,9 @@ class StructuredOutputManager:
if TYPE_CHECKING:
assert isinstance(grammar, StructuredOutputGrammar)
apply_bitmask = self._get_constraint_start(request, ()) == 0
apply_bitmask = (
self._get_constraint_bounds(request, ()).constraint_start == 0
)
batch.append((grammar, cumulative_index, apply_bitmask))
if len(batch) == self.fill_bitmask_parallel_batch_size:
promises.append(self._async_submit_fill_bitmask(batch))
@@ -384,7 +402,7 @@ class StructuredOutputManager:
assert isinstance(grammar, StructuredOutputGrammar)
req_tokens = scheduled_spec_decode_tokens.get(req_id, list())
constraint_start = self._get_constraint_start(
grammar_start, constraint_start = self._get_constraint_bounds(
request, strip_speculative_padding(req_tokens)
)
state_advancements = 0
@@ -402,7 +420,7 @@ class StructuredOutputManager:
self._fill_bitmasks(((grammar, cumulative_index, apply_bitmask),))
if token == -1:
seen_padding = True
elif apply_bitmask:
elif not seen_padding and i >= grammar_start:
if not grammar.is_terminated() and grammar.accept_tokens(
req_id, [token]
):
@@ -453,17 +471,15 @@ class StructuredOutputManager:
if TYPE_CHECKING:
assert isinstance(grammar, StructuredOutputGrammar)
constraint_start = self._get_constraint_start(
grammar_start, _ = self._get_constraint_bounds(
request, new_token_ids, spec_tokens_committed=True
)
# Early return only when the constraint hasn't started.
# Early return only when no reasoning-end token is present.
# Otherwise, latch structured_req.reasoning_ended.
if constraint_start > len(new_token_ids):
if grammar_start > len(new_token_ids):
return True
structured_req.reasoning_ended = True
return grammar.accept_tokens(
request.request_id, new_token_ids[constraint_start:]
)
return grammar.accept_tokens(request.request_id, new_token_ids[grammar_start:])
def clear_backend(self) -> None:
if self.backend is not None: