diff --git a/tests/parser/engine/test_deepseek_v4.py b/tests/parser/engine/test_deepseek_v4.py index 2cf3a8bb85..f5b66d9dc1 100644 --- a/tests/parser/engine/test_deepseek_v4.py +++ b/tests/parser/engine/test_deepseek_v4.py @@ -643,8 +643,8 @@ class TestImplicitReasoningEnd: # token-ID fast path; the implicit tool-call exit still goes through # the text lexer. assert thinking_parser.reasoning_end_token_ids == {_THINK_END_ID} - assert thinking_parser.find_reasoning_end_offset([7, _THINK_END_ID, 8]) == 1 - assert thinking_parser.find_reasoning_end_offset([7, 8]) == 2 + assert thinking_parser.find_reasoning_end([7, _THINK_END_ID, 8]) == (1, False) + assert thinking_parser.find_reasoning_end([7, 8]) == (2, False) def test_streaming_reasoning_implicit_end(self, thinking_parser): chunks = [ diff --git a/tests/parser/engine/test_parser_engine.py b/tests/parser/engine/test_parser_engine.py index 8fd7c4d262..dc91cef740 100644 --- a/tests/parser/engine/test_parser_engine.py +++ b/tests/parser/engine/test_parser_engine.py @@ -28,12 +28,13 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( from vllm.parser.abstract_parser import DelegatingParser from vllm.parser.engine.adapters import make_adapters from vllm.parser.engine.events import EventType, SemanticEvent -from vllm.parser.engine.parser_engine import ParserEngine +from vllm.parser.engine.parser_engine import ParserEngine, ReasoningEnd from vllm.parser.engine.parser_engine_config import ( ParserEngineConfig, ParserState, Transition, ) +from vllm.parser.glm47_moe import glm47_moe_config from vllm.parser.parser_manager import ParserManager # ── Shared test configs ────────────────────────────────────────────── @@ -158,7 +159,10 @@ class TestReasoningEndTokenIds: (EventType.REASONING_END, EventType.TOOL_CALL_START), ), ) - assert _make_engine(cfg).reasoning_end_token_ids == {201, 202} + engine = _make_engine(cfg) + assert engine.reasoning_end_token_ids == {201, 202} + assert engine.find_reasoning_end([5, 201]) == ReasoningEnd(1, False) + assert engine.find_reasoning_end([5, 202]) == ReasoningEnd(1, True) def test_transition_staying_in_reasoning_is_ignored(self): cfg = _with_reasoning_exits(("THINK_START", ParserState.REASONING, ())) @@ -182,17 +186,40 @@ class TestReasoningEndTokenIds: def test_config_without_reasoning_has_empty_set(self): assert _make_engine(_hermes_config()).reasoning_end_token_ids == frozenset() - def test_find_reasoning_end_offset_returns_first_match(self): + def test_find_reasoning_end_returns_first_match(self): engine = _make_engine() - assert engine.find_reasoning_end_offset([5, 201, 6, 201]) == 1 - assert engine.find_reasoning_end_offset([5, 6]) == 2 - assert engine.find_reasoning_end_offset([]) == 0 + assert engine.find_reasoning_end([5, 201, 6, 201]) == ReasoningEnd(1, False) + assert engine.find_reasoning_end([5, 6]) == ReasoningEnd(2, False) + assert engine.find_reasoning_end([]) == ReasoningEnd(0, False) # Rejected-draft placeholders never match. - assert engine.find_reasoning_end_offset([-1, 201]) == 1 + assert engine.find_reasoning_end([-1, 201]) == ReasoningEnd(1, False) - def test_find_reasoning_end_offset_with_empty_set_returns_none(self): + def test_find_reasoning_end_with_empty_set_returns_none(self): engine = _make_engine(_hermes_config()) - assert engine.find_reasoning_end_offset([201]) is None + assert engine.find_reasoning_end([201]) is None + + +@pytest.mark.parametrize( + ("config", "implicit_token"), + [ + pytest.param(glm47_moe_config(), "", id="glm47_moe"), + ], +) +def test_implicit_reasoning_end_of_parsers( + config: ParserEngineConfig, implicit_token: str +): + literals = sorted(set(config.token_id_terminals.values())) + vocab = {literal: 1000 + i for i, literal in enumerate(literals)} + engine = _make_engine(config, vocab=vocab) + implicit_ids = set() + for token_id in engine.reasoning_end_token_ids: + end = engine.find_reasoning_end([5, token_id, 6]) + assert end is not None and end.offset == 1 + kept = engine.extract_content_ids([5, token_id, 6]) == [token_id, 6] + assert end.implicit == kept + if end.implicit: + implicit_ids.add(token_id) + assert implicit_ids == {vocab[implicit_token]} # ── TestEventsToDelta ──────────────────────────────────────────────── diff --git a/tests/v1/spec_decode/test_mtp_structured_output.py b/tests/v1/spec_decode/test_mtp_structured_output.py index 5e4312227d..8a72765db1 100644 --- a/tests/v1/spec_decode/test_mtp_structured_output.py +++ b/tests/v1/spec_decode/test_mtp_structured_output.py @@ -11,6 +11,7 @@ from vllm.config import StructuredOutputsConfig, VllmConfig from vllm.config.model import ModelConfig from vllm.config.speculative import SpeculativeConfig from vllm.parser.engine.adapters import ParserEngineReasoningAdapter +from vllm.parser.engine.parser_engine import ReasoningEnd from vllm.sampling_params import SamplingParams, StructuredOutputsParams from vllm.v1.request import Request from vllm.v1.structured_output import StructuredOutputManager @@ -113,12 +114,12 @@ class _EngineReasonerStub(ParserEngineReasoningAdapter): def reasoning_end_token_ids(self): return self._end_token_ids - def find_reasoning_end_offset(self, token_ids): + def find_reasoning_end(self, token_ids): self.windows.append(list(token_ids)) for offset, token in enumerate(token_ids): if token in self._end_token_ids: - return offset - return len(token_ids) + return ReasoningEnd(offset, False) + return ReasoningEnd(len(token_ids), False) def is_reasoning_end(self, input_ids): return any(token in self._end_token_ids for token in input_ids) diff --git a/tests/v1/structured_output/test_structured_output_manager.py b/tests/v1/structured_output/test_structured_output_manager.py index 6082bcd6a7..fbceb3bd07 100644 --- a/tests/v1/structured_output/test_structured_output_manager.py +++ b/tests/v1/structured_output/test_structured_output_manager.py @@ -11,6 +11,8 @@ from vllm.config import DeviceConfig, StructuredOutputsConfig, VllmConfig from vllm.config.model import ModelConfig from vllm.config.speculative import SpeculativeConfig from vllm.config.structured_outputs import StructuredOutputsBackend +from vllm.parser.engine.adapters import ParserEngineReasoningAdapter +from vllm.parser.engine.parser_engine import ReasoningEnd from vllm.sampling_params import SamplingParams, StructuredOutputsParams from vllm.v1.request import Request from vllm.v1.structured_output import StructuredOutputManager @@ -18,6 +20,7 @@ from vllm.v1.structured_output.backend_outlines import OutlinesGrammar TOKENIZER = "gpt2" THINK_END = "\n" # reasoning-end marker (single GPT-2 token) +IMPLICIT_THINK_END = "z" # accepted in strings, rejected at the object start EOS = "<|eos|>" # resolved to tokenizer.eos_token_id JSON_SCHEMA = '{"type": "object"}' BACKENDS = ("xgrammar", "guidance") @@ -43,6 +46,7 @@ class FlowCase: prefix: str = "" expected_validated: tuple[str, ...] | None = None reasoning_ended: bool | None = False + engine_reasoner: bool = False xfail_guidance: str | None = None @@ -59,6 +63,23 @@ class MockReasoner: return self.is_reasoning_end(delta_ids) +class MockEngineReasoner(ParserEngineReasoningAdapter): + """Ends reasoning implicitly on ``marker``, which is also content.""" + + def __init__(self, tokenizer, marker: int): + self.marker = marker + + @property + def reasoning_end_token_ids(self): + return frozenset({self.marker}) + + def find_reasoning_end(self, token_ids): + ids = list(token_ids) + if self.marker in ids: + return ReasoningEnd(ids.index(self.marker), True) + return ReasoningEnd(len(ids), False) + + def _single_token(tokenizer, text: str) -> int: token_ids = tokenizer.encode(text) assert len(token_ids) == 1, (text, token_ids) @@ -331,6 +352,28 @@ FLOW_CASES = [ ), id="becomes_active_terminates", ), + pytest.param( + FlowCase( + prefix='{"a": "', + raw_drafts=(IMPLICIT_THINK_END,), + expected_row_pattern="UC", + expected_reasoning=True, + expect_terminated=False, + engine_reasoner=True, + ), + id="implicit_end_accepted", + ), + pytest.param( + FlowCase( + raw_drafts=(" ", IMPLICIT_THINK_END, "{", "}"), + expected_validated=(" ",), + expected_row_pattern="UUUUU", + expected_reasoning=False, + expect_terminated=False, + engine_reasoner=True, + ), + id="implicit_end_rejected", + ), ] @@ -344,7 +387,8 @@ def test_real_flow( if backend == "guidance" and case.xfail_guidance: pytest.xfail(case.xfail_guidance) - reasoner_kwargs = {"marker": _single_token(tokenizer, THINK_END)} + marker = IMPLICIT_THINK_END if case.engine_reasoner else THINK_END + reasoner_kwargs = {"marker": _single_token(tokenizer, marker)} manager, request = _build_harness( tokenizer, backend, @@ -352,6 +396,8 @@ def test_real_flow( reasoning_ended=case.reasoning_ended, reasoning_parser_kwargs=reasoner_kwargs, ) + if case.engine_reasoner: + manager.reasoner_cls = MockEngineReasoner raw_drafts = _to_token_ids(tokenizer, case.raw_drafts) expected_texts = ( diff --git a/vllm/parser/engine/adapters.py b/vllm/parser/engine/adapters.py index 4f4aa9f5a1..cd7aaa7297 100644 --- a/vllm/parser/engine/adapters.py +++ b/vllm/parser/engine/adapters.py @@ -27,7 +27,7 @@ if TYPE_CHECKING: ChatCompletionRequest, ) from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - from vllm.parser.engine.parser_engine import ParserEngine + from vllm.parser.engine.parser_engine import ParserEngine, ReasoningEnd from vllm.tokenizers import TokenizerLike from vllm.tool_parsers.utils import Tool @@ -65,8 +65,8 @@ class ParserEngineReasoningAdapter(ReasoningParser): def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: return self._parser_engine.is_reasoning_end(list(input_ids)) - def find_reasoning_end_offset(self, token_ids: Sequence[int]) -> int | None: - return self._parser_engine.find_reasoning_end_offset(token_ids) + def find_reasoning_end(self, token_ids: Sequence[int]) -> ReasoningEnd | None: + return self._parser_engine.find_reasoning_end(token_ids) def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None: self._parser_engine.adjust_initial_state_from_prompt(prompt_token_ids) diff --git a/vllm/parser/engine/parser_engine.py b/vllm/parser/engine/parser_engine.py index 527c310c14..b04fdaa794 100644 --- a/vllm/parser/engine/parser_engine.py +++ b/vllm/parser/engine/parser_engine.py @@ -9,7 +9,7 @@ from __future__ import annotations import json from collections.abc import Sequence from functools import cached_property -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, NamedTuple import regex as re @@ -45,6 +45,13 @@ if TYPE_CHECKING: logger = init_logger(__name__) +class ReasoningEnd(NamedTuple): + """Where reasoning ends in a token sequence.""" + + offset: int # index of the first end token, or the sequence length + implicit: bool # the end token is also content, e.g. a tool-call start + + class ToolCallSlot: __slots__ = ( "id", @@ -638,14 +645,15 @@ class ParserEngine(Parser): def reasoning_end_token_ids(self) -> frozenset[int]: return self._reasoning_end_token_ids - def find_reasoning_end_offset(self, token_ids: Sequence[int]) -> int | None: + def find_reasoning_end(self, token_ids: Sequence[int]) -> ReasoningEnd | None: + """First reasoning end in `token_ids`, and if its token is content.""" end_ids = self._reasoning_end_token_ids if not end_ids: return None for offset, token_id in enumerate(token_ids): if token_id in end_ids: - return offset - return len(token_ids) + return ReasoningEnd(offset, token_id != self._reasoning_end_token_id) + return ReasoningEnd(len(token_ids), False) def is_reasoning_end(self, input_ids: list[int]) -> bool: config = self.parser_engine_config diff --git a/vllm/v1/structured_output/__init__.py b/vllm/v1/structured_output/__init__.py index 3b35d39d14..2163a800a5 100644 --- a/vllm/v1/structured_output/__init__.py +++ b/vllm/v1/structured_output/__init__.py @@ -3,7 +3,7 @@ import multiprocessing from collections.abc import Iterable, Sequence from concurrent.futures import Future, ThreadPoolExecutor -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, NamedTuple from vllm.config import VllmConfig from vllm.logger import init_logger @@ -33,6 +33,17 @@ else: logger = init_logger(__name__) +class ConstraintBounds(NamedTuple): + """Where tokens start being fed to the grammar and being masked.""" + + grammar_start: int + constraint_start: int + + +_CONSTRAINED_FROM_START = ConstraintBounds(0, 0) +_NO_SPEC_TOKENS = ConstraintBounds(1, 1) + + class StructuredOutputManager: """Engine-level manager for structured output requests.""" @@ -217,32 +228,34 @@ class StructuredOutputManager: ) -> Future: return self.executor_for_fillmask.submit(self._fill_bitmasks, batch) - def _get_constraint_start( + def _get_constraint_bounds( self, request: "Request", spec_tokens: Sequence[int], spec_tokens_committed: bool = False, - ) -> int: + ) -> ConstraintBounds: """Return the index into `spec_tokens` where tokens should start being constrained by the grammar. Assumes `spec_tokens` are stripped of -1 padding tokens. Returns `len(spec_tokens) + 1` if no token should be constrained after accepting all `spec_tokens`. + This is `constraint_start`; `grammar_start` is one less if reasoning ends + implicitly on a content token (e.g. a tool-call start). `spec_tokens_committed` is True if spec_tokens is already in `request.all_token_ids`. """ if self.enable_in_reasoning: - return 0 + return _CONSTRAINED_FROM_START structured_req = request.structured_output_request assert structured_req is not None if structured_req.reasoning_ended: - return 0 + return _CONSTRAINED_FROM_START reasoner = self._get_reasoner(request) if reasoner is None: - return 0 + return _CONSTRAINED_FROM_START if structured_req.reasoning_ended is None: # This should be removed here, but since `openai_gptoss` @@ -250,30 +263,33 @@ class StructuredOutputManager: # After unifying the `openai_gptoss` and non-`openai_gptoss` styles, # it can be removed. if reasoner.is_reasoning_end(request.prompt_token_ids or []): - return 0 + return _CONSTRAINED_FROM_START structured_req.reasoning_ended = False num_spec_tokens = len(spec_tokens) if num_spec_tokens <= 0: - return num_spec_tokens + 1 + return _NO_SPEC_TOKENS - # Use `find_reasoning_end_offset` to find constraint start if supported + # Engine-based parsers also report whether the end token is content. if ( isinstance(reasoner, ParserEngineReasoningAdapter) and reasoner.reasoning_end_token_ids ): - offset = reasoner.find_reasoning_end_offset(spec_tokens) - if offset is not None: - return offset + 1 + end = reasoner.find_reasoning_end(spec_tokens) + if end is not None: + constraint_start = end.offset + 1 + if end.implicit: + return ConstraintBounds(end.offset, constraint_start) + return ConstraintBounds(constraint_start, constraint_start) - # Fallback to `find_reasoning_end_offset` + # Fallback: the grammar starts after the end token. # TODO: Build a read-only Sequence view over all_token_ids # instead of copying the entire all_token_ids into input_ids. if spec_tokens_committed: if not spec_tokens or not reasoner.is_reasoning_end_streaming( request.all_token_ids, spec_tokens ): - return num_spec_tokens + 1 + return ConstraintBounds(num_spec_tokens + 1, num_spec_tokens + 1) input_ids = request.all_token_ids.copy() delta_ids = list(spec_tokens) @@ -281,15 +297,15 @@ class StructuredOutputManager: input_ids = request.all_token_ids.copy() input_ids.extend(spec_tokens) if not reasoner.is_reasoning_end_streaming(input_ids, spec_tokens): - return num_spec_tokens + 1 + return ConstraintBounds(num_spec_tokens + 1, num_spec_tokens + 1) delta_ids = list(spec_tokens) for i in range(num_spec_tokens - 1, 0, -1): input_ids.pop() delta_ids.pop() if not reasoner.is_reasoning_end_streaming(input_ids, delta_ids): - return i + 1 - return 1 + return ConstraintBounds(i + 1, i + 1) + return ConstraintBounds(1, 1) def validate_tokens(self, request: "Request", spec_tokens: list[int]) -> list[int]: """Return the longest unconstrained or grammar-valid prefix of `spec_tokens`.""" @@ -297,8 +313,8 @@ class StructuredOutputManager: return spec_tokens spec_tokens = strip_speculative_padding(spec_tokens) - constraint_start = self._get_constraint_start(request, spec_tokens) - if constraint_start >= len(spec_tokens): + grammar_start, _ = self._get_constraint_bounds(request, spec_tokens) + if grammar_start >= len(spec_tokens): return spec_tokens structured_req = request.structured_output_request @@ -307,8 +323,8 @@ class StructuredOutputManager: grammar = structured_req.grammar if TYPE_CHECKING: assert isinstance(grammar, StructuredOutputGrammar) - prefix = spec_tokens[:constraint_start] - validated = grammar.validate_tokens(spec_tokens[constraint_start:]) + prefix = spec_tokens[:grammar_start] + validated = grammar.validate_tokens(spec_tokens[grammar_start:]) return prefix + validated def grammar_bitmask( @@ -358,7 +374,9 @@ class StructuredOutputManager: if TYPE_CHECKING: assert isinstance(grammar, StructuredOutputGrammar) - apply_bitmask = self._get_constraint_start(request, ()) == 0 + apply_bitmask = ( + self._get_constraint_bounds(request, ()).constraint_start == 0 + ) batch.append((grammar, cumulative_index, apply_bitmask)) if len(batch) == self.fill_bitmask_parallel_batch_size: promises.append(self._async_submit_fill_bitmask(batch)) @@ -384,7 +402,7 @@ class StructuredOutputManager: assert isinstance(grammar, StructuredOutputGrammar) req_tokens = scheduled_spec_decode_tokens.get(req_id, list()) - constraint_start = self._get_constraint_start( + grammar_start, constraint_start = self._get_constraint_bounds( request, strip_speculative_padding(req_tokens) ) state_advancements = 0 @@ -402,7 +420,7 @@ class StructuredOutputManager: self._fill_bitmasks(((grammar, cumulative_index, apply_bitmask),)) if token == -1: seen_padding = True - elif apply_bitmask: + elif not seen_padding and i >= grammar_start: if not grammar.is_terminated() and grammar.accept_tokens( req_id, [token] ): @@ -453,17 +471,15 @@ class StructuredOutputManager: if TYPE_CHECKING: assert isinstance(grammar, StructuredOutputGrammar) - constraint_start = self._get_constraint_start( + grammar_start, _ = self._get_constraint_bounds( request, new_token_ids, spec_tokens_committed=True ) - # Early return only when the constraint hasn't started. + # Early return only when no reasoning-end token is present. # Otherwise, latch structured_req.reasoning_ended. - if constraint_start > len(new_token_ids): + if grammar_start > len(new_token_ids): return True structured_req.reasoning_ended = True - return grammar.accept_tokens( - request.request_id, new_token_ids[constraint_start:] - ) + return grammar.accept_tokens(request.request_id, new_token_ids[grammar_start:]) def clear_backend(self) -> None: if self.backend is not None: