[Bugfix][Structured Output] Mask grammar-invalid implicit reasoning ends (#61574)
Signed-off-by: Maroon Ayoub <mayoub@redhat.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude
Robert Shaw
parent
92dc3dd393
commit
4ac1f8d2ac
@@ -5,6 +5,8 @@ from collections.abc import Iterable, Sequence
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from typing import TYPE_CHECKING, NamedTuple
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser.engine.adapters import ParserEngineReasoningAdapter
|
||||
@@ -20,7 +22,6 @@ from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend
|
||||
from vllm.v1.structured_output.utils import strip_speculative_padding
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
import torch
|
||||
|
||||
@@ -211,10 +212,16 @@ class StructuredOutputManager:
|
||||
raise
|
||||
|
||||
def _fill_bitmasks(
|
||||
self, batch: Iterable[tuple[StructuredOutputGrammar, int, bool]]
|
||||
self,
|
||||
batch: Iterable[tuple[StructuredOutputGrammar, int, bool, tuple[int, ...]]],
|
||||
) -> None:
|
||||
"""Fill one bitmask row per entry.
|
||||
|
||||
A row the grammar does not constrain allows every token except
|
||||
`masked_ids`, the implicit reasoning ends masked while reasoning.
|
||||
"""
|
||||
assert self._grammar_bitmask is not None
|
||||
for grammar, index, apply_bitmask in batch:
|
||||
for grammar, index, apply_bitmask, masked_ids in batch:
|
||||
if apply_bitmask and not grammar.is_terminated():
|
||||
grammar.fill_bitmask(self._grammar_bitmask, index)
|
||||
else:
|
||||
@@ -222,12 +229,41 @@ class StructuredOutputManager:
|
||||
# reset the relevant part of the bitmask for consequent
|
||||
# requests here.
|
||||
self._grammar_bitmask[index].fill_(self._full_mask)
|
||||
if masked_ids:
|
||||
row = self._grammar_bitmask[index].numpy().view(np.uint32)
|
||||
for token_id in masked_ids:
|
||||
row[token_id >> 5] &= ~np.uint32(1 << (token_id & 31))
|
||||
|
||||
def _async_submit_fill_bitmask(
|
||||
self, batch: list[tuple[StructuredOutputGrammar, int, bool]]
|
||||
self, batch: list[tuple[StructuredOutputGrammar, int, bool, tuple[int, ...]]]
|
||||
) -> Future:
|
||||
return self.executor_for_fillmask.submit(self._fill_bitmasks, batch)
|
||||
|
||||
def _masked_reasoning_ends(self, request: "Request") -> tuple[int, ...]:
|
||||
"""Implicit reasoning-end tokens the grammar rejects as first content.
|
||||
|
||||
Such a token (e.g. a tool-call start) ends reasoning and is also the
|
||||
first content token, so sampling one where the grammar cannot start
|
||||
with it would make the output invalid. These stay masked while the
|
||||
request reasons. The grammar does not advance before reasoning ends,
|
||||
so the set is fixed per request.
|
||||
"""
|
||||
structured_req = request.structured_output_request
|
||||
assert structured_req is not None
|
||||
if structured_req.masked_reasoning_end_ids is None:
|
||||
masked: tuple[int, ...] = ()
|
||||
reasoner = self._get_reasoner(request)
|
||||
if isinstance(reasoner, ParserEngineReasoningAdapter):
|
||||
grammar = structured_req.grammar
|
||||
assert isinstance(grammar, StructuredOutputGrammar)
|
||||
masked = tuple(
|
||||
token_id
|
||||
for token_id in sorted(reasoner.implicit_reasoning_end_token_ids)
|
||||
if not grammar.validate_tokens([token_id])
|
||||
)
|
||||
structured_req.masked_reasoning_end_ids = masked
|
||||
return structured_req.masked_reasoning_end_ids
|
||||
|
||||
def _get_constraint_bounds(
|
||||
self,
|
||||
request: "Request",
|
||||
@@ -377,7 +413,10 @@ class StructuredOutputManager:
|
||||
apply_bitmask = (
|
||||
self._get_constraint_bounds(request, ()).constraint_start == 0
|
||||
)
|
||||
batch.append((grammar, cumulative_index, apply_bitmask))
|
||||
masked_ids = (
|
||||
() if apply_bitmask else self._masked_reasoning_ends(request)
|
||||
)
|
||||
batch.append((grammar, cumulative_index, apply_bitmask, masked_ids))
|
||||
if len(batch) == self.fill_bitmask_parallel_batch_size:
|
||||
promises.append(self._async_submit_fill_bitmask(batch))
|
||||
batch = []
|
||||
@@ -405,6 +444,10 @@ class StructuredOutputManager:
|
||||
grammar_start, constraint_start = self._get_constraint_bounds(
|
||||
request, strip_speculative_padding(req_tokens)
|
||||
)
|
||||
# Rows before constraint_start are still reasoning.
|
||||
masked_ids = (
|
||||
self._masked_reasoning_ends(request) if constraint_start else ()
|
||||
)
|
||||
state_advancements = 0
|
||||
seen_padding = False
|
||||
# Row filled from the last valid grammar state before a draft
|
||||
@@ -417,7 +460,11 @@ class StructuredOutputManager:
|
||||
cumulative_index += 1
|
||||
continue
|
||||
apply_bitmask = not seen_padding and i >= constraint_start
|
||||
self._fill_bitmasks(((grammar, cumulative_index, apply_bitmask),))
|
||||
reasoning = not seen_padding and i < constraint_start
|
||||
row_masked = masked_ids if reasoning else ()
|
||||
self._fill_bitmasks(
|
||||
((grammar, cumulative_index, apply_bitmask, row_masked),)
|
||||
)
|
||||
if token == -1:
|
||||
seen_padding = True
|
||||
elif not seen_padding and i >= grammar_start:
|
||||
@@ -444,7 +491,11 @@ class StructuredOutputManager:
|
||||
bonus_apply = not seen_padding and constraint_start <= len(
|
||||
req_tokens
|
||||
)
|
||||
self._fill_bitmasks(((grammar, cumulative_index, bonus_apply),))
|
||||
bonus_reasoning = not seen_padding and not bonus_apply
|
||||
bonus_masked = masked_ids if bonus_reasoning else ()
|
||||
self._fill_bitmasks(
|
||||
((grammar, cumulative_index, bonus_apply, bonus_masked),)
|
||||
)
|
||||
cumulative_index += 1
|
||||
|
||||
if state_advancements > 0:
|
||||
|
||||
@@ -28,6 +28,9 @@ class StructuredOutputRequest:
|
||||
# Cached per request; do not share reasoning parsers across requests because
|
||||
# their behavior can depend on reasoning_parser_kwargs.
|
||||
reasoner: "ReasoningParser | None" = None
|
||||
# Implicit reasoning-end tokens the grammar rejects as the first content
|
||||
# token; masked while the request reasons. Computed on first use.
|
||||
masked_reasoning_end_ids: tuple[int, ...] | None = None
|
||||
|
||||
@staticmethod
|
||||
def from_sampling_params(
|
||||
|
||||
Reference in New Issue
Block a user