[Bugfix][Outlines] Fix EOS termination and unconstrained masks after rejected drafts (#58612)

This commit is contained in:
Nick Hill
2026-09-24 21:55:56 +00:00
committed by GitHub
parent 04730e8270
commit 36f94d5fe8
3 changed files with 172 additions and 34 deletions
@@ -19,6 +19,8 @@ THINK_END = "\n" # reasoning-end marker (single GPT-2 token)
EOS = "<|eos|>" # resolved to tokenizer.eos_token_id
JSON_SCHEMA = '{"type": "object"}'
BACKENDS = ("xgrammar", "guidance")
REGEX_BACKENDS = (*BACKENDS, "outlines")
DIGITS_REGEX = "[0-9]+"
MAX_WAIT_SECONDS = 5
NUM_SPEC_TOKENS = 8
@@ -88,6 +90,7 @@ def _build_harness(
reasoning_ended: bool | None = None,
enable_in_reasoning: bool = False,
reasoning_parser_kwargs: dict | None = None,
structured_outputs: StructuredOutputsParams | None = None,
) -> tuple[StructuredOutputManager, Request]:
vllm_config = VllmConfig(
model_config=ModelConfig(tokenizer=TOKENIZER),
@@ -106,7 +109,8 @@ def _build_harness(
manager.reasoner_cls = MockReasoner
sampling_params = SamplingParams(
structured_outputs=StructuredOutputsParams(json=JSON_SCHEMA)
structured_outputs=structured_outputs
or StructuredOutputsParams(json=JSON_SCHEMA)
)
sampling_params.structured_outputs._backend = backend
sampling_params.update_from_generation_config({}, tokenizer.eos_token_id)
@@ -412,3 +416,96 @@ def test_initial_constraint_activation(
structured_req = request.structured_output_request
assert structured_req is not None
assert structured_req.reasoner is None
@pytest.mark.parametrize("backend", REGEX_BACKENDS)
@pytest.mark.parametrize(
("raw_drafts", "expected_validated", "expected_row_pattern", "terminated"),
[
pytest.param(("1", "2"), None, "CCC", False, id="all_valid"),
# EOS is only valid once the regex has matched; rows after EOS are
# unconstrained since the request stops there.
pytest.param(("1", EOS, "2"), ("1", EOS), "CCUU", True, id="terminates"),
pytest.param((EOS, "1"), (), "CUU", False, id="early_eos"),
],
)
def test_regex_flow(
tokenizer,
backend: str,
raw_drafts: tuple[str, ...],
expected_validated: tuple[str, ...] | None,
expected_row_pattern: str,
terminated: bool,
):
manager, request = _build_harness(
tokenizer,
backend,
use_reasoner=False,
structured_outputs=StructuredOutputsParams(regex=DIGITS_REGEX),
)
validated = _to_token_ids(
tokenizer, raw_drafts if expected_validated is None else expected_validated
)
_run_real_flow(
manager,
request,
raw_drafts=_to_token_ids(tokenizer, raw_drafts),
expected_validated=validated,
expected_row_pattern=expected_row_pattern,
# accept_tokens() latches reasoning_ended once it accepts any token.
expected_reasoning=True if validated else None,
expect_terminated=terminated,
)
@pytest.mark.parametrize("backend", REGEX_BACKENDS)
def test_rejected_draft_keeps_later_rows_constrained(tokenizer, backend: str):
"""A draft the grammar rejects during bitmask fill must not unconstrain
the remaining rows or the bonus row."""
manager, request = _build_harness(
tokenizer,
backend,
use_reasoner=False,
structured_outputs=StructuredOutputsParams(regex=DIGITS_REGEX),
)
drafts = _to_token_ids(tokenizer, ("1", "z", "2"))
bitmask = manager.grammar_bitmask(
requests={request.request_id: request},
structured_output_request_ids=[request.request_id],
scheduled_spec_decode_tokens={request.request_id: drafts},
)
assert bitmask is not None
assert _row_pattern(bitmask) == "CCCC"
assert (bitmask[2] == bitmask[1]).all()
assert (bitmask[3] == bitmask[1]).all()
def test_outlines_termination(tokenizer):
"""outlines_core never advances on EOS; termination must still be
tracked exactly once per accepted EOS and undone by rollback."""
manager, request = _build_harness(
tokenizer,
"outlines",
use_reasoner=False,
structured_outputs=StructuredOutputsParams(regex=DIGITS_REGEX),
)
structured_req = request.structured_output_request
assert structured_req is not None
grammar = structured_req.grammar
assert grammar is not None
one, eos = _to_token_ids(tokenizer, ("1", EOS))
assert not grammar.accept_tokens(request.request_id, [eos])
assert grammar.accept_tokens(request.request_id, [one])
# Polling must not change the result.
assert not any(grammar.is_terminated() for _ in range(3))
assert grammar.accept_tokens(request.request_id, [eos])
assert all(grammar.is_terminated() for _ in range(3))
assert grammar.num_processed_tokens == 2
grammar.rollback(2)
assert not grammar.is_terminated()
assert grammar.num_processed_tokens == 0
assert grammar.validate_tokens([eos]) == []
assert grammar.accept_tokens(request.request_id, [one, eos, one])
assert grammar.is_terminated()
+17 -11
View File
@@ -389,11 +389,16 @@ class StructuredOutputManager:
)
state_advancements = 0
seen_padding = False
failed = False
# Row filled from the last valid grammar state before a draft
# was rejected; later rows reuse it rather than unconstraining.
failed_index: int | None = None
bitmask = self._grammar_bitmask
for i, token in enumerate(req_tokens):
apply_bitmask = (
not failed and not seen_padding and i >= constraint_start
)
if failed_index is not None:
bitmask[cumulative_index].copy_(bitmask[failed_index])
cumulative_index += 1
continue
apply_bitmask = not seen_padding and i >= constraint_start
self._fill_bitmasks(((grammar, cumulative_index, apply_bitmask),))
if token == -1:
seen_padding = True
@@ -403,7 +408,7 @@ class StructuredOutputManager:
):
state_advancements += 1
else:
failed = True
failed_index = cumulative_index
logger.error(
"Unexpected: grammar terminated or rejected draft "
"token %s for request %s during bitmask fill.",
@@ -415,12 +420,13 @@ class StructuredOutputManager:
# Diffusion LLMs don't sample a bonus token after the
# scheduled positions, so skip its bitmask in that case.
if not (self.vllm_config.model_config.is_diffusion and req_tokens):
bonus_apply = (
not failed
and not seen_padding
and constraint_start <= len(req_tokens)
)
self._fill_bitmasks(((grammar, cumulative_index, bonus_apply),))
if failed_index is not None:
bitmask[cumulative_index].copy_(bitmask[failed_index])
else:
bonus_apply = not seen_padding and constraint_start <= len(
req_tokens
)
self._fill_bitmasks(((grammar, cumulative_index, bonus_apply),))
cumulative_index += 1
if state_advancements > 0:
+57 -22
View File
@@ -97,6 +97,7 @@ class OutlinesBackend(StructuredOutputBackend):
)
return OutlinesGrammar(
vocab_size=self.vocab_size,
eos_token_id=self.vocabulary.inner.get_eos_token_id(),
guide=oc.Guide(index, max_rollback=max_rollback_tokens),
)
@@ -115,60 +116,94 @@ class OutlinesBackend(StructuredOutputBackend):
@dataclass
class OutlinesGrammar(StructuredOutputGrammar):
vocab_size: int
eos_token_id: int
guide: oc.Guide = field(hash=False)
num_processed_tokens: int = field(
default_factory=lambda: 0, repr=False, hash=False, init=False
)
# outlines_core signals done on DFA accept; vLLM expects done after EOS.
# We delay the finished flag by one step so EOS can still be emitted.
_prev_finished: bool = field(default=False, init=False, repr=False, hash=False)
# outlines_core signals done on DFA accept and never advances on EOS
# (its mask allows only EOS in a final state, but advance(EOS) fails),
# so accepting EOS in a final state is tracked here instead.
_is_terminated: bool = field(default=False, init=False, repr=False, hash=False)
def accept_tokens(self, request_id: str, tokens: list[int]) -> bool:
"""Accepts a list of tokens and advances the FSM.
Returns True if the FSM was advanced successfully.
Returns False if the FSM failed to advance.
Returns True if all grammar-constrained tokens were accepted.
Tokens after termination (EOS) are ignored. Returns False if the FSM
failed to advance.
"""
if self.guide.accepts_tokens(tokens):
# Advance can fail when the next state reached after advancing with
# the current tokens is a dead state. This is because Guide.accepts_tokens()
# only checks whether the current tokens can be accepted,
# whereas guide.advance() additionally checks the next state
# after all tokens are accepted.
# We need to be aware that the FSM must be prepared without dead states.
for t in tokens:
self.guide.advance(t)
self.num_processed_tokens += 1
if self._is_terminated:
return True
return False
eos_index = next(
(i for i, t in enumerate(tokens) if t == self.eos_token_id), None
)
if eos_index is not None:
tokens = tokens[:eos_index]
# Advance can fail when the next state reached after advancing with
# the current tokens is a dead state. This is because Guide.accepts_tokens()
# only checks whether the current tokens can be accepted,
# whereas guide.advance() additionally checks the next state
# after all tokens are accepted.
# We need to be aware that the FSM must be prepared without dead states.
if tokens and not self.guide.accepts_tokens(tokens):
return False
for t in tokens:
self.guide.advance(t)
if eos_index is not None:
if not self.guide.is_finished():
if tokens:
self.guide.rollback_state(len(tokens))
return False
self._is_terminated = True
self.num_processed_tokens += len(tokens) + self._is_terminated
return True
def rollback(self, num_tokens: int) -> None:
self.guide.rollback_state(num_tokens)
if num_tokens <= 0:
return
self.num_processed_tokens -= num_tokens
if self._is_terminated:
self._is_terminated = False
num_tokens -= 1
if num_tokens:
self.guide.rollback_state(num_tokens)
def validate_tokens(self, tokens: list[int]) -> list[int]:
if self._is_terminated:
return []
accepted: list[int] = []
for tok in tokens:
if tok == self.eos_token_id:
if self._is_finished_after(accepted):
accepted.append(tok)
break
accepted.append(tok)
if not self.guide.accepts_tokens(accepted):
accepted.pop()
break
return accepted
def _is_finished_after(self, tokens: list[int]) -> bool:
if not tokens:
return self.guide.is_finished()
for t in tokens:
self.guide.advance(t)
finished = self.guide.is_finished()
self.guide.rollback_state(len(tokens))
return finished
def fill_bitmask(self, bitmask: torch.Tensor, idx: int) -> None:
mask = bitmask[idx]
self.guide.write_mask_into(mask.data_ptr(), mask.numel(), mask.element_size())
def is_terminated(self) -> bool:
curr = self.guide.is_finished()
prev = self._prev_finished
self._prev_finished = curr
return prev
return self._is_terminated
def reset(self):
self.num_processed_tokens = 0
self._prev_finished = False
self._is_terminated = False
self.guide.reset()