[Bugfix][Outlines] Fix EOS termination and unconstrained masks after rejected drafts (#58612)
This commit is contained in:
@@ -19,6 +19,8 @@ THINK_END = "\n" # reasoning-end marker (single GPT-2 token)
|
||||
EOS = "<|eos|>" # resolved to tokenizer.eos_token_id
|
||||
JSON_SCHEMA = '{"type": "object"}'
|
||||
BACKENDS = ("xgrammar", "guidance")
|
||||
REGEX_BACKENDS = (*BACKENDS, "outlines")
|
||||
DIGITS_REGEX = "[0-9]+"
|
||||
MAX_WAIT_SECONDS = 5
|
||||
NUM_SPEC_TOKENS = 8
|
||||
|
||||
@@ -88,6 +90,7 @@ def _build_harness(
|
||||
reasoning_ended: bool | None = None,
|
||||
enable_in_reasoning: bool = False,
|
||||
reasoning_parser_kwargs: dict | None = None,
|
||||
structured_outputs: StructuredOutputsParams | None = None,
|
||||
) -> tuple[StructuredOutputManager, Request]:
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(tokenizer=TOKENIZER),
|
||||
@@ -106,7 +109,8 @@ def _build_harness(
|
||||
manager.reasoner_cls = MockReasoner
|
||||
|
||||
sampling_params = SamplingParams(
|
||||
structured_outputs=StructuredOutputsParams(json=JSON_SCHEMA)
|
||||
structured_outputs=structured_outputs
|
||||
or StructuredOutputsParams(json=JSON_SCHEMA)
|
||||
)
|
||||
sampling_params.structured_outputs._backend = backend
|
||||
sampling_params.update_from_generation_config({}, tokenizer.eos_token_id)
|
||||
@@ -412,3 +416,96 @@ def test_initial_constraint_activation(
|
||||
structured_req = request.structured_output_request
|
||||
assert structured_req is not None
|
||||
assert structured_req.reasoner is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", REGEX_BACKENDS)
|
||||
@pytest.mark.parametrize(
|
||||
("raw_drafts", "expected_validated", "expected_row_pattern", "terminated"),
|
||||
[
|
||||
pytest.param(("1", "2"), None, "CCC", False, id="all_valid"),
|
||||
# EOS is only valid once the regex has matched; rows after EOS are
|
||||
# unconstrained since the request stops there.
|
||||
pytest.param(("1", EOS, "2"), ("1", EOS), "CCUU", True, id="terminates"),
|
||||
pytest.param((EOS, "1"), (), "CUU", False, id="early_eos"),
|
||||
],
|
||||
)
|
||||
def test_regex_flow(
|
||||
tokenizer,
|
||||
backend: str,
|
||||
raw_drafts: tuple[str, ...],
|
||||
expected_validated: tuple[str, ...] | None,
|
||||
expected_row_pattern: str,
|
||||
terminated: bool,
|
||||
):
|
||||
manager, request = _build_harness(
|
||||
tokenizer,
|
||||
backend,
|
||||
use_reasoner=False,
|
||||
structured_outputs=StructuredOutputsParams(regex=DIGITS_REGEX),
|
||||
)
|
||||
validated = _to_token_ids(
|
||||
tokenizer, raw_drafts if expected_validated is None else expected_validated
|
||||
)
|
||||
_run_real_flow(
|
||||
manager,
|
||||
request,
|
||||
raw_drafts=_to_token_ids(tokenizer, raw_drafts),
|
||||
expected_validated=validated,
|
||||
expected_row_pattern=expected_row_pattern,
|
||||
# accept_tokens() latches reasoning_ended once it accepts any token.
|
||||
expected_reasoning=True if validated else None,
|
||||
expect_terminated=terminated,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", REGEX_BACKENDS)
|
||||
def test_rejected_draft_keeps_later_rows_constrained(tokenizer, backend: str):
|
||||
"""A draft the grammar rejects during bitmask fill must not unconstrain
|
||||
the remaining rows or the bonus row."""
|
||||
manager, request = _build_harness(
|
||||
tokenizer,
|
||||
backend,
|
||||
use_reasoner=False,
|
||||
structured_outputs=StructuredOutputsParams(regex=DIGITS_REGEX),
|
||||
)
|
||||
drafts = _to_token_ids(tokenizer, ("1", "z", "2"))
|
||||
bitmask = manager.grammar_bitmask(
|
||||
requests={request.request_id: request},
|
||||
structured_output_request_ids=[request.request_id],
|
||||
scheduled_spec_decode_tokens={request.request_id: drafts},
|
||||
)
|
||||
assert bitmask is not None
|
||||
assert _row_pattern(bitmask) == "CCCC"
|
||||
assert (bitmask[2] == bitmask[1]).all()
|
||||
assert (bitmask[3] == bitmask[1]).all()
|
||||
|
||||
|
||||
def test_outlines_termination(tokenizer):
|
||||
"""outlines_core never advances on EOS; termination must still be
|
||||
tracked exactly once per accepted EOS and undone by rollback."""
|
||||
manager, request = _build_harness(
|
||||
tokenizer,
|
||||
"outlines",
|
||||
use_reasoner=False,
|
||||
structured_outputs=StructuredOutputsParams(regex=DIGITS_REGEX),
|
||||
)
|
||||
structured_req = request.structured_output_request
|
||||
assert structured_req is not None
|
||||
grammar = structured_req.grammar
|
||||
assert grammar is not None
|
||||
one, eos = _to_token_ids(tokenizer, ("1", EOS))
|
||||
|
||||
assert not grammar.accept_tokens(request.request_id, [eos])
|
||||
assert grammar.accept_tokens(request.request_id, [one])
|
||||
# Polling must not change the result.
|
||||
assert not any(grammar.is_terminated() for _ in range(3))
|
||||
assert grammar.accept_tokens(request.request_id, [eos])
|
||||
assert all(grammar.is_terminated() for _ in range(3))
|
||||
assert grammar.num_processed_tokens == 2
|
||||
|
||||
grammar.rollback(2)
|
||||
assert not grammar.is_terminated()
|
||||
assert grammar.num_processed_tokens == 0
|
||||
assert grammar.validate_tokens([eos]) == []
|
||||
assert grammar.accept_tokens(request.request_id, [one, eos, one])
|
||||
assert grammar.is_terminated()
|
||||
|
||||
@@ -389,11 +389,16 @@ class StructuredOutputManager:
|
||||
)
|
||||
state_advancements = 0
|
||||
seen_padding = False
|
||||
failed = False
|
||||
# Row filled from the last valid grammar state before a draft
|
||||
# was rejected; later rows reuse it rather than unconstraining.
|
||||
failed_index: int | None = None
|
||||
bitmask = self._grammar_bitmask
|
||||
for i, token in enumerate(req_tokens):
|
||||
apply_bitmask = (
|
||||
not failed and not seen_padding and i >= constraint_start
|
||||
)
|
||||
if failed_index is not None:
|
||||
bitmask[cumulative_index].copy_(bitmask[failed_index])
|
||||
cumulative_index += 1
|
||||
continue
|
||||
apply_bitmask = not seen_padding and i >= constraint_start
|
||||
self._fill_bitmasks(((grammar, cumulative_index, apply_bitmask),))
|
||||
if token == -1:
|
||||
seen_padding = True
|
||||
@@ -403,7 +408,7 @@ class StructuredOutputManager:
|
||||
):
|
||||
state_advancements += 1
|
||||
else:
|
||||
failed = True
|
||||
failed_index = cumulative_index
|
||||
logger.error(
|
||||
"Unexpected: grammar terminated or rejected draft "
|
||||
"token %s for request %s during bitmask fill.",
|
||||
@@ -415,12 +420,13 @@ class StructuredOutputManager:
|
||||
# Diffusion LLMs don't sample a bonus token after the
|
||||
# scheduled positions, so skip its bitmask in that case.
|
||||
if not (self.vllm_config.model_config.is_diffusion and req_tokens):
|
||||
bonus_apply = (
|
||||
not failed
|
||||
and not seen_padding
|
||||
and constraint_start <= len(req_tokens)
|
||||
)
|
||||
self._fill_bitmasks(((grammar, cumulative_index, bonus_apply),))
|
||||
if failed_index is not None:
|
||||
bitmask[cumulative_index].copy_(bitmask[failed_index])
|
||||
else:
|
||||
bonus_apply = not seen_padding and constraint_start <= len(
|
||||
req_tokens
|
||||
)
|
||||
self._fill_bitmasks(((grammar, cumulative_index, bonus_apply),))
|
||||
cumulative_index += 1
|
||||
|
||||
if state_advancements > 0:
|
||||
|
||||
@@ -97,6 +97,7 @@ class OutlinesBackend(StructuredOutputBackend):
|
||||
)
|
||||
return OutlinesGrammar(
|
||||
vocab_size=self.vocab_size,
|
||||
eos_token_id=self.vocabulary.inner.get_eos_token_id(),
|
||||
guide=oc.Guide(index, max_rollback=max_rollback_tokens),
|
||||
)
|
||||
|
||||
@@ -115,60 +116,94 @@ class OutlinesBackend(StructuredOutputBackend):
|
||||
@dataclass
|
||||
class OutlinesGrammar(StructuredOutputGrammar):
|
||||
vocab_size: int
|
||||
eos_token_id: int
|
||||
guide: oc.Guide = field(hash=False)
|
||||
num_processed_tokens: int = field(
|
||||
default_factory=lambda: 0, repr=False, hash=False, init=False
|
||||
)
|
||||
|
||||
# outlines_core signals done on DFA accept; vLLM expects done after EOS.
|
||||
# We delay the finished flag by one step so EOS can still be emitted.
|
||||
_prev_finished: bool = field(default=False, init=False, repr=False, hash=False)
|
||||
# outlines_core signals done on DFA accept and never advances on EOS
|
||||
# (its mask allows only EOS in a final state, but advance(EOS) fails),
|
||||
# so accepting EOS in a final state is tracked here instead.
|
||||
_is_terminated: bool = field(default=False, init=False, repr=False, hash=False)
|
||||
|
||||
def accept_tokens(self, request_id: str, tokens: list[int]) -> bool:
|
||||
"""Accepts a list of tokens and advances the FSM.
|
||||
|
||||
Returns True if the FSM was advanced successfully.
|
||||
Returns False if the FSM failed to advance.
|
||||
Returns True if all grammar-constrained tokens were accepted.
|
||||
Tokens after termination (EOS) are ignored. Returns False if the FSM
|
||||
failed to advance.
|
||||
"""
|
||||
if self.guide.accepts_tokens(tokens):
|
||||
# Advance can fail when the next state reached after advancing with
|
||||
# the current tokens is a dead state. This is because Guide.accepts_tokens()
|
||||
# only checks whether the current tokens can be accepted,
|
||||
# whereas guide.advance() additionally checks the next state
|
||||
# after all tokens are accepted.
|
||||
# We need to be aware that the FSM must be prepared without dead states.
|
||||
for t in tokens:
|
||||
self.guide.advance(t)
|
||||
self.num_processed_tokens += 1
|
||||
if self._is_terminated:
|
||||
return True
|
||||
return False
|
||||
eos_index = next(
|
||||
(i for i, t in enumerate(tokens) if t == self.eos_token_id), None
|
||||
)
|
||||
if eos_index is not None:
|
||||
tokens = tokens[:eos_index]
|
||||
# Advance can fail when the next state reached after advancing with
|
||||
# the current tokens is a dead state. This is because Guide.accepts_tokens()
|
||||
# only checks whether the current tokens can be accepted,
|
||||
# whereas guide.advance() additionally checks the next state
|
||||
# after all tokens are accepted.
|
||||
# We need to be aware that the FSM must be prepared without dead states.
|
||||
if tokens and not self.guide.accepts_tokens(tokens):
|
||||
return False
|
||||
for t in tokens:
|
||||
self.guide.advance(t)
|
||||
if eos_index is not None:
|
||||
if not self.guide.is_finished():
|
||||
if tokens:
|
||||
self.guide.rollback_state(len(tokens))
|
||||
return False
|
||||
self._is_terminated = True
|
||||
self.num_processed_tokens += len(tokens) + self._is_terminated
|
||||
return True
|
||||
|
||||
def rollback(self, num_tokens: int) -> None:
|
||||
self.guide.rollback_state(num_tokens)
|
||||
if num_tokens <= 0:
|
||||
return
|
||||
self.num_processed_tokens -= num_tokens
|
||||
if self._is_terminated:
|
||||
self._is_terminated = False
|
||||
num_tokens -= 1
|
||||
if num_tokens:
|
||||
self.guide.rollback_state(num_tokens)
|
||||
|
||||
def validate_tokens(self, tokens: list[int]) -> list[int]:
|
||||
if self._is_terminated:
|
||||
return []
|
||||
accepted: list[int] = []
|
||||
for tok in tokens:
|
||||
if tok == self.eos_token_id:
|
||||
if self._is_finished_after(accepted):
|
||||
accepted.append(tok)
|
||||
break
|
||||
accepted.append(tok)
|
||||
if not self.guide.accepts_tokens(accepted):
|
||||
accepted.pop()
|
||||
break
|
||||
return accepted
|
||||
|
||||
def _is_finished_after(self, tokens: list[int]) -> bool:
|
||||
if not tokens:
|
||||
return self.guide.is_finished()
|
||||
for t in tokens:
|
||||
self.guide.advance(t)
|
||||
finished = self.guide.is_finished()
|
||||
self.guide.rollback_state(len(tokens))
|
||||
return finished
|
||||
|
||||
def fill_bitmask(self, bitmask: torch.Tensor, idx: int) -> None:
|
||||
mask = bitmask[idx]
|
||||
self.guide.write_mask_into(mask.data_ptr(), mask.numel(), mask.element_size())
|
||||
|
||||
def is_terminated(self) -> bool:
|
||||
curr = self.guide.is_finished()
|
||||
prev = self._prev_finished
|
||||
self._prev_finished = curr
|
||||
return prev
|
||||
return self._is_terminated
|
||||
|
||||
def reset(self):
|
||||
self.num_processed_tokens = 0
|
||||
self._prev_finished = False
|
||||
self._is_terminated = False
|
||||
self.guide.reset()
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user