[Bugfix][Outlines] Fix EOS termination and unconstrained masks after rejected drafts (#58612)

This commit is contained in:
Nick Hill
2026-09-24 21:55:56 +00:00
committed by GitHub
parent 04730e8270
commit 36f94d5fe8
3 changed files with 172 additions and 34 deletions
@@ -19,6 +19,8 @@ THINK_END = "\n" # reasoning-end marker (single GPT-2 token)
EOS = "<|eos|>" # resolved to tokenizer.eos_token_id
JSON_SCHEMA = '{"type": "object"}'
BACKENDS = ("xgrammar", "guidance")
REGEX_BACKENDS = (*BACKENDS, "outlines")
DIGITS_REGEX = "[0-9]+"
MAX_WAIT_SECONDS = 5
NUM_SPEC_TOKENS = 8
@@ -88,6 +90,7 @@ def _build_harness(
reasoning_ended: bool | None = None,
enable_in_reasoning: bool = False,
reasoning_parser_kwargs: dict | None = None,
structured_outputs: StructuredOutputsParams | None = None,
) -> tuple[StructuredOutputManager, Request]:
vllm_config = VllmConfig(
model_config=ModelConfig(tokenizer=TOKENIZER),
@@ -106,7 +109,8 @@ def _build_harness(
manager.reasoner_cls = MockReasoner
sampling_params = SamplingParams(
structured_outputs=StructuredOutputsParams(json=JSON_SCHEMA)
structured_outputs=structured_outputs
or StructuredOutputsParams(json=JSON_SCHEMA)
)
sampling_params.structured_outputs._backend = backend
sampling_params.update_from_generation_config({}, tokenizer.eos_token_id)
@@ -412,3 +416,96 @@ def test_initial_constraint_activation(
structured_req = request.structured_output_request
assert structured_req is not None
assert structured_req.reasoner is None
@pytest.mark.parametrize("backend", REGEX_BACKENDS)
@pytest.mark.parametrize(
("raw_drafts", "expected_validated", "expected_row_pattern", "terminated"),
[
pytest.param(("1", "2"), None, "CCC", False, id="all_valid"),
# EOS is only valid once the regex has matched; rows after EOS are
# unconstrained since the request stops there.
pytest.param(("1", EOS, "2"), ("1", EOS), "CCUU", True, id="terminates"),
pytest.param((EOS, "1"), (), "CUU", False, id="early_eos"),
],
)
def test_regex_flow(
tokenizer,
backend: str,
raw_drafts: tuple[str, ...],
expected_validated: tuple[str, ...] | None,
expected_row_pattern: str,
terminated: bool,
):
manager, request = _build_harness(
tokenizer,
backend,
use_reasoner=False,
structured_outputs=StructuredOutputsParams(regex=DIGITS_REGEX),
)
validated = _to_token_ids(
tokenizer, raw_drafts if expected_validated is None else expected_validated
)
_run_real_flow(
manager,
request,
raw_drafts=_to_token_ids(tokenizer, raw_drafts),
expected_validated=validated,
expected_row_pattern=expected_row_pattern,
# accept_tokens() latches reasoning_ended once it accepts any token.
expected_reasoning=True if validated else None,
expect_terminated=terminated,
)
@pytest.mark.parametrize("backend", REGEX_BACKENDS)
def test_rejected_draft_keeps_later_rows_constrained(tokenizer, backend: str):
"""A draft the grammar rejects during bitmask fill must not unconstrain
the remaining rows or the bonus row."""
manager, request = _build_harness(
tokenizer,
backend,
use_reasoner=False,
structured_outputs=StructuredOutputsParams(regex=DIGITS_REGEX),
)
drafts = _to_token_ids(tokenizer, ("1", "z", "2"))
bitmask = manager.grammar_bitmask(
requests={request.request_id: request},
structured_output_request_ids=[request.request_id],
scheduled_spec_decode_tokens={request.request_id: drafts},
)
assert bitmask is not None
assert _row_pattern(bitmask) == "CCCC"
assert (bitmask[2] == bitmask[1]).all()
assert (bitmask[3] == bitmask[1]).all()
def test_outlines_termination(tokenizer):
"""outlines_core never advances on EOS; termination must still be
tracked exactly once per accepted EOS and undone by rollback."""
manager, request = _build_harness(
tokenizer,
"outlines",
use_reasoner=False,
structured_outputs=StructuredOutputsParams(regex=DIGITS_REGEX),
)
structured_req = request.structured_output_request
assert structured_req is not None
grammar = structured_req.grammar
assert grammar is not None
one, eos = _to_token_ids(tokenizer, ("1", EOS))
assert not grammar.accept_tokens(request.request_id, [eos])
assert grammar.accept_tokens(request.request_id, [one])
# Polling must not change the result.
assert not any(grammar.is_terminated() for _ in range(3))
assert grammar.accept_tokens(request.request_id, [eos])
assert all(grammar.is_terminated() for _ in range(3))
assert grammar.num_processed_tokens == 2
grammar.rollback(2)
assert not grammar.is_terminated()
assert grammar.num_processed_tokens == 0
assert grammar.validate_tokens([eos]) == []
assert grammar.accept_tokens(request.request_id, [one, eos, one])
assert grammar.is_terminated()