[Bugfix][Outlines] Fix EOS termination and unconstrained masks after rejected drafts (#58612)
This commit is contained in:
@@ -19,6 +19,8 @@ THINK_END = "\n" # reasoning-end marker (single GPT-2 token)
|
||||
EOS = "<|eos|>" # resolved to tokenizer.eos_token_id
|
||||
JSON_SCHEMA = '{"type": "object"}'
|
||||
BACKENDS = ("xgrammar", "guidance")
|
||||
REGEX_BACKENDS = (*BACKENDS, "outlines")
|
||||
DIGITS_REGEX = "[0-9]+"
|
||||
MAX_WAIT_SECONDS = 5
|
||||
NUM_SPEC_TOKENS = 8
|
||||
|
||||
@@ -88,6 +90,7 @@ def _build_harness(
|
||||
reasoning_ended: bool | None = None,
|
||||
enable_in_reasoning: bool = False,
|
||||
reasoning_parser_kwargs: dict | None = None,
|
||||
structured_outputs: StructuredOutputsParams | None = None,
|
||||
) -> tuple[StructuredOutputManager, Request]:
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(tokenizer=TOKENIZER),
|
||||
@@ -106,7 +109,8 @@ def _build_harness(
|
||||
manager.reasoner_cls = MockReasoner
|
||||
|
||||
sampling_params = SamplingParams(
|
||||
structured_outputs=StructuredOutputsParams(json=JSON_SCHEMA)
|
||||
structured_outputs=structured_outputs
|
||||
or StructuredOutputsParams(json=JSON_SCHEMA)
|
||||
)
|
||||
sampling_params.structured_outputs._backend = backend
|
||||
sampling_params.update_from_generation_config({}, tokenizer.eos_token_id)
|
||||
@@ -412,3 +416,96 @@ def test_initial_constraint_activation(
|
||||
structured_req = request.structured_output_request
|
||||
assert structured_req is not None
|
||||
assert structured_req.reasoner is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", REGEX_BACKENDS)
|
||||
@pytest.mark.parametrize(
|
||||
("raw_drafts", "expected_validated", "expected_row_pattern", "terminated"),
|
||||
[
|
||||
pytest.param(("1", "2"), None, "CCC", False, id="all_valid"),
|
||||
# EOS is only valid once the regex has matched; rows after EOS are
|
||||
# unconstrained since the request stops there.
|
||||
pytest.param(("1", EOS, "2"), ("1", EOS), "CCUU", True, id="terminates"),
|
||||
pytest.param((EOS, "1"), (), "CUU", False, id="early_eos"),
|
||||
],
|
||||
)
|
||||
def test_regex_flow(
|
||||
tokenizer,
|
||||
backend: str,
|
||||
raw_drafts: tuple[str, ...],
|
||||
expected_validated: tuple[str, ...] | None,
|
||||
expected_row_pattern: str,
|
||||
terminated: bool,
|
||||
):
|
||||
manager, request = _build_harness(
|
||||
tokenizer,
|
||||
backend,
|
||||
use_reasoner=False,
|
||||
structured_outputs=StructuredOutputsParams(regex=DIGITS_REGEX),
|
||||
)
|
||||
validated = _to_token_ids(
|
||||
tokenizer, raw_drafts if expected_validated is None else expected_validated
|
||||
)
|
||||
_run_real_flow(
|
||||
manager,
|
||||
request,
|
||||
raw_drafts=_to_token_ids(tokenizer, raw_drafts),
|
||||
expected_validated=validated,
|
||||
expected_row_pattern=expected_row_pattern,
|
||||
# accept_tokens() latches reasoning_ended once it accepts any token.
|
||||
expected_reasoning=True if validated else None,
|
||||
expect_terminated=terminated,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", REGEX_BACKENDS)
|
||||
def test_rejected_draft_keeps_later_rows_constrained(tokenizer, backend: str):
|
||||
"""A draft the grammar rejects during bitmask fill must not unconstrain
|
||||
the remaining rows or the bonus row."""
|
||||
manager, request = _build_harness(
|
||||
tokenizer,
|
||||
backend,
|
||||
use_reasoner=False,
|
||||
structured_outputs=StructuredOutputsParams(regex=DIGITS_REGEX),
|
||||
)
|
||||
drafts = _to_token_ids(tokenizer, ("1", "z", "2"))
|
||||
bitmask = manager.grammar_bitmask(
|
||||
requests={request.request_id: request},
|
||||
structured_output_request_ids=[request.request_id],
|
||||
scheduled_spec_decode_tokens={request.request_id: drafts},
|
||||
)
|
||||
assert bitmask is not None
|
||||
assert _row_pattern(bitmask) == "CCCC"
|
||||
assert (bitmask[2] == bitmask[1]).all()
|
||||
assert (bitmask[3] == bitmask[1]).all()
|
||||
|
||||
|
||||
def test_outlines_termination(tokenizer):
|
||||
"""outlines_core never advances on EOS; termination must still be
|
||||
tracked exactly once per accepted EOS and undone by rollback."""
|
||||
manager, request = _build_harness(
|
||||
tokenizer,
|
||||
"outlines",
|
||||
use_reasoner=False,
|
||||
structured_outputs=StructuredOutputsParams(regex=DIGITS_REGEX),
|
||||
)
|
||||
structured_req = request.structured_output_request
|
||||
assert structured_req is not None
|
||||
grammar = structured_req.grammar
|
||||
assert grammar is not None
|
||||
one, eos = _to_token_ids(tokenizer, ("1", EOS))
|
||||
|
||||
assert not grammar.accept_tokens(request.request_id, [eos])
|
||||
assert grammar.accept_tokens(request.request_id, [one])
|
||||
# Polling must not change the result.
|
||||
assert not any(grammar.is_terminated() for _ in range(3))
|
||||
assert grammar.accept_tokens(request.request_id, [eos])
|
||||
assert all(grammar.is_terminated() for _ in range(3))
|
||||
assert grammar.num_processed_tokens == 2
|
||||
|
||||
grammar.rollback(2)
|
||||
assert not grammar.is_terminated()
|
||||
assert grammar.num_processed_tokens == 0
|
||||
assert grammar.validate_tokens([eos]) == []
|
||||
assert grammar.accept_tokens(request.request_id, [one, eos, one])
|
||||
assert grammar.is_terminated()
|
||||
|
||||
Reference in New Issue
Block a user