[Bugfix][Structured Output] Stop XGrammar token batches at termination (#52805)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -262,6 +262,54 @@ def test_validate_tokens_then_bitmask_round_trip(backend):
|
||||
assert not grammar.is_terminated()
|
||||
|
||||
|
||||
def test_xgrammar_accept_tokens_stops_at_termination(capfd):
|
||||
"""Tokens after a terminating EOS do not reach the matcher."""
|
||||
tokenizer, _, request, prompt = _make_manager_and_request("xgrammar")
|
||||
grammar = request.structured_output_request.grammar
|
||||
|
||||
assert grammar.accept_tokens(request.request_id, prompt)
|
||||
|
||||
eos = tokenizer.eos_token_id
|
||||
trailing = tokenizer.encode("\n")[0]
|
||||
processed_before = grammar.num_processed_tokens
|
||||
|
||||
assert grammar.accept_tokens(request.request_id, [eos, trailing])
|
||||
assert grammar.is_terminated()
|
||||
assert grammar.num_processed_tokens == processed_before + 1
|
||||
assert "trying to accept new token" not in capfd.readouterr().err
|
||||
|
||||
processed_after_eos = grammar.num_processed_tokens
|
||||
assert grammar.accept_tokens(request.request_id, [trailing])
|
||||
assert grammar.num_processed_tokens == processed_after_eos
|
||||
assert "trying to accept new token" not in capfd.readouterr().err
|
||||
|
||||
grammar.reset()
|
||||
assert not grammar.is_terminated()
|
||||
assert grammar.num_processed_tokens == 0
|
||||
|
||||
|
||||
def test_xgrammar_validate_tokens_stops_at_termination(capfd):
|
||||
"""Validation rolls back after reaching a terminating EOS."""
|
||||
tokenizer, _, request, prompt = _make_manager_and_request("xgrammar")
|
||||
grammar = request.structured_output_request.grammar
|
||||
|
||||
assert grammar.accept_tokens(request.request_id, prompt)
|
||||
|
||||
eos = tokenizer.eos_token_id
|
||||
trailing = tokenizer.encode("\n")[0]
|
||||
|
||||
assert grammar.validate_tokens([eos, trailing]) == [eos]
|
||||
assert "trying to accept new token" not in capfd.readouterr().err
|
||||
# Check matcher state directly to verify validation rolled it back.
|
||||
assert not grammar.matcher.is_terminated()
|
||||
|
||||
assert grammar.accept_tokens(request.request_id, [eos])
|
||||
assert grammar.is_terminated()
|
||||
|
||||
assert grammar.validate_tokens([trailing]) == []
|
||||
assert "trying to accept new token" not in capfd.readouterr().err
|
||||
|
||||
|
||||
class _MarkerReasoner:
|
||||
"""Stub reasoner whose reasoning-end marker is a single fixed token."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user