[Bugfix][Structured Output] Stop XGrammar token batches at termination (#52805)

Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
Flora Feng
2026-08-18 21:21:43 +00:00
committed by GitHub
parent 7ddb50788d
commit 12f64b39d2
2 changed files with 62 additions and 5 deletions
@@ -262,6 +262,54 @@ def test_validate_tokens_then_bitmask_round_trip(backend):
assert not grammar.is_terminated()
def test_xgrammar_accept_tokens_stops_at_termination(capfd):
"""Tokens after a terminating EOS do not reach the matcher."""
tokenizer, _, request, prompt = _make_manager_and_request("xgrammar")
grammar = request.structured_output_request.grammar
assert grammar.accept_tokens(request.request_id, prompt)
eos = tokenizer.eos_token_id
trailing = tokenizer.encode("\n")[0]
processed_before = grammar.num_processed_tokens
assert grammar.accept_tokens(request.request_id, [eos, trailing])
assert grammar.is_terminated()
assert grammar.num_processed_tokens == processed_before + 1
assert "trying to accept new token" not in capfd.readouterr().err
processed_after_eos = grammar.num_processed_tokens
assert grammar.accept_tokens(request.request_id, [trailing])
assert grammar.num_processed_tokens == processed_after_eos
assert "trying to accept new token" not in capfd.readouterr().err
grammar.reset()
assert not grammar.is_terminated()
assert grammar.num_processed_tokens == 0
def test_xgrammar_validate_tokens_stops_at_termination(capfd):
"""Validation rolls back after reaching a terminating EOS."""
tokenizer, _, request, prompt = _make_manager_and_request("xgrammar")
grammar = request.structured_output_request.grammar
assert grammar.accept_tokens(request.request_id, prompt)
eos = tokenizer.eos_token_id
trailing = tokenizer.encode("\n")[0]
assert grammar.validate_tokens([eos, trailing]) == [eos]
assert "trying to accept new token" not in capfd.readouterr().err
# Check matcher state directly to verify validation rolled it back.
assert not grammar.matcher.is_terminated()
assert grammar.accept_tokens(request.request_id, [eos])
assert grammar.is_terminated()
assert grammar.validate_tokens([trailing]) == []
assert "trying to accept new token" not in capfd.readouterr().err
class _MarkerReasoner:
"""Stub reasoner whose reasoning-end marker is a single fixed token."""
+14 -5
View File
@@ -157,11 +157,12 @@ class XgrammarGrammar(StructuredOutputGrammar):
def accept_tokens(self, request_id: str, tokens: list[int]) -> bool:
"""Accepts a list of tokens and advances the FSM.
Returns True if the FSM was advanced successfully.
Returns False if the FSM failed to advance.
Returns True if all grammar-constrained tokens were accepted.
Tokens after termination are ignored. Returns False if the FSM
failed to advance.
"""
if self._is_terminated:
return False
return True
for token in tokens:
if not self.matcher.accept_token(token):
logger.error(
@@ -172,7 +173,9 @@ class XgrammarGrammar(StructuredOutputGrammar):
)
return False
self.num_processed_tokens += 1
self._is_terminated = self.matcher.is_terminated()
self._is_terminated = self.matcher.is_terminated()
if self._is_terminated:
break
return True
def validate_tokens(self, tokens: list[int]) -> list[int]:
@@ -181,10 +184,15 @@ class XgrammarGrammar(StructuredOutputGrammar):
Returns the prefix list of tokens that are accepted by the FSM.
"""
if self._is_terminated:
return []
accepted_tokens = []
for token in tokens:
if self.matcher.accept_token(token):
accepted_tokens.append(token)
if self.matcher.is_terminated():
break
else:
break
if len(accepted_tokens) > 0:
@@ -204,8 +212,9 @@ class XgrammarGrammar(StructuredOutputGrammar):
return self._is_terminated
def reset(self):
self.num_processed_tokens = 0
self.matcher.reset()
self.num_processed_tokens = 0
self._is_terminated = False
# cf https://github.com/mlc-ai/xgrammar/blob/a32ac892676d2eedc0327416105b9b06edfb94b2/cpp/json_schema_converter.cc