[Bugfix][Structured Output] Stop XGrammar token batches at termination (#52805)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -262,6 +262,54 @@ def test_validate_tokens_then_bitmask_round_trip(backend):
|
||||
assert not grammar.is_terminated()
|
||||
|
||||
|
||||
def test_xgrammar_accept_tokens_stops_at_termination(capfd):
|
||||
"""Tokens after a terminating EOS do not reach the matcher."""
|
||||
tokenizer, _, request, prompt = _make_manager_and_request("xgrammar")
|
||||
grammar = request.structured_output_request.grammar
|
||||
|
||||
assert grammar.accept_tokens(request.request_id, prompt)
|
||||
|
||||
eos = tokenizer.eos_token_id
|
||||
trailing = tokenizer.encode("\n")[0]
|
||||
processed_before = grammar.num_processed_tokens
|
||||
|
||||
assert grammar.accept_tokens(request.request_id, [eos, trailing])
|
||||
assert grammar.is_terminated()
|
||||
assert grammar.num_processed_tokens == processed_before + 1
|
||||
assert "trying to accept new token" not in capfd.readouterr().err
|
||||
|
||||
processed_after_eos = grammar.num_processed_tokens
|
||||
assert grammar.accept_tokens(request.request_id, [trailing])
|
||||
assert grammar.num_processed_tokens == processed_after_eos
|
||||
assert "trying to accept new token" not in capfd.readouterr().err
|
||||
|
||||
grammar.reset()
|
||||
assert not grammar.is_terminated()
|
||||
assert grammar.num_processed_tokens == 0
|
||||
|
||||
|
||||
def test_xgrammar_validate_tokens_stops_at_termination(capfd):
|
||||
"""Validation rolls back after reaching a terminating EOS."""
|
||||
tokenizer, _, request, prompt = _make_manager_and_request("xgrammar")
|
||||
grammar = request.structured_output_request.grammar
|
||||
|
||||
assert grammar.accept_tokens(request.request_id, prompt)
|
||||
|
||||
eos = tokenizer.eos_token_id
|
||||
trailing = tokenizer.encode("\n")[0]
|
||||
|
||||
assert grammar.validate_tokens([eos, trailing]) == [eos]
|
||||
assert "trying to accept new token" not in capfd.readouterr().err
|
||||
# Check matcher state directly to verify validation rolled it back.
|
||||
assert not grammar.matcher.is_terminated()
|
||||
|
||||
assert grammar.accept_tokens(request.request_id, [eos])
|
||||
assert grammar.is_terminated()
|
||||
|
||||
assert grammar.validate_tokens([trailing]) == []
|
||||
assert "trying to accept new token" not in capfd.readouterr().err
|
||||
|
||||
|
||||
class _MarkerReasoner:
|
||||
"""Stub reasoner whose reasoning-end marker is a single fixed token."""
|
||||
|
||||
|
||||
@@ -157,11 +157,12 @@ class XgrammarGrammar(StructuredOutputGrammar):
|
||||
def accept_tokens(self, request_id: str, tokens: list[int]) -> bool:
|
||||
"""Accepts a list of tokens and advances the FSM.
|
||||
|
||||
Returns True if the FSM was advanced successfully.
|
||||
Returns False if the FSM failed to advance.
|
||||
Returns True if all grammar-constrained tokens were accepted.
|
||||
Tokens after termination are ignored. Returns False if the FSM
|
||||
failed to advance.
|
||||
"""
|
||||
if self._is_terminated:
|
||||
return False
|
||||
return True
|
||||
for token in tokens:
|
||||
if not self.matcher.accept_token(token):
|
||||
logger.error(
|
||||
@@ -172,7 +173,9 @@ class XgrammarGrammar(StructuredOutputGrammar):
|
||||
)
|
||||
return False
|
||||
self.num_processed_tokens += 1
|
||||
self._is_terminated = self.matcher.is_terminated()
|
||||
self._is_terminated = self.matcher.is_terminated()
|
||||
if self._is_terminated:
|
||||
break
|
||||
return True
|
||||
|
||||
def validate_tokens(self, tokens: list[int]) -> list[int]:
|
||||
@@ -181,10 +184,15 @@ class XgrammarGrammar(StructuredOutputGrammar):
|
||||
|
||||
Returns the prefix list of tokens that are accepted by the FSM.
|
||||
"""
|
||||
if self._is_terminated:
|
||||
return []
|
||||
|
||||
accepted_tokens = []
|
||||
for token in tokens:
|
||||
if self.matcher.accept_token(token):
|
||||
accepted_tokens.append(token)
|
||||
if self.matcher.is_terminated():
|
||||
break
|
||||
else:
|
||||
break
|
||||
if len(accepted_tokens) > 0:
|
||||
@@ -204,8 +212,9 @@ class XgrammarGrammar(StructuredOutputGrammar):
|
||||
return self._is_terminated
|
||||
|
||||
def reset(self):
|
||||
self.num_processed_tokens = 0
|
||||
self.matcher.reset()
|
||||
self.num_processed_tokens = 0
|
||||
self._is_terminated = False
|
||||
|
||||
|
||||
# cf https://github.com/mlc-ai/xgrammar/blob/a32ac892676d2eedc0327416105b9b06edfb94b2/cpp/json_schema_converter.cc
|
||||
|
||||
Reference in New Issue
Block a user