[Feat][Core] safely abort requests when FSM fails to advance (#38663)

Signed-off-by: walterbm <walter.beller.morales@gmail.com>
This commit is contained in:
Walter Beller-Morales
2026-04-06 08:00:16 -07:00
committed by GitHub
parent fef56c1855
commit e69a265135
3 changed files with 164 additions and 13 deletions
+17 -12
View File
@@ -1406,6 +1406,23 @@ class Scheduler(SchedulerInterface):
request.status = RequestStatus.FINISHED_STOPPED
stopped = True
if new_token_ids and self.structured_output_manager.should_advance(request):
struct_output_request = request.structured_output_request
assert struct_output_request is not None
assert struct_output_request.grammar is not None
if not struct_output_request.grammar.accept_tokens( # type: ignore[union-attr]
req_id, new_token_ids
):
logger.error(
"Unexpected: grammar rejected tokens %s for request %s. "
"Terminating request.",
new_token_ids,
req_id,
)
request.status = RequestStatus.FINISHED_ERROR
request.resumable = False
stopped = True
routed_experts = None
finish_reason = None
if stopped:
@@ -1431,18 +1448,6 @@ class Scheduler(SchedulerInterface):
):
new_logprobs = logprobs.slice_request(req_index, len(new_token_ids))
if new_token_ids and self.structured_output_manager.should_advance(request):
struct_output_request = request.structured_output_request
assert struct_output_request is not None
assert struct_output_request.grammar is not None
ok = struct_output_request.grammar.accept_tokens(req_id, new_token_ids)
if not ok:
logger.warning(
"Unexpected: grammar rejected tokens %s for request %s.",
new_token_ids,
req_id,
)
if num_nans_in_logits is not None and req_id in num_nans_in_logits:
request.num_nans_in_logits = num_nans_in_logits[req_id]