[Feat][Core] safely abort requests when FSM fails to advance (#38663)
Signed-off-by: walterbm <walter.beller.morales@gmail.com>
This commit is contained in:
@@ -1406,6 +1406,23 @@ class Scheduler(SchedulerInterface):
|
||||
request.status = RequestStatus.FINISHED_STOPPED
|
||||
stopped = True
|
||||
|
||||
if new_token_ids and self.structured_output_manager.should_advance(request):
|
||||
struct_output_request = request.structured_output_request
|
||||
assert struct_output_request is not None
|
||||
assert struct_output_request.grammar is not None
|
||||
if not struct_output_request.grammar.accept_tokens( # type: ignore[union-attr]
|
||||
req_id, new_token_ids
|
||||
):
|
||||
logger.error(
|
||||
"Unexpected: grammar rejected tokens %s for request %s. "
|
||||
"Terminating request.",
|
||||
new_token_ids,
|
||||
req_id,
|
||||
)
|
||||
request.status = RequestStatus.FINISHED_ERROR
|
||||
request.resumable = False
|
||||
stopped = True
|
||||
|
||||
routed_experts = None
|
||||
finish_reason = None
|
||||
if stopped:
|
||||
@@ -1431,18 +1448,6 @@ class Scheduler(SchedulerInterface):
|
||||
):
|
||||
new_logprobs = logprobs.slice_request(req_index, len(new_token_ids))
|
||||
|
||||
if new_token_ids and self.structured_output_manager.should_advance(request):
|
||||
struct_output_request = request.structured_output_request
|
||||
assert struct_output_request is not None
|
||||
assert struct_output_request.grammar is not None
|
||||
ok = struct_output_request.grammar.accept_tokens(req_id, new_token_ids)
|
||||
if not ok:
|
||||
logger.warning(
|
||||
"Unexpected: grammar rejected tokens %s for request %s.",
|
||||
new_token_ids,
|
||||
req_id,
|
||||
)
|
||||
|
||||
if num_nans_in_logits is not None and req_id in num_nans_in_logits:
|
||||
request.num_nans_in_logits = num_nans_in_logits[req_id]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user