[Bugfix] handle grammar compilation failures to avoid engine crash (#47312)
Signed-off-by: zhuhaoran <zhuhaoran.zhr@alibaba-inc.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -9,6 +9,7 @@ from vllm.v1.core.sched.async_scheduler import AsyncScheduler
|
||||
from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
from vllm.v1.request import RequestStatus
|
||||
from vllm.v1.structured_output import StructuredOutputGrammar
|
||||
from vllm.v1.utils import ConstantList
|
||||
|
||||
from .utils import create_requests, create_scheduler
|
||||
@@ -262,7 +263,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
|
||||
scheduler = object.__new__(AsyncScheduler)
|
||||
request = create_requests(num_requests=1, num_tokens=1)[0]
|
||||
request.structured_output_request = Mock()
|
||||
request.structured_output_request.grammar = Mock()
|
||||
request.structured_output_request.grammar = Mock(spec=StructuredOutputGrammar)
|
||||
request.structured_output_request.grammar.accept_tokens.return_value = False
|
||||
request.status = RequestStatus.RUNNING
|
||||
request.num_computed_tokens = request.num_tokens
|
||||
@@ -284,6 +285,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
|
||||
scheduler.kv_event_publisher = Mock()
|
||||
scheduler.finished_req_ids = set()
|
||||
scheduler.finished_req_ids_dict = None
|
||||
scheduler.grammar_compile_error_reqs = set()
|
||||
scheduler.vllm_config = Mock()
|
||||
scheduler.vllm_config.model_config.enable_return_routed_experts = False
|
||||
scheduler.enable_return_routed_experts = False
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import dataclasses
|
||||
from concurrent.futures import Future
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
@@ -39,7 +40,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
)
|
||||
from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
from vllm.v1.structured_output import StructuredOutputManager
|
||||
from vllm.v1.structured_output import StructuredOutputGrammar, StructuredOutputManager
|
||||
|
||||
from .utils import EOS_TOKEN_ID, create_requests, create_scheduler, mock_kv
|
||||
|
||||
@@ -3144,6 +3145,58 @@ def test_schedule_skip_tokenizer_init_structured_output_request():
|
||||
assert len(scheduler.skipped_waiting) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_grammar", [True, False])
|
||||
def test_grammar_compile_error_finishes_only_request(async_grammar: bool):
|
||||
scheduler = create_scheduler()
|
||||
manager = scheduler.structured_output_manager
|
||||
manager.backend = Mock()
|
||||
manager.backend.compile_grammar.side_effect = RuntimeError(
|
||||
"forced FSM compilation error"
|
||||
)
|
||||
manager._use_async_grammar_compilation = async_grammar
|
||||
|
||||
sampling_params = SamplingParams(
|
||||
max_tokens=16,
|
||||
structured_outputs=StructuredOutputsParams(json='{"type": "object"}'),
|
||||
)
|
||||
sampling_params.update_from_generation_config({}, EOS_TOKEN_ID)
|
||||
request = Request(
|
||||
request_id="grammar-error",
|
||||
prompt_token_ids=[0, 1],
|
||||
sampling_params=sampling_params,
|
||||
pooling_params=None,
|
||||
)
|
||||
|
||||
manager.grammar_init(request)
|
||||
assert request.structured_output_request is not None
|
||||
grammar_future = request.structured_output_request._grammar
|
||||
assert isinstance(grammar_future, Future)
|
||||
assert isinstance(grammar_future.exception(timeout=5), RuntimeError)
|
||||
|
||||
scheduler.add_request(request)
|
||||
scheduler_output = scheduler.schedule()
|
||||
assert not scheduler_output.num_scheduled_tokens
|
||||
|
||||
engine_core_outputs = scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
ModelRunnerOutput(req_ids=[], req_id_to_index={}),
|
||||
)
|
||||
|
||||
assert request.status == RequestStatus.FINISHED_ERROR
|
||||
assert request.request_id not in scheduler.requests
|
||||
output = engine_core_outputs[0].outputs[0]
|
||||
assert output.request_id == request.request_id
|
||||
assert output.finish_reason == FinishReason.ERROR
|
||||
assert output.stop_reason is None
|
||||
|
||||
healthy_request = create_requests(num_requests=1, req_ids=["healthy-request"])[0]
|
||||
scheduler.add_request(healthy_request)
|
||||
next_output = scheduler.schedule()
|
||||
assert [req.req_id for req in next_output.scheduled_new_reqs] == [
|
||||
healthy_request.request_id
|
||||
]
|
||||
|
||||
|
||||
def test_abort_request_when_structured_output_fsm_cannot_advance():
|
||||
scheduler = object.__new__(Scheduler)
|
||||
sampling_params = SamplingParams(ignore_eos=True, max_tokens=4)
|
||||
@@ -3157,7 +3210,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
|
||||
pooling_params=None,
|
||||
)
|
||||
request.structured_output_request = Mock()
|
||||
request.structured_output_request.grammar = Mock()
|
||||
request.structured_output_request.grammar = Mock(spec=StructuredOutputGrammar)
|
||||
request.structured_output_request.grammar.accept_tokens.return_value = False
|
||||
request.status = RequestStatus.RUNNING
|
||||
request.num_computed_tokens = request.num_tokens
|
||||
@@ -3178,6 +3231,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
|
||||
scheduler.kv_event_publisher = Mock()
|
||||
scheduler.finished_req_ids = set()
|
||||
scheduler.finished_req_ids_dict = None
|
||||
scheduler.grammar_compile_error_reqs = set()
|
||||
scheduler.vllm_config = Mock()
|
||||
scheduler.vllm_config.model_config.enable_return_routed_experts = False
|
||||
scheduler.enable_return_routed_experts = False
|
||||
|
||||
@@ -145,7 +145,7 @@ class SchedulerInterface(ABC):
|
||||
self,
|
||||
request_ids: str | Iterable[str] | None,
|
||||
finished_status: "RequestStatus",
|
||||
) -> list[tuple[str, int]]:
|
||||
) -> "list[Request]":
|
||||
"""Finish the requests in the scheduler's internal queue. If the request
|
||||
is not in the queue, this method will do nothing for that request.
|
||||
|
||||
@@ -159,8 +159,8 @@ class SchedulerInterface(ABC):
|
||||
finished_status: The finished status of the given requests.
|
||||
|
||||
Returns:
|
||||
Tuple of (req_id, client_index) for requests that were aborted. Will not
|
||||
include any that were already finished.
|
||||
List of requests that were aborted. Will not include any that were
|
||||
already finished.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@@ -60,7 +60,7 @@ from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput
|
||||
from vllm.v1.request import Request, RequestStatus, StreamingUpdate
|
||||
from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
|
||||
from vllm.v1.spec_decode.metrics import SpecDecodingStats
|
||||
from vllm.v1.structured_output import StructuredOutputManager
|
||||
from vllm.v1.structured_output import StructuredOutputGrammar, StructuredOutputManager
|
||||
from vllm.v1.utils import record_function_or_nullcontext
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -201,6 +201,10 @@ class Scheduler(SchedulerInterface):
|
||||
self.finished_recving_kv_req_ids: set[str] = set()
|
||||
self.failed_recving_kv_req_ids: set[str] = set()
|
||||
|
||||
# Grammar compilation failures to finish as per-request errors in
|
||||
# update_from_output.
|
||||
self.grammar_compile_error_reqs: set[str] = set()
|
||||
|
||||
# Encoder-related.
|
||||
# Calculate encoder cache size if applicable
|
||||
supports_mm_inputs = mm_registry.supports_multimodal_inputs(
|
||||
@@ -1716,7 +1720,7 @@ class Scheduler(SchedulerInterface):
|
||||
struct_output_request = request.structured_output_request
|
||||
assert struct_output_request is not None
|
||||
grammar = struct_output_request.grammar
|
||||
assert grammar is not None
|
||||
assert isinstance(grammar, StructuredOutputGrammar)
|
||||
# new_token_ids can be a mixed block of reasoning content, then
|
||||
# the reasoning end marker, then the start of the grammar content.
|
||||
# Trim the reasoning content so the grammar only sees grammar content.
|
||||
@@ -1846,10 +1850,16 @@ class Scheduler(SchedulerInterface):
|
||||
# This is a rare case and unlikely to impact performance.
|
||||
self.waiting.remove_requests(stopped_preempted_reqs)
|
||||
|
||||
error_req_ids = set(self.grammar_compile_error_reqs)
|
||||
self.grammar_compile_error_reqs.clear()
|
||||
if failed_kv_load_req_ids and not self.recompute_kv_load_failures:
|
||||
requests = [self.requests[req_id] for req_id in failed_kv_load_req_ids]
|
||||
self.finish_requests(failed_kv_load_req_ids, RequestStatus.FINISHED_ERROR)
|
||||
for request in requests:
|
||||
error_req_ids.update(failed_kv_load_req_ids)
|
||||
|
||||
if error_req_ids:
|
||||
error_reqs = self.finish_requests(
|
||||
error_req_ids, RequestStatus.FINISHED_ERROR
|
||||
)
|
||||
for request in error_reqs:
|
||||
outputs[request.client_index].append(
|
||||
EngineCoreOutput(
|
||||
request_id=request.request_id,
|
||||
@@ -2079,8 +2089,7 @@ class Scheduler(SchedulerInterface):
|
||||
# Filter out spec tokens which do not adhere to the grammar.
|
||||
if self.structured_output_manager.should_advance(request):
|
||||
metadata = request.structured_output_request
|
||||
assert metadata is not None and metadata.grammar is not None
|
||||
spec_token_ids = metadata.grammar.validate_tokens(spec_token_ids)
|
||||
spec_token_ids = metadata.grammar.validate_tokens(spec_token_ids) # type: ignore[union-attr]
|
||||
# Pad to original number of spec tokens.
|
||||
num_invalid_tokens = orig_num_spec_tokens - len(spec_token_ids)
|
||||
if num_invalid_tokens:
|
||||
@@ -2121,7 +2130,7 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
def finish_requests(
|
||||
self, request_ids: str | Iterable[str] | None, finished_status: RequestStatus
|
||||
) -> list[tuple[str, int]]:
|
||||
) -> list[Request]:
|
||||
"""Handles the finish signal from outside the scheduler.
|
||||
|
||||
For example, the API server can abort a request when the client
|
||||
@@ -2130,8 +2139,8 @@ class Scheduler(SchedulerInterface):
|
||||
If request_ids is None, all requests will be finished.
|
||||
|
||||
Returns:
|
||||
Tuple of (req_id, client_index) for requests that were aborted. Will not
|
||||
include any that were already finished.
|
||||
List of requests that were aborted. Will not include any that were
|
||||
already finished.
|
||||
"""
|
||||
assert RequestStatus.is_finished(finished_status)
|
||||
if isinstance(request_ids, str):
|
||||
@@ -2180,7 +2189,7 @@ class Scheduler(SchedulerInterface):
|
||||
request.status = finished_status
|
||||
self._free_request(request, delay_free_blocks=delay_free_blocks)
|
||||
|
||||
return [(r.request_id, r.client_index) for r in valid_requests]
|
||||
return valid_requests
|
||||
|
||||
def _free_request(
|
||||
self, request: Request, delay_free_blocks: bool = False
|
||||
@@ -2580,7 +2589,10 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
if request.status == RequestStatus.WAITING_FOR_STRUCTURED_OUTPUT_GRAMMAR:
|
||||
structured_output_req = request.structured_output_request
|
||||
if not (structured_output_req and structured_output_req.grammar):
|
||||
if not structured_output_req or structured_output_req.grammar is None:
|
||||
return False
|
||||
if isinstance(structured_output_req.grammar, Exception):
|
||||
self.grammar_compile_error_reqs.add(request.request_id)
|
||||
return False
|
||||
request.status = RequestStatus.WAITING
|
||||
return True
|
||||
|
||||
@@ -1830,13 +1830,13 @@ class EngineCoreProc(EngineCore):
|
||||
) -> None:
|
||||
self._send_finish_outputs_to_client(req_ids, client_index, FinishReason.ERROR)
|
||||
|
||||
def _send_abort_outputs(self, aborted_reqs: list[tuple[str, int]]) -> None:
|
||||
def _send_abort_outputs(self, aborted_reqs: list[Request]) -> None:
|
||||
# TODO(nick) this will be moved inside the scheduler
|
||||
if aborted_reqs:
|
||||
# Map client_index to list of request_ids that belong to that client.
|
||||
by_client = defaultdict[int, set[str]](set)
|
||||
for req_id, client_index in aborted_reqs:
|
||||
by_client[client_index].add(req_id)
|
||||
for request in aborted_reqs:
|
||||
by_client[request.client_index].add(request.request_id)
|
||||
for client_index, req_ids in by_client.items():
|
||||
self._send_abort_outputs_to_client(list(req_ids), client_index)
|
||||
|
||||
|
||||
@@ -164,24 +164,33 @@ class StructuredOutputManager:
|
||||
else:
|
||||
raise ValueError(f"Unsupported structured output backend: {backend}")
|
||||
|
||||
grammar: Future[StructuredOutputGrammar] | StructuredOutputGrammar
|
||||
if self._use_async_grammar_compilation:
|
||||
grammar = self.executor.submit(self._create_grammar, request)
|
||||
else:
|
||||
grammar = self._create_grammar(request) # type: ignore[assignment]
|
||||
request.structured_output_request.grammar = grammar # type: ignore[assignment]
|
||||
try:
|
||||
grammar = self._create_grammar(request)
|
||||
except Exception as e:
|
||||
grammar = Future()
|
||||
grammar.set_exception(e)
|
||||
request.structured_output_request.grammar = grammar
|
||||
|
||||
def _create_grammar(self, request: "Request") -> StructuredOutputGrammar:
|
||||
key = request.structured_output_request.structured_output_key # type: ignore[union-attr]
|
||||
|
||||
struct_request = request.structured_output_request
|
||||
assert struct_request is not None
|
||||
# Note that the request was validated in the engine core client,
|
||||
# so at this point we know it is a supported type of request.
|
||||
#
|
||||
# TODO: we still need to handle xgrammar compilation failures,
|
||||
# though it should be unlikely as we test that up front as well.
|
||||
request_type, grammar_spec = key
|
||||
|
||||
assert self.backend is not None
|
||||
return self.backend.compile_grammar(request_type, grammar_spec)
|
||||
# so at this point we know it is a supported type of request. Grammar
|
||||
# compilation may still fail; the Future carries that error to the
|
||||
# scheduler so it can fail only this request.
|
||||
try:
|
||||
request_type, grammar_spec = struct_request.structured_output_key
|
||||
assert self.backend is not None
|
||||
return self.backend.compile_grammar(request_type, grammar_spec)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to compile grammar for request %s", request.request_id
|
||||
)
|
||||
raise
|
||||
|
||||
def _fill_bitmasks(
|
||||
self, batch: Iterable[tuple[StructuredOutputGrammar, int, bool]]
|
||||
@@ -244,8 +253,9 @@ class StructuredOutputManager:
|
||||
structured_output_request = request.structured_output_request
|
||||
if TYPE_CHECKING:
|
||||
assert structured_output_request is not None
|
||||
assert structured_output_request.grammar is not None
|
||||
grammar = structured_output_request.grammar
|
||||
if TYPE_CHECKING:
|
||||
assert isinstance(grammar, StructuredOutputGrammar)
|
||||
|
||||
apply_bitmask = self.should_fill_bitmask(request)
|
||||
batch.append((grammar, cumulative_index, apply_bitmask))
|
||||
@@ -268,8 +278,9 @@ class StructuredOutputManager:
|
||||
|
||||
if TYPE_CHECKING:
|
||||
assert structured_output_request is not None
|
||||
assert structured_output_request.grammar is not None
|
||||
grammar = structured_output_request.grammar
|
||||
if TYPE_CHECKING:
|
||||
assert isinstance(grammar, StructuredOutputGrammar)
|
||||
apply_bitmask = self.should_fill_bitmask(request)
|
||||
|
||||
reasoner = self._get_reasoner(request)
|
||||
|
||||
@@ -21,7 +21,9 @@ if TYPE_CHECKING:
|
||||
@dataclasses.dataclass
|
||||
class StructuredOutputRequest:
|
||||
params: StructuredOutputsParams
|
||||
_grammar: Future[StructuredOutputGrammar] | StructuredOutputGrammar | None = None
|
||||
_grammar: (
|
||||
Future[StructuredOutputGrammar] | StructuredOutputGrammar | Exception | None
|
||||
) = None
|
||||
reasoning_ended: bool | None = None
|
||||
# Absolute index into the request's all_token_ids of the last reasoning
|
||||
# token (the reasoning-end marker). Tokens at or before this index are
|
||||
@@ -52,6 +54,8 @@ class StructuredOutputRequest:
|
||||
self._grammar = self._grammar.result(timeout=0.0001)
|
||||
except TimeoutError:
|
||||
return False
|
||||
except Exception as e:
|
||||
self._grammar = e
|
||||
return True
|
||||
|
||||
@property
|
||||
@@ -59,11 +63,10 @@ class StructuredOutputRequest:
|
||||
return self._check_grammar_completion()
|
||||
|
||||
@property
|
||||
def grammar(self) -> StructuredOutputGrammar | None:
|
||||
completed = self._check_grammar_completion()
|
||||
return (
|
||||
cast(StructuredOutputGrammar | None, self._grammar) if completed else None
|
||||
)
|
||||
def grammar(self) -> StructuredOutputGrammar | Exception | None:
|
||||
if not self._check_grammar_completion():
|
||||
return None
|
||||
return cast(StructuredOutputGrammar | Exception | None, self._grammar)
|
||||
|
||||
@grammar.setter
|
||||
def grammar(
|
||||
|
||||
Reference in New Issue
Block a user