diff --git a/tests/v1/core/test_async_scheduler.py b/tests/v1/core/test_async_scheduler.py index cd3efa8ee6..e34a0da54d 100644 --- a/tests/v1/core/test_async_scheduler.py +++ b/tests/v1/core/test_async_scheduler.py @@ -9,6 +9,7 @@ from vllm.v1.core.sched.async_scheduler import AsyncScheduler from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput from vllm.v1.outputs import ModelRunnerOutput from vllm.v1.request import RequestStatus +from vllm.v1.structured_output import StructuredOutputGrammar from vllm.v1.utils import ConstantList from .utils import create_requests, create_scheduler @@ -262,7 +263,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance(): scheduler = object.__new__(AsyncScheduler) request = create_requests(num_requests=1, num_tokens=1)[0] request.structured_output_request = Mock() - request.structured_output_request.grammar = Mock() + request.structured_output_request.grammar = Mock(spec=StructuredOutputGrammar) request.structured_output_request.grammar.accept_tokens.return_value = False request.status = RequestStatus.RUNNING request.num_computed_tokens = request.num_tokens @@ -284,6 +285,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance(): scheduler.kv_event_publisher = Mock() scheduler.finished_req_ids = set() scheduler.finished_req_ids_dict = None + scheduler.grammar_compile_error_reqs = set() scheduler.vllm_config = Mock() scheduler.vllm_config.model_config.enable_return_routed_experts = False scheduler.enable_return_routed_experts = False diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index b782e34b01..66eafb39af 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import dataclasses +from concurrent.futures import Future from unittest.mock import Mock import pytest @@ -39,7 +40,7 @@ from vllm.v1.kv_cache_interface import ( ) from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput from vllm.v1.request import Request, RequestStatus -from vllm.v1.structured_output import StructuredOutputManager +from vllm.v1.structured_output import StructuredOutputGrammar, StructuredOutputManager from .utils import EOS_TOKEN_ID, create_requests, create_scheduler, mock_kv @@ -3144,6 +3145,58 @@ def test_schedule_skip_tokenizer_init_structured_output_request(): assert len(scheduler.skipped_waiting) == 1 +@pytest.mark.parametrize("async_grammar", [True, False]) +def test_grammar_compile_error_finishes_only_request(async_grammar: bool): + scheduler = create_scheduler() + manager = scheduler.structured_output_manager + manager.backend = Mock() + manager.backend.compile_grammar.side_effect = RuntimeError( + "forced FSM compilation error" + ) + manager._use_async_grammar_compilation = async_grammar + + sampling_params = SamplingParams( + max_tokens=16, + structured_outputs=StructuredOutputsParams(json='{"type": "object"}'), + ) + sampling_params.update_from_generation_config({}, EOS_TOKEN_ID) + request = Request( + request_id="grammar-error", + prompt_token_ids=[0, 1], + sampling_params=sampling_params, + pooling_params=None, + ) + + manager.grammar_init(request) + assert request.structured_output_request is not None + grammar_future = request.structured_output_request._grammar + assert isinstance(grammar_future, Future) + assert isinstance(grammar_future.exception(timeout=5), RuntimeError) + + scheduler.add_request(request) + scheduler_output = scheduler.schedule() + assert not scheduler_output.num_scheduled_tokens + + engine_core_outputs = scheduler.update_from_output( + scheduler_output, + ModelRunnerOutput(req_ids=[], req_id_to_index={}), + ) + + assert request.status == RequestStatus.FINISHED_ERROR + assert request.request_id not in scheduler.requests + output = engine_core_outputs[0].outputs[0] + assert output.request_id == request.request_id + assert output.finish_reason == FinishReason.ERROR + assert output.stop_reason is None + + healthy_request = create_requests(num_requests=1, req_ids=["healthy-request"])[0] + scheduler.add_request(healthy_request) + next_output = scheduler.schedule() + assert [req.req_id for req in next_output.scheduled_new_reqs] == [ + healthy_request.request_id + ] + + def test_abort_request_when_structured_output_fsm_cannot_advance(): scheduler = object.__new__(Scheduler) sampling_params = SamplingParams(ignore_eos=True, max_tokens=4) @@ -3157,7 +3210,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance(): pooling_params=None, ) request.structured_output_request = Mock() - request.structured_output_request.grammar = Mock() + request.structured_output_request.grammar = Mock(spec=StructuredOutputGrammar) request.structured_output_request.grammar.accept_tokens.return_value = False request.status = RequestStatus.RUNNING request.num_computed_tokens = request.num_tokens @@ -3178,6 +3231,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance(): scheduler.kv_event_publisher = Mock() scheduler.finished_req_ids = set() scheduler.finished_req_ids_dict = None + scheduler.grammar_compile_error_reqs = set() scheduler.vllm_config = Mock() scheduler.vllm_config.model_config.enable_return_routed_experts = False scheduler.enable_return_routed_experts = False diff --git a/vllm/v1/core/sched/interface.py b/vllm/v1/core/sched/interface.py index 98866154c9..4f13aa4b72 100644 --- a/vllm/v1/core/sched/interface.py +++ b/vllm/v1/core/sched/interface.py @@ -145,7 +145,7 @@ class SchedulerInterface(ABC): self, request_ids: str | Iterable[str] | None, finished_status: "RequestStatus", - ) -> list[tuple[str, int]]: + ) -> "list[Request]": """Finish the requests in the scheduler's internal queue. If the request is not in the queue, this method will do nothing for that request. @@ -159,8 +159,8 @@ class SchedulerInterface(ABC): finished_status: The finished status of the given requests. Returns: - Tuple of (req_id, client_index) for requests that were aborted. Will not - include any that were already finished. + List of requests that were aborted. Will not include any that were + already finished. """ raise NotImplementedError diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index fcd60804a3..aca76d9cca 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -60,7 +60,7 @@ from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput from vllm.v1.request import Request, RequestStatus, StreamingUpdate from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup from vllm.v1.spec_decode.metrics import SpecDecodingStats -from vllm.v1.structured_output import StructuredOutputManager +from vllm.v1.structured_output import StructuredOutputGrammar, StructuredOutputManager from vllm.v1.utils import record_function_or_nullcontext logger = init_logger(__name__) @@ -201,6 +201,10 @@ class Scheduler(SchedulerInterface): self.finished_recving_kv_req_ids: set[str] = set() self.failed_recving_kv_req_ids: set[str] = set() + # Grammar compilation failures to finish as per-request errors in + # update_from_output. + self.grammar_compile_error_reqs: set[str] = set() + # Encoder-related. # Calculate encoder cache size if applicable supports_mm_inputs = mm_registry.supports_multimodal_inputs( @@ -1716,7 +1720,7 @@ class Scheduler(SchedulerInterface): struct_output_request = request.structured_output_request assert struct_output_request is not None grammar = struct_output_request.grammar - assert grammar is not None + assert isinstance(grammar, StructuredOutputGrammar) # new_token_ids can be a mixed block of reasoning content, then # the reasoning end marker, then the start of the grammar content. # Trim the reasoning content so the grammar only sees grammar content. @@ -1846,10 +1850,16 @@ class Scheduler(SchedulerInterface): # This is a rare case and unlikely to impact performance. self.waiting.remove_requests(stopped_preempted_reqs) + error_req_ids = set(self.grammar_compile_error_reqs) + self.grammar_compile_error_reqs.clear() if failed_kv_load_req_ids and not self.recompute_kv_load_failures: - requests = [self.requests[req_id] for req_id in failed_kv_load_req_ids] - self.finish_requests(failed_kv_load_req_ids, RequestStatus.FINISHED_ERROR) - for request in requests: + error_req_ids.update(failed_kv_load_req_ids) + + if error_req_ids: + error_reqs = self.finish_requests( + error_req_ids, RequestStatus.FINISHED_ERROR + ) + for request in error_reqs: outputs[request.client_index].append( EngineCoreOutput( request_id=request.request_id, @@ -2079,8 +2089,7 @@ class Scheduler(SchedulerInterface): # Filter out spec tokens which do not adhere to the grammar. if self.structured_output_manager.should_advance(request): metadata = request.structured_output_request - assert metadata is not None and metadata.grammar is not None - spec_token_ids = metadata.grammar.validate_tokens(spec_token_ids) + spec_token_ids = metadata.grammar.validate_tokens(spec_token_ids) # type: ignore[union-attr] # Pad to original number of spec tokens. num_invalid_tokens = orig_num_spec_tokens - len(spec_token_ids) if num_invalid_tokens: @@ -2121,7 +2130,7 @@ class Scheduler(SchedulerInterface): def finish_requests( self, request_ids: str | Iterable[str] | None, finished_status: RequestStatus - ) -> list[tuple[str, int]]: + ) -> list[Request]: """Handles the finish signal from outside the scheduler. For example, the API server can abort a request when the client @@ -2130,8 +2139,8 @@ class Scheduler(SchedulerInterface): If request_ids is None, all requests will be finished. Returns: - Tuple of (req_id, client_index) for requests that were aborted. Will not - include any that were already finished. + List of requests that were aborted. Will not include any that were + already finished. """ assert RequestStatus.is_finished(finished_status) if isinstance(request_ids, str): @@ -2180,7 +2189,7 @@ class Scheduler(SchedulerInterface): request.status = finished_status self._free_request(request, delay_free_blocks=delay_free_blocks) - return [(r.request_id, r.client_index) for r in valid_requests] + return valid_requests def _free_request( self, request: Request, delay_free_blocks: bool = False @@ -2580,7 +2589,10 @@ class Scheduler(SchedulerInterface): if request.status == RequestStatus.WAITING_FOR_STRUCTURED_OUTPUT_GRAMMAR: structured_output_req = request.structured_output_request - if not (structured_output_req and structured_output_req.grammar): + if not structured_output_req or structured_output_req.grammar is None: + return False + if isinstance(structured_output_req.grammar, Exception): + self.grammar_compile_error_reqs.add(request.request_id) return False request.status = RequestStatus.WAITING return True diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index 383853807d..476f53d461 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -1830,13 +1830,13 @@ class EngineCoreProc(EngineCore): ) -> None: self._send_finish_outputs_to_client(req_ids, client_index, FinishReason.ERROR) - def _send_abort_outputs(self, aborted_reqs: list[tuple[str, int]]) -> None: + def _send_abort_outputs(self, aborted_reqs: list[Request]) -> None: # TODO(nick) this will be moved inside the scheduler if aborted_reqs: # Map client_index to list of request_ids that belong to that client. by_client = defaultdict[int, set[str]](set) - for req_id, client_index in aborted_reqs: - by_client[client_index].add(req_id) + for request in aborted_reqs: + by_client[request.client_index].add(request.request_id) for client_index, req_ids in by_client.items(): self._send_abort_outputs_to_client(list(req_ids), client_index) diff --git a/vllm/v1/structured_output/__init__.py b/vllm/v1/structured_output/__init__.py index 34f775257b..939a387ee9 100644 --- a/vllm/v1/structured_output/__init__.py +++ b/vllm/v1/structured_output/__init__.py @@ -164,24 +164,33 @@ class StructuredOutputManager: else: raise ValueError(f"Unsupported structured output backend: {backend}") + grammar: Future[StructuredOutputGrammar] | StructuredOutputGrammar if self._use_async_grammar_compilation: grammar = self.executor.submit(self._create_grammar, request) else: - grammar = self._create_grammar(request) # type: ignore[assignment] - request.structured_output_request.grammar = grammar # type: ignore[assignment] + try: + grammar = self._create_grammar(request) + except Exception as e: + grammar = Future() + grammar.set_exception(e) + request.structured_output_request.grammar = grammar def _create_grammar(self, request: "Request") -> StructuredOutputGrammar: - key = request.structured_output_request.structured_output_key # type: ignore[union-attr] - + struct_request = request.structured_output_request + assert struct_request is not None # Note that the request was validated in the engine core client, - # so at this point we know it is a supported type of request. - # - # TODO: we still need to handle xgrammar compilation failures, - # though it should be unlikely as we test that up front as well. - request_type, grammar_spec = key - - assert self.backend is not None - return self.backend.compile_grammar(request_type, grammar_spec) + # so at this point we know it is a supported type of request. Grammar + # compilation may still fail; the Future carries that error to the + # scheduler so it can fail only this request. + try: + request_type, grammar_spec = struct_request.structured_output_key + assert self.backend is not None + return self.backend.compile_grammar(request_type, grammar_spec) + except Exception: + logger.exception( + "Failed to compile grammar for request %s", request.request_id + ) + raise def _fill_bitmasks( self, batch: Iterable[tuple[StructuredOutputGrammar, int, bool]] @@ -244,8 +253,9 @@ class StructuredOutputManager: structured_output_request = request.structured_output_request if TYPE_CHECKING: assert structured_output_request is not None - assert structured_output_request.grammar is not None grammar = structured_output_request.grammar + if TYPE_CHECKING: + assert isinstance(grammar, StructuredOutputGrammar) apply_bitmask = self.should_fill_bitmask(request) batch.append((grammar, cumulative_index, apply_bitmask)) @@ -268,8 +278,9 @@ class StructuredOutputManager: if TYPE_CHECKING: assert structured_output_request is not None - assert structured_output_request.grammar is not None grammar = structured_output_request.grammar + if TYPE_CHECKING: + assert isinstance(grammar, StructuredOutputGrammar) apply_bitmask = self.should_fill_bitmask(request) reasoner = self._get_reasoner(request) diff --git a/vllm/v1/structured_output/request.py b/vllm/v1/structured_output/request.py index 6e0a643745..b0f4f3c3a8 100644 --- a/vllm/v1/structured_output/request.py +++ b/vllm/v1/structured_output/request.py @@ -21,7 +21,9 @@ if TYPE_CHECKING: @dataclasses.dataclass class StructuredOutputRequest: params: StructuredOutputsParams - _grammar: Future[StructuredOutputGrammar] | StructuredOutputGrammar | None = None + _grammar: ( + Future[StructuredOutputGrammar] | StructuredOutputGrammar | Exception | None + ) = None reasoning_ended: bool | None = None # Absolute index into the request's all_token_ids of the last reasoning # token (the reasoning-end marker). Tokens at or before this index are @@ -52,6 +54,8 @@ class StructuredOutputRequest: self._grammar = self._grammar.result(timeout=0.0001) except TimeoutError: return False + except Exception as e: + self._grammar = e return True @property @@ -59,11 +63,10 @@ class StructuredOutputRequest: return self._check_grammar_completion() @property - def grammar(self) -> StructuredOutputGrammar | None: - completed = self._check_grammar_completion() - return ( - cast(StructuredOutputGrammar | None, self._grammar) if completed else None - ) + def grammar(self) -> StructuredOutputGrammar | Exception | None: + if not self._check_grammar_completion(): + return None + return cast(StructuredOutputGrammar | Exception | None, self._grammar) @grammar.setter def grammar(