diff --git a/tests/v1/engine/test_admission_control.py b/tests/v1/engine/test_admission_control.py index 219850f5d4..15be166c8c 100644 --- a/tests/v1/engine/test_admission_control.py +++ b/tests/v1/engine/test_admission_control.py @@ -15,6 +15,7 @@ These tests cover: import argparse import asyncio import multiprocessing +from collections.abc import Awaitable, Callable from http import HTTPStatus from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock @@ -35,6 +36,7 @@ from vllm.exceptions import ( from vllm.pooling_params import PoolingParams from vllm.sampling_params import SamplingParams from vllm.utils.argparse_utils import human_readable_int +from vllm.v1.engine import EngineCoreRequest from vllm.v1.engine.admission_control import SharedAdmissionStats from vllm.v1.engine.async_llm import AsyncLLM from vllm.v1.engine.output_processor import OutputProcessor @@ -78,6 +80,45 @@ def _make_output_processor(**request_states) -> OutputProcessor: return op +def _make_engine_request(request_id: str, n: int) -> EngineCoreRequest: + return EngineCoreRequest( + request_id=request_id, + prompt_token_ids=[1], + mm_features=None, + sampling_params=SamplingParams(n=n), + pooling_params=None, + arrival_time=0, + lora_request=None, + cache_salt=None, + data_parallel_rank=None, + ) + + +def _make_request_test_llm( + max_num_queued_reqs: int, + add_request_async: Callable[[EngineCoreRequest], Awaitable[None]], +) -> AsyncLLM: + llm = _make_async_llm(max_num_queued_reqs=max_num_queued_reqs) + llm.output_processor = OutputProcessor(None, log_stats=False) + llm.engine_core = SimpleNamespace( + resources=SimpleNamespace(engine_dead=False), + add_request_async=AsyncMock(side_effect=add_request_async), + abort_requests_async=AsyncMock(), + shutdown=MagicMock(), + ) + llm.vllm_config = SimpleNamespace( + cache_config=SimpleNamespace(kv_sharing_fast_prefill=False) + ) + llm.output_handler = None + llm.log_requests = False + llm._run_output_handler = MagicMock() + llm.input_processor = MagicMock() + llm.input_processor.assign_request_id.side_effect = lambda request: setattr( + request, "external_req_id", request.request_id + ) + return llm + + def _make_scheduler_config(**kwargs) -> SchedulerConfig: return SchedulerConfig( runner_type="generate", @@ -340,6 +381,55 @@ async def test_concurrent_single_request_admission_respects_limit(): assert len(request_states) == 1 +@pytest.mark.asyncio +@pytest.mark.parametrize("first_n", [1, 3]) +async def test_parallel_admission_is_all_or_nothing(first_n: int): + """An n>1 request reserves either all its capacity or none of it.""" + + async def add_request_async(_request): + await asyncio.sleep(0) + + llm = _make_request_test_llm(3, add_request_async) + second_n = 3 if first_n == 1 else 1 + first = _make_engine_request("first", first_n) + second = _make_engine_request("second", second_n) + results = await asyncio.gather( + llm.add_request("first", first, first.params), + llm.add_request("second", second, second.params), + return_exceptions=True, + ) + + assert not isinstance(results[0], Exception) + assert isinstance(results[1], QueueOverflowError) + assert llm.output_processor.get_num_unfinished_requests() == first_n + assert llm.engine_core.add_request_async.await_count == first_n + + +@pytest.mark.asyncio +async def test_parallel_admission_cancellation_cleans_up_all_children(): + """Cancellation during core submission must release every reserved slot.""" + first_submission_started = asyncio.Event() + + async def add_request_async(_request): + first_submission_started.set() + await asyncio.Event().wait() + + llm = _make_request_test_llm(3, add_request_async) + request = _make_engine_request("parallel", 3) + output = llm.generate(request, request.params, request.request_id) + generate_task = asyncio.create_task(anext(output)) + + await first_submission_started.wait() + generate_task.cancel() + with pytest.raises(asyncio.CancelledError): + await generate_task + + assert llm.engine_core.add_request_async.await_count == 1 + assert llm.output_processor.get_num_unfinished_requests() == 0 + aborted_ids = llm.engine_core.abort_requests_async.await_args.args[0] + assert set(aborted_ids) == {"0_parallel", "1_parallel", "2_parallel"} + + # --------------------------------------------------------------------------- # create_error_response integration # --------------------------------------------------------------------------- diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index ad9d9bf6c7..7a57d0ff18 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -401,10 +401,6 @@ class AsyncLLM(EngineClient): "prompt logprobs" ) - if isinstance(params, SamplingParams) and params.n > 1: - # TODO (NickLucche) Batch check admission check for all n requests - self.check_admission(params.n, request_id) - if isinstance(prompt, AsyncGenerator): if reasoning_ended is not None or reasoning_parser_kwargs is not None: raise NotImplementedError @@ -500,14 +496,28 @@ class AsyncLLM(EngineClient): # Fan out child requests (for n>1). parent_request = ParentRequest(request) + child_requests: list[EngineCoreRequest] = [] for idx in range(parent_params.n): request_id, child_params = parent_request.get_child_info(idx) child_request = request if idx == parent_params.n - 1 else copy(request) child_request.request_id = request_id child_request.sampling_params = child_params - await self._add_request( - child_request, prompt_text, parent_request, idx, queue - ) + child_requests.append(child_request) + + self.check_admission(parent_params.n, parent_request.request_id) + try: + for idx, child_request in enumerate(child_requests): + self.output_processor.add_request( + child_request, prompt_text, parent_request, idx, queue + ) + + for child_request in child_requests: + await self.engine_core.add_request_async(child_request) + if self.log_requests: + logger.info("Added request %s.", child_request.request_id) + except BaseException: + await self.abort(parent_request.request_id, internal=True) + raise return queue async def _add_request(