[Core] Make parallel sampling (n>1) reqs admission atomic (#53936)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-09-18 18:08:44 +02:00
committed by GitHub
parent 4991f97669
commit a7156060c1
2 changed files with 107 additions and 7 deletions
+90
View File
@@ -15,6 +15,7 @@ These tests cover:
import argparse
import asyncio
import multiprocessing
from collections.abc import Awaitable, Callable
from http import HTTPStatus
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
@@ -35,6 +36,7 @@ from vllm.exceptions import (
from vllm.pooling_params import PoolingParams
from vllm.sampling_params import SamplingParams
from vllm.utils.argparse_utils import human_readable_int
from vllm.v1.engine import EngineCoreRequest
from vllm.v1.engine.admission_control import SharedAdmissionStats
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.engine.output_processor import OutputProcessor
@@ -78,6 +80,45 @@ def _make_output_processor(**request_states) -> OutputProcessor:
return op
def _make_engine_request(request_id: str, n: int) -> EngineCoreRequest:
return EngineCoreRequest(
request_id=request_id,
prompt_token_ids=[1],
mm_features=None,
sampling_params=SamplingParams(n=n),
pooling_params=None,
arrival_time=0,
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
)
def _make_request_test_llm(
max_num_queued_reqs: int,
add_request_async: Callable[[EngineCoreRequest], Awaitable[None]],
) -> AsyncLLM:
llm = _make_async_llm(max_num_queued_reqs=max_num_queued_reqs)
llm.output_processor = OutputProcessor(None, log_stats=False)
llm.engine_core = SimpleNamespace(
resources=SimpleNamespace(engine_dead=False),
add_request_async=AsyncMock(side_effect=add_request_async),
abort_requests_async=AsyncMock(),
shutdown=MagicMock(),
)
llm.vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(kv_sharing_fast_prefill=False)
)
llm.output_handler = None
llm.log_requests = False
llm._run_output_handler = MagicMock()
llm.input_processor = MagicMock()
llm.input_processor.assign_request_id.side_effect = lambda request: setattr(
request, "external_req_id", request.request_id
)
return llm
def _make_scheduler_config(**kwargs) -> SchedulerConfig:
return SchedulerConfig(
runner_type="generate",
@@ -340,6 +381,55 @@ async def test_concurrent_single_request_admission_respects_limit():
assert len(request_states) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("first_n", [1, 3])
async def test_parallel_admission_is_all_or_nothing(first_n: int):
"""An n>1 request reserves either all its capacity or none of it."""
async def add_request_async(_request):
await asyncio.sleep(0)
llm = _make_request_test_llm(3, add_request_async)
second_n = 3 if first_n == 1 else 1
first = _make_engine_request("first", first_n)
second = _make_engine_request("second", second_n)
results = await asyncio.gather(
llm.add_request("first", first, first.params),
llm.add_request("second", second, second.params),
return_exceptions=True,
)
assert not isinstance(results[0], Exception)
assert isinstance(results[1], QueueOverflowError)
assert llm.output_processor.get_num_unfinished_requests() == first_n
assert llm.engine_core.add_request_async.await_count == first_n
@pytest.mark.asyncio
async def test_parallel_admission_cancellation_cleans_up_all_children():
"""Cancellation during core submission must release every reserved slot."""
first_submission_started = asyncio.Event()
async def add_request_async(_request):
first_submission_started.set()
await asyncio.Event().wait()
llm = _make_request_test_llm(3, add_request_async)
request = _make_engine_request("parallel", 3)
output = llm.generate(request, request.params, request.request_id)
generate_task = asyncio.create_task(anext(output))
await first_submission_started.wait()
generate_task.cancel()
with pytest.raises(asyncio.CancelledError):
await generate_task
assert llm.engine_core.add_request_async.await_count == 1
assert llm.output_processor.get_num_unfinished_requests() == 0
aborted_ids = llm.engine_core.abort_requests_async.await_args.args[0]
assert set(aborted_ids) == {"0_parallel", "1_parallel", "2_parallel"}
# ---------------------------------------------------------------------------
# create_error_response integration
# ---------------------------------------------------------------------------
+17 -7
View File
@@ -401,10 +401,6 @@ class AsyncLLM(EngineClient):
"prompt logprobs"
)
if isinstance(params, SamplingParams) and params.n > 1:
# TODO (NickLucche) Batch check admission check for all n requests
self.check_admission(params.n, request_id)
if isinstance(prompt, AsyncGenerator):
if reasoning_ended is not None or reasoning_parser_kwargs is not None:
raise NotImplementedError
@@ -500,14 +496,28 @@ class AsyncLLM(EngineClient):
# Fan out child requests (for n>1).
parent_request = ParentRequest(request)
child_requests: list[EngineCoreRequest] = []
for idx in range(parent_params.n):
request_id, child_params = parent_request.get_child_info(idx)
child_request = request if idx == parent_params.n - 1 else copy(request)
child_request.request_id = request_id
child_request.sampling_params = child_params
await self._add_request(
child_request, prompt_text, parent_request, idx, queue
)
child_requests.append(child_request)
self.check_admission(parent_params.n, parent_request.request_id)
try:
for idx, child_request in enumerate(child_requests):
self.output_processor.add_request(
child_request, prompt_text, parent_request, idx, queue
)
for child_request in child_requests:
await self.engine_core.add_request_async(child_request)
if self.log_requests:
logger.info("Added request %s.", child_request.request_id)
except BaseException:
await self.abort(parent_request.request_id, internal=True)
raise
return queue
async def _add_request(