[Core] Make parallel sampling (n>1) reqs admission atomic (#53936)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -15,6 +15,7 @@ These tests cover:
|
||||
import argparse
|
||||
import asyncio
|
||||
import multiprocessing
|
||||
from collections.abc import Awaitable, Callable
|
||||
from http import HTTPStatus
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
@@ -35,6 +36,7 @@ from vllm.exceptions import (
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.argparse_utils import human_readable_int
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.engine.admission_control import SharedAdmissionStats
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.engine.output_processor import OutputProcessor
|
||||
@@ -78,6 +80,45 @@ def _make_output_processor(**request_states) -> OutputProcessor:
|
||||
return op
|
||||
|
||||
|
||||
def _make_engine_request(request_id: str, n: int) -> EngineCoreRequest:
|
||||
return EngineCoreRequest(
|
||||
request_id=request_id,
|
||||
prompt_token_ids=[1],
|
||||
mm_features=None,
|
||||
sampling_params=SamplingParams(n=n),
|
||||
pooling_params=None,
|
||||
arrival_time=0,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=None,
|
||||
)
|
||||
|
||||
|
||||
def _make_request_test_llm(
|
||||
max_num_queued_reqs: int,
|
||||
add_request_async: Callable[[EngineCoreRequest], Awaitable[None]],
|
||||
) -> AsyncLLM:
|
||||
llm = _make_async_llm(max_num_queued_reqs=max_num_queued_reqs)
|
||||
llm.output_processor = OutputProcessor(None, log_stats=False)
|
||||
llm.engine_core = SimpleNamespace(
|
||||
resources=SimpleNamespace(engine_dead=False),
|
||||
add_request_async=AsyncMock(side_effect=add_request_async),
|
||||
abort_requests_async=AsyncMock(),
|
||||
shutdown=MagicMock(),
|
||||
)
|
||||
llm.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(kv_sharing_fast_prefill=False)
|
||||
)
|
||||
llm.output_handler = None
|
||||
llm.log_requests = False
|
||||
llm._run_output_handler = MagicMock()
|
||||
llm.input_processor = MagicMock()
|
||||
llm.input_processor.assign_request_id.side_effect = lambda request: setattr(
|
||||
request, "external_req_id", request.request_id
|
||||
)
|
||||
return llm
|
||||
|
||||
|
||||
def _make_scheduler_config(**kwargs) -> SchedulerConfig:
|
||||
return SchedulerConfig(
|
||||
runner_type="generate",
|
||||
@@ -340,6 +381,55 @@ async def test_concurrent_single_request_admission_respects_limit():
|
||||
assert len(request_states) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("first_n", [1, 3])
|
||||
async def test_parallel_admission_is_all_or_nothing(first_n: int):
|
||||
"""An n>1 request reserves either all its capacity or none of it."""
|
||||
|
||||
async def add_request_async(_request):
|
||||
await asyncio.sleep(0)
|
||||
|
||||
llm = _make_request_test_llm(3, add_request_async)
|
||||
second_n = 3 if first_n == 1 else 1
|
||||
first = _make_engine_request("first", first_n)
|
||||
second = _make_engine_request("second", second_n)
|
||||
results = await asyncio.gather(
|
||||
llm.add_request("first", first, first.params),
|
||||
llm.add_request("second", second, second.params),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
assert not isinstance(results[0], Exception)
|
||||
assert isinstance(results[1], QueueOverflowError)
|
||||
assert llm.output_processor.get_num_unfinished_requests() == first_n
|
||||
assert llm.engine_core.add_request_async.await_count == first_n
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parallel_admission_cancellation_cleans_up_all_children():
|
||||
"""Cancellation during core submission must release every reserved slot."""
|
||||
first_submission_started = asyncio.Event()
|
||||
|
||||
async def add_request_async(_request):
|
||||
first_submission_started.set()
|
||||
await asyncio.Event().wait()
|
||||
|
||||
llm = _make_request_test_llm(3, add_request_async)
|
||||
request = _make_engine_request("parallel", 3)
|
||||
output = llm.generate(request, request.params, request.request_id)
|
||||
generate_task = asyncio.create_task(anext(output))
|
||||
|
||||
await first_submission_started.wait()
|
||||
generate_task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await generate_task
|
||||
|
||||
assert llm.engine_core.add_request_async.await_count == 1
|
||||
assert llm.output_processor.get_num_unfinished_requests() == 0
|
||||
aborted_ids = llm.engine_core.abort_requests_async.await_args.args[0]
|
||||
assert set(aborted_ids) == {"0_parallel", "1_parallel", "2_parallel"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# create_error_response integration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -401,10 +401,6 @@ class AsyncLLM(EngineClient):
|
||||
"prompt logprobs"
|
||||
)
|
||||
|
||||
if isinstance(params, SamplingParams) and params.n > 1:
|
||||
# TODO (NickLucche) Batch check admission check for all n requests
|
||||
self.check_admission(params.n, request_id)
|
||||
|
||||
if isinstance(prompt, AsyncGenerator):
|
||||
if reasoning_ended is not None or reasoning_parser_kwargs is not None:
|
||||
raise NotImplementedError
|
||||
@@ -500,14 +496,28 @@ class AsyncLLM(EngineClient):
|
||||
|
||||
# Fan out child requests (for n>1).
|
||||
parent_request = ParentRequest(request)
|
||||
child_requests: list[EngineCoreRequest] = []
|
||||
for idx in range(parent_params.n):
|
||||
request_id, child_params = parent_request.get_child_info(idx)
|
||||
child_request = request if idx == parent_params.n - 1 else copy(request)
|
||||
child_request.request_id = request_id
|
||||
child_request.sampling_params = child_params
|
||||
await self._add_request(
|
||||
child_request, prompt_text, parent_request, idx, queue
|
||||
)
|
||||
child_requests.append(child_request)
|
||||
|
||||
self.check_admission(parent_params.n, parent_request.request_id)
|
||||
try:
|
||||
for idx, child_request in enumerate(child_requests):
|
||||
self.output_processor.add_request(
|
||||
child_request, prompt_text, parent_request, idx, queue
|
||||
)
|
||||
|
||||
for child_request in child_requests:
|
||||
await self.engine_core.add_request_async(child_request)
|
||||
if self.log_requests:
|
||||
logger.info("Added request %s.", child_request.request_id)
|
||||
except BaseException:
|
||||
await self.abort(parent_request.request_id, internal=True)
|
||||
raise
|
||||
return queue
|
||||
|
||||
async def _add_request(
|
||||
|
||||
Reference in New Issue
Block a user