[Core] Move pause and resume functions into engine (#34125)
Signed-off-by: ahao-anyscale <ahao@anyscale.com> Signed-off-by: Aaron Hao <ahao@anyscale.com> Signed-off-by: hao-aaron <ahao@anyscale.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Nick Hill
parent
47e9b63e1a
commit
dddbff4624
@@ -12,6 +12,7 @@ from vllm import SamplingParams
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.inputs import PromptType
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sampling_params import RequestOutputKind
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
@@ -181,3 +182,145 @@ async def test_load(
|
||||
assert slogger.finished_req_count > NUM_REQUESTS // (DP_SIZE + 1), (
|
||||
f"requests are imbalanced: {stats_loggers}"
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# DP Pause/Resume Tests
|
||||
# =============================================================================
|
||||
|
||||
DP_PAUSE_MODEL = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
DP_PAUSE_PROMPT = "This is a test of data parallel pause"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dp_pause_resume_basic():
|
||||
"""Pausing from the client (one call) pauses all DP ranks; resume clears it."""
|
||||
if current_platform.is_rocm():
|
||||
pytest.skip("DP pause tests use mp backend only")
|
||||
with ExitStack() as after:
|
||||
engine_args = AsyncEngineArgs(
|
||||
model=DP_PAUSE_MODEL,
|
||||
enforce_eager=True,
|
||||
tensor_parallel_size=int(os.getenv("TP_SIZE", 1)),
|
||||
data_parallel_size=DP_SIZE,
|
||||
data_parallel_backend="mp",
|
||||
)
|
||||
engine = AsyncLLM.from_engine_args(engine_args)
|
||||
after.callback(engine.shutdown)
|
||||
|
||||
assert not await engine.is_paused()
|
||||
await engine.pause_generation(mode="abort")
|
||||
assert await engine.is_paused()
|
||||
await engine.resume_generation()
|
||||
assert not await engine.is_paused()
|
||||
|
||||
# Engine still works after resume
|
||||
sampling_params = SamplingParams(max_tokens=5)
|
||||
async for out in engine.generate(
|
||||
request_id="after-resume",
|
||||
prompt=DP_PAUSE_PROMPT,
|
||||
sampling_params=sampling_params,
|
||||
):
|
||||
pass
|
||||
assert out.finished
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dp_pause_abort():
|
||||
"""Pause with abort from one client aborts in-flight requests on all DP ranks."""
|
||||
if current_platform.is_rocm():
|
||||
pytest.skip("DP pause tests use mp backend only")
|
||||
with ExitStack() as after:
|
||||
engine_args = AsyncEngineArgs(
|
||||
model=DP_PAUSE_MODEL,
|
||||
enforce_eager=True,
|
||||
tensor_parallel_size=int(os.getenv("TP_SIZE", 1)),
|
||||
data_parallel_size=DP_SIZE,
|
||||
data_parallel_backend="mp",
|
||||
)
|
||||
engine = AsyncLLM.from_engine_args(engine_args)
|
||||
after.callback(engine.shutdown)
|
||||
|
||||
# Start several requests so they are distributed across ranks
|
||||
sampling_params = SamplingParams(max_tokens=500, ignore_eos=True)
|
||||
num_requests = 4
|
||||
outputs_by_id: dict[str, list[RequestOutput]] = {}
|
||||
|
||||
async def gen(rid: str):
|
||||
out_list: list[RequestOutput] = []
|
||||
outputs_by_id[rid] = out_list
|
||||
async for out in engine.generate(
|
||||
request_id=rid,
|
||||
prompt=DP_PAUSE_PROMPT,
|
||||
sampling_params=sampling_params,
|
||||
):
|
||||
out_list.append(out)
|
||||
return out_list[-1] if out_list else None
|
||||
|
||||
tasks = [asyncio.create_task(gen(f"req-{i}")) for i in range(num_requests)]
|
||||
# Wait for some tokens on at least one request
|
||||
while not any(len(o) >= 2 for o in outputs_by_id.values()):
|
||||
await asyncio.sleep(0.02)
|
||||
|
||||
await engine.pause_generation(mode="abort")
|
||||
|
||||
finals = await asyncio.gather(*tasks)
|
||||
for i, final in enumerate(finals):
|
||||
assert final is not None, f"req-{i} had no output"
|
||||
assert final.finished
|
||||
assert final.outputs[0].finish_reason == "abort"
|
||||
|
||||
assert await engine.is_paused()
|
||||
await engine.resume_generation()
|
||||
assert not await engine.is_paused()
|
||||
|
||||
# New request completes after resume
|
||||
async for out in engine.generate(
|
||||
request_id="after-abort",
|
||||
prompt=DP_PAUSE_PROMPT,
|
||||
sampling_params=SamplingParams(max_tokens=5),
|
||||
):
|
||||
pass
|
||||
assert out.finished
|
||||
assert not engine.output_processor.has_unfinished_requests()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dp_pause_keep_then_resume():
|
||||
"""Pause with keep queues new requests; resume allows them to run."""
|
||||
if current_platform.is_rocm():
|
||||
pytest.skip("DP pause tests use mp backend only")
|
||||
with ExitStack() as after:
|
||||
engine_args = AsyncEngineArgs(
|
||||
model=DP_PAUSE_MODEL,
|
||||
enforce_eager=True,
|
||||
tensor_parallel_size=int(os.getenv("TP_SIZE", 1)),
|
||||
data_parallel_size=DP_SIZE,
|
||||
data_parallel_backend="mp",
|
||||
)
|
||||
engine = AsyncLLM.from_engine_args(engine_args)
|
||||
after.callback(engine.shutdown)
|
||||
|
||||
await engine.pause_generation(mode="keep")
|
||||
assert await engine.is_paused()
|
||||
|
||||
request_done = asyncio.Event()
|
||||
|
||||
async def gen():
|
||||
async for out in engine.generate(
|
||||
request_id="queued-keep",
|
||||
prompt=DP_PAUSE_PROMPT,
|
||||
sampling_params=SamplingParams(max_tokens=5),
|
||||
):
|
||||
pass
|
||||
request_done.set()
|
||||
return out
|
||||
|
||||
task = asyncio.create_task(gen())
|
||||
await asyncio.sleep(0.2)
|
||||
assert not request_done.is_set()
|
||||
|
||||
await engine.resume_generation()
|
||||
final = await asyncio.wait_for(task, timeout=10.0)
|
||||
assert final.finished
|
||||
assert not await engine.is_paused()
|
||||
|
||||
@@ -708,9 +708,7 @@ async def test_pause_resume_basic():
|
||||
# Test all modes with no requests in flight
|
||||
for mode in ("abort", "wait", "keep"):
|
||||
await engine.pause_generation(mode=mode)
|
||||
# "keep" only freezes the scheduler; it does not set _paused
|
||||
if mode != "keep":
|
||||
assert await engine.is_paused()
|
||||
assert await engine.is_paused()
|
||||
await engine.resume_generation()
|
||||
assert not await engine.is_paused()
|
||||
|
||||
@@ -808,6 +806,53 @@ async def test_pause_abort():
|
||||
assert final_output2.finished
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pause_then_abort_queued_request():
|
||||
"""Test that aborting a request that was submitted while paused (in
|
||||
_paused_adds_queue) aborts it and notifies the client; the request does
|
||||
not run after resume.
|
||||
"""
|
||||
with ExitStack() as after:
|
||||
with set_default_torch_num_threads(1):
|
||||
engine = AsyncLLM.from_engine_args(TEXT_ENGINE_ARGS)
|
||||
after.callback(engine.shutdown)
|
||||
|
||||
request_id = "abort-queued-request"
|
||||
sampling_params = SamplingParams(max_tokens=20, ignore_eos=True)
|
||||
outputs: list[RequestOutput] = []
|
||||
|
||||
# Pause first so the next add goes to _paused_adds_queue
|
||||
await engine.pause_generation(mode="keep")
|
||||
assert await engine.is_paused()
|
||||
|
||||
async def gen():
|
||||
async for out in engine.generate(
|
||||
request_id=request_id,
|
||||
prompt=TEXT_PROMPT,
|
||||
sampling_params=sampling_params,
|
||||
):
|
||||
outputs.append(out)
|
||||
return outputs[-1] if outputs else None
|
||||
|
||||
gen_task = asyncio.create_task(gen())
|
||||
|
||||
# Give the request time to reach the engine and sit in _paused_adds_queue
|
||||
await asyncio.sleep(0.2)
|
||||
|
||||
# Abort the queued request
|
||||
await engine.abort(request_id, internal=False)
|
||||
|
||||
# Resume so the engine can process and deliver the abort output
|
||||
await engine.resume_generation()
|
||||
|
||||
final_output = await asyncio.wait_for(gen_task, timeout=10.0)
|
||||
assert final_output is not None
|
||||
assert final_output.finished
|
||||
assert final_output.outputs[0].finish_reason == "abort"
|
||||
# Request was never run, so no tokens
|
||||
assert len(final_output.outputs[0].token_ids) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pause_wait():
|
||||
"""Test that mode='wait' waits for in-flight requests to complete."""
|
||||
|
||||
@@ -8,6 +8,7 @@ import os
|
||||
import signal
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import Future
|
||||
from dataclasses import dataclass
|
||||
from threading import Thread
|
||||
from types import SimpleNamespace
|
||||
@@ -278,6 +279,24 @@ def echo_dc_nested(
|
||||
return structures.get(structure_type, val)
|
||||
|
||||
|
||||
def future_echo(self, value: Any, num_wait_loops: int = 2) -> Future:
|
||||
"""Utility that returns a Future completed by a per_step_hook after
|
||||
num_wait_loops engine steps (tests deferred utility path).
|
||||
"""
|
||||
future: Future = Future()
|
||||
remaining = [num_wait_loops]
|
||||
|
||||
def _step(engine: EngineCore) -> bool:
|
||||
remaining[0] -= 1
|
||||
if remaining[0] <= 0:
|
||||
future.set_result(value)
|
||||
return True # remove hook
|
||||
return False
|
||||
|
||||
self.per_step_hooks.add(_step)
|
||||
return future
|
||||
|
||||
|
||||
# --- Fixtures for subprocess patching ---
|
||||
# These create sitecustomize.py files that patch EngineCore in spawned
|
||||
# subprocesses. This is necessary because ROCm requires 'spawn' multiprocessing
|
||||
@@ -383,6 +402,28 @@ def subprocess_echo_dc_nested_patch(monkeypatch, tmp_path):
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def subprocess_future_echo_patch(monkeypatch, tmp_path):
|
||||
"""Create sitecustomize.py so spawned subprocesses have future_echo method."""
|
||||
sc = tmp_path / "sitecustomize.py"
|
||||
sc.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"from concurrent.futures import Future",
|
||||
"from typing import Any",
|
||||
"",
|
||||
"from vllm.v1.engine.core import EngineCore",
|
||||
inspect.getsource(future_echo),
|
||||
"EngineCore.future_echo = future_echo",
|
||||
]
|
||||
)
|
||||
)
|
||||
monkeypatch.setenv(
|
||||
"PYTHONPATH",
|
||||
os.pathsep.join(filter(None, [str(tmp_path), os.getenv("PYTHONPATH")])),
|
||||
)
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.parametrize("multiprocessing_mode", [True, False])
|
||||
def test_engine_core_client(
|
||||
@@ -786,6 +827,48 @@ async def test_engine_core_client_util_method_nested_structures(
|
||||
client.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="function")
|
||||
async def test_engine_core_client_future_utility_async(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
subprocess_future_echo_patch,
|
||||
):
|
||||
"""Test that a utility returning a Future (completed by a per_step_hook
|
||||
after N steps) completes when the future is done (engine uses add_done_callback).
|
||||
"""
|
||||
with monkeypatch.context() as m:
|
||||
m.setattr(EngineCore, "future_echo", future_echo, raising=False)
|
||||
|
||||
engine_args = EngineArgs(model=MODEL_NAME, enforce_eager=True)
|
||||
vllm_config = engine_args.create_engine_config(
|
||||
usage_context=UsageContext.UNKNOWN_CONTEXT
|
||||
)
|
||||
executor_class = Executor.get_class(vllm_config)
|
||||
|
||||
with set_default_torch_num_threads(1):
|
||||
client = EngineCoreClient.make_client(
|
||||
multiprocess_mode=True,
|
||||
asyncio_mode=True,
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
log_stats=True,
|
||||
)
|
||||
|
||||
try:
|
||||
core_client: AsyncMPClient = client
|
||||
|
||||
# Completes after 2 engine steps (num_wait_loops=2)
|
||||
result = await core_client.call_utility_async(
|
||||
"future_echo", "future_result", 2
|
||||
)
|
||||
assert result == "future_result"
|
||||
|
||||
# None is a valid result (num_wait_loops=0 → completes on first step)
|
||||
result = await core_client.call_utility_async("future_echo", None, 0)
|
||||
assert result is None
|
||||
finally:
|
||||
client.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"multiprocessing_mode,publisher_config",
|
||||
[(True, "tcp"), (False, "inproc")],
|
||||
|
||||
Reference in New Issue
Block a user