[Core] Move pause and resume functions into engine (#34125)

Signed-off-by: ahao-anyscale <ahao@anyscale.com>
Signed-off-by: Aaron Hao <ahao@anyscale.com>
Signed-off-by: hao-aaron <ahao@anyscale.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Aaron Hao
2026-02-13 00:15:10 -08:00
committed by GitHub
co-authored by gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Nick Hill
parent 47e9b63e1a
commit dddbff4624
9 changed files with 621 additions and 136 deletions
+143
View File
@@ -12,6 +12,7 @@ from vllm import SamplingParams
from vllm.config import VllmConfig
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.inputs import PromptType
from vllm.outputs import RequestOutput
from vllm.platforms import current_platform
from vllm.sampling_params import RequestOutputKind
from vllm.v1.engine.async_llm import AsyncLLM
@@ -181,3 +182,145 @@ async def test_load(
assert slogger.finished_req_count > NUM_REQUESTS // (DP_SIZE + 1), (
f"requests are imbalanced: {stats_loggers}"
)
# =============================================================================
# DP Pause/Resume Tests
# =============================================================================
DP_PAUSE_MODEL = "hmellor/tiny-random-LlamaForCausalLM"
DP_PAUSE_PROMPT = "This is a test of data parallel pause"
@pytest.mark.asyncio
async def test_dp_pause_resume_basic():
"""Pausing from the client (one call) pauses all DP ranks; resume clears it."""
if current_platform.is_rocm():
pytest.skip("DP pause tests use mp backend only")
with ExitStack() as after:
engine_args = AsyncEngineArgs(
model=DP_PAUSE_MODEL,
enforce_eager=True,
tensor_parallel_size=int(os.getenv("TP_SIZE", 1)),
data_parallel_size=DP_SIZE,
data_parallel_backend="mp",
)
engine = AsyncLLM.from_engine_args(engine_args)
after.callback(engine.shutdown)
assert not await engine.is_paused()
await engine.pause_generation(mode="abort")
assert await engine.is_paused()
await engine.resume_generation()
assert not await engine.is_paused()
# Engine still works after resume
sampling_params = SamplingParams(max_tokens=5)
async for out in engine.generate(
request_id="after-resume",
prompt=DP_PAUSE_PROMPT,
sampling_params=sampling_params,
):
pass
assert out.finished
@pytest.mark.asyncio
async def test_dp_pause_abort():
"""Pause with abort from one client aborts in-flight requests on all DP ranks."""
if current_platform.is_rocm():
pytest.skip("DP pause tests use mp backend only")
with ExitStack() as after:
engine_args = AsyncEngineArgs(
model=DP_PAUSE_MODEL,
enforce_eager=True,
tensor_parallel_size=int(os.getenv("TP_SIZE", 1)),
data_parallel_size=DP_SIZE,
data_parallel_backend="mp",
)
engine = AsyncLLM.from_engine_args(engine_args)
after.callback(engine.shutdown)
# Start several requests so they are distributed across ranks
sampling_params = SamplingParams(max_tokens=500, ignore_eos=True)
num_requests = 4
outputs_by_id: dict[str, list[RequestOutput]] = {}
async def gen(rid: str):
out_list: list[RequestOutput] = []
outputs_by_id[rid] = out_list
async for out in engine.generate(
request_id=rid,
prompt=DP_PAUSE_PROMPT,
sampling_params=sampling_params,
):
out_list.append(out)
return out_list[-1] if out_list else None
tasks = [asyncio.create_task(gen(f"req-{i}")) for i in range(num_requests)]
# Wait for some tokens on at least one request
while not any(len(o) >= 2 for o in outputs_by_id.values()):
await asyncio.sleep(0.02)
await engine.pause_generation(mode="abort")
finals = await asyncio.gather(*tasks)
for i, final in enumerate(finals):
assert final is not None, f"req-{i} had no output"
assert final.finished
assert final.outputs[0].finish_reason == "abort"
assert await engine.is_paused()
await engine.resume_generation()
assert not await engine.is_paused()
# New request completes after resume
async for out in engine.generate(
request_id="after-abort",
prompt=DP_PAUSE_PROMPT,
sampling_params=SamplingParams(max_tokens=5),
):
pass
assert out.finished
assert not engine.output_processor.has_unfinished_requests()
@pytest.mark.asyncio
async def test_dp_pause_keep_then_resume():
"""Pause with keep queues new requests; resume allows them to run."""
if current_platform.is_rocm():
pytest.skip("DP pause tests use mp backend only")
with ExitStack() as after:
engine_args = AsyncEngineArgs(
model=DP_PAUSE_MODEL,
enforce_eager=True,
tensor_parallel_size=int(os.getenv("TP_SIZE", 1)),
data_parallel_size=DP_SIZE,
data_parallel_backend="mp",
)
engine = AsyncLLM.from_engine_args(engine_args)
after.callback(engine.shutdown)
await engine.pause_generation(mode="keep")
assert await engine.is_paused()
request_done = asyncio.Event()
async def gen():
async for out in engine.generate(
request_id="queued-keep",
prompt=DP_PAUSE_PROMPT,
sampling_params=SamplingParams(max_tokens=5),
):
pass
request_done.set()
return out
task = asyncio.create_task(gen())
await asyncio.sleep(0.2)
assert not request_done.is_set()
await engine.resume_generation()
final = await asyncio.wait_for(task, timeout=10.0)
assert final.finished
assert not await engine.is_paused()
+48 -3
View File
@@ -708,9 +708,7 @@ async def test_pause_resume_basic():
# Test all modes with no requests in flight
for mode in ("abort", "wait", "keep"):
await engine.pause_generation(mode=mode)
# "keep" only freezes the scheduler; it does not set _paused
if mode != "keep":
assert await engine.is_paused()
assert await engine.is_paused()
await engine.resume_generation()
assert not await engine.is_paused()
@@ -808,6 +806,53 @@ async def test_pause_abort():
assert final_output2.finished
@pytest.mark.asyncio
async def test_pause_then_abort_queued_request():
"""Test that aborting a request that was submitted while paused (in
_paused_adds_queue) aborts it and notifies the client; the request does
not run after resume.
"""
with ExitStack() as after:
with set_default_torch_num_threads(1):
engine = AsyncLLM.from_engine_args(TEXT_ENGINE_ARGS)
after.callback(engine.shutdown)
request_id = "abort-queued-request"
sampling_params = SamplingParams(max_tokens=20, ignore_eos=True)
outputs: list[RequestOutput] = []
# Pause first so the next add goes to _paused_adds_queue
await engine.pause_generation(mode="keep")
assert await engine.is_paused()
async def gen():
async for out in engine.generate(
request_id=request_id,
prompt=TEXT_PROMPT,
sampling_params=sampling_params,
):
outputs.append(out)
return outputs[-1] if outputs else None
gen_task = asyncio.create_task(gen())
# Give the request time to reach the engine and sit in _paused_adds_queue
await asyncio.sleep(0.2)
# Abort the queued request
await engine.abort(request_id, internal=False)
# Resume so the engine can process and deliver the abort output
await engine.resume_generation()
final_output = await asyncio.wait_for(gen_task, timeout=10.0)
assert final_output is not None
assert final_output.finished
assert final_output.outputs[0].finish_reason == "abort"
# Request was never run, so no tokens
assert len(final_output.outputs[0].token_ids) == 0
@pytest.mark.asyncio
async def test_pause_wait():
"""Test that mode='wait' waits for in-flight requests to complete."""
@@ -8,6 +8,7 @@ import os
import signal
import time
import uuid
from concurrent.futures import Future
from dataclasses import dataclass
from threading import Thread
from types import SimpleNamespace
@@ -278,6 +279,24 @@ def echo_dc_nested(
return structures.get(structure_type, val)
def future_echo(self, value: Any, num_wait_loops: int = 2) -> Future:
"""Utility that returns a Future completed by a per_step_hook after
num_wait_loops engine steps (tests deferred utility path).
"""
future: Future = Future()
remaining = [num_wait_loops]
def _step(engine: EngineCore) -> bool:
remaining[0] -= 1
if remaining[0] <= 0:
future.set_result(value)
return True # remove hook
return False
self.per_step_hooks.add(_step)
return future
# --- Fixtures for subprocess patching ---
# These create sitecustomize.py files that patch EngineCore in spawned
# subprocesses. This is necessary because ROCm requires 'spawn' multiprocessing
@@ -383,6 +402,28 @@ def subprocess_echo_dc_nested_patch(monkeypatch, tmp_path):
)
@pytest.fixture
def subprocess_future_echo_patch(monkeypatch, tmp_path):
"""Create sitecustomize.py so spawned subprocesses have future_echo method."""
sc = tmp_path / "sitecustomize.py"
sc.write_text(
"\n".join(
[
"from concurrent.futures import Future",
"from typing import Any",
"",
"from vllm.v1.engine.core import EngineCore",
inspect.getsource(future_echo),
"EngineCore.future_echo = future_echo",
]
)
)
monkeypatch.setenv(
"PYTHONPATH",
os.pathsep.join(filter(None, [str(tmp_path), os.getenv("PYTHONPATH")])),
)
@create_new_process_for_each_test()
@pytest.mark.parametrize("multiprocessing_mode", [True, False])
def test_engine_core_client(
@@ -786,6 +827,48 @@ async def test_engine_core_client_util_method_nested_structures(
client.shutdown()
@pytest.mark.asyncio(loop_scope="function")
async def test_engine_core_client_future_utility_async(
monkeypatch: pytest.MonkeyPatch,
subprocess_future_echo_patch,
):
"""Test that a utility returning a Future (completed by a per_step_hook
after N steps) completes when the future is done (engine uses add_done_callback).
"""
with monkeypatch.context() as m:
m.setattr(EngineCore, "future_echo", future_echo, raising=False)
engine_args = EngineArgs(model=MODEL_NAME, enforce_eager=True)
vllm_config = engine_args.create_engine_config(
usage_context=UsageContext.UNKNOWN_CONTEXT
)
executor_class = Executor.get_class(vllm_config)
with set_default_torch_num_threads(1):
client = EngineCoreClient.make_client(
multiprocess_mode=True,
asyncio_mode=True,
vllm_config=vllm_config,
executor_class=executor_class,
log_stats=True,
)
try:
core_client: AsyncMPClient = client
# Completes after 2 engine steps (num_wait_loops=2)
result = await core_client.call_utility_async(
"future_echo", "future_result", 2
)
assert result == "future_result"
# None is a valid result (num_wait_loops=0 → completes on first step)
result = await core_client.call_utility_async("future_echo", None, 0)
assert result is None
finally:
client.shutdown()
@pytest.mark.parametrize(
"multiprocessing_mode,publisher_config",
[(True, "tcp"), (False, "inproc")],