[Core] Move pause and resume functions into engine (#34125)

Signed-off-by: ahao-anyscale <ahao@anyscale.com>
Signed-off-by: Aaron Hao <ahao@anyscale.com>
Signed-off-by: hao-aaron <ahao@anyscale.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Aaron Hao
2026-02-13 00:15:10 -08:00
committed by GitHub
co-authored by gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Nick Hill
parent 47e9b63e1a
commit dddbff4624
9 changed files with 621 additions and 136 deletions
+48 -3
View File
@@ -708,9 +708,7 @@ async def test_pause_resume_basic():
# Test all modes with no requests in flight
for mode in ("abort", "wait", "keep"):
await engine.pause_generation(mode=mode)
# "keep" only freezes the scheduler; it does not set _paused
if mode != "keep":
assert await engine.is_paused()
assert await engine.is_paused()
await engine.resume_generation()
assert not await engine.is_paused()
@@ -808,6 +806,53 @@ async def test_pause_abort():
assert final_output2.finished
@pytest.mark.asyncio
async def test_pause_then_abort_queued_request():
"""Test that aborting a request that was submitted while paused (in
_paused_adds_queue) aborts it and notifies the client; the request does
not run after resume.
"""
with ExitStack() as after:
with set_default_torch_num_threads(1):
engine = AsyncLLM.from_engine_args(TEXT_ENGINE_ARGS)
after.callback(engine.shutdown)
request_id = "abort-queued-request"
sampling_params = SamplingParams(max_tokens=20, ignore_eos=True)
outputs: list[RequestOutput] = []
# Pause first so the next add goes to _paused_adds_queue
await engine.pause_generation(mode="keep")
assert await engine.is_paused()
async def gen():
async for out in engine.generate(
request_id=request_id,
prompt=TEXT_PROMPT,
sampling_params=sampling_params,
):
outputs.append(out)
return outputs[-1] if outputs else None
gen_task = asyncio.create_task(gen())
# Give the request time to reach the engine and sit in _paused_adds_queue
await asyncio.sleep(0.2)
# Abort the queued request
await engine.abort(request_id, internal=False)
# Resume so the engine can process and deliver the abort output
await engine.resume_generation()
final_output = await asyncio.wait_for(gen_task, timeout=10.0)
assert final_output is not None
assert final_output.finished
assert final_output.outputs[0].finish_reason == "abort"
# Request was never run, so no tokens
assert len(final_output.outputs[0].token_ids) == 0
@pytest.mark.asyncio
async def test_pause_wait():
"""Test that mode='wait' waits for in-flight requests to complete."""
@@ -8,6 +8,7 @@ import os
import signal
import time
import uuid
from concurrent.futures import Future
from dataclasses import dataclass
from threading import Thread
from types import SimpleNamespace
@@ -278,6 +279,24 @@ def echo_dc_nested(
return structures.get(structure_type, val)
def future_echo(self, value: Any, num_wait_loops: int = 2) -> Future:
"""Utility that returns a Future completed by a per_step_hook after
num_wait_loops engine steps (tests deferred utility path).
"""
future: Future = Future()
remaining = [num_wait_loops]
def _step(engine: EngineCore) -> bool:
remaining[0] -= 1
if remaining[0] <= 0:
future.set_result(value)
return True # remove hook
return False
self.per_step_hooks.add(_step)
return future
# --- Fixtures for subprocess patching ---
# These create sitecustomize.py files that patch EngineCore in spawned
# subprocesses. This is necessary because ROCm requires 'spawn' multiprocessing
@@ -383,6 +402,28 @@ def subprocess_echo_dc_nested_patch(monkeypatch, tmp_path):
)
@pytest.fixture
def subprocess_future_echo_patch(monkeypatch, tmp_path):
"""Create sitecustomize.py so spawned subprocesses have future_echo method."""
sc = tmp_path / "sitecustomize.py"
sc.write_text(
"\n".join(
[
"from concurrent.futures import Future",
"from typing import Any",
"",
"from vllm.v1.engine.core import EngineCore",
inspect.getsource(future_echo),
"EngineCore.future_echo = future_echo",
]
)
)
monkeypatch.setenv(
"PYTHONPATH",
os.pathsep.join(filter(None, [str(tmp_path), os.getenv("PYTHONPATH")])),
)
@create_new_process_for_each_test()
@pytest.mark.parametrize("multiprocessing_mode", [True, False])
def test_engine_core_client(
@@ -786,6 +827,48 @@ async def test_engine_core_client_util_method_nested_structures(
client.shutdown()
@pytest.mark.asyncio(loop_scope="function")
async def test_engine_core_client_future_utility_async(
monkeypatch: pytest.MonkeyPatch,
subprocess_future_echo_patch,
):
"""Test that a utility returning a Future (completed by a per_step_hook
after N steps) completes when the future is done (engine uses add_done_callback).
"""
with monkeypatch.context() as m:
m.setattr(EngineCore, "future_echo", future_echo, raising=False)
engine_args = EngineArgs(model=MODEL_NAME, enforce_eager=True)
vllm_config = engine_args.create_engine_config(
usage_context=UsageContext.UNKNOWN_CONTEXT
)
executor_class = Executor.get_class(vllm_config)
with set_default_torch_num_threads(1):
client = EngineCoreClient.make_client(
multiprocess_mode=True,
asyncio_mode=True,
vllm_config=vllm_config,
executor_class=executor_class,
log_stats=True,
)
try:
core_client: AsyncMPClient = client
# Completes after 2 engine steps (num_wait_loops=2)
result = await core_client.call_utility_async(
"future_echo", "future_result", 2
)
assert result == "future_result"
# None is a valid result (num_wait_loops=0 → completes on first step)
result = await core_client.call_utility_async("future_echo", None, 0)
assert result is None
finally:
client.shutdown()
@pytest.mark.parametrize(
"multiprocessing_mode,publisher_config",
[(True, "tcp"), (False, "inproc")],