[Core] Move pause and resume functions into engine (#34125)
Signed-off-by: ahao-anyscale <ahao@anyscale.com> Signed-off-by: Aaron Hao <ahao@anyscale.com> Signed-off-by: hao-aaron <ahao@anyscale.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Nick Hill
parent
47e9b63e1a
commit
dddbff4624
@@ -708,9 +708,7 @@ async def test_pause_resume_basic():
|
||||
# Test all modes with no requests in flight
|
||||
for mode in ("abort", "wait", "keep"):
|
||||
await engine.pause_generation(mode=mode)
|
||||
# "keep" only freezes the scheduler; it does not set _paused
|
||||
if mode != "keep":
|
||||
assert await engine.is_paused()
|
||||
assert await engine.is_paused()
|
||||
await engine.resume_generation()
|
||||
assert not await engine.is_paused()
|
||||
|
||||
@@ -808,6 +806,53 @@ async def test_pause_abort():
|
||||
assert final_output2.finished
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pause_then_abort_queued_request():
|
||||
"""Test that aborting a request that was submitted while paused (in
|
||||
_paused_adds_queue) aborts it and notifies the client; the request does
|
||||
not run after resume.
|
||||
"""
|
||||
with ExitStack() as after:
|
||||
with set_default_torch_num_threads(1):
|
||||
engine = AsyncLLM.from_engine_args(TEXT_ENGINE_ARGS)
|
||||
after.callback(engine.shutdown)
|
||||
|
||||
request_id = "abort-queued-request"
|
||||
sampling_params = SamplingParams(max_tokens=20, ignore_eos=True)
|
||||
outputs: list[RequestOutput] = []
|
||||
|
||||
# Pause first so the next add goes to _paused_adds_queue
|
||||
await engine.pause_generation(mode="keep")
|
||||
assert await engine.is_paused()
|
||||
|
||||
async def gen():
|
||||
async for out in engine.generate(
|
||||
request_id=request_id,
|
||||
prompt=TEXT_PROMPT,
|
||||
sampling_params=sampling_params,
|
||||
):
|
||||
outputs.append(out)
|
||||
return outputs[-1] if outputs else None
|
||||
|
||||
gen_task = asyncio.create_task(gen())
|
||||
|
||||
# Give the request time to reach the engine and sit in _paused_adds_queue
|
||||
await asyncio.sleep(0.2)
|
||||
|
||||
# Abort the queued request
|
||||
await engine.abort(request_id, internal=False)
|
||||
|
||||
# Resume so the engine can process and deliver the abort output
|
||||
await engine.resume_generation()
|
||||
|
||||
final_output = await asyncio.wait_for(gen_task, timeout=10.0)
|
||||
assert final_output is not None
|
||||
assert final_output.finished
|
||||
assert final_output.outputs[0].finish_reason == "abort"
|
||||
# Request was never run, so no tokens
|
||||
assert len(final_output.outputs[0].token_ids) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pause_wait():
|
||||
"""Test that mode='wait' waits for in-flight requests to complete."""
|
||||
|
||||
@@ -8,6 +8,7 @@ import os
|
||||
import signal
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import Future
|
||||
from dataclasses import dataclass
|
||||
from threading import Thread
|
||||
from types import SimpleNamespace
|
||||
@@ -278,6 +279,24 @@ def echo_dc_nested(
|
||||
return structures.get(structure_type, val)
|
||||
|
||||
|
||||
def future_echo(self, value: Any, num_wait_loops: int = 2) -> Future:
|
||||
"""Utility that returns a Future completed by a per_step_hook after
|
||||
num_wait_loops engine steps (tests deferred utility path).
|
||||
"""
|
||||
future: Future = Future()
|
||||
remaining = [num_wait_loops]
|
||||
|
||||
def _step(engine: EngineCore) -> bool:
|
||||
remaining[0] -= 1
|
||||
if remaining[0] <= 0:
|
||||
future.set_result(value)
|
||||
return True # remove hook
|
||||
return False
|
||||
|
||||
self.per_step_hooks.add(_step)
|
||||
return future
|
||||
|
||||
|
||||
# --- Fixtures for subprocess patching ---
|
||||
# These create sitecustomize.py files that patch EngineCore in spawned
|
||||
# subprocesses. This is necessary because ROCm requires 'spawn' multiprocessing
|
||||
@@ -383,6 +402,28 @@ def subprocess_echo_dc_nested_patch(monkeypatch, tmp_path):
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def subprocess_future_echo_patch(monkeypatch, tmp_path):
|
||||
"""Create sitecustomize.py so spawned subprocesses have future_echo method."""
|
||||
sc = tmp_path / "sitecustomize.py"
|
||||
sc.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"from concurrent.futures import Future",
|
||||
"from typing import Any",
|
||||
"",
|
||||
"from vllm.v1.engine.core import EngineCore",
|
||||
inspect.getsource(future_echo),
|
||||
"EngineCore.future_echo = future_echo",
|
||||
]
|
||||
)
|
||||
)
|
||||
monkeypatch.setenv(
|
||||
"PYTHONPATH",
|
||||
os.pathsep.join(filter(None, [str(tmp_path), os.getenv("PYTHONPATH")])),
|
||||
)
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.parametrize("multiprocessing_mode", [True, False])
|
||||
def test_engine_core_client(
|
||||
@@ -786,6 +827,48 @@ async def test_engine_core_client_util_method_nested_structures(
|
||||
client.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="function")
|
||||
async def test_engine_core_client_future_utility_async(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
subprocess_future_echo_patch,
|
||||
):
|
||||
"""Test that a utility returning a Future (completed by a per_step_hook
|
||||
after N steps) completes when the future is done (engine uses add_done_callback).
|
||||
"""
|
||||
with monkeypatch.context() as m:
|
||||
m.setattr(EngineCore, "future_echo", future_echo, raising=False)
|
||||
|
||||
engine_args = EngineArgs(model=MODEL_NAME, enforce_eager=True)
|
||||
vllm_config = engine_args.create_engine_config(
|
||||
usage_context=UsageContext.UNKNOWN_CONTEXT
|
||||
)
|
||||
executor_class = Executor.get_class(vllm_config)
|
||||
|
||||
with set_default_torch_num_threads(1):
|
||||
client = EngineCoreClient.make_client(
|
||||
multiprocess_mode=True,
|
||||
asyncio_mode=True,
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
log_stats=True,
|
||||
)
|
||||
|
||||
try:
|
||||
core_client: AsyncMPClient = client
|
||||
|
||||
# Completes after 2 engine steps (num_wait_loops=2)
|
||||
result = await core_client.call_utility_async(
|
||||
"future_echo", "future_result", 2
|
||||
)
|
||||
assert result == "future_result"
|
||||
|
||||
# None is a valid result (num_wait_loops=0 → completes on first step)
|
||||
result = await core_client.call_utility_async("future_echo", None, 0)
|
||||
assert result is None
|
||||
finally:
|
||||
client.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"multiprocessing_mode,publisher_config",
|
||||
[(True, "tcp"), (False, "inproc")],
|
||||
|
||||
Reference in New Issue
Block a user