[Bugfix][DP] Synchronize the device on pause completion (#52914)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
aoshen02
2026-08-26 09:50:33 -07:00
committed by GitHub
parent 8d301f075b
commit 17da48596c
5 changed files with 164 additions and 8 deletions
+110
View File
@@ -713,3 +713,113 @@ async def test_dp_pause_barrier_request_deadlock():
assert not await engine.is_paused() assert not await engine.is_paused()
# Let the two requests we sent mid-barrier complete. # Let the two requests we sent mid-barrier complete.
await asyncio.gather(*mid_barrier_tasks) await asyncio.gather(*mid_barrier_tasks)
@pytest.mark.asyncio
async def test_dp_pause_wait_mode_drains_in_flight():
"""mode="wait" through the DP consensus path: the pause lets the
in-flight request run to completion while the consensus is pending,
resolves only once drained, and the engine generates again on resume."""
with ExitStack() as after:
engine = AsyncLLM.from_engine_args(_get_dp_pause_engine_args(True))
after.callback(engine.shutdown)
collector = await engine.add_request(
request_id="inflight",
prompt=DP_PAUSE_PROMPT,
params=SamplingParams(max_tokens=64),
)
await engine.pause_generation(mode="wait")
assert await engine.is_paused()
while True:
out = await asyncio.wait_for(collector.get(), timeout=30)
if out.finished:
break
await engine.resume_generation()
async for out in engine.generate(
request_id="after-wait",
prompt=DP_PAUSE_PROMPT,
sampling_params=SamplingParams(max_tokens=5),
):
pass
assert out.finished
@pytest.mark.asyncio
async def test_dp_pause_while_asleep():
"""Pausing a sleeping DP engine: sleeping ranks skip dummy batches yet
must still reach the pause consensus, and the completion barrier must
be harmless on workers whose memory is unmapped."""
with ExitStack() as after:
engine = AsyncLLM.from_engine_args(_get_dp_pause_engine_args(True))
after.callback(engine.shutdown)
async for _ in engine.generate(
request_id="warmup",
prompt=DP_PAUSE_PROMPT,
sampling_params=SamplingParams(max_tokens=5),
):
pass
await engine.sleep(level=1)
assert await engine.is_sleeping()
await asyncio.wait_for(engine.pause_generation(mode="abort"), timeout=30)
assert await engine.is_paused()
# A full wake makes the memory resident and resumes the scheduler.
await engine.wake_up()
assert not await engine.is_sleeping()
async for out in engine.generate(
request_id="after-wake",
prompt=DP_PAUSE_PROMPT,
sampling_params=SamplingParams(max_tokens=5),
):
pass
assert out.finished
@pytest.mark.asyncio
async def test_dp_pause_completion_implies_device_idle(
monkeypatch: pytest.MonkeyPatch,
):
"""A resolved pause future promises an idle device. Inflate the dummy
batches the pause consensus manufactures so any work the pause fails to
wait on stays visible, then require a quiet worker stream the moment
pause_generation() returns."""
with ExitStack() as after:
# The probes below ship functions through collective_rpc.
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
engine = AsyncLLM.from_engine_args(_get_dp_pause_engine_args(True))
after.callback(engine.shutdown)
async for _ in engine.generate(
request_id="warmup",
prompt=DP_PAUSE_PROMPT,
sampling_params=SamplingParams(max_tokens=5),
):
pass
def inflate_dummy_batches(worker: Any) -> bool:
import torch
inner = worker.execute_dummy_batch
def slow_dummy_batch() -> None:
inner()
# Keep the stream busy after launch, like a large-MoE forward.
torch.cuda._sleep(200_000_000)
worker.execute_dummy_batch = slow_dummy_batch
return True
assert all(await engine.engine_core.collective_rpc_async(inflate_dummy_batches))
await asyncio.wait_for(engine.pause_generation(mode="abort"), timeout=60)
def device_idle(worker: Any) -> bool:
import torch
return bool(torch.cuda.current_stream().query())
assert all(await engine.engine_core.collective_rpc_async(device_idle))
+36 -2
View File
@@ -5,7 +5,7 @@ import copy
import time import time
import uuid import uuid
from concurrent.futures import Future, ThreadPoolExecutor from concurrent.futures import Future, ThreadPoolExecutor
from unittest.mock import PropertyMock, patch from unittest.mock import MagicMock, PropertyMock, patch
import pytest import pytest
from transformers import AutoTokenizer from transformers import AutoTokenizer
@@ -23,7 +23,7 @@ from vllm.engine.arg_utils import EngineArgs
from vllm.platforms import current_platform from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_default_torch_num_threads from vllm.utils.torch_utils import set_default_torch_num_threads
from vllm.v1.engine import EngineCoreRequest from vllm.v1.engine import EngineCoreRequest
from vllm.v1.engine.core import EngineCore from vllm.v1.engine.core import EngineCore, EngineCoreProc
from vllm.v1.executor.abstract import Executor from vllm.v1.executor.abstract import Executor
from vllm.v1.executor.uniproc_executor import UniProcExecutor from vllm.v1.executor.uniproc_executor import UniProcExecutor
from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.kv_cache_interface import KVCacheConfig
@@ -608,3 +608,37 @@ def test_encoder_instance_zero_kv_cache(
assert not engine_core.scheduler.ec_connector.is_producer, ( assert not engine_core.scheduler.ec_connector.is_producer, (
"Consumer instance EC connector should be consumer" "Consumer instance EC connector should be consumer"
) )
def _pausable_engine_core_proc() -> EngineCoreProc:
"""A bare EngineCoreProc holding just the state pause_scheduler touches."""
core = object.__new__(EngineCoreProc)
core.model_executor = MagicMock()
core.scheduler = MagicMock()
core.scheduler.has_requests.return_value = False
core.batch_queue = None
core.engines_running = False
core._idle_state_callbacks = []
return core
@pytest.mark.parametrize("deferred", [False, True])
def test_pause_synchronizes_device_before_cache_reset(deferred: bool):
"""A resolved pause promises an idle device: the barrier must run before
caches are cleared and before the caller is unblocked."""
core = _pausable_engine_core_proc()
core.engines_running = deferred
order: list[str] = []
core.model_executor.collective_rpc.side_effect = lambda method: order.append(method)
core._reset_caches = lambda: order.append("reset_caches")
result = EngineCoreProc.pause_scheduler(core, mode="keep", clear_cache=True)
if deferred:
assert isinstance(result, Future)
assert not result.done() and order == []
core.engines_running = False
core._notify_idle_state_callbacks()
assert result.result(timeout=0) is None
else:
assert result is None
assert order == ["synchronize_device", "reset_caches"]
+10 -6
View File
@@ -839,6 +839,13 @@ class EngineCore:
self.reset_mm_cache() self.reset_mm_cache()
self.reset_encoder_cache() self.reset_encoder_cache()
def _finish_pause(self, clear_cache: bool) -> None:
# A completed pause promises an idle device: nothing else waits on
# the last dummy batch an idle DP rank launches.
self.model_executor.collective_rpc("synchronize_device")
if clear_cache:
self._reset_caches()
def pause_scheduler( def pause_scheduler(
self, mode: PauseMode = "abort", clear_cache: bool = True self, mode: PauseMode = "abort", clear_cache: bool = True
) -> Future | None: ) -> Future | None:
@@ -865,8 +872,7 @@ class EngineCore:
pause_state = PauseState.PAUSED_ALL if mode == "keep" else PauseState.PAUSED_NEW pause_state = PauseState.PAUSED_ALL if mode == "keep" else PauseState.PAUSED_NEW
self.scheduler.set_pause_state(pause_state) self.scheduler.set_pause_state(pause_state)
if clear_cache: self._finish_pause(clear_cache)
self._reset_caches()
return None return None
@@ -1937,8 +1943,7 @@ class EngineCoreProc(EngineCore):
raise ValueError(f"Invalid pause mode: {mode}") raise ValueError(f"Invalid pause mode: {mode}")
def engine_idle_callback(engine: "EngineCoreProc", future: Future[Any]) -> None: def engine_idle_callback(engine: "EngineCoreProc", future: Future[Any]) -> None:
if clear_cache: engine._finish_pause(clear_cache)
engine._reset_caches()
future.set_result(None) future.set_result(None)
if mode == "abort": if mode == "abort":
@@ -1951,8 +1956,7 @@ class EngineCoreProc(EngineCore):
self.scheduler.set_pause_state(pause_state) self.scheduler.set_pause_state(pause_state)
if self._pause_complete(): if self._pause_complete():
if clear_cache: self._finish_pause(clear_cache)
self._reset_caches()
return None return None
future = Future[Any]() future = Future[Any]()
+2
View File
@@ -259,6 +259,8 @@ class Worker(WorkerBase):
if tags is None or "kv_cache" in tags: if tags is None or "kv_cache" in tags:
self.model_runner.post_kv_cache_wake_up() self.model_runner.post_kv_cache_wake_up()
self.synchronize_device()
def checkpoint_prepare(self) -> None: def checkpoint_prepare(self) -> None:
checkpoint_prepare_distributed_state() checkpoint_prepare_distributed_state()
+6
View File
@@ -125,6 +125,12 @@ class WorkerBase:
"""Basic health check (override for device-specific checks).""" """Basic health check (override for device-specific checks)."""
return return
def synchronize_device(self) -> None:
"""Block until in-flight device work completes; backends outside
``torch.accelerator`` must override with their own wait."""
if torch.accelerator.is_available():
torch.accelerator.synchronize()
def init_device(self) -> None: def init_device(self) -> None:
"""Initialize device state, such as loading the model or other on-device """Initialize device state, such as loading the model or other on-device
memory allocations. memory allocations.