[DP] Engines own wave wake-ups; drop the front-end FIRST_REQ (#59559)

Signed-off-by: fh3 <a@b>
Co-authored-by: fh3 <a@b>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-16 19:41:54 -04:00
committed by GitHub
co-authored by fh3 Claude Opus 5.5
parent b544587408
commit a94c315711
13 changed files with 226 additions and 523 deletions
+53 -33
View File
@@ -18,6 +18,7 @@ from vllm.inputs import PromptType
from vllm.outputs import RequestOutput
from vllm.platforms import current_platform
from vllm.sampling_params import RequestOutputKind
from vllm.v1.engine import EngineCoreRequestType
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.engine.core_client import DPLBAsyncMPClient
from vllm.v1.metrics.loggers import StatLoggerBase
@@ -336,12 +337,12 @@ async def _poll_flag(engine: AsyncLLM, want: bool, timeout: float) -> bool:
@pytest.mark.asyncio
async def test_dp_pause_late_request_does_not_block_drain():
"""A request arriving after pause must not leave the coordinator believing
"""A request arriving after pause must not make the coordinator believe
the engines are running.
Paused engines discard START_DP_WAVE, so nothing can report wave
completion afterwards; if forwarding the wake also marks the engines as
running, drain has no way back and can only end in a timeout.
Paused engines start no wave for it, so nothing could report wave
completion afterwards; a running flag set on its behalf would leave drain
no way back but a timeout.
MoE only: wave coordination is enabled iff the model is MoE, so a dense
model never reaches the coordinator state this exercises.
@@ -365,11 +366,9 @@ async def test_dp_pause_late_request_does_not_block_drain():
# A design that never reports the engines as running would satisfy
# every drain assertion below while destroying the signal, so pin it
# down first. The front-end sets its own copy optimistically when it
# forwards the wake, so sample only after several coordinator
# publishes (every 100ms while stats change) have overwritten it.
# Rank 0 reports the wave only after its first step, which can outlast
# the 2s on a cold device, so wait for the report rather than sample it.
# down first. Rank 0 reports the wave only after its first step, which
# can outlast the 2s on a cold device, so wait for the report rather
# than sample it.
await asyncio.sleep(2)
assert not long_request.done(), "the warmup request was too short to sample"
assert await _poll_flag(engine, True, timeout=30), (
@@ -382,24 +381,12 @@ async def test_dp_pause_late_request_does_not_block_drain():
await engine.pause_generation(mode="abort")
await engine.wait_for_requests_to_drain(drain_timeout=30)
# Awaiting add_request guarantees the new-request notification has been
# sent to the coordinator - the message that used to latch the flag.
collector = await engine.add_request(
request_id="late",
prompt=DP_PAUSE_PROMPT,
params=SamplingParams(max_tokens=5),
)
# The front-end marks the engines running off the back of that
# notification. This is what makes the test non-vacuous: it is the
# path that used to leave the coordinator stuck.
assert await _poll_flag(engine, True, timeout=5), (
"the late request did not notify the coordinator"
)
# It must settle back by itself. Unfixed it never does, because the
# paused engines discard the wake and so never report wave completion.
assert await _poll_flag(engine, False, timeout=60)
assert not await _poll_flag(engine, True, timeout=5)
# The late request was held rather than dropped: it completes on resume.
await engine.resume_generation()
@@ -409,6 +396,46 @@ async def test_dp_pause_late_request_does_not_block_drain():
break
@pytest.mark.asyncio
async def test_dp_request_reaching_rank_after_resume_wakes_peers():
"""A request can reach its rank only after resume (e.g. from another API
server), with no other work anywhere. That rank must wake its peers itself,
or it steps alone. Modelled here by holding the request's ADD on the client
side across pause and resume."""
with ExitStack() as after:
engine = AsyncLLM.from_engine_args(
_get_dp_pause_engine_args(expert_parallel=True)
)
after.callback(engine.shutdown)
client = engine.engine_core
params = SamplingParams(max_tokens=5)
await _consume(engine.generate(DP_PAUSE_PROMPT, params, "warmup"))
await engine.pause_generation(mode="abort")
assert await _poll_flag(engine, False, timeout=30)
held: list[tuple] = []
send_input = client._send_input
def hold_add(request_type, request, engine_id=None):
if request_type == EngineCoreRequestType.ADD:
held.append((request_type, request, engine_id))
return asyncio.sleep(0)
return send_input(request_type, request, engine_id)
client._send_input = hold_add
task = asyncio.create_task(
_consume(engine.generate(DP_PAUSE_PROMPT, params, "late"))
)
await asyncio.sleep(2)
assert held
await engine.resume_generation()
client._send_input = send_input
await send_input(*held[0])
await asyncio.wait_for(task, timeout=60)
@pytest.mark.asyncio
async def test_dp_sleep_late_request_does_not_block_drain():
"""The same latch, reached through sleep rather than pause.
@@ -441,14 +468,7 @@ async def test_dp_sleep_late_request_does_not_block_drain():
prompt=DP_PAUSE_PROMPT,
params=SamplingParams(max_tokens=5),
)
assert await _poll_flag(engine, True, timeout=5), (
"the request did not notify the coordinator"
)
# Sleeping engines cannot report wave completion, so a coordinator that
# marked them running when it forwarded the wake never hears otherwise.
assert await _poll_flag(engine, False, timeout=60)
assert not await _poll_flag(engine, True, timeout=5)
await engine.wake_up()
assert not await engine.is_sleeping()
@@ -628,7 +648,7 @@ async def test_dp_pause_keep_race_staggered_engines():
@pytest.mark.asyncio
async def test_dp_pause_barrier_request_deadlock():
"""Test that start_dp_wave is ignored while paused.
"""Test that a request sent while paused does not wake any engine.
Sequence:
1. Pause all engines (PAUSED_ALL).
@@ -697,7 +717,7 @@ async def test_dp_pause_barrier_request_deadlock():
# Yield so generate() preprocessing completes and
# add_request_async is called (which, in buggy code,
# would send FIRST_REQ and wake engine 1).
# would wake engine 1).
for _ in range(200):
await asyncio.sleep(0)
@@ -725,7 +745,7 @@ async def test_dp_pause_barrier_request_deadlock():
for t in mid_barrier_tasks:
t.cancel()
pytest.fail(
"Staggered barrier deadlocked — FIRST_REQ sent while "
"Staggered barrier deadlocked — a request sent while "
"paused caused collective-op mismatch between engines"
)
+49 -1
View File
@@ -25,7 +25,7 @@ from vllm.engine.arg_utils import EngineArgs
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_default_torch_num_threads
from vllm.v1.core.sched.interface import PauseState
from vllm.v1.engine import EngineCoreRequest
from vllm.v1.engine import EngineCoreRequest, EngineCoreRequestType
from vllm.v1.engine.core import DPEngineCoreProc, EngineCore, EngineCoreProc
from vllm.v1.executor.abstract import Executor
from vllm.v1.executor.uniproc_executor import UniProcExecutor
@@ -689,6 +689,54 @@ def test_dp_sync_interval_idle_pause_consensus_on_first_step(monkeypatch):
assert not core.pending_pause
@pytest.mark.parametrize(
"request_wave,engines_running,pause_state,announces",
[
pytest.param(3, False, PauseState.UNPAUSED, True, id="current-wave"),
pytest.param(2, False, PauseState.UNPAUSED, True, id="stale-wave"),
pytest.param(4, False, PauseState.UNPAUSED, True, id="newer-wave"),
pytest.param(3, True, PauseState.UNPAUSED, False, id="running"),
pytest.param(3, False, PauseState.PAUSED_ALL, False, id="paused"),
],
)
def test_idle_dp_rank_announces_wave_for_new_work(
request_wave, engines_running, pause_state, announces
):
"""An idle rank handed work announces the wave itself, since only engines
start waves; a paused rank waits for resume."""
core = object.__new__(DPEngineCoreProc)
core.has_coordinator = True
core.current_wave = 3
core.engines_running = engines_running
core.scheduler = MagicMock(pause_state=pause_state)
core.output_queue = MagicMock()
with patch.object(EngineCore, "add_request"):
core.add_request(MagicMock(), request_wave)
if announces:
assert core.engines_running
_, outputs = core.output_queue.put_nowait.call_args.args[0]
assert outputs.start_wave == max(request_wave, 3)
else:
core.output_queue.put_nowait.assert_not_called()
@pytest.mark.parametrize("paused", [False, True])
def test_paused_dp_rank_ignores_start_wave(paused):
"""A START_DP_WAVE still in flight when the pause completes must not wake
the paused rank."""
core = object.__new__(DPEngineCoreProc)
core.ignore_start_dp_wave = paused
core.engine_index = 0
core.current_wave = 3
core.engines_running = False
core._handle_client_request(EngineCoreRequestType.START_DP_WAVE, (3, 1))
assert core.engines_running != paused
def _pausable_engine_core_proc() -> EngineCoreProc:
"""A bare EngineCoreProc holding just the state pause_scheduler touches."""
core = object.__new__(EngineCoreProc)
+3 -3
View File
@@ -314,7 +314,7 @@ async def test_dplb_scale_down_routes_after_stale_stats_snapshot():
client.current_wave = 0
client.resources = SimpleNamespace(stats_update_task=None)
client.stats_update_address = "inproc://scale-down-stats"
client.first_req_sock_addr = "inproc://scale-down-first-request"
client.scale_sock_addr = "inproc://scale-down-notify"
pause_started = asyncio.Event()
async def pause_scheduler(*args, **kwargs):
@@ -326,11 +326,11 @@ async def test_dplb_scale_down_routes_after_stale_stats_snapshot():
with (
zmq.asyncio.Context() as ctx,
ctx.socket(zmq.XPUB) as coordinator,
ctx.socket(zmq.PAIR) as first_req_socket,
ctx.socket(zmq.PAIR) as scale_socket,
):
client.ctx = ctx
coordinator.bind(client.stats_update_address)
first_req_socket.bind(client.first_req_sock_addr)
scale_socket.bind(client.scale_sock_addr)
client._ensure_stats_update_task()
scale_task = None
try: