[DP] Engines own wave wake-ups; drop the front-end FIRST_REQ (#59559)
Signed-off-by: fh3 <a@b> Co-authored-by: fh3 <a@b> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
fh3
Claude Opus 5.5
parent
b544587408
commit
a94c315711
@@ -18,6 +18,7 @@ from vllm.inputs import PromptType
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sampling_params import RequestOutputKind
|
||||
from vllm.v1.engine import EngineCoreRequestType
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.engine.core_client import DPLBAsyncMPClient
|
||||
from vllm.v1.metrics.loggers import StatLoggerBase
|
||||
@@ -336,12 +337,12 @@ async def _poll_flag(engine: AsyncLLM, want: bool, timeout: float) -> bool:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dp_pause_late_request_does_not_block_drain():
|
||||
"""A request arriving after pause must not leave the coordinator believing
|
||||
"""A request arriving after pause must not make the coordinator believe
|
||||
the engines are running.
|
||||
|
||||
Paused engines discard START_DP_WAVE, so nothing can report wave
|
||||
completion afterwards; if forwarding the wake also marks the engines as
|
||||
running, drain has no way back and can only end in a timeout.
|
||||
Paused engines start no wave for it, so nothing could report wave
|
||||
completion afterwards; a running flag set on its behalf would leave drain
|
||||
no way back but a timeout.
|
||||
|
||||
MoE only: wave coordination is enabled iff the model is MoE, so a dense
|
||||
model never reaches the coordinator state this exercises.
|
||||
@@ -365,11 +366,9 @@ async def test_dp_pause_late_request_does_not_block_drain():
|
||||
|
||||
# A design that never reports the engines as running would satisfy
|
||||
# every drain assertion below while destroying the signal, so pin it
|
||||
# down first. The front-end sets its own copy optimistically when it
|
||||
# forwards the wake, so sample only after several coordinator
|
||||
# publishes (every 100ms while stats change) have overwritten it.
|
||||
# Rank 0 reports the wave only after its first step, which can outlast
|
||||
# the 2s on a cold device, so wait for the report rather than sample it.
|
||||
# down first. Rank 0 reports the wave only after its first step, which
|
||||
# can outlast the 2s on a cold device, so wait for the report rather
|
||||
# than sample it.
|
||||
await asyncio.sleep(2)
|
||||
assert not long_request.done(), "the warmup request was too short to sample"
|
||||
assert await _poll_flag(engine, True, timeout=30), (
|
||||
@@ -382,24 +381,12 @@ async def test_dp_pause_late_request_does_not_block_drain():
|
||||
await engine.pause_generation(mode="abort")
|
||||
await engine.wait_for_requests_to_drain(drain_timeout=30)
|
||||
|
||||
# Awaiting add_request guarantees the new-request notification has been
|
||||
# sent to the coordinator - the message that used to latch the flag.
|
||||
collector = await engine.add_request(
|
||||
request_id="late",
|
||||
prompt=DP_PAUSE_PROMPT,
|
||||
params=SamplingParams(max_tokens=5),
|
||||
)
|
||||
|
||||
# The front-end marks the engines running off the back of that
|
||||
# notification. This is what makes the test non-vacuous: it is the
|
||||
# path that used to leave the coordinator stuck.
|
||||
assert await _poll_flag(engine, True, timeout=5), (
|
||||
"the late request did not notify the coordinator"
|
||||
)
|
||||
|
||||
# It must settle back by itself. Unfixed it never does, because the
|
||||
# paused engines discard the wake and so never report wave completion.
|
||||
assert await _poll_flag(engine, False, timeout=60)
|
||||
assert not await _poll_flag(engine, True, timeout=5)
|
||||
|
||||
# The late request was held rather than dropped: it completes on resume.
|
||||
await engine.resume_generation()
|
||||
@@ -409,6 +396,46 @@ async def test_dp_pause_late_request_does_not_block_drain():
|
||||
break
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dp_request_reaching_rank_after_resume_wakes_peers():
|
||||
"""A request can reach its rank only after resume (e.g. from another API
|
||||
server), with no other work anywhere. That rank must wake its peers itself,
|
||||
or it steps alone. Modelled here by holding the request's ADD on the client
|
||||
side across pause and resume."""
|
||||
with ExitStack() as after:
|
||||
engine = AsyncLLM.from_engine_args(
|
||||
_get_dp_pause_engine_args(expert_parallel=True)
|
||||
)
|
||||
after.callback(engine.shutdown)
|
||||
client = engine.engine_core
|
||||
params = SamplingParams(max_tokens=5)
|
||||
|
||||
await _consume(engine.generate(DP_PAUSE_PROMPT, params, "warmup"))
|
||||
await engine.pause_generation(mode="abort")
|
||||
assert await _poll_flag(engine, False, timeout=30)
|
||||
|
||||
held: list[tuple] = []
|
||||
send_input = client._send_input
|
||||
|
||||
def hold_add(request_type, request, engine_id=None):
|
||||
if request_type == EngineCoreRequestType.ADD:
|
||||
held.append((request_type, request, engine_id))
|
||||
return asyncio.sleep(0)
|
||||
return send_input(request_type, request, engine_id)
|
||||
|
||||
client._send_input = hold_add
|
||||
task = asyncio.create_task(
|
||||
_consume(engine.generate(DP_PAUSE_PROMPT, params, "late"))
|
||||
)
|
||||
await asyncio.sleep(2)
|
||||
assert held
|
||||
|
||||
await engine.resume_generation()
|
||||
client._send_input = send_input
|
||||
await send_input(*held[0])
|
||||
await asyncio.wait_for(task, timeout=60)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dp_sleep_late_request_does_not_block_drain():
|
||||
"""The same latch, reached through sleep rather than pause.
|
||||
@@ -441,14 +468,7 @@ async def test_dp_sleep_late_request_does_not_block_drain():
|
||||
prompt=DP_PAUSE_PROMPT,
|
||||
params=SamplingParams(max_tokens=5),
|
||||
)
|
||||
|
||||
assert await _poll_flag(engine, True, timeout=5), (
|
||||
"the request did not notify the coordinator"
|
||||
)
|
||||
|
||||
# Sleeping engines cannot report wave completion, so a coordinator that
|
||||
# marked them running when it forwarded the wake never hears otherwise.
|
||||
assert await _poll_flag(engine, False, timeout=60)
|
||||
assert not await _poll_flag(engine, True, timeout=5)
|
||||
|
||||
await engine.wake_up()
|
||||
assert not await engine.is_sleeping()
|
||||
@@ -628,7 +648,7 @@ async def test_dp_pause_keep_race_staggered_engines():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dp_pause_barrier_request_deadlock():
|
||||
"""Test that start_dp_wave is ignored while paused.
|
||||
"""Test that a request sent while paused does not wake any engine.
|
||||
|
||||
Sequence:
|
||||
1. Pause all engines (PAUSED_ALL).
|
||||
@@ -697,7 +717,7 @@ async def test_dp_pause_barrier_request_deadlock():
|
||||
|
||||
# Yield so generate() preprocessing completes and
|
||||
# add_request_async is called (which, in buggy code,
|
||||
# would send FIRST_REQ and wake engine 1).
|
||||
# would wake engine 1).
|
||||
for _ in range(200):
|
||||
await asyncio.sleep(0)
|
||||
|
||||
@@ -725,7 +745,7 @@ async def test_dp_pause_barrier_request_deadlock():
|
||||
for t in mid_barrier_tasks:
|
||||
t.cancel()
|
||||
pytest.fail(
|
||||
"Staggered barrier deadlocked — FIRST_REQ sent while "
|
||||
"Staggered barrier deadlocked — a request sent while "
|
||||
"paused caused collective-op mismatch between engines"
|
||||
)
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_default_torch_num_threads
|
||||
from vllm.v1.core.sched.interface import PauseState
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.engine import EngineCoreRequest, EngineCoreRequestType
|
||||
from vllm.v1.engine.core import DPEngineCoreProc, EngineCore, EngineCoreProc
|
||||
from vllm.v1.executor.abstract import Executor
|
||||
from vllm.v1.executor.uniproc_executor import UniProcExecutor
|
||||
@@ -689,6 +689,54 @@ def test_dp_sync_interval_idle_pause_consensus_on_first_step(monkeypatch):
|
||||
assert not core.pending_pause
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"request_wave,engines_running,pause_state,announces",
|
||||
[
|
||||
pytest.param(3, False, PauseState.UNPAUSED, True, id="current-wave"),
|
||||
pytest.param(2, False, PauseState.UNPAUSED, True, id="stale-wave"),
|
||||
pytest.param(4, False, PauseState.UNPAUSED, True, id="newer-wave"),
|
||||
pytest.param(3, True, PauseState.UNPAUSED, False, id="running"),
|
||||
pytest.param(3, False, PauseState.PAUSED_ALL, False, id="paused"),
|
||||
],
|
||||
)
|
||||
def test_idle_dp_rank_announces_wave_for_new_work(
|
||||
request_wave, engines_running, pause_state, announces
|
||||
):
|
||||
"""An idle rank handed work announces the wave itself, since only engines
|
||||
start waves; a paused rank waits for resume."""
|
||||
core = object.__new__(DPEngineCoreProc)
|
||||
core.has_coordinator = True
|
||||
core.current_wave = 3
|
||||
core.engines_running = engines_running
|
||||
core.scheduler = MagicMock(pause_state=pause_state)
|
||||
core.output_queue = MagicMock()
|
||||
|
||||
with patch.object(EngineCore, "add_request"):
|
||||
core.add_request(MagicMock(), request_wave)
|
||||
|
||||
if announces:
|
||||
assert core.engines_running
|
||||
_, outputs = core.output_queue.put_nowait.call_args.args[0]
|
||||
assert outputs.start_wave == max(request_wave, 3)
|
||||
else:
|
||||
core.output_queue.put_nowait.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("paused", [False, True])
|
||||
def test_paused_dp_rank_ignores_start_wave(paused):
|
||||
"""A START_DP_WAVE still in flight when the pause completes must not wake
|
||||
the paused rank."""
|
||||
core = object.__new__(DPEngineCoreProc)
|
||||
core.ignore_start_dp_wave = paused
|
||||
core.engine_index = 0
|
||||
core.current_wave = 3
|
||||
core.engines_running = False
|
||||
|
||||
core._handle_client_request(EngineCoreRequestType.START_DP_WAVE, (3, 1))
|
||||
|
||||
assert core.engines_running != paused
|
||||
|
||||
|
||||
def _pausable_engine_core_proc() -> EngineCoreProc:
|
||||
"""A bare EngineCoreProc holding just the state pause_scheduler touches."""
|
||||
core = object.__new__(EngineCoreProc)
|
||||
|
||||
@@ -314,7 +314,7 @@ async def test_dplb_scale_down_routes_after_stale_stats_snapshot():
|
||||
client.current_wave = 0
|
||||
client.resources = SimpleNamespace(stats_update_task=None)
|
||||
client.stats_update_address = "inproc://scale-down-stats"
|
||||
client.first_req_sock_addr = "inproc://scale-down-first-request"
|
||||
client.scale_sock_addr = "inproc://scale-down-notify"
|
||||
pause_started = asyncio.Event()
|
||||
|
||||
async def pause_scheduler(*args, **kwargs):
|
||||
@@ -326,11 +326,11 @@ async def test_dplb_scale_down_routes_after_stale_stats_snapshot():
|
||||
with (
|
||||
zmq.asyncio.Context() as ctx,
|
||||
ctx.socket(zmq.XPUB) as coordinator,
|
||||
ctx.socket(zmq.PAIR) as first_req_socket,
|
||||
ctx.socket(zmq.PAIR) as scale_socket,
|
||||
):
|
||||
client.ctx = ctx
|
||||
coordinator.bind(client.stats_update_address)
|
||||
first_req_socket.bind(client.first_req_sock_addr)
|
||||
scale_socket.bind(client.scale_sock_addr)
|
||||
client._ensure_stats_update_task()
|
||||
scale_task = None
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user