From 7ddb50788ddf0846518feff972028534e39bcb86 Mon Sep 17 00:00:00 2001 From: aoshen02 Date: Wed, 19 Aug 2026 04:49:35 +0800 Subject: [PATCH] [Bugfix][DP] Don't assume the engines started when forwarding a wake (#51481) Signed-off-by: aoshen02 Co-authored-by: Claude Opus 5 (1M context) --- tests/v1/distributed/test_async_llm_dp.py | 137 ++++++++++++++++++++++ vllm/v1/engine/coordinator.py | 6 +- vllm/v1/engine/core.py | 14 +++ 3 files changed, 155 insertions(+), 2 deletions(-) diff --git a/tests/v1/distributed/test_async_llm_dp.py b/tests/v1/distributed/test_async_llm_dp.py index 711daaef74..f51fde9053 100644 --- a/tests/v1/distributed/test_async_llm_dp.py +++ b/tests/v1/distributed/test_async_llm_dp.py @@ -299,6 +299,143 @@ async def test_dp_pause_resume_basic(expert_parallel: bool): assert out.finished +async def _consume(generator) -> None: + async for _ in generator: + pass + + +async def _poll_flag(engine: AsyncLLM, want: bool, timeout: float) -> bool: + """Wait for the front-end's view of the DP engines to reach ``want``.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if engine.engine_core.dp_engines_running() == want: + return True + await asyncio.sleep(0.05) + return False + + +@pytest.mark.asyncio +async def test_dp_pause_late_request_does_not_block_drain(): + """A request arriving after pause must not leave the coordinator believing + the engines are running. + + Paused engines discard START_DP_WAVE, so nothing can report wave + completion afterwards; if forwarding the wake also marks the engines as + running, drain has no way back and can only end in a timeout. + + MoE only: wave coordination is enabled iff the model is MoE, so a dense + model never reaches the coordinator state this exercises. + """ + with ExitStack() as after: + engine_args = _get_dp_pause_engine_args(expert_parallel=True) + engine = AsyncLLM.from_engine_args(engine_args) + after.callback(engine.shutdown) + + # Run a wave first, so the engines are quiesced by the pause rather + # than by never having started, and drain has something to observe. + long_request = asyncio.create_task( + _consume( + engine.generate( + request_id="warmup", + prompt=DP_PAUSE_PROMPT, + sampling_params=SamplingParams(max_tokens=400, ignore_eos=True), + ) + ) + ) + + # A design that never reports the engines as running would satisfy + # every drain assertion below while destroying the signal, so pin it + # down first. The front-end sets its own copy optimistically when it + # forwards the wake, so sample only after several coordinator + # publishes (every 100ms while stats change) have overwritten it. + await asyncio.sleep(2) + assert not long_request.done(), "the warmup request was too short to sample" + assert engine.engine_core.dp_engines_running(), ( + "the coordinator does not report the engines as running while they are" + ) + + await long_request + assert await _poll_flag(engine, False, timeout=30) + + await engine.pause_generation(mode="abort") + await engine.wait_for_requests_to_drain(drain_timeout=30) + + # Awaiting add_request guarantees the new-request notification has been + # sent to the coordinator - the message that used to latch the flag. + collector = await engine.add_request( + request_id="late", + prompt=DP_PAUSE_PROMPT, + params=SamplingParams(max_tokens=5), + ) + + # The front-end marks the engines running off the back of that + # notification. This is what makes the test non-vacuous: it is the + # path that used to leave the coordinator stuck. + assert await _poll_flag(engine, True, timeout=5), ( + "the late request did not notify the coordinator" + ) + + # It must settle back by itself. Unfixed it never does, because the + # paused engines discard the wake and so never report wave completion. + assert await _poll_flag(engine, False, timeout=60) + + # The late request was held rather than dropped: it completes on resume. + await engine.resume_generation() + while True: + out = await asyncio.wait_for(collector.get(), timeout=60) + if out.finished: + break + + +@pytest.mark.asyncio +async def test_dp_sleep_late_request_does_not_block_drain(): + """The same latch, reached through sleep rather than pause. + + Sleep stops the engines stepping just as pause does, so a request arriving + while they are asleep can leave the coordinator believing they are running + with nothing able to say otherwise. This is worth pinning separately from + the pause case because it is the shape that reaches + `_drain_requests_for_elastic_ep`, which decides from the same signal + whether it is safe to scale. + """ + with ExitStack() as after: + engine_args = _get_dp_pause_engine_args(expert_parallel=True) + engine = AsyncLLM.from_engine_args(engine_args) + after.callback(engine.shutdown) + + async for _ in engine.generate( + request_id="warmup", + prompt=DP_PAUSE_PROMPT, + sampling_params=SamplingParams(max_tokens=5), + ): + pass + assert await _poll_flag(engine, False, timeout=30) + + await engine.sleep(level=1) + assert await engine.is_sleeping() + + collector = await engine.add_request( + request_id="while-asleep", + prompt=DP_PAUSE_PROMPT, + params=SamplingParams(max_tokens=5), + ) + + assert await _poll_flag(engine, True, timeout=5), ( + "the request did not notify the coordinator" + ) + + # Sleeping engines cannot report wave completion, so a coordinator that + # marked them running when it forwarded the wake never hears otherwise. + assert await _poll_flag(engine, False, timeout=60) + + await engine.wake_up() + assert not await engine.is_sleeping() + while True: + out = await asyncio.wait_for(collector.get(), timeout=60) + if out.finished: + break + + @pytest.mark.asyncio @pytest.mark.parametrize("expert_parallel", [False, True]) async def test_dp_pause_abort(expert_parallel: bool): diff --git a/vllm/v1/engine/coordinator.py b/vllm/v1/engine/coordinator.py index d7f05cffc8..efadbfea26 100644 --- a/vllm/v1/engine/coordinator.py +++ b/vllm/v1/engine/coordinator.py @@ -358,8 +358,10 @@ class DPCoordinatorProc: # is handled by all the engines. engine_to_exclude = None - engines_running = True - wave_state_changed = True + # engines_running is only set from the engines' + # own notifications; a paused engine discards + # START_DP_WAVE, so sending it is not evidence + # that the engines are running. self._send_start_wave( publish_back, current_wave, engine_to_exclude ) diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index 641c5ecc4e..71a791d721 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -2156,6 +2156,7 @@ class DPEngineCoreProc(EngineCoreProc): # Loop until process is sent a SIGINT or SIGTERM while self._handle_shutdown(): # 1) Poll the input queue until there is work to do. + was_running = self.engines_running self._process_input_queue() # Publish request counts before and after GPU step to ensure freshness. self._maybe_publish_request_counts() @@ -2216,6 +2217,19 @@ class DPEngineCoreProc(EngineCoreProc): # Increment wave count and reset step counter. self.current_wave += 1 self.step_counter = 0 + elif ( + not was_running + and self.has_coordinator + and self.dp_rank == 0 + and not self.pending_pause + ): + # Mirror of the wave_complete notification above: the + # coordinator must observe this edge too rather than assume + # that a START_DP_WAVE it sent was acted upon, since a paused + # engine discards it. + self.output_queue.put_nowait( + (-1, EngineCoreOutputs(start_wave=self.current_wave)) + ) raise SystemExit