[Bugfix][DP] Don't assume the engines started when forwarding a wake (#51481)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
6a391a931a
commit
7ddb50788d
@@ -299,6 +299,143 @@ async def test_dp_pause_resume_basic(expert_parallel: bool):
|
||||
assert out.finished
|
||||
|
||||
|
||||
async def _consume(generator) -> None:
|
||||
async for _ in generator:
|
||||
pass
|
||||
|
||||
|
||||
async def _poll_flag(engine: AsyncLLM, want: bool, timeout: float) -> bool:
|
||||
"""Wait for the front-end's view of the DP engines to reach ``want``."""
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if engine.engine_core.dp_engines_running() == want:
|
||||
return True
|
||||
await asyncio.sleep(0.05)
|
||||
return False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dp_pause_late_request_does_not_block_drain():
|
||||
"""A request arriving after pause must not leave the coordinator believing
|
||||
the engines are running.
|
||||
|
||||
Paused engines discard START_DP_WAVE, so nothing can report wave
|
||||
completion afterwards; if forwarding the wake also marks the engines as
|
||||
running, drain has no way back and can only end in a timeout.
|
||||
|
||||
MoE only: wave coordination is enabled iff the model is MoE, so a dense
|
||||
model never reaches the coordinator state this exercises.
|
||||
"""
|
||||
with ExitStack() as after:
|
||||
engine_args = _get_dp_pause_engine_args(expert_parallel=True)
|
||||
engine = AsyncLLM.from_engine_args(engine_args)
|
||||
after.callback(engine.shutdown)
|
||||
|
||||
# Run a wave first, so the engines are quiesced by the pause rather
|
||||
# than by never having started, and drain has something to observe.
|
||||
long_request = asyncio.create_task(
|
||||
_consume(
|
||||
engine.generate(
|
||||
request_id="warmup",
|
||||
prompt=DP_PAUSE_PROMPT,
|
||||
sampling_params=SamplingParams(max_tokens=400, ignore_eos=True),
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
# A design that never reports the engines as running would satisfy
|
||||
# every drain assertion below while destroying the signal, so pin it
|
||||
# down first. The front-end sets its own copy optimistically when it
|
||||
# forwards the wake, so sample only after several coordinator
|
||||
# publishes (every 100ms while stats change) have overwritten it.
|
||||
await asyncio.sleep(2)
|
||||
assert not long_request.done(), "the warmup request was too short to sample"
|
||||
assert engine.engine_core.dp_engines_running(), (
|
||||
"the coordinator does not report the engines as running while they are"
|
||||
)
|
||||
|
||||
await long_request
|
||||
assert await _poll_flag(engine, False, timeout=30)
|
||||
|
||||
await engine.pause_generation(mode="abort")
|
||||
await engine.wait_for_requests_to_drain(drain_timeout=30)
|
||||
|
||||
# Awaiting add_request guarantees the new-request notification has been
|
||||
# sent to the coordinator - the message that used to latch the flag.
|
||||
collector = await engine.add_request(
|
||||
request_id="late",
|
||||
prompt=DP_PAUSE_PROMPT,
|
||||
params=SamplingParams(max_tokens=5),
|
||||
)
|
||||
|
||||
# The front-end marks the engines running off the back of that
|
||||
# notification. This is what makes the test non-vacuous: it is the
|
||||
# path that used to leave the coordinator stuck.
|
||||
assert await _poll_flag(engine, True, timeout=5), (
|
||||
"the late request did not notify the coordinator"
|
||||
)
|
||||
|
||||
# It must settle back by itself. Unfixed it never does, because the
|
||||
# paused engines discard the wake and so never report wave completion.
|
||||
assert await _poll_flag(engine, False, timeout=60)
|
||||
|
||||
# The late request was held rather than dropped: it completes on resume.
|
||||
await engine.resume_generation()
|
||||
while True:
|
||||
out = await asyncio.wait_for(collector.get(), timeout=60)
|
||||
if out.finished:
|
||||
break
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dp_sleep_late_request_does_not_block_drain():
|
||||
"""The same latch, reached through sleep rather than pause.
|
||||
|
||||
Sleep stops the engines stepping just as pause does, so a request arriving
|
||||
while they are asleep can leave the coordinator believing they are running
|
||||
with nothing able to say otherwise. This is worth pinning separately from
|
||||
the pause case because it is the shape that reaches
|
||||
`_drain_requests_for_elastic_ep`, which decides from the same signal
|
||||
whether it is safe to scale.
|
||||
"""
|
||||
with ExitStack() as after:
|
||||
engine_args = _get_dp_pause_engine_args(expert_parallel=True)
|
||||
engine = AsyncLLM.from_engine_args(engine_args)
|
||||
after.callback(engine.shutdown)
|
||||
|
||||
async for _ in engine.generate(
|
||||
request_id="warmup",
|
||||
prompt=DP_PAUSE_PROMPT,
|
||||
sampling_params=SamplingParams(max_tokens=5),
|
||||
):
|
||||
pass
|
||||
assert await _poll_flag(engine, False, timeout=30)
|
||||
|
||||
await engine.sleep(level=1)
|
||||
assert await engine.is_sleeping()
|
||||
|
||||
collector = await engine.add_request(
|
||||
request_id="while-asleep",
|
||||
prompt=DP_PAUSE_PROMPT,
|
||||
params=SamplingParams(max_tokens=5),
|
||||
)
|
||||
|
||||
assert await _poll_flag(engine, True, timeout=5), (
|
||||
"the request did not notify the coordinator"
|
||||
)
|
||||
|
||||
# Sleeping engines cannot report wave completion, so a coordinator that
|
||||
# marked them running when it forwarded the wake never hears otherwise.
|
||||
assert await _poll_flag(engine, False, timeout=60)
|
||||
|
||||
await engine.wake_up()
|
||||
assert not await engine.is_sleeping()
|
||||
while True:
|
||||
out = await asyncio.wait_for(collector.get(), timeout=60)
|
||||
if out.finished:
|
||||
break
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("expert_parallel", [False, True])
|
||||
async def test_dp_pause_abort(expert_parallel: bool):
|
||||
|
||||
@@ -358,8 +358,10 @@ class DPCoordinatorProc:
|
||||
# is handled by all the engines.
|
||||
engine_to_exclude = None
|
||||
|
||||
engines_running = True
|
||||
wave_state_changed = True
|
||||
# engines_running is only set from the engines'
|
||||
# own notifications; a paused engine discards
|
||||
# START_DP_WAVE, so sending it is not evidence
|
||||
# that the engines are running.
|
||||
self._send_start_wave(
|
||||
publish_back, current_wave, engine_to_exclude
|
||||
)
|
||||
|
||||
@@ -2156,6 +2156,7 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
# Loop until process is sent a SIGINT or SIGTERM
|
||||
while self._handle_shutdown():
|
||||
# 1) Poll the input queue until there is work to do.
|
||||
was_running = self.engines_running
|
||||
self._process_input_queue()
|
||||
# Publish request counts before and after GPU step to ensure freshness.
|
||||
self._maybe_publish_request_counts()
|
||||
@@ -2216,6 +2217,19 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
# Increment wave count and reset step counter.
|
||||
self.current_wave += 1
|
||||
self.step_counter = 0
|
||||
elif (
|
||||
not was_running
|
||||
and self.has_coordinator
|
||||
and self.dp_rank == 0
|
||||
and not self.pending_pause
|
||||
):
|
||||
# Mirror of the wave_complete notification above: the
|
||||
# coordinator must observe this edge too rather than assume
|
||||
# that a START_DP_WAVE it sent was acted upon, since a paused
|
||||
# engine discards it.
|
||||
self.output_queue.put_nowait(
|
||||
(-1, EngineCoreOutputs(start_wave=self.current_wave))
|
||||
)
|
||||
|
||||
raise SystemExit
|
||||
|
||||
|
||||
Reference in New Issue
Block a user