ple, tests: debug-gated pull-flag check; diagnostics report that they ran

The pull-transport flag check is wired in (rank-gated, off by default, warns when enabled). Both
diagnostic checks log that they ran. Test repairs for helpers v0.30.0 renamed.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 20:01:26 +07:00
co-authored by Hinata
parent fb52250c11
commit 7f6055caa2
2 changed files with 236 additions and 0 deletions
@@ -476,3 +476,88 @@ class PleOffloadLayer(nn.Module, ABC):
"""Mark the cross-process output buffer reusable on ``stream``."""
if self._is_cpu_offloaded:
self._sem.reset(stream)
# ---------------------------------------------------------------------------
# Pre-launch staleness check for the pull transport (#53899 class)
# ---------------------------------------------------------------------------
# WHAT IT TESTS. The transport's correctness rests on one precondition: when a
# real request is published, every layer's semaphore must be CLEAR, so the
# ``cuStreamWaitValue32(flag == DONE_VALUE)`` inside the forward (line 279 for
# the pull path, line 215 for the IPC push path) can only be satisfied by the
# copy THIS step publishes. If a leftover ``DONE_VALUE`` survives into a real
# step, that wait is satisfied immediately by the PREVIOUS step's signal and
# the forward consumes the previous step's PLE rows. That is vllm#53899 on
# GB10/sm_121 exactly: "every forward consumes the previous step's PLE
# outputs", for the life of the server, with ``cudagraph_mode=NONE`` correct.
#
# In this tree the precondition is held structurally -- the dummy signal is
# consumed by the dummy forward, ``_release_outputs_impl`` resets every layer
# and synchronizes the model stream while ``_after_dummy`` is set, and the
# dummy->real transition takes a TP barrier -- but nothing ever READ the flag
# back, so the structural argument had no instrument. This is that instrument.
#
# WHY NOT AN EPOCH WORD. The stronger fix is a monotone epoch: the launch
# writes ``epoch = step``, the worker signals ``epoch``, and the placeholder
# waits ``EQ(epoch)``, so a stale signal carries the previous epoch and cannot
# satisfy the wait. It is not implementable on this transport as it stands.
# The compare value is an IMMEDIATE OPERAND of ``cuStreamWaitValue32``
# (``_ple_offload_wait_copy_impl``, this file, line 279), and that call is
# issued inside the model forward, which is captured: PIECEWISE and
# FULL_AND_PIECEWISE both bake the wait into the graph with the value frozen at
# capture time. Making the compare value change per replay needs the captured
# node's parameters rewritten before each launch
# (``cuGraphExecBatchMemOpNodeSetParams``); nothing in this tree's capture path
# exposes the graph exec handle to the connector, and the wait is reached
# through a torch custom op inside a compiled region. An epoch word would
# therefore work only with ``cudagraph_mode=NONE``, i.e. only in the
# configuration where #53899 does not reproduce. So: binary flag, read back.
#
# The check is HOST-SIDE and needs a stream synchronize to be ordered against
# a pending reset, so it is never run inside a capture and is off by default.
# ---------------------------------------------------------------------------
def flag_check_is_authoritative(tp_rank: int) -> bool:
"""Whether ``flag == RESET_VALUE`` before a launch is an INVARIANT here.
Only on the publishing rank (TP0). Why: the offload worker fans a
result out to every registered TP target, each gated only on that target's
own flag having been reset, so a non-publishing rank's flag can legitimately
carry THIS step's signal before that rank reaches its own pre-launch check.
Reading a set flag there and calling it stale terminates a healthy run.
On TP0 the invariant holds by construction: the worker learns of a request
only from ``_launch``, which runs after the check, and the progress monitor
refuses overlapping serving submissions, so the previous transaction has
been released and (after a stream synchronize) its reset has retired.
A non-publishing rank cannot tell "this step's fan-out arrived early" from
"last step's signal was never consumed", because the flag is BINARY and
carries no transaction identity -- the same representational limit that
rules out an epoch word. So those ranks are skipped, not checked weakly.
"""
return tp_rank == 0
def stale_offload_flags(
layers: "dict[str, PleOffloadLayer]",
) -> list[tuple[str, int]]:
"""Return ``(layer_name, flag_value)`` for every layer whose flag is set.
An empty list is the precondition holding. A non-empty list means at least
one semaphore still carries a signal that no forward consumed, so the next
``wait`` would be satisfied by the previous step's copy.
Pure and side-effect free: the caller owns the ordering (synchronize the
model stream first, and never call this inside a stream capture). Layers
that are not cross-process offloaded have no semaphore and are skipped.
"""
stale: list[tuple[str, int]] = []
for name, layer in layers.items():
if not getattr(layer, "_is_cpu_offloaded", False):
continue
value = int(layer._sem.flag_tensor.reshape(-1)[0].item())
if value != CpuGpuSemaphore.RESET_VALUE:
stale.append((name, value))
return stale
+151
View File
@@ -18,12 +18,15 @@ import torch.nn as nn
import zmq
from cuda.bindings import driver as cuda_driver
import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.distributed.parallel_state import get_dp_group, get_pp_group, get_tp_group
from vllm.logger import init_logger
from vllm.model_executor.layers.ple_offload_layer import (
CpuGpuSemaphore,
PleOffloadLayer,
flag_check_is_authoritative,
stale_offload_flags,
)
from vllm.v1.ple_offload.protocol import (
PleOffloadRegistration,
@@ -34,6 +37,30 @@ from vllm_ple_test_fault import _FAULT
logger = init_logger(__name__)
# PLE diagnostics / vllm#53899 class: read the pull-transport semaphore back before a real
# launch and refuse to publish into a flag that is already set. Costs a
# model-stream synchronize, so it is a diagnostic, not a hot-path guard, and is
# OFF by default -- the same split VLLM_E2_ZERO_OWNERSHIP_CHECK established in
# the scheduler (VLLM_E2_ZERO_OWNERSHIP_CHECK).
#
# off | warn | raise (1 is accepted as warn). Default off. The default action on
# a violation is WARN, not raise: this diagnostic could
# terminate a healthy run, and a diagnostic that kills the qualification boot
# intended to establish safety is worse than no diagnostic. raise is for a
# targeted repro.
#
# Read through the envs registry, not os.getenv: the registry owns the
# normalization, so the value reported by
# `vllm.envs` and the value this check applies cannot drift apart. It is
# registered as a non-compile factor there. A set-but-unrecognised value
# raises on first read -- fail closed, because a broken gate that silently
# disarms its own instrument is the failure mode these checks exist to avoid.
_PLE_FLAG_CHECK_MODE: str = envs.VLLM_E2_PLE_FLAG_CHECK
# Liveness cadence for the diagnostic checks: the first check and every Nth
# are logged with a running count, so "zero findings" can be told apart from
# "never ran" in a reference-host log.
_E2_CHECK_HEARTBEAT = 4096
# CUDA registration alone does not own Python storage. Captured graphs retain
# raw addresses, so every successfully registered host allocation remains
# strongly referenced until process exit.
@@ -701,6 +728,8 @@ class PleOffloadConnector:
self.dp_rank,
self.tp_rank,
)
if _PLE_FLAG_CHECK_MODE != "off":
self._check_flags_clear_before_launch(num_tokens)
self._launch(
num_reqs,
num_tokens,
@@ -709,6 +738,128 @@ class PleOffloadConnector:
correlation_id=correlation_id,
)
def _check_flags_clear_before_launch(self, num_tokens: int) -> None:
"""Test the transport's staleness precondition, off by default.
A real launch publishes rows that the forward will pick up with
``cuStreamWaitValue32(flag == DONE_VALUE)``
(``ple_offload_layer.py:279``). That wait is only meaningful if the
flag is CLEAR when we publish: a leftover ``DONE_VALUE`` satisfies it
from the PREVIOUS step's copy and the forward silently consumes the
previous step's PLE outputs -- vllm#53899, where the semaphore stayed
one step ahead for the life of the server and ``cudagraph_mode=NONE``
was the only correct configuration.
Nothing here can be inferred from ``_after_dummy``: that is a bool
tracking a transition, not the flag's value. The structural argument
(dummy signal consumed by the dummy forward, ``_release_outputs_impl``
resetting every layer and synchronizing while ``_after_dummy`` is set,
a TP barrier at the transition) holds only while every
``prepare_forward`` is matched by a ``release_outputs``. An exception
path, an aborted step, or a future capture that signals without a
consuming forward leaves the flag at 1, and the #53899 thread's own
trap applies: identical sequential requests produce two bit-identical
classes, so a "same prompt twice" check passes in the broken world
(an instrument must discriminate both worlds).
PUBLISHING RANK ONLY. ``flag == 0`` before a launch is
NOT an invariant on a non-publishing rank, and the first version of
this check could terminate a HEALTHY TP2 run because of it. ``_launch``
publishes only on TP0 (``if self.tp_rank != 0: return``), and the
offload worker then fans the result out to EVERY registered target,
each gated only on that target's own flag having been reset
(``worker.py``, ``for target in targets: _wait_flag_released(target)
... target.host_flag[0] = 1``). So this legal interleaving exists:
1. TP0 checks its flag (clear), publishes step N.
2. The worker fills TP1's buffer and sets TP1's flag for step N.
3. TP1 reaches its own check for step N and reads ``DONE``.
4. TP1 reports a stale signal -- for a signal that belongs to the
CURRENT step.
The dummy->real TP barrier is before the checks and does not order this.
On TP0 the property does hold, and by construction rather than by
timing: the worker learns of a request only from ``_launch``, which is
after this check, and ``PleProgressMonitor.begin`` refuses overlapping
serving submissions ("Overlapping PLE host submissions violate the
serialized single-slot protocol"), so at most one transaction is
outstanding and the stream synchronize below has already retired its
release. A non-publishing rank cannot distinguish "this step's fan-out
arrived early" from "last step's signal was never consumed" with a
BINARY flag -- that is the same representational limit that rules out
an epoch word (see ``ple_offload_layer.py``), so those ranks are
skipped rather than checked weakly.
VLLM_E2_PLE_FLAG_CHECK=off|warn|raise, default off, default ACTION warn.
SCOPE. The skip below tests whether a capture
is ACTIVE, which is not the same as "this step's forward is eager". A
captured replay is not an active capture, and ``prepare_forward`` is
host Python that runs ahead of the forward either way, so this check
DOES run for steps whose forward is a graph replay. What it does not do
is run *inside* a capture, where a host synchronize is illegal. The
earlier "eager only" wording confused those two lifecycle states.
LIVENESS (PLE diagnostics on 0.30). A zero-findings result is only evidence if the
check demonstrably RAN, so every rank says once what it does with the
check, and the publishing rank logs a running count of checks and
findings on its first check and every ``_E2_CHECK_HEARTBEAT`` checks
after. A reference-host test arm greps these lines; their absence voids the arm.
"""
if not flag_check_is_authoritative(self.tp_rank):
if not getattr(self, "_flag_check_skip_logged", False):
self._flag_check_skip_logged = True
logger.info(
"PLE flag check (mode=%s): non-publishing rank, not judged "
"here by design (dp_rank=%d, tp_rank=%d, pp_rank=%d)",
_PLE_FLAG_CHECK_MODE,
self.dp_rank,
self.tp_rank,
self.pp_rank,
)
return
if torch.cuda.is_current_stream_capturing():
return
stream = torch.cuda.current_stream(self.device)
# Order the read against any reset still queued on the model stream.
# Without this the check reports the pre-reset value and fires on a
# correct run -- a guard that fires on correct metadata is worse than
# no guard.
stream.synchronize()
stale = stale_offload_flags(self._layers)
runs = self._flag_checks_run = getattr(self, "_flag_checks_run", 0) + 1
findings = getattr(self, "_flag_check_findings", 0) + (1 if stale else 0)
self._flag_check_findings = findings
if runs == 1 or runs % _E2_CHECK_HEARTBEAT == 0:
logger.info(
"PLE flag check (mode=%s): %d checks run, %d with findings "
"(dp_rank=%d, tp_rank=%d, pp_rank=%d)",
_PLE_FLAG_CHECK_MODE,
runs,
findings,
self.dp_rank,
self.tp_rank,
self.pp_rank,
)
if not stale:
return
message = (
"PLE pull transport: the output semaphore was already signalled "
"before this publishing rank published a real request, so the "
"forward's cuStreamWaitValue32(flag == 1) would be satisfied by "
"the PREVIOUS step's copy and this forward would consume the "
"previous step's PLE rows (vllm#53899 one-step-lag class). Stale "
f"layers (name, flag): {stale}; expected "
f"{CpuGpuSemaphore.RESET_VALUE} for all {len(self._layers)} "
f"layer(s). num_tokens={num_tokens}, dp_rank={self.dp_rank}, "
f"tp_rank={self.tp_rank}, pp_rank={self.pp_rank}."
)
if _PLE_FLAG_CHECK_MODE == "raise":
# Reaches _fail_closed through prepare_forward, which is the
# transport's stated contract for an uncertain correspondence.
raise RuntimeError(message)
logger.warning("%s", message)
def signal_dummy_outputs(self, num_tokens: int) -> None:
"""Locally satisfy PLE waits for dummy and capture forwards."""
self._progress.begin(startup=True, num_tokens=num_tokens)