diff --git a/vllm/model_executor/layers/ple_offload_layer.py b/vllm/model_executor/layers/ple_offload_layer.py index cc8bd73bfe..6eab455176 100644 --- a/vllm/model_executor/layers/ple_offload_layer.py +++ b/vllm/model_executor/layers/ple_offload_layer.py @@ -476,3 +476,88 @@ class PleOffloadLayer(nn.Module, ABC): """Mark the cross-process output buffer reusable on ``stream``.""" if self._is_cpu_offloaded: self._sem.reset(stream) + + +# --------------------------------------------------------------------------- +# Pre-launch staleness check for the pull transport (#53899 class) +# --------------------------------------------------------------------------- +# WHAT IT TESTS. The transport's correctness rests on one precondition: when a +# real request is published, every layer's semaphore must be CLEAR, so the +# ``cuStreamWaitValue32(flag == DONE_VALUE)`` inside the forward (line 279 for +# the pull path, line 215 for the IPC push path) can only be satisfied by the +# copy THIS step publishes. If a leftover ``DONE_VALUE`` survives into a real +# step, that wait is satisfied immediately by the PREVIOUS step's signal and +# the forward consumes the previous step's PLE rows. That is vllm#53899 on +# GB10/sm_121 exactly: "every forward consumes the previous step's PLE +# outputs", for the life of the server, with ``cudagraph_mode=NONE`` correct. +# +# In this tree the precondition is held structurally -- the dummy signal is +# consumed by the dummy forward, ``_release_outputs_impl`` resets every layer +# and synchronizes the model stream while ``_after_dummy`` is set, and the +# dummy->real transition takes a TP barrier -- but nothing ever READ the flag +# back, so the structural argument had no instrument. This is that instrument. +# +# WHY NOT AN EPOCH WORD. The stronger fix is a monotone epoch: the launch +# writes ``epoch = step``, the worker signals ``epoch``, and the placeholder +# waits ``EQ(epoch)``, so a stale signal carries the previous epoch and cannot +# satisfy the wait. It is not implementable on this transport as it stands. +# The compare value is an IMMEDIATE OPERAND of ``cuStreamWaitValue32`` +# (``_ple_offload_wait_copy_impl``, this file, line 279), and that call is +# issued inside the model forward, which is captured: PIECEWISE and +# FULL_AND_PIECEWISE both bake the wait into the graph with the value frozen at +# capture time. Making the compare value change per replay needs the captured +# node's parameters rewritten before each launch +# (``cuGraphExecBatchMemOpNodeSetParams``); nothing in this tree's capture path +# exposes the graph exec handle to the connector, and the wait is reached +# through a torch custom op inside a compiled region. An epoch word would +# therefore work only with ``cudagraph_mode=NONE``, i.e. only in the +# configuration where #53899 does not reproduce. So: binary flag, read back. +# +# The check is HOST-SIDE and needs a stream synchronize to be ordered against +# a pending reset, so it is never run inside a capture and is off by default. +# --------------------------------------------------------------------------- + + +def flag_check_is_authoritative(tp_rank: int) -> bool: + """Whether ``flag == RESET_VALUE`` before a launch is an INVARIANT here. + + Only on the publishing rank (TP0). Why: the offload worker fans a + result out to every registered TP target, each gated only on that target's + own flag having been reset, so a non-publishing rank's flag can legitimately + carry THIS step's signal before that rank reaches its own pre-launch check. + Reading a set flag there and calling it stale terminates a healthy run. + + On TP0 the invariant holds by construction: the worker learns of a request + only from ``_launch``, which runs after the check, and the progress monitor + refuses overlapping serving submissions, so the previous transaction has + been released and (after a stream synchronize) its reset has retired. + + A non-publishing rank cannot tell "this step's fan-out arrived early" from + "last step's signal was never consumed", because the flag is BINARY and + carries no transaction identity -- the same representational limit that + rules out an epoch word. So those ranks are skipped, not checked weakly. + """ + return tp_rank == 0 + + +def stale_offload_flags( + layers: "dict[str, PleOffloadLayer]", +) -> list[tuple[str, int]]: + """Return ``(layer_name, flag_value)`` for every layer whose flag is set. + + An empty list is the precondition holding. A non-empty list means at least + one semaphore still carries a signal that no forward consumed, so the next + ``wait`` would be satisfied by the previous step's copy. + + Pure and side-effect free: the caller owns the ordering (synchronize the + model stream first, and never call this inside a stream capture). Layers + that are not cross-process offloaded have no semaphore and are skipped. + """ + stale: list[tuple[str, int]] = [] + for name, layer in layers.items(): + if not getattr(layer, "_is_cpu_offloaded", False): + continue + value = int(layer._sem.flag_tensor.reshape(-1)[0].item()) + if value != CpuGpuSemaphore.RESET_VALUE: + stale.append((name, value)) + return stale diff --git a/vllm/v1/ple_offload/connector.py b/vllm/v1/ple_offload/connector.py index 2779edf3b5..0d5265115b 100644 --- a/vllm/v1/ple_offload/connector.py +++ b/vllm/v1/ple_offload/connector.py @@ -18,12 +18,15 @@ import torch.nn as nn import zmq from cuda.bindings import driver as cuda_driver +import vllm.envs as envs from vllm.config import VllmConfig from vllm.distributed.parallel_state import get_dp_group, get_pp_group, get_tp_group from vllm.logger import init_logger from vllm.model_executor.layers.ple_offload_layer import ( CpuGpuSemaphore, PleOffloadLayer, + flag_check_is_authoritative, + stale_offload_flags, ) from vllm.v1.ple_offload.protocol import ( PleOffloadRegistration, @@ -34,6 +37,30 @@ from vllm_ple_test_fault import _FAULT logger = init_logger(__name__) +# PLE diagnostics / vllm#53899 class: read the pull-transport semaphore back before a real +# launch and refuse to publish into a flag that is already set. Costs a +# model-stream synchronize, so it is a diagnostic, not a hot-path guard, and is +# OFF by default -- the same split VLLM_E2_ZERO_OWNERSHIP_CHECK established in +# the scheduler (VLLM_E2_ZERO_OWNERSHIP_CHECK). +# +# off | warn | raise (1 is accepted as warn). Default off. The default action on +# a violation is WARN, not raise: this diagnostic could +# terminate a healthy run, and a diagnostic that kills the qualification boot +# intended to establish safety is worse than no diagnostic. raise is for a +# targeted repro. +# +# Read through the envs registry, not os.getenv: the registry owns the +# normalization, so the value reported by +# `vllm.envs` and the value this check applies cannot drift apart. It is +# registered as a non-compile factor there. A set-but-unrecognised value +# raises on first read -- fail closed, because a broken gate that silently +# disarms its own instrument is the failure mode these checks exist to avoid. +_PLE_FLAG_CHECK_MODE: str = envs.VLLM_E2_PLE_FLAG_CHECK +# Liveness cadence for the diagnostic checks: the first check and every Nth +# are logged with a running count, so "zero findings" can be told apart from +# "never ran" in a reference-host log. +_E2_CHECK_HEARTBEAT = 4096 + # CUDA registration alone does not own Python storage. Captured graphs retain # raw addresses, so every successfully registered host allocation remains # strongly referenced until process exit. @@ -701,6 +728,8 @@ class PleOffloadConnector: self.dp_rank, self.tp_rank, ) + if _PLE_FLAG_CHECK_MODE != "off": + self._check_flags_clear_before_launch(num_tokens) self._launch( num_reqs, num_tokens, @@ -709,6 +738,128 @@ class PleOffloadConnector: correlation_id=correlation_id, ) + def _check_flags_clear_before_launch(self, num_tokens: int) -> None: + """Test the transport's staleness precondition, off by default. + + A real launch publishes rows that the forward will pick up with + ``cuStreamWaitValue32(flag == DONE_VALUE)`` + (``ple_offload_layer.py:279``). That wait is only meaningful if the + flag is CLEAR when we publish: a leftover ``DONE_VALUE`` satisfies it + from the PREVIOUS step's copy and the forward silently consumes the + previous step's PLE outputs -- vllm#53899, where the semaphore stayed + one step ahead for the life of the server and ``cudagraph_mode=NONE`` + was the only correct configuration. + + Nothing here can be inferred from ``_after_dummy``: that is a bool + tracking a transition, not the flag's value. The structural argument + (dummy signal consumed by the dummy forward, ``_release_outputs_impl`` + resetting every layer and synchronizing while ``_after_dummy`` is set, + a TP barrier at the transition) holds only while every + ``prepare_forward`` is matched by a ``release_outputs``. An exception + path, an aborted step, or a future capture that signals without a + consuming forward leaves the flag at 1, and the #53899 thread's own + trap applies: identical sequential requests produce two bit-identical + classes, so a "same prompt twice" check passes in the broken world + (an instrument must discriminate both worlds). + + PUBLISHING RANK ONLY. ``flag == 0`` before a launch is + NOT an invariant on a non-publishing rank, and the first version of + this check could terminate a HEALTHY TP2 run because of it. ``_launch`` + publishes only on TP0 (``if self.tp_rank != 0: return``), and the + offload worker then fans the result out to EVERY registered target, + each gated only on that target's own flag having been reset + (``worker.py``, ``for target in targets: _wait_flag_released(target) + ... target.host_flag[0] = 1``). So this legal interleaving exists: + + 1. TP0 checks its flag (clear), publishes step N. + 2. The worker fills TP1's buffer and sets TP1's flag for step N. + 3. TP1 reaches its own check for step N and reads ``DONE``. + 4. TP1 reports a stale signal -- for a signal that belongs to the + CURRENT step. + + The dummy->real TP barrier is before the checks and does not order this. + On TP0 the property does hold, and by construction rather than by + timing: the worker learns of a request only from ``_launch``, which is + after this check, and ``PleProgressMonitor.begin`` refuses overlapping + serving submissions ("Overlapping PLE host submissions violate the + serialized single-slot protocol"), so at most one transaction is + outstanding and the stream synchronize below has already retired its + release. A non-publishing rank cannot distinguish "this step's fan-out + arrived early" from "last step's signal was never consumed" with a + BINARY flag -- that is the same representational limit that rules out + an epoch word (see ``ple_offload_layer.py``), so those ranks are + skipped rather than checked weakly. + + VLLM_E2_PLE_FLAG_CHECK=off|warn|raise, default off, default ACTION warn. + + SCOPE. The skip below tests whether a capture + is ACTIVE, which is not the same as "this step's forward is eager". A + captured replay is not an active capture, and ``prepare_forward`` is + host Python that runs ahead of the forward either way, so this check + DOES run for steps whose forward is a graph replay. What it does not do + is run *inside* a capture, where a host synchronize is illegal. The + earlier "eager only" wording confused those two lifecycle states. + + LIVENESS (PLE diagnostics on 0.30). A zero-findings result is only evidence if the + check demonstrably RAN, so every rank says once what it does with the + check, and the publishing rank logs a running count of checks and + findings on its first check and every ``_E2_CHECK_HEARTBEAT`` checks + after. A reference-host test arm greps these lines; their absence voids the arm. + """ + if not flag_check_is_authoritative(self.tp_rank): + if not getattr(self, "_flag_check_skip_logged", False): + self._flag_check_skip_logged = True + logger.info( + "PLE flag check (mode=%s): non-publishing rank, not judged " + "here by design (dp_rank=%d, tp_rank=%d, pp_rank=%d)", + _PLE_FLAG_CHECK_MODE, + self.dp_rank, + self.tp_rank, + self.pp_rank, + ) + return + if torch.cuda.is_current_stream_capturing(): + return + stream = torch.cuda.current_stream(self.device) + # Order the read against any reset still queued on the model stream. + # Without this the check reports the pre-reset value and fires on a + # correct run -- a guard that fires on correct metadata is worse than + # no guard. + stream.synchronize() + stale = stale_offload_flags(self._layers) + runs = self._flag_checks_run = getattr(self, "_flag_checks_run", 0) + 1 + findings = getattr(self, "_flag_check_findings", 0) + (1 if stale else 0) + self._flag_check_findings = findings + if runs == 1 or runs % _E2_CHECK_HEARTBEAT == 0: + logger.info( + "PLE flag check (mode=%s): %d checks run, %d with findings " + "(dp_rank=%d, tp_rank=%d, pp_rank=%d)", + _PLE_FLAG_CHECK_MODE, + runs, + findings, + self.dp_rank, + self.tp_rank, + self.pp_rank, + ) + if not stale: + return + message = ( + "PLE pull transport: the output semaphore was already signalled " + "before this publishing rank published a real request, so the " + "forward's cuStreamWaitValue32(flag == 1) would be satisfied by " + "the PREVIOUS step's copy and this forward would consume the " + "previous step's PLE rows (vllm#53899 one-step-lag class). Stale " + f"layers (name, flag): {stale}; expected " + f"{CpuGpuSemaphore.RESET_VALUE} for all {len(self._layers)} " + f"layer(s). num_tokens={num_tokens}, dp_rank={self.dp_rank}, " + f"tp_rank={self.tp_rank}, pp_rank={self.pp_rank}." + ) + if _PLE_FLAG_CHECK_MODE == "raise": + # Reaches _fail_closed through prepare_forward, which is the + # transport's stated contract for an uncertain correspondence. + raise RuntimeError(message) + logger.warning("%s", message) + def signal_dummy_outputs(self, num_tokens: int) -> None: """Locally satisfy PLE waits for dummy and capture forwards.""" self._progress.begin(startup=True, num_tokens=num_tokens)