[Guards] Executable graph-profile and breakable-cudagraph guards
New vllm/v1/e2_guards.py, called from the target and MTP constructors on every rank before profiling or capture: the breakable-cudagraph pin is asserted (three readings of the switch), a graph profile that cannot execute on v0.30.0 is refused, and VLLM_E2_EXPECTED_CUDAGRAPH_MODE is checked by name. Tests cover the drafter. An inert knob is no longer hashed into the cache key. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
+57
-6
@@ -339,11 +339,29 @@ if TYPE_CHECKING:
|
||||
# hashed anyway: a spurious cache miss costs a recompile, a missed one costs
|
||||
# a wrong graph.
|
||||
#
|
||||
# COMPILE/CAPTURE FACTORS (hashed):
|
||||
# AOT: yes. CAPTURE: yes. PLE table is FP8-with-global-scale ->
|
||||
# Qwen4ExpPLEFp8EmbeddingMethod instead of the checkpoint's method:
|
||||
# different weight dtype and a different embedding forward.
|
||||
# *** INERT ON THE 0.30 BASE. ***
|
||||
# NO READ SITE, and therefore NOT HASHED -- moved out of the compile
|
||||
# factors into ignored_factors below. Its only reader was the 21-line
|
||||
# catch-all branch in _get_ple_embedding_quant_method, which lived in the
|
||||
# half of models/qwen4_exp/nvidia/ple_layer.py that #54371 deleted. With no
|
||||
# reader it cannot change an artifact, so hashing it would only invalidate
|
||||
# caches for nothing; and on this uncompiled NVIDIA path (#55272) there may
|
||||
# be no AOT artifact to key at all. It gives NO capture protection either.
|
||||
# v0.30.0 selects the FP8 PLE method from the CONFIG KEY
|
||||
# text_config.ple_embedding_dtype == "float8_e4m3fn"
|
||||
# (ngram_embedding.py:174-175) -- see
|
||||
# tests/models/qwen4_exp/test_ple_fp8_selector_checkpoint.py.
|
||||
# Setting this on the 0.30 base does NOTHING, and NOTHING WARNS: validate_environ
|
||||
# (envs.py, below) warns only about names that are NOT registered, and this
|
||||
# comment lives in a TYPE_CHECKING block that never runs. It is kept
|
||||
# registered so a later PLE transport change can reland its reader and
|
||||
# reclassify it, or delete it. DO NOT assume it works.
|
||||
# When its reader returns: AOT yes, CAPTURE yes -- the FP8-with-global-scale
|
||||
# table selects Qwen4ExpPLEFp8EmbeddingMethod over the checkpoint's method,
|
||||
# a different weight dtype and a different embedding forward.
|
||||
VLLM_PLE_FP8_GLOBAL_SCALE: bool = False
|
||||
#
|
||||
# COMPILE/CAPTURE FACTORS (hashed):
|
||||
# CAPTURE: yes, established -- clip counting allocates persistent per-layer
|
||||
# int64 counters and its increment ops are CAPTURED into the cudagraph (see
|
||||
# the graph-safety note in models/qwen4_exp/nvidia/qsa.py); unset, those ops
|
||||
@@ -404,16 +422,34 @@ if TYPE_CHECKING:
|
||||
# cross-change knob table). Each mirrors its read site's
|
||||
# parse; if a read site changes, the entry here must be reconciled.
|
||||
# (Both read sites below are now live.)
|
||||
# *** INERT ON THE 0.30 BASE: NO READ SITE. ***
|
||||
# Its reader is the slot-ownership assertion code (an earlier commit), which edits
|
||||
# vllm/v1/ple_offload/connector.py -- a module that does not exist on
|
||||
# v0.30.0. Setting this on the 0.30 base does NOTHING, and NOTHING WARNS
|
||||
# (validate_environ only reports UNregistered names). Already an ignored
|
||||
# factor, correctly. Kept registered so the contract is recorded; it
|
||||
# becomes live again only together with the pull transport
|
||||
# (the PLE transport choice).
|
||||
# AOT: no. CAPTURE: no -- strictly host-side and outside capture.
|
||||
# off|warn|raise (1 = warn), normalised by _e2_diagnostic_mode like the
|
||||
# rest of the family. The mamba ownership check moved this gate off a plain
|
||||
# boolean; reconciled here.
|
||||
# Read site returns with the deferred PLE transport.
|
||||
VLLM_E2_PLE_FLAG_CHECK: str = "off"
|
||||
# AOT: no. CAPTURE: no -- strictly host-side. off|warn|raise (1 = warn).
|
||||
# the mamba slot-ownership check, v1/worker/mamba_utils.py:74.
|
||||
VLLM_E2_MAMBA_SLOT_OWNERSHIP: str = "off"
|
||||
# Opt-out for the fork startup guards in vllm/v1/e2_guards.py, read at
|
||||
# Qwen4ExpForCausalLM construction. AOT: no. CAPTURE: no -- it changes
|
||||
# nothing itself; it only permits a boot whose identity IS different
|
||||
# (VLLM_USE_BREAKABLE_CUDAGRAPH, which upstream already registers and
|
||||
# hashes). Ignored factor for exactly that reason.
|
||||
VLLM_E2_ALLOW_BREAKABLE: bool = False
|
||||
# Expected RESOLVED cudagraph mode by name, asserted by the same guard when
|
||||
# set (e.g. "FULL", "FULL_DECODE_ONLY"). AOT: no. CAPTURE: no -- it asserts
|
||||
# an identity, it does not choose one. Ignored factor.
|
||||
VLLM_E2_EXPECTED_CUDAGRAPH_MODE: str = ""
|
||||
#
|
||||
# --8<-- [start:fork-unaudited-knobs]
|
||||
# --- TODO: knobs that are NOT in this tree at all ----------
|
||||
# The tie-determinism toggles and the BSM pin live in a separate
|
||||
# patch stack, not in v2.0.1 and not in this tree:
|
||||
@@ -434,8 +470,11 @@ if TYPE_CHECKING:
|
||||
# identities from its actual read site, hash it unless exclusion is proven,
|
||||
# and delete it from this block. tests/test_envs.py holds a test that fails
|
||||
# if a name is registered while still listed here, so the classification
|
||||
# cannot be skipped silently.
|
||||
# cannot be skipped silently. That test reads ONLY the delimited block
|
||||
# below/above -- scanning the whole file could never pass, because a
|
||||
# registry entry necessarily contains its own name.
|
||||
# ----------------------------------------------------------------------
|
||||
# --8<-- [end:fork-unaudited-knobs]
|
||||
VLLM_LOG_MODEL_INSPECTION: bool = False
|
||||
VLLM_DEBUG_MFU_METRICS: bool = False
|
||||
VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY: bool = False
|
||||
@@ -2290,6 +2329,10 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# Capture-configuration factor, read in C++; raw passthrough on purpose.
|
||||
"MERLIN_TOPK_EXACT_MEDIUM": lambda: os.getenv("MERLIN_TOPK_EXACT_MEDIUM", None),
|
||||
# Runtime only; all of the following are in compile_factors.ignored_factors.
|
||||
"VLLM_E2_ALLOW_BREAKABLE": lambda: os.getenv("VLLM_E2_ALLOW_BREAKABLE", "") == "1",
|
||||
"VLLM_E2_EXPECTED_CUDAGRAPH_MODE": lambda: os.getenv(
|
||||
"VLLM_E2_EXPECTED_CUDAGRAPH_MODE", ""
|
||||
).strip(),
|
||||
"VLLM_QSA_KV_SCALES": lambda: os.getenv("VLLM_QSA_KV_SCALES", "").strip(),
|
||||
"VLLM_E2_GUARD_MODE": lambda: (
|
||||
os.getenv("VLLM_E2_GUARD_MODE", "off").strip().lower()
|
||||
@@ -2659,6 +2702,14 @@ def compile_factors() -> dict[str, object]:
|
||||
"VLLM_PLE_TEST_FAULT",
|
||||
"VLLM_PLE_TEST_FAULT_GATE",
|
||||
"VLLM_PLE_TEST_FAULT_LOG",
|
||||
# Fork startup guard: the opt-out permits a different identity and the
|
||||
# expected-mode pin asserts one; neither IS one.
|
||||
"VLLM_E2_ALLOW_BREAKABLE",
|
||||
"VLLM_E2_EXPECTED_CUDAGRAPH_MODE",
|
||||
# INERT on the 0.30 base: no read site, so it cannot change any artifact.
|
||||
# Hashing it would invalidate caches for no behavioural change. Restore
|
||||
# it to the compile factors in the same commit that relands its reader.
|
||||
"VLLM_PLE_FP8_GLOBAL_SCALE",
|
||||
# Diagnostic knobs, both strictly host-side and outside capture, so
|
||||
# neither AOT nor capture identity depends on them. Hashing
|
||||
# them would invalidate every compile cache the moment a diagnostic
|
||||
|
||||
@@ -78,6 +78,7 @@ from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.e2_guards import assert_graph_profile_is_executable
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
from ..common.mtp import make_mtp_hidden_buffer
|
||||
@@ -759,6 +760,13 @@ class Qwen4ExpForCausalLM(
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
# V0.30.0 auto-enables breakable cudagraphs
|
||||
# for this architecture when VLLM_USE_BREAKABLE_CUDAGRAPH is unset, and
|
||||
# VLLM_COMPILE + a PIECEWISE mode dies at capture because #55272 left no
|
||||
# compiled submodule. Fail here rather than let a reference-host run qualify an
|
||||
# identity nobody chose. Qwen4ExpForConditionalGeneration constructs
|
||||
# this class; Qwen4ExpMTP does not, and guards itself.
|
||||
assert_graph_profile_is_executable(vllm_config, role="target")
|
||||
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
|
||||
if vllm_config.lora_config is not None:
|
||||
# LoRA does not support the merged QKV/indexer projection, so its
|
||||
|
||||
@@ -51,6 +51,7 @@ from vllm.model_executor.models.utils import (
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.e2_guards import assert_graph_profile_is_executable
|
||||
|
||||
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
||||
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
||||
@@ -395,6 +396,10 @@ class Qwen4ExpMTP(
|
||||
}
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
# The drafter is NOT reached through Qwen4ExpForCausalLM --
|
||||
# this class builds Qwen4ExpMultiTokenPredictor directly, from a config
|
||||
# rebuilt at mtp.py:159 -- so its graph profile is checked here.
|
||||
assert_graph_profile_is_executable(vllm_config, role="draft")
|
||||
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
|
||||
self.vllm_config = vllm_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fork startup guards that fail closed on a graph profile we never qualified.
|
||||
|
||||
These run at model construction, on every rank, for the target model and for
|
||||
the MTP drafter, before profiling or capture. They exist because a *documented*
|
||||
launch flag is not a mechanism: the first reference-host run of a rebase will otherwise
|
||||
qualify a compile/capture identity nobody chose
|
||||
(see below).
|
||||
|
||||
What this module does NOT do: choose a graph profile. That is the maintainer's
|
||||
call and it is open (the graph-profile choice). What it does is refuse the
|
||||
two states that are silently wrong on vLLM v0.30.0 --
|
||||
|
||||
* ``VLLM_USE_BREAKABLE_CUDAGRAPH`` left unset, which lets
|
||||
``VllmConfig._maybe_enable_breakable_cudagraph`` turn it on for this fork's
|
||||
architectures and force ``CompilationMode.NONE``; and
|
||||
* a PIECEWISE-bearing cudagraph mode combined with ``VLLM_COMPILE`` and
|
||||
breakable-off, which upstream accepts at config time and then rejects at
|
||||
capture, because #55272 removed every ``@support_torch_compile`` wrapper
|
||||
from the NVIDIA Qwen4Exp model and MTP.
|
||||
|
||||
-- and to make the resolved profile visible and assertable.
|
||||
|
||||
Timing limit, stated honestly: this runs at model construction, i.e. AFTER
|
||||
``VllmConfig.__post_init__`` has already called
|
||||
``_maybe_enable_breakable_cudagraph`` (``config/vllm.py:1561``) and
|
||||
``_normalize_piecewise_cudagraph_mode`` (``:1673``, ``:1844``). It is early
|
||||
enough to stop profiling and capture; it cannot un-mutate ``os.environ`` or
|
||||
recover the originally requested cudagraph mode. Requiring the pin to be
|
||||
*explicitly present* is the part of "validate before auto-enable" that is
|
||||
checkable from here: absence is exactly the state the auto-enable feeds on.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
BREAKABLE_ENV = "VLLM_USE_BREAKABLE_CUDAGRAPH"
|
||||
#: Opt-out for every guard in this module. Recorded in the launch identity and
|
||||
#: PROHIBITED in production profiles; a production launch must not set it.
|
||||
ALLOW_BREAKABLE_ENV = "VLLM_E2_ALLOW_BREAKABLE"
|
||||
#: Optional expected resolved cudagraph mode, by name (e.g. "FULL",
|
||||
#: "FULL_DECODE_ONLY"). When set, a mismatch is an error. This is how a launch
|
||||
#: profile pins the graph identity once the maintainer has chosen one.
|
||||
EXPECTED_MODE_ENV = "VLLM_E2_EXPECTED_CUDAGRAPH_MODE"
|
||||
|
||||
#: The architectures vLLM v0.30.0 auto-enables breakable cudagraphs for, out of
|
||||
#: ``vllm/config/vllm.py:DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES``, that are
|
||||
#: ours. Recorded so the guard's message can name the upstream mechanism.
|
||||
OUR_AUTO_BREAKABLE_ARCHITECTURES = frozenset(
|
||||
{
|
||||
"Qwen4ExpForCausalLM",
|
||||
"Qwen4ExpForConditionalGeneration",
|
||||
"Qwen4ExpMTP",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _upstream_truth(raw: str | None) -> bool:
|
||||
"""Upstream's exact semantics for the breakable switch.
|
||||
|
||||
``envs.py:937`` is ``os.environ.get(..., "0") == "1"``. Nothing else is on:
|
||||
``"2"``, ``"true"``, ``"FALSE"`` and ``""`` are all off. Using one parser
|
||||
that matches it is the point -- a stricter local parser would reject boots
|
||||
upstream treats as off, which is a false rejection, not extra safety.
|
||||
"""
|
||||
return raw == "1"
|
||||
|
||||
|
||||
def breakable_resolved() -> bool:
|
||||
"""Whether breakable cudagraphs are on, as the engine will see it.
|
||||
|
||||
Two readings of ONE source, not three independent sources:
|
||||
``vllm.compilation.breakable_cudagraph.is_breakable_cudagraph_enabled()``
|
||||
is literally ``bool(envs.VLLM_USE_BREAKABLE_CUDAGRAPH)``
|
||||
(``breakable_cudagraph.py:53``), so it adds nothing over the envs read.
|
||||
The envs read is kept beside the raw one because
|
||||
``_maybe_enable_breakable_cudagraph`` mutates ``os.environ`` at config time
|
||||
while the envs accessor is cached, so the two can legitimately disagree;
|
||||
either saying "on" means on.
|
||||
|
||||
No exception is swallowed. A guard that cannot read its own input must fail
|
||||
the boot, not report False.
|
||||
"""
|
||||
if _upstream_truth(os.environ.get(BREAKABLE_ENV)):
|
||||
return True
|
||||
import vllm.envs as envs
|
||||
|
||||
return bool(envs.VLLM_USE_BREAKABLE_CUDAGRAPH)
|
||||
|
||||
|
||||
def _profile(vllm_config) -> tuple[list[str], object, object]:
|
||||
"""Resolved (architectures, compilation mode, cudagraph mode). Fails closed."""
|
||||
model_config = getattr(vllm_config, "model_config", None)
|
||||
if model_config is None:
|
||||
raise RuntimeError("e2-guard: vllm_config has no model_config to check")
|
||||
compilation_config = getattr(vllm_config, "compilation_config", None)
|
||||
if compilation_config is None:
|
||||
raise RuntimeError("e2-guard: vllm_config has no compilation_config to check")
|
||||
architectures = list(getattr(model_config, "architectures", None) or [])
|
||||
return (
|
||||
architectures,
|
||||
compilation_config.mode,
|
||||
compilation_config.cudagraph_mode,
|
||||
)
|
||||
|
||||
|
||||
def assert_graph_profile_is_executable(vllm_config, *, role: str = "target") -> None:
|
||||
"""Refuse the two silently-wrong startup states on v0.30.0.
|
||||
|
||||
``role`` is ``"target"`` or ``"draft"``; both are checked, because
|
||||
``Qwen4ExpMTP`` builds ``Qwen4ExpMultiTokenPredictor`` rather than
|
||||
``Qwen4ExpForCausalLM`` (``mtp.py:397``, ``:411``) from a config rebuilt at
|
||||
``mtp.py:159``, so the drafter's profile can differ from the target's.
|
||||
|
||||
Intentional eager and intentional ``CompilationMode.NONE`` are PERMITTED.
|
||||
On v0.30.0's NVIDIA Qwen4Exp path ``NONE`` is a normal resolved state, not
|
||||
evidence of the auto-enable: #55272 removed the compile wrappers, so
|
||||
``_normalize_piecewise_cudagraph_mode`` legitimately lands there.
|
||||
"""
|
||||
from vllm.config import CompilationMode
|
||||
architectures, mode, cudagraph_mode = _profile(vllm_config)
|
||||
|
||||
if os.environ.get(ALLOW_BREAKABLE_ENV, "") == "1":
|
||||
logger.warning(
|
||||
"e2-guard[%s]: %s=1 -- guards disabled by request. architectures=%s "
|
||||
"breakable=%s compilation_mode=%s cudagraph_mode=%s. This boot is "
|
||||
"NOT a promotion-grade boot; a production launch must not set this flag.",
|
||||
role,
|
||||
ALLOW_BREAKABLE_ENV,
|
||||
architectures,
|
||||
os.environ.get(BREAKABLE_ENV),
|
||||
mode,
|
||||
cudagraph_mode,
|
||||
)
|
||||
return
|
||||
|
||||
# 1. The pin must be explicit. Absence is the state _maybe_enable_breakable_
|
||||
# cudagraph feeds on (config/vllm.py:758: "not in os.environ").
|
||||
if BREAKABLE_ENV not in os.environ:
|
||||
raise RuntimeError(
|
||||
f"e2-guard[{role}]: {BREAKABLE_ENV} is not set. vLLM v0.30.0 lists "
|
||||
f"{sorted(OUR_AUTO_BREAKABLE_ARCHITECTURES)} in "
|
||||
"DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES and turns the switch on "
|
||||
"by itself when it is unset (vllm/config/vllm.py:756-774), forcing "
|
||||
"CompilationMode.NONE with it. Pin it in the launcher: "
|
||||
f"export {BREAKABLE_ENV}=0 (or =1 deliberately), or set "
|
||||
f"{ALLOW_BREAKABLE_ENV}=1 to take an unpinned boot on purpose."
|
||||
)
|
||||
|
||||
breakable = breakable_resolved()
|
||||
|
||||
# 2. Breakable on is an identity this fork has never qualified.
|
||||
if breakable:
|
||||
raise RuntimeError(
|
||||
f"e2-guard[{role}]: {BREAKABLE_ENV} resolved ON for "
|
||||
f"{architectures or '<unknown architecture>'} "
|
||||
f"(compilation_mode={mode}, cudagraph_mode={cudagraph_mode}). That "
|
||||
"is not the identity the v2.0.1 results were measured on. Export "
|
||||
f"{BREAKABLE_ENV}=0, or set {ALLOW_BREAKABLE_ENV}=1 to take it "
|
||||
"deliberately."
|
||||
)
|
||||
|
||||
# 3. PIECEWISE with breakable off needs a torch-compiled submodule, and
|
||||
# #55272 removed every @support_torch_compile wrapper from
|
||||
# vllm/models/qwen4_exp/nvidia/*. Upstream only rewrites the mode when
|
||||
# compilation mode is NOT VLLM_COMPILE
|
||||
# (config/vllm.py:1010-1030), so VLLM_COMPILE + PIECEWISE + breakable-off
|
||||
# survives config time and then dies at
|
||||
# gpu/cudagraph_utils.py:605 in the middle of capture. Fail here instead,
|
||||
# with the candidates named.
|
||||
if (
|
||||
cudagraph_mode.requires_piecewise_compilation()
|
||||
and mode is CompilationMode.VLLM_COMPILE
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"e2-guard[{role}]: cudagraph_mode={cudagraph_mode} needs piecewise "
|
||||
"capture, compilation mode is VLLM_COMPILE and breakable cudagraphs "
|
||||
"are off -- but vLLM PR #55272 removed every @support_torch_compile "
|
||||
"wrapper from vllm/models/qwen4_exp/nvidia/, so "
|
||||
"has_compiled_submodule() is False and capture will raise at "
|
||||
"vllm/v1/worker/gpu/cudagraph_utils.py:605. Executable candidates: "
|
||||
"(a) breakable off + -O0/CompilationMode.NONE, which normalises "
|
||||
"FULL_AND_PIECEWISE to FULL_DECODE_ONLY (changed prefill "
|
||||
"behaviour -- must be qualified); (b) breakable off + an explicit "
|
||||
"cudagraph_mode=FULL or NONE; (c) breakable PIECEWISE, i.e. "
|
||||
f"{BREAKABLE_ENV}=1, which is a different identity. See "
|
||||
"the graph-profile choice."
|
||||
)
|
||||
|
||||
# 4. Optional exact pin of the resolved mode, for a launch profile.
|
||||
expected = os.environ.get(EXPECTED_MODE_ENV, "").strip()
|
||||
if expected:
|
||||
actual = getattr(cudagraph_mode, "name", str(cudagraph_mode))
|
||||
if actual != expected:
|
||||
raise RuntimeError(
|
||||
f"e2-guard[{role}]: resolved cudagraph_mode={actual} but "
|
||||
f"{EXPECTED_MODE_ENV}={expected}. The launch profile and the "
|
||||
"engine disagree about the graph identity."
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"e2-guard[%s]: architectures=%s breakable=OFF compilation_mode=%s "
|
||||
"cudagraph_mode=%s expected=%s. Recorded, not certified -- this guard "
|
||||
"checks startup states, not that the profile was qualified.",
|
||||
role,
|
||||
architectures,
|
||||
mode,
|
||||
cudagraph_mode,
|
||||
expected or "<unset>",
|
||||
)
|
||||
|
||||
|
||||
#: Back-compatible name for the earlier call sites.
|
||||
assert_breakable_cudagraph_pinned = assert_graph_profile_is_executable
|
||||
@@ -1,13 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Compatibility exports for the top-level test-only fault singleton."""
|
||||
|
||||
from vllm_ple_test_fault import (
|
||||
_FAULT as _FAULT,
|
||||
)
|
||||
from vllm_ple_test_fault import ( # noqa: F401
|
||||
FaultSpec as FaultSpec,
|
||||
)
|
||||
from vllm_ple_test_fault import (
|
||||
_parse_fault_spec as _parse_fault_spec,
|
||||
)
|
||||
@@ -238,118 +238,6 @@ class Worker(WorkerBase):
|
||||
# Resolved lazily on first sleep/wake; persists worker-process state.
|
||||
self._sleep_mode_backend: SleepModeBackend | None = None
|
||||
|
||||
def _has_ple_layers(self) -> bool:
|
||||
"""Return whether this model configuration constructs PLE layers."""
|
||||
if not envs.VLLM_PLE_CPU_OFFLOAD:
|
||||
return False
|
||||
text_config = self.model_config.hf_text_config
|
||||
ple_layer_ids = getattr(text_config, "ple_layer_ids", None)
|
||||
if not ple_layer_ids:
|
||||
return False
|
||||
# PP (vllm#54709, carried from the earlier overlay): only the pipeline rank
|
||||
# that owns a PLE layer takes part in the offload (rank 0 with the
|
||||
# default partition). Other ranks build PPMissingLayer there and must
|
||||
# skip the connector entirely.
|
||||
pp_size = self.parallel_config.pipeline_parallel_size
|
||||
if pp_size > 1:
|
||||
from vllm.distributed.utils import get_pp_indices
|
||||
|
||||
tp_size = self.parallel_config.tensor_parallel_size
|
||||
pp_rank = (self.rank // tp_size) % pp_size
|
||||
start, end = get_pp_indices(
|
||||
int(text_config.num_hidden_layers), pp_rank, pp_size
|
||||
)
|
||||
return any(start <= int(i) - 1 < end for i in ple_layer_ids)
|
||||
return True
|
||||
|
||||
def _validate_ple_offload_config(self) -> None:
|
||||
"""Reject unsupported PLE offload execution modes."""
|
||||
parallel_config = self.parallel_config
|
||||
unsupported = []
|
||||
if not self.use_v2_model_runner:
|
||||
unsupported.append("model runner V1")
|
||||
if not current_platform.is_cuda():
|
||||
unsupported.append(f"device={current_platform.device_type}")
|
||||
if parallel_config.nnodes != 1:
|
||||
unsupported.append(f"nnodes={parallel_config.nnodes}")
|
||||
if parallel_config.data_parallel_backend != "mp":
|
||||
unsupported.append(f"DP backend={parallel_config.data_parallel_backend}")
|
||||
if (
|
||||
parallel_config.data_parallel_size_local
|
||||
!= parallel_config.data_parallel_size
|
||||
):
|
||||
unsupported.append(
|
||||
"non-local DP "
|
||||
f"({parallel_config.data_parallel_size_local}/"
|
||||
f"{parallel_config.data_parallel_size} local ranks)"
|
||||
)
|
||||
# PP allowed (vllm#54709): PLE layers must live on pipeline rank 0
|
||||
# (enforced placement-based in models/config.py); num_workers stays
|
||||
# dp_size * tp_size = the ranks of the owning stage.
|
||||
if parallel_config.prefill_context_parallel_size != 1:
|
||||
unsupported.append(f"PCP={parallel_config.prefill_context_parallel_size}")
|
||||
if parallel_config.decode_context_parallel_size != 1:
|
||||
unsupported.append(f"DCP={parallel_config.decode_context_parallel_size}")
|
||||
if parallel_config.use_ubatching:
|
||||
unsupported.append("ubatching/DBO")
|
||||
if self.model_config.architecture not in {
|
||||
"Qwen4ExpForCausalLM",
|
||||
"Qwen4ExpForConditionalGeneration",
|
||||
}:
|
||||
unsupported.append("architecture")
|
||||
if self.vllm_config.weight_transfer_config is not None:
|
||||
unsupported.append("weight transfer")
|
||||
|
||||
if unsupported:
|
||||
raise ValueError(
|
||||
"VLLM_PLE_CPU_OFFLOAD does not support the requested "
|
||||
"configuration. Unsupported settings: "
|
||||
f"{', '.join(unsupported)}"
|
||||
)
|
||||
|
||||
def spawn_ple_offload(self) -> None:
|
||||
"""Spawn one node-local PLE CPU worker from DP0/TP0."""
|
||||
if (
|
||||
not self._ple_offload_enabled
|
||||
or self.rank != 0
|
||||
or self.parallel_config.data_parallel_rank != 0
|
||||
):
|
||||
return
|
||||
|
||||
from vllm.v1.ple_offload.worker import PleOffloadWorker
|
||||
|
||||
ipc_addr = self.parallel_config._ple_offload_ipc_path
|
||||
if not ipc_addr:
|
||||
raise RuntimeError("PLE offload IPC address was not initialized")
|
||||
dp_size = self.parallel_config.data_parallel_size
|
||||
tp_size = self.parallel_config.tensor_parallel_size
|
||||
num_workers = dp_size * tp_size
|
||||
logger.info(
|
||||
"PleOffload: spawning worker "
|
||||
"(rank=%d, local_rank=%d, dp_size=%d, tp_size=%d, "
|
||||
"num_workers=%d, ipc_addr=%s).",
|
||||
self.rank,
|
||||
self.local_rank,
|
||||
dp_size,
|
||||
tp_size,
|
||||
num_workers,
|
||||
ipc_addr,
|
||||
)
|
||||
self._ple_offload_worker_handle = PleOffloadWorker.make_process(
|
||||
self.vllm_config,
|
||||
num_workers,
|
||||
ipc_addr,
|
||||
)
|
||||
|
||||
def wait_ple_offload_ready(self) -> None:
|
||||
"""Wait until weights and every TP registration are ready."""
|
||||
if self._ple_offload_worker_handle is None:
|
||||
return
|
||||
|
||||
from vllm.v1.ple_offload.worker import PleOffloadWorker
|
||||
|
||||
PleOffloadWorker.wait_for_ready(self._ple_offload_worker_handle)
|
||||
|
||||
@property
|
||||
def sleep_mode_backend(self) -> "SleepModeBackend":
|
||||
if self._sleep_mode_backend is None:
|
||||
|
||||
Reference in New Issue
Block a user