diff --git a/vllm/config/engram.py b/vllm/config/engram.py index b47226ad1a..987d93d28e 100644 --- a/vllm/config/engram.py +++ b/vllm/config/engram.py @@ -3,9 +3,10 @@ from typing import TYPE_CHECKING -from pydantic import model_validator +from pydantic import Field, model_validator from typing_extensions import Self +import vllm.envs as envs from vllm.config.utils import config, get_hash_factors, hash_factors if TYPE_CHECKING: @@ -21,6 +22,17 @@ _NGRAM_LAYER_FIELDS = { } +def _default_cpu_offload() -> bool: + # Fork: with VLLM_E2_PLE_PULL_TRANSPORT=1 the default is False and a + # VLLM_PLE_CPU_OFFLOAD value that would enable UVA fails the boot. With the + # pull transport off, the upstream default below is returned unchanged. + from vllm.v1.e2_guards import pull_transport_cpu_offload_default + + if not envs.VLLM_E2_PLE_PULL_TRANSPORT: + return envs.VLLM_PLE_CPU_OFFLOAD + return pull_transport_cpu_offload_default(False) + + def model_has_engram_layers(model_config: "ModelConfig | None") -> bool: """Whether the model carries n-gram embedding layers.""" if model_config is None: @@ -35,8 +47,12 @@ def model_has_engram_layers(model_config: "ModelConfig | None") -> bool: class EngramConfig: """Configuration for Engram embedding storage and sharding.""" - cpu_offload: bool = True - """Store embedding weights in pinned CPU memory for UVA lookup.""" + cpu_offload: bool = Field(default_factory=_default_cpu_offload) + """Store embedding weights in pinned CPU memory for UVA lookup. + Defaults to VLLM_PLE_CPU_OFFLOAD, which is enabled by default. An explicit + value takes precedence over the environment variable. With + VLLM_E2_PLE_PULL_TRANSPORT=1 the default is False and a value that would + enable UVA fails the boot (vllm/v1/e2_guards.py).""" host_file_gather: bool = False """Serve Qwen4Exp PLE rows by gathering them on the CPU from the loader's diff --git a/vllm/config/parallel.py b/vllm/config/parallel.py index 7df8f4e0a4..2c6564bb22 100644 --- a/vllm/config/parallel.py +++ b/vllm/config/parallel.py @@ -18,7 +18,7 @@ from vllm.config.fault_tolerance import FaultToleranceConfig from vllm.config.utils import config from vllm.logger import init_logger from vllm.platforms import current_platform -from vllm.utils.network_utils import get_open_ports_list +from vllm.utils.network_utils import get_open_ports_list, get_open_zmq_ipc_path if TYPE_CHECKING: from ray.runtime_env import RuntimeEnv @@ -369,6 +369,10 @@ class ParallelConfig: """Port of the coordination TCPStore. Can be set by the API server; workers connect as clients to exchange self-picked group ports at runtime.""" + _ple_offload_ipc_path: str = "" + """Node-local ZMQ IPC address for the fork's PLE pull-transport offload + process. Set only when VLLM_E2_PLE_PULL_TRANSPORT=1.""" + decode_context_parallel_size: int = Field(default=1, ge=1) """Number of ranks that shard the decode KV cache. DCP does not expand the process world size. Without PCP, DCP reuses TP ranks. With PCP, DCP @@ -517,6 +521,9 @@ class ParallelConfig: "The FT system assumes one AsyncMPClient manages all engines." ) + if envs.VLLM_E2_PLE_PULL_TRANSPORT and not self._ple_offload_ipc_path: + self._ple_offload_ipc_path = get_open_zmq_ipc_path() + if self.all2all_backend in ["pplx", "naive"]: logger.warning( "The '%s' all2all backend has been removed. " @@ -910,6 +917,7 @@ class ParallelConfig: "worker_extension_cls", "_api_process_count", "_api_process_rank", + "_ple_offload_ipc_path", # NUMA binding is per-rank host-side memory locality; it does # not affect collective-communication semantics. When numa_bind # is enabled with auto-detection, each DP rank stores its own diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index a6c979400b..52868756b0 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1249,6 +1249,11 @@ class VllmConfig: # PyTorch's expandable_segments allocator uses CUDA VMM, which can # remap a virtual address range to different physical pages over the # engine's lifetime. KV connectors that pin KV cache memory (e.g. + # Fork: an explicit cpu_offload=True beside the pull transport is + # a double offload; refused here, before any worker starts. + from vllm.v1.e2_guards import verify_engram_against_pull + + verify_engram_against_pull(self.engram_config) # NixlConnector via ibv_reg_mr, MooncakeConnector) end up with their # registrations pointing at stale physical pages after any remap, # producing RDMA failures like IBV_WC_REM_ACCESS_ERR / diff --git a/vllm/envs.py b/vllm/envs.py index 97b2b8d216..38ae7fd01b 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -382,6 +382,18 @@ if TYPE_CHECKING: # to decide behaviour. Registered with the medium-path exact restart # so the cache-factor contract is complete. MERLIN_TOPK_EXACT_MEDIUM: str | None = None + # The fork's PLE pull transport -- the host-mapped, + # fail-closed offload process in vllm/v1/ple_offload/. Renamed from the E2 + # base's VLLM_PLE_CPU_OFFLOAD, which vLLM v0.30.0 reuses for a DIFFERENT + # subsystem with the OPPOSITE default (upstream: pinned-host UVA lookup, + # default ON). The two subsystems no longer share a switch. + # Strict: unset or "0" = off, "1" = on, anything else is a boot error. + # On: the PLE table lives in the offload process, the GPU placeholder waits + # on a host flag and DMAs rows inside the captured forward, and upstream's + # UVA backend is refused (vllm/v1/e2_guards.py). Off: the tree is upstream. + # AOT: yes (different module class and forward). CAPTURE: yes (the captured + # graph contains the wait/copy ops instead of a table lookup). Hashed. + VLLM_E2_PLE_PULL_TRANSPORT: bool = False # # RUNTIME-ONLY (registered for validate_environ + a single cached read): # Static K/V scale sidecar: fill_()ed into existing fp32 device buffers @@ -780,6 +792,21 @@ def _resolve_rust_cli_path() -> str | None: return raw +def _e2_strict_switch(name: str) -> bool: + """Parse a fork on/off switch that accepts exactly ``"0"`` and ``"1"``. + + Unset means off. Anything else -- ``"true"``, ``"yes"``, ``""``, ``" 1"``, + ``"01"`` -- is a boot error: a switch that selects a different transport + must not be satisfiable by a spelling nobody meant. + """ + raw = os.environ.get(name) + if raw is None or raw == "0": + return False + if raw == "1": + return True + raise ValueError(f'{name} must be exactly "0" or "1" (or unset), got {raw!r}') + + def _e2_diagnostic_mode(name: str) -> str: """Normalize an E2 diagnostic gate to off|warn|raise. @@ -2328,6 +2355,10 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_QSA_KV_COLLECT": lambda: os.getenv("VLLM_QSA_KV_COLLECT", "").strip(), # Capture-configuration factor, read in C++; raw passthrough on purpose. "MERLIN_TOPK_EXACT_MEDIUM": lambda: os.getenv("MERLIN_TOPK_EXACT_MEDIUM", None), + # PLE pull transport switch; strict "0"/"1" (see the TYPE_CHECKING block). + "VLLM_E2_PLE_PULL_TRANSPORT": lambda: _e2_strict_switch( + "VLLM_E2_PLE_PULL_TRANSPORT" + ), # Runtime only; all of the following are in compile_factors.ignored_factors. "VLLM_E2_ALLOW_BREAKABLE": lambda: os.getenv("VLLM_E2_ALLOW_BREAKABLE", "") == "1", "VLLM_E2_EXPECTED_CUDAGRAPH_MODE": lambda: os.getenv( diff --git a/vllm/model_executor/layers/ple_offload_layer.py b/vllm/model_executor/layers/ple_offload_layer.py index 1b67578d9b..d1f668380b 100644 --- a/vllm/model_executor/layers/ple_offload_layer.py +++ b/vllm/model_executor/layers/ple_offload_layer.py @@ -346,7 +346,7 @@ class PleOffloadLayer(nn.Module, ABC): def guarded_init( self: "PleOffloadLayer", *args: object, **kwargs: object ) -> None: - if envs.VLLM_PLE_CPU_OFFLOAD and not is_offload_process(): + if envs.VLLM_E2_PLE_PULL_TRANSPORT and not is_offload_process(): nn.Module.__init__(self) return original_init(self, *args, **kwargs) @@ -356,7 +356,7 @@ class PleOffloadLayer(nn.Module, ABC): @classmethod def get_target_device(cls) -> torch.device: """Return CPU for the offload process and the active GPU otherwise.""" - if envs.VLLM_PLE_CPU_OFFLOAD: + if envs.VLLM_E2_PLE_PULL_TRANSPORT: return torch.device("cpu") return torch.device("cuda", torch.accelerator.current_device_index()) diff --git a/vllm/models/qwen4_exp/common/ngram_embedding.py b/vllm/models/qwen4_exp/common/ngram_embedding.py index b387d3f364..b15aa0c2ea 100644 --- a/vllm/models/qwen4_exp/common/ngram_embedding.py +++ b/vllm/models/qwen4_exp/common/ngram_embedding.py @@ -15,6 +15,7 @@ import torch import torch.nn.functional as F from torch import nn +import vllm.envs as envs from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.distributed import get_dp_group, get_etp_group, get_tp_group from vllm.forward_context import DPMetadata, get_forward_context @@ -422,6 +423,13 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding): max_total_tokens: int = 0, data_parallel_rank: int = 0, ) -> None: + # Fork: never build the UVA table beside the E2 pull transport -- + # that would be a second offload of the same PLE table. + if envs.VLLM_E2_PLE_PULL_TRANSPORT: + raise RuntimeError( + "Qwen4ExpPLEPinnedHostEmbedding (upstream UVA PLE backend) " + "refused: VLLM_E2_PLE_PULL_TRANSPORT=1 owns the PLE table" + ) if not is_uva_available(): raise RuntimeError("Engram CPU offload requires UVA support") super().__init__( diff --git a/vllm/models/qwen4_exp/nvidia/model.py b/vllm/models/qwen4_exp/nvidia/model.py index f097b0d425..255aee6ae5 100644 --- a/vllm/models/qwen4_exp/nvidia/model.py +++ b/vllm/models/qwen4_exp/nvidia/model.py @@ -78,7 +78,7 @@ from vllm.multimodal.inputs import MultiModalFeatureSpec from vllm.sequence import IntermediateTensors from vllm.tokenizers.registry import cached_tokenizer_from_config from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum -from vllm.v1.e2_guards import assert_graph_profile_is_executable +from vllm.v1.e2_guards import assert_graph_profile_is_executable, assert_ple_mode from vllm.v1.kv_cache_interface import MambaSpec from ..common.mtp import make_mtp_hidden_buffer @@ -767,6 +767,7 @@ class Qwen4ExpForCausalLM( # identity nobody chose. Qwen4ExpForConditionalGeneration constructs # this class; Qwen4ExpMTP does not, and guards itself. assert_graph_profile_is_executable(vllm_config, role="target") + assert_ple_mode(vllm_config, role="target") config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config if vllm_config.lora_config is not None: # LoRA does not support the merged QKV/indexer projection, so its diff --git a/vllm/models/qwen4_exp/nvidia/mtp.py b/vllm/models/qwen4_exp/nvidia/mtp.py index 6adefb785b..d4f29257a9 100644 --- a/vllm/models/qwen4_exp/nvidia/mtp.py +++ b/vllm/models/qwen4_exp/nvidia/mtp.py @@ -51,7 +51,7 @@ from vllm.model_executor.models.utils import ( maybe_prefix, ) from vllm.sequence import IntermediateTensors -from vllm.v1.e2_guards import assert_graph_profile_is_executable +from vllm.v1.e2_guards import assert_graph_profile_is_executable, assert_ple_mode from .hyperconnection import GatedResidual, HyperConnectionConfig from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm @@ -400,6 +400,7 @@ class Qwen4ExpMTP( # this class builds Qwen4ExpMultiTokenPredictor directly, from a config # rebuilt at mtp.py:159 -- so its graph profile is checked here. assert_graph_profile_is_executable(vllm_config, role="draft") + assert_ple_mode(vllm_config, role="draft") config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config self.vllm_config = vllm_config self.quant_config = vllm_config.quant_config diff --git a/vllm/models/qwen4_exp/nvidia/ngram_embedding.py b/vllm/models/qwen4_exp/nvidia/ngram_embedding.py index eedde22e6d..15b8f96f80 100644 --- a/vllm/models/qwen4_exp/nvidia/ngram_embedding.py +++ b/vllm/models/qwen4_exp/nvidia/ngram_embedding.py @@ -10,6 +10,7 @@ import torch from torch import nn from transformers import Qwen4ExpTextConfig +import vllm.envs as envs from vllm.config import get_current_vllm_config from vllm.logger import init_logger from vllm.model_executor.layers.quantization.base_config import ( @@ -357,11 +358,18 @@ class Qwen4ExpNGramEmbedding(nn.Module): if params_dtype is None: params_dtype = torch.get_default_dtype() engram_config = get_current_vllm_config().engram_config + use_uva = engram_config is not None and engram_config.cpu_offload + if use_uva and envs.VLLM_E2_PLE_PULL_TRANSPORT: + # Fork: no silent fallback between transports. + raise RuntimeError( + "PLE backend selection: cpu_offload=True (upstream UVA) with " + "VLLM_E2_PLE_PULL_TRANSPORT=1 -- refusing a double offload" + ) embedding_cls = ( Qwen4ExpPLEFileGatherEmbedding if engram_config is not None and engram_config.host_file_gather else Qwen4ExpPLEPinnedHostEmbedding - if engram_config is not None and engram_config.cpu_offload + if use_uva else Qwen4ExpPLEDeviceEmbedding ) self.ngram_embedding = embedding_cls( @@ -382,14 +390,21 @@ class Qwen4ExpNGramEmbedding(nn.Module): max_total_tokens, self.ngram_heads, dtype=torch.long ) weight = self.ngram_embedding.weight + # Fork: backend= names the class (or, inside the pull transport's + # offload process, the table it serves), so a smoke need not derive + # the backend from pinned=. Fields before it are upstream's, unchanged. + backend = type(self.ngram_embedding).__name__ + if envs.VLLM_E2_PLE_PULL_TRANSPORT: + backend = f"E2PullTransport.offload_table:{backend}" logger.info( "Initialized PLE embedding %s: quantization_method=%s, " - "weight_dtype=%s, weight_device=%s, pinned=%s", + "weight_dtype=%s, weight_device=%s, pinned=%s, backend=%s", embedding_prefix, type(embedding_quant_method).__name__, weight.dtype, weight.device, weight.is_pinned(), + backend, ) @staticmethod @@ -402,6 +417,13 @@ class Qwen4ExpNGramEmbedding(nn.Module): positions = torch.arange(seq_len, device=tokens.device, dtype=torch.int64) eos_positions = torch.where(tokens == eos_token_id, positions, -1) previous_eos_inclusive = torch.cummax(eos_positions, dim=1).values + # Fork: never build the UVA table beside the E2 pull transport -- + # that would be a second offload of the same PLE table. + if envs.VLLM_E2_PLE_PULL_TRANSPORT: + raise RuntimeError( + "Qwen4ExpPLEPinnedHostEmbedding (upstream UVA PLE backend) " + "refused: VLLM_E2_PLE_PULL_TRANSPORT=1 owns the PLE table" + ) previous_eos = torch.cat( [ eos_positions.new_full((batch_size, 1), -1), diff --git a/vllm/models/qwen4_exp/nvidia/ple_layer.py b/vllm/models/qwen4_exp/nvidia/ple_layer.py index 95f832a365..b48c1d9bd2 100644 --- a/vllm/models/qwen4_exp/nvidia/ple_layer.py +++ b/vllm/models/qwen4_exp/nvidia/ple_layer.py @@ -8,6 +8,7 @@ import torch from torch import nn from transformers import Qwen4ExpTextConfig +import vllm.envs as envs from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config from vllm.forward_context import get_forward_context @@ -91,16 +92,32 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): self.conv_state_len = (self.conv_kernel_size - 1) * self.short_conv_dilation self.num_spec_tokens = vllm_config.num_speculative_tokens self.activation = "silu" - self.ple_embedding = Qwen4ExpNGramEmbedding( - config, - int(config.ple_embed_dim), - self.ple_dense_layer_id, - vllm_config.scheduler_config.max_num_batched_tokens, - data_parallel_rank=vllm_config.parallel_config.data_parallel_rank, - prefix=f"{prefix}.ple_embedding", - quant_config=quant_config, - params_dtype=model_config.dtype, - ) + if envs.VLLM_E2_PLE_PULL_TRANSPORT: + # Fork: the E2 pull transport owns the table in a CPU-only + # offload process; upstream's device/UVA backends are never built. + from .ple_pull import Qwen4ExpNGramEmbeddingPull + + self.ple_embedding = Qwen4ExpNGramEmbeddingPull.build( + config, + int(config.ple_embed_dim), + self.ple_dense_layer_id, + vllm_config.scheduler_config.max_num_batched_tokens, + data_parallel_rank=vllm_config.parallel_config.data_parallel_rank, + prefix=f"{prefix}.ple_embedding", + quant_config=quant_config, + params_dtype=model_config.dtype, + ) + else: + self.ple_embedding = Qwen4ExpNGramEmbedding( + config, + int(config.ple_embed_dim), + self.ple_dense_layer_id, + vllm_config.scheduler_config.max_num_batched_tokens, + data_parallel_rank=vllm_config.parallel_config.data_parallel_rank, + prefix=f"{prefix}.ple_embedding", + quant_config=quant_config, + params_dtype=model_config.dtype, + ) # The PLE cache is TP-replicated, so this merged projection is too. self.kv_proj = MergedColumnParallelLinear( int(config.ple_embed_dim), @@ -145,6 +162,10 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): ) -> torch.Tensor: """Dequantize PLE lookup output.""" return self.ple_embedding.ngram_embedding.dequantize( + dequantize_pulled = getattr(self.ple_embedding, "dequantize_pulled", None) + if dequantize_pulled is not None: + # Fork pull transport: no local table; FP8 scale kept here. + return dequantize_pulled(embeddings, output_dtype) embeddings, output_dtype, ) diff --git a/vllm/models/qwen4_exp/nvidia/ple_pull.py b/vllm/models/qwen4_exp/nvidia/ple_pull.py new file mode 100644 index 0000000000..f52000c8d9 --- /dev/null +++ b/vllm/models/qwen4_exp/nvidia/ple_pull.py @@ -0,0 +1,342 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""The fork's PLE pull transport, re-seated on vLLM v0.30.0. + +v0.30.0 split the E2 base's offload-aware ``Qwen4ExpNGramEmbedding`` into an +upstream-only module (``ngram_embedding.py``, device or pinned-host UVA table). +This module puts the E2 pull transport back WITHOUT editing that upstream +class: ``Qwen4ExpNGramEmbeddingPull`` inherits it and ``PleOffloadLayer``. + +* In a GPU worker, ``PleOffloadLayer.__init_subclass__`` skips the constructor: + the module is an empty placeholder with no table. ``PleOffloadConnector`` + binds it to a host-mapped flag/buffer; its forward is the captured + ``ple_offload_wait_copy`` op. Only the FP8 global scale is kept on the GPU. +* In the CPU-only offload process the constructor runs in full, on CPU. The + table is upstream's ``Qwen4ExpPLEDeviceEmbedding`` allocated on CPU; + ``forward_impl`` computes n-gram IDs and gathers raw (FP8) rows. + +Selected only when ``VLLM_E2_PLE_PULL_TRANSPORT=1``; with the switch off this +module is never imported by the model and the tree is upstream. + +Supported table formats: FP8 (global scale) and unquantized -- the two +``Qwen4ExpPLEEmbeddingMethod`` families v0.30.0 itself supports. The E2 NVFP4 +path is not carried (v0.30.0 has no NVFP4 PLE method to select). +""" + +from collections.abc import Iterable + +import torch + +from vllm.config import get_current_vllm_config +from vllm.logger import init_logger +from vllm.model_executor.layers.ple_offload_layer import ( + PleOffloadLayer, + is_offload_process, +) +from vllm.model_executor.layers.quantization.base_config import QuantizationConfig +from vllm.transformers_utils.configs.qwen4_exp import Qwen4ExpTextConfig + +from .ngram_embedding import ( + Qwen4ExpNGramEmbedding, + Qwen4ExpPLEEmbeddingMethod, + Qwen4ExpPLEFp8EmbeddingMethod, + Qwen4ExpPLEUnquantizedEmbeddingMethod, +) + +logger = init_logger(__name__) + +#: The backend name this module reports in the PLE producer log line. +PULL_BACKEND_NAME = "E2PullTransport" + + +class Qwen4ExpNGramEmbeddingPull(PleOffloadLayer, Qwen4ExpNGramEmbedding): + """Upstream n-gram embedding whose table lives in the offload process.""" + + #: GPU placeholder: the method the offload process will use for the table, + #: resolved with upstream's own selector (set by ``build``). + _offload_quant_method: Qwen4ExpPLEEmbeddingMethod | None = None + + def __init__( + self, + config: Qwen4ExpTextConfig, + embedding_dim: int, + ple_dense_layer_id: int, + max_total_tokens: int, + *, + data_parallel_rank: int, + prefix: str, + quant_config: QuantizationConfig | None = None, + params_dtype: torch.dtype | None = None, + ) -> None: + # Runs only in the offload process (guarded by PleOffloadLayer). + engram_config = get_current_vllm_config().engram_config + if engram_config is not None and engram_config.cpu_offload: + raise RuntimeError( + "PLE pull transport: the offload process resolved " + "cpu_offload=True; refusing to build upstream's UVA table here." + ) + Qwen4ExpNGramEmbedding.__init__( + self, + config, + embedding_dim, + ple_dense_layer_id, + max_total_tokens, + data_parallel_rank=data_parallel_rank, + prefix=prefix, + quant_config=quant_config, + params_dtype=params_dtype, + ) + + @classmethod + def build( + cls, + config: Qwen4ExpTextConfig, + embedding_dim: int, + ple_dense_layer_id: int, + max_total_tokens: int, + *, + data_parallel_rank: int, + prefix: str, + quant_config: QuantizationConfig | None, + params_dtype: torch.dtype | None, + ) -> "Qwen4ExpNGramEmbeddingPull": + """Construct the placeholder (GPU) or the full CPU module (offload).""" + # The offload process builds the surrounding model on meta; this + # subtree must own real CPU storage. In a GPU worker the constructor is + # skipped, so the device context allocates nothing. + with torch.device(cls.get_target_device()): + module = cls( + config, + embedding_dim, + ple_dense_layer_id, + max_total_tokens, + data_parallel_rank=data_parallel_rank, + prefix=prefix, + quant_config=quant_config, + params_dtype=params_dtype, + ) + embedding_prefix = f"{prefix}.ngram_embedding" + method = Qwen4ExpPLEEmbeddingMethod.from_quant_config( + quant_config, + embedding_prefix, + getattr(config, "ple_embedding_dtype", None), + ) + if not isinstance( + method, + (Qwen4ExpPLEFp8EmbeddingMethod, Qwen4ExpPLEUnquantizedEmbeddingMethod), + ): + raise NotImplementedError( + f"PLE pull transport does not support {type(method).__name__}" + ) + module._offload_quant_method = method + module._pull_params_dtype = params_dtype or torch.get_default_dtype() + module._pull_embedding_prefix = embedding_prefix + if not is_offload_process(): + # The GPU-rank producer line, in upstream's format plus backend= + # (the offload process logs its own table line from upstream's + # constructor). The table lives in the offload process's CPU memory + # and is not pinned; rows cross via host-mapped buffers. + logger.info( + "Initialized PLE embedding %s: quantization_method=%s, " + "weight_dtype=%s, weight_device=%s, pinned=%s, backend=%s", + embedding_prefix, + type(method).__name__, + module.get_offload_output_dtype(module._pull_params_dtype), + "cpu", + False, + PULL_BACKEND_NAME, + ) + return module + + # ------------------------------------------------------------- formats + + def get_offload_output_dtype(self, default_dtype: torch.dtype) -> torch.dtype: + """Rows cross the transport in their table storage dtype.""" + embedding = getattr(self, "ngram_embedding", None) + weight = getattr(embedding, "weight", None) + if weight is not None: + return weight.dtype + if isinstance(self._offload_quant_method, Qwen4ExpPLEFp8EmbeddingMethod): + return torch.float8_e4m3fn + return getattr(self, "_pull_params_dtype", default_dtype) + + def get_offload_output_dim(self, default_dim: int) -> int: + return default_dim + + def initialize_dummy_offload_metadata(self, device: torch.device) -> None: + """Dummy loads skip checkpoint scales; give FP8 a unit scale.""" + if isinstance(self._offload_quant_method, Qwen4ExpPLEFp8EmbeddingMethod): + self.register_buffer( + "_offload_weight_scale", + torch.ones((), dtype=torch.float32, device=device), + persistent=False, + ) + + def dequantize_pulled( + self, embeddings: torch.Tensor, output_dtype: torch.dtype + ) -> torch.Tensor: + """GPU-side dequantization of rows DMA'd from the offload process. + + Same arithmetic as upstream ``Qwen4ExpPLEFp8EmbeddingMethod.dequantize`` + (and the E2 ``_dequantize_embeddings`` FP8 branch). + """ + method = self._offload_quant_method + if isinstance(method, Qwen4ExpPLEUnquantizedEmbeddingMethod): + return embeddings + weight_scale = getattr(self, "_offload_weight_scale", None) + if weight_scale is None: + raise RuntimeError( + "FP8 PLE pull transport is missing its global scale on the GPU" + ) + if weight_scale.device != embeddings.device: + raise RuntimeError("FP8 PLE embedding scale must be on the output device") + return embeddings.to(output_dtype) * weight_scale.to(output_dtype) + + # ------------------------------------------------------------- execution + + def start_prefetch( + self, + hidden_states: torch.Tensor, + input_ids: torch.Tensor, + query_start_loc: torch.Tensor, + ngram_context: torch.Tensor, + ) -> None: + """No prefetch: rows are published by the offload process.""" + return None + + def _compute_ngram_ids_offload( + self, + input_ids: torch.Tensor, + query_start_loc: torch.Tensor, + ngram_context: torch.Tensor, + ) -> torch.Tensor: + """CPU n-gram IDs, carried unchanged from the E2 offload path. + + Differs from upstream's CPU branch in two E2 fixes: the packed layout + is (num_reqs, max_seq_len) rather than (num_reqs, num_tokens), and + CUDA-graph padding rows beyond ``query_start_loc[-1]`` are not written, + so they cannot overwrite a request's final real token. + """ + input_ids = input_ids.reshape(-1).long() + query_start_loc = query_start_loc.long() + num_reqs = query_start_loc.numel() - 1 + num_tokens = input_ids.shape[0] + if num_reqs <= 0: + raise ValueError("PLE requires at least one request") + max_seq_len = max( + 1, + int((query_start_loc[1:] - query_start_loc[:-1]).max().item()), + ) + num_valid_tokens = min(int(query_start_loc[-1].item()), num_tokens) + + positions = torch.arange(num_tokens, device=input_ids.device, dtype=torch.int64) + packed = torch.full( + (num_reqs, max_seq_len), + self.eos_token_id, + device=input_ids.device, + dtype=torch.int64, + ) + request_indices = torch.searchsorted(query_start_loc, positions, right=True) - 1 + request_indices.clamp_(max=num_reqs - 1) + columns = (positions - query_start_loc[request_indices]).clamp( + 0, packed.shape[1] - 1 + ) + packed[request_indices[:num_valid_tokens], columns[:num_valid_tokens]] = ( + input_ids[:num_valid_tokens] + ) + ngram_context = ngram_context[:num_reqs].to( + device=input_ids.device, dtype=torch.long + ) + + context = torch.cat([ngram_context, packed], dim=-1) + positions_2d, position_in_segment = self._shift_precompute( + context, self.eos_token_id + ) + shifted = [context] + for shift in range(1, self.ngram_size): + shifted.append( + self._shift_apply( + context, + positions_2d, + position_in_segment, + shift, + self.eos_token_id, + ) + ) + adjusted_columns = columns + self.ngram_size - 1 + id_blocks = [] + for ngram in range(2, self.ngram_size + 1): + start = (ngram - 2) * self.heads_per_ngram + end = start + self.heads_per_ngram + mixed = shifted[0] * self.layer_multipliers[0] + for index in range(1, ngram): + mixed = torch.bitwise_xor( + mixed, shifted[index] * self.layer_multipliers[index] + ) + sizes = self.ngram_heads_vocab_sizes[start:end] + offsets = self.ngram_heads_offsets[start:end] + ids = torch.remainder(mixed.unsqueeze(-1), sizes) + offsets + id_blocks.append(ids[request_indices, adjusted_columns]) + return torch.cat(id_blocks, dim=-1) + + def forward_impl( # type: ignore[override] + self, + hidden_states: torch.Tensor, + input_ids: torch.Tensor, + query_start_loc: torch.Tensor, + ngram_context: torch.Tensor, + output_buffer: torch.Tensor | None = None, + ) -> torch.Tensor: + """Offload process only: gather raw table rows for these tokens.""" + del hidden_states + if not is_offload_process(): + raise RuntimeError( + "PLE pull transport: forward_impl ran outside the offload " + "process -- the GPU placeholder was never bound by the connector" + ) + ngram_ids = self._compute_ngram_ids_offload( + input_ids, query_start_loc, ngram_context + ) + num_tokens = input_ids.reshape(-1).shape[0] + weight = self.ngram_embedding.weight + if output_buffer is not None: + output = output_buffer[:num_tokens, : self.embedding_dim] + torch.index_select( + weight, + 0, + ngram_ids.reshape(-1), + out=output.reshape(-1, self.head_dim), + ) + return output + return torch.index_select(weight, 0, ngram_ids.reshape(-1)).reshape( + num_tokens, self.embedding_dim + ) + + # ------------------------------------------------------------- loading + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + """Offload process: upstream loading. GPU worker: keep the FP8 scale. + + The GPU placeholder owns no table; every other checkpoint tensor for + this subtree is consumed and dropped here (the offload process loads + the table itself). Prefix grouping may call this more than once. + """ + if is_offload_process(): + return Qwen4ExpNGramEmbedding.load_weights(self, weights) + retained: set[str] = set() + fp8 = isinstance(self._offload_quant_method, Qwen4ExpPLEFp8EmbeddingMethod) + for name, loaded_weight in weights: + if fp8 and name == "ngram_embedding.weight_scale": + self.register_buffer( + "_offload_weight_scale", + loaded_weight.reshape(()).to( + device=torch.accelerator.current_accelerator(), + dtype=torch.float32, + ), + persistent=False, + ) + retained.add(name) + return retained + + +__all__ = ["PULL_BACKEND_NAME", "Qwen4ExpNGramEmbeddingPull"] diff --git a/vllm/v1/e2_guards.py b/vllm/v1/e2_guards.py index 9971ccd39a..12a0d4864e 100644 --- a/vllm/v1/e2_guards.py +++ b/vllm/v1/e2_guards.py @@ -214,5 +214,151 @@ def assert_graph_profile_is_executable(vllm_config, *, role: str = "target") -> ) +# -------------------------------------------------------------------------- +# PLE transport: carry the pull transport, UVA disabled. +# -------------------------------------------------------------------------- + +#: The fork's pull-transport switch (strict "0"/"1"; registered and hashed in +#: vllm/envs.py). Renamed from the E2 base's VLLM_PLE_CPU_OFFLOAD. +PULL_TRANSPORT_ENV = "VLLM_E2_PLE_PULL_TRANSPORT" +#: Upstream v0.30.0's switch for the pinned-host UVA PLE backend. Same name as +#: the E2 base's pull switch, opposite default (upstream: "1"). +UPSTREAM_UVA_ENV = "VLLM_PLE_CPU_OFFLOAD" + + +def pull_transport_enabled() -> bool: + """Whether the fork's pull transport is on. Strict; raises on bad values. + + One source: the registered accessor ``envs.VLLM_E2_PLE_PULL_TRANSPORT``, + whose parser (``envs._e2_strict_switch``) accepts exactly unset/"0"/"1". + """ + import vllm.envs as envs + + return bool(envs.VLLM_E2_PLE_PULL_TRANSPORT) + + +def check_upstream_uva_env_against_pull() -> None: + """Fail loud if the old name would turn upstream's UVA path on. + + With the pull transport on, ``VLLM_PLE_CPU_OFFLOAD`` may be unset (the + fork then supplies ``cpu_offload=False`` as the Engram default) or exactly + ``"0"``. ``"1"`` -- the E2-era spelling of "enable our offload" -- would + enable upstream's pinned-host UVA backend beside ours: a double offload. + Any other value is rejected rather than interpreted. With the pull + transport off this function does nothing: the name is upstream's. + """ + if not pull_transport_enabled(): + return + raw = os.environ.get(UPSTREAM_UVA_ENV) + if raw is None or raw == "0": + return + if raw == "1": + raise RuntimeError( + f"e2-guard: {PULL_TRANSPORT_ENV}=1 and {UPSTREAM_UVA_ENV}=1. On " + f"vLLM v0.30.0 {UPSTREAM_UVA_ENV} is UPSTREAM's switch for the " + "pinned-host UVA PLE backend (default on), not the E2 pull " + "transport, so this launch asks for both transports at once. Unset " + f"{UPSTREAM_UVA_ENV} or export {UPSTREAM_UVA_ENV}=0 " + "(the pull transport replaces that switch)." + ) + raise ValueError( + f"e2-guard: {PULL_TRANSPORT_ENV}=1 and {UPSTREAM_UVA_ENV}={raw!r}. " + f'With the pull transport on, {UPSTREAM_UVA_ENV} must be unset or ' + 'exactly "0"; any other value is refused rather than guessed.' + ) + + +def pull_transport_cpu_offload_default(upstream_default: bool) -> bool: + """The Engram ``cpu_offload`` default, as this fork resolves it. + + Pull off: ``upstream_default`` unchanged (the tree behaves as upstream). + Pull on: the old-name check above, then ``False`` -- upstream's UVA + backend is never the default beside the pull transport. + """ + if not pull_transport_enabled(): + return upstream_default + check_upstream_uva_env_against_pull() + return False + + +def verify_engram_against_pull(engram_config) -> None: + """Refuse an explicit ``cpu_offload=True`` while the pull transport is on. + + ``--engram-config '{"cpu_offload": true}'`` takes precedence over every + default, so it is checked on the resolved object, not the environment. + """ + if not pull_transport_enabled(): + return + check_upstream_uva_env_against_pull() + if engram_config is not None and bool(engram_config.cpu_offload): + raise RuntimeError( + f"e2-guard: {PULL_TRANSPORT_ENV}=1 but the resolved EngramConfig " + "has cpu_offload=True (upstream's pinned-host UVA PLE backend). " + "Both transports would own the PLE table. Drop cpu_offload from " + "--engram-config (or set it false), or turn the pull transport off." + ) + + +#: Effective PLE modes, as the guard names them. +PLE_MODE_PULL = "pull" # E2 pull transport (VLLM_E2_PLE_PULL_TRANSPORT=1) +PLE_MODE_UVA = "uva" # upstream Qwen4ExpPLEPinnedHostEmbedding +PLE_MODE_DEVICE = "device" # upstream Qwen4ExpPLEDeviceEmbedding +PLE_MODE_NONE = "none" # no Engram config resolved (no PLE layers / not CUDA) + + +def resolve_ple_mode(vllm_config) -> str: + """The PLE transport this process will build, or an error if mixed. + + Reads the two inputs that decide it -- the pull switch (strict) and the + RESOLVED ``engram_config.cpu_offload`` -- and nothing else. Mixed states + raise; there is no fallback from one transport to another. + """ + pull = pull_transport_enabled() + check_upstream_uva_env_against_pull() + engram_config = getattr(vllm_config, "engram_config", None) + cpu_offload = ( + None if engram_config is None else bool(engram_config.cpu_offload) + ) + if pull: + if cpu_offload: + raise RuntimeError( + f"e2-guard: {PULL_TRANSPORT_ENV}=1 with a resolved Engram " + "cpu_offload=True: the pull transport and upstream's UVA " + "backend would both own the PLE table." + ) + return PLE_MODE_PULL + if cpu_offload is None: + return PLE_MODE_NONE + return PLE_MODE_UVA if cpu_offload else PLE_MODE_DEVICE + + +def assert_ple_mode(vllm_config, *, role: str = "target") -> str: + """Startup guard: resolve, assert and log the effective PLE mode. + + Runs beside ``assert_graph_profile_is_executable`` at model construction + for ``role="target"`` and ``role="draft"`` (the MTP drafter carries the + target's EngramConfig and the same process environment, so a mixed + configuration fails there too). NOT disabled by VLLM_E2_ALLOW_BREAKABLE: + that opt-out is about graph identity, not about owning the PLE table twice. + """ + mode = resolve_ple_mode(vllm_config) + model_config = getattr(vllm_config, "model_config", None) + text_config = getattr(model_config, "hf_text_config", None) + ple_layer_ids = list(getattr(text_config, "ple_layer_ids", None) or []) + engram_config = getattr(vllm_config, "engram_config", None) + logger.info( + "e2-guard[%s]: ple_mode=%s pull_transport=%d engram_cpu_offload=%s " + "%s=%s ple_layer_ids=%s.", + role, + mode, + 1 if mode == PLE_MODE_PULL else 0, + None if engram_config is None else bool(engram_config.cpu_offload), + UPSTREAM_UVA_ENV, + os.environ.get(UPSTREAM_UVA_ENV, ""), + ple_layer_ids, + ) + return mode + + #: Back-compatible name for the earlier call sites. assert_breakable_cudagraph_pinned = assert_graph_profile_is_executable diff --git a/vllm/v1/executor/multiproc_executor.py b/vllm/v1/executor/multiproc_executor.py index bd164656b5..fe11014b13 100644 --- a/vllm/v1/executor/multiproc_executor.py +++ b/vllm/v1/executor/multiproc_executor.py @@ -716,6 +716,10 @@ class WorkerProc: # Load model self.worker.init_device() + # PLE pull transport: DP0/TP0 starts the CPU-only PLE process before + # load_model, so its weight load overlaps ours; READY is awaited after. + if envs.VLLM_E2_PLE_PULL_TRANSPORT: + self.worker.spawn_ple_offload() # Update process title now that parallel groups are initialized self.setup_proc_title_and_log_prefix( enable_ep=vllm_config.parallel_config.enable_expert_parallel @@ -724,6 +728,8 @@ class WorkerProc: self.worker.elastic_ep_execute("load_model") else: self.worker.load_model() + if envs.VLLM_E2_PLE_PULL_TRANSPORT: + self.worker.wait_ple_offload_ready() scheduler_config = vllm_config.scheduler_config self.use_async_scheduling = scheduler_config.async_scheduling diff --git a/vllm/v1/executor/uniproc_executor.py b/vllm/v1/executor/uniproc_executor.py index dc798cdc5c..eeb0786297 100644 --- a/vllm/v1/executor/uniproc_executor.py +++ b/vllm/v1/executor/uniproc_executor.py @@ -67,11 +67,15 @@ class UniProcExecutor(Executor): self.driver_worker.init_worker(all_kwargs=[kwargs]) self.driver_worker.init_device() + if envs.VLLM_E2_PLE_PULL_TRANSPORT: + self.driver_worker.spawn_ple_offload() if envs.VLLM_ELASTIC_EP_SCALE_UP_LAUNCH: self.driver_worker.elastic_ep_execute("load_model") else: self.driver_worker.load_model() + if envs.VLLM_E2_PLE_PULL_TRANSPORT: + self.driver_worker.wait_ple_offload_ready() current_platform.update_block_size_for_backend(self.vllm_config) def _distributed_args(self) -> tuple[str, int, int]: diff --git a/vllm/v1/ple_offload/connector.py b/vllm/v1/ple_offload/connector.py index 52bc4622e7..2779edf3b5 100644 --- a/vllm/v1/ple_offload/connector.py +++ b/vllm/v1/ple_offload/connector.py @@ -257,7 +257,7 @@ class PleOffloadConnector: } if not layers: raise RuntimeError( - "VLLM_PLE_CPU_OFFLOAD is enabled, but the model has no PleOffloadLayer" + "VLLM_E2_PLE_PULL_TRANSPORT is enabled, but the model has no PleOffloadLayer" ) config = vllm_config.model_config.hf_text_config diff --git a/vllm/v1/ple_offload/test_fault.py b/vllm/v1/ple_offload/test_fault.py new file mode 100644 index 0000000000..9ae74f7205 --- /dev/null +++ b/vllm/v1/ple_offload/test_fault.py @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Compatibility exports for the top-level test-only fault singleton.""" + +from vllm_ple_test_fault import ( + _FAULT as _FAULT, +) +from vllm_ple_test_fault import ( # noqa: F401 + FaultSpec as FaultSpec, +) +from vllm_ple_test_fault import ( + _parse_fault_spec as _parse_fault_spec, +) diff --git a/vllm/v1/ple_offload/worker.py b/vllm/v1/ple_offload/worker.py index 24beebdea6..a5f039e407 100644 --- a/vllm/v1/ple_offload/worker.py +++ b/vllm/v1/ple_offload/worker.py @@ -830,7 +830,7 @@ class PleOffloadRunner: } if not offload_layers: raise RuntimeError( - "VLLM_PLE_CPU_OFFLOAD is enabled, but no PleOffloadLayer " + "VLLM_E2_PLE_PULL_TRANSPORT is enabled, but no PleOffloadLayer " "was found in the initialized model" ) logger.info( diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 9b96775453..fbb630243f 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -385,6 +385,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): # For transferring state from execute_model to subsequent sample_tokens call. self.execute_model_state: ExecuteModelState | None = None + # PLE pull transport: set by _setup_ple_offload() from Worker.load_model + # when VLLM_E2_PLE_PULL_TRANSPORT=1 and this rank owns a PLE layer. + # None otherwise, and every hook below is then a no-op (upstream path). + self._ple_offload_connector: PleOffloadConnector | None = None # Expert parallelism load balancer. self.eplb = EPLBController(self.parallel_config, self.device) @@ -408,6 +412,26 @@ class GPUModelRunner(LoRAModelRunnerMixin): tasks.extend(PoolingRunner.get_supported_tasks(self.model)) return tuple(tasks) + def _setup_ple_offload(self, ipc_addr: str) -> None: + """Initialize PLE offload after the model state is available.""" + # PLE placeholders are created by model loading, so CUDA output + # registration cannot happen in the runner constructor. + query_start_loc_source = getattr(self.model_state, "ple_query_start_loc", None) + ngram_context_source = getattr(self.model_state, "ngram_context", None) + if not isinstance(query_start_loc_source, torch.Tensor): + raise RuntimeError("PLE offload requires a query_start_loc source") + if not isinstance(ngram_context_source, torch.Tensor): + raise RuntimeError("PLE offload requires an ngram_context source") + self._ple_offload_connector = PleOffloadConnector( + self.vllm_config, + self.model, + self.device, + ipc_addr, + input_ids_source=self.input_buffers.input_ids, + query_start_loc_source=query_start_loc_source, + ngram_context_source=ngram_context_source, + ) + def load_model(self, load_dummy_weights: bool = False, *args, **kwargs) -> None: time_before_load = time.perf_counter() if load_dummy_weights: @@ -1102,6 +1126,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): torch.accelerator.empty_cache() start_free_gpu_memory = torch.accelerator.get_memory_info()[0] + # Decoder capture calls the model outside execute_model. Keep dummy + # PLE outputs signaled until all graphs capture their semaphore wait. + if self._ple_offload_connector is not None and capture_decoder: + self._ple_offload_connector.signal_dummy_outputs(self.max_num_tokens) with self.maybe_setup_dummy_loras(self.lora_config): if capture_encoder: self.model_state.encoder_runner.capture() @@ -1137,6 +1165,8 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.adaptive_verification.set_initial_cost_curves(timings) self.kv_connector.reset_capture_state() + if self._ple_offload_connector is not None and capture_decoder: + self._ple_offload_connector.release_outputs() end_free_gpu_memory = torch.accelerator.get_memory_info()[0] if not profile_only: @@ -2479,6 +2509,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): return None input_batch = self.execute_model_state.input_batch + if self._ple_offload_connector is not None: + self._ple_offload_connector.close() + self._ple_offload_connector = None hidden_states = self.execute_model_state.hidden_states finished_req_ids = self.execute_model_state.finished_req_ids ec_connector_output = self.execute_model_state.ec_connector_output diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index a90501415f..74c970594d 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -233,6 +233,22 @@ class Worker(WorkerBase): validate_worker_profiler_config(self.profiler_config) self.use_v2_model_runner = vllm_config.use_v2_model_runner + # PLE pull transport (VLLM_E2_PLE_PULL_TRANSPORT). Off: both stay + # None/False and every PLE hook in this class is a no-op. + self._ple_offload_worker_handle: Any | None = None + self._ple_offload_stopping = False + self._ple_offload_enabled = self._has_ple_layers() + if envs.VLLM_E2_PLE_PULL_TRANSPORT: + if self._ple_offload_enabled: + self._validate_ple_offload_config() + elif self.rank == 0 and self.parallel_config.data_parallel_rank == 0: + text_config = self.model_config.hf_text_config + logger.warning( + "VLLM_E2_PLE_PULL_TRANSPORT is enabled, but this rank owns " + "no PLE layer (ple_layer_ids=%s); no PLE offload connector " + "is created here.", + getattr(text_config, "ple_layer_ids", None), + ) # Device handles of the previous step's PP intermediate-tensor send. self._pp_send_work: list[Handle] = [] @@ -242,7 +258,7 @@ class Worker(WorkerBase): def _has_ple_layers(self) -> bool: """Return whether this model configuration constructs PLE layers.""" - if not envs.VLLM_PLE_CPU_OFFLOAD: + if not envs.VLLM_E2_PLE_PULL_TRANSPORT: return False text_config = self.model_config.hf_text_config ple_layer_ids = getattr(text_config, "ple_layer_ids", None) @@ -304,7 +320,7 @@ class Worker(WorkerBase): if unsupported: raise ValueError( - "VLLM_PLE_CPU_OFFLOAD does not support the requested " + "VLLM_E2_PLE_PULL_TRANSPORT does not support the requested " "configuration. Unsupported settings: " f"{', '.join(unsupported)}" ) @@ -718,6 +734,12 @@ class Worker(WorkerBase): ): self.model_runner.load_model(load_dummy_weights=load_dummy_weights) + # Model loading constructs the PLE placeholders whose buffers are + # registered with the offload process by the model runner V2 connector. + if self._ple_offload_enabled: + model_runner = cast("GPUModelRunnerV2", self.model_runner) + model_runner._setup_ple_offload(self.parallel_config._ple_offload_ipc_path) + if has_ec_transfer(): get_ec_transfer().start_worker_services() @@ -738,6 +760,10 @@ class Worker(WorkerBase): self.model_runner.update_config(overrides) def reload_weights(self, *args, **kwargs) -> None: + if self._ple_offload_enabled: + raise NotImplementedError( + "Weight reload is not supported with the PLE pull transport" + ) with set_current_vllm_config(self.vllm_config): self.model_runner.reload_weights(*args, **kwargs) @@ -1759,6 +1785,10 @@ class Worker(WorkerBase): self.elastic_ep_executor.shutdown() + if self._ple_offload_worker_handle is not None: + self._ple_offload_worker_handle.close() + self._ple_offload_worker_handle = None + # Release GPU resources held by the model runner so that memory # can be reclaimed when running in-process if model_runner := getattr(self, "model_runner", None):