diff --git a/tests/model_executor/model_loader/test_weight_cache.py b/tests/model_executor/model_loader/test_weight_cache.py new file mode 100644 index 0000000000..6084d5e7ed --- /dev/null +++ b/tests/model_executor/model_loader/test_weight_cache.py @@ -0,0 +1,153 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""End-to-end tests for the IPC weight cache loader. + +A cold start (no weight cache daemon, so the loader falls back to disk) and +warm restarts (weights mapped from the daemon via CUDA IPC) must both serve +identical outputs. +""" + +import shutil +import subprocess +import sys +import tempfile +import threading +import time + +from vllm import SamplingParams + +MODEL = "Qwen/Qwen3.5-0.8B" +PROMPTS = [ + "Hello, my name is", + "The capital of France is", +] +DAEMON_TIMEOUT_S = 600 + + +class WeightCacheDaemon: + """Context manager running the real weight cache daemon as a subprocess.""" + + _READY_MARKER = "Weight cache daemon READY" + + def __init__(self, model: str, tp_size: int): + # Short base path: Unix socket paths are limited to ~107 characters. + self.socket_dir = tempfile.mkdtemp(prefix="vllm_ipc_") + self._cmd = [ + sys.executable, + "-m", + "vllm.model_executor.model_loader.weight_cache.daemon", + "--model", + model, + "--tensor-parallel-size", + str(tp_size), + "--weight-cache-socket-dir", + self.socket_dir, + "--enforce-eager", + ] + self._proc: subprocess.Popen | None = None + self._lines: list[str] = [] + + def __enter__(self) -> "WeightCacheDaemon": + self._proc = subprocess.Popen( + self._cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True + ) + threading.Thread(target=self._drain_stderr, daemon=True).start() + try: + self._wait_ready(DAEMON_TIMEOUT_S) + except Exception: + self._stop() + raise + return self + + def __exit__(self, *exc_info) -> None: + self._stop() + + def _drain_stderr(self) -> None: + assert self._proc is not None and self._proc.stderr is not None + for line in self._proc.stderr: + self._lines.append(line) + + def _logs(self) -> str: + return "".join(self._lines) + + def _wait_ready(self, timeout_s: float) -> None: + assert self._proc is not None + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + if self._proc.poll() is not None: + raise RuntimeError( + f"Weight cache daemon exited with {self._proc.returncode}:\n" + f"{self._logs()}" + ) + if self._READY_MARKER in self._logs(): + return + time.sleep(1.0) + raise TimeoutError( + f"Weight cache daemon not ready after {timeout_s}s:\n{self._logs()}" + ) + + def _stop(self) -> None: + assert self._proc is not None + self._proc.terminate() + try: + self._proc.wait(timeout=60) + except subprocess.TimeoutExpired: + self._proc.kill() + self._proc.wait() + shutil.rmtree(self.socket_dir, ignore_errors=True) + + +def generate(vllm_runner, socket_dir: str | None, fallback: bool, **kwargs): + extra_config = ( + {} if socket_dir is None else {"socket_dir": socket_dir, "fallback": fallback} + ) + with vllm_runner( + MODEL, + load_format="auto" if socket_dir is None else "ipc_cache", + model_loader_extra_config=extra_config, + # Greedy outputs are compared across engine restarts; cap the batch + # size so scheduling cannot change the results. + max_num_seqs=1, + **kwargs, + ) as llm: + sampling_params = SamplingParams(temperature=0, max_tokens=16, ignore_eos=True) + return llm.generate(PROMPTS, sampling_params) + + +def test_ipc_cache_cold_start_and_warm_restart(vllm_runner): + """Cold start falls back to disk; warm restarts load weights via CUDA IPC. + + All runs must produce outputs identical to a default-loader baseline. The + warm runs disable the disk fallback, so they only pass if the weights + really came from the daemon. + """ + llm_kwargs = dict( + gpu_memory_utilization=0.3, + enforce_eager=True, + # Qwen3.5 is a hybrid mamba model; its mamba cache mode requires + # chunked prefill. + enable_chunked_prefill=True, + ) + + # Baseline: plain disk loading with the default loader. + baseline_outputs = generate(vllm_runner, None, fallback=True, **llm_kwargs) + assert all(text for _, texts in baseline_outputs for text in texts) + + # Cold start: no daemon is serving, so the loader falls back to disk. + with tempfile.TemporaryDirectory(prefix="vllm_ipc_empty_") as empty_socket_dir: + cold_outputs = generate( + vllm_runner, empty_socket_dir, fallback=True, **llm_kwargs + ) + + with WeightCacheDaemon(MODEL, tp_size=1) as daemon: + warm_outputs = generate( + vllm_runner, daemon.socket_dir, fallback=False, **llm_kwargs + ) + # Warm restart: a second engine lifetime against the same daemon. + restart_outputs = generate( + vllm_runner, daemon.socket_dir, fallback=False, **llm_kwargs + ) + + assert cold_outputs == baseline_outputs + assert warm_outputs == baseline_outputs + assert restart_outputs == baseline_outputs diff --git a/tools/pre_commit/check_forbidden_imports.py b/tools/pre_commit/check_forbidden_imports.py index 59163a23e0..3be7bb243b 100644 --- a/tools/pre_commit/check_forbidden_imports.py +++ b/tools/pre_commit/check_forbidden_imports.py @@ -48,6 +48,7 @@ CHECK_IMPORTS = { "vllm/distributed/device_communicators/shm_object_storage.py", "vllm/distributed/weight_transfer/ipc_engine.py", "vllm/distributed/weight_transfer/clients.py", + "vllm/model_executor/model_loader/weight_cache/protocol.py", "tests/distributed/test_shm_broadcast.py", "tests/distributed/test_weight_transfer.py", "vllm/utils/hashing.py", diff --git a/vllm/config/load.py b/vllm/config/load.py index e090214403..5e24a617ba 100644 --- a/vllm/config/load.py +++ b/vllm/config/load.py @@ -38,6 +38,10 @@ class LoadConfig: - "instanttensor" will load the Safetensors weights on CUDA devices using InstantTensor, which enables distributed loading with pipelined prefetching and fast direct I/O. + - "ipc_cache" will map post-quantized weights from a local weight cache + daemon via CUDA IPC for fast engine restarts. See + `vllm/model_executor/model_loader/weight_cache/daemon.py` for how to + launch the daemon. - "npcache" will load the weights in pytorch format and store a numpy cache to speed up the loading. - "dummy" will initialize the weights with random values, which is mainly diff --git a/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py b/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py index cc20467f82..523d1df0d8 100644 --- a/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py +++ b/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py @@ -27,7 +27,11 @@ from vllm.model_executor.layers.fused_moe.oracle.unquantized import ( from vllm.model_executor.layers.fused_moe.runner.shared_experts import ( SharedExperts, ) -from vllm.model_executor.utils import replace_parameter, set_weight_attrs +from vllm.model_executor.utils import ( + is_weights_pre_processed, + replace_parameter, + set_weight_attrs, +) from vllm.platforms import current_platform if TYPE_CHECKING: @@ -43,6 +47,8 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): # --8<-- [end:unquantized_fused_moe] + supports_pre_processed_weights = True + def __init__(self, moe: FusedMoEConfig): super().__init__(moe) self.unquantized_backend, self.experts_cls = select_unquantized_moe_backend( @@ -153,16 +159,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): # optional w{13,2}_bias references and SwiGLU gate params. Since # weight updates mutate those bias tensors in place, the kernel # does not need to be re-built. - self.moe_quant_config = self.get_fused_moe_quant_config(layer) - assert self.moe_quant_config is not None - assert self.experts_cls is not None - self.moe_kernel = make_unquantized_moe_kernel( - quant_config=self.moe_quant_config, - moe_config=self.moe, - backend=self.unquantized_backend, - experts_cls=self.experts_cls, - routing_tables=layer._expert_routing_tables(), - ) + self._init_moe_kernel(layer) if self.unquantized_backend == UnquantizedMoeBackend.CPU: # The CPU experts need the layer itself for the setup that @@ -170,11 +167,30 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): # it only sees the two weight tensors: padding and prepacking # into the grouped-gemm layout (bias included), and capturing # the router config that monolithic apply() cannot carry. + assert self.moe_kernel is not None self.moe_kernel.fused_experts.process_weights_after_loading(layer) + def _init_moe_kernel(self, layer: "RoutedExperts") -> None: + """Build the MoE kernel from the layer's current (shuffled) weights.""" + self.moe_quant_config = self.get_fused_moe_quant_config(layer) + assert self.moe_quant_config is not None + assert self.experts_cls is not None + self.moe_kernel = make_unquantized_moe_kernel( + quant_config=self.moe_quant_config, + moe_config=self.moe, + backend=self.unquantized_backend, + experts_cls=self.experts_cls, + routing_tables=layer._expert_routing_tables(), + ) + def process_weights_after_loading(self, layer: "RoutedExperts") -> None: super().process_weights_after_loading(layer) + if is_weights_pre_processed(): + # Weights are already in runtime format; rebuild the kernel only. + self._init_moe_kernel(layer) + return + # Padding may allocate a same-shaped strided view. Copy it back instead # of rebinding `.data`, since CUDA graphs capture the parameter storage # address. diff --git a/vllm/model_executor/layers/linear.py b/vllm/model_executor/layers/linear.py index fdccabae6f..c7a366012a 100644 --- a/vllm/model_executor/layers/linear.py +++ b/vllm/model_executor/layers/linear.py @@ -163,6 +163,8 @@ class LinearMethodBase(QuantizeMethodBase): class UnquantizedLinearMethod(LinearMethodBase): """Linear method without quantization.""" + supports_pre_processed_weights = True + def __init__(self) -> None: config = get_current_vllm_config_or_none() linear_backend = ( diff --git a/vllm/model_executor/layers/quantization/base_config.py b/vllm/model_executor/layers/quantization/base_config.py index c76259616c..917d229b17 100644 --- a/vllm/model_executor/layers/quantization/base_config.py +++ b/vllm/model_executor/layers/quantization/base_config.py @@ -27,6 +27,13 @@ class QuantizeMethodBase(ABC): in process_weights_after_loading, reducing peak memory during loading. """ + supports_pre_processed_weights: bool = False + """ + Whether ``process_weights_after_loading`` supports running under + ``weights_already_processed``. Methods must skip tensor transforms in + that mode; the loader driver rejects methods that do not declare support. + """ + @abstractmethod def create_weights( self, layer: torch.nn.Module, *weight_args, **extra_weight_attrs diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index 7d74bd9876..0b057f7fd0 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -74,7 +74,11 @@ from vllm.model_executor.parameter import ( BlockQuantScaleParameter, PerTensorScaleParameter, ) -from vllm.model_executor.utils import replace_parameter, set_weight_attrs +from vllm.model_executor.utils import ( + is_weights_pre_processed, + replace_parameter, + set_weight_attrs, +) from vllm.platforms import current_platform from vllm.utils.deep_gemm import ( is_deep_gemm_supported, @@ -248,6 +252,8 @@ class Fp8LinearMethod(LinearMethodBase): quant_config: The quantization config. """ + supports_pre_processed_weights = True + def __init__(self, quant_config: Fp8Config): self.quant_config = quant_config self.is_scale_e8m0 = getattr(quant_config, "is_scale_e8m0", False) @@ -366,6 +372,19 @@ class Fp8LinearMethod(LinearMethodBase): self.use_marlin = isinstance(self.fp8_linear, MarlinFP8ScaledMMLinearKernel) def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + if is_weights_pre_processed(): + # Weights are already in runtime format; the only state tensor + # export cannot carry is the `input_scale = None` stamp that dynamic + # activation quantization relies on, since create_weights never + # registered that parameter. + if not self.act_q_static and not hasattr(layer, "input_scale"): + layer.input_scale = None + # Marlin reads marlin_input_dtype in apply_weights; it lives on the + # method (not exported with the weights), so restore it here too. + if self.use_marlin and hasattr(self.fp8_linear, "marlin_input_dtype"): + self.fp8_linear.marlin_input_dtype = self.marlin_input_dtype + return + if self.use_marlin: if not self.block_quant: # Canonicalize to (K, N) for the kernel. @@ -472,6 +491,8 @@ class Fp8MoEMethod(FusedMoEMethodBase): quant_config: The quantization config. """ + supports_pre_processed_weights = True + def __init__(self, quant_config: Fp8Config, layer: RoutedExperts): super().__init__(layer.moe_config) self.quant_config = quant_config @@ -713,6 +734,10 @@ class Fp8MoEMethod(FusedMoEMethodBase): replace_parameter(layer, f"w13_{self.weight_scale_name}", w13_scale) replace_parameter(layer, f"w2_{self.weight_scale_name}", w2_scale) + self._init_moe_kernel(layer) + + def _init_moe_kernel(self, layer: RoutedExperts) -> None: + """Build the MoE kernel from the layer's current (converted) weights.""" self.moe_quant_config = self.get_fused_moe_quant_config(layer) assert self.moe_quant_config is not None assert self.experts_cls is not None @@ -725,6 +750,11 @@ class Fp8MoEMethod(FusedMoEMethodBase): ) def process_weights_after_loading(self, layer: RoutedExperts) -> None: + if is_weights_pre_processed(): + # Weights are already in kernel format; rebuild the kernel only. + self._init_moe_kernel(layer) + return + # Allow for accessing weights and scales in standard way. w13 = layer.w13_weight w2 = layer.w2_weight diff --git a/vllm/model_executor/layers/quantization/kv_cache.py b/vllm/model_executor/layers/quantization/kv_cache.py index ee535bab2d..447cfb955a 100644 --- a/vllm/model_executor/layers/quantization/kv_cache.py +++ b/vllm/model_executor/layers/quantization/kv_cache.py @@ -8,6 +8,7 @@ from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig, QuantizeMethodBase, ) +from vllm.model_executor.utils import is_weights_pre_processed from vllm.platforms import current_platform from vllm.utils.torch_utils import is_quantized_kv_cache from vllm.v1.kv_cache_interface import kv_cache_uses_per_token_head_scales @@ -51,6 +52,8 @@ class BaseKVCacheMethod(QuantizeMethodBase): quant_config: the appropriate QuantizationConfig """ + supports_pre_processed_weights = True + def __init__(self, quant_config: QuantizationConfig): self.quant_config = quant_config @@ -72,6 +75,20 @@ class BaseKVCacheMethod(QuantizeMethodBase): raise RuntimeError(f"{self.__class__.__name__}.apply should not be called.") def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + if is_weights_pre_processed(): + # Scale buffers were exported with post-processed values; drop the + # consumed scale placeholders and rebuild the host-side copies. + for name in ("k_scale", "v_scale", "q_scale", "prob_scale"): + if hasattr(layer, name): + delattr(layer, name) + if hasattr(layer, "_k_scale"): + layer._k_scale_float = layer._k_scale.item() + layer._v_scale_float = layer._v_scale.item() + layer._q_scale_float = layer._q_scale.item() + layer._k_scale_cpu = layer._k_scale.detach().to("cpu") + layer._v_scale_cpu = layer._v_scale.detach().to("cpu") + return + # skip if there are no weights to process (for example, weight reloading) if not hasattr(layer, "q_scale"): assert not hasattr(layer, "k_scale") diff --git a/vllm/model_executor/layers/vocab_parallel_embedding.py b/vllm/model_executor/layers/vocab_parallel_embedding.py index e2d9072c25..2eddd0ffec 100644 --- a/vllm/model_executor/layers/vocab_parallel_embedding.py +++ b/vllm/model_executor/layers/vocab_parallel_embedding.py @@ -36,6 +36,8 @@ DEFAULT_VOCAB_PADDING_SIZE = 64 class UnquantizedEmbeddingMethod(QuantizeMethodBase): """Unquantized method for embeddings.""" + supports_pre_processed_weights = True + def create_weights( self, layer: torch.nn.Module, diff --git a/vllm/model_executor/model_loader/__init__.py b/vllm/model_executor/model_loader/__init__.py index a902c61da1..6305c45f5d 100644 --- a/vllm/model_executor/model_loader/__init__.py +++ b/vllm/model_executor/model_loader/__init__.py @@ -24,6 +24,7 @@ from vllm.model_executor.model_loader.utils import ( get_model_architecture, get_model_cls, ) +from vllm.model_executor.model_loader.weight_cache.ipc_loader import IpcModelLoader logger = init_logger(__name__) @@ -35,6 +36,7 @@ LoadFormats = Literal[ "dummy", "fastsafetensors", "instanttensor", + "ipc_cache", "mistral", "modelexpress", "npcache", @@ -51,6 +53,7 @@ _LOAD_FORMAT_TO_MODEL_LOADER: dict[str, type[BaseModelLoader]] = { "dummy": DummyModelLoader, "fastsafetensors": DefaultModelLoader, "instanttensor": DefaultModelLoader, + "ipc_cache": IpcModelLoader, "mistral": DefaultModelLoader, "modelexpress": ModelExpressModelLoader, "npcache": DefaultModelLoader, @@ -150,6 +153,7 @@ __all__ = [ "ModelExpressModelLoader", "DefaultModelLoader", "DummyModelLoader", + "IpcModelLoader", "RunaiModelStreamerLoader", "ShardedStateLoader", "TensorizerLoader", diff --git a/vllm/model_executor/model_loader/utils.py b/vllm/model_executor/model_loader/utils.py index e904f07fc4..03039cf019 100644 --- a/vllm/model_executor/model_loader/utils.py +++ b/vllm/model_executor/model_loader/utils.py @@ -26,6 +26,7 @@ from vllm.model_executor.model_loader.reload import ( ) from vllm.model_executor.model_loader.weight_tying import maybe_retie_word_embeddings from vllm.model_executor.models.interfaces import SupportsQuant +from vllm.model_executor.utils import is_weights_pre_processed from vllm.tracing import instrument from vllm.utils.mem_utils import release_device_memory_under_pressure from vllm.utils.platform_utils import is_pin_memory_available @@ -97,13 +98,27 @@ def initialize_model( def process_weights_after_loading( model: nn.Module, model_config: ModelConfig, target_device: torch.device ) -> None: + """Post-process loaded weights into runtime format. + + Under ``weights_already_processed`` (weight cache IPC loader), quant + methods skip tensor transforms and must declare + ``supports_pre_processed_weights``, otherwise this raises ``RuntimeError``. + """ # Reclaim memory when an explicit lm_head has been # loaded, but it is identical to the input embeddings. maybe_retie_word_embeddings(model, model_config) - for _, module in model.named_modules(): + for name, module in model.named_modules(): quant_method = getattr(module, "quant_method", None) if isinstance(quant_method, QuantizeMethodBase): + if ( + is_weights_pre_processed() + and not quant_method.supports_pre_processed_weights + ): + raise RuntimeError( + f"layer {name or ''}: {type(quant_method).__name__} " + "does not support pre-processed weights" + ) # When quant methods need to process weights after loading # (for repacking, quantizing, etc), they expect parameters # to be on the global target device. This scope is for the diff --git a/vllm/model_executor/model_loader/weight_cache/__init__.py b/vllm/model_executor/model_loader/weight_cache/__init__.py new file mode 100644 index 0000000000..022770f893 --- /dev/null +++ b/vllm/model_executor/model_loader/weight_cache/__init__.py @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.model_executor.model_loader.weight_cache.ipc_loader import IpcModelLoader +from vllm.model_executor.model_loader.weight_cache.protocol import ( + CacheConfigMismatchError, + TensorEntry, + UnsupportedQuantForIPCError, + WeightCacheKey, + WeightCacheUnavailableError, + check_ipc_quant_support, + is_ipc_quant_supported, +) + +__all__ = [ + "CacheConfigMismatchError", + "IpcModelLoader", + "TensorEntry", + "UnsupportedQuantForIPCError", + "WeightCacheKey", + "WeightCacheUnavailableError", + "check_ipc_quant_support", + "is_ipc_quant_supported", +] diff --git a/vllm/model_executor/model_loader/weight_cache/daemon.py b/vllm/model_executor/model_loader/weight_cache/daemon.py new file mode 100644 index 0000000000..7bb10b936e --- /dev/null +++ b/vllm/model_executor/model_loader/weight_cache/daemon.py @@ -0,0 +1,412 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Weight cache daemon for fast engine restarts. + +One daemon process per GPU holds the post-quantized, TP-sharded weights of its +rank in GPU memory and serves CUDA IPC handles to vLLM engines over a Unix +domain socket. Restarting engines map the weights via zero-copy IPC instead of +reloading from disk. + +Launch one daemon per TP rank with a single command: + + python -m vllm.model_executor.model_loader.weight_cache.daemon \\ + --model /path/to/model --tensor-parallel-size 4 + +Engines then load from the daemons with: + + vllm serve /path/to/model --tensor-parallel-size 4 \\ + --load-format ipc_cache + +Only tensor parallelism is supported; pipeline, data, and expert parallelism +are rejected at launch. +""" + +import contextlib +import fcntl +import multiprocessing +import os +import queue +import signal +import socket +import sys +from collections.abc import Callable + +import torch + +from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config +from vllm.distributed import ( + ensure_model_parallel_initialized, + init_distributed_environment, +) +from vllm.logger import init_logger +from vllm.model_executor.model_loader.weight_cache.protocol import ( + TensorEntry, + WeightCacheKey, + WeightCacheUnavailableError, + check_ipc_quant_support, + ensure_private_socket_dir, + get_physical_device_id, + get_socket_path, + recv_msg, + send_msg, + verify_peer_is_owner, +) +from vllm.platforms import current_platform + +logger = init_logger(__name__) + + +def _report_ready(message: str) -> None: + """Write a readiness message straight to the original stderr descriptor. + + Loading the model pulls in FlashInfer's CuTeDSL JIT compiler, which swaps + ``sys.stdout``/``sys.stderr`` for in-memory buffers while compiling kernels + across worker threads. That save/restore races on the interpreter-global + streams and can leave them (and vLLM's logging handler, which caches the + original stream object) detached, silently swallowing everything logged + afterwards -- including the daemon's readiness announcement. Since operators + rely on that line to know the daemon is serving, write it directly to file + descriptor 2 so it bypasses the logging machinery entirely. + """ + with contextlib.suppress(OSError): + os.write(2, (message + "\n").encode()) + + +def export_entries( + model: torch.nn.Module, +) -> tuple[dict[str, TensorEntry], dict[str, str]]: + """Export a model's tensors, preserving tied-parameter aliases. + + ``named_parameters``/``named_buffers`` are iterated with + ``remove_duplicate=False`` so tied weights (e.g. ``lm_head.weight`` sharing + storage with ``embed_tokens.weight``) are not silently dropped. Each unique + tensor is exported once; every additional name that refers to the same + tensor object is recorded in the returned alias map so the client can + re-establish the shared identity instead of allocating uninitialized + memory for it. + + Returns: + A ``(entries, aliases)`` pair where ``entries`` maps a canonical name to + its ``TensorEntry`` and ``aliases`` maps each duplicate name to its + canonical name. + """ + entries: dict[str, TensorEntry] = {} + aliases: dict[str, str] = {} + canonical_by_id: dict[int, str] = {} + + def _add(name: str, tensor: torch.Tensor, kind: str) -> None: + canonical = canonical_by_id.get(id(tensor)) + if canonical is not None: + aliases[name] = canonical + return + canonical_by_id[id(tensor)] = name + entries[name] = TensorEntry.from_tensor(tensor, kind) + + for name, param in model.named_parameters(remove_duplicate=False): + _add(name, param, "param") + # named_buffers includes non-persistent buffers (e.g. rotary embedding + # caches) that state_dict would miss. + for name, buffer in model.named_buffers(remove_duplicate=False): + if name in entries or name in aliases: + continue + _add(name, buffer, "buffer") + return entries, aliases + + +class WeightCacheDaemon: + """Per-GPU process that loads one TP shard and serves CUDA IPC handles.""" + + def __init__( + self, + vllm_config: VllmConfig, + tp_rank: int, + distributed_init_method: str, + socket_dir: str | None = None, + ): + self.vllm_config = vllm_config + self.tp_rank = tp_rank + self.distributed_init_method = distributed_init_method + self.socket_dir = socket_dir + self.model: torch.nn.Module | None = None + self.entries: dict[str, TensorEntry] = {} + self.aliases: dict[str, str] = {} + # Fingerprint before loading: process_weights_after_loading may + # mutate hf_config.quantization_config. + self.cache_config = WeightCacheKey.from_model_config( + vllm_config.model_config, + tp_size=vllm_config.parallel_config.tensor_parallel_size, + tp_rank=tp_rank, + ) + + def load_model(self) -> None: + from vllm.model_executor.model_loader import get_model + + tp_size = self.cache_config.tp_size + torch.accelerator.set_device_index(self.tp_rank) + init_distributed_environment( + world_size=tp_size, + rank=self.tp_rank, + distributed_init_method=self.distributed_init_method, + local_rank=self.tp_rank, + backend=current_platform.dist_backend, + ) + with set_current_vllm_config(self.vllm_config): + ensure_model_parallel_initialized(tp_size, 1) + self.model = get_model(vllm_config=self.vllm_config) + self._export_entries() + logger.info( + "Weight cache daemon rank %d cached %d tensors", + self.tp_rank, + len(self.entries), + ) + + def _export_entries(self) -> None: + assert self.model is not None + self.entries, self.aliases = export_entries(self.model) + + def serve_forever(self, ready_callback: Callable[[], None] | None = None) -> None: + """Serve requests until terminated. + + The socket is only bound once the model is fully cached, so clients + get a connection error (and fall back to disk) until the daemon is + ready. + + Args: + ready_callback: Invoked once the socket is bound and listening, + so the launcher can report overall readiness. + """ + socket_path = self._socket_path() + ensure_private_socket_dir( + os.path.dirname(socket_path), strict_perms=self.socket_dir is None + ) + # Hold an exclusive per-GPU lock for the daemon's lifetime so a second + # daemon cannot remove this daemon's live socket and hijack the path. + lock_fd = self._acquire_gpu_lock(socket_path) + if os.path.exists(socket_path): + os.unlink(socket_path) + server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + server.bind(socket_path) + os.chmod(socket_path, 0o600) + server.listen() + logger.info( + "Weight cache daemon rank %d serving on %s", self.tp_rank, socket_path + ) + print( + f"Weight cache daemon rank {self.tp_rank} ready: serving on {socket_path}" + ) + if ready_callback is not None: + ready_callback() + try: + while True: + conn, _ = server.accept() + with conn: + try: + verify_peer_is_owner(conn) + self._handle_connection(conn) + except (ConnectionError, EOFError): + logger.warning("Client disconnected mid-request") + except Exception: + # A single malformed or malicious request must not take + # down the daemon for every other engine on this GPU. + logger.exception( + "Error handling weight cache client; continuing" + ) + finally: + server.close() + if os.path.exists(socket_path): + os.unlink(socket_path) + os.close(lock_fd) + + def _acquire_gpu_lock(self, socket_path: str) -> int: + """Take an exclusive lock guarding this GPU's socket path. + + The lock is advisory and released automatically when the daemon exits + (or crashes), so a stale socket is only ever removed by whoever owns + the lock. A running daemon holding it makes a second daemon fail fast + instead of clobbering the live socket. + """ + lock_fd = os.open(f"{socket_path}.lock", os.O_CREAT | os.O_RDWR, 0o600) + try: + fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError as e: + os.close(lock_fd) + raise WeightCacheUnavailableError( + f"Another weight cache daemon already owns {socket_path}" + ) from e + return lock_fd + + def _socket_path(self) -> str: + device_index = torch.accelerator.current_device_index() + gpu_id = get_physical_device_id(device_index) + if gpu_id is None: + gpu_id = device_index + return get_socket_path(gpu_id, self.socket_dir) + + def _handle_connection(self, conn: socket.socket) -> None: + request = recv_msg(conn) + cmd = request.get("cmd") + if cmd == "get_state": + self._handle_get_state(conn, request) + elif cmd == "release": + self._handle_release(conn) + else: + send_msg(conn, {"status": "error", "message": f"Unknown command {cmd!r}"}) + + def _handle_get_state(self, conn: socket.socket, request: dict) -> None: + client_config = request.get("cache_config") + if not isinstance(client_config, WeightCacheKey): + send_msg(conn, {"status": "error", "message": "Missing cache_config"}) + return + mismatched = self.cache_config.mismatched_fields(client_config) + if mismatched: + logger.warning("WeightCacheKey mismatch on fields: %s", mismatched) + send_msg(conn, {"status": "mismatch", "fields": mismatched}) + return + if not self.entries: + send_msg(conn, {"status": "error", "message": "Weights were released"}) + return + send_msg( + conn, + { + "status": "ok", + "entries": self.entries, + "aliases": self.aliases, + "gpu_uuid": self._gpu_uuid(), + }, + ) + + def _handle_release(self, conn: socket.socket) -> None: + self.entries.clear() + self.aliases.clear() + self.model = None + torch.accelerator.empty_cache() + logger.info("Weight cache daemon rank %d released cached weights", self.tp_rank) + send_msg(conn, {"status": "ok"}) + + def _gpu_uuid(self) -> str: + props = torch.cuda.get_device_properties( + torch.accelerator.current_device_index() + ) + return str(props.uuid) + + +def _run_daemon( + tp_rank: int, + vllm_config: VllmConfig, + distributed_init_method: str, + socket_dir: str | None, + ready_queue: "multiprocessing.Queue[int]", +) -> None: + daemon = WeightCacheDaemon( + vllm_config, tp_rank, distributed_init_method, socket_dir + ) + daemon.load_model() + daemon.serve_forever(ready_callback=lambda: ready_queue.put(tp_rank)) + + +def _reject_unsupported_parallelism(parallel_config: ParallelConfig) -> None: + """Reject every parallelism mode other than tensor parallelism.""" + unsupported = { + "pipeline parallelism": parallel_config.pipeline_parallel_size > 1, + "data parallelism": parallel_config.data_parallel_size > 1, + "expert parallelism": parallel_config.enable_expert_parallel, + } + for name, enabled in unsupported.items(): + if enabled: + raise ValueError( + f"The weight cache daemon only supports tensor parallelism; " + f"{name} is not supported" + ) + + +def main() -> None: + from vllm.engine.arg_utils import EngineArgs + from vllm.utils.argparse_utils import FlexibleArgumentParser + from vllm.utils.network_utils import get_distributed_init_method, get_open_port + + parser = FlexibleArgumentParser( + description="Launch weight cache daemons (one per TP rank)." + ) + EngineArgs.add_cli_args(parser) + parser.add_argument( + "--weight-cache-socket-dir", + type=str, + default=None, + help="Directory for the daemon Unix sockets (default: tempdir).", + ) + args = parser.parse_args() + engine_args = EngineArgs.from_cli_args(args) + vllm_config = engine_args.create_engine_config() + if vllm_config.load_config.load_format == "ipc_cache": + raise ValueError( + "The weight cache daemon itself must load from disk; use the " + "default --load-format" + ) + # Checked before loading anything: an unsupported quantization method would + # otherwise only surface in the engine, after a full load. + check_ipc_quant_support(vllm_config.model_config, where="daemon") + parallel_config = vllm_config.parallel_config + _reject_unsupported_parallelism(parallel_config) + tp_size = parallel_config.tensor_parallel_size + + distributed_init_method = get_distributed_init_method("127.0.0.1", get_open_port()) + ctx = multiprocessing.get_context("spawn") + ready_queue: multiprocessing.Queue[int] = ctx.Queue() + procs = [ + ctx.Process( + target=_run_daemon, + args=( + rank, + vllm_config, + distributed_init_method, + args.weight_cache_socket_dir, + ready_queue, + ), + name=f"vllm-weight-cache-daemon-{rank}", + ) + for rank in range(tp_size) + ] + for proc in procs: + proc.start() + + def _shutdown(signum, frame): + for proc in procs: + proc.terminate() + + signal.signal(signal.SIGINT, _shutdown) + signal.signal(signal.SIGTERM, _shutdown) + + ready_ranks: set[int] = set() + while len(ready_ranks) < tp_size: + try: + ready_ranks.add(ready_queue.get(timeout=1.0)) + except queue.Empty: + dead = [p for p in procs if p.exitcode is not None] + if dead: + logger.error( + "Weight cache daemon rank(s) exited during startup " + "(exitcodes=%s); shutting down.", + [p.exitcode for p in dead], + ) + for proc in procs: + proc.terminate() + for proc in procs: + proc.join() + sys.exit(max((p.exitcode or 0) for p in procs)) + logger.info_once( + "===== Weight cache daemon READY: all %d ranks serving in %s =====", + tp_size, + args.weight_cache_socket_dir or "the default socket dir", + ) + _report_ready( + f"===== Weight cache daemon READY: all {tp_size} rank(s) serving in " + f"{args.weight_cache_socket_dir or 'the default socket dir'} =====" + ) + + for proc in procs: + proc.join() + sys.exit(max(proc.exitcode or 0 for proc in procs)) + + +if __name__ == "__main__": + main() diff --git a/vllm/model_executor/model_loader/weight_cache/ipc_loader.py b/vllm/model_executor/model_loader/weight_cache/ipc_loader.py new file mode 100644 index 0000000000..93c5bfc016 --- /dev/null +++ b/vllm/model_executor/model_loader/weight_cache/ipc_loader.py @@ -0,0 +1,394 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""IPC model loader: maps post-quantized weights from a local weight cache +daemon via CUDA IPC instead of loading from disk.""" + +import dataclasses +import socket +from copy import copy + +import torch +import torch.nn as nn + +from vllm.config import ModelConfig, VllmConfig +from vllm.config.load import LoadConfig +from vllm.logger import init_logger +from vllm.model_executor.model_loader.base_loader import BaseModelLoader +from vllm.model_executor.model_loader.default_loader import DefaultModelLoader +from vllm.model_executor.model_loader.utils import ( + initialize_model, + process_weights_after_loading, +) +from vllm.model_executor.model_loader.weight_cache.protocol import ( + CacheConfigMismatchError, + TensorEntry, + UnsupportedQuantForIPCError, + WeightCacheKey, + WeightCacheUnavailableError, + check_ipc_quant_support, + get_physical_device_id, + get_socket_path, + recv_msg, + send_msg, + verify_socket_owner, +) +from vllm.model_executor.utils import weights_already_processed +from vllm.tracing import instrument +from vllm.utils.torch_utils import set_default_torch_dtype + +logger = init_logger(__name__) + +_CONNECT_TIMEOUT_S = 5.0 +_STATE_TIMEOUT_S = 300.0 + + +class IpcModelLoader(BaseModelLoader): + """Loads a model by mapping the weight cache daemon's tensors via CUDA IPC. + + The model is initialized on the meta device and every parameter/buffer is + replaced by the daemon's post-quantized tensor, so + process_weights_after_loading is skipped entirely. In "zero_copy" mode the + engine shares the daemon's GPU memory; in "copy" mode the tensors are + cloned into engine-owned memory and the daemon is asked to release its + cache afterwards. + + Extra config keys (via --model-loader-extra-config): + + - socket_path: explicit daemon socket path. Defaults to a per-GPU path + derived from the physical GPU id. + - socket_dir: directory containing the daemon sockets. + - mode: "zero_copy" (default) or "copy". + - fallback: fall back to disk loading when the daemon is unavailable or + the fingerprints mismatch (default: True). + - connect_timeout_s: socket connect timeout (default: 5.0). + - state_timeout_s: timeout for the weight-transfer request (default: 300.0). + + Note: in zero-copy mode the weights live in the daemon's CUDA IPC + allocations, so sleep mode (CuMemAllocator weight offloading) must not be + used with this loader. + """ + + def __init__(self, load_config: LoadConfig): + super().__init__(load_config) + extra_config = copy(load_config.model_loader_extra_config or {}) + self.socket_path: str | None = extra_config.pop("socket_path", None) + self.socket_dir: str | None = extra_config.pop("socket_dir", None) + self.mode: str = extra_config.pop("mode", "zero_copy") + self.fallback: bool = extra_config.pop("fallback", True) + self.connect_timeout_s: float = float( + extra_config.pop("connect_timeout_s", _CONNECT_TIMEOUT_S) + ) + self.state_timeout_s: float = float( + extra_config.pop("state_timeout_s", _STATE_TIMEOUT_S) + ) + if self.mode not in ("zero_copy", "copy"): + raise ValueError( + f"Invalid weight cache mode {self.mode!r}, " + "expected 'zero_copy' or 'copy'" + ) + if extra_config: + raise ValueError( + f"Unexpected extra config keys for load format " + f"{load_config.load_format}: {sorted(extra_config)}" + ) + + def download_model(self, model_config: ModelConfig) -> None: + DefaultModelLoader(self._fallback_load_config()).download_model(model_config) + + def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None: + """Best-effort in-place reload for an already-initialized model. + + Copies daemon tensors into matching parameters/buffers. The model is + expected to already be in the post-quantized layout (e.g. previously + loaded through this loader). + """ + device_index = torch.accelerator.current_device_index() + entries, _ = self._fetch_entries(model_config) + params = dict(model.named_parameters()) + buffers = dict(model.named_buffers()) + for name, entry in entries.items(): + target = params.get(name, buffers.get(name)) + source = entry.rebuild(device_index) + if target is None or target.shape != source.shape: + logger.warning("Skipping mismatched cached tensor %s", name) + continue + target.data.copy_(source) + + @instrument(span_name="Load model") + def load_model( + self, vllm_config: VllmConfig, model_config: ModelConfig, prefix: str = "" + ) -> nn.Module: + # Unsupported quantization is a permanent misconfiguration rather than a + # transient daemon outage, so it is raised even when fallback is on. + self._check_supported(vllm_config, model_config) + state_fetched = False + try: + entries, aliases = self._fetch_entries(model_config) + state_fetched = True + return self._build_model( + vllm_config, model_config, prefix, entries, aliases + ) + except (WeightCacheUnavailableError, CacheConfigMismatchError) as e: + if not self.fallback: + raise + logger.warning( + "Weight cache unusable (%s); falling back to disk loading", e + ) + except Exception: + if not self.fallback: + raise + logger.exception( + "Weight cache IPC loading failed; falling back to disk loading" + ) + # _build_model failed after fetching state without reaching its + # copy-mode release, so the daemon still holds the full cache; + # release it so the disk fallback does not OOM against it. + if state_fetched and self.mode == "copy": + self._send_release() + torch.accelerator.empty_cache() + return self._fallback_load(vllm_config, model_config, prefix) + + def _build_model( + self, + vllm_config: VllmConfig, + model_config: ModelConfig, + prefix: str, + entries: dict[str, TensorEntry], + aliases: dict[str, str], + ) -> nn.Module: + device_config = vllm_config.device_config + load_device = ( + device_config.device + if self.load_config.device is None + else self.load_config.device + ) + target_device = torch.device(load_device) + device_index = ( + target_device.index + if target_device.index is not None + else torch.accelerator.current_device_index() + ) + with set_default_torch_dtype(model_config.dtype): + with torch.device("meta"): + model = initialize_model( + vllm_config=vllm_config, + model_config=model_config, + prefix=prefix, + ) + self._apply_entries(model, entries, aliases, device_index) + # The daemon exports tensors that already went through + # process_weights_after_loading; re-run it in pre-processed mode + # so quant methods only rebuild Python-side state (e.g. the MoE + # kernel). Leftovers are materialized afterwards so that + # placeholders the daemon-side post-processing consumed are + # dropped rather than filled with uninitialized memory. + with weights_already_processed(): + process_weights_after_loading(model, model_config, target_device) + _materialize_remaining_meta_tensors( + model, torch.device(target_device.type, device_index) + ) + if self.mode == "copy": + self._send_release() + logger.info( + "Mapped %d tensors from the weight cache daemon (%s mode)", + len(entries), + self.mode, + ) + return model.eval() + + @staticmethod + def _check_supported(vllm_config: VllmConfig, model_config: ModelConfig) -> None: + check_ipc_quant_support(model_config, where="engine") + cache_dtype = vllm_config.cache_config.cache_dtype + if cache_dtype != "auto" and not str(cache_dtype).startswith("fp8"): + # BaseKVCacheMethod.process_weights_after_loading turns the loaded + # k/v scale parameters into plain float attributes. For fp8 cache + # dtypes those are rebuilt from the exported scale buffers when + # process_weights_after_loading runs in pre-processed mode; other + # quantized cache dtypes are not verified. + raise UnsupportedQuantForIPCError( + f"[weight_cache:engine] kv cache dtype {cache_dtype!r} is not " + "supported by the weight cache; use --kv-cache-dtype auto." + ) + + def _apply_entries( + self, + model: nn.Module, + entries: dict[str, TensorEntry], + aliases: dict[str, str], + device_index: int, + ) -> None: + # remove_duplicate=False keeps tied module aliases reachable by name: + # a tied lm_head *is* the embedding module, so the deduplicated view + # would not contain "lm_head" at all. + modules = dict(model.named_modules(remove_duplicate=False)) + registered: dict[str, torch.Tensor] = {} + + def _register(name: str, tensor: torch.Tensor, is_param: bool) -> None: + module_name, _, leaf = name.rpartition(".") + module = modules.get(module_name) + if module is None: + raise RuntimeError(f"Cached tensor {name} has no matching module") + # Replace via registration rather than param.data assignment, + # which fails for meta tensors. Entries may also introduce + # post-quantization tensors absent from the meta model. + module._parameters.pop(leaf, None) + module._buffers.pop(leaf, None) + if is_param: + obj: torch.Tensor = ( + tensor + if isinstance(tensor, nn.Parameter) + else nn.Parameter(tensor, requires_grad=False) + ) + module.register_parameter(leaf, obj) + else: + obj = tensor + module.register_buffer(leaf, obj) + registered[name] = obj + + for name, entry in entries.items(): + tensor = entry.rebuild(device_index) + if self.mode == "copy": + tensor = tensor.clone() + _register(name, tensor, entry.kind == "param") + + # Re-establish tied-weight aliases by registering the *same* object the + # canonical name resolved to, so parameter identity (and the tie) is + # preserved instead of allocating uninitialized memory. + for alias_name, canonical_name in aliases.items(): + obj = registered.get(canonical_name) + if obj is None: + logger.warning( + "Cached alias %s references missing canonical tensor %s", + alias_name, + canonical_name, + ) + continue + _register(alias_name, obj, isinstance(obj, nn.Parameter)) + + def _fetch_entries( + self, model_config: ModelConfig + ) -> tuple[dict[str, TensorEntry], dict[str, str]]: + from vllm.distributed import ( + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + ) + + cache_config = WeightCacheKey.from_model_config( + model_config, + tp_size=get_tensor_model_parallel_world_size(), + tp_rank=get_tensor_model_parallel_rank(), + ) + return self._request_state(cache_config) + + def _request_state( + self, cache_config: WeightCacheKey + ) -> tuple[dict[str, TensorEntry], dict[str, str]]: + with self._connect(self.state_timeout_s) as conn: + send_msg(conn, {"cmd": "get_state", "cache_config": cache_config}) + response = recv_msg(conn) + status = response.get("status") + if status == "mismatch": + raise CacheConfigMismatchError( + f"WeightCacheKey mismatch on fields: {response.get('fields')}" + ) + if status != "ok": + raise WeightCacheUnavailableError( + f"Weight cache daemon error: {response.get('message')}" + ) + self._check_gpu_uuid(response.get("gpu_uuid")) + return response["entries"], response.get("aliases", {}) + + def _connect(self, timeout: float) -> socket.socket: + socket_path = self._resolve_socket_path() + # The auto-derived per-user directory is locked to 0700 and checked + # strictly. When the operator explicitly configures a path they own the + # trust decision, so only ownership/symlink safety is enforced. + strict_perms = self.socket_path is None and self.socket_dir is None + try: + verify_socket_owner(socket_path, strict_perms=strict_perms) + except OSError as e: + raise WeightCacheUnavailableError( + f"Weight cache socket {socket_path} is unavailable: {e}" + ) from e + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.settimeout(timeout) + try: + sock.connect(socket_path) + except OSError as e: + sock.close() + raise WeightCacheUnavailableError( + f"Cannot connect to weight cache daemon at {socket_path}: {e}" + ) from e + return sock + + def _resolve_socket_path(self) -> str: + if self.socket_path is not None: + return self.socket_path + device_index = torch.accelerator.current_device_index() + gpu_id = get_physical_device_id(device_index) + if gpu_id is None: + raise WeightCacheUnavailableError( + "Cannot infer the physical GPU id from CUDA_VISIBLE_DEVICES; " + "pass socket_path via --model-loader-extra-config" + ) + return get_socket_path(gpu_id, self.socket_dir) + + def _check_gpu_uuid(self, daemon_uuid: str | None) -> None: + if daemon_uuid is None: + return + props = torch.cuda.get_device_properties( + torch.accelerator.current_device_index() + ) + local_uuid = str(props.uuid) + if daemon_uuid != local_uuid: + raise CacheConfigMismatchError( + f"Daemon GPU {daemon_uuid} != engine GPU {local_uuid}; " + "check the socket path / GPU mapping" + ) + + def _send_release(self) -> None: + try: + with self._connect(self.connect_timeout_s) as conn: + send_msg(conn, {"cmd": "release"}) + recv_msg(conn) + except (WeightCacheUnavailableError, ConnectionError, OSError): + logger.warning("Failed to ask the weight cache daemon to release") + + def _fallback_load_config(self) -> LoadConfig: + # DefaultModelLoader must not see load_format="ipc_cache" or the ipc + # extra config keys. + return dataclasses.replace( + self.load_config, load_format="auto", model_loader_extra_config={} + ) + + def _fallback_load( + self, vllm_config: VllmConfig, model_config: ModelConfig, prefix: str + ) -> nn.Module: + loader = DefaultModelLoader(self._fallback_load_config()) + return loader.load_model( + vllm_config=vllm_config, model_config=model_config, prefix=prefix + ) + + +def _materialize_remaining_meta_tensors(model: nn.Module, device: torch.device) -> None: + """Allocate any tensors the daemon did not provide. + + These are typically parameters removed on the daemon side by + process_weights_after_loading; they are not expected to be read at + runtime, so they are left uninitialized. + """ + for module_name, module in model.named_modules(): + for leaf, param in list(module._parameters.items()): + if param is not None and param.device.type == "meta": + logger.warning( + "Materializing empty parameter %s.%s missing from the weight cache", + module_name, + leaf, + ) + module._parameters[leaf] = nn.Parameter( + torch.empty_like(param, device=device), requires_grad=False + ) + for leaf, buffer in list(module._buffers.items()): + if buffer is not None and buffer.device.type == "meta": + module._buffers[leaf] = torch.empty_like(buffer, device=device) diff --git a/vllm/model_executor/model_loader/weight_cache/protocol.py b/vllm/model_executor/model_loader/weight_cache/protocol.py new file mode 100644 index 0000000000..3866cc773b --- /dev/null +++ b/vllm/model_executor/model_loader/weight_cache/protocol.py @@ -0,0 +1,412 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""WeightCacheKey fingerprinting and socket protocol for the weight cache daemon. + +The protocol uses pickle over a Unix domain socket and is only intended for +communication between trusted local processes owned by the same user. The +sockets live in a per-user private directory (mode 0700) and the daemon +restricts the socket file permissions to the owner (0600). Both the daemon and +the engine verify that the directory and socket are owned by the current user +and are not group/world accessible before trusting them, so a different local +user cannot pre-plant a malicious socket at a predictable path. +""" + +import glob +import json +import os +import pickle +import socket +import stat +import struct +import tempfile +from dataclasses import dataclass, fields +from typing import Any + +import torch + +import vllm.version +from vllm.config import ModelConfig +from vllm.utils.hashing import safe_hash + +SOCKET_NAME_TEMPLATE = "vllm_weight_cache_gpu{gpu_id}.sock" +SOCKET_DIR_TEMPLATE = "vllm_weight_cache_{uid}" + +_LEN_STRUCT = struct.Struct("!Q") +# Sanity bound for a single message. IPC handles are tiny; only small +# non-CUDA tensors are ever shipped by value. +MAX_MSG_SIZE = 1 << 34 + + +def _current_uid() -> int: + getuid = getattr(os, "getuid", None) + return getuid() if getuid is not None else -1 + + +class WeightCacheUnavailableError(Exception): + """Raised when no weight cache daemon is reachable or usable.""" + + +class CacheConfigMismatchError(Exception): + """Raised when the daemon's cached weights don't match the engine.""" + + +class UnsupportedQuantForIPCError(Exception): + """Raised when a quantization method is not verified for IPC weight sharing.""" + + +# The daemon exports tensor data only, so sharing is correct just for methods +# whose post-load effect is either fully captured by that data or rebuilt when +# process_weights_after_loading runs under weights_already_processed. Anything +# absent from this registry hard-errors: a method that silently repacks into +# shapes the client cannot reproduce or stamps unrebuildable Python-side state +# would serve wrong numerics. Extend it only after an end-to-end check against +# a disk-loaded baseline. +# +# This check is config-only so the daemon fails fast before a slow full load, +# rather than surfacing the incompatibility only after the engine has built the +# model (where process_weights_after_loading's supports_pre_processed_weights +# guard would eventually catch it). + + +def _quant_config_field(quant_config: Any, key: str) -> Any: + if quant_config is None: + return None + if isinstance(quant_config, dict): + return quant_config.get(key) + return getattr(quant_config, key, None) + + +def _fp8_round_trips_via_ipc(quant_config: Any) -> bool: + """Only block-wise FP8 is verified. + + Block-wise FP8 preserves the weight shape, while per-tensor FP8 transposes + ``layer.weight`` during post-processing -- a shape the client's + meta-initialized model cannot reproduce. + """ + return _quant_config_field(quant_config, "weight_block_size") is not None + + +# quantization name -> predicate(quant_config) -> True when verified safe. +IPC_QUANT_ALLOWLIST: dict[str | None, Any] = { + None: lambda _quant_config: True, # unquantized + "fp8": _fp8_round_trips_via_ipc, +} + + +def is_ipc_quant_supported(quantization: str | None, quant_config: Any) -> bool: + predicate = IPC_QUANT_ALLOWLIST.get(quantization) + return False if predicate is None else bool(predicate(quant_config)) + + +def check_ipc_quant_support(model_config: ModelConfig, *, where: str) -> None: + """Hard-error unless the model's quantization is verified for IPC sharing. + + Args: + model_config: Model configuration to inspect. + where: Short tag ("daemon"/"engine") used in the error message. + + Raises: + UnsupportedQuantForIPCError: If the quantization method is not on the + verified allowlist. + """ + quantization = model_config.quantization + # Prefer the canonical, nested-aware config (multimodal models keep it under + # text_config); fall back to the raw hf_config for older code paths. + quant_config = getattr( + getattr(model_config, "model_arch_config", None), "quantization_config", None + ) + if quant_config is None: + quant_config = getattr(model_config.hf_config, "quantization_config", None) + if is_ipc_quant_supported(quantization, quant_config): + return + verified = ", ".join( + "unquantized" if name is None else repr(name) for name in IPC_QUANT_ALLOWLIST + ) + raise UnsupportedQuantForIPCError( + f"[weight_cache:{where}] quantization {quantization!r} is not verified " + f"for CUDA IPC weight sharing: its post-load processing may repack " + f"weights into shapes the client cannot reproduce or stamp Python-side " + f"state that tensor export cannot carry, which would silently serve " + f"wrong numerics. Verified: {verified} (FP8 only with weight_block_size " + f"set, i.e. block-wise). Use the default --load-format for this model." + ) + + +def get_physical_device_id(device_index: int) -> int | None: + """Map a local CUDA device index to the physical GPU id. + + Returns None if CUDA_VISIBLE_DEVICES contains non-integer entries + (e.g. GPU UUIDs), in which case an explicit socket path is required. + """ + visible = os.environ.get("CUDA_VISIBLE_DEVICES") + if not visible: + return device_index + entries = visible.split(",") + try: + return int(entries[device_index]) + except (IndexError, ValueError): + return None + + +def get_socket_dir(socket_dir: str | None = None) -> str: + """Return the directory that holds the daemon sockets. + + When no explicit directory is given, use a per-user private directory + under the system temp dir so its path is unpredictable to other users and + can be locked down to mode 0700. + """ + if socket_dir is not None: + return socket_dir + return os.path.join( + tempfile.gettempdir(), SOCKET_DIR_TEMPLATE.format(uid=_current_uid()) + ) + + +def get_socket_path(gpu_id: int, socket_dir: str | None = None) -> str: + directory = get_socket_dir(socket_dir) + return os.path.join(directory, SOCKET_NAME_TEMPLATE.format(gpu_id=gpu_id)) + + +def ensure_private_socket_dir(directory: str, strict_perms: bool = True) -> None: + """Create the socket directory (if needed) locked down to the owner. + + Called by the daemon before binding. Existing directories are re-checked + and, for the auto-derived path, tightened so a pre-existing world-writable + directory is rejected. + """ + os.makedirs(directory, mode=0o700, exist_ok=True) + if strict_perms: + os.chmod(directory, 0o700) + verify_private_dir(directory, strict_perms=strict_perms) + + +def verify_private_dir(directory: str, strict_perms: bool = True) -> None: + """Verify a directory is a real dir owned by us and not world/group readable. + + When ``strict_perms`` is False the group/world permission bits are not + checked; this is used for directories the operator explicitly configured + (they own the trust decision), while the auto-derived per-user directory is + always checked strictly. + """ + info = os.lstat(directory) + if stat.S_ISLNK(info.st_mode): + raise WeightCacheUnavailableError( + f"Refusing to use symlinked socket directory {directory}" + ) + if not stat.S_ISDIR(info.st_mode): + raise WeightCacheUnavailableError(f"{directory} is not a directory") + uid = _current_uid() + if uid != -1 and info.st_uid != uid: + raise WeightCacheUnavailableError( + f"Socket directory {directory} is not owned by the current user" + ) + if strict_perms and info.st_mode & 0o077: + raise WeightCacheUnavailableError( + f"Socket directory {directory} is group/world accessible" + ) + + +def verify_socket_owner(socket_path: str, strict_perms: bool = True) -> None: + """Verify the socket lives in a private dir and is owned by the current user. + + Called by the engine before connecting so it never talks to a socket a + different user could have planted. + """ + verify_private_dir(os.path.dirname(socket_path), strict_perms=strict_perms) + info = os.lstat(socket_path) + if stat.S_ISLNK(info.st_mode): + raise WeightCacheUnavailableError( + f"Refusing to connect to symlinked socket {socket_path}" + ) + uid = _current_uid() + if uid != -1 and info.st_uid != uid: + raise WeightCacheUnavailableError( + f"Socket {socket_path} is not owned by the current user" + ) + + +def verify_peer_is_owner(conn: socket.socket) -> None: + """Best-effort check that the connecting peer runs as the current user. + + Uses SO_PEERCRED where available (Linux). Silently returns on platforms + that do not expose peer credentials. + """ + so_peercred = getattr(socket, "SO_PEERCRED", None) + if so_peercred is None: + return + try: + creds = conn.getsockopt(socket.SOL_SOCKET, so_peercred, struct.calcsize("3i")) + _, peer_uid, _ = struct.unpack("3i", creds) + except OSError: + return + uid = _current_uid() + if uid != -1 and peer_uid != uid: + raise PermissionError(f"Rejecting weight cache connection from uid {peer_uid}") + + +def _hash_quant_config(quant_config: Any) -> str: + if quant_config is None: + return "" + if hasattr(quant_config, "to_dict"): + quant_config = quant_config.to_dict() + payload = json.dumps(quant_config, sort_keys=True, default=str) + return safe_hash(payload.encode(), usedforsecurity=False).hexdigest() + + +def _safetensors_header(path: str) -> bytes: + """Return the raw safetensors header (length prefix + JSON) of a file. + + The header carries tensor names, dtypes, shapes and byte offsets, so it is + a content fingerprint of the shard without reading any weight bytes. + """ + with open(path, "rb") as f: + size_bytes = f.read(8) + (header_len,) = struct.unpack(" str | None: + """Fingerprint checkpoint content from local safetensors metadata. + + Hashes each shard's safetensors header so a daemon and an engine pointing + at identical weights in different directories produce the same key. Returns + None when local safetensors files can't be located (e.g. an undownloaded + Hugging Face repo id), leaving the caller to fall back to the model path. + """ + if not os.path.isdir(model): + return None + from transformers.utils import SAFE_WEIGHTS_INDEX_NAME + + from vllm.model_executor.model_loader.weight_utils import ( + filter_duplicate_safetensors_files, + ) + + files = glob.glob(os.path.join(model, "*.safetensors")) + if os.path.isfile(os.path.join(model, SAFE_WEIGHTS_INDEX_NAME)): + files = filter_duplicate_safetensors_files( + files, model, SAFE_WEIGHTS_INDEX_NAME + ) + if not files: + return None + hasher = safe_hash(b"", usedforsecurity=False) + for path in sorted(files, key=os.path.basename): + hasher.update(os.path.basename(path).encode()) + hasher.update(_safetensors_header(path)) + return hasher.hexdigest() + + +@dataclass(frozen=True) +class WeightCacheKey: + """Fingerprint of the cached weights. + + Any mismatch between the daemon's and the engine's fingerprint means the + cached weights cannot be reused and the engine must load from disk. + """ + + checkpoint: str + model_arch: str + tp_size: int + tp_rank: int + dtype: str + quantization: str | None + quant_config_hash: str + revision: str | None + vllm_version: str + + @classmethod + def from_model_config( + cls, model_config: ModelConfig, tp_size: int, tp_rank: int + ) -> "WeightCacheKey": + """Build the fingerprint for a model configuration. + + Must be called before weight loading: process_weights_after_loading + may mutate hf_config.quantization_config, which would change the hash + between the daemon and the engine. + + The checkpoint is identified by a hash of its safetensors metadata when + the weights are available locally, so a daemon and engine referencing + identical weights in different directories still match; otherwise it + falls back to the model path. + """ + hf_config = model_config.hf_config + arch = ",".join(getattr(hf_config, "architectures", None) or []) + quant_config = getattr(hf_config, "quantization_config", None) + checkpoint = hash_checkpoint(model_config.model) or model_config.model + return cls( + checkpoint=checkpoint, + model_arch=arch, + tp_size=tp_size, + tp_rank=tp_rank, + dtype=str(model_config.dtype), + quantization=model_config.quantization, + quant_config_hash=_hash_quant_config(quant_config), + revision=model_config.revision, + vllm_version=vllm.version.__version__, + ) + + def mismatched_fields(self, other: "WeightCacheKey") -> list[str]: + return [ + f.name + for f in fields(self) + if getattr(self, f.name) != getattr(other, f.name) + ] + + +@dataclass +class TensorEntry: + """A single cached tensor. + + CUDA tensors are exported as `torch.multiprocessing` reduction args + (CUDA IPC handles); non-CUDA tensors are shipped by value. + """ + + kind: str + """Either "param" or "buffer".""" + ipc_args: tuple | None = None + cpu_tensor: torch.Tensor | None = None + + @classmethod + def from_tensor(cls, tensor: torch.Tensor, kind: str) -> "TensorEntry": + from torch.multiprocessing.reductions import reduce_tensor + + tensor = tensor.detach() + if tensor.is_cuda: + _, ipc_args = reduce_tensor(tensor) + return cls(kind=kind, ipc_args=ipc_args) + return cls(kind=kind, cpu_tensor=tensor.cpu()) + + def rebuild(self, device_index: int) -> torch.Tensor: + if self.ipc_args is None: + assert self.cpu_tensor is not None + return self.cpu_tensor + from torch.multiprocessing.reductions import rebuild_cuda_tensor + + args = list(self.ipc_args) + # Index 6 of the args from reduce_tensor is the device index. It must + # be retargeted to the local index since the daemon and the engine may + # have different CUDA_VISIBLE_DEVICES mappings. + args[6] = device_index + return rebuild_cuda_tensor(*args) + + +def send_msg(sock: socket.socket, obj: Any) -> None: + payload = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL) + sock.sendall(_LEN_STRUCT.pack(len(payload))) + sock.sendall(payload) + + +def recv_msg(sock: socket.socket) -> Any: + (length,) = _LEN_STRUCT.unpack(_recv_exact(sock, _LEN_STRUCT.size)) + if length > MAX_MSG_SIZE: + raise ValueError(f"Message size {length} exceeds limit {MAX_MSG_SIZE}") + return pickle.loads(_recv_exact(sock, length)) + + +def _recv_exact(sock: socket.socket, num_bytes: int) -> bytes: + buf = bytearray() + while len(buf) < num_bytes: + chunk = sock.recv(num_bytes - len(buf)) + if not chunk: + raise ConnectionError("Socket closed while receiving message") + buf.extend(chunk) + return bytes(buf) diff --git a/vllm/model_executor/utils.py b/vllm/model_executor/utils.py index 180820d61b..6c5c517d9c 100644 --- a/vllm/model_executor/utils.py +++ b/vllm/model_executor/utils.py @@ -3,12 +3,35 @@ """Utils for model executor.""" import copy +from contextlib import contextmanager +from contextvars import ContextVar from typing import Any import torch from vllm.utils.torch_utils import is_torch_equal_or_newer +_weights_pre_processed: ContextVar[bool] = ContextVar( + "weights_pre_processed", default=False +) + + +@contextmanager +def weights_already_processed(): + """Mark that weights are already in post-processed (runtime) format, so + ``process_weights_after_loading`` skips tensor transforms (used by the + weight cache IPC loader). + """ + token = _weights_pre_processed.set(True) + try: + yield + finally: + _weights_pre_processed.reset(token) + + +def is_weights_pre_processed() -> bool: + return _weights_pre_processed.get() + def set_weight_attrs( weight: torch.Tensor,