Fast Start (#54921)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: liusy58 <liusy58@smail.nju.edu.cn>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
siyu
2026-09-04 06:32:06 -07:00
committed by GitHub
co-authored by liusy58 Isotr0py
parent 07ee97e023
commit a69e75b9b6
16 changed files with 1529 additions and 13 deletions
@@ -0,0 +1,153 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end tests for the IPC weight cache loader.
A cold start (no weight cache daemon, so the loader falls back to disk) and
warm restarts (weights mapped from the daemon via CUDA IPC) must both serve
identical outputs.
"""
import shutil
import subprocess
import sys
import tempfile
import threading
import time
from vllm import SamplingParams
MODEL = "Qwen/Qwen3.5-0.8B"
PROMPTS = [
"Hello, my name is",
"The capital of France is",
]
DAEMON_TIMEOUT_S = 600
class WeightCacheDaemon:
"""Context manager running the real weight cache daemon as a subprocess."""
_READY_MARKER = "Weight cache daemon READY"
def __init__(self, model: str, tp_size: int):
# Short base path: Unix socket paths are limited to ~107 characters.
self.socket_dir = tempfile.mkdtemp(prefix="vllm_ipc_")
self._cmd = [
sys.executable,
"-m",
"vllm.model_executor.model_loader.weight_cache.daemon",
"--model",
model,
"--tensor-parallel-size",
str(tp_size),
"--weight-cache-socket-dir",
self.socket_dir,
"--enforce-eager",
]
self._proc: subprocess.Popen | None = None
self._lines: list[str] = []
def __enter__(self) -> "WeightCacheDaemon":
self._proc = subprocess.Popen(
self._cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True
)
threading.Thread(target=self._drain_stderr, daemon=True).start()
try:
self._wait_ready(DAEMON_TIMEOUT_S)
except Exception:
self._stop()
raise
return self
def __exit__(self, *exc_info) -> None:
self._stop()
def _drain_stderr(self) -> None:
assert self._proc is not None and self._proc.stderr is not None
for line in self._proc.stderr:
self._lines.append(line)
def _logs(self) -> str:
return "".join(self._lines)
def _wait_ready(self, timeout_s: float) -> None:
assert self._proc is not None
deadline = time.monotonic() + timeout_s
while time.monotonic() < deadline:
if self._proc.poll() is not None:
raise RuntimeError(
f"Weight cache daemon exited with {self._proc.returncode}:\n"
f"{self._logs()}"
)
if self._READY_MARKER in self._logs():
return
time.sleep(1.0)
raise TimeoutError(
f"Weight cache daemon not ready after {timeout_s}s:\n{self._logs()}"
)
def _stop(self) -> None:
assert self._proc is not None
self._proc.terminate()
try:
self._proc.wait(timeout=60)
except subprocess.TimeoutExpired:
self._proc.kill()
self._proc.wait()
shutil.rmtree(self.socket_dir, ignore_errors=True)
def generate(vllm_runner, socket_dir: str | None, fallback: bool, **kwargs):
extra_config = (
{} if socket_dir is None else {"socket_dir": socket_dir, "fallback": fallback}
)
with vllm_runner(
MODEL,
load_format="auto" if socket_dir is None else "ipc_cache",
model_loader_extra_config=extra_config,
# Greedy outputs are compared across engine restarts; cap the batch
# size so scheduling cannot change the results.
max_num_seqs=1,
**kwargs,
) as llm:
sampling_params = SamplingParams(temperature=0, max_tokens=16, ignore_eos=True)
return llm.generate(PROMPTS, sampling_params)
def test_ipc_cache_cold_start_and_warm_restart(vllm_runner):
"""Cold start falls back to disk; warm restarts load weights via CUDA IPC.
All runs must produce outputs identical to a default-loader baseline. The
warm runs disable the disk fallback, so they only pass if the weights
really came from the daemon.
"""
llm_kwargs = dict(
gpu_memory_utilization=0.3,
enforce_eager=True,
# Qwen3.5 is a hybrid mamba model; its mamba cache mode requires
# chunked prefill.
enable_chunked_prefill=True,
)
# Baseline: plain disk loading with the default loader.
baseline_outputs = generate(vllm_runner, None, fallback=True, **llm_kwargs)
assert all(text for _, texts in baseline_outputs for text in texts)
# Cold start: no daemon is serving, so the loader falls back to disk.
with tempfile.TemporaryDirectory(prefix="vllm_ipc_empty_") as empty_socket_dir:
cold_outputs = generate(
vllm_runner, empty_socket_dir, fallback=True, **llm_kwargs
)
with WeightCacheDaemon(MODEL, tp_size=1) as daemon:
warm_outputs = generate(
vllm_runner, daemon.socket_dir, fallback=False, **llm_kwargs
)
# Warm restart: a second engine lifetime against the same daemon.
restart_outputs = generate(
vllm_runner, daemon.socket_dir, fallback=False, **llm_kwargs
)
assert cold_outputs == baseline_outputs
assert warm_outputs == baseline_outputs
assert restart_outputs == baseline_outputs
@@ -48,6 +48,7 @@ CHECK_IMPORTS = {
"vllm/distributed/device_communicators/shm_object_storage.py",
"vllm/distributed/weight_transfer/ipc_engine.py",
"vllm/distributed/weight_transfer/clients.py",
"vllm/model_executor/model_loader/weight_cache/protocol.py",
"tests/distributed/test_shm_broadcast.py",
"tests/distributed/test_weight_transfer.py",
"vllm/utils/hashing.py",
+4
View File
@@ -38,6 +38,10 @@ class LoadConfig:
- "instanttensor" will load the Safetensors weights on CUDA devices using
InstantTensor, which enables distributed loading with pipelined prefetching
and fast direct I/O.
- "ipc_cache" will map post-quantized weights from a local weight cache
daemon via CUDA IPC for fast engine restarts. See
`vllm/model_executor/model_loader/weight_cache/daemon.py` for how to
launch the daemon.
- "npcache" will load the weights in pytorch format and store a numpy cache
to speed up the loading.
- "dummy" will initialize the weights with random values, which is mainly
@@ -27,7 +27,11 @@ from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
SharedExperts,
)
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
from vllm.model_executor.utils import (
is_weights_pre_processed,
replace_parameter,
set_weight_attrs,
)
from vllm.platforms import current_platform
if TYPE_CHECKING:
@@ -43,6 +47,8 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
# --8<-- [end:unquantized_fused_moe]
supports_pre_processed_weights = True
def __init__(self, moe: FusedMoEConfig):
super().__init__(moe)
self.unquantized_backend, self.experts_cls = select_unquantized_moe_backend(
@@ -153,16 +159,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
# optional w{13,2}_bias references and SwiGLU gate params. Since
# weight updates mutate those bias tensors in place, the kernel
# does not need to be re-built.
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
assert self.moe_quant_config is not None
assert self.experts_cls is not None
self.moe_kernel = make_unquantized_moe_kernel(
quant_config=self.moe_quant_config,
moe_config=self.moe,
backend=self.unquantized_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
)
self._init_moe_kernel(layer)
if self.unquantized_backend == UnquantizedMoeBackend.CPU:
# The CPU experts need the layer itself for the setup that
@@ -170,11 +167,30 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
# it only sees the two weight tensors: padding and prepacking
# into the grouped-gemm layout (bias included), and capturing
# the router config that monolithic apply() cannot carry.
assert self.moe_kernel is not None
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
def _init_moe_kernel(self, layer: "RoutedExperts") -> None:
"""Build the MoE kernel from the layer's current (shuffled) weights."""
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
assert self.moe_quant_config is not None
assert self.experts_cls is not None
self.moe_kernel = make_unquantized_moe_kernel(
quant_config=self.moe_quant_config,
moe_config=self.moe,
backend=self.unquantized_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
)
def process_weights_after_loading(self, layer: "RoutedExperts") -> None:
super().process_weights_after_loading(layer)
if is_weights_pre_processed():
# Weights are already in runtime format; rebuild the kernel only.
self._init_moe_kernel(layer)
return
# Padding may allocate a same-shaped strided view. Copy it back instead
# of rebinding `.data`, since CUDA graphs capture the parameter storage
# address.
+2
View File
@@ -163,6 +163,8 @@ class LinearMethodBase(QuantizeMethodBase):
class UnquantizedLinearMethod(LinearMethodBase):
"""Linear method without quantization."""
supports_pre_processed_weights = True
def __init__(self) -> None:
config = get_current_vllm_config_or_none()
linear_backend = (
@@ -27,6 +27,13 @@ class QuantizeMethodBase(ABC):
in process_weights_after_loading, reducing peak memory during loading.
"""
supports_pre_processed_weights: bool = False
"""
Whether ``process_weights_after_loading`` supports running under
``weights_already_processed``. Methods must skip tensor transforms in
that mode; the loader driver rejects methods that do not declare support.
"""
@abstractmethod
def create_weights(
self, layer: torch.nn.Module, *weight_args, **extra_weight_attrs
+31 -1
View File
@@ -74,7 +74,11 @@ from vllm.model_executor.parameter import (
BlockQuantScaleParameter,
PerTensorScaleParameter,
)
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
from vllm.model_executor.utils import (
is_weights_pre_processed,
replace_parameter,
set_weight_attrs,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import (
is_deep_gemm_supported,
@@ -248,6 +252,8 @@ class Fp8LinearMethod(LinearMethodBase):
quant_config: The quantization config.
"""
supports_pre_processed_weights = True
def __init__(self, quant_config: Fp8Config):
self.quant_config = quant_config
self.is_scale_e8m0 = getattr(quant_config, "is_scale_e8m0", False)
@@ -366,6 +372,19 @@ class Fp8LinearMethod(LinearMethodBase):
self.use_marlin = isinstance(self.fp8_linear, MarlinFP8ScaledMMLinearKernel)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
if is_weights_pre_processed():
# Weights are already in runtime format; the only state tensor
# export cannot carry is the `input_scale = None` stamp that dynamic
# activation quantization relies on, since create_weights never
# registered that parameter.
if not self.act_q_static and not hasattr(layer, "input_scale"):
layer.input_scale = None
# Marlin reads marlin_input_dtype in apply_weights; it lives on the
# method (not exported with the weights), so restore it here too.
if self.use_marlin and hasattr(self.fp8_linear, "marlin_input_dtype"):
self.fp8_linear.marlin_input_dtype = self.marlin_input_dtype
return
if self.use_marlin:
if not self.block_quant:
# Canonicalize to (K, N) for the kernel.
@@ -472,6 +491,8 @@ class Fp8MoEMethod(FusedMoEMethodBase):
quant_config: The quantization config.
"""
supports_pre_processed_weights = True
def __init__(self, quant_config: Fp8Config, layer: RoutedExperts):
super().__init__(layer.moe_config)
self.quant_config = quant_config
@@ -713,6 +734,10 @@ class Fp8MoEMethod(FusedMoEMethodBase):
replace_parameter(layer, f"w13_{self.weight_scale_name}", w13_scale)
replace_parameter(layer, f"w2_{self.weight_scale_name}", w2_scale)
self._init_moe_kernel(layer)
def _init_moe_kernel(self, layer: RoutedExperts) -> None:
"""Build the MoE kernel from the layer's current (converted) weights."""
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
assert self.moe_quant_config is not None
assert self.experts_cls is not None
@@ -725,6 +750,11 @@ class Fp8MoEMethod(FusedMoEMethodBase):
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
if is_weights_pre_processed():
# Weights are already in kernel format; rebuild the kernel only.
self._init_moe_kernel(layer)
return
# Allow for accessing weights and scales in standard way.
w13 = layer.w13_weight
w2 = layer.w2_weight
@@ -8,6 +8,7 @@ from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
QuantizeMethodBase,
)
from vllm.model_executor.utils import is_weights_pre_processed
from vllm.platforms import current_platform
from vllm.utils.torch_utils import is_quantized_kv_cache
from vllm.v1.kv_cache_interface import kv_cache_uses_per_token_head_scales
@@ -51,6 +52,8 @@ class BaseKVCacheMethod(QuantizeMethodBase):
quant_config: the appropriate QuantizationConfig
"""
supports_pre_processed_weights = True
def __init__(self, quant_config: QuantizationConfig):
self.quant_config = quant_config
@@ -72,6 +75,20 @@ class BaseKVCacheMethod(QuantizeMethodBase):
raise RuntimeError(f"{self.__class__.__name__}.apply should not be called.")
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
if is_weights_pre_processed():
# Scale buffers were exported with post-processed values; drop the
# consumed scale placeholders and rebuild the host-side copies.
for name in ("k_scale", "v_scale", "q_scale", "prob_scale"):
if hasattr(layer, name):
delattr(layer, name)
if hasattr(layer, "_k_scale"):
layer._k_scale_float = layer._k_scale.item()
layer._v_scale_float = layer._v_scale.item()
layer._q_scale_float = layer._q_scale.item()
layer._k_scale_cpu = layer._k_scale.detach().to("cpu")
layer._v_scale_cpu = layer._v_scale.detach().to("cpu")
return
# skip if there are no weights to process (for example, weight reloading)
if not hasattr(layer, "q_scale"):
assert not hasattr(layer, "k_scale")
@@ -36,6 +36,8 @@ DEFAULT_VOCAB_PADDING_SIZE = 64
class UnquantizedEmbeddingMethod(QuantizeMethodBase):
"""Unquantized method for embeddings."""
supports_pre_processed_weights = True
def create_weights(
self,
layer: torch.nn.Module,
@@ -24,6 +24,7 @@ from vllm.model_executor.model_loader.utils import (
get_model_architecture,
get_model_cls,
)
from vllm.model_executor.model_loader.weight_cache.ipc_loader import IpcModelLoader
logger = init_logger(__name__)
@@ -35,6 +36,7 @@ LoadFormats = Literal[
"dummy",
"fastsafetensors",
"instanttensor",
"ipc_cache",
"mistral",
"modelexpress",
"npcache",
@@ -51,6 +53,7 @@ _LOAD_FORMAT_TO_MODEL_LOADER: dict[str, type[BaseModelLoader]] = {
"dummy": DummyModelLoader,
"fastsafetensors": DefaultModelLoader,
"instanttensor": DefaultModelLoader,
"ipc_cache": IpcModelLoader,
"mistral": DefaultModelLoader,
"modelexpress": ModelExpressModelLoader,
"npcache": DefaultModelLoader,
@@ -150,6 +153,7 @@ __all__ = [
"ModelExpressModelLoader",
"DefaultModelLoader",
"DummyModelLoader",
"IpcModelLoader",
"RunaiModelStreamerLoader",
"ShardedStateLoader",
"TensorizerLoader",
+16 -1
View File
@@ -26,6 +26,7 @@ from vllm.model_executor.model_loader.reload import (
)
from vllm.model_executor.model_loader.weight_tying import maybe_retie_word_embeddings
from vllm.model_executor.models.interfaces import SupportsQuant
from vllm.model_executor.utils import is_weights_pre_processed
from vllm.tracing import instrument
from vllm.utils.mem_utils import release_device_memory_under_pressure
from vllm.utils.platform_utils import is_pin_memory_available
@@ -97,13 +98,27 @@ def initialize_model(
def process_weights_after_loading(
model: nn.Module, model_config: ModelConfig, target_device: torch.device
) -> None:
"""Post-process loaded weights into runtime format.
Under ``weights_already_processed`` (weight cache IPC loader), quant
methods skip tensor transforms and must declare
``supports_pre_processed_weights``, otherwise this raises ``RuntimeError``.
"""
# Reclaim memory when an explicit lm_head has been
# loaded, but it is identical to the input embeddings.
maybe_retie_word_embeddings(model, model_config)
for _, module in model.named_modules():
for name, module in model.named_modules():
quant_method = getattr(module, "quant_method", None)
if isinstance(quant_method, QuantizeMethodBase):
if (
is_weights_pre_processed()
and not quant_method.supports_pre_processed_weights
):
raise RuntimeError(
f"layer {name or '<root>'}: {type(quant_method).__name__} "
"does not support pre-processed weights"
)
# When quant methods need to process weights after loading
# (for repacking, quantizing, etc), they expect parameters
# to be on the global target device. This scope is for the
@@ -0,0 +1,24 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.model_executor.model_loader.weight_cache.ipc_loader import IpcModelLoader
from vllm.model_executor.model_loader.weight_cache.protocol import (
CacheConfigMismatchError,
TensorEntry,
UnsupportedQuantForIPCError,
WeightCacheKey,
WeightCacheUnavailableError,
check_ipc_quant_support,
is_ipc_quant_supported,
)
__all__ = [
"CacheConfigMismatchError",
"IpcModelLoader",
"TensorEntry",
"UnsupportedQuantForIPCError",
"WeightCacheKey",
"WeightCacheUnavailableError",
"check_ipc_quant_support",
"is_ipc_quant_supported",
]
@@ -0,0 +1,412 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Weight cache daemon for fast engine restarts.
One daemon process per GPU holds the post-quantized, TP-sharded weights of its
rank in GPU memory and serves CUDA IPC handles to vLLM engines over a Unix
domain socket. Restarting engines map the weights via zero-copy IPC instead of
reloading from disk.
Launch one daemon per TP rank with a single command:
python -m vllm.model_executor.model_loader.weight_cache.daemon \\
--model /path/to/model --tensor-parallel-size 4
Engines then load from the daemons with:
vllm serve /path/to/model --tensor-parallel-size 4 \\
--load-format ipc_cache
Only tensor parallelism is supported; pipeline, data, and expert parallelism
are rejected at launch.
"""
import contextlib
import fcntl
import multiprocessing
import os
import queue
import signal
import socket
import sys
from collections.abc import Callable
import torch
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
from vllm.distributed import (
ensure_model_parallel_initialized,
init_distributed_environment,
)
from vllm.logger import init_logger
from vllm.model_executor.model_loader.weight_cache.protocol import (
TensorEntry,
WeightCacheKey,
WeightCacheUnavailableError,
check_ipc_quant_support,
ensure_private_socket_dir,
get_physical_device_id,
get_socket_path,
recv_msg,
send_msg,
verify_peer_is_owner,
)
from vllm.platforms import current_platform
logger = init_logger(__name__)
def _report_ready(message: str) -> None:
"""Write a readiness message straight to the original stderr descriptor.
Loading the model pulls in FlashInfer's CuTeDSL JIT compiler, which swaps
``sys.stdout``/``sys.stderr`` for in-memory buffers while compiling kernels
across worker threads. That save/restore races on the interpreter-global
streams and can leave them (and vLLM's logging handler, which caches the
original stream object) detached, silently swallowing everything logged
afterwards -- including the daemon's readiness announcement. Since operators
rely on that line to know the daemon is serving, write it directly to file
descriptor 2 so it bypasses the logging machinery entirely.
"""
with contextlib.suppress(OSError):
os.write(2, (message + "\n").encode())
def export_entries(
model: torch.nn.Module,
) -> tuple[dict[str, TensorEntry], dict[str, str]]:
"""Export a model's tensors, preserving tied-parameter aliases.
``named_parameters``/``named_buffers`` are iterated with
``remove_duplicate=False`` so tied weights (e.g. ``lm_head.weight`` sharing
storage with ``embed_tokens.weight``) are not silently dropped. Each unique
tensor is exported once; every additional name that refers to the same
tensor object is recorded in the returned alias map so the client can
re-establish the shared identity instead of allocating uninitialized
memory for it.
Returns:
A ``(entries, aliases)`` pair where ``entries`` maps a canonical name to
its ``TensorEntry`` and ``aliases`` maps each duplicate name to its
canonical name.
"""
entries: dict[str, TensorEntry] = {}
aliases: dict[str, str] = {}
canonical_by_id: dict[int, str] = {}
def _add(name: str, tensor: torch.Tensor, kind: str) -> None:
canonical = canonical_by_id.get(id(tensor))
if canonical is not None:
aliases[name] = canonical
return
canonical_by_id[id(tensor)] = name
entries[name] = TensorEntry.from_tensor(tensor, kind)
for name, param in model.named_parameters(remove_duplicate=False):
_add(name, param, "param")
# named_buffers includes non-persistent buffers (e.g. rotary embedding
# caches) that state_dict would miss.
for name, buffer in model.named_buffers(remove_duplicate=False):
if name in entries or name in aliases:
continue
_add(name, buffer, "buffer")
return entries, aliases
class WeightCacheDaemon:
"""Per-GPU process that loads one TP shard and serves CUDA IPC handles."""
def __init__(
self,
vllm_config: VllmConfig,
tp_rank: int,
distributed_init_method: str,
socket_dir: str | None = None,
):
self.vllm_config = vllm_config
self.tp_rank = tp_rank
self.distributed_init_method = distributed_init_method
self.socket_dir = socket_dir
self.model: torch.nn.Module | None = None
self.entries: dict[str, TensorEntry] = {}
self.aliases: dict[str, str] = {}
# Fingerprint before loading: process_weights_after_loading may
# mutate hf_config.quantization_config.
self.cache_config = WeightCacheKey.from_model_config(
vllm_config.model_config,
tp_size=vllm_config.parallel_config.tensor_parallel_size,
tp_rank=tp_rank,
)
def load_model(self) -> None:
from vllm.model_executor.model_loader import get_model
tp_size = self.cache_config.tp_size
torch.accelerator.set_device_index(self.tp_rank)
init_distributed_environment(
world_size=tp_size,
rank=self.tp_rank,
distributed_init_method=self.distributed_init_method,
local_rank=self.tp_rank,
backend=current_platform.dist_backend,
)
with set_current_vllm_config(self.vllm_config):
ensure_model_parallel_initialized(tp_size, 1)
self.model = get_model(vllm_config=self.vllm_config)
self._export_entries()
logger.info(
"Weight cache daemon rank %d cached %d tensors",
self.tp_rank,
len(self.entries),
)
def _export_entries(self) -> None:
assert self.model is not None
self.entries, self.aliases = export_entries(self.model)
def serve_forever(self, ready_callback: Callable[[], None] | None = None) -> None:
"""Serve requests until terminated.
The socket is only bound once the model is fully cached, so clients
get a connection error (and fall back to disk) until the daemon is
ready.
Args:
ready_callback: Invoked once the socket is bound and listening,
so the launcher can report overall readiness.
"""
socket_path = self._socket_path()
ensure_private_socket_dir(
os.path.dirname(socket_path), strict_perms=self.socket_dir is None
)
# Hold an exclusive per-GPU lock for the daemon's lifetime so a second
# daemon cannot remove this daemon's live socket and hijack the path.
lock_fd = self._acquire_gpu_lock(socket_path)
if os.path.exists(socket_path):
os.unlink(socket_path)
server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
server.bind(socket_path)
os.chmod(socket_path, 0o600)
server.listen()
logger.info(
"Weight cache daemon rank %d serving on %s", self.tp_rank, socket_path
)
print(
f"Weight cache daemon rank {self.tp_rank} ready: serving on {socket_path}"
)
if ready_callback is not None:
ready_callback()
try:
while True:
conn, _ = server.accept()
with conn:
try:
verify_peer_is_owner(conn)
self._handle_connection(conn)
except (ConnectionError, EOFError):
logger.warning("Client disconnected mid-request")
except Exception:
# A single malformed or malicious request must not take
# down the daemon for every other engine on this GPU.
logger.exception(
"Error handling weight cache client; continuing"
)
finally:
server.close()
if os.path.exists(socket_path):
os.unlink(socket_path)
os.close(lock_fd)
def _acquire_gpu_lock(self, socket_path: str) -> int:
"""Take an exclusive lock guarding this GPU's socket path.
The lock is advisory and released automatically when the daemon exits
(or crashes), so a stale socket is only ever removed by whoever owns
the lock. A running daemon holding it makes a second daemon fail fast
instead of clobbering the live socket.
"""
lock_fd = os.open(f"{socket_path}.lock", os.O_CREAT | os.O_RDWR, 0o600)
try:
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError as e:
os.close(lock_fd)
raise WeightCacheUnavailableError(
f"Another weight cache daemon already owns {socket_path}"
) from e
return lock_fd
def _socket_path(self) -> str:
device_index = torch.accelerator.current_device_index()
gpu_id = get_physical_device_id(device_index)
if gpu_id is None:
gpu_id = device_index
return get_socket_path(gpu_id, self.socket_dir)
def _handle_connection(self, conn: socket.socket) -> None:
request = recv_msg(conn)
cmd = request.get("cmd")
if cmd == "get_state":
self._handle_get_state(conn, request)
elif cmd == "release":
self._handle_release(conn)
else:
send_msg(conn, {"status": "error", "message": f"Unknown command {cmd!r}"})
def _handle_get_state(self, conn: socket.socket, request: dict) -> None:
client_config = request.get("cache_config")
if not isinstance(client_config, WeightCacheKey):
send_msg(conn, {"status": "error", "message": "Missing cache_config"})
return
mismatched = self.cache_config.mismatched_fields(client_config)
if mismatched:
logger.warning("WeightCacheKey mismatch on fields: %s", mismatched)
send_msg(conn, {"status": "mismatch", "fields": mismatched})
return
if not self.entries:
send_msg(conn, {"status": "error", "message": "Weights were released"})
return
send_msg(
conn,
{
"status": "ok",
"entries": self.entries,
"aliases": self.aliases,
"gpu_uuid": self._gpu_uuid(),
},
)
def _handle_release(self, conn: socket.socket) -> None:
self.entries.clear()
self.aliases.clear()
self.model = None
torch.accelerator.empty_cache()
logger.info("Weight cache daemon rank %d released cached weights", self.tp_rank)
send_msg(conn, {"status": "ok"})
def _gpu_uuid(self) -> str:
props = torch.cuda.get_device_properties(
torch.accelerator.current_device_index()
)
return str(props.uuid)
def _run_daemon(
tp_rank: int,
vllm_config: VllmConfig,
distributed_init_method: str,
socket_dir: str | None,
ready_queue: "multiprocessing.Queue[int]",
) -> None:
daemon = WeightCacheDaemon(
vllm_config, tp_rank, distributed_init_method, socket_dir
)
daemon.load_model()
daemon.serve_forever(ready_callback=lambda: ready_queue.put(tp_rank))
def _reject_unsupported_parallelism(parallel_config: ParallelConfig) -> None:
"""Reject every parallelism mode other than tensor parallelism."""
unsupported = {
"pipeline parallelism": parallel_config.pipeline_parallel_size > 1,
"data parallelism": parallel_config.data_parallel_size > 1,
"expert parallelism": parallel_config.enable_expert_parallel,
}
for name, enabled in unsupported.items():
if enabled:
raise ValueError(
f"The weight cache daemon only supports tensor parallelism; "
f"{name} is not supported"
)
def main() -> None:
from vllm.engine.arg_utils import EngineArgs
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.network_utils import get_distributed_init_method, get_open_port
parser = FlexibleArgumentParser(
description="Launch weight cache daemons (one per TP rank)."
)
EngineArgs.add_cli_args(parser)
parser.add_argument(
"--weight-cache-socket-dir",
type=str,
default=None,
help="Directory for the daemon Unix sockets (default: tempdir).",
)
args = parser.parse_args()
engine_args = EngineArgs.from_cli_args(args)
vllm_config = engine_args.create_engine_config()
if vllm_config.load_config.load_format == "ipc_cache":
raise ValueError(
"The weight cache daemon itself must load from disk; use the "
"default --load-format"
)
# Checked before loading anything: an unsupported quantization method would
# otherwise only surface in the engine, after a full load.
check_ipc_quant_support(vllm_config.model_config, where="daemon")
parallel_config = vllm_config.parallel_config
_reject_unsupported_parallelism(parallel_config)
tp_size = parallel_config.tensor_parallel_size
distributed_init_method = get_distributed_init_method("127.0.0.1", get_open_port())
ctx = multiprocessing.get_context("spawn")
ready_queue: multiprocessing.Queue[int] = ctx.Queue()
procs = [
ctx.Process(
target=_run_daemon,
args=(
rank,
vllm_config,
distributed_init_method,
args.weight_cache_socket_dir,
ready_queue,
),
name=f"vllm-weight-cache-daemon-{rank}",
)
for rank in range(tp_size)
]
for proc in procs:
proc.start()
def _shutdown(signum, frame):
for proc in procs:
proc.terminate()
signal.signal(signal.SIGINT, _shutdown)
signal.signal(signal.SIGTERM, _shutdown)
ready_ranks: set[int] = set()
while len(ready_ranks) < tp_size:
try:
ready_ranks.add(ready_queue.get(timeout=1.0))
except queue.Empty:
dead = [p for p in procs if p.exitcode is not None]
if dead:
logger.error(
"Weight cache daemon rank(s) exited during startup "
"(exitcodes=%s); shutting down.",
[p.exitcode for p in dead],
)
for proc in procs:
proc.terminate()
for proc in procs:
proc.join()
sys.exit(max((p.exitcode or 0) for p in procs))
logger.info_once(
"===== Weight cache daemon READY: all %d ranks serving in %s =====",
tp_size,
args.weight_cache_socket_dir or "the default socket dir",
)
_report_ready(
f"===== Weight cache daemon READY: all {tp_size} rank(s) serving in "
f"{args.weight_cache_socket_dir or 'the default socket dir'} ====="
)
for proc in procs:
proc.join()
sys.exit(max(proc.exitcode or 0 for proc in procs))
if __name__ == "__main__":
main()
@@ -0,0 +1,394 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""IPC model loader: maps post-quantized weights from a local weight cache
daemon via CUDA IPC instead of loading from disk."""
import dataclasses
import socket
from copy import copy
import torch
import torch.nn as nn
from vllm.config import ModelConfig, VllmConfig
from vllm.config.load import LoadConfig
from vllm.logger import init_logger
from vllm.model_executor.model_loader.base_loader import BaseModelLoader
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
from vllm.model_executor.model_loader.utils import (
initialize_model,
process_weights_after_loading,
)
from vllm.model_executor.model_loader.weight_cache.protocol import (
CacheConfigMismatchError,
TensorEntry,
UnsupportedQuantForIPCError,
WeightCacheKey,
WeightCacheUnavailableError,
check_ipc_quant_support,
get_physical_device_id,
get_socket_path,
recv_msg,
send_msg,
verify_socket_owner,
)
from vllm.model_executor.utils import weights_already_processed
from vllm.tracing import instrument
from vllm.utils.torch_utils import set_default_torch_dtype
logger = init_logger(__name__)
_CONNECT_TIMEOUT_S = 5.0
_STATE_TIMEOUT_S = 300.0
class IpcModelLoader(BaseModelLoader):
"""Loads a model by mapping the weight cache daemon's tensors via CUDA IPC.
The model is initialized on the meta device and every parameter/buffer is
replaced by the daemon's post-quantized tensor, so
process_weights_after_loading is skipped entirely. In "zero_copy" mode the
engine shares the daemon's GPU memory; in "copy" mode the tensors are
cloned into engine-owned memory and the daemon is asked to release its
cache afterwards.
Extra config keys (via --model-loader-extra-config):
- socket_path: explicit daemon socket path. Defaults to a per-GPU path
derived from the physical GPU id.
- socket_dir: directory containing the daemon sockets.
- mode: "zero_copy" (default) or "copy".
- fallback: fall back to disk loading when the daemon is unavailable or
the fingerprints mismatch (default: True).
- connect_timeout_s: socket connect timeout (default: 5.0).
- state_timeout_s: timeout for the weight-transfer request (default: 300.0).
Note: in zero-copy mode the weights live in the daemon's CUDA IPC
allocations, so sleep mode (CuMemAllocator weight offloading) must not be
used with this loader.
"""
def __init__(self, load_config: LoadConfig):
super().__init__(load_config)
extra_config = copy(load_config.model_loader_extra_config or {})
self.socket_path: str | None = extra_config.pop("socket_path", None)
self.socket_dir: str | None = extra_config.pop("socket_dir", None)
self.mode: str = extra_config.pop("mode", "zero_copy")
self.fallback: bool = extra_config.pop("fallback", True)
self.connect_timeout_s: float = float(
extra_config.pop("connect_timeout_s", _CONNECT_TIMEOUT_S)
)
self.state_timeout_s: float = float(
extra_config.pop("state_timeout_s", _STATE_TIMEOUT_S)
)
if self.mode not in ("zero_copy", "copy"):
raise ValueError(
f"Invalid weight cache mode {self.mode!r}, "
"expected 'zero_copy' or 'copy'"
)
if extra_config:
raise ValueError(
f"Unexpected extra config keys for load format "
f"{load_config.load_format}: {sorted(extra_config)}"
)
def download_model(self, model_config: ModelConfig) -> None:
DefaultModelLoader(self._fallback_load_config()).download_model(model_config)
def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:
"""Best-effort in-place reload for an already-initialized model.
Copies daemon tensors into matching parameters/buffers. The model is
expected to already be in the post-quantized layout (e.g. previously
loaded through this loader).
"""
device_index = torch.accelerator.current_device_index()
entries, _ = self._fetch_entries(model_config)
params = dict(model.named_parameters())
buffers = dict(model.named_buffers())
for name, entry in entries.items():
target = params.get(name, buffers.get(name))
source = entry.rebuild(device_index)
if target is None or target.shape != source.shape:
logger.warning("Skipping mismatched cached tensor %s", name)
continue
target.data.copy_(source)
@instrument(span_name="Load model")
def load_model(
self, vllm_config: VllmConfig, model_config: ModelConfig, prefix: str = ""
) -> nn.Module:
# Unsupported quantization is a permanent misconfiguration rather than a
# transient daemon outage, so it is raised even when fallback is on.
self._check_supported(vllm_config, model_config)
state_fetched = False
try:
entries, aliases = self._fetch_entries(model_config)
state_fetched = True
return self._build_model(
vllm_config, model_config, prefix, entries, aliases
)
except (WeightCacheUnavailableError, CacheConfigMismatchError) as e:
if not self.fallback:
raise
logger.warning(
"Weight cache unusable (%s); falling back to disk loading", e
)
except Exception:
if not self.fallback:
raise
logger.exception(
"Weight cache IPC loading failed; falling back to disk loading"
)
# _build_model failed after fetching state without reaching its
# copy-mode release, so the daemon still holds the full cache;
# release it so the disk fallback does not OOM against it.
if state_fetched and self.mode == "copy":
self._send_release()
torch.accelerator.empty_cache()
return self._fallback_load(vllm_config, model_config, prefix)
def _build_model(
self,
vllm_config: VllmConfig,
model_config: ModelConfig,
prefix: str,
entries: dict[str, TensorEntry],
aliases: dict[str, str],
) -> nn.Module:
device_config = vllm_config.device_config
load_device = (
device_config.device
if self.load_config.device is None
else self.load_config.device
)
target_device = torch.device(load_device)
device_index = (
target_device.index
if target_device.index is not None
else torch.accelerator.current_device_index()
)
with set_default_torch_dtype(model_config.dtype):
with torch.device("meta"):
model = initialize_model(
vllm_config=vllm_config,
model_config=model_config,
prefix=prefix,
)
self._apply_entries(model, entries, aliases, device_index)
# The daemon exports tensors that already went through
# process_weights_after_loading; re-run it in pre-processed mode
# so quant methods only rebuild Python-side state (e.g. the MoE
# kernel). Leftovers are materialized afterwards so that
# placeholders the daemon-side post-processing consumed are
# dropped rather than filled with uninitialized memory.
with weights_already_processed():
process_weights_after_loading(model, model_config, target_device)
_materialize_remaining_meta_tensors(
model, torch.device(target_device.type, device_index)
)
if self.mode == "copy":
self._send_release()
logger.info(
"Mapped %d tensors from the weight cache daemon (%s mode)",
len(entries),
self.mode,
)
return model.eval()
@staticmethod
def _check_supported(vllm_config: VllmConfig, model_config: ModelConfig) -> None:
check_ipc_quant_support(model_config, where="engine")
cache_dtype = vllm_config.cache_config.cache_dtype
if cache_dtype != "auto" and not str(cache_dtype).startswith("fp8"):
# BaseKVCacheMethod.process_weights_after_loading turns the loaded
# k/v scale parameters into plain float attributes. For fp8 cache
# dtypes those are rebuilt from the exported scale buffers when
# process_weights_after_loading runs in pre-processed mode; other
# quantized cache dtypes are not verified.
raise UnsupportedQuantForIPCError(
f"[weight_cache:engine] kv cache dtype {cache_dtype!r} is not "
"supported by the weight cache; use --kv-cache-dtype auto."
)
def _apply_entries(
self,
model: nn.Module,
entries: dict[str, TensorEntry],
aliases: dict[str, str],
device_index: int,
) -> None:
# remove_duplicate=False keeps tied module aliases reachable by name:
# a tied lm_head *is* the embedding module, so the deduplicated view
# would not contain "lm_head" at all.
modules = dict(model.named_modules(remove_duplicate=False))
registered: dict[str, torch.Tensor] = {}
def _register(name: str, tensor: torch.Tensor, is_param: bool) -> None:
module_name, _, leaf = name.rpartition(".")
module = modules.get(module_name)
if module is None:
raise RuntimeError(f"Cached tensor {name} has no matching module")
# Replace via registration rather than param.data assignment,
# which fails for meta tensors. Entries may also introduce
# post-quantization tensors absent from the meta model.
module._parameters.pop(leaf, None)
module._buffers.pop(leaf, None)
if is_param:
obj: torch.Tensor = (
tensor
if isinstance(tensor, nn.Parameter)
else nn.Parameter(tensor, requires_grad=False)
)
module.register_parameter(leaf, obj)
else:
obj = tensor
module.register_buffer(leaf, obj)
registered[name] = obj
for name, entry in entries.items():
tensor = entry.rebuild(device_index)
if self.mode == "copy":
tensor = tensor.clone()
_register(name, tensor, entry.kind == "param")
# Re-establish tied-weight aliases by registering the *same* object the
# canonical name resolved to, so parameter identity (and the tie) is
# preserved instead of allocating uninitialized memory.
for alias_name, canonical_name in aliases.items():
obj = registered.get(canonical_name)
if obj is None:
logger.warning(
"Cached alias %s references missing canonical tensor %s",
alias_name,
canonical_name,
)
continue
_register(alias_name, obj, isinstance(obj, nn.Parameter))
def _fetch_entries(
self, model_config: ModelConfig
) -> tuple[dict[str, TensorEntry], dict[str, str]]:
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
cache_config = WeightCacheKey.from_model_config(
model_config,
tp_size=get_tensor_model_parallel_world_size(),
tp_rank=get_tensor_model_parallel_rank(),
)
return self._request_state(cache_config)
def _request_state(
self, cache_config: WeightCacheKey
) -> tuple[dict[str, TensorEntry], dict[str, str]]:
with self._connect(self.state_timeout_s) as conn:
send_msg(conn, {"cmd": "get_state", "cache_config": cache_config})
response = recv_msg(conn)
status = response.get("status")
if status == "mismatch":
raise CacheConfigMismatchError(
f"WeightCacheKey mismatch on fields: {response.get('fields')}"
)
if status != "ok":
raise WeightCacheUnavailableError(
f"Weight cache daemon error: {response.get('message')}"
)
self._check_gpu_uuid(response.get("gpu_uuid"))
return response["entries"], response.get("aliases", {})
def _connect(self, timeout: float) -> socket.socket:
socket_path = self._resolve_socket_path()
# The auto-derived per-user directory is locked to 0700 and checked
# strictly. When the operator explicitly configures a path they own the
# trust decision, so only ownership/symlink safety is enforced.
strict_perms = self.socket_path is None and self.socket_dir is None
try:
verify_socket_owner(socket_path, strict_perms=strict_perms)
except OSError as e:
raise WeightCacheUnavailableError(
f"Weight cache socket {socket_path} is unavailable: {e}"
) from e
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.settimeout(timeout)
try:
sock.connect(socket_path)
except OSError as e:
sock.close()
raise WeightCacheUnavailableError(
f"Cannot connect to weight cache daemon at {socket_path}: {e}"
) from e
return sock
def _resolve_socket_path(self) -> str:
if self.socket_path is not None:
return self.socket_path
device_index = torch.accelerator.current_device_index()
gpu_id = get_physical_device_id(device_index)
if gpu_id is None:
raise WeightCacheUnavailableError(
"Cannot infer the physical GPU id from CUDA_VISIBLE_DEVICES; "
"pass socket_path via --model-loader-extra-config"
)
return get_socket_path(gpu_id, self.socket_dir)
def _check_gpu_uuid(self, daemon_uuid: str | None) -> None:
if daemon_uuid is None:
return
props = torch.cuda.get_device_properties(
torch.accelerator.current_device_index()
)
local_uuid = str(props.uuid)
if daemon_uuid != local_uuid:
raise CacheConfigMismatchError(
f"Daemon GPU {daemon_uuid} != engine GPU {local_uuid}; "
"check the socket path / GPU mapping"
)
def _send_release(self) -> None:
try:
with self._connect(self.connect_timeout_s) as conn:
send_msg(conn, {"cmd": "release"})
recv_msg(conn)
except (WeightCacheUnavailableError, ConnectionError, OSError):
logger.warning("Failed to ask the weight cache daemon to release")
def _fallback_load_config(self) -> LoadConfig:
# DefaultModelLoader must not see load_format="ipc_cache" or the ipc
# extra config keys.
return dataclasses.replace(
self.load_config, load_format="auto", model_loader_extra_config={}
)
def _fallback_load(
self, vllm_config: VllmConfig, model_config: ModelConfig, prefix: str
) -> nn.Module:
loader = DefaultModelLoader(self._fallback_load_config())
return loader.load_model(
vllm_config=vllm_config, model_config=model_config, prefix=prefix
)
def _materialize_remaining_meta_tensors(model: nn.Module, device: torch.device) -> None:
"""Allocate any tensors the daemon did not provide.
These are typically parameters removed on the daemon side by
process_weights_after_loading; they are not expected to be read at
runtime, so they are left uninitialized.
"""
for module_name, module in model.named_modules():
for leaf, param in list(module._parameters.items()):
if param is not None and param.device.type == "meta":
logger.warning(
"Materializing empty parameter %s.%s missing from the weight cache",
module_name,
leaf,
)
module._parameters[leaf] = nn.Parameter(
torch.empty_like(param, device=device), requires_grad=False
)
for leaf, buffer in list(module._buffers.items()):
if buffer is not None and buffer.device.type == "meta":
module._buffers[leaf] = torch.empty_like(buffer, device=device)
@@ -0,0 +1,412 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""WeightCacheKey fingerprinting and socket protocol for the weight cache daemon.
The protocol uses pickle over a Unix domain socket and is only intended for
communication between trusted local processes owned by the same user. The
sockets live in a per-user private directory (mode 0700) and the daemon
restricts the socket file permissions to the owner (0600). Both the daemon and
the engine verify that the directory and socket are owned by the current user
and are not group/world accessible before trusting them, so a different local
user cannot pre-plant a malicious socket at a predictable path.
"""
import glob
import json
import os
import pickle
import socket
import stat
import struct
import tempfile
from dataclasses import dataclass, fields
from typing import Any
import torch
import vllm.version
from vllm.config import ModelConfig
from vllm.utils.hashing import safe_hash
SOCKET_NAME_TEMPLATE = "vllm_weight_cache_gpu{gpu_id}.sock"
SOCKET_DIR_TEMPLATE = "vllm_weight_cache_{uid}"
_LEN_STRUCT = struct.Struct("!Q")
# Sanity bound for a single message. IPC handles are tiny; only small
# non-CUDA tensors are ever shipped by value.
MAX_MSG_SIZE = 1 << 34
def _current_uid() -> int:
getuid = getattr(os, "getuid", None)
return getuid() if getuid is not None else -1
class WeightCacheUnavailableError(Exception):
"""Raised when no weight cache daemon is reachable or usable."""
class CacheConfigMismatchError(Exception):
"""Raised when the daemon's cached weights don't match the engine."""
class UnsupportedQuantForIPCError(Exception):
"""Raised when a quantization method is not verified for IPC weight sharing."""
# The daemon exports tensor data only, so sharing is correct just for methods
# whose post-load effect is either fully captured by that data or rebuilt when
# process_weights_after_loading runs under weights_already_processed. Anything
# absent from this registry hard-errors: a method that silently repacks into
# shapes the client cannot reproduce or stamps unrebuildable Python-side state
# would serve wrong numerics. Extend it only after an end-to-end check against
# a disk-loaded baseline.
#
# This check is config-only so the daemon fails fast before a slow full load,
# rather than surfacing the incompatibility only after the engine has built the
# model (where process_weights_after_loading's supports_pre_processed_weights
# guard would eventually catch it).
def _quant_config_field(quant_config: Any, key: str) -> Any:
if quant_config is None:
return None
if isinstance(quant_config, dict):
return quant_config.get(key)
return getattr(quant_config, key, None)
def _fp8_round_trips_via_ipc(quant_config: Any) -> bool:
"""Only block-wise FP8 is verified.
Block-wise FP8 preserves the weight shape, while per-tensor FP8 transposes
``layer.weight`` during post-processing -- a shape the client's
meta-initialized model cannot reproduce.
"""
return _quant_config_field(quant_config, "weight_block_size") is not None
# quantization name -> predicate(quant_config) -> True when verified safe.
IPC_QUANT_ALLOWLIST: dict[str | None, Any] = {
None: lambda _quant_config: True, # unquantized
"fp8": _fp8_round_trips_via_ipc,
}
def is_ipc_quant_supported(quantization: str | None, quant_config: Any) -> bool:
predicate = IPC_QUANT_ALLOWLIST.get(quantization)
return False if predicate is None else bool(predicate(quant_config))
def check_ipc_quant_support(model_config: ModelConfig, *, where: str) -> None:
"""Hard-error unless the model's quantization is verified for IPC sharing.
Args:
model_config: Model configuration to inspect.
where: Short tag ("daemon"/"engine") used in the error message.
Raises:
UnsupportedQuantForIPCError: If the quantization method is not on the
verified allowlist.
"""
quantization = model_config.quantization
# Prefer the canonical, nested-aware config (multimodal models keep it under
# text_config); fall back to the raw hf_config for older code paths.
quant_config = getattr(
getattr(model_config, "model_arch_config", None), "quantization_config", None
)
if quant_config is None:
quant_config = getattr(model_config.hf_config, "quantization_config", None)
if is_ipc_quant_supported(quantization, quant_config):
return
verified = ", ".join(
"unquantized" if name is None else repr(name) for name in IPC_QUANT_ALLOWLIST
)
raise UnsupportedQuantForIPCError(
f"[weight_cache:{where}] quantization {quantization!r} is not verified "
f"for CUDA IPC weight sharing: its post-load processing may repack "
f"weights into shapes the client cannot reproduce or stamp Python-side "
f"state that tensor export cannot carry, which would silently serve "
f"wrong numerics. Verified: {verified} (FP8 only with weight_block_size "
f"set, i.e. block-wise). Use the default --load-format for this model."
)
def get_physical_device_id(device_index: int) -> int | None:
"""Map a local CUDA device index to the physical GPU id.
Returns None if CUDA_VISIBLE_DEVICES contains non-integer entries
(e.g. GPU UUIDs), in which case an explicit socket path is required.
"""
visible = os.environ.get("CUDA_VISIBLE_DEVICES")
if not visible:
return device_index
entries = visible.split(",")
try:
return int(entries[device_index])
except (IndexError, ValueError):
return None
def get_socket_dir(socket_dir: str | None = None) -> str:
"""Return the directory that holds the daemon sockets.
When no explicit directory is given, use a per-user private directory
under the system temp dir so its path is unpredictable to other users and
can be locked down to mode 0700.
"""
if socket_dir is not None:
return socket_dir
return os.path.join(
tempfile.gettempdir(), SOCKET_DIR_TEMPLATE.format(uid=_current_uid())
)
def get_socket_path(gpu_id: int, socket_dir: str | None = None) -> str:
directory = get_socket_dir(socket_dir)
return os.path.join(directory, SOCKET_NAME_TEMPLATE.format(gpu_id=gpu_id))
def ensure_private_socket_dir(directory: str, strict_perms: bool = True) -> None:
"""Create the socket directory (if needed) locked down to the owner.
Called by the daemon before binding. Existing directories are re-checked
and, for the auto-derived path, tightened so a pre-existing world-writable
directory is rejected.
"""
os.makedirs(directory, mode=0o700, exist_ok=True)
if strict_perms:
os.chmod(directory, 0o700)
verify_private_dir(directory, strict_perms=strict_perms)
def verify_private_dir(directory: str, strict_perms: bool = True) -> None:
"""Verify a directory is a real dir owned by us and not world/group readable.
When ``strict_perms`` is False the group/world permission bits are not
checked; this is used for directories the operator explicitly configured
(they own the trust decision), while the auto-derived per-user directory is
always checked strictly.
"""
info = os.lstat(directory)
if stat.S_ISLNK(info.st_mode):
raise WeightCacheUnavailableError(
f"Refusing to use symlinked socket directory {directory}"
)
if not stat.S_ISDIR(info.st_mode):
raise WeightCacheUnavailableError(f"{directory} is not a directory")
uid = _current_uid()
if uid != -1 and info.st_uid != uid:
raise WeightCacheUnavailableError(
f"Socket directory {directory} is not owned by the current user"
)
if strict_perms and info.st_mode & 0o077:
raise WeightCacheUnavailableError(
f"Socket directory {directory} is group/world accessible"
)
def verify_socket_owner(socket_path: str, strict_perms: bool = True) -> None:
"""Verify the socket lives in a private dir and is owned by the current user.
Called by the engine before connecting so it never talks to a socket a
different user could have planted.
"""
verify_private_dir(os.path.dirname(socket_path), strict_perms=strict_perms)
info = os.lstat(socket_path)
if stat.S_ISLNK(info.st_mode):
raise WeightCacheUnavailableError(
f"Refusing to connect to symlinked socket {socket_path}"
)
uid = _current_uid()
if uid != -1 and info.st_uid != uid:
raise WeightCacheUnavailableError(
f"Socket {socket_path} is not owned by the current user"
)
def verify_peer_is_owner(conn: socket.socket) -> None:
"""Best-effort check that the connecting peer runs as the current user.
Uses SO_PEERCRED where available (Linux). Silently returns on platforms
that do not expose peer credentials.
"""
so_peercred = getattr(socket, "SO_PEERCRED", None)
if so_peercred is None:
return
try:
creds = conn.getsockopt(socket.SOL_SOCKET, so_peercred, struct.calcsize("3i"))
_, peer_uid, _ = struct.unpack("3i", creds)
except OSError:
return
uid = _current_uid()
if uid != -1 and peer_uid != uid:
raise PermissionError(f"Rejecting weight cache connection from uid {peer_uid}")
def _hash_quant_config(quant_config: Any) -> str:
if quant_config is None:
return ""
if hasattr(quant_config, "to_dict"):
quant_config = quant_config.to_dict()
payload = json.dumps(quant_config, sort_keys=True, default=str)
return safe_hash(payload.encode(), usedforsecurity=False).hexdigest()
def _safetensors_header(path: str) -> bytes:
"""Return the raw safetensors header (length prefix + JSON) of a file.
The header carries tensor names, dtypes, shapes and byte offsets, so it is
a content fingerprint of the shard without reading any weight bytes.
"""
with open(path, "rb") as f:
size_bytes = f.read(8)
(header_len,) = struct.unpack("<Q", size_bytes)
return size_bytes + f.read(header_len)
def hash_checkpoint(model: str) -> str | None:
"""Fingerprint checkpoint content from local safetensors metadata.
Hashes each shard's safetensors header so a daemon and an engine pointing
at identical weights in different directories produce the same key. Returns
None when local safetensors files can't be located (e.g. an undownloaded
Hugging Face repo id), leaving the caller to fall back to the model path.
"""
if not os.path.isdir(model):
return None
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
from vllm.model_executor.model_loader.weight_utils import (
filter_duplicate_safetensors_files,
)
files = glob.glob(os.path.join(model, "*.safetensors"))
if os.path.isfile(os.path.join(model, SAFE_WEIGHTS_INDEX_NAME)):
files = filter_duplicate_safetensors_files(
files, model, SAFE_WEIGHTS_INDEX_NAME
)
if not files:
return None
hasher = safe_hash(b"", usedforsecurity=False)
for path in sorted(files, key=os.path.basename):
hasher.update(os.path.basename(path).encode())
hasher.update(_safetensors_header(path))
return hasher.hexdigest()
@dataclass(frozen=True)
class WeightCacheKey:
"""Fingerprint of the cached weights.
Any mismatch between the daemon's and the engine's fingerprint means the
cached weights cannot be reused and the engine must load from disk.
"""
checkpoint: str
model_arch: str
tp_size: int
tp_rank: int
dtype: str
quantization: str | None
quant_config_hash: str
revision: str | None
vllm_version: str
@classmethod
def from_model_config(
cls, model_config: ModelConfig, tp_size: int, tp_rank: int
) -> "WeightCacheKey":
"""Build the fingerprint for a model configuration.
Must be called before weight loading: process_weights_after_loading
may mutate hf_config.quantization_config, which would change the hash
between the daemon and the engine.
The checkpoint is identified by a hash of its safetensors metadata when
the weights are available locally, so a daemon and engine referencing
identical weights in different directories still match; otherwise it
falls back to the model path.
"""
hf_config = model_config.hf_config
arch = ",".join(getattr(hf_config, "architectures", None) or [])
quant_config = getattr(hf_config, "quantization_config", None)
checkpoint = hash_checkpoint(model_config.model) or model_config.model
return cls(
checkpoint=checkpoint,
model_arch=arch,
tp_size=tp_size,
tp_rank=tp_rank,
dtype=str(model_config.dtype),
quantization=model_config.quantization,
quant_config_hash=_hash_quant_config(quant_config),
revision=model_config.revision,
vllm_version=vllm.version.__version__,
)
def mismatched_fields(self, other: "WeightCacheKey") -> list[str]:
return [
f.name
for f in fields(self)
if getattr(self, f.name) != getattr(other, f.name)
]
@dataclass
class TensorEntry:
"""A single cached tensor.
CUDA tensors are exported as `torch.multiprocessing` reduction args
(CUDA IPC handles); non-CUDA tensors are shipped by value.
"""
kind: str
"""Either "param" or "buffer"."""
ipc_args: tuple | None = None
cpu_tensor: torch.Tensor | None = None
@classmethod
def from_tensor(cls, tensor: torch.Tensor, kind: str) -> "TensorEntry":
from torch.multiprocessing.reductions import reduce_tensor
tensor = tensor.detach()
if tensor.is_cuda:
_, ipc_args = reduce_tensor(tensor)
return cls(kind=kind, ipc_args=ipc_args)
return cls(kind=kind, cpu_tensor=tensor.cpu())
def rebuild(self, device_index: int) -> torch.Tensor:
if self.ipc_args is None:
assert self.cpu_tensor is not None
return self.cpu_tensor
from torch.multiprocessing.reductions import rebuild_cuda_tensor
args = list(self.ipc_args)
# Index 6 of the args from reduce_tensor is the device index. It must
# be retargeted to the local index since the daemon and the engine may
# have different CUDA_VISIBLE_DEVICES mappings.
args[6] = device_index
return rebuild_cuda_tensor(*args)
def send_msg(sock: socket.socket, obj: Any) -> None:
payload = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL)
sock.sendall(_LEN_STRUCT.pack(len(payload)))
sock.sendall(payload)
def recv_msg(sock: socket.socket) -> Any:
(length,) = _LEN_STRUCT.unpack(_recv_exact(sock, _LEN_STRUCT.size))
if length > MAX_MSG_SIZE:
raise ValueError(f"Message size {length} exceeds limit {MAX_MSG_SIZE}")
return pickle.loads(_recv_exact(sock, length))
def _recv_exact(sock: socket.socket, num_bytes: int) -> bytes:
buf = bytearray()
while len(buf) < num_bytes:
chunk = sock.recv(num_bytes - len(buf))
if not chunk:
raise ConnectionError("Socket closed while receiving message")
buf.extend(chunk)
return bytes(buf)
+23
View File
@@ -3,12 +3,35 @@
"""Utils for model executor."""
import copy
from contextlib import contextmanager
from contextvars import ContextVar
from typing import Any
import torch
from vllm.utils.torch_utils import is_torch_equal_or_newer
_weights_pre_processed: ContextVar[bool] = ContextVar(
"weights_pre_processed", default=False
)
@contextmanager
def weights_already_processed():
"""Mark that weights are already in post-processed (runtime) format, so
``process_weights_after_loading`` skips tensor transforms (used by the
weight cache IPC loader).
"""
token = _weights_pre_processed.set(True)
try:
yield
finally:
_weights_pre_processed.reset(token)
def is_weights_pre_processed() -> bool:
return _weights_pre_processed.get()
def set_weight_attrs(
weight: torch.Tensor,