Fast Start (#54921)
Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: liusy58 <liusy58@smail.nju.edu.cn> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -0,0 +1,153 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end tests for the IPC weight cache loader.
|
||||
|
||||
A cold start (no weight cache daemon, so the loader falls back to disk) and
|
||||
warm restarts (weights mapped from the daemon via CUDA IPC) must both serve
|
||||
identical outputs.
|
||||
"""
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
|
||||
from vllm import SamplingParams
|
||||
|
||||
MODEL = "Qwen/Qwen3.5-0.8B"
|
||||
PROMPTS = [
|
||||
"Hello, my name is",
|
||||
"The capital of France is",
|
||||
]
|
||||
DAEMON_TIMEOUT_S = 600
|
||||
|
||||
|
||||
class WeightCacheDaemon:
|
||||
"""Context manager running the real weight cache daemon as a subprocess."""
|
||||
|
||||
_READY_MARKER = "Weight cache daemon READY"
|
||||
|
||||
def __init__(self, model: str, tp_size: int):
|
||||
# Short base path: Unix socket paths are limited to ~107 characters.
|
||||
self.socket_dir = tempfile.mkdtemp(prefix="vllm_ipc_")
|
||||
self._cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"vllm.model_executor.model_loader.weight_cache.daemon",
|
||||
"--model",
|
||||
model,
|
||||
"--tensor-parallel-size",
|
||||
str(tp_size),
|
||||
"--weight-cache-socket-dir",
|
||||
self.socket_dir,
|
||||
"--enforce-eager",
|
||||
]
|
||||
self._proc: subprocess.Popen | None = None
|
||||
self._lines: list[str] = []
|
||||
|
||||
def __enter__(self) -> "WeightCacheDaemon":
|
||||
self._proc = subprocess.Popen(
|
||||
self._cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True
|
||||
)
|
||||
threading.Thread(target=self._drain_stderr, daemon=True).start()
|
||||
try:
|
||||
self._wait_ready(DAEMON_TIMEOUT_S)
|
||||
except Exception:
|
||||
self._stop()
|
||||
raise
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc_info) -> None:
|
||||
self._stop()
|
||||
|
||||
def _drain_stderr(self) -> None:
|
||||
assert self._proc is not None and self._proc.stderr is not None
|
||||
for line in self._proc.stderr:
|
||||
self._lines.append(line)
|
||||
|
||||
def _logs(self) -> str:
|
||||
return "".join(self._lines)
|
||||
|
||||
def _wait_ready(self, timeout_s: float) -> None:
|
||||
assert self._proc is not None
|
||||
deadline = time.monotonic() + timeout_s
|
||||
while time.monotonic() < deadline:
|
||||
if self._proc.poll() is not None:
|
||||
raise RuntimeError(
|
||||
f"Weight cache daemon exited with {self._proc.returncode}:\n"
|
||||
f"{self._logs()}"
|
||||
)
|
||||
if self._READY_MARKER in self._logs():
|
||||
return
|
||||
time.sleep(1.0)
|
||||
raise TimeoutError(
|
||||
f"Weight cache daemon not ready after {timeout_s}s:\n{self._logs()}"
|
||||
)
|
||||
|
||||
def _stop(self) -> None:
|
||||
assert self._proc is not None
|
||||
self._proc.terminate()
|
||||
try:
|
||||
self._proc.wait(timeout=60)
|
||||
except subprocess.TimeoutExpired:
|
||||
self._proc.kill()
|
||||
self._proc.wait()
|
||||
shutil.rmtree(self.socket_dir, ignore_errors=True)
|
||||
|
||||
|
||||
def generate(vllm_runner, socket_dir: str | None, fallback: bool, **kwargs):
|
||||
extra_config = (
|
||||
{} if socket_dir is None else {"socket_dir": socket_dir, "fallback": fallback}
|
||||
)
|
||||
with vllm_runner(
|
||||
MODEL,
|
||||
load_format="auto" if socket_dir is None else "ipc_cache",
|
||||
model_loader_extra_config=extra_config,
|
||||
# Greedy outputs are compared across engine restarts; cap the batch
|
||||
# size so scheduling cannot change the results.
|
||||
max_num_seqs=1,
|
||||
**kwargs,
|
||||
) as llm:
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=16, ignore_eos=True)
|
||||
return llm.generate(PROMPTS, sampling_params)
|
||||
|
||||
|
||||
def test_ipc_cache_cold_start_and_warm_restart(vllm_runner):
|
||||
"""Cold start falls back to disk; warm restarts load weights via CUDA IPC.
|
||||
|
||||
All runs must produce outputs identical to a default-loader baseline. The
|
||||
warm runs disable the disk fallback, so they only pass if the weights
|
||||
really came from the daemon.
|
||||
"""
|
||||
llm_kwargs = dict(
|
||||
gpu_memory_utilization=0.3,
|
||||
enforce_eager=True,
|
||||
# Qwen3.5 is a hybrid mamba model; its mamba cache mode requires
|
||||
# chunked prefill.
|
||||
enable_chunked_prefill=True,
|
||||
)
|
||||
|
||||
# Baseline: plain disk loading with the default loader.
|
||||
baseline_outputs = generate(vllm_runner, None, fallback=True, **llm_kwargs)
|
||||
assert all(text for _, texts in baseline_outputs for text in texts)
|
||||
|
||||
# Cold start: no daemon is serving, so the loader falls back to disk.
|
||||
with tempfile.TemporaryDirectory(prefix="vllm_ipc_empty_") as empty_socket_dir:
|
||||
cold_outputs = generate(
|
||||
vllm_runner, empty_socket_dir, fallback=True, **llm_kwargs
|
||||
)
|
||||
|
||||
with WeightCacheDaemon(MODEL, tp_size=1) as daemon:
|
||||
warm_outputs = generate(
|
||||
vllm_runner, daemon.socket_dir, fallback=False, **llm_kwargs
|
||||
)
|
||||
# Warm restart: a second engine lifetime against the same daemon.
|
||||
restart_outputs = generate(
|
||||
vllm_runner, daemon.socket_dir, fallback=False, **llm_kwargs
|
||||
)
|
||||
|
||||
assert cold_outputs == baseline_outputs
|
||||
assert warm_outputs == baseline_outputs
|
||||
assert restart_outputs == baseline_outputs
|
||||
@@ -48,6 +48,7 @@ CHECK_IMPORTS = {
|
||||
"vllm/distributed/device_communicators/shm_object_storage.py",
|
||||
"vllm/distributed/weight_transfer/ipc_engine.py",
|
||||
"vllm/distributed/weight_transfer/clients.py",
|
||||
"vllm/model_executor/model_loader/weight_cache/protocol.py",
|
||||
"tests/distributed/test_shm_broadcast.py",
|
||||
"tests/distributed/test_weight_transfer.py",
|
||||
"vllm/utils/hashing.py",
|
||||
|
||||
@@ -38,6 +38,10 @@ class LoadConfig:
|
||||
- "instanttensor" will load the Safetensors weights on CUDA devices using
|
||||
InstantTensor, which enables distributed loading with pipelined prefetching
|
||||
and fast direct I/O.
|
||||
- "ipc_cache" will map post-quantized weights from a local weight cache
|
||||
daemon via CUDA IPC for fast engine restarts. See
|
||||
`vllm/model_executor/model_loader/weight_cache/daemon.py` for how to
|
||||
launch the daemon.
|
||||
- "npcache" will load the weights in pytorch format and store a numpy cache
|
||||
to speed up the loading.
|
||||
- "dummy" will initialize the weights with random values, which is mainly
|
||||
|
||||
@@ -27,7 +27,11 @@ from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
|
||||
from vllm.model_executor.utils import (
|
||||
is_weights_pre_processed,
|
||||
replace_parameter,
|
||||
set_weight_attrs,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -43,6 +47,8 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
|
||||
# --8<-- [end:unquantized_fused_moe]
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
def __init__(self, moe: FusedMoEConfig):
|
||||
super().__init__(moe)
|
||||
self.unquantized_backend, self.experts_cls = select_unquantized_moe_backend(
|
||||
@@ -153,16 +159,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
# optional w{13,2}_bias references and SwiGLU gate params. Since
|
||||
# weight updates mutate those bias tensors in place, the kernel
|
||||
# does not need to be re-built.
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
assert self.moe_quant_config is not None
|
||||
assert self.experts_cls is not None
|
||||
self.moe_kernel = make_unquantized_moe_kernel(
|
||||
quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
backend=self.unquantized_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
self._init_moe_kernel(layer)
|
||||
|
||||
if self.unquantized_backend == UnquantizedMoeBackend.CPU:
|
||||
# The CPU experts need the layer itself for the setup that
|
||||
@@ -170,11 +167,30 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
# it only sees the two weight tensors: padding and prepacking
|
||||
# into the grouped-gemm layout (bias included), and capturing
|
||||
# the router config that monolithic apply() cannot carry.
|
||||
assert self.moe_kernel is not None
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def _init_moe_kernel(self, layer: "RoutedExperts") -> None:
|
||||
"""Build the MoE kernel from the layer's current (shuffled) weights."""
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
assert self.moe_quant_config is not None
|
||||
assert self.experts_cls is not None
|
||||
self.moe_kernel = make_unquantized_moe_kernel(
|
||||
quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
backend=self.unquantized_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: "RoutedExperts") -> None:
|
||||
super().process_weights_after_loading(layer)
|
||||
|
||||
if is_weights_pre_processed():
|
||||
# Weights are already in runtime format; rebuild the kernel only.
|
||||
self._init_moe_kernel(layer)
|
||||
return
|
||||
|
||||
# Padding may allocate a same-shaped strided view. Copy it back instead
|
||||
# of rebinding `.data`, since CUDA graphs capture the parameter storage
|
||||
# address.
|
||||
|
||||
@@ -163,6 +163,8 @@ class LinearMethodBase(QuantizeMethodBase):
|
||||
class UnquantizedLinearMethod(LinearMethodBase):
|
||||
"""Linear method without quantization."""
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
config = get_current_vllm_config_or_none()
|
||||
linear_backend = (
|
||||
|
||||
@@ -27,6 +27,13 @@ class QuantizeMethodBase(ABC):
|
||||
in process_weights_after_loading, reducing peak memory during loading.
|
||||
"""
|
||||
|
||||
supports_pre_processed_weights: bool = False
|
||||
"""
|
||||
Whether ``process_weights_after_loading`` supports running under
|
||||
``weights_already_processed``. Methods must skip tensor transforms in
|
||||
that mode; the loader driver rejects methods that do not declare support.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def create_weights(
|
||||
self, layer: torch.nn.Module, *weight_args, **extra_weight_attrs
|
||||
|
||||
@@ -74,7 +74,11 @@ from vllm.model_executor.parameter import (
|
||||
BlockQuantScaleParameter,
|
||||
PerTensorScaleParameter,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
|
||||
from vllm.model_executor.utils import (
|
||||
is_weights_pre_processed,
|
||||
replace_parameter,
|
||||
set_weight_attrs,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import (
|
||||
is_deep_gemm_supported,
|
||||
@@ -248,6 +252,8 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
quant_config: The quantization config.
|
||||
"""
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
def __init__(self, quant_config: Fp8Config):
|
||||
self.quant_config = quant_config
|
||||
self.is_scale_e8m0 = getattr(quant_config, "is_scale_e8m0", False)
|
||||
@@ -366,6 +372,19 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
self.use_marlin = isinstance(self.fp8_linear, MarlinFP8ScaledMMLinearKernel)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
if is_weights_pre_processed():
|
||||
# Weights are already in runtime format; the only state tensor
|
||||
# export cannot carry is the `input_scale = None` stamp that dynamic
|
||||
# activation quantization relies on, since create_weights never
|
||||
# registered that parameter.
|
||||
if not self.act_q_static and not hasattr(layer, "input_scale"):
|
||||
layer.input_scale = None
|
||||
# Marlin reads marlin_input_dtype in apply_weights; it lives on the
|
||||
# method (not exported with the weights), so restore it here too.
|
||||
if self.use_marlin and hasattr(self.fp8_linear, "marlin_input_dtype"):
|
||||
self.fp8_linear.marlin_input_dtype = self.marlin_input_dtype
|
||||
return
|
||||
|
||||
if self.use_marlin:
|
||||
if not self.block_quant:
|
||||
# Canonicalize to (K, N) for the kernel.
|
||||
@@ -472,6 +491,8 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
quant_config: The quantization config.
|
||||
"""
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
def __init__(self, quant_config: Fp8Config, layer: RoutedExperts):
|
||||
super().__init__(layer.moe_config)
|
||||
self.quant_config = quant_config
|
||||
@@ -713,6 +734,10 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
replace_parameter(layer, f"w13_{self.weight_scale_name}", w13_scale)
|
||||
replace_parameter(layer, f"w2_{self.weight_scale_name}", w2_scale)
|
||||
|
||||
self._init_moe_kernel(layer)
|
||||
|
||||
def _init_moe_kernel(self, layer: RoutedExperts) -> None:
|
||||
"""Build the MoE kernel from the layer's current (converted) weights."""
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
assert self.moe_quant_config is not None
|
||||
assert self.experts_cls is not None
|
||||
@@ -725,6 +750,11 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
if is_weights_pre_processed():
|
||||
# Weights are already in kernel format; rebuild the kernel only.
|
||||
self._init_moe_kernel(layer)
|
||||
return
|
||||
|
||||
# Allow for accessing weights and scales in standard way.
|
||||
w13 = layer.w13_weight
|
||||
w2 = layer.w2_weight
|
||||
|
||||
@@ -8,6 +8,7 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from vllm.model_executor.utils import is_weights_pre_processed
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
from vllm.v1.kv_cache_interface import kv_cache_uses_per_token_head_scales
|
||||
@@ -51,6 +52,8 @@ class BaseKVCacheMethod(QuantizeMethodBase):
|
||||
quant_config: the appropriate QuantizationConfig
|
||||
"""
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
def __init__(self, quant_config: QuantizationConfig):
|
||||
self.quant_config = quant_config
|
||||
|
||||
@@ -72,6 +75,20 @@ class BaseKVCacheMethod(QuantizeMethodBase):
|
||||
raise RuntimeError(f"{self.__class__.__name__}.apply should not be called.")
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
if is_weights_pre_processed():
|
||||
# Scale buffers were exported with post-processed values; drop the
|
||||
# consumed scale placeholders and rebuild the host-side copies.
|
||||
for name in ("k_scale", "v_scale", "q_scale", "prob_scale"):
|
||||
if hasattr(layer, name):
|
||||
delattr(layer, name)
|
||||
if hasattr(layer, "_k_scale"):
|
||||
layer._k_scale_float = layer._k_scale.item()
|
||||
layer._v_scale_float = layer._v_scale.item()
|
||||
layer._q_scale_float = layer._q_scale.item()
|
||||
layer._k_scale_cpu = layer._k_scale.detach().to("cpu")
|
||||
layer._v_scale_cpu = layer._v_scale.detach().to("cpu")
|
||||
return
|
||||
|
||||
# skip if there are no weights to process (for example, weight reloading)
|
||||
if not hasattr(layer, "q_scale"):
|
||||
assert not hasattr(layer, "k_scale")
|
||||
|
||||
@@ -36,6 +36,8 @@ DEFAULT_VOCAB_PADDING_SIZE = 64
|
||||
class UnquantizedEmbeddingMethod(QuantizeMethodBase):
|
||||
"""Unquantized method for embeddings."""
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm.model_executor.model_loader.utils import (
|
||||
get_model_architecture,
|
||||
get_model_cls,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_cache.ipc_loader import IpcModelLoader
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -35,6 +36,7 @@ LoadFormats = Literal[
|
||||
"dummy",
|
||||
"fastsafetensors",
|
||||
"instanttensor",
|
||||
"ipc_cache",
|
||||
"mistral",
|
||||
"modelexpress",
|
||||
"npcache",
|
||||
@@ -51,6 +53,7 @@ _LOAD_FORMAT_TO_MODEL_LOADER: dict[str, type[BaseModelLoader]] = {
|
||||
"dummy": DummyModelLoader,
|
||||
"fastsafetensors": DefaultModelLoader,
|
||||
"instanttensor": DefaultModelLoader,
|
||||
"ipc_cache": IpcModelLoader,
|
||||
"mistral": DefaultModelLoader,
|
||||
"modelexpress": ModelExpressModelLoader,
|
||||
"npcache": DefaultModelLoader,
|
||||
@@ -150,6 +153,7 @@ __all__ = [
|
||||
"ModelExpressModelLoader",
|
||||
"DefaultModelLoader",
|
||||
"DummyModelLoader",
|
||||
"IpcModelLoader",
|
||||
"RunaiModelStreamerLoader",
|
||||
"ShardedStateLoader",
|
||||
"TensorizerLoader",
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.model_executor.model_loader.reload import (
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_tying import maybe_retie_word_embeddings
|
||||
from vllm.model_executor.models.interfaces import SupportsQuant
|
||||
from vllm.model_executor.utils import is_weights_pre_processed
|
||||
from vllm.tracing import instrument
|
||||
from vllm.utils.mem_utils import release_device_memory_under_pressure
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
@@ -97,13 +98,27 @@ def initialize_model(
|
||||
def process_weights_after_loading(
|
||||
model: nn.Module, model_config: ModelConfig, target_device: torch.device
|
||||
) -> None:
|
||||
"""Post-process loaded weights into runtime format.
|
||||
|
||||
Under ``weights_already_processed`` (weight cache IPC loader), quant
|
||||
methods skip tensor transforms and must declare
|
||||
``supports_pre_processed_weights``, otherwise this raises ``RuntimeError``.
|
||||
"""
|
||||
# Reclaim memory when an explicit lm_head has been
|
||||
# loaded, but it is identical to the input embeddings.
|
||||
maybe_retie_word_embeddings(model, model_config)
|
||||
|
||||
for _, module in model.named_modules():
|
||||
for name, module in model.named_modules():
|
||||
quant_method = getattr(module, "quant_method", None)
|
||||
if isinstance(quant_method, QuantizeMethodBase):
|
||||
if (
|
||||
is_weights_pre_processed()
|
||||
and not quant_method.supports_pre_processed_weights
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"layer {name or '<root>'}: {type(quant_method).__name__} "
|
||||
"does not support pre-processed weights"
|
||||
)
|
||||
# When quant methods need to process weights after loading
|
||||
# (for repacking, quantizing, etc), they expect parameters
|
||||
# to be on the global target device. This scope is for the
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.model_executor.model_loader.weight_cache.ipc_loader import IpcModelLoader
|
||||
from vllm.model_executor.model_loader.weight_cache.protocol import (
|
||||
CacheConfigMismatchError,
|
||||
TensorEntry,
|
||||
UnsupportedQuantForIPCError,
|
||||
WeightCacheKey,
|
||||
WeightCacheUnavailableError,
|
||||
check_ipc_quant_support,
|
||||
is_ipc_quant_supported,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"CacheConfigMismatchError",
|
||||
"IpcModelLoader",
|
||||
"TensorEntry",
|
||||
"UnsupportedQuantForIPCError",
|
||||
"WeightCacheKey",
|
||||
"WeightCacheUnavailableError",
|
||||
"check_ipc_quant_support",
|
||||
"is_ipc_quant_supported",
|
||||
]
|
||||
@@ -0,0 +1,412 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Weight cache daemon for fast engine restarts.
|
||||
|
||||
One daemon process per GPU holds the post-quantized, TP-sharded weights of its
|
||||
rank in GPU memory and serves CUDA IPC handles to vLLM engines over a Unix
|
||||
domain socket. Restarting engines map the weights via zero-copy IPC instead of
|
||||
reloading from disk.
|
||||
|
||||
Launch one daemon per TP rank with a single command:
|
||||
|
||||
python -m vllm.model_executor.model_loader.weight_cache.daemon \\
|
||||
--model /path/to/model --tensor-parallel-size 4
|
||||
|
||||
Engines then load from the daemons with:
|
||||
|
||||
vllm serve /path/to/model --tensor-parallel-size 4 \\
|
||||
--load-format ipc_cache
|
||||
|
||||
Only tensor parallelism is supported; pipeline, data, and expert parallelism
|
||||
are rejected at launch.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import fcntl
|
||||
import multiprocessing
|
||||
import os
|
||||
import queue
|
||||
import signal
|
||||
import socket
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
ensure_model_parallel_initialized,
|
||||
init_distributed_environment,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader.weight_cache.protocol import (
|
||||
TensorEntry,
|
||||
WeightCacheKey,
|
||||
WeightCacheUnavailableError,
|
||||
check_ipc_quant_support,
|
||||
ensure_private_socket_dir,
|
||||
get_physical_device_id,
|
||||
get_socket_path,
|
||||
recv_msg,
|
||||
send_msg,
|
||||
verify_peer_is_owner,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _report_ready(message: str) -> None:
|
||||
"""Write a readiness message straight to the original stderr descriptor.
|
||||
|
||||
Loading the model pulls in FlashInfer's CuTeDSL JIT compiler, which swaps
|
||||
``sys.stdout``/``sys.stderr`` for in-memory buffers while compiling kernels
|
||||
across worker threads. That save/restore races on the interpreter-global
|
||||
streams and can leave them (and vLLM's logging handler, which caches the
|
||||
original stream object) detached, silently swallowing everything logged
|
||||
afterwards -- including the daemon's readiness announcement. Since operators
|
||||
rely on that line to know the daemon is serving, write it directly to file
|
||||
descriptor 2 so it bypasses the logging machinery entirely.
|
||||
"""
|
||||
with contextlib.suppress(OSError):
|
||||
os.write(2, (message + "\n").encode())
|
||||
|
||||
|
||||
def export_entries(
|
||||
model: torch.nn.Module,
|
||||
) -> tuple[dict[str, TensorEntry], dict[str, str]]:
|
||||
"""Export a model's tensors, preserving tied-parameter aliases.
|
||||
|
||||
``named_parameters``/``named_buffers`` are iterated with
|
||||
``remove_duplicate=False`` so tied weights (e.g. ``lm_head.weight`` sharing
|
||||
storage with ``embed_tokens.weight``) are not silently dropped. Each unique
|
||||
tensor is exported once; every additional name that refers to the same
|
||||
tensor object is recorded in the returned alias map so the client can
|
||||
re-establish the shared identity instead of allocating uninitialized
|
||||
memory for it.
|
||||
|
||||
Returns:
|
||||
A ``(entries, aliases)`` pair where ``entries`` maps a canonical name to
|
||||
its ``TensorEntry`` and ``aliases`` maps each duplicate name to its
|
||||
canonical name.
|
||||
"""
|
||||
entries: dict[str, TensorEntry] = {}
|
||||
aliases: dict[str, str] = {}
|
||||
canonical_by_id: dict[int, str] = {}
|
||||
|
||||
def _add(name: str, tensor: torch.Tensor, kind: str) -> None:
|
||||
canonical = canonical_by_id.get(id(tensor))
|
||||
if canonical is not None:
|
||||
aliases[name] = canonical
|
||||
return
|
||||
canonical_by_id[id(tensor)] = name
|
||||
entries[name] = TensorEntry.from_tensor(tensor, kind)
|
||||
|
||||
for name, param in model.named_parameters(remove_duplicate=False):
|
||||
_add(name, param, "param")
|
||||
# named_buffers includes non-persistent buffers (e.g. rotary embedding
|
||||
# caches) that state_dict would miss.
|
||||
for name, buffer in model.named_buffers(remove_duplicate=False):
|
||||
if name in entries or name in aliases:
|
||||
continue
|
||||
_add(name, buffer, "buffer")
|
||||
return entries, aliases
|
||||
|
||||
|
||||
class WeightCacheDaemon:
|
||||
"""Per-GPU process that loads one TP shard and serves CUDA IPC handles."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
tp_rank: int,
|
||||
distributed_init_method: str,
|
||||
socket_dir: str | None = None,
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
self.tp_rank = tp_rank
|
||||
self.distributed_init_method = distributed_init_method
|
||||
self.socket_dir = socket_dir
|
||||
self.model: torch.nn.Module | None = None
|
||||
self.entries: dict[str, TensorEntry] = {}
|
||||
self.aliases: dict[str, str] = {}
|
||||
# Fingerprint before loading: process_weights_after_loading may
|
||||
# mutate hf_config.quantization_config.
|
||||
self.cache_config = WeightCacheKey.from_model_config(
|
||||
vllm_config.model_config,
|
||||
tp_size=vllm_config.parallel_config.tensor_parallel_size,
|
||||
tp_rank=tp_rank,
|
||||
)
|
||||
|
||||
def load_model(self) -> None:
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
|
||||
tp_size = self.cache_config.tp_size
|
||||
torch.accelerator.set_device_index(self.tp_rank)
|
||||
init_distributed_environment(
|
||||
world_size=tp_size,
|
||||
rank=self.tp_rank,
|
||||
distributed_init_method=self.distributed_init_method,
|
||||
local_rank=self.tp_rank,
|
||||
backend=current_platform.dist_backend,
|
||||
)
|
||||
with set_current_vllm_config(self.vllm_config):
|
||||
ensure_model_parallel_initialized(tp_size, 1)
|
||||
self.model = get_model(vllm_config=self.vllm_config)
|
||||
self._export_entries()
|
||||
logger.info(
|
||||
"Weight cache daemon rank %d cached %d tensors",
|
||||
self.tp_rank,
|
||||
len(self.entries),
|
||||
)
|
||||
|
||||
def _export_entries(self) -> None:
|
||||
assert self.model is not None
|
||||
self.entries, self.aliases = export_entries(self.model)
|
||||
|
||||
def serve_forever(self, ready_callback: Callable[[], None] | None = None) -> None:
|
||||
"""Serve requests until terminated.
|
||||
|
||||
The socket is only bound once the model is fully cached, so clients
|
||||
get a connection error (and fall back to disk) until the daemon is
|
||||
ready.
|
||||
|
||||
Args:
|
||||
ready_callback: Invoked once the socket is bound and listening,
|
||||
so the launcher can report overall readiness.
|
||||
"""
|
||||
socket_path = self._socket_path()
|
||||
ensure_private_socket_dir(
|
||||
os.path.dirname(socket_path), strict_perms=self.socket_dir is None
|
||||
)
|
||||
# Hold an exclusive per-GPU lock for the daemon's lifetime so a second
|
||||
# daemon cannot remove this daemon's live socket and hijack the path.
|
||||
lock_fd = self._acquire_gpu_lock(socket_path)
|
||||
if os.path.exists(socket_path):
|
||||
os.unlink(socket_path)
|
||||
server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
server.bind(socket_path)
|
||||
os.chmod(socket_path, 0o600)
|
||||
server.listen()
|
||||
logger.info(
|
||||
"Weight cache daemon rank %d serving on %s", self.tp_rank, socket_path
|
||||
)
|
||||
print(
|
||||
f"Weight cache daemon rank {self.tp_rank} ready: serving on {socket_path}"
|
||||
)
|
||||
if ready_callback is not None:
|
||||
ready_callback()
|
||||
try:
|
||||
while True:
|
||||
conn, _ = server.accept()
|
||||
with conn:
|
||||
try:
|
||||
verify_peer_is_owner(conn)
|
||||
self._handle_connection(conn)
|
||||
except (ConnectionError, EOFError):
|
||||
logger.warning("Client disconnected mid-request")
|
||||
except Exception:
|
||||
# A single malformed or malicious request must not take
|
||||
# down the daemon for every other engine on this GPU.
|
||||
logger.exception(
|
||||
"Error handling weight cache client; continuing"
|
||||
)
|
||||
finally:
|
||||
server.close()
|
||||
if os.path.exists(socket_path):
|
||||
os.unlink(socket_path)
|
||||
os.close(lock_fd)
|
||||
|
||||
def _acquire_gpu_lock(self, socket_path: str) -> int:
|
||||
"""Take an exclusive lock guarding this GPU's socket path.
|
||||
|
||||
The lock is advisory and released automatically when the daemon exits
|
||||
(or crashes), so a stale socket is only ever removed by whoever owns
|
||||
the lock. A running daemon holding it makes a second daemon fail fast
|
||||
instead of clobbering the live socket.
|
||||
"""
|
||||
lock_fd = os.open(f"{socket_path}.lock", os.O_CREAT | os.O_RDWR, 0o600)
|
||||
try:
|
||||
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError as e:
|
||||
os.close(lock_fd)
|
||||
raise WeightCacheUnavailableError(
|
||||
f"Another weight cache daemon already owns {socket_path}"
|
||||
) from e
|
||||
return lock_fd
|
||||
|
||||
def _socket_path(self) -> str:
|
||||
device_index = torch.accelerator.current_device_index()
|
||||
gpu_id = get_physical_device_id(device_index)
|
||||
if gpu_id is None:
|
||||
gpu_id = device_index
|
||||
return get_socket_path(gpu_id, self.socket_dir)
|
||||
|
||||
def _handle_connection(self, conn: socket.socket) -> None:
|
||||
request = recv_msg(conn)
|
||||
cmd = request.get("cmd")
|
||||
if cmd == "get_state":
|
||||
self._handle_get_state(conn, request)
|
||||
elif cmd == "release":
|
||||
self._handle_release(conn)
|
||||
else:
|
||||
send_msg(conn, {"status": "error", "message": f"Unknown command {cmd!r}"})
|
||||
|
||||
def _handle_get_state(self, conn: socket.socket, request: dict) -> None:
|
||||
client_config = request.get("cache_config")
|
||||
if not isinstance(client_config, WeightCacheKey):
|
||||
send_msg(conn, {"status": "error", "message": "Missing cache_config"})
|
||||
return
|
||||
mismatched = self.cache_config.mismatched_fields(client_config)
|
||||
if mismatched:
|
||||
logger.warning("WeightCacheKey mismatch on fields: %s", mismatched)
|
||||
send_msg(conn, {"status": "mismatch", "fields": mismatched})
|
||||
return
|
||||
if not self.entries:
|
||||
send_msg(conn, {"status": "error", "message": "Weights were released"})
|
||||
return
|
||||
send_msg(
|
||||
conn,
|
||||
{
|
||||
"status": "ok",
|
||||
"entries": self.entries,
|
||||
"aliases": self.aliases,
|
||||
"gpu_uuid": self._gpu_uuid(),
|
||||
},
|
||||
)
|
||||
|
||||
def _handle_release(self, conn: socket.socket) -> None:
|
||||
self.entries.clear()
|
||||
self.aliases.clear()
|
||||
self.model = None
|
||||
torch.accelerator.empty_cache()
|
||||
logger.info("Weight cache daemon rank %d released cached weights", self.tp_rank)
|
||||
send_msg(conn, {"status": "ok"})
|
||||
|
||||
def _gpu_uuid(self) -> str:
|
||||
props = torch.cuda.get_device_properties(
|
||||
torch.accelerator.current_device_index()
|
||||
)
|
||||
return str(props.uuid)
|
||||
|
||||
|
||||
def _run_daemon(
|
||||
tp_rank: int,
|
||||
vllm_config: VllmConfig,
|
||||
distributed_init_method: str,
|
||||
socket_dir: str | None,
|
||||
ready_queue: "multiprocessing.Queue[int]",
|
||||
) -> None:
|
||||
daemon = WeightCacheDaemon(
|
||||
vllm_config, tp_rank, distributed_init_method, socket_dir
|
||||
)
|
||||
daemon.load_model()
|
||||
daemon.serve_forever(ready_callback=lambda: ready_queue.put(tp_rank))
|
||||
|
||||
|
||||
def _reject_unsupported_parallelism(parallel_config: ParallelConfig) -> None:
|
||||
"""Reject every parallelism mode other than tensor parallelism."""
|
||||
unsupported = {
|
||||
"pipeline parallelism": parallel_config.pipeline_parallel_size > 1,
|
||||
"data parallelism": parallel_config.data_parallel_size > 1,
|
||||
"expert parallelism": parallel_config.enable_expert_parallel,
|
||||
}
|
||||
for name, enabled in unsupported.items():
|
||||
if enabled:
|
||||
raise ValueError(
|
||||
f"The weight cache daemon only supports tensor parallelism; "
|
||||
f"{name} is not supported"
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.utils.network_utils import get_distributed_init_method, get_open_port
|
||||
|
||||
parser = FlexibleArgumentParser(
|
||||
description="Launch weight cache daemons (one per TP rank)."
|
||||
)
|
||||
EngineArgs.add_cli_args(parser)
|
||||
parser.add_argument(
|
||||
"--weight-cache-socket-dir",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Directory for the daemon Unix sockets (default: tempdir).",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
engine_args = EngineArgs.from_cli_args(args)
|
||||
vllm_config = engine_args.create_engine_config()
|
||||
if vllm_config.load_config.load_format == "ipc_cache":
|
||||
raise ValueError(
|
||||
"The weight cache daemon itself must load from disk; use the "
|
||||
"default --load-format"
|
||||
)
|
||||
# Checked before loading anything: an unsupported quantization method would
|
||||
# otherwise only surface in the engine, after a full load.
|
||||
check_ipc_quant_support(vllm_config.model_config, where="daemon")
|
||||
parallel_config = vllm_config.parallel_config
|
||||
_reject_unsupported_parallelism(parallel_config)
|
||||
tp_size = parallel_config.tensor_parallel_size
|
||||
|
||||
distributed_init_method = get_distributed_init_method("127.0.0.1", get_open_port())
|
||||
ctx = multiprocessing.get_context("spawn")
|
||||
ready_queue: multiprocessing.Queue[int] = ctx.Queue()
|
||||
procs = [
|
||||
ctx.Process(
|
||||
target=_run_daemon,
|
||||
args=(
|
||||
rank,
|
||||
vllm_config,
|
||||
distributed_init_method,
|
||||
args.weight_cache_socket_dir,
|
||||
ready_queue,
|
||||
),
|
||||
name=f"vllm-weight-cache-daemon-{rank}",
|
||||
)
|
||||
for rank in range(tp_size)
|
||||
]
|
||||
for proc in procs:
|
||||
proc.start()
|
||||
|
||||
def _shutdown(signum, frame):
|
||||
for proc in procs:
|
||||
proc.terminate()
|
||||
|
||||
signal.signal(signal.SIGINT, _shutdown)
|
||||
signal.signal(signal.SIGTERM, _shutdown)
|
||||
|
||||
ready_ranks: set[int] = set()
|
||||
while len(ready_ranks) < tp_size:
|
||||
try:
|
||||
ready_ranks.add(ready_queue.get(timeout=1.0))
|
||||
except queue.Empty:
|
||||
dead = [p for p in procs if p.exitcode is not None]
|
||||
if dead:
|
||||
logger.error(
|
||||
"Weight cache daemon rank(s) exited during startup "
|
||||
"(exitcodes=%s); shutting down.",
|
||||
[p.exitcode for p in dead],
|
||||
)
|
||||
for proc in procs:
|
||||
proc.terminate()
|
||||
for proc in procs:
|
||||
proc.join()
|
||||
sys.exit(max((p.exitcode or 0) for p in procs))
|
||||
logger.info_once(
|
||||
"===== Weight cache daemon READY: all %d ranks serving in %s =====",
|
||||
tp_size,
|
||||
args.weight_cache_socket_dir or "the default socket dir",
|
||||
)
|
||||
_report_ready(
|
||||
f"===== Weight cache daemon READY: all {tp_size} rank(s) serving in "
|
||||
f"{args.weight_cache_socket_dir or 'the default socket dir'} ====="
|
||||
)
|
||||
|
||||
for proc in procs:
|
||||
proc.join()
|
||||
sys.exit(max(proc.exitcode or 0 for proc in procs))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,394 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""IPC model loader: maps post-quantized weights from a local weight cache
|
||||
daemon via CUDA IPC instead of loading from disk."""
|
||||
|
||||
import dataclasses
|
||||
import socket
|
||||
from copy import copy
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader.base_loader import BaseModelLoader
|
||||
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
|
||||
from vllm.model_executor.model_loader.utils import (
|
||||
initialize_model,
|
||||
process_weights_after_loading,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_cache.protocol import (
|
||||
CacheConfigMismatchError,
|
||||
TensorEntry,
|
||||
UnsupportedQuantForIPCError,
|
||||
WeightCacheKey,
|
||||
WeightCacheUnavailableError,
|
||||
check_ipc_quant_support,
|
||||
get_physical_device_id,
|
||||
get_socket_path,
|
||||
recv_msg,
|
||||
send_msg,
|
||||
verify_socket_owner,
|
||||
)
|
||||
from vllm.model_executor.utils import weights_already_processed
|
||||
from vllm.tracing import instrument
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_CONNECT_TIMEOUT_S = 5.0
|
||||
_STATE_TIMEOUT_S = 300.0
|
||||
|
||||
|
||||
class IpcModelLoader(BaseModelLoader):
|
||||
"""Loads a model by mapping the weight cache daemon's tensors via CUDA IPC.
|
||||
|
||||
The model is initialized on the meta device and every parameter/buffer is
|
||||
replaced by the daemon's post-quantized tensor, so
|
||||
process_weights_after_loading is skipped entirely. In "zero_copy" mode the
|
||||
engine shares the daemon's GPU memory; in "copy" mode the tensors are
|
||||
cloned into engine-owned memory and the daemon is asked to release its
|
||||
cache afterwards.
|
||||
|
||||
Extra config keys (via --model-loader-extra-config):
|
||||
|
||||
- socket_path: explicit daemon socket path. Defaults to a per-GPU path
|
||||
derived from the physical GPU id.
|
||||
- socket_dir: directory containing the daemon sockets.
|
||||
- mode: "zero_copy" (default) or "copy".
|
||||
- fallback: fall back to disk loading when the daemon is unavailable or
|
||||
the fingerprints mismatch (default: True).
|
||||
- connect_timeout_s: socket connect timeout (default: 5.0).
|
||||
- state_timeout_s: timeout for the weight-transfer request (default: 300.0).
|
||||
|
||||
Note: in zero-copy mode the weights live in the daemon's CUDA IPC
|
||||
allocations, so sleep mode (CuMemAllocator weight offloading) must not be
|
||||
used with this loader.
|
||||
"""
|
||||
|
||||
def __init__(self, load_config: LoadConfig):
|
||||
super().__init__(load_config)
|
||||
extra_config = copy(load_config.model_loader_extra_config or {})
|
||||
self.socket_path: str | None = extra_config.pop("socket_path", None)
|
||||
self.socket_dir: str | None = extra_config.pop("socket_dir", None)
|
||||
self.mode: str = extra_config.pop("mode", "zero_copy")
|
||||
self.fallback: bool = extra_config.pop("fallback", True)
|
||||
self.connect_timeout_s: float = float(
|
||||
extra_config.pop("connect_timeout_s", _CONNECT_TIMEOUT_S)
|
||||
)
|
||||
self.state_timeout_s: float = float(
|
||||
extra_config.pop("state_timeout_s", _STATE_TIMEOUT_S)
|
||||
)
|
||||
if self.mode not in ("zero_copy", "copy"):
|
||||
raise ValueError(
|
||||
f"Invalid weight cache mode {self.mode!r}, "
|
||||
"expected 'zero_copy' or 'copy'"
|
||||
)
|
||||
if extra_config:
|
||||
raise ValueError(
|
||||
f"Unexpected extra config keys for load format "
|
||||
f"{load_config.load_format}: {sorted(extra_config)}"
|
||||
)
|
||||
|
||||
def download_model(self, model_config: ModelConfig) -> None:
|
||||
DefaultModelLoader(self._fallback_load_config()).download_model(model_config)
|
||||
|
||||
def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:
|
||||
"""Best-effort in-place reload for an already-initialized model.
|
||||
|
||||
Copies daemon tensors into matching parameters/buffers. The model is
|
||||
expected to already be in the post-quantized layout (e.g. previously
|
||||
loaded through this loader).
|
||||
"""
|
||||
device_index = torch.accelerator.current_device_index()
|
||||
entries, _ = self._fetch_entries(model_config)
|
||||
params = dict(model.named_parameters())
|
||||
buffers = dict(model.named_buffers())
|
||||
for name, entry in entries.items():
|
||||
target = params.get(name, buffers.get(name))
|
||||
source = entry.rebuild(device_index)
|
||||
if target is None or target.shape != source.shape:
|
||||
logger.warning("Skipping mismatched cached tensor %s", name)
|
||||
continue
|
||||
target.data.copy_(source)
|
||||
|
||||
@instrument(span_name="Load model")
|
||||
def load_model(
|
||||
self, vllm_config: VllmConfig, model_config: ModelConfig, prefix: str = ""
|
||||
) -> nn.Module:
|
||||
# Unsupported quantization is a permanent misconfiguration rather than a
|
||||
# transient daemon outage, so it is raised even when fallback is on.
|
||||
self._check_supported(vllm_config, model_config)
|
||||
state_fetched = False
|
||||
try:
|
||||
entries, aliases = self._fetch_entries(model_config)
|
||||
state_fetched = True
|
||||
return self._build_model(
|
||||
vllm_config, model_config, prefix, entries, aliases
|
||||
)
|
||||
except (WeightCacheUnavailableError, CacheConfigMismatchError) as e:
|
||||
if not self.fallback:
|
||||
raise
|
||||
logger.warning(
|
||||
"Weight cache unusable (%s); falling back to disk loading", e
|
||||
)
|
||||
except Exception:
|
||||
if not self.fallback:
|
||||
raise
|
||||
logger.exception(
|
||||
"Weight cache IPC loading failed; falling back to disk loading"
|
||||
)
|
||||
# _build_model failed after fetching state without reaching its
|
||||
# copy-mode release, so the daemon still holds the full cache;
|
||||
# release it so the disk fallback does not OOM against it.
|
||||
if state_fetched and self.mode == "copy":
|
||||
self._send_release()
|
||||
torch.accelerator.empty_cache()
|
||||
return self._fallback_load(vllm_config, model_config, prefix)
|
||||
|
||||
def _build_model(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
model_config: ModelConfig,
|
||||
prefix: str,
|
||||
entries: dict[str, TensorEntry],
|
||||
aliases: dict[str, str],
|
||||
) -> nn.Module:
|
||||
device_config = vllm_config.device_config
|
||||
load_device = (
|
||||
device_config.device
|
||||
if self.load_config.device is None
|
||||
else self.load_config.device
|
||||
)
|
||||
target_device = torch.device(load_device)
|
||||
device_index = (
|
||||
target_device.index
|
||||
if target_device.index is not None
|
||||
else torch.accelerator.current_device_index()
|
||||
)
|
||||
with set_default_torch_dtype(model_config.dtype):
|
||||
with torch.device("meta"):
|
||||
model = initialize_model(
|
||||
vllm_config=vllm_config,
|
||||
model_config=model_config,
|
||||
prefix=prefix,
|
||||
)
|
||||
self._apply_entries(model, entries, aliases, device_index)
|
||||
# The daemon exports tensors that already went through
|
||||
# process_weights_after_loading; re-run it in pre-processed mode
|
||||
# so quant methods only rebuild Python-side state (e.g. the MoE
|
||||
# kernel). Leftovers are materialized afterwards so that
|
||||
# placeholders the daemon-side post-processing consumed are
|
||||
# dropped rather than filled with uninitialized memory.
|
||||
with weights_already_processed():
|
||||
process_weights_after_loading(model, model_config, target_device)
|
||||
_materialize_remaining_meta_tensors(
|
||||
model, torch.device(target_device.type, device_index)
|
||||
)
|
||||
if self.mode == "copy":
|
||||
self._send_release()
|
||||
logger.info(
|
||||
"Mapped %d tensors from the weight cache daemon (%s mode)",
|
||||
len(entries),
|
||||
self.mode,
|
||||
)
|
||||
return model.eval()
|
||||
|
||||
@staticmethod
|
||||
def _check_supported(vllm_config: VllmConfig, model_config: ModelConfig) -> None:
|
||||
check_ipc_quant_support(model_config, where="engine")
|
||||
cache_dtype = vllm_config.cache_config.cache_dtype
|
||||
if cache_dtype != "auto" and not str(cache_dtype).startswith("fp8"):
|
||||
# BaseKVCacheMethod.process_weights_after_loading turns the loaded
|
||||
# k/v scale parameters into plain float attributes. For fp8 cache
|
||||
# dtypes those are rebuilt from the exported scale buffers when
|
||||
# process_weights_after_loading runs in pre-processed mode; other
|
||||
# quantized cache dtypes are not verified.
|
||||
raise UnsupportedQuantForIPCError(
|
||||
f"[weight_cache:engine] kv cache dtype {cache_dtype!r} is not "
|
||||
"supported by the weight cache; use --kv-cache-dtype auto."
|
||||
)
|
||||
|
||||
def _apply_entries(
|
||||
self,
|
||||
model: nn.Module,
|
||||
entries: dict[str, TensorEntry],
|
||||
aliases: dict[str, str],
|
||||
device_index: int,
|
||||
) -> None:
|
||||
# remove_duplicate=False keeps tied module aliases reachable by name:
|
||||
# a tied lm_head *is* the embedding module, so the deduplicated view
|
||||
# would not contain "lm_head" at all.
|
||||
modules = dict(model.named_modules(remove_duplicate=False))
|
||||
registered: dict[str, torch.Tensor] = {}
|
||||
|
||||
def _register(name: str, tensor: torch.Tensor, is_param: bool) -> None:
|
||||
module_name, _, leaf = name.rpartition(".")
|
||||
module = modules.get(module_name)
|
||||
if module is None:
|
||||
raise RuntimeError(f"Cached tensor {name} has no matching module")
|
||||
# Replace via registration rather than param.data assignment,
|
||||
# which fails for meta tensors. Entries may also introduce
|
||||
# post-quantization tensors absent from the meta model.
|
||||
module._parameters.pop(leaf, None)
|
||||
module._buffers.pop(leaf, None)
|
||||
if is_param:
|
||||
obj: torch.Tensor = (
|
||||
tensor
|
||||
if isinstance(tensor, nn.Parameter)
|
||||
else nn.Parameter(tensor, requires_grad=False)
|
||||
)
|
||||
module.register_parameter(leaf, obj)
|
||||
else:
|
||||
obj = tensor
|
||||
module.register_buffer(leaf, obj)
|
||||
registered[name] = obj
|
||||
|
||||
for name, entry in entries.items():
|
||||
tensor = entry.rebuild(device_index)
|
||||
if self.mode == "copy":
|
||||
tensor = tensor.clone()
|
||||
_register(name, tensor, entry.kind == "param")
|
||||
|
||||
# Re-establish tied-weight aliases by registering the *same* object the
|
||||
# canonical name resolved to, so parameter identity (and the tie) is
|
||||
# preserved instead of allocating uninitialized memory.
|
||||
for alias_name, canonical_name in aliases.items():
|
||||
obj = registered.get(canonical_name)
|
||||
if obj is None:
|
||||
logger.warning(
|
||||
"Cached alias %s references missing canonical tensor %s",
|
||||
alias_name,
|
||||
canonical_name,
|
||||
)
|
||||
continue
|
||||
_register(alias_name, obj, isinstance(obj, nn.Parameter))
|
||||
|
||||
def _fetch_entries(
|
||||
self, model_config: ModelConfig
|
||||
) -> tuple[dict[str, TensorEntry], dict[str, str]]:
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
|
||||
cache_config = WeightCacheKey.from_model_config(
|
||||
model_config,
|
||||
tp_size=get_tensor_model_parallel_world_size(),
|
||||
tp_rank=get_tensor_model_parallel_rank(),
|
||||
)
|
||||
return self._request_state(cache_config)
|
||||
|
||||
def _request_state(
|
||||
self, cache_config: WeightCacheKey
|
||||
) -> tuple[dict[str, TensorEntry], dict[str, str]]:
|
||||
with self._connect(self.state_timeout_s) as conn:
|
||||
send_msg(conn, {"cmd": "get_state", "cache_config": cache_config})
|
||||
response = recv_msg(conn)
|
||||
status = response.get("status")
|
||||
if status == "mismatch":
|
||||
raise CacheConfigMismatchError(
|
||||
f"WeightCacheKey mismatch on fields: {response.get('fields')}"
|
||||
)
|
||||
if status != "ok":
|
||||
raise WeightCacheUnavailableError(
|
||||
f"Weight cache daemon error: {response.get('message')}"
|
||||
)
|
||||
self._check_gpu_uuid(response.get("gpu_uuid"))
|
||||
return response["entries"], response.get("aliases", {})
|
||||
|
||||
def _connect(self, timeout: float) -> socket.socket:
|
||||
socket_path = self._resolve_socket_path()
|
||||
# The auto-derived per-user directory is locked to 0700 and checked
|
||||
# strictly. When the operator explicitly configures a path they own the
|
||||
# trust decision, so only ownership/symlink safety is enforced.
|
||||
strict_perms = self.socket_path is None and self.socket_dir is None
|
||||
try:
|
||||
verify_socket_owner(socket_path, strict_perms=strict_perms)
|
||||
except OSError as e:
|
||||
raise WeightCacheUnavailableError(
|
||||
f"Weight cache socket {socket_path} is unavailable: {e}"
|
||||
) from e
|
||||
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
sock.settimeout(timeout)
|
||||
try:
|
||||
sock.connect(socket_path)
|
||||
except OSError as e:
|
||||
sock.close()
|
||||
raise WeightCacheUnavailableError(
|
||||
f"Cannot connect to weight cache daemon at {socket_path}: {e}"
|
||||
) from e
|
||||
return sock
|
||||
|
||||
def _resolve_socket_path(self) -> str:
|
||||
if self.socket_path is not None:
|
||||
return self.socket_path
|
||||
device_index = torch.accelerator.current_device_index()
|
||||
gpu_id = get_physical_device_id(device_index)
|
||||
if gpu_id is None:
|
||||
raise WeightCacheUnavailableError(
|
||||
"Cannot infer the physical GPU id from CUDA_VISIBLE_DEVICES; "
|
||||
"pass socket_path via --model-loader-extra-config"
|
||||
)
|
||||
return get_socket_path(gpu_id, self.socket_dir)
|
||||
|
||||
def _check_gpu_uuid(self, daemon_uuid: str | None) -> None:
|
||||
if daemon_uuid is None:
|
||||
return
|
||||
props = torch.cuda.get_device_properties(
|
||||
torch.accelerator.current_device_index()
|
||||
)
|
||||
local_uuid = str(props.uuid)
|
||||
if daemon_uuid != local_uuid:
|
||||
raise CacheConfigMismatchError(
|
||||
f"Daemon GPU {daemon_uuid} != engine GPU {local_uuid}; "
|
||||
"check the socket path / GPU mapping"
|
||||
)
|
||||
|
||||
def _send_release(self) -> None:
|
||||
try:
|
||||
with self._connect(self.connect_timeout_s) as conn:
|
||||
send_msg(conn, {"cmd": "release"})
|
||||
recv_msg(conn)
|
||||
except (WeightCacheUnavailableError, ConnectionError, OSError):
|
||||
logger.warning("Failed to ask the weight cache daemon to release")
|
||||
|
||||
def _fallback_load_config(self) -> LoadConfig:
|
||||
# DefaultModelLoader must not see load_format="ipc_cache" or the ipc
|
||||
# extra config keys.
|
||||
return dataclasses.replace(
|
||||
self.load_config, load_format="auto", model_loader_extra_config={}
|
||||
)
|
||||
|
||||
def _fallback_load(
|
||||
self, vllm_config: VllmConfig, model_config: ModelConfig, prefix: str
|
||||
) -> nn.Module:
|
||||
loader = DefaultModelLoader(self._fallback_load_config())
|
||||
return loader.load_model(
|
||||
vllm_config=vllm_config, model_config=model_config, prefix=prefix
|
||||
)
|
||||
|
||||
|
||||
def _materialize_remaining_meta_tensors(model: nn.Module, device: torch.device) -> None:
|
||||
"""Allocate any tensors the daemon did not provide.
|
||||
|
||||
These are typically parameters removed on the daemon side by
|
||||
process_weights_after_loading; they are not expected to be read at
|
||||
runtime, so they are left uninitialized.
|
||||
"""
|
||||
for module_name, module in model.named_modules():
|
||||
for leaf, param in list(module._parameters.items()):
|
||||
if param is not None and param.device.type == "meta":
|
||||
logger.warning(
|
||||
"Materializing empty parameter %s.%s missing from the weight cache",
|
||||
module_name,
|
||||
leaf,
|
||||
)
|
||||
module._parameters[leaf] = nn.Parameter(
|
||||
torch.empty_like(param, device=device), requires_grad=False
|
||||
)
|
||||
for leaf, buffer in list(module._buffers.items()):
|
||||
if buffer is not None and buffer.device.type == "meta":
|
||||
module._buffers[leaf] = torch.empty_like(buffer, device=device)
|
||||
@@ -0,0 +1,412 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""WeightCacheKey fingerprinting and socket protocol for the weight cache daemon.
|
||||
|
||||
The protocol uses pickle over a Unix domain socket and is only intended for
|
||||
communication between trusted local processes owned by the same user. The
|
||||
sockets live in a per-user private directory (mode 0700) and the daemon
|
||||
restricts the socket file permissions to the owner (0600). Both the daemon and
|
||||
the engine verify that the directory and socket are owned by the current user
|
||||
and are not group/world accessible before trusting them, so a different local
|
||||
user cannot pre-plant a malicious socket at a predictable path.
|
||||
"""
|
||||
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import pickle
|
||||
import socket
|
||||
import stat
|
||||
import struct
|
||||
import tempfile
|
||||
from dataclasses import dataclass, fields
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.version
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.utils.hashing import safe_hash
|
||||
|
||||
SOCKET_NAME_TEMPLATE = "vllm_weight_cache_gpu{gpu_id}.sock"
|
||||
SOCKET_DIR_TEMPLATE = "vllm_weight_cache_{uid}"
|
||||
|
||||
_LEN_STRUCT = struct.Struct("!Q")
|
||||
# Sanity bound for a single message. IPC handles are tiny; only small
|
||||
# non-CUDA tensors are ever shipped by value.
|
||||
MAX_MSG_SIZE = 1 << 34
|
||||
|
||||
|
||||
def _current_uid() -> int:
|
||||
getuid = getattr(os, "getuid", None)
|
||||
return getuid() if getuid is not None else -1
|
||||
|
||||
|
||||
class WeightCacheUnavailableError(Exception):
|
||||
"""Raised when no weight cache daemon is reachable or usable."""
|
||||
|
||||
|
||||
class CacheConfigMismatchError(Exception):
|
||||
"""Raised when the daemon's cached weights don't match the engine."""
|
||||
|
||||
|
||||
class UnsupportedQuantForIPCError(Exception):
|
||||
"""Raised when a quantization method is not verified for IPC weight sharing."""
|
||||
|
||||
|
||||
# The daemon exports tensor data only, so sharing is correct just for methods
|
||||
# whose post-load effect is either fully captured by that data or rebuilt when
|
||||
# process_weights_after_loading runs under weights_already_processed. Anything
|
||||
# absent from this registry hard-errors: a method that silently repacks into
|
||||
# shapes the client cannot reproduce or stamps unrebuildable Python-side state
|
||||
# would serve wrong numerics. Extend it only after an end-to-end check against
|
||||
# a disk-loaded baseline.
|
||||
#
|
||||
# This check is config-only so the daemon fails fast before a slow full load,
|
||||
# rather than surfacing the incompatibility only after the engine has built the
|
||||
# model (where process_weights_after_loading's supports_pre_processed_weights
|
||||
# guard would eventually catch it).
|
||||
|
||||
|
||||
def _quant_config_field(quant_config: Any, key: str) -> Any:
|
||||
if quant_config is None:
|
||||
return None
|
||||
if isinstance(quant_config, dict):
|
||||
return quant_config.get(key)
|
||||
return getattr(quant_config, key, None)
|
||||
|
||||
|
||||
def _fp8_round_trips_via_ipc(quant_config: Any) -> bool:
|
||||
"""Only block-wise FP8 is verified.
|
||||
|
||||
Block-wise FP8 preserves the weight shape, while per-tensor FP8 transposes
|
||||
``layer.weight`` during post-processing -- a shape the client's
|
||||
meta-initialized model cannot reproduce.
|
||||
"""
|
||||
return _quant_config_field(quant_config, "weight_block_size") is not None
|
||||
|
||||
|
||||
# quantization name -> predicate(quant_config) -> True when verified safe.
|
||||
IPC_QUANT_ALLOWLIST: dict[str | None, Any] = {
|
||||
None: lambda _quant_config: True, # unquantized
|
||||
"fp8": _fp8_round_trips_via_ipc,
|
||||
}
|
||||
|
||||
|
||||
def is_ipc_quant_supported(quantization: str | None, quant_config: Any) -> bool:
|
||||
predicate = IPC_QUANT_ALLOWLIST.get(quantization)
|
||||
return False if predicate is None else bool(predicate(quant_config))
|
||||
|
||||
|
||||
def check_ipc_quant_support(model_config: ModelConfig, *, where: str) -> None:
|
||||
"""Hard-error unless the model's quantization is verified for IPC sharing.
|
||||
|
||||
Args:
|
||||
model_config: Model configuration to inspect.
|
||||
where: Short tag ("daemon"/"engine") used in the error message.
|
||||
|
||||
Raises:
|
||||
UnsupportedQuantForIPCError: If the quantization method is not on the
|
||||
verified allowlist.
|
||||
"""
|
||||
quantization = model_config.quantization
|
||||
# Prefer the canonical, nested-aware config (multimodal models keep it under
|
||||
# text_config); fall back to the raw hf_config for older code paths.
|
||||
quant_config = getattr(
|
||||
getattr(model_config, "model_arch_config", None), "quantization_config", None
|
||||
)
|
||||
if quant_config is None:
|
||||
quant_config = getattr(model_config.hf_config, "quantization_config", None)
|
||||
if is_ipc_quant_supported(quantization, quant_config):
|
||||
return
|
||||
verified = ", ".join(
|
||||
"unquantized" if name is None else repr(name) for name in IPC_QUANT_ALLOWLIST
|
||||
)
|
||||
raise UnsupportedQuantForIPCError(
|
||||
f"[weight_cache:{where}] quantization {quantization!r} is not verified "
|
||||
f"for CUDA IPC weight sharing: its post-load processing may repack "
|
||||
f"weights into shapes the client cannot reproduce or stamp Python-side "
|
||||
f"state that tensor export cannot carry, which would silently serve "
|
||||
f"wrong numerics. Verified: {verified} (FP8 only with weight_block_size "
|
||||
f"set, i.e. block-wise). Use the default --load-format for this model."
|
||||
)
|
||||
|
||||
|
||||
def get_physical_device_id(device_index: int) -> int | None:
|
||||
"""Map a local CUDA device index to the physical GPU id.
|
||||
|
||||
Returns None if CUDA_VISIBLE_DEVICES contains non-integer entries
|
||||
(e.g. GPU UUIDs), in which case an explicit socket path is required.
|
||||
"""
|
||||
visible = os.environ.get("CUDA_VISIBLE_DEVICES")
|
||||
if not visible:
|
||||
return device_index
|
||||
entries = visible.split(",")
|
||||
try:
|
||||
return int(entries[device_index])
|
||||
except (IndexError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def get_socket_dir(socket_dir: str | None = None) -> str:
|
||||
"""Return the directory that holds the daemon sockets.
|
||||
|
||||
When no explicit directory is given, use a per-user private directory
|
||||
under the system temp dir so its path is unpredictable to other users and
|
||||
can be locked down to mode 0700.
|
||||
"""
|
||||
if socket_dir is not None:
|
||||
return socket_dir
|
||||
return os.path.join(
|
||||
tempfile.gettempdir(), SOCKET_DIR_TEMPLATE.format(uid=_current_uid())
|
||||
)
|
||||
|
||||
|
||||
def get_socket_path(gpu_id: int, socket_dir: str | None = None) -> str:
|
||||
directory = get_socket_dir(socket_dir)
|
||||
return os.path.join(directory, SOCKET_NAME_TEMPLATE.format(gpu_id=gpu_id))
|
||||
|
||||
|
||||
def ensure_private_socket_dir(directory: str, strict_perms: bool = True) -> None:
|
||||
"""Create the socket directory (if needed) locked down to the owner.
|
||||
|
||||
Called by the daemon before binding. Existing directories are re-checked
|
||||
and, for the auto-derived path, tightened so a pre-existing world-writable
|
||||
directory is rejected.
|
||||
"""
|
||||
os.makedirs(directory, mode=0o700, exist_ok=True)
|
||||
if strict_perms:
|
||||
os.chmod(directory, 0o700)
|
||||
verify_private_dir(directory, strict_perms=strict_perms)
|
||||
|
||||
|
||||
def verify_private_dir(directory: str, strict_perms: bool = True) -> None:
|
||||
"""Verify a directory is a real dir owned by us and not world/group readable.
|
||||
|
||||
When ``strict_perms`` is False the group/world permission bits are not
|
||||
checked; this is used for directories the operator explicitly configured
|
||||
(they own the trust decision), while the auto-derived per-user directory is
|
||||
always checked strictly.
|
||||
"""
|
||||
info = os.lstat(directory)
|
||||
if stat.S_ISLNK(info.st_mode):
|
||||
raise WeightCacheUnavailableError(
|
||||
f"Refusing to use symlinked socket directory {directory}"
|
||||
)
|
||||
if not stat.S_ISDIR(info.st_mode):
|
||||
raise WeightCacheUnavailableError(f"{directory} is not a directory")
|
||||
uid = _current_uid()
|
||||
if uid != -1 and info.st_uid != uid:
|
||||
raise WeightCacheUnavailableError(
|
||||
f"Socket directory {directory} is not owned by the current user"
|
||||
)
|
||||
if strict_perms and info.st_mode & 0o077:
|
||||
raise WeightCacheUnavailableError(
|
||||
f"Socket directory {directory} is group/world accessible"
|
||||
)
|
||||
|
||||
|
||||
def verify_socket_owner(socket_path: str, strict_perms: bool = True) -> None:
|
||||
"""Verify the socket lives in a private dir and is owned by the current user.
|
||||
|
||||
Called by the engine before connecting so it never talks to a socket a
|
||||
different user could have planted.
|
||||
"""
|
||||
verify_private_dir(os.path.dirname(socket_path), strict_perms=strict_perms)
|
||||
info = os.lstat(socket_path)
|
||||
if stat.S_ISLNK(info.st_mode):
|
||||
raise WeightCacheUnavailableError(
|
||||
f"Refusing to connect to symlinked socket {socket_path}"
|
||||
)
|
||||
uid = _current_uid()
|
||||
if uid != -1 and info.st_uid != uid:
|
||||
raise WeightCacheUnavailableError(
|
||||
f"Socket {socket_path} is not owned by the current user"
|
||||
)
|
||||
|
||||
|
||||
def verify_peer_is_owner(conn: socket.socket) -> None:
|
||||
"""Best-effort check that the connecting peer runs as the current user.
|
||||
|
||||
Uses SO_PEERCRED where available (Linux). Silently returns on platforms
|
||||
that do not expose peer credentials.
|
||||
"""
|
||||
so_peercred = getattr(socket, "SO_PEERCRED", None)
|
||||
if so_peercred is None:
|
||||
return
|
||||
try:
|
||||
creds = conn.getsockopt(socket.SOL_SOCKET, so_peercred, struct.calcsize("3i"))
|
||||
_, peer_uid, _ = struct.unpack("3i", creds)
|
||||
except OSError:
|
||||
return
|
||||
uid = _current_uid()
|
||||
if uid != -1 and peer_uid != uid:
|
||||
raise PermissionError(f"Rejecting weight cache connection from uid {peer_uid}")
|
||||
|
||||
|
||||
def _hash_quant_config(quant_config: Any) -> str:
|
||||
if quant_config is None:
|
||||
return ""
|
||||
if hasattr(quant_config, "to_dict"):
|
||||
quant_config = quant_config.to_dict()
|
||||
payload = json.dumps(quant_config, sort_keys=True, default=str)
|
||||
return safe_hash(payload.encode(), usedforsecurity=False).hexdigest()
|
||||
|
||||
|
||||
def _safetensors_header(path: str) -> bytes:
|
||||
"""Return the raw safetensors header (length prefix + JSON) of a file.
|
||||
|
||||
The header carries tensor names, dtypes, shapes and byte offsets, so it is
|
||||
a content fingerprint of the shard without reading any weight bytes.
|
||||
"""
|
||||
with open(path, "rb") as f:
|
||||
size_bytes = f.read(8)
|
||||
(header_len,) = struct.unpack("<Q", size_bytes)
|
||||
return size_bytes + f.read(header_len)
|
||||
|
||||
|
||||
def hash_checkpoint(model: str) -> str | None:
|
||||
"""Fingerprint checkpoint content from local safetensors metadata.
|
||||
|
||||
Hashes each shard's safetensors header so a daemon and an engine pointing
|
||||
at identical weights in different directories produce the same key. Returns
|
||||
None when local safetensors files can't be located (e.g. an undownloaded
|
||||
Hugging Face repo id), leaving the caller to fall back to the model path.
|
||||
"""
|
||||
if not os.path.isdir(model):
|
||||
return None
|
||||
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
|
||||
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
filter_duplicate_safetensors_files,
|
||||
)
|
||||
|
||||
files = glob.glob(os.path.join(model, "*.safetensors"))
|
||||
if os.path.isfile(os.path.join(model, SAFE_WEIGHTS_INDEX_NAME)):
|
||||
files = filter_duplicate_safetensors_files(
|
||||
files, model, SAFE_WEIGHTS_INDEX_NAME
|
||||
)
|
||||
if not files:
|
||||
return None
|
||||
hasher = safe_hash(b"", usedforsecurity=False)
|
||||
for path in sorted(files, key=os.path.basename):
|
||||
hasher.update(os.path.basename(path).encode())
|
||||
hasher.update(_safetensors_header(path))
|
||||
return hasher.hexdigest()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WeightCacheKey:
|
||||
"""Fingerprint of the cached weights.
|
||||
|
||||
Any mismatch between the daemon's and the engine's fingerprint means the
|
||||
cached weights cannot be reused and the engine must load from disk.
|
||||
"""
|
||||
|
||||
checkpoint: str
|
||||
model_arch: str
|
||||
tp_size: int
|
||||
tp_rank: int
|
||||
dtype: str
|
||||
quantization: str | None
|
||||
quant_config_hash: str
|
||||
revision: str | None
|
||||
vllm_version: str
|
||||
|
||||
@classmethod
|
||||
def from_model_config(
|
||||
cls, model_config: ModelConfig, tp_size: int, tp_rank: int
|
||||
) -> "WeightCacheKey":
|
||||
"""Build the fingerprint for a model configuration.
|
||||
|
||||
Must be called before weight loading: process_weights_after_loading
|
||||
may mutate hf_config.quantization_config, which would change the hash
|
||||
between the daemon and the engine.
|
||||
|
||||
The checkpoint is identified by a hash of its safetensors metadata when
|
||||
the weights are available locally, so a daemon and engine referencing
|
||||
identical weights in different directories still match; otherwise it
|
||||
falls back to the model path.
|
||||
"""
|
||||
hf_config = model_config.hf_config
|
||||
arch = ",".join(getattr(hf_config, "architectures", None) or [])
|
||||
quant_config = getattr(hf_config, "quantization_config", None)
|
||||
checkpoint = hash_checkpoint(model_config.model) or model_config.model
|
||||
return cls(
|
||||
checkpoint=checkpoint,
|
||||
model_arch=arch,
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
dtype=str(model_config.dtype),
|
||||
quantization=model_config.quantization,
|
||||
quant_config_hash=_hash_quant_config(quant_config),
|
||||
revision=model_config.revision,
|
||||
vllm_version=vllm.version.__version__,
|
||||
)
|
||||
|
||||
def mismatched_fields(self, other: "WeightCacheKey") -> list[str]:
|
||||
return [
|
||||
f.name
|
||||
for f in fields(self)
|
||||
if getattr(self, f.name) != getattr(other, f.name)
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class TensorEntry:
|
||||
"""A single cached tensor.
|
||||
|
||||
CUDA tensors are exported as `torch.multiprocessing` reduction args
|
||||
(CUDA IPC handles); non-CUDA tensors are shipped by value.
|
||||
"""
|
||||
|
||||
kind: str
|
||||
"""Either "param" or "buffer"."""
|
||||
ipc_args: tuple | None = None
|
||||
cpu_tensor: torch.Tensor | None = None
|
||||
|
||||
@classmethod
|
||||
def from_tensor(cls, tensor: torch.Tensor, kind: str) -> "TensorEntry":
|
||||
from torch.multiprocessing.reductions import reduce_tensor
|
||||
|
||||
tensor = tensor.detach()
|
||||
if tensor.is_cuda:
|
||||
_, ipc_args = reduce_tensor(tensor)
|
||||
return cls(kind=kind, ipc_args=ipc_args)
|
||||
return cls(kind=kind, cpu_tensor=tensor.cpu())
|
||||
|
||||
def rebuild(self, device_index: int) -> torch.Tensor:
|
||||
if self.ipc_args is None:
|
||||
assert self.cpu_tensor is not None
|
||||
return self.cpu_tensor
|
||||
from torch.multiprocessing.reductions import rebuild_cuda_tensor
|
||||
|
||||
args = list(self.ipc_args)
|
||||
# Index 6 of the args from reduce_tensor is the device index. It must
|
||||
# be retargeted to the local index since the daemon and the engine may
|
||||
# have different CUDA_VISIBLE_DEVICES mappings.
|
||||
args[6] = device_index
|
||||
return rebuild_cuda_tensor(*args)
|
||||
|
||||
|
||||
def send_msg(sock: socket.socket, obj: Any) -> None:
|
||||
payload = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL)
|
||||
sock.sendall(_LEN_STRUCT.pack(len(payload)))
|
||||
sock.sendall(payload)
|
||||
|
||||
|
||||
def recv_msg(sock: socket.socket) -> Any:
|
||||
(length,) = _LEN_STRUCT.unpack(_recv_exact(sock, _LEN_STRUCT.size))
|
||||
if length > MAX_MSG_SIZE:
|
||||
raise ValueError(f"Message size {length} exceeds limit {MAX_MSG_SIZE}")
|
||||
return pickle.loads(_recv_exact(sock, length))
|
||||
|
||||
|
||||
def _recv_exact(sock: socket.socket, num_bytes: int) -> bytes:
|
||||
buf = bytearray()
|
||||
while len(buf) < num_bytes:
|
||||
chunk = sock.recv(num_bytes - len(buf))
|
||||
if not chunk:
|
||||
raise ConnectionError("Socket closed while receiving message")
|
||||
buf.extend(chunk)
|
||||
return bytes(buf)
|
||||
@@ -3,12 +3,35 @@
|
||||
"""Utils for model executor."""
|
||||
|
||||
import copy
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.utils.torch_utils import is_torch_equal_or_newer
|
||||
|
||||
_weights_pre_processed: ContextVar[bool] = ContextVar(
|
||||
"weights_pre_processed", default=False
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def weights_already_processed():
|
||||
"""Mark that weights are already in post-processed (runtime) format, so
|
||||
``process_weights_after_loading`` skips tensor transforms (used by the
|
||||
weight cache IPC loader).
|
||||
"""
|
||||
token = _weights_pre_processed.set(True)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_weights_pre_processed.reset(token)
|
||||
|
||||
|
||||
def is_weights_pre_processed() -> bool:
|
||||
return _weights_pre_processed.get()
|
||||
|
||||
|
||||
def set_weight_attrs(
|
||||
weight: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user