[Fast Start] Cache the MTP draft model in a separate daemon group (#57312)

Signed-off-by: liusy58 <mg21330037@smail.nju.edu.cn>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
siyu
2026-09-22 03:13:38 +00:00
committed by GitHub
co-authored by Isotr0py
parent f07e227ee9
commit c723a831a8
16 changed files with 417 additions and 140 deletions
@@ -30,10 +30,19 @@ DAEMON_TIMEOUT_S = 600
class WeightCacheDaemon:
"""Context manager running the real weight cache daemon as a subprocess."""
def __init__(self, model: str, tp_size: int, extra_args: list[str] | None = None):
def __init__(
self,
model: str,
tp_size: int,
extra_args: list[str] | None = None,
num_groups: int = 1,
):
# Short base path: Unix socket paths are limited to ~107 characters.
self.socket_dir = tempfile.mkdtemp(prefix="vllm_ipc_")
self.tp_size = tp_size
# Each daemon group (target plus cached drafts) binds one socket per
# local rank.
self.num_sockets = tp_size * num_groups
self._cmd = [
sys.executable,
"-m",
@@ -87,7 +96,7 @@ class WeightCacheDaemon:
f"Weight cache daemon exited with {self._proc.returncode}:\n"
f"{self._logs()}"
)
if len(glob.glob(pattern)) >= self.tp_size:
if len(glob.glob(pattern)) >= self.num_sockets:
return
time.sleep(1.0)
raise TimeoutError(
@@ -112,6 +121,9 @@ class ModelCase:
images: list | None = None
llm_kwargs: dict[str, Any] = field(default_factory=dict)
daemon_args: list[str] = field(default_factory=list)
# Daemon groups the launcher starts; 2 when a cached draft group joins the
# target group (MTP/EAGLE speculative decoding).
daemon_groups: int = 1
def generate(
@@ -172,14 +184,41 @@ K3_CASE = ModelCase(
daemon_args=["--trust-remote-code"],
)
# Qwen3.5-0.8B ships one MTP layer in the target checkpoint, so method="mtp"
# loads the draft from the same model. The daemon must cache it in its draft
# group for the warm runs (fallback=False) to succeed.
QWEN_MTP_CASE = ModelCase(
model="Qwen/Qwen3.5-0.8B",
prompts=[
"Hello, my name is",
"The capital of France is",
],
llm_kwargs=dict(
gpu_memory_utilization=0.3,
enforce_eager=True,
enable_chunked_prefill=True,
speculative_config={"method": "mtp", "num_speculative_tokens": 1},
),
daemon_args=[
"--speculative-config",
'{"method": "mtp", "num_speculative_tokens": 1}',
],
daemon_groups=2,
)
@pytest.mark.parametrize("case", [QWEN_CASE, K3_CASE], ids=["qwen3.5", "kimi-k3"])
@pytest.mark.parametrize(
"case",
[QWEN_CASE, K3_CASE, QWEN_MTP_CASE],
ids=["qwen3.5", "kimi-k3", "qwen3.5-mtp"],
)
def test_ipc_cache_cold_start_and_warm_restart(vllm_runner, case: ModelCase):
"""Cold start falls back to disk; warm restarts load weights via CUDA IPC.
All runs must produce outputs identical to a default-loader baseline. The
warm runs disable the disk fallback, so they only pass if the weights
really came from the daemon.
really came from the daemon — for the MTP case, both the target's and the
draft's daemon groups.
"""
if not current_platform.is_cuda_alike():
pytest.skip("Weight cache IPC sharing requires CUDA or ROCm")
@@ -199,7 +238,12 @@ def test_ipc_cache_cold_start_and_warm_restart(vllm_runner, case: ModelCase):
with tempfile.TemporaryDirectory(prefix="vllm_ipc_empty_") as empty_socket_dir:
cold_outputs = generate(vllm_runner, case, empty_socket_dir, fallback=True)
with WeightCacheDaemon(case.model, tp_size=1, extra_args=case.daemon_args) as d:
with WeightCacheDaemon(
case.model,
tp_size=1,
extra_args=case.daemon_args,
num_groups=case.daemon_groups,
) as d:
warm_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
# Warm restart: a second engine lifetime against the same daemon.
restart_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
+2 -1
View File
@@ -369,11 +369,12 @@ def test_dspark_shares_target_embedding_with_smaller_draft_vocabulary():
draft_parallel_config=SimpleNamespace(tensor_parallel_size=1),
attention_backend=None,
kv_cache_dtype=None,
draft_load_config=None,
),
parallel_config=ParallelConfig(),
attention_config=SimpleNamespace(backend=None),
cache_config=SimpleNamespace(),
load_config=SimpleNamespace(),
load_config=SimpleNamespace(load_format="auto"),
model_config=SimpleNamespace(get_vocab_size=Mock(return_value=100)),
)
@@ -12,7 +12,7 @@ from unittest.mock import patch
import pytest
from vllm.config import LoadConfig
from vllm.config import LoadConfig, SpeculativeConfig
from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
@@ -37,6 +37,9 @@ class _SpeculativeConfig:
moe_backend: str | None = None
kv_cache_dtype: str | None = None
draft_model_config: object = None
draft_load_config: object = None
apply_draft_overrides = SpeculativeConfig.apply_draft_overrides
@dataclass
@@ -14,7 +14,7 @@ from unittest.mock import patch
import pytest
from vllm.config import LoadConfig
from vllm.config import LoadConfig, SpeculativeConfig
from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
@@ -34,6 +34,9 @@ class _SpeculativeConfig:
moe_backend: str | None = None
kv_cache_dtype: str | None = None
draft_model_config: object = None
draft_load_config: object = None
apply_draft_overrides = SpeculativeConfig.apply_draft_overrides
@dataclass
+26 -1
View File
@@ -14,7 +14,7 @@ from vllm.config.cache import CacheDType
from vllm.config.kernel import MoEBackend
from vllm.config.model import HfOverrides, ModelConfig
from vllm.config.parallel import ParallelConfig
from vllm.config.utils import config
from vllm.config.utils import config, replace
from vllm.logger import init_logger
from vllm.transformers_utils.config import get_hf_text_config
from vllm.utils.hashing import safe_hash
@@ -25,8 +25,10 @@ if TYPE_CHECKING:
from transformers import PretrainedConfig
import vllm.model_executor.layers.quantization as me_quant
from vllm.config.vllm import VllmConfig
else:
PretrainedConfig = Any
VllmConfig = Any
me_quant = LazyLoader(
"model_executor", globals(), "vllm.model_executor.layers.quantization"
@@ -371,6 +373,17 @@ def _validate_qwen3_omni_dspark(
)
# (SpeculativeConfig field, VllmConfig sub-config, overridden field)
_DRAFT_VLLM_CONFIG_OVERRIDES = (
# Otherwise the draft inherits the target's --moe-backend, which fails
# when the draft is unquantized and that backend is not.
("moe_backend", "kernel_config", "moe_backend"),
# Only when set, so the draft keeps a KV cache layout the target shares.
("attention_backend", "attention_config", "backend"),
("kv_cache_dtype", "cache_config", "cache_dtype"),
)
@config
class SpeculativeConfig:
"""Configuration for speculative decoding."""
@@ -1771,6 +1784,18 @@ class SpeculativeConfig:
return draft_parallel_config
def apply_draft_overrides(self, vllm_config: VllmConfig) -> VllmConfig:
"""Overlay this config's kernel overrides onto a target VllmConfig.
Only non-None fields override, so an unset field keeps whatever the
target resolved.
"""
for src, config_name, dst in _DRAFT_VLLM_CONFIG_OVERRIDES:
if (value := getattr(self, src)) is not None:
sub_config = replace(getattr(vllm_config, config_name), **{dst: value})
vllm_config = replace(vllm_config, **{config_name: sub_config})
return vllm_config
@field_validator("attention_backend", mode="before")
@classmethod
def _parse_attention_backend(cls, value: Any) -> Any:
+37 -1
View File
@@ -12,7 +12,13 @@ from torch import nn
from typing_extensions import assert_never
import vllm.envs as envs
from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
from vllm.config import (
LoadConfig,
ModelConfig,
VllmConfig,
replace,
set_current_vllm_config,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import is_deferred_attention_layer
from vllm.model_executor.layers.quantization.base_config import (
@@ -23,6 +29,9 @@ from vllm.model_executor.model_loader.reload import (
record_metadata_for_reloading,
set_torchao_reload_attrs,
)
from vllm.model_executor.model_loader.weight_cache.utils import (
is_draft_model_cacheable,
)
from vllm.model_executor.model_loader.weight_tying import maybe_retie_word_embeddings
from vllm.model_executor.models.interfaces import SupportsQuant
from vllm.model_executor.utils import is_weights_pre_processed
@@ -34,6 +43,33 @@ from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor
logger = init_logger(__name__)
def get_draft_load_config(vllm_config: VllmConfig) -> LoadConfig:
"""Get load config for the speculative draft model."""
speculative_config = vllm_config.speculative_config
if (
speculative_config is not None
and speculative_config.draft_load_config is not None
):
return speculative_config.draft_load_config
load_config = vllm_config.load_config
if load_config is not None and load_config.load_format != "ipc_cache":
return load_config
kwargs = (
# Route the draft to the daemon's draft group.
{
"model_loader_extra_config": {
**load_config.model_loader_extra_config,
"is_draft": True,
}
}
if is_draft_model_cacheable(speculative_config)
# No daemon draft group for this method; load from disk instead of
# hitting the target daemon with a mismatching fingerprint.
else {"load_format": "auto", "model_loader_extra_config": {}}
)
return replace(load_config, **kwargs)
@instrument(span_name="Initialize model")
def initialize_model(
vllm_config: VllmConfig,
@@ -1,26 +1,3 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.model_executor.model_loader.weight_cache.ipc_loader import IpcModelLoader
from vllm.model_executor.model_loader.weight_cache.protocol import (
CacheConfigMismatchError,
TensorEntry,
UnsupportedPlatformForIPCError,
UnsupportedQuantForIPCError,
WeightCacheKey,
WeightCacheUnavailableError,
check_ipc_platform_support,
check_ipc_quant_support,
)
__all__ = [
"CacheConfigMismatchError",
"IpcModelLoader",
"TensorEntry",
"UnsupportedPlatformForIPCError",
"UnsupportedQuantForIPCError",
"WeightCacheKey",
"WeightCacheUnavailableError",
"check_ipc_platform_support",
"check_ipc_quant_support",
]
@@ -41,6 +41,13 @@ The global TP rank of local GPU ``i`` on node ``r`` is
``r * (tp_size // nnodes) + i``, matching vLLM's contiguous per-node rank
assignment, so each engine worker maps its shard from the daemon on its own
node.
With MTP, EAGLE or EAGLE3 speculative decoding the launcher additionally
starts a draft daemon group that caches the draft model. It uses its own cache
key, Unix sockets (``*_draft.sock``) and rendezvous port
(``--weight-cache-draft-master-port``, default ``--weight-cache-master-port +
1``), so each process serves exactly one model role. Other draft types are not
cached and keep loading from disk in the engine.
"""
import contextlib
@@ -52,10 +59,15 @@ import signal
import socket
import sys
from collections.abc import Callable
from itertools import product
import torch
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
from vllm.config import (
ParallelConfig,
VllmConfig,
set_current_vllm_config,
)
from vllm.distributed import (
ensure_model_parallel_initialized,
init_distributed_environment,
@@ -77,6 +89,11 @@ from vllm.model_executor.model_loader.weight_cache.protocol import (
send_msg,
verify_peer_is_owner,
)
from vllm.model_executor.model_loader.weight_cache.utils import (
export_model_attrs,
format_daemon_role,
is_draft_model_cacheable,
)
from vllm.platforms import current_platform
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.network_utils import get_distributed_init_method, get_open_port
@@ -130,30 +147,6 @@ def export_entries(
return entries, aliases
def get_daemon_model(vllm_config: VllmConfig) -> torch.nn.Module:
"""Load the daemon's model, composed from the configured loader.
Runs the quantization check after model creation but before the slow
weight load, so an unsupported method fails fast. Online quantization
always fails the check, so load_model's finalize step for it is
unnecessary here.
"""
model_config = vllm_config.model_config
load_config = vllm_config.load_config
loader = get_model_loader(load_config)
device_config = vllm_config.device_config
target_device = torch.device(
device_config.device if load_config.device is None else load_config.device
)
with set_default_torch_dtype(model_config.dtype):
with target_device:
model = loader.create_model(vllm_config, model_config)
check_ipc_quant_support(model)
loader.load_weights(model, model_config)
process_weights_after_loading(model, model_config, target_device)
return model.eval()
class WeightCacheDaemon:
"""Per-GPU process that loads one TP shard and serves CUDA IPC handles."""
@@ -164,19 +157,31 @@ class WeightCacheDaemon:
local_rank: int,
distributed_init_method: str,
socket_dir: str | None = None,
is_draft: bool = False,
):
self.vllm_config = vllm_config
# A draft daemon builds the draft with the target's VllmConfig, like
# the engine does; only the ModelConfig differs.
if is_draft:
spec = vllm_config.speculative_config
assert spec is not None and spec.draft_model_config is not None
self.model_config = spec.draft_model_config
else:
self.model_config = vllm_config.model_config
self.tp_rank = tp_rank
self.local_rank = local_rank
self.distributed_init_method = distributed_init_method
self.socket_dir = socket_dir
self.is_draft = is_draft
self.role = format_daemon_role(is_draft)
self.model: torch.nn.Module | None = None
# Fingerprint before loading: process_weights_after_loading may
# mutate hf_config.quantization_config.
self.cache_config = WeightCacheKey.from_model_config(
vllm_config.model_config,
self.model_config,
tp_size=vllm_config.parallel_config.tensor_parallel_size,
tp_rank=tp_rank,
is_draft=is_draft,
)
def load_model(self) -> None:
@@ -191,12 +196,37 @@ class WeightCacheDaemon:
)
with set_current_vllm_config(self.vllm_config):
ensure_model_parallel_initialized(tp_size, 1)
self.model = get_daemon_model(self.vllm_config)
self.model = self.get_model()
logger.info(
"Weight cache daemon rank %d loaded model",
"Weight cache %s daemon rank %d loaded model",
self.role,
self.tp_rank,
)
def get_model(self) -> torch.nn.Module:
"""Load the daemon's model, composed from the configured loader.
Runs the quantization check after model creation but before the slow
weight load, so an unsupported method fails fast. Online quantization
always fails the check, so load_model's finalize step for it is
unnecessary here.
"""
vllm_config = self.vllm_config
model_config = self.model_config
load_config = vllm_config.load_config
loader = get_model_loader(load_config)
device_config = vllm_config.device_config
target_device = torch.device(
device_config.device if load_config.device is None else load_config.device
)
with set_default_torch_dtype(model_config.dtype):
with target_device:
model = loader.create_model(vllm_config, model_config)
check_ipc_quant_support(model)
loader.load_weights(model, model_config)
process_weights_after_loading(model, model_config, target_device)
return model.eval()
def serve_forever(self, ready_callback: Callable[[], None] | None = None) -> None:
"""Serve requests until terminated.
@@ -223,7 +253,10 @@ class WeightCacheDaemon:
os.chmod(socket_path, 0o600)
server.listen()
logger.info(
"Weight cache daemon rank %d serving on %s", self.tp_rank, socket_path
"Weight cache %s daemon rank %d serving on %s",
self.role,
self.tp_rank,
socket_path,
)
if ready_callback is not None:
ready_callback()
@@ -270,7 +303,11 @@ class WeightCacheDaemon:
@property
def _socket_path(self) -> str:
return get_socket_path(get_current_device_uuid(), self.socket_dir)
return get_socket_path(
get_current_device_uuid(),
self.socket_dir,
is_draft=self.is_draft,
)
def _handle_connection(self, conn: socket.socket) -> None:
request = recv_msg(conn)
@@ -303,11 +340,13 @@ class WeightCacheDaemon:
"status": "ok",
"entries": entries,
"aliases": aliases,
"attrs": export_model_attrs(self.model),
"gpu_uuid": gpu_uuid,
},
)
logger.info_once(
"Weight cache daemon rank %d sent %d tensors (+%d aliases) to engine",
"Weight cache %s daemon rank %d sent %d tensors (+%d aliases) to engine",
self.role,
self.tp_rank,
len(entries),
len(aliases),
@@ -316,7 +355,11 @@ class WeightCacheDaemon:
def _handle_release(self, conn: socket.socket) -> None:
self.model = None
torch.accelerator.empty_cache()
logger.info("Weight cache daemon rank %d released cached weights", self.tp_rank)
logger.info(
"Weight cache %s daemon rank %d released cached weights",
self.role,
self.tp_rank,
)
send_msg(conn, {"status": "ok"})
@@ -326,13 +369,27 @@ def _run_daemon(
vllm_config: VllmConfig,
distributed_init_method: str,
socket_dir: str | None,
ready_queue: "multiprocessing.Queue[int]",
ready_queue: "multiprocessing.Queue[tuple[str, int]]",
is_draft: bool = False,
) -> None:
daemon = WeightCacheDaemon(
vllm_config, tp_rank, local_rank, distributed_init_method, socket_dir
vllm_config,
tp_rank,
local_rank,
distributed_init_method,
socket_dir,
is_draft,
)
daemon.load_model()
daemon.serve_forever(ready_callback=lambda: ready_queue.put(tp_rank))
daemon.serve_forever(ready_callback=lambda: ready_queue.put((daemon.role, tp_rank)))
def get_draft_daemon_config(vllm_config: VllmConfig) -> VllmConfig | None:
"""VllmConfig for the draft daemon group"""
if not is_draft_model_cacheable(vllm_config.speculative_config):
return None
assert vllm_config.speculative_config is not None
return vllm_config.speculative_config.apply_draft_overrides(vllm_config)
def _reject_unsupported_parallelism(parallel_config: ParallelConfig) -> None:
@@ -369,6 +426,14 @@ def main() -> None:
"serving) and match across nodes. Required when --nnodes > 1; defaults "
"to a free port for single-node.",
)
parser.add_argument(
"--weight-cache-draft-master-port",
type=int,
default=None,
help="Rendezvous port for the MTP draft daemon group. Defaults to "
"--weight-cache-master-port + 1 for multi-node, or a free port for "
"single-node.",
)
args = parser.parse_args()
engine_args = EngineArgs.from_cli_args(args)
vllm_config = engine_args.create_engine_config()
@@ -398,27 +463,56 @@ def main() -> None:
master_port = args.weight_cache_master_port or get_open_port()
distributed_init_method = get_distributed_init_method(master_addr, master_port)
# (is_draft, vllm_config, rendezvous) per daemon group.
groups: list[tuple[bool, VllmConfig, str]] = [
(False, vllm_config, distributed_init_method)
]
draft_vllm_config = get_draft_daemon_config(vllm_config)
if draft_vllm_config is not None:
draft_master_port = args.weight_cache_draft_master_port or (
master_port + 1 if nnodes > 1 else get_open_port()
)
if draft_master_port == master_port:
raise ValueError(
"--weight-cache-draft-master-port must differ from "
"--weight-cache-master-port"
)
groups.append(
(
True,
draft_vllm_config,
get_distributed_init_method(master_addr, draft_master_port),
)
)
ctx = multiprocessing.get_context("spawn")
ready_queue: multiprocessing.Queue[int] = ctx.Queue()
ready_queue: multiprocessing.Queue[tuple[str, int]] = ctx.Queue()
# Global TP rank of local GPU i on this node; local index == device index.
global_ranks = [
node_rank * local_world_size + local_rank
for local_rank in range(local_world_size)
]
expected_ready = {
(format_daemon_role(is_draft), global_rank)
for (is_draft, _, _), global_rank in product(groups, global_ranks)
}
procs = [
ctx.Process(
target=_run_daemon,
args=(
global_rank,
local_rank,
vllm_config,
distributed_init_method,
config,
init_method,
args.weight_cache_socket_dir,
ready_queue,
is_draft,
),
name=f"vllm-weight-cache-daemon-{global_rank}",
name=f"vllm-weight-cache-{format_daemon_role(is_draft)}-{global_rank}",
)
for (is_draft, config, init_method), (local_rank, global_rank) in product(
groups, enumerate(global_ranks)
)
for local_rank, global_rank in enumerate(global_ranks)
]
for proc in procs:
proc.start()
@@ -430,10 +524,10 @@ def main() -> None:
signal.signal(signal.SIGINT, _shutdown)
signal.signal(signal.SIGTERM, _shutdown)
ready_ranks: set[int] = set()
while len(ready_ranks) < local_world_size:
ready: set[tuple[str, int]] = set()
while len(ready) < len(expected_ready):
try:
ready_ranks.add(ready_queue.get(timeout=1.0))
ready.add(ready_queue.get(timeout=1.0))
except queue.Empty:
dead = [p for p in procs if p.exitcode is not None]
if dead:
@@ -450,10 +544,11 @@ def main() -> None:
socket_dir_msg = args.weight_cache_socket_dir or "the default socket dir"
logger.info_once(
"===== Weight cache daemon READY: node %d/%d serving %d local rank(s) "
"in %s =====",
"x %d role(s) in %s =====",
node_rank,
nnodes,
local_world_size,
len(groups),
socket_dir_msg,
)
@@ -25,9 +25,9 @@ from vllm.model_executor.model_loader.utils import (
)
from vllm.model_executor.model_loader.weight_cache.protocol import (
CacheConfigMismatchError,
TensorEntry,
UnsupportedQuantForIPCError,
WeightCacheKey,
WeightCacheState,
WeightCacheUnavailableError,
check_ipc_platform_support,
check_ipc_quant_support,
@@ -60,7 +60,7 @@ class IpcModelLoader(BaseModelLoader):
Extra config keys (via --model-loader-extra-config):
- socket_path: explicit daemon socket path. Defaults to a per-GPU path
derived from the physical GPU uuid.
derived from the physical GPU uuid and the cache role (target/draft).
- socket_dir: directory containing the daemon sockets.
- mode: "zero_copy" (default) or "copy".
- fallback: fall back to disk loading when the daemon is unavailable or
@@ -78,6 +78,15 @@ class IpcModelLoader(BaseModelLoader):
extra_config = copy(load_config.model_loader_extra_config or {})
self.socket_path: str | None = extra_config.pop("socket_path", None)
self.socket_dir: str | None = extra_config.pop("socket_dir", None)
# Internal: set by the engine when routing a speculative draft to the
# daemon's draft group.
self.is_draft = bool(extra_config.pop("is_draft", False))
if self.is_draft and self.socket_path is not None:
raise ValueError(
"socket_path cannot be combined with the draft weight cache role; "
"use socket_dir so the target and draft sockets are derived "
"independently"
)
self.mode: str = extra_config.pop("mode", "zero_copy")
self.fallback: bool = extra_config.pop("fallback", True)
self.connect_timeout_s: float = float(
@@ -108,7 +117,7 @@ class IpcModelLoader(BaseModelLoader):
loaded through this loader).
"""
device_index = torch.accelerator.current_device_index()
entries, _ = self._fetch_entries(model_config)
entries = self._fetch_entries(model_config).entries
params = dict(model.named_parameters())
buffers = dict(model.named_buffers())
for name, entry in entries.items():
@@ -128,11 +137,22 @@ class IpcModelLoader(BaseModelLoader):
check_ipc_platform_support()
state_fetched = False
try:
entries, aliases = self._fetch_entries(model_config)
# Cross-check the routing flag against the identity of the model
# being loaded: a draft load that lost its flag (or a target load
# that got one) would hit the wrong daemon group and
# fingerprint-mismatch.
spec = vllm_config.speculative_config
inferred = spec is not None and model_config is spec.draft_model_config
if inferred != self.is_draft:
raise CacheConfigMismatchError(
f"Weight cache role mismatch: loading "
f"{'draft' if inferred else 'target'} model but the loader "
f"was configured for the "
f"{'draft' if self.is_draft else 'target'} group"
)
state = self._fetch_entries(model_config)
state_fetched = True
return self._build_model(
vllm_config, model_config, prefix, entries, aliases
)
return self._build_model(vllm_config, model_config, prefix, state)
except (WeightCacheUnavailableError, CacheConfigMismatchError) as e:
if not self.fallback:
raise
@@ -163,8 +183,7 @@ class IpcModelLoader(BaseModelLoader):
vllm_config: VllmConfig,
model_config: ModelConfig,
prefix: str,
entries: dict[str, TensorEntry],
aliases: dict[str, str],
state: WeightCacheState,
) -> nn.Module:
device_config = vllm_config.device_config
load_device = (
@@ -186,7 +205,11 @@ class IpcModelLoader(BaseModelLoader):
prefix=prefix,
)
check_ipc_quant_support(model)
self._apply_entries(model, entries, aliases, device_index)
self._apply_entries(model, state, device_index)
# Flags that load_weights would have set (e.g. EAGLE ownership of
# embed_tokens / lm_head); the daemon ran it, this process did not.
for name, value in state.attrs.items():
setattr(model, name, value)
# The daemon exports tensors that already went through
# process_weights_after_loading; re-run it in pre-processed mode
# so quant methods only rebuild Python-side state (e.g. the MoE
@@ -202,7 +225,7 @@ class IpcModelLoader(BaseModelLoader):
self._send_release()
logger.info(
"Mapped %d tensors from the weight cache daemon (%s mode)",
len(entries),
len(state.entries),
self.mode,
)
return model.eval()
@@ -210,8 +233,7 @@ class IpcModelLoader(BaseModelLoader):
def _apply_entries(
self,
model: nn.Module,
entries: dict[str, TensorEntry],
aliases: dict[str, str],
state: WeightCacheState,
device_index: int,
) -> None:
# remove_duplicate=False keeps tied module aliases reachable by name:
@@ -242,7 +264,7 @@ class IpcModelLoader(BaseModelLoader):
module.register_buffer(leaf, obj)
registered[name] = obj
for name, entry in entries.items():
for name, entry in state.entries.items():
tensor = entry.rebuild(device_index)
if self.mode == "copy":
tensor = tensor.clone()
@@ -251,7 +273,7 @@ class IpcModelLoader(BaseModelLoader):
# Re-establish tied-weight aliases by registering the *same* object the
# canonical name resolved to, so parameter identity (and the tie) is
# preserved instead of allocating uninitialized memory.
for alias_name, canonical_name in aliases.items():
for alias_name, canonical_name in state.aliases.items():
obj = registered.get(canonical_name)
if obj is None:
logger.warning(
@@ -262,19 +284,16 @@ class IpcModelLoader(BaseModelLoader):
continue
_register(alias_name, obj, isinstance(obj, nn.Parameter))
def _fetch_entries(
self, model_config: ModelConfig
) -> tuple[dict[str, TensorEntry], dict[str, str]]:
def _fetch_entries(self, model_config: ModelConfig) -> WeightCacheState:
cache_config = WeightCacheKey.from_model_config(
model_config,
tp_size=get_tensor_model_parallel_world_size(),
tp_rank=get_tensor_model_parallel_rank(),
is_draft=self.is_draft,
)
return self._request_state(cache_config)
def _request_state(
self, cache_config: WeightCacheKey
) -> tuple[dict[str, TensorEntry], dict[str, str]]:
def _request_state(self, cache_config: WeightCacheKey) -> WeightCacheState:
with self._connect(self.state_timeout_s) as conn:
send_msg(conn, {"cmd": "get_state", "cache_config": cache_config})
response = recv_msg(conn)
@@ -288,7 +307,11 @@ class IpcModelLoader(BaseModelLoader):
f"Weight cache daemon error: {response.get('message')}"
)
self._check_gpu_uuid(response.get("gpu_uuid"))
return response["entries"], response.get("aliases", {})
return WeightCacheState(
entries=response["entries"],
aliases=response.get("aliases", {}),
attrs=response.get("attrs", {}),
)
def _connect(self, timeout: float) -> socket.socket:
socket_path = self._resolve_socket_path()
@@ -316,7 +339,11 @@ class IpcModelLoader(BaseModelLoader):
def _resolve_socket_path(self) -> str:
if self.socket_path is not None:
return self.socket_path
return get_socket_path(get_current_device_uuid(), self.socket_dir)
return get_socket_path(
get_current_device_uuid(),
self.socket_dir,
is_draft=self.is_draft,
)
def _check_gpu_uuid(self, daemon_uuid: str | None) -> None:
if daemon_uuid is None:
@@ -340,7 +367,9 @@ class IpcModelLoader(BaseModelLoader):
# DefaultModelLoader must not see load_format="ipc_cache" or the ipc
# extra config keys.
return dataclasses.replace(
self.load_config, load_format="auto", model_loader_extra_config={}
self.load_config,
load_format="auto",
model_loader_extra_config={},
)
def _fallback_load(
@@ -20,7 +20,7 @@ import stat
import struct
import tempfile
from dataclasses import dataclass, fields
from typing import Any
from typing import Any, NamedTuple
import torch
from torch.multiprocessing.reductions import rebuild_cuda_tensor, reduce_tensor
@@ -29,13 +29,16 @@ from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
import vllm.version
from vllm.config import ModelConfig
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
from vllm.model_executor.model_loader.weight_cache.utils import (
format_socket_role_suffix,
)
from vllm.model_executor.model_loader.weight_utils import (
filter_duplicate_safetensors_files,
)
from vllm.platforms import current_platform
from vllm.utils.hashing import safe_hash
SOCKET_NAME_TEMPLATE = "vllm_weight_cache_{gpu_uuid}.sock"
SOCKET_NAME_TEMPLATE = "vllm_weight_cache_{gpu_uuid}{role}.sock"
SOCKET_DIR_TEMPLATE = "vllm_weight_cache_{uid}"
_LEN_STRUCT = struct.Struct("!Q")
@@ -128,9 +131,23 @@ def get_socket_dir(socket_dir: str | None = None) -> str:
)
def get_socket_path(gpu_uuid: str, socket_dir: str | None = None) -> str:
directory = get_socket_dir(socket_dir)
return os.path.join(directory, SOCKET_NAME_TEMPLATE.format(gpu_uuid=gpu_uuid))
def get_socket_path(
gpu_uuid: str,
socket_dir: str | None = None,
*,
is_draft: bool = False,
) -> str:
"""Socket path of a daemon group; ``is_draft=False`` is the target.
The GPU uuid is hashed to keep the name well under the AF_UNIX path
limit (~108 bytes) even with the draft role suffix.
"""
gpu_id = safe_hash(gpu_uuid.encode()).hexdigest()
name = SOCKET_NAME_TEMPLATE.format(
gpu_uuid=gpu_id,
role=format_socket_role_suffix(is_draft),
)
return os.path.join(get_socket_dir(socket_dir), name)
def ensure_private_socket_dir(directory: str, strict_perms: bool = True) -> None:
@@ -272,10 +289,17 @@ class WeightCacheKey:
quant_config_hash: str
revision: str | None
vllm_version: str
is_draft: bool = False
"""Daemon group the weights come from; False is the target model."""
@classmethod
def from_model_config(
cls, model_config: ModelConfig, tp_size: int, tp_rank: int
cls,
model_config: ModelConfig,
tp_size: int,
tp_rank: int,
*,
is_draft: bool = False,
) -> "WeightCacheKey":
"""Build the fingerprint for a model configuration.
@@ -302,6 +326,7 @@ class WeightCacheKey:
quant_config_hash=_hash_quant_config(quant_config),
revision=model_config.revision,
vllm_version=vllm.version.__version__,
is_draft=is_draft,
)
def mismatched_fields(self, other: "WeightCacheKey") -> list[str]:
@@ -345,6 +370,17 @@ class TensorEntry:
return rebuild_cuda_tensor(*args)
class WeightCacheState(NamedTuple):
"""Client-side decode of a daemon's get_state response payload."""
entries: dict[str, TensorEntry]
"""Model tensors, exported as CUDA IPC handles or shipped by value."""
aliases: dict[str, str]
"""Duplicate (tied) weight names aliased to their canonical entry."""
attrs: dict[str, bool]
"""Python-side flags set by load_weights, e.g. EAGLE ownership flags."""
def send_msg(sock: socket.socket, obj: Any) -> None:
payload = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL)
sock.sendall(_LEN_STRUCT.pack(len(payload)))
@@ -0,0 +1,48 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Helpers shared by the weight cache daemon, the IPC loader and the engine.
These live outside ``protocol`` so callers that only need to know whether a
draft is cached, or how a daemon group is named, do not have to import the
wire format.
"""
from typing import Any
from vllm.config import SpeculativeConfig
from vllm.model_executor.models.interfaces import SupportsEagleBase
# Speculative methods whose draft model the daemon caches in its own group.
# Other drafts keep loading from disk in the engine.
WEIGHT_CACHE_DRAFT_METHODS = frozenset({"mtp", "eagle", "eagle3"})
# Python-side flags that weight loading sets on EAGLE-style drafts; the engine
# never runs load_weights for cached models, so the daemon ships them.
EXPORTED_MODEL_ATTRS = tuple(SupportsEagleBase.__annotations__)
def export_model_attrs(model: Any) -> dict[str, bool]:
return {
name: bool(getattr(model, name))
for name in EXPORTED_MODEL_ATTRS
if hasattr(model, name)
}
def is_draft_model_cacheable(speculative_config: SpeculativeConfig | None) -> bool:
"""Whether the daemon serves the speculative draft as a separate role."""
return (
speculative_config is not None
and speculative_config.method in WEIGHT_CACHE_DRAFT_METHODS
and speculative_config.draft_model_config is not None
)
def format_daemon_role(is_draft: bool) -> str:
"""Name of a daemon group: the target model or the draft."""
return "draft" if is_draft else "target"
def format_socket_role_suffix(is_draft: bool) -> str:
"""Socket-name suffix keeping the draft group distinct from the target."""
return "_draft" if is_draft else ""
+2 -1
View File
@@ -25,6 +25,7 @@ from vllm.forward_context import set_forward_context
from vllm.logger import init_logger
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.model_loader import get_model
from vllm.model_executor.model_loader.utils import get_draft_load_config
from vllm.model_executor.models import (
supports_multimodal,
supports_multimodal_embeddings,
@@ -1311,7 +1312,7 @@ class SpecDecodeBaseProposer:
model = get_model(
vllm_config=draft_vllm_config,
model_config=self.speculative_config.draft_model_config,
load_config=self.speculative_config.draft_load_config,
load_config=get_draft_load_config(draft_vllm_config),
)
return model
@@ -4,6 +4,7 @@ import torch.nn as nn
from vllm.config import VllmConfig, replace
from vllm.model_executor.model_loader import get_model
from vllm.model_executor.model_loader.utils import get_draft_load_config
from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
_should_share,
get_target_lm_head,
@@ -38,7 +39,7 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
if speculative_config.kv_cache_dtype is not None
else vllm_config.cache_config
),
load_config=get_pp_safe_draft_load_config(vllm_config.load_config),
load_config=get_pp_safe_draft_load_config(get_draft_load_config(vllm_config)),
)
with set_model_tag("dflash_head"):
dflash_model = get_model(
@@ -5,6 +5,7 @@ import torch.nn as nn
from vllm.config import ModelConfig, ParallelConfig, VllmConfig, replace
from vllm.logger import init_logger
from vllm.model_executor.model_loader.utils import get_draft_load_config
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.worker.gpu.spec_decode.utils import get_pp_safe_draft_load_config
@@ -94,7 +95,7 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
if speculative_config.kv_cache_dtype is not None
else vllm_config.cache_config
),
load_config=get_pp_safe_draft_load_config(vllm_config.load_config),
load_config=get_pp_safe_draft_load_config(get_draft_load_config(vllm_config)),
)
# VllmConfig post-init restores the target's quant config because the target
# config is retained for DSpark's target-layer metadata, so we must override it.
+5 -29
View File
@@ -7,6 +7,7 @@ from vllm.config import VllmConfig, replace
from vllm.distributed.parallel_state import get_pp_group
from vllm.lora.layers.base import BaseLayerWithLoRA
from vllm.model_executor.model_loader import get_model
from vllm.model_executor.model_loader.utils import get_draft_load_config
from vllm.model_executor.models.utils import PPMissingLayer
from vllm.v1.worker.gpu.spec_decode.utils import get_pp_safe_draft_load_config
@@ -75,35 +76,10 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
speculative_config = vllm_config.speculative_config
assert speculative_config is not None
draft_model_config = speculative_config.draft_model_config
if speculative_config.moe_backend is not None:
# Otherwise the draft inherits the target's --moe-backend, which
# fails when the draft is unquantized and that backend is not.
vllm_config = replace(
vllm_config,
kernel_config=replace(
vllm_config.kernel_config,
moe_backend=speculative_config.moe_backend,
),
)
if speculative_config.kv_cache_dtype is not None:
vllm_config = replace(
vllm_config,
cache_config=replace(
vllm_config.cache_config,
cache_dtype=speculative_config.kv_cache_dtype,
),
)
if speculative_config.attention_backend is not None:
# Before get_model(): the backend is read off the constructed layers.
# Only when set, so the draft keeps a KV cache layout the target shares.
vllm_config = replace(
vllm_config,
attention_config=replace(
vllm_config.attention_config,
backend=speculative_config.attention_backend,
),
)
draft_load_config = get_pp_safe_draft_load_config(vllm_config.load_config)
vllm_config = speculative_config.apply_draft_overrides(vllm_config)
draft_load_config = get_pp_safe_draft_load_config(
get_draft_load_config(vllm_config)
)
if draft_load_config is not vllm_config.load_config:
vllm_config = replace(vllm_config, load_config=draft_load_config)
with set_model_tag("eagle_head"):
@@ -16,6 +16,7 @@ from vllm.config import VllmConfig, replace
from vllm.distributed.parallel_state import get_pp_group
from vllm.logger import init_logger
from vllm.model_executor.model_loader import get_model
from vllm.model_executor.model_loader.utils import get_draft_load_config
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
AutoRegressiveSpeculator,
)
@@ -40,7 +41,7 @@ class Gemma4Speculator(AutoRegressiveSpeculator):
draft_model = get_model(
vllm_config=draft_vllm_config,
model_config=self.speculative_config.draft_model_config,
load_config=self.speculative_config.draft_load_config,
load_config=get_draft_load_config(draft_vllm_config),
)
self._setup_gemma4_kv_sharing(draft_model, target_attn_layer_names)
self._share_embeddings(draft_model, target_model)