[Fast Start] Cache the MTP draft model in a separate daemon group (#57312)

Signed-off-by: liusy58 <mg21330037@smail.nju.edu.cn>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
siyu
2026-09-22 03:13:38 +00:00
committed by GitHub
co-authored by Isotr0py
parent f07e227ee9
commit c723a831a8
16 changed files with 417 additions and 140 deletions
@@ -1,26 +1,3 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.model_executor.model_loader.weight_cache.ipc_loader import IpcModelLoader
from vllm.model_executor.model_loader.weight_cache.protocol import (
CacheConfigMismatchError,
TensorEntry,
UnsupportedPlatformForIPCError,
UnsupportedQuantForIPCError,
WeightCacheKey,
WeightCacheUnavailableError,
check_ipc_platform_support,
check_ipc_quant_support,
)
__all__ = [
"CacheConfigMismatchError",
"IpcModelLoader",
"TensorEntry",
"UnsupportedPlatformForIPCError",
"UnsupportedQuantForIPCError",
"WeightCacheKey",
"WeightCacheUnavailableError",
"check_ipc_platform_support",
"check_ipc_quant_support",
]
@@ -41,6 +41,13 @@ The global TP rank of local GPU ``i`` on node ``r`` is
``r * (tp_size // nnodes) + i``, matching vLLM's contiguous per-node rank
assignment, so each engine worker maps its shard from the daemon on its own
node.
With MTP, EAGLE or EAGLE3 speculative decoding the launcher additionally
starts a draft daemon group that caches the draft model. It uses its own cache
key, Unix sockets (``*_draft.sock``) and rendezvous port
(``--weight-cache-draft-master-port``, default ``--weight-cache-master-port +
1``), so each process serves exactly one model role. Other draft types are not
cached and keep loading from disk in the engine.
"""
import contextlib
@@ -52,10 +59,15 @@ import signal
import socket
import sys
from collections.abc import Callable
from itertools import product
import torch
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
from vllm.config import (
ParallelConfig,
VllmConfig,
set_current_vllm_config,
)
from vllm.distributed import (
ensure_model_parallel_initialized,
init_distributed_environment,
@@ -77,6 +89,11 @@ from vllm.model_executor.model_loader.weight_cache.protocol import (
send_msg,
verify_peer_is_owner,
)
from vllm.model_executor.model_loader.weight_cache.utils import (
export_model_attrs,
format_daemon_role,
is_draft_model_cacheable,
)
from vllm.platforms import current_platform
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.network_utils import get_distributed_init_method, get_open_port
@@ -130,30 +147,6 @@ def export_entries(
return entries, aliases
def get_daemon_model(vllm_config: VllmConfig) -> torch.nn.Module:
"""Load the daemon's model, composed from the configured loader.
Runs the quantization check after model creation but before the slow
weight load, so an unsupported method fails fast. Online quantization
always fails the check, so load_model's finalize step for it is
unnecessary here.
"""
model_config = vllm_config.model_config
load_config = vllm_config.load_config
loader = get_model_loader(load_config)
device_config = vllm_config.device_config
target_device = torch.device(
device_config.device if load_config.device is None else load_config.device
)
with set_default_torch_dtype(model_config.dtype):
with target_device:
model = loader.create_model(vllm_config, model_config)
check_ipc_quant_support(model)
loader.load_weights(model, model_config)
process_weights_after_loading(model, model_config, target_device)
return model.eval()
class WeightCacheDaemon:
"""Per-GPU process that loads one TP shard and serves CUDA IPC handles."""
@@ -164,19 +157,31 @@ class WeightCacheDaemon:
local_rank: int,
distributed_init_method: str,
socket_dir: str | None = None,
is_draft: bool = False,
):
self.vllm_config = vllm_config
# A draft daemon builds the draft with the target's VllmConfig, like
# the engine does; only the ModelConfig differs.
if is_draft:
spec = vllm_config.speculative_config
assert spec is not None and spec.draft_model_config is not None
self.model_config = spec.draft_model_config
else:
self.model_config = vllm_config.model_config
self.tp_rank = tp_rank
self.local_rank = local_rank
self.distributed_init_method = distributed_init_method
self.socket_dir = socket_dir
self.is_draft = is_draft
self.role = format_daemon_role(is_draft)
self.model: torch.nn.Module | None = None
# Fingerprint before loading: process_weights_after_loading may
# mutate hf_config.quantization_config.
self.cache_config = WeightCacheKey.from_model_config(
vllm_config.model_config,
self.model_config,
tp_size=vllm_config.parallel_config.tensor_parallel_size,
tp_rank=tp_rank,
is_draft=is_draft,
)
def load_model(self) -> None:
@@ -191,12 +196,37 @@ class WeightCacheDaemon:
)
with set_current_vllm_config(self.vllm_config):
ensure_model_parallel_initialized(tp_size, 1)
self.model = get_daemon_model(self.vllm_config)
self.model = self.get_model()
logger.info(
"Weight cache daemon rank %d loaded model",
"Weight cache %s daemon rank %d loaded model",
self.role,
self.tp_rank,
)
def get_model(self) -> torch.nn.Module:
"""Load the daemon's model, composed from the configured loader.
Runs the quantization check after model creation but before the slow
weight load, so an unsupported method fails fast. Online quantization
always fails the check, so load_model's finalize step for it is
unnecessary here.
"""
vllm_config = self.vllm_config
model_config = self.model_config
load_config = vllm_config.load_config
loader = get_model_loader(load_config)
device_config = vllm_config.device_config
target_device = torch.device(
device_config.device if load_config.device is None else load_config.device
)
with set_default_torch_dtype(model_config.dtype):
with target_device:
model = loader.create_model(vllm_config, model_config)
check_ipc_quant_support(model)
loader.load_weights(model, model_config)
process_weights_after_loading(model, model_config, target_device)
return model.eval()
def serve_forever(self, ready_callback: Callable[[], None] | None = None) -> None:
"""Serve requests until terminated.
@@ -223,7 +253,10 @@ class WeightCacheDaemon:
os.chmod(socket_path, 0o600)
server.listen()
logger.info(
"Weight cache daemon rank %d serving on %s", self.tp_rank, socket_path
"Weight cache %s daemon rank %d serving on %s",
self.role,
self.tp_rank,
socket_path,
)
if ready_callback is not None:
ready_callback()
@@ -270,7 +303,11 @@ class WeightCacheDaemon:
@property
def _socket_path(self) -> str:
return get_socket_path(get_current_device_uuid(), self.socket_dir)
return get_socket_path(
get_current_device_uuid(),
self.socket_dir,
is_draft=self.is_draft,
)
def _handle_connection(self, conn: socket.socket) -> None:
request = recv_msg(conn)
@@ -303,11 +340,13 @@ class WeightCacheDaemon:
"status": "ok",
"entries": entries,
"aliases": aliases,
"attrs": export_model_attrs(self.model),
"gpu_uuid": gpu_uuid,
},
)
logger.info_once(
"Weight cache daemon rank %d sent %d tensors (+%d aliases) to engine",
"Weight cache %s daemon rank %d sent %d tensors (+%d aliases) to engine",
self.role,
self.tp_rank,
len(entries),
len(aliases),
@@ -316,7 +355,11 @@ class WeightCacheDaemon:
def _handle_release(self, conn: socket.socket) -> None:
self.model = None
torch.accelerator.empty_cache()
logger.info("Weight cache daemon rank %d released cached weights", self.tp_rank)
logger.info(
"Weight cache %s daemon rank %d released cached weights",
self.role,
self.tp_rank,
)
send_msg(conn, {"status": "ok"})
@@ -326,13 +369,27 @@ def _run_daemon(
vllm_config: VllmConfig,
distributed_init_method: str,
socket_dir: str | None,
ready_queue: "multiprocessing.Queue[int]",
ready_queue: "multiprocessing.Queue[tuple[str, int]]",
is_draft: bool = False,
) -> None:
daemon = WeightCacheDaemon(
vllm_config, tp_rank, local_rank, distributed_init_method, socket_dir
vllm_config,
tp_rank,
local_rank,
distributed_init_method,
socket_dir,
is_draft,
)
daemon.load_model()
daemon.serve_forever(ready_callback=lambda: ready_queue.put(tp_rank))
daemon.serve_forever(ready_callback=lambda: ready_queue.put((daemon.role, tp_rank)))
def get_draft_daemon_config(vllm_config: VllmConfig) -> VllmConfig | None:
"""VllmConfig for the draft daemon group"""
if not is_draft_model_cacheable(vllm_config.speculative_config):
return None
assert vllm_config.speculative_config is not None
return vllm_config.speculative_config.apply_draft_overrides(vllm_config)
def _reject_unsupported_parallelism(parallel_config: ParallelConfig) -> None:
@@ -369,6 +426,14 @@ def main() -> None:
"serving) and match across nodes. Required when --nnodes > 1; defaults "
"to a free port for single-node.",
)
parser.add_argument(
"--weight-cache-draft-master-port",
type=int,
default=None,
help="Rendezvous port for the MTP draft daemon group. Defaults to "
"--weight-cache-master-port + 1 for multi-node, or a free port for "
"single-node.",
)
args = parser.parse_args()
engine_args = EngineArgs.from_cli_args(args)
vllm_config = engine_args.create_engine_config()
@@ -398,27 +463,56 @@ def main() -> None:
master_port = args.weight_cache_master_port or get_open_port()
distributed_init_method = get_distributed_init_method(master_addr, master_port)
# (is_draft, vllm_config, rendezvous) per daemon group.
groups: list[tuple[bool, VllmConfig, str]] = [
(False, vllm_config, distributed_init_method)
]
draft_vllm_config = get_draft_daemon_config(vllm_config)
if draft_vllm_config is not None:
draft_master_port = args.weight_cache_draft_master_port or (
master_port + 1 if nnodes > 1 else get_open_port()
)
if draft_master_port == master_port:
raise ValueError(
"--weight-cache-draft-master-port must differ from "
"--weight-cache-master-port"
)
groups.append(
(
True,
draft_vllm_config,
get_distributed_init_method(master_addr, draft_master_port),
)
)
ctx = multiprocessing.get_context("spawn")
ready_queue: multiprocessing.Queue[int] = ctx.Queue()
ready_queue: multiprocessing.Queue[tuple[str, int]] = ctx.Queue()
# Global TP rank of local GPU i on this node; local index == device index.
global_ranks = [
node_rank * local_world_size + local_rank
for local_rank in range(local_world_size)
]
expected_ready = {
(format_daemon_role(is_draft), global_rank)
for (is_draft, _, _), global_rank in product(groups, global_ranks)
}
procs = [
ctx.Process(
target=_run_daemon,
args=(
global_rank,
local_rank,
vllm_config,
distributed_init_method,
config,
init_method,
args.weight_cache_socket_dir,
ready_queue,
is_draft,
),
name=f"vllm-weight-cache-daemon-{global_rank}",
name=f"vllm-weight-cache-{format_daemon_role(is_draft)}-{global_rank}",
)
for (is_draft, config, init_method), (local_rank, global_rank) in product(
groups, enumerate(global_ranks)
)
for local_rank, global_rank in enumerate(global_ranks)
]
for proc in procs:
proc.start()
@@ -430,10 +524,10 @@ def main() -> None:
signal.signal(signal.SIGINT, _shutdown)
signal.signal(signal.SIGTERM, _shutdown)
ready_ranks: set[int] = set()
while len(ready_ranks) < local_world_size:
ready: set[tuple[str, int]] = set()
while len(ready) < len(expected_ready):
try:
ready_ranks.add(ready_queue.get(timeout=1.0))
ready.add(ready_queue.get(timeout=1.0))
except queue.Empty:
dead = [p for p in procs if p.exitcode is not None]
if dead:
@@ -450,10 +544,11 @@ def main() -> None:
socket_dir_msg = args.weight_cache_socket_dir or "the default socket dir"
logger.info_once(
"===== Weight cache daemon READY: node %d/%d serving %d local rank(s) "
"in %s =====",
"x %d role(s) in %s =====",
node_rank,
nnodes,
local_world_size,
len(groups),
socket_dir_msg,
)
@@ -25,9 +25,9 @@ from vllm.model_executor.model_loader.utils import (
)
from vllm.model_executor.model_loader.weight_cache.protocol import (
CacheConfigMismatchError,
TensorEntry,
UnsupportedQuantForIPCError,
WeightCacheKey,
WeightCacheState,
WeightCacheUnavailableError,
check_ipc_platform_support,
check_ipc_quant_support,
@@ -60,7 +60,7 @@ class IpcModelLoader(BaseModelLoader):
Extra config keys (via --model-loader-extra-config):
- socket_path: explicit daemon socket path. Defaults to a per-GPU path
derived from the physical GPU uuid.
derived from the physical GPU uuid and the cache role (target/draft).
- socket_dir: directory containing the daemon sockets.
- mode: "zero_copy" (default) or "copy".
- fallback: fall back to disk loading when the daemon is unavailable or
@@ -78,6 +78,15 @@ class IpcModelLoader(BaseModelLoader):
extra_config = copy(load_config.model_loader_extra_config or {})
self.socket_path: str | None = extra_config.pop("socket_path", None)
self.socket_dir: str | None = extra_config.pop("socket_dir", None)
# Internal: set by the engine when routing a speculative draft to the
# daemon's draft group.
self.is_draft = bool(extra_config.pop("is_draft", False))
if self.is_draft and self.socket_path is not None:
raise ValueError(
"socket_path cannot be combined with the draft weight cache role; "
"use socket_dir so the target and draft sockets are derived "
"independently"
)
self.mode: str = extra_config.pop("mode", "zero_copy")
self.fallback: bool = extra_config.pop("fallback", True)
self.connect_timeout_s: float = float(
@@ -108,7 +117,7 @@ class IpcModelLoader(BaseModelLoader):
loaded through this loader).
"""
device_index = torch.accelerator.current_device_index()
entries, _ = self._fetch_entries(model_config)
entries = self._fetch_entries(model_config).entries
params = dict(model.named_parameters())
buffers = dict(model.named_buffers())
for name, entry in entries.items():
@@ -128,11 +137,22 @@ class IpcModelLoader(BaseModelLoader):
check_ipc_platform_support()
state_fetched = False
try:
entries, aliases = self._fetch_entries(model_config)
# Cross-check the routing flag against the identity of the model
# being loaded: a draft load that lost its flag (or a target load
# that got one) would hit the wrong daemon group and
# fingerprint-mismatch.
spec = vllm_config.speculative_config
inferred = spec is not None and model_config is spec.draft_model_config
if inferred != self.is_draft:
raise CacheConfigMismatchError(
f"Weight cache role mismatch: loading "
f"{'draft' if inferred else 'target'} model but the loader "
f"was configured for the "
f"{'draft' if self.is_draft else 'target'} group"
)
state = self._fetch_entries(model_config)
state_fetched = True
return self._build_model(
vllm_config, model_config, prefix, entries, aliases
)
return self._build_model(vllm_config, model_config, prefix, state)
except (WeightCacheUnavailableError, CacheConfigMismatchError) as e:
if not self.fallback:
raise
@@ -163,8 +183,7 @@ class IpcModelLoader(BaseModelLoader):
vllm_config: VllmConfig,
model_config: ModelConfig,
prefix: str,
entries: dict[str, TensorEntry],
aliases: dict[str, str],
state: WeightCacheState,
) -> nn.Module:
device_config = vllm_config.device_config
load_device = (
@@ -186,7 +205,11 @@ class IpcModelLoader(BaseModelLoader):
prefix=prefix,
)
check_ipc_quant_support(model)
self._apply_entries(model, entries, aliases, device_index)
self._apply_entries(model, state, device_index)
# Flags that load_weights would have set (e.g. EAGLE ownership of
# embed_tokens / lm_head); the daemon ran it, this process did not.
for name, value in state.attrs.items():
setattr(model, name, value)
# The daemon exports tensors that already went through
# process_weights_after_loading; re-run it in pre-processed mode
# so quant methods only rebuild Python-side state (e.g. the MoE
@@ -202,7 +225,7 @@ class IpcModelLoader(BaseModelLoader):
self._send_release()
logger.info(
"Mapped %d tensors from the weight cache daemon (%s mode)",
len(entries),
len(state.entries),
self.mode,
)
return model.eval()
@@ -210,8 +233,7 @@ class IpcModelLoader(BaseModelLoader):
def _apply_entries(
self,
model: nn.Module,
entries: dict[str, TensorEntry],
aliases: dict[str, str],
state: WeightCacheState,
device_index: int,
) -> None:
# remove_duplicate=False keeps tied module aliases reachable by name:
@@ -242,7 +264,7 @@ class IpcModelLoader(BaseModelLoader):
module.register_buffer(leaf, obj)
registered[name] = obj
for name, entry in entries.items():
for name, entry in state.entries.items():
tensor = entry.rebuild(device_index)
if self.mode == "copy":
tensor = tensor.clone()
@@ -251,7 +273,7 @@ class IpcModelLoader(BaseModelLoader):
# Re-establish tied-weight aliases by registering the *same* object the
# canonical name resolved to, so parameter identity (and the tie) is
# preserved instead of allocating uninitialized memory.
for alias_name, canonical_name in aliases.items():
for alias_name, canonical_name in state.aliases.items():
obj = registered.get(canonical_name)
if obj is None:
logger.warning(
@@ -262,19 +284,16 @@ class IpcModelLoader(BaseModelLoader):
continue
_register(alias_name, obj, isinstance(obj, nn.Parameter))
def _fetch_entries(
self, model_config: ModelConfig
) -> tuple[dict[str, TensorEntry], dict[str, str]]:
def _fetch_entries(self, model_config: ModelConfig) -> WeightCacheState:
cache_config = WeightCacheKey.from_model_config(
model_config,
tp_size=get_tensor_model_parallel_world_size(),
tp_rank=get_tensor_model_parallel_rank(),
is_draft=self.is_draft,
)
return self._request_state(cache_config)
def _request_state(
self, cache_config: WeightCacheKey
) -> tuple[dict[str, TensorEntry], dict[str, str]]:
def _request_state(self, cache_config: WeightCacheKey) -> WeightCacheState:
with self._connect(self.state_timeout_s) as conn:
send_msg(conn, {"cmd": "get_state", "cache_config": cache_config})
response = recv_msg(conn)
@@ -288,7 +307,11 @@ class IpcModelLoader(BaseModelLoader):
f"Weight cache daemon error: {response.get('message')}"
)
self._check_gpu_uuid(response.get("gpu_uuid"))
return response["entries"], response.get("aliases", {})
return WeightCacheState(
entries=response["entries"],
aliases=response.get("aliases", {}),
attrs=response.get("attrs", {}),
)
def _connect(self, timeout: float) -> socket.socket:
socket_path = self._resolve_socket_path()
@@ -316,7 +339,11 @@ class IpcModelLoader(BaseModelLoader):
def _resolve_socket_path(self) -> str:
if self.socket_path is not None:
return self.socket_path
return get_socket_path(get_current_device_uuid(), self.socket_dir)
return get_socket_path(
get_current_device_uuid(),
self.socket_dir,
is_draft=self.is_draft,
)
def _check_gpu_uuid(self, daemon_uuid: str | None) -> None:
if daemon_uuid is None:
@@ -340,7 +367,9 @@ class IpcModelLoader(BaseModelLoader):
# DefaultModelLoader must not see load_format="ipc_cache" or the ipc
# extra config keys.
return dataclasses.replace(
self.load_config, load_format="auto", model_loader_extra_config={}
self.load_config,
load_format="auto",
model_loader_extra_config={},
)
def _fallback_load(
@@ -20,7 +20,7 @@ import stat
import struct
import tempfile
from dataclasses import dataclass, fields
from typing import Any
from typing import Any, NamedTuple
import torch
from torch.multiprocessing.reductions import rebuild_cuda_tensor, reduce_tensor
@@ -29,13 +29,16 @@ from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
import vllm.version
from vllm.config import ModelConfig
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
from vllm.model_executor.model_loader.weight_cache.utils import (
format_socket_role_suffix,
)
from vllm.model_executor.model_loader.weight_utils import (
filter_duplicate_safetensors_files,
)
from vllm.platforms import current_platform
from vllm.utils.hashing import safe_hash
SOCKET_NAME_TEMPLATE = "vllm_weight_cache_{gpu_uuid}.sock"
SOCKET_NAME_TEMPLATE = "vllm_weight_cache_{gpu_uuid}{role}.sock"
SOCKET_DIR_TEMPLATE = "vllm_weight_cache_{uid}"
_LEN_STRUCT = struct.Struct("!Q")
@@ -128,9 +131,23 @@ def get_socket_dir(socket_dir: str | None = None) -> str:
)
def get_socket_path(gpu_uuid: str, socket_dir: str | None = None) -> str:
directory = get_socket_dir(socket_dir)
return os.path.join(directory, SOCKET_NAME_TEMPLATE.format(gpu_uuid=gpu_uuid))
def get_socket_path(
gpu_uuid: str,
socket_dir: str | None = None,
*,
is_draft: bool = False,
) -> str:
"""Socket path of a daemon group; ``is_draft=False`` is the target.
The GPU uuid is hashed to keep the name well under the AF_UNIX path
limit (~108 bytes) even with the draft role suffix.
"""
gpu_id = safe_hash(gpu_uuid.encode()).hexdigest()
name = SOCKET_NAME_TEMPLATE.format(
gpu_uuid=gpu_id,
role=format_socket_role_suffix(is_draft),
)
return os.path.join(get_socket_dir(socket_dir), name)
def ensure_private_socket_dir(directory: str, strict_perms: bool = True) -> None:
@@ -272,10 +289,17 @@ class WeightCacheKey:
quant_config_hash: str
revision: str | None
vllm_version: str
is_draft: bool = False
"""Daemon group the weights come from; False is the target model."""
@classmethod
def from_model_config(
cls, model_config: ModelConfig, tp_size: int, tp_rank: int
cls,
model_config: ModelConfig,
tp_size: int,
tp_rank: int,
*,
is_draft: bool = False,
) -> "WeightCacheKey":
"""Build the fingerprint for a model configuration.
@@ -302,6 +326,7 @@ class WeightCacheKey:
quant_config_hash=_hash_quant_config(quant_config),
revision=model_config.revision,
vllm_version=vllm.version.__version__,
is_draft=is_draft,
)
def mismatched_fields(self, other: "WeightCacheKey") -> list[str]:
@@ -345,6 +370,17 @@ class TensorEntry:
return rebuild_cuda_tensor(*args)
class WeightCacheState(NamedTuple):
"""Client-side decode of a daemon's get_state response payload."""
entries: dict[str, TensorEntry]
"""Model tensors, exported as CUDA IPC handles or shipped by value."""
aliases: dict[str, str]
"""Duplicate (tied) weight names aliased to their canonical entry."""
attrs: dict[str, bool]
"""Python-side flags set by load_weights, e.g. EAGLE ownership flags."""
def send_msg(sock: socket.socket, obj: Any) -> None:
payload = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL)
sock.sendall(_LEN_STRUCT.pack(len(payload)))
@@ -0,0 +1,48 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Helpers shared by the weight cache daemon, the IPC loader and the engine.
These live outside ``protocol`` so callers that only need to know whether a
draft is cached, or how a daemon group is named, do not have to import the
wire format.
"""
from typing import Any
from vllm.config import SpeculativeConfig
from vllm.model_executor.models.interfaces import SupportsEagleBase
# Speculative methods whose draft model the daemon caches in its own group.
# Other drafts keep loading from disk in the engine.
WEIGHT_CACHE_DRAFT_METHODS = frozenset({"mtp", "eagle", "eagle3"})
# Python-side flags that weight loading sets on EAGLE-style drafts; the engine
# never runs load_weights for cached models, so the daemon ships them.
EXPORTED_MODEL_ATTRS = tuple(SupportsEagleBase.__annotations__)
def export_model_attrs(model: Any) -> dict[str, bool]:
return {
name: bool(getattr(model, name))
for name in EXPORTED_MODEL_ATTRS
if hasattr(model, name)
}
def is_draft_model_cacheable(speculative_config: SpeculativeConfig | None) -> bool:
"""Whether the daemon serves the speculative draft as a separate role."""
return (
speculative_config is not None
and speculative_config.method in WEIGHT_CACHE_DRAFT_METHODS
and speculative_config.draft_model_config is not None
)
def format_daemon_role(is_draft: bool) -> str:
"""Name of a daemon group: the target model or the draft."""
return "draft" if is_draft else "target"
def format_socket_role_suffix(is_draft: bool) -> str:
"""Socket-name suffix keeping the draft group distinct from the target."""
return "_draft" if is_draft else ""