[Fast Start] Cache the MTP draft model in a separate daemon group (#57312)

Signed-off-by: liusy58 <mg21330037@smail.nju.edu.cn>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
siyu
2026-09-22 03:13:38 +00:00
committed by GitHub
co-authored by Isotr0py
parent f07e227ee9
commit c723a831a8
16 changed files with 417 additions and 140 deletions
@@ -30,10 +30,19 @@ DAEMON_TIMEOUT_S = 600
class WeightCacheDaemon:
"""Context manager running the real weight cache daemon as a subprocess."""
def __init__(self, model: str, tp_size: int, extra_args: list[str] | None = None):
def __init__(
self,
model: str,
tp_size: int,
extra_args: list[str] | None = None,
num_groups: int = 1,
):
# Short base path: Unix socket paths are limited to ~107 characters.
self.socket_dir = tempfile.mkdtemp(prefix="vllm_ipc_")
self.tp_size = tp_size
# Each daemon group (target plus cached drafts) binds one socket per
# local rank.
self.num_sockets = tp_size * num_groups
self._cmd = [
sys.executable,
"-m",
@@ -87,7 +96,7 @@ class WeightCacheDaemon:
f"Weight cache daemon exited with {self._proc.returncode}:\n"
f"{self._logs()}"
)
if len(glob.glob(pattern)) >= self.tp_size:
if len(glob.glob(pattern)) >= self.num_sockets:
return
time.sleep(1.0)
raise TimeoutError(
@@ -112,6 +121,9 @@ class ModelCase:
images: list | None = None
llm_kwargs: dict[str, Any] = field(default_factory=dict)
daemon_args: list[str] = field(default_factory=list)
# Daemon groups the launcher starts; 2 when a cached draft group joins the
# target group (MTP/EAGLE speculative decoding).
daemon_groups: int = 1
def generate(
@@ -172,14 +184,41 @@ K3_CASE = ModelCase(
daemon_args=["--trust-remote-code"],
)
# Qwen3.5-0.8B ships one MTP layer in the target checkpoint, so method="mtp"
# loads the draft from the same model. The daemon must cache it in its draft
# group for the warm runs (fallback=False) to succeed.
QWEN_MTP_CASE = ModelCase(
model="Qwen/Qwen3.5-0.8B",
prompts=[
"Hello, my name is",
"The capital of France is",
],
llm_kwargs=dict(
gpu_memory_utilization=0.3,
enforce_eager=True,
enable_chunked_prefill=True,
speculative_config={"method": "mtp", "num_speculative_tokens": 1},
),
daemon_args=[
"--speculative-config",
'{"method": "mtp", "num_speculative_tokens": 1}',
],
daemon_groups=2,
)
@pytest.mark.parametrize("case", [QWEN_CASE, K3_CASE], ids=["qwen3.5", "kimi-k3"])
@pytest.mark.parametrize(
"case",
[QWEN_CASE, K3_CASE, QWEN_MTP_CASE],
ids=["qwen3.5", "kimi-k3", "qwen3.5-mtp"],
)
def test_ipc_cache_cold_start_and_warm_restart(vllm_runner, case: ModelCase):
"""Cold start falls back to disk; warm restarts load weights via CUDA IPC.
All runs must produce outputs identical to a default-loader baseline. The
warm runs disable the disk fallback, so they only pass if the weights
really came from the daemon.
really came from the daemon — for the MTP case, both the target's and the
draft's daemon groups.
"""
if not current_platform.is_cuda_alike():
pytest.skip("Weight cache IPC sharing requires CUDA or ROCm")
@@ -199,7 +238,12 @@ def test_ipc_cache_cold_start_and_warm_restart(vllm_runner, case: ModelCase):
with tempfile.TemporaryDirectory(prefix="vllm_ipc_empty_") as empty_socket_dir:
cold_outputs = generate(vllm_runner, case, empty_socket_dir, fallback=True)
with WeightCacheDaemon(case.model, tp_size=1, extra_args=case.daemon_args) as d:
with WeightCacheDaemon(
case.model,
tp_size=1,
extra_args=case.daemon_args,
num_groups=case.daemon_groups,
) as d:
warm_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
# Warm restart: a second engine lifetime against the same daemon.
restart_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
+2 -1
View File
@@ -369,11 +369,12 @@ def test_dspark_shares_target_embedding_with_smaller_draft_vocabulary():
draft_parallel_config=SimpleNamespace(tensor_parallel_size=1),
attention_backend=None,
kv_cache_dtype=None,
draft_load_config=None,
),
parallel_config=ParallelConfig(),
attention_config=SimpleNamespace(backend=None),
cache_config=SimpleNamespace(),
load_config=SimpleNamespace(),
load_config=SimpleNamespace(load_format="auto"),
model_config=SimpleNamespace(get_vocab_size=Mock(return_value=100)),
)