[Fast Start] Cache the MTP draft model in a separate daemon group (#57312)

Signed-off-by: liusy58 <mg21330037@smail.nju.edu.cn>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
siyu
2026-09-22 03:13:38 +00:00
committed by GitHub
co-authored by Isotr0py
parent f07e227ee9
commit c723a831a8
16 changed files with 417 additions and 140 deletions
@@ -4,6 +4,7 @@ import torch.nn as nn
from vllm.config import VllmConfig, replace
from vllm.model_executor.model_loader import get_model
from vllm.model_executor.model_loader.utils import get_draft_load_config
from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
_should_share,
get_target_lm_head,
@@ -38,7 +39,7 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
if speculative_config.kv_cache_dtype is not None
else vllm_config.cache_config
),
load_config=get_pp_safe_draft_load_config(vllm_config.load_config),
load_config=get_pp_safe_draft_load_config(get_draft_load_config(vllm_config)),
)
with set_model_tag("dflash_head"):
dflash_model = get_model(
@@ -5,6 +5,7 @@ import torch.nn as nn
from vllm.config import ModelConfig, ParallelConfig, VllmConfig, replace
from vllm.logger import init_logger
from vllm.model_executor.model_loader.utils import get_draft_load_config
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.worker.gpu.spec_decode.utils import get_pp_safe_draft_load_config
@@ -94,7 +95,7 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
if speculative_config.kv_cache_dtype is not None
else vllm_config.cache_config
),
load_config=get_pp_safe_draft_load_config(vllm_config.load_config),
load_config=get_pp_safe_draft_load_config(get_draft_load_config(vllm_config)),
)
# VllmConfig post-init restores the target's quant config because the target
# config is retained for DSpark's target-layer metadata, so we must override it.
+5 -29
View File
@@ -7,6 +7,7 @@ from vllm.config import VllmConfig, replace
from vllm.distributed.parallel_state import get_pp_group
from vllm.lora.layers.base import BaseLayerWithLoRA
from vllm.model_executor.model_loader import get_model
from vllm.model_executor.model_loader.utils import get_draft_load_config
from vllm.model_executor.models.utils import PPMissingLayer
from vllm.v1.worker.gpu.spec_decode.utils import get_pp_safe_draft_load_config
@@ -75,35 +76,10 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
speculative_config = vllm_config.speculative_config
assert speculative_config is not None
draft_model_config = speculative_config.draft_model_config
if speculative_config.moe_backend is not None:
# Otherwise the draft inherits the target's --moe-backend, which
# fails when the draft is unquantized and that backend is not.
vllm_config = replace(
vllm_config,
kernel_config=replace(
vllm_config.kernel_config,
moe_backend=speculative_config.moe_backend,
),
)
if speculative_config.kv_cache_dtype is not None:
vllm_config = replace(
vllm_config,
cache_config=replace(
vllm_config.cache_config,
cache_dtype=speculative_config.kv_cache_dtype,
),
)
if speculative_config.attention_backend is not None:
# Before get_model(): the backend is read off the constructed layers.
# Only when set, so the draft keeps a KV cache layout the target shares.
vllm_config = replace(
vllm_config,
attention_config=replace(
vllm_config.attention_config,
backend=speculative_config.attention_backend,
),
)
draft_load_config = get_pp_safe_draft_load_config(vllm_config.load_config)
vllm_config = speculative_config.apply_draft_overrides(vllm_config)
draft_load_config = get_pp_safe_draft_load_config(
get_draft_load_config(vllm_config)
)
if draft_load_config is not vllm_config.load_config:
vllm_config = replace(vllm_config, load_config=draft_load_config)
with set_model_tag("eagle_head"):
@@ -16,6 +16,7 @@ from vllm.config import VllmConfig, replace
from vllm.distributed.parallel_state import get_pp_group
from vllm.logger import init_logger
from vllm.model_executor.model_loader import get_model
from vllm.model_executor.model_loader.utils import get_draft_load_config
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
AutoRegressiveSpeculator,
)
@@ -40,7 +41,7 @@ class Gemma4Speculator(AutoRegressiveSpeculator):
draft_model = get_model(
vllm_config=draft_vllm_config,
model_config=self.speculative_config.draft_model_config,
load_config=self.speculative_config.draft_load_config,
load_config=get_draft_load_config(draft_vllm_config),
)
self._setup_gemma4_kv_sharing(draft_model, target_attn_layer_names)
self._share_embeddings(draft_model, target_model)