[Fast Start] Cache the MTP draft model in a separate daemon group (#57312)
Signed-off-by: liusy58 <mg21330037@smail.nju.edu.cn> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -4,6 +4,7 @@ import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig, replace
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.model_executor.model_loader.utils import get_draft_load_config
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
|
||||
_should_share,
|
||||
get_target_lm_head,
|
||||
@@ -38,7 +39,7 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
|
||||
if speculative_config.kv_cache_dtype is not None
|
||||
else vllm_config.cache_config
|
||||
),
|
||||
load_config=get_pp_safe_draft_load_config(vllm_config.load_config),
|
||||
load_config=get_pp_safe_draft_load_config(get_draft_load_config(vllm_config)),
|
||||
)
|
||||
with set_model_tag("dflash_head"):
|
||||
dflash_model = get_model(
|
||||
|
||||
@@ -5,6 +5,7 @@ import torch.nn as nn
|
||||
|
||||
from vllm.config import ModelConfig, ParallelConfig, VllmConfig, replace
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader.utils import get_draft_load_config
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.worker.gpu.spec_decode.utils import get_pp_safe_draft_load_config
|
||||
|
||||
@@ -94,7 +95,7 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
|
||||
if speculative_config.kv_cache_dtype is not None
|
||||
else vllm_config.cache_config
|
||||
),
|
||||
load_config=get_pp_safe_draft_load_config(vllm_config.load_config),
|
||||
load_config=get_pp_safe_draft_load_config(get_draft_load_config(vllm_config)),
|
||||
)
|
||||
# VllmConfig post-init restores the target's quant config because the target
|
||||
# config is retained for DSpark's target-layer metadata, so we must override it.
|
||||
|
||||
@@ -7,6 +7,7 @@ from vllm.config import VllmConfig, replace
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.lora.layers.base import BaseLayerWithLoRA
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.model_executor.model_loader.utils import get_draft_load_config
|
||||
from vllm.model_executor.models.utils import PPMissingLayer
|
||||
from vllm.v1.worker.gpu.spec_decode.utils import get_pp_safe_draft_load_config
|
||||
|
||||
@@ -75,35 +76,10 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
draft_model_config = speculative_config.draft_model_config
|
||||
if speculative_config.moe_backend is not None:
|
||||
# Otherwise the draft inherits the target's --moe-backend, which
|
||||
# fails when the draft is unquantized and that backend is not.
|
||||
vllm_config = replace(
|
||||
vllm_config,
|
||||
kernel_config=replace(
|
||||
vllm_config.kernel_config,
|
||||
moe_backend=speculative_config.moe_backend,
|
||||
),
|
||||
)
|
||||
if speculative_config.kv_cache_dtype is not None:
|
||||
vllm_config = replace(
|
||||
vllm_config,
|
||||
cache_config=replace(
|
||||
vllm_config.cache_config,
|
||||
cache_dtype=speculative_config.kv_cache_dtype,
|
||||
),
|
||||
)
|
||||
if speculative_config.attention_backend is not None:
|
||||
# Before get_model(): the backend is read off the constructed layers.
|
||||
# Only when set, so the draft keeps a KV cache layout the target shares.
|
||||
vllm_config = replace(
|
||||
vllm_config,
|
||||
attention_config=replace(
|
||||
vllm_config.attention_config,
|
||||
backend=speculative_config.attention_backend,
|
||||
),
|
||||
)
|
||||
draft_load_config = get_pp_safe_draft_load_config(vllm_config.load_config)
|
||||
vllm_config = speculative_config.apply_draft_overrides(vllm_config)
|
||||
draft_load_config = get_pp_safe_draft_load_config(
|
||||
get_draft_load_config(vllm_config)
|
||||
)
|
||||
if draft_load_config is not vllm_config.load_config:
|
||||
vllm_config = replace(vllm_config, load_config=draft_load_config)
|
||||
with set_model_tag("eagle_head"):
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.config import VllmConfig, replace
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.model_executor.model_loader.utils import get_draft_load_config
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
|
||||
AutoRegressiveSpeculator,
|
||||
)
|
||||
@@ -40,7 +41,7 @@ class Gemma4Speculator(AutoRegressiveSpeculator):
|
||||
draft_model = get_model(
|
||||
vllm_config=draft_vllm_config,
|
||||
model_config=self.speculative_config.draft_model_config,
|
||||
load_config=self.speculative_config.draft_load_config,
|
||||
load_config=get_draft_load_config(draft_vllm_config),
|
||||
)
|
||||
self._setup_gemma4_kv_sharing(draft_model, target_attn_layer_names)
|
||||
self._share_embeddings(draft_model, target_model)
|
||||
|
||||
Reference in New Issue
Block a user