[Bugfix][Spec Decode] Scope DSpark backend inheritance to DeepSeek V4 (#52809)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Codex
OpenAI Codex
parent
d29f7f5c92
commit
91a893de64
@@ -3,13 +3,32 @@
|
||||
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig, replace
|
||||
from vllm.config import ModelConfig, VllmConfig, replace
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
|
||||
_should_share,
|
||||
get_target_lm_head,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _resolve_dspark_attention_backend(
|
||||
draft_model_config: ModelConfig,
|
||||
draft_backend: AttentionBackendEnum | None,
|
||||
target_backend: AttentionBackendEnum | None,
|
||||
) -> AttentionBackendEnum | None:
|
||||
if draft_backend is not None:
|
||||
return draft_backend
|
||||
# DeepSeek-V4 draft layers share the target's KV-cache layout. Other
|
||||
# DSpark architectures may use a different attention kind.
|
||||
if draft_model_config.hf_config.model_type == "deepseek_v4":
|
||||
if target_backend is not None:
|
||||
logger.info_once(
|
||||
"Using the target model's %s attention backend for the "
|
||||
"DeepSeek-V4 DSpark drafter.",
|
||||
target_backend.name,
|
||||
)
|
||||
return target_backend
|
||||
return None
|
||||
|
||||
|
||||
def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module:
|
||||
@@ -18,13 +37,18 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
|
||||
draft_model_config = speculative_config.draft_model_config
|
||||
|
||||
from vllm.compilation.backends import set_model_tag
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.model_executor.models.qwen3_dflash import dflash_has_any_non_causal
|
||||
from vllm.model_executor.models.utils import get_draft_quant_config
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
|
||||
_should_share,
|
||||
get_target_lm_head,
|
||||
)
|
||||
|
||||
# None re-runs backend auto-selection for the draft, which can pick a
|
||||
# different attention class than the target; fall back to the target's.
|
||||
draft_attention_backend = (
|
||||
speculative_config.attention_backend or vllm_config.attention_config.backend
|
||||
draft_attention_backend = _resolve_dspark_attention_backend(
|
||||
draft_model_config,
|
||||
speculative_config.attention_backend,
|
||||
vllm_config.attention_config.backend,
|
||||
)
|
||||
|
||||
draft_vllm_config = replace(
|
||||
|
||||
Reference in New Issue
Block a user