Enable DeepSeek V4 and GLM-5.1 on SM120 (#43477)
Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Yongye Zhu
parent
3ce15fd574
commit
44d95069e9
@@ -60,9 +60,11 @@ from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
|
||||
from vllm.models.deepseek_v4.nvidia.flashinfer_sparse import (
|
||||
DeepseekV4FlashInferMLAAttention,
|
||||
DeepseekV4FlashInferSM120Attention,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.flashmla import DeepseekV4FlashMLAAttention
|
||||
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
@@ -736,14 +738,34 @@ class DeepseekV4MoE(nn.Module):
|
||||
def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
|
||||
"""Pick the CUDA sparse-MLA attention class for the configured backend.
|
||||
|
||||
An explicit ``--attention-backend FLASHINFER_MLA_SPARSE_DSV4`` selects the
|
||||
FlashInfer TRTLLM-gen path; otherwise the FlashMLA path is used.
|
||||
The generic CUDA backend selector does not instantiate DSv4 layers directly,
|
||||
so map generic sparse-MLA choices to the DSv4-specialized attention class.
|
||||
Without an explicit backend, SM12 defaults to FlashInfer while the other
|
||||
CUDA arches keep the FlashMLA path.
|
||||
"""
|
||||
if (
|
||||
vllm_config.attention_config.backend
|
||||
== AttentionBackendEnum.FLASHINFER_MLA_SPARSE_DSV4
|
||||
backend = vllm_config.attention_config.backend
|
||||
device_capability = current_platform.get_device_capability()
|
||||
if backend in (
|
||||
AttentionBackendEnum.FLASHINFER_MLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM120,
|
||||
):
|
||||
raise ValueError(
|
||||
f"{backend.name} is not a DeepSeek V4 attention backend. "
|
||||
"Use FLASHINFER_MLA_SPARSE_DSV4 for DeepSeek V4 FlashInfer "
|
||||
"sparse MLA."
|
||||
)
|
||||
if backend == AttentionBackendEnum.FLASHINFER_MLA_SPARSE_DSV4:
|
||||
if device_capability is not None and device_capability.major == 12:
|
||||
return DeepseekV4FlashInferSM120Attention
|
||||
return DeepseekV4FlashInferMLAAttention
|
||||
if backend in (
|
||||
AttentionBackendEnum.FLASHMLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHMLA_SPARSE_DSV4,
|
||||
):
|
||||
return DeepseekV4FlashMLAAttention
|
||||
|
||||
if device_capability is not None and device_capability.major == 12:
|
||||
return DeepseekV4FlashInferSM120Attention
|
||||
return DeepseekV4FlashMLAAttention
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user