Enable DeepSeek V4 and GLM-5.1 on SM120 (#43477)

Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
Gabriel Wu
2026-06-22 11:54:14 -07:00
committed by GitHub
co-authored by Claude Opus 4.8 Yongye Zhu
parent 3ce15fd574
commit 44d95069e9
37 changed files with 2347 additions and 476 deletions
+27 -5
View File
@@ -60,9 +60,11 @@ from vllm.model_executor.utils import set_weight_attrs
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
from vllm.models.deepseek_v4.nvidia.flashinfer_sparse import (
DeepseekV4FlashInferMLAAttention,
DeepseekV4FlashInferSM120Attention,
)
from vllm.models.deepseek_v4.nvidia.flashmla import DeepseekV4FlashMLAAttention
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -736,14 +738,34 @@ class DeepseekV4MoE(nn.Module):
def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
"""Pick the CUDA sparse-MLA attention class for the configured backend.
An explicit ``--attention-backend FLASHINFER_MLA_SPARSE_DSV4`` selects the
FlashInfer TRTLLM-gen path; otherwise the FlashMLA path is used.
The generic CUDA backend selector does not instantiate DSv4 layers directly,
so map generic sparse-MLA choices to the DSv4-specialized attention class.
Without an explicit backend, SM12 defaults to FlashInfer while the other
CUDA arches keep the FlashMLA path.
"""
if (
vllm_config.attention_config.backend
== AttentionBackendEnum.FLASHINFER_MLA_SPARSE_DSV4
backend = vllm_config.attention_config.backend
device_capability = current_platform.get_device_capability()
if backend in (
AttentionBackendEnum.FLASHINFER_MLA_SPARSE,
AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM120,
):
raise ValueError(
f"{backend.name} is not a DeepSeek V4 attention backend. "
"Use FLASHINFER_MLA_SPARSE_DSV4 for DeepSeek V4 FlashInfer "
"sparse MLA."
)
if backend == AttentionBackendEnum.FLASHINFER_MLA_SPARSE_DSV4:
if device_capability is not None and device_capability.major == 12:
return DeepseekV4FlashInferSM120Attention
return DeepseekV4FlashInferMLAAttention
if backend in (
AttentionBackendEnum.FLASHMLA_SPARSE,
AttentionBackendEnum.FLASHMLA_SPARSE_DSV4,
):
return DeepseekV4FlashMLAAttention
if device_capability is not None and device_capability.major == 12:
return DeepseekV4FlashInferSM120Attention
return DeepseekV4FlashMLAAttention