Enable DeepSeek V4 and GLM-5.1 on SM120 (#43477)

Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
Gabriel Wu
2026-06-22 11:54:14 -07:00
committed by GitHub
co-authored by Claude Opus 4.8 Yongye Zhu
parent 3ce15fd574
commit 44d95069e9
37 changed files with 2347 additions and 476 deletions
+46 -15
View File
@@ -11,7 +11,7 @@ import platform
from collections.abc import Callable
from datetime import timedelta
from functools import cache, lru_cache, wraps
from typing import TYPE_CHECKING, TypeVar
from typing import TYPE_CHECKING, NamedTuple, TypeVar
import torch
from torch.distributed import PrefixStore, ProcessGroup
@@ -31,6 +31,7 @@ if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
from vllm.config.kernel import IrOpPriorityConfig
from vllm.v1.attention.backend import AttentionBackend
from vllm.v1.attention.selector import AttentionSelectorConfig
else:
VllmConfig = None
@@ -126,6 +127,11 @@ def _get_backend_priorities(
AttentionBackendEnum.TRITON_MLA,
*sparse_backends,
]
elif device_capability.major == 12:
return [
AttentionBackendEnum.TRITON_MLA,
AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM120,
]
else:
return [
AttentionBackendEnum.FLASH_ATTN_MLA,
@@ -153,6 +159,21 @@ def _get_backend_priorities(
]
def _backend_cls_path(backend_cls: type[AttentionBackend]) -> str:
module, qualname = backend_cls.full_cls_name()
return f"{module}.{qualname}"
def _get_attn_backend_class(backend: AttentionBackendEnum) -> type[AttentionBackend]:
return backend.get_class()
class _BackendCandidate(NamedTuple):
backend_class: type[AttentionBackend]
backend: AttentionBackendEnum
priority: int
def with_nvml_context(fn: Callable[_P, _R]) -> Callable[_P, _R]:
@wraps(fn)
def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R:
@@ -340,7 +361,7 @@ class CudaPlatformBase(Platform):
attn_selector_config: AttentionSelectorConfig,
num_heads: int | None = None,
) -> tuple[
list[tuple[AttentionBackendEnum, int]],
list[_BackendCandidate],
dict[AttentionBackendEnum, tuple[int, list[str]]],
]:
valid_backends_priorities = []
@@ -354,7 +375,7 @@ class CudaPlatformBase(Platform):
)
for priority, backend in enumerate(backend_priorities):
try:
backend_class = backend.get_class()
backend_class = _get_attn_backend_class(backend)
invalid_reasons_i = backend_class.validate_configuration(
device_capability=device_capability,
**attn_selector_config._asdict(),
@@ -364,7 +385,9 @@ class CudaPlatformBase(Platform):
if invalid_reasons_i:
invalid_reasons[backend] = (priority, invalid_reasons_i)
else:
valid_backends_priorities.append((backend, priority))
valid_backends_priorities.append(
_BackendCandidate(backend_class, backend, priority)
)
return valid_backends_priorities, invalid_reasons
@@ -381,7 +404,7 @@ class CudaPlatformBase(Platform):
# First try checking just the selected backend, if there is one.
if selected_backend is not None:
try:
backend_class = selected_backend.get_class()
backend_class = _get_attn_backend_class(selected_backend)
invalid_reasons = backend_class.validate_configuration(
device_capability=device_capability,
**attn_selector_config._asdict(),
@@ -395,7 +418,7 @@ class CudaPlatformBase(Platform):
)
else:
logger.info("Using %s backend.", selected_backend)
return selected_backend.get_path()
return _backend_cls_path(backend_class)
# No selected backend or the selected backend is invalid,
# so we try finding a valid backend.
@@ -425,13 +448,13 @@ class CudaPlatformBase(Platform):
# We have found some valid backends. Select the one with the
# highest priority.
sorted_indices = sorted(
range(len(valid_backends_priorities)),
key=lambda i: valid_backends_priorities[i][1],
selected_candidate = min(
valid_backends_priorities,
key=lambda candidate: candidate.priority,
)
selected_index = sorted_indices[0]
selected_backend = valid_backends_priorities[selected_index][0]
selected_priority = valid_backends_priorities[selected_index][1]
selected_backend_class = selected_candidate.backend_class
selected_backend = selected_candidate.backend
selected_priority = selected_candidate.priority
# If the user specified --block-size (but not --attention-backend),
# check whether that constraint precluded any higher-priority backends.
@@ -457,10 +480,14 @@ class CudaPlatformBase(Platform):
logger.info_once(
"Using %s attention backend out of potential backends: %s.",
selected_backend.name,
"[" + ", ".join(f"'{b[0].name}'" for b in valid_backends_priorities) + "]",
"["
+ ", ".join(
f"'{candidate.backend.name}'" for candidate in valid_backends_priorities
)
+ "]",
)
return selected_backend.get_path()
return _backend_cls_path(selected_backend_class)
@classmethod
def get_supported_vit_attn_backends(cls) -> list[AttentionBackendEnum]:
@@ -635,7 +662,11 @@ class CudaPlatformBase(Platform):
@classmethod
def support_deep_gemm(cls) -> bool:
"""Currently, only Hopper and Blackwell GPUs are supported."""
return cls.is_device_capability(90) or cls.is_device_capability_family(100)
return (
cls.is_device_capability(90)
or cls.is_device_capability_family(100)
or cls.is_device_capability_family(120)
)
@classmethod
def is_integrated_gpu(cls, device_id: int = 0) -> bool: