Enable DeepSeek V4 and GLM-5.1 on SM120 (#43477)
Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Yongye Zhu
parent
3ce15fd574
commit
44d95069e9
+46
-15
@@ -11,7 +11,7 @@ import platform
|
||||
from collections.abc import Callable
|
||||
from datetime import timedelta
|
||||
from functools import cache, lru_cache, wraps
|
||||
from typing import TYPE_CHECKING, TypeVar
|
||||
from typing import TYPE_CHECKING, NamedTuple, TypeVar
|
||||
|
||||
import torch
|
||||
from torch.distributed import PrefixStore, ProcessGroup
|
||||
@@ -31,6 +31,7 @@ if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.config.kernel import IrOpPriorityConfig
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
from vllm.v1.attention.selector import AttentionSelectorConfig
|
||||
else:
|
||||
VllmConfig = None
|
||||
@@ -126,6 +127,11 @@ def _get_backend_priorities(
|
||||
AttentionBackendEnum.TRITON_MLA,
|
||||
*sparse_backends,
|
||||
]
|
||||
elif device_capability.major == 12:
|
||||
return [
|
||||
AttentionBackendEnum.TRITON_MLA,
|
||||
AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM120,
|
||||
]
|
||||
else:
|
||||
return [
|
||||
AttentionBackendEnum.FLASH_ATTN_MLA,
|
||||
@@ -153,6 +159,21 @@ def _get_backend_priorities(
|
||||
]
|
||||
|
||||
|
||||
def _backend_cls_path(backend_cls: type[AttentionBackend]) -> str:
|
||||
module, qualname = backend_cls.full_cls_name()
|
||||
return f"{module}.{qualname}"
|
||||
|
||||
|
||||
def _get_attn_backend_class(backend: AttentionBackendEnum) -> type[AttentionBackend]:
|
||||
return backend.get_class()
|
||||
|
||||
|
||||
class _BackendCandidate(NamedTuple):
|
||||
backend_class: type[AttentionBackend]
|
||||
backend: AttentionBackendEnum
|
||||
priority: int
|
||||
|
||||
|
||||
def with_nvml_context(fn: Callable[_P, _R]) -> Callable[_P, _R]:
|
||||
@wraps(fn)
|
||||
def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R:
|
||||
@@ -340,7 +361,7 @@ class CudaPlatformBase(Platform):
|
||||
attn_selector_config: AttentionSelectorConfig,
|
||||
num_heads: int | None = None,
|
||||
) -> tuple[
|
||||
list[tuple[AttentionBackendEnum, int]],
|
||||
list[_BackendCandidate],
|
||||
dict[AttentionBackendEnum, tuple[int, list[str]]],
|
||||
]:
|
||||
valid_backends_priorities = []
|
||||
@@ -354,7 +375,7 @@ class CudaPlatformBase(Platform):
|
||||
)
|
||||
for priority, backend in enumerate(backend_priorities):
|
||||
try:
|
||||
backend_class = backend.get_class()
|
||||
backend_class = _get_attn_backend_class(backend)
|
||||
invalid_reasons_i = backend_class.validate_configuration(
|
||||
device_capability=device_capability,
|
||||
**attn_selector_config._asdict(),
|
||||
@@ -364,7 +385,9 @@ class CudaPlatformBase(Platform):
|
||||
if invalid_reasons_i:
|
||||
invalid_reasons[backend] = (priority, invalid_reasons_i)
|
||||
else:
|
||||
valid_backends_priorities.append((backend, priority))
|
||||
valid_backends_priorities.append(
|
||||
_BackendCandidate(backend_class, backend, priority)
|
||||
)
|
||||
|
||||
return valid_backends_priorities, invalid_reasons
|
||||
|
||||
@@ -381,7 +404,7 @@ class CudaPlatformBase(Platform):
|
||||
# First try checking just the selected backend, if there is one.
|
||||
if selected_backend is not None:
|
||||
try:
|
||||
backend_class = selected_backend.get_class()
|
||||
backend_class = _get_attn_backend_class(selected_backend)
|
||||
invalid_reasons = backend_class.validate_configuration(
|
||||
device_capability=device_capability,
|
||||
**attn_selector_config._asdict(),
|
||||
@@ -395,7 +418,7 @@ class CudaPlatformBase(Platform):
|
||||
)
|
||||
else:
|
||||
logger.info("Using %s backend.", selected_backend)
|
||||
return selected_backend.get_path()
|
||||
return _backend_cls_path(backend_class)
|
||||
|
||||
# No selected backend or the selected backend is invalid,
|
||||
# so we try finding a valid backend.
|
||||
@@ -425,13 +448,13 @@ class CudaPlatformBase(Platform):
|
||||
|
||||
# We have found some valid backends. Select the one with the
|
||||
# highest priority.
|
||||
sorted_indices = sorted(
|
||||
range(len(valid_backends_priorities)),
|
||||
key=lambda i: valid_backends_priorities[i][1],
|
||||
selected_candidate = min(
|
||||
valid_backends_priorities,
|
||||
key=lambda candidate: candidate.priority,
|
||||
)
|
||||
selected_index = sorted_indices[0]
|
||||
selected_backend = valid_backends_priorities[selected_index][0]
|
||||
selected_priority = valid_backends_priorities[selected_index][1]
|
||||
selected_backend_class = selected_candidate.backend_class
|
||||
selected_backend = selected_candidate.backend
|
||||
selected_priority = selected_candidate.priority
|
||||
|
||||
# If the user specified --block-size (but not --attention-backend),
|
||||
# check whether that constraint precluded any higher-priority backends.
|
||||
@@ -457,10 +480,14 @@ class CudaPlatformBase(Platform):
|
||||
logger.info_once(
|
||||
"Using %s attention backend out of potential backends: %s.",
|
||||
selected_backend.name,
|
||||
"[" + ", ".join(f"'{b[0].name}'" for b in valid_backends_priorities) + "]",
|
||||
"["
|
||||
+ ", ".join(
|
||||
f"'{candidate.backend.name}'" for candidate in valid_backends_priorities
|
||||
)
|
||||
+ "]",
|
||||
)
|
||||
|
||||
return selected_backend.get_path()
|
||||
return _backend_cls_path(selected_backend_class)
|
||||
|
||||
@classmethod
|
||||
def get_supported_vit_attn_backends(cls) -> list[AttentionBackendEnum]:
|
||||
@@ -635,7 +662,11 @@ class CudaPlatformBase(Platform):
|
||||
@classmethod
|
||||
def support_deep_gemm(cls) -> bool:
|
||||
"""Currently, only Hopper and Blackwell GPUs are supported."""
|
||||
return cls.is_device_capability(90) or cls.is_device_capability_family(100)
|
||||
return (
|
||||
cls.is_device_capability(90)
|
||||
or cls.is_device_capability_family(100)
|
||||
or cls.is_device_capability_family(120)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def is_integrated_gpu(cls, device_id: int = 0) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user