Enable DeepSeek V4 and GLM-5.1 on SM120 (#43477)

Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
Gabriel Wu
2026-06-22 11:54:14 -07:00
committed by GitHub
co-authored by Claude Opus 4.8 Yongye Zhu
parent 3ce15fd574
commit 44d95069e9
37 changed files with 2347 additions and 476 deletions
+144 -3
View File
@@ -5,6 +5,7 @@
Users of vLLM should always import **only** these wrappers.
"""
import contextlib
import functools
import importlib
import os
@@ -37,7 +38,10 @@ def should_auto_disable_deep_gemm(model_type: str | None) -> bool:
"""
if model_type is None:
return False
if not current_platform.is_device_capability_family(100):
if not (
current_platform.is_device_capability_family(100)
or current_platform.is_device_capability_family(120)
):
return False
return model_type in _DEEPGEMM_BLACKWELL_EXCLUDED_MODEL_TYPES
@@ -71,7 +75,10 @@ class DeepGemmQuantScaleFMT(Enum):
cls._oracle_cache = ( # type: ignore
cls.UE8M0
if current_platform.is_device_capability_family(100)
if (
current_platform.is_device_capability_family(100)
or current_platform.is_device_capability_family(120)
)
else cls.FLOAT32_CEIL_UE8M0
)
@@ -138,7 +145,15 @@ _get_paged_mqa_logits_metadata_impl: Callable[..., Any] | None = None
_tf32_hc_prenorm_gemm_impl: Callable[..., Any] | None = None
_get_mn_major_tma_aligned_tensor_impl: Callable[..., Any] | None = None
_get_mk_alignment_for_contiguous_layout_impl: Callable[..., Any] | None = None
_get_theoretical_mk_alignment_for_contiguous_layout_impl: Callable[..., Any] | None = (
None
)
_transform_sf_into_required_layout_impl: Callable[..., Any] | None = None
_pack_ue8m0_to_int_impl: Callable[..., Any] | None = None
_get_mn_major_tma_aligned_packed_ue8m0_tensor_impl: Callable[..., Any] | None = None
_get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor_impl: (
Callable[..., Any] | None
) = None
@functools.cache
@@ -203,7 +218,11 @@ def _lazy_init() -> None:
global _tf32_hc_prenorm_gemm_impl
global _get_mn_major_tma_aligned_tensor_impl
global _get_mk_alignment_for_contiguous_layout_impl
global _get_theoretical_mk_alignment_for_contiguous_layout_impl
global _transform_sf_into_required_layout_impl
global _pack_ue8m0_to_int_impl
global _get_mn_major_tma_aligned_packed_ue8m0_tensor_impl
global _get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor_impl
# fast path
if (
_cublaslt_gemm_nt_impl is not None
@@ -218,6 +237,9 @@ def _lazy_init() -> None:
or _tf32_hc_prenorm_gemm_impl is not None
or _get_mk_alignment_for_contiguous_layout_impl is not None
or _transform_sf_into_required_layout_impl is not None
or _pack_ue8m0_to_int_impl is not None
or _get_mn_major_tma_aligned_packed_ue8m0_tensor_impl is not None
or _get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor_impl is not None
):
return
@@ -258,9 +280,19 @@ def _lazy_init() -> None:
_get_mk_alignment_for_contiguous_layout_impl = getattr(
_dg, "get_mk_alignment_for_contiguous_layout", None
)
_get_theoretical_mk_alignment_for_contiguous_layout_impl = getattr(
_dg, "get_theoretical_mk_alignment_for_contiguous_layout", None
)
_transform_sf_into_required_layout_impl = getattr(
_dg, "transform_sf_into_required_layout", None
)
_pack_ue8m0_to_int_impl = getattr(_dg, "pack_ue8m0_to_int", None)
_get_mn_major_tma_aligned_packed_ue8m0_tensor_impl = getattr(
_dg, "get_mn_major_tma_aligned_packed_ue8m0_tensor", None
)
_get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor_impl = getattr(
_dg, "get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor", None
)
DeepGemmQuantScaleFMT.init_oracle_cache()
@@ -280,7 +312,6 @@ def set_num_sms(num_sms: int) -> None:
dg.set_num_sms(num_sms)
@functools.cache
def get_mk_alignment_for_contiguous_layout() -> list[int]:
_lazy_init()
if _get_mk_alignment_for_contiguous_layout_impl is None:
@@ -289,6 +320,70 @@ def get_mk_alignment_for_contiguous_layout() -> list[int]:
return [mk_align_size, mk_align_size]
def get_theoretical_mk_alignment_for_contiguous_layout(
expected_m: int | None = None,
num_groups: int | None = None,
) -> int:
"""Per-call optimal M alignment for grouped contiguous GEMMs.
`expected_m` is the TOTAL routed tokens (sum across experts, typically
M × num_topk). `num_groups` is the number of experts on this rank.
The helper divides to recover per-expert em and picks an alignment based
on data-driven thresholds (see deep_gemm runtime.hpp comments).
Older callers that omit `num_groups` are interpreted as passing already
per-expert em (legacy behaviour preserved for backward compat).
"""
_lazy_init()
if _get_theoretical_mk_alignment_for_contiguous_layout_impl is None:
return _missing()
if num_groups is None:
return _get_theoretical_mk_alignment_for_contiguous_layout_impl(expected_m)
if num_groups <= 0:
raise ValueError(f"num_groups must be positive, got {num_groups}")
try:
return _get_theoretical_mk_alignment_for_contiguous_layout_impl(
expected_m, num_groups
)
except TypeError:
per_group_m = None if expected_m is None else cdiv(expected_m, num_groups)
return _get_theoretical_mk_alignment_for_contiguous_layout_impl(per_group_m)
def set_mk_alignment_for_contiguous_layout(value: int) -> None:
"""Set DeepGEMM's BLOCK_M cap for grouped contiguous GEMMs.
The DG heuristic constrains BLOCK_M ≤ this value when picking a kernel
layout. Use this in concert with `compute_aligned_M_and_alignment`'s
per-call alignment so the workspace's per-expert padding matches the
kernel's BLOCK_M; a mismatch leads to the scheduler reading the wrong
expert_id from `m_indices` at `m_block_idx * BLOCK_M` stride and
OOB-indexing the B-weights tensor (manifests as IMA under CUDA-graph
replay).
"""
_lazy_init()
dg = _import_deep_gemm()
if dg is None:
raise RuntimeError("DeepGEMM is not available")
dg.set_mk_alignment_for_contiguous_layout(value)
@contextlib.contextmanager
def mk_alignment_scope(value: int):
"""Temporarily set DeepGEMM's BLOCK_M cap, restoring on exit.
Use around a sequence of grouped-contiguous GEMM calls whose workspace
is padded to `value` (typically the per_call_align returned by
`compute_aligned_M_and_alignment`).
"""
prev = get_mk_alignment_for_contiguous_layout()[0]
set_mk_alignment_for_contiguous_layout(value)
try:
yield
finally:
set_mk_alignment_for_contiguous_layout(prev)
def get_col_major_tma_aligned_tensor(x: torch.Tensor) -> torch.Tensor:
"""Wrapper for DeepGEMM's get_mn_major_tma_aligned_tensor"""
_lazy_init()
@@ -297,6 +392,48 @@ def get_col_major_tma_aligned_tensor(x: torch.Tensor) -> torch.Tensor:
return _get_mn_major_tma_aligned_tensor_impl(x)
def pack_ue8m0_to_int(x: torch.Tensor) -> torch.Tensor:
"""Pack 4 UE8M0 (uint8) scales into one int32.
DeepGEMM's SM100/SM120 FP8/FP4 kernels accept either ``float32`` scales
(legacy format, 4 B/scale) or ``int32`` packed UE8M0 scales (1 B/scale
after 4:1 packing — 4× smaller than the legacy fp32 representation).
"""
_lazy_init()
if _pack_ue8m0_to_int_impl is None:
return _missing()
return _pack_ue8m0_to_int_impl(x)
def get_mn_major_tma_aligned_packed_ue8m0_tensor(x: torch.Tensor) -> torch.Tensor:
"""Pack UE8M0 (uint8) → int32 with the MN-major TMA-aligned layout the
DeepGEMM kernels consume directly. 16× smaller than the fp32 legacy SF
format. Use for non-grouped 2D scale tensors.
"""
_lazy_init()
if _get_mn_major_tma_aligned_packed_ue8m0_tensor_impl is None:
return _missing()
return _get_mn_major_tma_aligned_packed_ue8m0_tensor_impl(x)
def get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor(
sf: torch.Tensor,
ks_tensor: torch.Tensor,
ks: list[int],
gran_k: int,
) -> torch.Tensor:
"""Grouped (3D, expert-batched) variant of
``get_mn_major_tma_aligned_packed_ue8m0_tensor``. Use for MoE weight
scale tensors of shape ``(num_experts, mn, k_scale)``.
"""
_lazy_init()
if _get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor_impl is None:
return _missing()
return _get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor_impl(
sf, ks_tensor, ks, gran_k
)
def cublaslt_gemm_nt(*args, **kwargs):
_lazy_init()
if _cublaslt_gemm_nt_impl is None:
@@ -601,4 +738,8 @@ __all__ = [
"should_use_deepgemm_for_fp8_linear",
"get_col_major_tma_aligned_tensor",
"get_mk_alignment_for_contiguous_layout",
"get_theoretical_mk_alignment_for_contiguous_layout",
"pack_ue8m0_to_int",
"get_mn_major_tma_aligned_packed_ue8m0_tensor",
"get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor",
]