Enable DeepSeek V4 and GLM-5.1 on SM120 (#43477)

Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
Gabriel Wu
2026-06-22 11:54:14 -07:00
committed by GitHub
co-authored by Claude Opus 4.8 Yongye Zhu
parent 3ce15fd574
commit 44d95069e9
37 changed files with 2347 additions and 476 deletions
+144 -3
View File
@@ -5,6 +5,7 @@
Users of vLLM should always import **only** these wrappers.
"""
import contextlib
import functools
import importlib
import os
@@ -37,7 +38,10 @@ def should_auto_disable_deep_gemm(model_type: str | None) -> bool:
"""
if model_type is None:
return False
if not current_platform.is_device_capability_family(100):
if not (
current_platform.is_device_capability_family(100)
or current_platform.is_device_capability_family(120)
):
return False
return model_type in _DEEPGEMM_BLACKWELL_EXCLUDED_MODEL_TYPES
@@ -71,7 +75,10 @@ class DeepGemmQuantScaleFMT(Enum):
cls._oracle_cache = ( # type: ignore
cls.UE8M0
if current_platform.is_device_capability_family(100)
if (
current_platform.is_device_capability_family(100)
or current_platform.is_device_capability_family(120)
)
else cls.FLOAT32_CEIL_UE8M0
)
@@ -138,7 +145,15 @@ _get_paged_mqa_logits_metadata_impl: Callable[..., Any] | None = None
_tf32_hc_prenorm_gemm_impl: Callable[..., Any] | None = None
_get_mn_major_tma_aligned_tensor_impl: Callable[..., Any] | None = None
_get_mk_alignment_for_contiguous_layout_impl: Callable[..., Any] | None = None
_get_theoretical_mk_alignment_for_contiguous_layout_impl: Callable[..., Any] | None = (
None
)
_transform_sf_into_required_layout_impl: Callable[..., Any] | None = None
_pack_ue8m0_to_int_impl: Callable[..., Any] | None = None
_get_mn_major_tma_aligned_packed_ue8m0_tensor_impl: Callable[..., Any] | None = None
_get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor_impl: (
Callable[..., Any] | None
) = None
@functools.cache
@@ -203,7 +218,11 @@ def _lazy_init() -> None:
global _tf32_hc_prenorm_gemm_impl
global _get_mn_major_tma_aligned_tensor_impl
global _get_mk_alignment_for_contiguous_layout_impl
global _get_theoretical_mk_alignment_for_contiguous_layout_impl
global _transform_sf_into_required_layout_impl
global _pack_ue8m0_to_int_impl
global _get_mn_major_tma_aligned_packed_ue8m0_tensor_impl
global _get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor_impl
# fast path
if (
_cublaslt_gemm_nt_impl is not None
@@ -218,6 +237,9 @@ def _lazy_init() -> None:
or _tf32_hc_prenorm_gemm_impl is not None
or _get_mk_alignment_for_contiguous_layout_impl is not None
or _transform_sf_into_required_layout_impl is not None
or _pack_ue8m0_to_int_impl is not None
or _get_mn_major_tma_aligned_packed_ue8m0_tensor_impl is not None
or _get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor_impl is not None
):
return
@@ -258,9 +280,19 @@ def _lazy_init() -> None:
_get_mk_alignment_for_contiguous_layout_impl = getattr(
_dg, "get_mk_alignment_for_contiguous_layout", None
)
_get_theoretical_mk_alignment_for_contiguous_layout_impl = getattr(
_dg, "get_theoretical_mk_alignment_for_contiguous_layout", None
)
_transform_sf_into_required_layout_impl = getattr(
_dg, "transform_sf_into_required_layout", None
)
_pack_ue8m0_to_int_impl = getattr(_dg, "pack_ue8m0_to_int", None)
_get_mn_major_tma_aligned_packed_ue8m0_tensor_impl = getattr(
_dg, "get_mn_major_tma_aligned_packed_ue8m0_tensor", None
)
_get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor_impl = getattr(
_dg, "get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor", None
)
DeepGemmQuantScaleFMT.init_oracle_cache()
@@ -280,7 +312,6 @@ def set_num_sms(num_sms: int) -> None:
dg.set_num_sms(num_sms)
@functools.cache
def get_mk_alignment_for_contiguous_layout() -> list[int]:
_lazy_init()
if _get_mk_alignment_for_contiguous_layout_impl is None:
@@ -289,6 +320,70 @@ def get_mk_alignment_for_contiguous_layout() -> list[int]:
return [mk_align_size, mk_align_size]
def get_theoretical_mk_alignment_for_contiguous_layout(
expected_m: int | None = None,
num_groups: int | None = None,
) -> int:
"""Per-call optimal M alignment for grouped contiguous GEMMs.
`expected_m` is the TOTAL routed tokens (sum across experts, typically
M × num_topk). `num_groups` is the number of experts on this rank.
The helper divides to recover per-expert em and picks an alignment based
on data-driven thresholds (see deep_gemm runtime.hpp comments).
Older callers that omit `num_groups` are interpreted as passing already
per-expert em (legacy behaviour preserved for backward compat).
"""
_lazy_init()
if _get_theoretical_mk_alignment_for_contiguous_layout_impl is None:
return _missing()
if num_groups is None:
return _get_theoretical_mk_alignment_for_contiguous_layout_impl(expected_m)
if num_groups <= 0:
raise ValueError(f"num_groups must be positive, got {num_groups}")
try:
return _get_theoretical_mk_alignment_for_contiguous_layout_impl(
expected_m, num_groups
)
except TypeError:
per_group_m = None if expected_m is None else cdiv(expected_m, num_groups)
return _get_theoretical_mk_alignment_for_contiguous_layout_impl(per_group_m)
def set_mk_alignment_for_contiguous_layout(value: int) -> None:
"""Set DeepGEMM's BLOCK_M cap for grouped contiguous GEMMs.
The DG heuristic constrains BLOCK_M ≤ this value when picking a kernel
layout. Use this in concert with `compute_aligned_M_and_alignment`'s
per-call alignment so the workspace's per-expert padding matches the
kernel's BLOCK_M; a mismatch leads to the scheduler reading the wrong
expert_id from `m_indices` at `m_block_idx * BLOCK_M` stride and
OOB-indexing the B-weights tensor (manifests as IMA under CUDA-graph
replay).
"""
_lazy_init()
dg = _import_deep_gemm()
if dg is None:
raise RuntimeError("DeepGEMM is not available")
dg.set_mk_alignment_for_contiguous_layout(value)
@contextlib.contextmanager
def mk_alignment_scope(value: int):
"""Temporarily set DeepGEMM's BLOCK_M cap, restoring on exit.
Use around a sequence of grouped-contiguous GEMM calls whose workspace
is padded to `value` (typically the per_call_align returned by
`compute_aligned_M_and_alignment`).
"""
prev = get_mk_alignment_for_contiguous_layout()[0]
set_mk_alignment_for_contiguous_layout(value)
try:
yield
finally:
set_mk_alignment_for_contiguous_layout(prev)
def get_col_major_tma_aligned_tensor(x: torch.Tensor) -> torch.Tensor:
"""Wrapper for DeepGEMM's get_mn_major_tma_aligned_tensor"""
_lazy_init()
@@ -297,6 +392,48 @@ def get_col_major_tma_aligned_tensor(x: torch.Tensor) -> torch.Tensor:
return _get_mn_major_tma_aligned_tensor_impl(x)
def pack_ue8m0_to_int(x: torch.Tensor) -> torch.Tensor:
"""Pack 4 UE8M0 (uint8) scales into one int32.
DeepGEMM's SM100/SM120 FP8/FP4 kernels accept either ``float32`` scales
(legacy format, 4 B/scale) or ``int32`` packed UE8M0 scales (1 B/scale
after 4:1 packing — 4× smaller than the legacy fp32 representation).
"""
_lazy_init()
if _pack_ue8m0_to_int_impl is None:
return _missing()
return _pack_ue8m0_to_int_impl(x)
def get_mn_major_tma_aligned_packed_ue8m0_tensor(x: torch.Tensor) -> torch.Tensor:
"""Pack UE8M0 (uint8) → int32 with the MN-major TMA-aligned layout the
DeepGEMM kernels consume directly. 16× smaller than the fp32 legacy SF
format. Use for non-grouped 2D scale tensors.
"""
_lazy_init()
if _get_mn_major_tma_aligned_packed_ue8m0_tensor_impl is None:
return _missing()
return _get_mn_major_tma_aligned_packed_ue8m0_tensor_impl(x)
def get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor(
sf: torch.Tensor,
ks_tensor: torch.Tensor,
ks: list[int],
gran_k: int,
) -> torch.Tensor:
"""Grouped (3D, expert-batched) variant of
``get_mn_major_tma_aligned_packed_ue8m0_tensor``. Use for MoE weight
scale tensors of shape ``(num_experts, mn, k_scale)``.
"""
_lazy_init()
if _get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor_impl is None:
return _missing()
return _get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor_impl(
sf, ks_tensor, ks, gran_k
)
def cublaslt_gemm_nt(*args, **kwargs):
_lazy_init()
if _cublaslt_gemm_nt_impl is None:
@@ -601,4 +738,8 @@ __all__ = [
"should_use_deepgemm_for_fp8_linear",
"get_col_major_tma_aligned_tensor",
"get_mk_alignment_for_contiguous_layout",
"get_theoretical_mk_alignment_for_contiguous_layout",
"pack_ue8m0_to_int",
"get_mn_major_tma_aligned_packed_ue8m0_tensor",
"get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor",
]
+35 -11
View File
@@ -72,11 +72,10 @@ def _missing(*_: Any, **__: Any) -> NoReturn:
)
def _missing_dsv4_sparse_mla(*_: Any, **__: Any) -> NoReturn:
def _missing_sparse_mla(*_: Any, **__: Any) -> NoReturn:
raise RuntimeError(
"flashinfer.mla.trtllm_batch_decode_sparse_mla_dsv4 is not available. "
"Install a FlashInfer build that includes DeepSeek V4 sparse MLA "
"TRTLLM-GEN support."
"FlashInfer sparse MLA decode APIs are not available. "
"Install a FlashInfer build that includes sparse MLA decode support."
)
@@ -149,14 +148,18 @@ flashinfer_b12x_fused_moe = _lazy_import_wrapper(
trtllm_fp4_block_scale_moe = _lazy_import_wrapper(
"flashinfer", "trtllm_fp4_block_scale_moe"
)
# DeepSeek V4 sparse MLA TRTLLM-GEN decode launcher (public wrapper). Handles
# the SWA + compressed KV pools, the concatenated sparse-index matrix, and
# per-tensor FP8 / BF16 inputs with BF16 output.
flashinfer_trtllm_batch_decode_sparse_mla_dsv4 = _lazy_import_wrapper(
"flashinfer.mla",
"trtllm_batch_decode_sparse_mla_dsv4",
fallback_fn=_missing_dsv4_sparse_mla,
flashinfer_trtllm_batch_decode_with_kv_cache_mla = _lazy_import_wrapper(
"flashinfer.decode",
"trtllm_batch_decode_with_kv_cache_mla",
fallback_fn=_missing_sparse_mla,
)
flashinfer_trtllm_batch_decode_sparse_mla_dsv4 = _lazy_import_wrapper(
"flashinfer.decode",
"trtllm_batch_decode_sparse_mla_dsv4",
fallback_fn=_missing_sparse_mla,
)
# Special case for autotune since it returns a context manager
autotune = _lazy_import_wrapper(
"flashinfer.autotuner",
@@ -209,6 +212,26 @@ def has_flashinfer_moe() -> bool:
)
@functools.cache
def has_flashinfer_sparse_mla_sm120() -> bool:
"""Return ``True`` if FlashInfer sparse MLA decode support is available."""
if not has_flashinfer():
return False
try:
from flashinfer.autotuner import autotune
from flashinfer.decode import (
trtllm_batch_decode_sparse_mla_dsv4,
trtllm_batch_decode_with_kv_cache_mla,
)
except ImportError:
return False
return (
callable(trtllm_batch_decode_sparse_mla_dsv4)
and callable(trtllm_batch_decode_with_kv_cache_mla)
and callable(autotune)
)
@functools.cache
def has_flashinfer_cutedsl() -> bool:
"""Return ``True`` if FlashInfer cutedsl module is available."""
@@ -988,6 +1011,7 @@ __all__ = [
"flashinfer_b12x_fused_moe",
"flashinfer_convert_sf_to_mma_layout",
"trtllm_fp4_block_scale_moe",
"flashinfer_trtllm_batch_decode_with_kv_cache_mla",
"flashinfer_trtllm_batch_decode_sparse_mla_dsv4",
"autotune",
"has_flashinfer_moe",