Enable DeepSeek V4 and GLM-5.1 on SM120 (#43477)
Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Yongye Zhu
parent
3ce15fd574
commit
44d95069e9
+144
-3
@@ -5,6 +5,7 @@
|
||||
Users of vLLM should always import **only** these wrappers.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import functools
|
||||
import importlib
|
||||
import os
|
||||
@@ -37,7 +38,10 @@ def should_auto_disable_deep_gemm(model_type: str | None) -> bool:
|
||||
"""
|
||||
if model_type is None:
|
||||
return False
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
if not (
|
||||
current_platform.is_device_capability_family(100)
|
||||
or current_platform.is_device_capability_family(120)
|
||||
):
|
||||
return False
|
||||
return model_type in _DEEPGEMM_BLACKWELL_EXCLUDED_MODEL_TYPES
|
||||
|
||||
@@ -71,7 +75,10 @@ class DeepGemmQuantScaleFMT(Enum):
|
||||
|
||||
cls._oracle_cache = ( # type: ignore
|
||||
cls.UE8M0
|
||||
if current_platform.is_device_capability_family(100)
|
||||
if (
|
||||
current_platform.is_device_capability_family(100)
|
||||
or current_platform.is_device_capability_family(120)
|
||||
)
|
||||
else cls.FLOAT32_CEIL_UE8M0
|
||||
)
|
||||
|
||||
@@ -138,7 +145,15 @@ _get_paged_mqa_logits_metadata_impl: Callable[..., Any] | None = None
|
||||
_tf32_hc_prenorm_gemm_impl: Callable[..., Any] | None = None
|
||||
_get_mn_major_tma_aligned_tensor_impl: Callable[..., Any] | None = None
|
||||
_get_mk_alignment_for_contiguous_layout_impl: Callable[..., Any] | None = None
|
||||
_get_theoretical_mk_alignment_for_contiguous_layout_impl: Callable[..., Any] | None = (
|
||||
None
|
||||
)
|
||||
_transform_sf_into_required_layout_impl: Callable[..., Any] | None = None
|
||||
_pack_ue8m0_to_int_impl: Callable[..., Any] | None = None
|
||||
_get_mn_major_tma_aligned_packed_ue8m0_tensor_impl: Callable[..., Any] | None = None
|
||||
_get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor_impl: (
|
||||
Callable[..., Any] | None
|
||||
) = None
|
||||
|
||||
|
||||
@functools.cache
|
||||
@@ -203,7 +218,11 @@ def _lazy_init() -> None:
|
||||
global _tf32_hc_prenorm_gemm_impl
|
||||
global _get_mn_major_tma_aligned_tensor_impl
|
||||
global _get_mk_alignment_for_contiguous_layout_impl
|
||||
global _get_theoretical_mk_alignment_for_contiguous_layout_impl
|
||||
global _transform_sf_into_required_layout_impl
|
||||
global _pack_ue8m0_to_int_impl
|
||||
global _get_mn_major_tma_aligned_packed_ue8m0_tensor_impl
|
||||
global _get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor_impl
|
||||
# fast path
|
||||
if (
|
||||
_cublaslt_gemm_nt_impl is not None
|
||||
@@ -218,6 +237,9 @@ def _lazy_init() -> None:
|
||||
or _tf32_hc_prenorm_gemm_impl is not None
|
||||
or _get_mk_alignment_for_contiguous_layout_impl is not None
|
||||
or _transform_sf_into_required_layout_impl is not None
|
||||
or _pack_ue8m0_to_int_impl is not None
|
||||
or _get_mn_major_tma_aligned_packed_ue8m0_tensor_impl is not None
|
||||
or _get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor_impl is not None
|
||||
):
|
||||
return
|
||||
|
||||
@@ -258,9 +280,19 @@ def _lazy_init() -> None:
|
||||
_get_mk_alignment_for_contiguous_layout_impl = getattr(
|
||||
_dg, "get_mk_alignment_for_contiguous_layout", None
|
||||
)
|
||||
_get_theoretical_mk_alignment_for_contiguous_layout_impl = getattr(
|
||||
_dg, "get_theoretical_mk_alignment_for_contiguous_layout", None
|
||||
)
|
||||
_transform_sf_into_required_layout_impl = getattr(
|
||||
_dg, "transform_sf_into_required_layout", None
|
||||
)
|
||||
_pack_ue8m0_to_int_impl = getattr(_dg, "pack_ue8m0_to_int", None)
|
||||
_get_mn_major_tma_aligned_packed_ue8m0_tensor_impl = getattr(
|
||||
_dg, "get_mn_major_tma_aligned_packed_ue8m0_tensor", None
|
||||
)
|
||||
_get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor_impl = getattr(
|
||||
_dg, "get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor", None
|
||||
)
|
||||
DeepGemmQuantScaleFMT.init_oracle_cache()
|
||||
|
||||
|
||||
@@ -280,7 +312,6 @@ def set_num_sms(num_sms: int) -> None:
|
||||
dg.set_num_sms(num_sms)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def get_mk_alignment_for_contiguous_layout() -> list[int]:
|
||||
_lazy_init()
|
||||
if _get_mk_alignment_for_contiguous_layout_impl is None:
|
||||
@@ -289,6 +320,70 @@ def get_mk_alignment_for_contiguous_layout() -> list[int]:
|
||||
return [mk_align_size, mk_align_size]
|
||||
|
||||
|
||||
def get_theoretical_mk_alignment_for_contiguous_layout(
|
||||
expected_m: int | None = None,
|
||||
num_groups: int | None = None,
|
||||
) -> int:
|
||||
"""Per-call optimal M alignment for grouped contiguous GEMMs.
|
||||
|
||||
`expected_m` is the TOTAL routed tokens (sum across experts, typically
|
||||
M × num_topk). `num_groups` is the number of experts on this rank.
|
||||
The helper divides to recover per-expert em and picks an alignment based
|
||||
on data-driven thresholds (see deep_gemm runtime.hpp comments).
|
||||
|
||||
Older callers that omit `num_groups` are interpreted as passing already
|
||||
per-expert em (legacy behaviour preserved for backward compat).
|
||||
"""
|
||||
_lazy_init()
|
||||
if _get_theoretical_mk_alignment_for_contiguous_layout_impl is None:
|
||||
return _missing()
|
||||
if num_groups is None:
|
||||
return _get_theoretical_mk_alignment_for_contiguous_layout_impl(expected_m)
|
||||
if num_groups <= 0:
|
||||
raise ValueError(f"num_groups must be positive, got {num_groups}")
|
||||
try:
|
||||
return _get_theoretical_mk_alignment_for_contiguous_layout_impl(
|
||||
expected_m, num_groups
|
||||
)
|
||||
except TypeError:
|
||||
per_group_m = None if expected_m is None else cdiv(expected_m, num_groups)
|
||||
return _get_theoretical_mk_alignment_for_contiguous_layout_impl(per_group_m)
|
||||
|
||||
|
||||
def set_mk_alignment_for_contiguous_layout(value: int) -> None:
|
||||
"""Set DeepGEMM's BLOCK_M cap for grouped contiguous GEMMs.
|
||||
|
||||
The DG heuristic constrains BLOCK_M ≤ this value when picking a kernel
|
||||
layout. Use this in concert with `compute_aligned_M_and_alignment`'s
|
||||
per-call alignment so the workspace's per-expert padding matches the
|
||||
kernel's BLOCK_M; a mismatch leads to the scheduler reading the wrong
|
||||
expert_id from `m_indices` at `m_block_idx * BLOCK_M` stride and
|
||||
OOB-indexing the B-weights tensor (manifests as IMA under CUDA-graph
|
||||
replay).
|
||||
"""
|
||||
_lazy_init()
|
||||
dg = _import_deep_gemm()
|
||||
if dg is None:
|
||||
raise RuntimeError("DeepGEMM is not available")
|
||||
dg.set_mk_alignment_for_contiguous_layout(value)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def mk_alignment_scope(value: int):
|
||||
"""Temporarily set DeepGEMM's BLOCK_M cap, restoring on exit.
|
||||
|
||||
Use around a sequence of grouped-contiguous GEMM calls whose workspace
|
||||
is padded to `value` (typically the per_call_align returned by
|
||||
`compute_aligned_M_and_alignment`).
|
||||
"""
|
||||
prev = get_mk_alignment_for_contiguous_layout()[0]
|
||||
set_mk_alignment_for_contiguous_layout(value)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
set_mk_alignment_for_contiguous_layout(prev)
|
||||
|
||||
|
||||
def get_col_major_tma_aligned_tensor(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Wrapper for DeepGEMM's get_mn_major_tma_aligned_tensor"""
|
||||
_lazy_init()
|
||||
@@ -297,6 +392,48 @@ def get_col_major_tma_aligned_tensor(x: torch.Tensor) -> torch.Tensor:
|
||||
return _get_mn_major_tma_aligned_tensor_impl(x)
|
||||
|
||||
|
||||
def pack_ue8m0_to_int(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Pack 4 UE8M0 (uint8) scales into one int32.
|
||||
|
||||
DeepGEMM's SM100/SM120 FP8/FP4 kernels accept either ``float32`` scales
|
||||
(legacy format, 4 B/scale) or ``int32`` packed UE8M0 scales (1 B/scale
|
||||
after 4:1 packing — 4× smaller than the legacy fp32 representation).
|
||||
"""
|
||||
_lazy_init()
|
||||
if _pack_ue8m0_to_int_impl is None:
|
||||
return _missing()
|
||||
return _pack_ue8m0_to_int_impl(x)
|
||||
|
||||
|
||||
def get_mn_major_tma_aligned_packed_ue8m0_tensor(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Pack UE8M0 (uint8) → int32 with the MN-major TMA-aligned layout the
|
||||
DeepGEMM kernels consume directly. 16× smaller than the fp32 legacy SF
|
||||
format. Use for non-grouped 2D scale tensors.
|
||||
"""
|
||||
_lazy_init()
|
||||
if _get_mn_major_tma_aligned_packed_ue8m0_tensor_impl is None:
|
||||
return _missing()
|
||||
return _get_mn_major_tma_aligned_packed_ue8m0_tensor_impl(x)
|
||||
|
||||
|
||||
def get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor(
|
||||
sf: torch.Tensor,
|
||||
ks_tensor: torch.Tensor,
|
||||
ks: list[int],
|
||||
gran_k: int,
|
||||
) -> torch.Tensor:
|
||||
"""Grouped (3D, expert-batched) variant of
|
||||
``get_mn_major_tma_aligned_packed_ue8m0_tensor``. Use for MoE weight
|
||||
scale tensors of shape ``(num_experts, mn, k_scale)``.
|
||||
"""
|
||||
_lazy_init()
|
||||
if _get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor_impl is None:
|
||||
return _missing()
|
||||
return _get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor_impl(
|
||||
sf, ks_tensor, ks, gran_k
|
||||
)
|
||||
|
||||
|
||||
def cublaslt_gemm_nt(*args, **kwargs):
|
||||
_lazy_init()
|
||||
if _cublaslt_gemm_nt_impl is None:
|
||||
@@ -601,4 +738,8 @@ __all__ = [
|
||||
"should_use_deepgemm_for_fp8_linear",
|
||||
"get_col_major_tma_aligned_tensor",
|
||||
"get_mk_alignment_for_contiguous_layout",
|
||||
"get_theoretical_mk_alignment_for_contiguous_layout",
|
||||
"pack_ue8m0_to_int",
|
||||
"get_mn_major_tma_aligned_packed_ue8m0_tensor",
|
||||
"get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor",
|
||||
]
|
||||
|
||||
+35
-11
@@ -72,11 +72,10 @@ def _missing(*_: Any, **__: Any) -> NoReturn:
|
||||
)
|
||||
|
||||
|
||||
def _missing_dsv4_sparse_mla(*_: Any, **__: Any) -> NoReturn:
|
||||
def _missing_sparse_mla(*_: Any, **__: Any) -> NoReturn:
|
||||
raise RuntimeError(
|
||||
"flashinfer.mla.trtllm_batch_decode_sparse_mla_dsv4 is not available. "
|
||||
"Install a FlashInfer build that includes DeepSeek V4 sparse MLA "
|
||||
"TRTLLM-GEN support."
|
||||
"FlashInfer sparse MLA decode APIs are not available. "
|
||||
"Install a FlashInfer build that includes sparse MLA decode support."
|
||||
)
|
||||
|
||||
|
||||
@@ -149,14 +148,18 @@ flashinfer_b12x_fused_moe = _lazy_import_wrapper(
|
||||
trtllm_fp4_block_scale_moe = _lazy_import_wrapper(
|
||||
"flashinfer", "trtllm_fp4_block_scale_moe"
|
||||
)
|
||||
# DeepSeek V4 sparse MLA TRTLLM-GEN decode launcher (public wrapper). Handles
|
||||
# the SWA + compressed KV pools, the concatenated sparse-index matrix, and
|
||||
# per-tensor FP8 / BF16 inputs with BF16 output.
|
||||
flashinfer_trtllm_batch_decode_sparse_mla_dsv4 = _lazy_import_wrapper(
|
||||
"flashinfer.mla",
|
||||
"trtllm_batch_decode_sparse_mla_dsv4",
|
||||
fallback_fn=_missing_dsv4_sparse_mla,
|
||||
flashinfer_trtllm_batch_decode_with_kv_cache_mla = _lazy_import_wrapper(
|
||||
"flashinfer.decode",
|
||||
"trtllm_batch_decode_with_kv_cache_mla",
|
||||
fallback_fn=_missing_sparse_mla,
|
||||
)
|
||||
flashinfer_trtllm_batch_decode_sparse_mla_dsv4 = _lazy_import_wrapper(
|
||||
"flashinfer.decode",
|
||||
"trtllm_batch_decode_sparse_mla_dsv4",
|
||||
fallback_fn=_missing_sparse_mla,
|
||||
)
|
||||
|
||||
|
||||
# Special case for autotune since it returns a context manager
|
||||
autotune = _lazy_import_wrapper(
|
||||
"flashinfer.autotuner",
|
||||
@@ -209,6 +212,26 @@ def has_flashinfer_moe() -> bool:
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def has_flashinfer_sparse_mla_sm120() -> bool:
|
||||
"""Return ``True`` if FlashInfer sparse MLA decode support is available."""
|
||||
if not has_flashinfer():
|
||||
return False
|
||||
try:
|
||||
from flashinfer.autotuner import autotune
|
||||
from flashinfer.decode import (
|
||||
trtllm_batch_decode_sparse_mla_dsv4,
|
||||
trtllm_batch_decode_with_kv_cache_mla,
|
||||
)
|
||||
except ImportError:
|
||||
return False
|
||||
return (
|
||||
callable(trtllm_batch_decode_sparse_mla_dsv4)
|
||||
and callable(trtllm_batch_decode_with_kv_cache_mla)
|
||||
and callable(autotune)
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def has_flashinfer_cutedsl() -> bool:
|
||||
"""Return ``True`` if FlashInfer cutedsl module is available."""
|
||||
@@ -988,6 +1011,7 @@ __all__ = [
|
||||
"flashinfer_b12x_fused_moe",
|
||||
"flashinfer_convert_sf_to_mma_layout",
|
||||
"trtllm_fp4_block_scale_moe",
|
||||
"flashinfer_trtllm_batch_decode_with_kv_cache_mla",
|
||||
"flashinfer_trtllm_batch_decode_sparse_mla_dsv4",
|
||||
"autotune",
|
||||
"has_flashinfer_moe",
|
||||
|
||||
Reference in New Issue
Block a user