Enable DeepSeek V4 and GLM-5.1 on SM120 (#43477)
Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Yongye Zhu
parent
3ce15fd574
commit
44d95069e9
+144
-3
@@ -5,6 +5,7 @@
|
||||
Users of vLLM should always import **only** these wrappers.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import functools
|
||||
import importlib
|
||||
import os
|
||||
@@ -37,7 +38,10 @@ def should_auto_disable_deep_gemm(model_type: str | None) -> bool:
|
||||
"""
|
||||
if model_type is None:
|
||||
return False
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
if not (
|
||||
current_platform.is_device_capability_family(100)
|
||||
or current_platform.is_device_capability_family(120)
|
||||
):
|
||||
return False
|
||||
return model_type in _DEEPGEMM_BLACKWELL_EXCLUDED_MODEL_TYPES
|
||||
|
||||
@@ -71,7 +75,10 @@ class DeepGemmQuantScaleFMT(Enum):
|
||||
|
||||
cls._oracle_cache = ( # type: ignore
|
||||
cls.UE8M0
|
||||
if current_platform.is_device_capability_family(100)
|
||||
if (
|
||||
current_platform.is_device_capability_family(100)
|
||||
or current_platform.is_device_capability_family(120)
|
||||
)
|
||||
else cls.FLOAT32_CEIL_UE8M0
|
||||
)
|
||||
|
||||
@@ -138,7 +145,15 @@ _get_paged_mqa_logits_metadata_impl: Callable[..., Any] | None = None
|
||||
_tf32_hc_prenorm_gemm_impl: Callable[..., Any] | None = None
|
||||
_get_mn_major_tma_aligned_tensor_impl: Callable[..., Any] | None = None
|
||||
_get_mk_alignment_for_contiguous_layout_impl: Callable[..., Any] | None = None
|
||||
_get_theoretical_mk_alignment_for_contiguous_layout_impl: Callable[..., Any] | None = (
|
||||
None
|
||||
)
|
||||
_transform_sf_into_required_layout_impl: Callable[..., Any] | None = None
|
||||
_pack_ue8m0_to_int_impl: Callable[..., Any] | None = None
|
||||
_get_mn_major_tma_aligned_packed_ue8m0_tensor_impl: Callable[..., Any] | None = None
|
||||
_get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor_impl: (
|
||||
Callable[..., Any] | None
|
||||
) = None
|
||||
|
||||
|
||||
@functools.cache
|
||||
@@ -203,7 +218,11 @@ def _lazy_init() -> None:
|
||||
global _tf32_hc_prenorm_gemm_impl
|
||||
global _get_mn_major_tma_aligned_tensor_impl
|
||||
global _get_mk_alignment_for_contiguous_layout_impl
|
||||
global _get_theoretical_mk_alignment_for_contiguous_layout_impl
|
||||
global _transform_sf_into_required_layout_impl
|
||||
global _pack_ue8m0_to_int_impl
|
||||
global _get_mn_major_tma_aligned_packed_ue8m0_tensor_impl
|
||||
global _get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor_impl
|
||||
# fast path
|
||||
if (
|
||||
_cublaslt_gemm_nt_impl is not None
|
||||
@@ -218,6 +237,9 @@ def _lazy_init() -> None:
|
||||
or _tf32_hc_prenorm_gemm_impl is not None
|
||||
or _get_mk_alignment_for_contiguous_layout_impl is not None
|
||||
or _transform_sf_into_required_layout_impl is not None
|
||||
or _pack_ue8m0_to_int_impl is not None
|
||||
or _get_mn_major_tma_aligned_packed_ue8m0_tensor_impl is not None
|
||||
or _get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor_impl is not None
|
||||
):
|
||||
return
|
||||
|
||||
@@ -258,9 +280,19 @@ def _lazy_init() -> None:
|
||||
_get_mk_alignment_for_contiguous_layout_impl = getattr(
|
||||
_dg, "get_mk_alignment_for_contiguous_layout", None
|
||||
)
|
||||
_get_theoretical_mk_alignment_for_contiguous_layout_impl = getattr(
|
||||
_dg, "get_theoretical_mk_alignment_for_contiguous_layout", None
|
||||
)
|
||||
_transform_sf_into_required_layout_impl = getattr(
|
||||
_dg, "transform_sf_into_required_layout", None
|
||||
)
|
||||
_pack_ue8m0_to_int_impl = getattr(_dg, "pack_ue8m0_to_int", None)
|
||||
_get_mn_major_tma_aligned_packed_ue8m0_tensor_impl = getattr(
|
||||
_dg, "get_mn_major_tma_aligned_packed_ue8m0_tensor", None
|
||||
)
|
||||
_get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor_impl = getattr(
|
||||
_dg, "get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor", None
|
||||
)
|
||||
DeepGemmQuantScaleFMT.init_oracle_cache()
|
||||
|
||||
|
||||
@@ -280,7 +312,6 @@ def set_num_sms(num_sms: int) -> None:
|
||||
dg.set_num_sms(num_sms)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def get_mk_alignment_for_contiguous_layout() -> list[int]:
|
||||
_lazy_init()
|
||||
if _get_mk_alignment_for_contiguous_layout_impl is None:
|
||||
@@ -289,6 +320,70 @@ def get_mk_alignment_for_contiguous_layout() -> list[int]:
|
||||
return [mk_align_size, mk_align_size]
|
||||
|
||||
|
||||
def get_theoretical_mk_alignment_for_contiguous_layout(
|
||||
expected_m: int | None = None,
|
||||
num_groups: int | None = None,
|
||||
) -> int:
|
||||
"""Per-call optimal M alignment for grouped contiguous GEMMs.
|
||||
|
||||
`expected_m` is the TOTAL routed tokens (sum across experts, typically
|
||||
M × num_topk). `num_groups` is the number of experts on this rank.
|
||||
The helper divides to recover per-expert em and picks an alignment based
|
||||
on data-driven thresholds (see deep_gemm runtime.hpp comments).
|
||||
|
||||
Older callers that omit `num_groups` are interpreted as passing already
|
||||
per-expert em (legacy behaviour preserved for backward compat).
|
||||
"""
|
||||
_lazy_init()
|
||||
if _get_theoretical_mk_alignment_for_contiguous_layout_impl is None:
|
||||
return _missing()
|
||||
if num_groups is None:
|
||||
return _get_theoretical_mk_alignment_for_contiguous_layout_impl(expected_m)
|
||||
if num_groups <= 0:
|
||||
raise ValueError(f"num_groups must be positive, got {num_groups}")
|
||||
try:
|
||||
return _get_theoretical_mk_alignment_for_contiguous_layout_impl(
|
||||
expected_m, num_groups
|
||||
)
|
||||
except TypeError:
|
||||
per_group_m = None if expected_m is None else cdiv(expected_m, num_groups)
|
||||
return _get_theoretical_mk_alignment_for_contiguous_layout_impl(per_group_m)
|
||||
|
||||
|
||||
def set_mk_alignment_for_contiguous_layout(value: int) -> None:
|
||||
"""Set DeepGEMM's BLOCK_M cap for grouped contiguous GEMMs.
|
||||
|
||||
The DG heuristic constrains BLOCK_M ≤ this value when picking a kernel
|
||||
layout. Use this in concert with `compute_aligned_M_and_alignment`'s
|
||||
per-call alignment so the workspace's per-expert padding matches the
|
||||
kernel's BLOCK_M; a mismatch leads to the scheduler reading the wrong
|
||||
expert_id from `m_indices` at `m_block_idx * BLOCK_M` stride and
|
||||
OOB-indexing the B-weights tensor (manifests as IMA under CUDA-graph
|
||||
replay).
|
||||
"""
|
||||
_lazy_init()
|
||||
dg = _import_deep_gemm()
|
||||
if dg is None:
|
||||
raise RuntimeError("DeepGEMM is not available")
|
||||
dg.set_mk_alignment_for_contiguous_layout(value)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def mk_alignment_scope(value: int):
|
||||
"""Temporarily set DeepGEMM's BLOCK_M cap, restoring on exit.
|
||||
|
||||
Use around a sequence of grouped-contiguous GEMM calls whose workspace
|
||||
is padded to `value` (typically the per_call_align returned by
|
||||
`compute_aligned_M_and_alignment`).
|
||||
"""
|
||||
prev = get_mk_alignment_for_contiguous_layout()[0]
|
||||
set_mk_alignment_for_contiguous_layout(value)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
set_mk_alignment_for_contiguous_layout(prev)
|
||||
|
||||
|
||||
def get_col_major_tma_aligned_tensor(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Wrapper for DeepGEMM's get_mn_major_tma_aligned_tensor"""
|
||||
_lazy_init()
|
||||
@@ -297,6 +392,48 @@ def get_col_major_tma_aligned_tensor(x: torch.Tensor) -> torch.Tensor:
|
||||
return _get_mn_major_tma_aligned_tensor_impl(x)
|
||||
|
||||
|
||||
def pack_ue8m0_to_int(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Pack 4 UE8M0 (uint8) scales into one int32.
|
||||
|
||||
DeepGEMM's SM100/SM120 FP8/FP4 kernels accept either ``float32`` scales
|
||||
(legacy format, 4 B/scale) or ``int32`` packed UE8M0 scales (1 B/scale
|
||||
after 4:1 packing — 4× smaller than the legacy fp32 representation).
|
||||
"""
|
||||
_lazy_init()
|
||||
if _pack_ue8m0_to_int_impl is None:
|
||||
return _missing()
|
||||
return _pack_ue8m0_to_int_impl(x)
|
||||
|
||||
|
||||
def get_mn_major_tma_aligned_packed_ue8m0_tensor(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Pack UE8M0 (uint8) → int32 with the MN-major TMA-aligned layout the
|
||||
DeepGEMM kernels consume directly. 16× smaller than the fp32 legacy SF
|
||||
format. Use for non-grouped 2D scale tensors.
|
||||
"""
|
||||
_lazy_init()
|
||||
if _get_mn_major_tma_aligned_packed_ue8m0_tensor_impl is None:
|
||||
return _missing()
|
||||
return _get_mn_major_tma_aligned_packed_ue8m0_tensor_impl(x)
|
||||
|
||||
|
||||
def get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor(
|
||||
sf: torch.Tensor,
|
||||
ks_tensor: torch.Tensor,
|
||||
ks: list[int],
|
||||
gran_k: int,
|
||||
) -> torch.Tensor:
|
||||
"""Grouped (3D, expert-batched) variant of
|
||||
``get_mn_major_tma_aligned_packed_ue8m0_tensor``. Use for MoE weight
|
||||
scale tensors of shape ``(num_experts, mn, k_scale)``.
|
||||
"""
|
||||
_lazy_init()
|
||||
if _get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor_impl is None:
|
||||
return _missing()
|
||||
return _get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor_impl(
|
||||
sf, ks_tensor, ks, gran_k
|
||||
)
|
||||
|
||||
|
||||
def cublaslt_gemm_nt(*args, **kwargs):
|
||||
_lazy_init()
|
||||
if _cublaslt_gemm_nt_impl is None:
|
||||
@@ -601,4 +738,8 @@ __all__ = [
|
||||
"should_use_deepgemm_for_fp8_linear",
|
||||
"get_col_major_tma_aligned_tensor",
|
||||
"get_mk_alignment_for_contiguous_layout",
|
||||
"get_theoretical_mk_alignment_for_contiguous_layout",
|
||||
"pack_ue8m0_to_int",
|
||||
"get_mn_major_tma_aligned_packed_ue8m0_tensor",
|
||||
"get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user