Enable DeepSeek V4 and GLM-5.1 on SM120 (#43477)

Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
Gabriel Wu
2026-06-22 11:54:14 -07:00
committed by GitHub
co-authored by Claude Opus 4.8 Yongye Zhu
parent 3ce15fd574
commit 44d95069e9
37 changed files with 2347 additions and 476 deletions
+35 -11
View File
@@ -72,11 +72,10 @@ def _missing(*_: Any, **__: Any) -> NoReturn:
)
def _missing_dsv4_sparse_mla(*_: Any, **__: Any) -> NoReturn:
def _missing_sparse_mla(*_: Any, **__: Any) -> NoReturn:
raise RuntimeError(
"flashinfer.mla.trtllm_batch_decode_sparse_mla_dsv4 is not available. "
"Install a FlashInfer build that includes DeepSeek V4 sparse MLA "
"TRTLLM-GEN support."
"FlashInfer sparse MLA decode APIs are not available. "
"Install a FlashInfer build that includes sparse MLA decode support."
)
@@ -149,14 +148,18 @@ flashinfer_b12x_fused_moe = _lazy_import_wrapper(
trtllm_fp4_block_scale_moe = _lazy_import_wrapper(
"flashinfer", "trtllm_fp4_block_scale_moe"
)
# DeepSeek V4 sparse MLA TRTLLM-GEN decode launcher (public wrapper). Handles
# the SWA + compressed KV pools, the concatenated sparse-index matrix, and
# per-tensor FP8 / BF16 inputs with BF16 output.
flashinfer_trtllm_batch_decode_sparse_mla_dsv4 = _lazy_import_wrapper(
"flashinfer.mla",
"trtllm_batch_decode_sparse_mla_dsv4",
fallback_fn=_missing_dsv4_sparse_mla,
flashinfer_trtllm_batch_decode_with_kv_cache_mla = _lazy_import_wrapper(
"flashinfer.decode",
"trtllm_batch_decode_with_kv_cache_mla",
fallback_fn=_missing_sparse_mla,
)
flashinfer_trtllm_batch_decode_sparse_mla_dsv4 = _lazy_import_wrapper(
"flashinfer.decode",
"trtllm_batch_decode_sparse_mla_dsv4",
fallback_fn=_missing_sparse_mla,
)
# Special case for autotune since it returns a context manager
autotune = _lazy_import_wrapper(
"flashinfer.autotuner",
@@ -209,6 +212,26 @@ def has_flashinfer_moe() -> bool:
)
@functools.cache
def has_flashinfer_sparse_mla_sm120() -> bool:
"""Return ``True`` if FlashInfer sparse MLA decode support is available."""
if not has_flashinfer():
return False
try:
from flashinfer.autotuner import autotune
from flashinfer.decode import (
trtllm_batch_decode_sparse_mla_dsv4,
trtllm_batch_decode_with_kv_cache_mla,
)
except ImportError:
return False
return (
callable(trtllm_batch_decode_sparse_mla_dsv4)
and callable(trtllm_batch_decode_with_kv_cache_mla)
and callable(autotune)
)
@functools.cache
def has_flashinfer_cutedsl() -> bool:
"""Return ``True`` if FlashInfer cutedsl module is available."""
@@ -988,6 +1011,7 @@ __all__ = [
"flashinfer_b12x_fused_moe",
"flashinfer_convert_sf_to_mma_layout",
"trtllm_fp4_block_scale_moe",
"flashinfer_trtllm_batch_decode_with_kv_cache_mla",
"flashinfer_trtllm_batch_decode_sparse_mla_dsv4",
"autotune",
"has_flashinfer_moe",