Enable DeepSeek V4 and GLM-5.1 on SM120 (#43477)
Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Yongye Zhu
parent
3ce15fd574
commit
44d95069e9
+35
-11
@@ -72,11 +72,10 @@ def _missing(*_: Any, **__: Any) -> NoReturn:
|
||||
)
|
||||
|
||||
|
||||
def _missing_dsv4_sparse_mla(*_: Any, **__: Any) -> NoReturn:
|
||||
def _missing_sparse_mla(*_: Any, **__: Any) -> NoReturn:
|
||||
raise RuntimeError(
|
||||
"flashinfer.mla.trtllm_batch_decode_sparse_mla_dsv4 is not available. "
|
||||
"Install a FlashInfer build that includes DeepSeek V4 sparse MLA "
|
||||
"TRTLLM-GEN support."
|
||||
"FlashInfer sparse MLA decode APIs are not available. "
|
||||
"Install a FlashInfer build that includes sparse MLA decode support."
|
||||
)
|
||||
|
||||
|
||||
@@ -149,14 +148,18 @@ flashinfer_b12x_fused_moe = _lazy_import_wrapper(
|
||||
trtllm_fp4_block_scale_moe = _lazy_import_wrapper(
|
||||
"flashinfer", "trtllm_fp4_block_scale_moe"
|
||||
)
|
||||
# DeepSeek V4 sparse MLA TRTLLM-GEN decode launcher (public wrapper). Handles
|
||||
# the SWA + compressed KV pools, the concatenated sparse-index matrix, and
|
||||
# per-tensor FP8 / BF16 inputs with BF16 output.
|
||||
flashinfer_trtllm_batch_decode_sparse_mla_dsv4 = _lazy_import_wrapper(
|
||||
"flashinfer.mla",
|
||||
"trtllm_batch_decode_sparse_mla_dsv4",
|
||||
fallback_fn=_missing_dsv4_sparse_mla,
|
||||
flashinfer_trtllm_batch_decode_with_kv_cache_mla = _lazy_import_wrapper(
|
||||
"flashinfer.decode",
|
||||
"trtllm_batch_decode_with_kv_cache_mla",
|
||||
fallback_fn=_missing_sparse_mla,
|
||||
)
|
||||
flashinfer_trtllm_batch_decode_sparse_mla_dsv4 = _lazy_import_wrapper(
|
||||
"flashinfer.decode",
|
||||
"trtllm_batch_decode_sparse_mla_dsv4",
|
||||
fallback_fn=_missing_sparse_mla,
|
||||
)
|
||||
|
||||
|
||||
# Special case for autotune since it returns a context manager
|
||||
autotune = _lazy_import_wrapper(
|
||||
"flashinfer.autotuner",
|
||||
@@ -209,6 +212,26 @@ def has_flashinfer_moe() -> bool:
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def has_flashinfer_sparse_mla_sm120() -> bool:
|
||||
"""Return ``True`` if FlashInfer sparse MLA decode support is available."""
|
||||
if not has_flashinfer():
|
||||
return False
|
||||
try:
|
||||
from flashinfer.autotuner import autotune
|
||||
from flashinfer.decode import (
|
||||
trtllm_batch_decode_sparse_mla_dsv4,
|
||||
trtllm_batch_decode_with_kv_cache_mla,
|
||||
)
|
||||
except ImportError:
|
||||
return False
|
||||
return (
|
||||
callable(trtllm_batch_decode_sparse_mla_dsv4)
|
||||
and callable(trtllm_batch_decode_with_kv_cache_mla)
|
||||
and callable(autotune)
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def has_flashinfer_cutedsl() -> bool:
|
||||
"""Return ``True`` if FlashInfer cutedsl module is available."""
|
||||
@@ -988,6 +1011,7 @@ __all__ = [
|
||||
"flashinfer_b12x_fused_moe",
|
||||
"flashinfer_convert_sf_to_mma_layout",
|
||||
"trtllm_fp4_block_scale_moe",
|
||||
"flashinfer_trtllm_batch_decode_with_kv_cache_mla",
|
||||
"flashinfer_trtllm_batch_decode_sparse_mla_dsv4",
|
||||
"autotune",
|
||||
"has_flashinfer_moe",
|
||||
|
||||
Reference in New Issue
Block a user