Enable DeepSeek V4 and GLM-5.1 on SM120 (#43477)
Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Yongye Zhu
parent
3ce15fd574
commit
44d95069e9
@@ -208,6 +208,7 @@ from vllm.config import (
|
||||
get_current_vllm_config,
|
||||
get_current_vllm_config_or_none,
|
||||
)
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_dcp_group,
|
||||
is_global_first_rank,
|
||||
@@ -319,6 +320,22 @@ def _detect_output_quant_key(
|
||||
return kFp8StaticTensorSym
|
||||
|
||||
|
||||
def _canonicalize_sparse_mla_kv_cache_dtype(
|
||||
attn_backend: type[AttentionBackend],
|
||||
kv_cache_dtype: CacheDType,
|
||||
) -> CacheDType:
|
||||
backend_name = attn_backend.get_name()
|
||||
if backend_name == "FLASHMLA_SPARSE" and is_quantized_kv_cache(kv_cache_dtype):
|
||||
return "fp8_ds_mla"
|
||||
if backend_name == "FLASHINFER_MLA_SPARSE_SM120" and kv_cache_dtype in (
|
||||
"auto",
|
||||
"fp8",
|
||||
"fp8_e4m3",
|
||||
):
|
||||
return "fp8_ds_mla"
|
||||
return kv_cache_dtype
|
||||
|
||||
|
||||
class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
"""Multi-Head Latent Attention layer.
|
||||
|
||||
@@ -369,7 +386,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
|
||||
|
||||
if cache_config is not None:
|
||||
kv_cache_dtype = cache_config.cache_dtype
|
||||
kv_cache_dtype: CacheDType = cache_config.cache_dtype
|
||||
calculate_kv_scales = cache_config.calculate_kv_scales
|
||||
else:
|
||||
kv_cache_dtype = "auto"
|
||||
@@ -393,24 +410,22 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
num_heads=self.num_heads,
|
||||
)
|
||||
|
||||
# FlashMLA Sparse Attention fp8 backend uses "fp8_ds_mla" kv-cache format
|
||||
# Automatically convert fp8 kv-cache format to "fp8_ds_mla"
|
||||
if (
|
||||
self.attn_backend.get_name() == "FLASHMLA_SPARSE"
|
||||
and is_quantized_kv_cache(kv_cache_dtype)
|
||||
and kv_cache_dtype != "fp8_ds_mla"
|
||||
):
|
||||
assert cache_config is not None
|
||||
cache_config.cache_dtype = "fp8_ds_mla"
|
||||
kv_cache_dtype = "fp8_ds_mla"
|
||||
normalized_kv_cache_dtype = _canonicalize_sparse_mla_kv_cache_dtype(
|
||||
self.attn_backend, kv_cache_dtype
|
||||
)
|
||||
if normalized_kv_cache_dtype != kv_cache_dtype:
|
||||
if cache_config is not None:
|
||||
cache_config.cache_dtype = normalized_kv_cache_dtype
|
||||
kv_cache_dtype = normalized_kv_cache_dtype
|
||||
logger.info_once(
|
||||
"Using DeepSeek's fp8_ds_mla KV cache format. To use standard "
|
||||
"fp8 kv-cache format, please set `--attention-backend "
|
||||
"FLASHINFER_MLA_SPARSE`"
|
||||
"Using %s KV cache format for %s backend.",
|
||||
kv_cache_dtype,
|
||||
self.attn_backend.get_name(),
|
||||
)
|
||||
|
||||
if (
|
||||
self.attn_backend.get_name() == "FLASHINFER_MLA_SPARSE"
|
||||
and kv_cache_dtype != "fp8_ds_mla"
|
||||
and is_quantized_kv_cache(kv_cache_dtype)
|
||||
):
|
||||
logger.info_once(
|
||||
|
||||
Reference in New Issue
Block a user