[Bugfix]Fix DeepSeek-V4 fp8_ds_mla KV cache reshape (#47716)
Co-authored-by: yy-fighting <23518844576@qq.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
yy-fighting
Michael Goin
parent
ae098abe3f
commit
04adc8843b
@@ -56,7 +56,11 @@ from vllm.v1.attention.backends.mla.indexer import (
|
|||||||
get_max_prefill_buffer_size,
|
get_max_prefill_buffer_size,
|
||||||
)
|
)
|
||||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekV4SWACache
|
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekV4SWACache
|
||||||
from vllm.v1.kv_cache_interface import KVCacheSpec, MLAAttentionSpec
|
from vllm.v1.kv_cache_interface import (
|
||||||
|
KVCacheSpec,
|
||||||
|
MLAAttentionSpec,
|
||||||
|
get_kv_quant_mode,
|
||||||
|
)
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
@@ -616,6 +620,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
|||||||
cache_dtype_str=self.kv_cache_dtype,
|
cache_dtype_str=self.kv_cache_dtype,
|
||||||
alignment=576 if uses_fp8_ds_mla_layout else None,
|
alignment=576 if uses_fp8_ds_mla_layout else None,
|
||||||
model_version="deepseek_v4",
|
model_version="deepseek_v4",
|
||||||
|
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ from vllm.v1.kv_cache_interface import (
|
|||||||
KVCacheSpec,
|
KVCacheSpec,
|
||||||
MLAAttentionSpec,
|
MLAAttentionSpec,
|
||||||
SlidingWindowMLASpec,
|
SlidingWindowMLASpec,
|
||||||
|
get_kv_quant_mode,
|
||||||
)
|
)
|
||||||
|
|
||||||
# DeepseekV4 decode layer types, keyed by compress_ratio. Each type has a distinct
|
# DeepseekV4 decode layer types, keyed by compress_ratio. Each type has a distinct
|
||||||
@@ -91,6 +92,7 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase):
|
|||||||
cache_dtype_str=self.cache_config.cache_dtype,
|
cache_dtype_str=self.cache_config.cache_dtype,
|
||||||
alignment=576 if uses_fp8_ds_mla_layout else None,
|
alignment=576 if uses_fp8_ds_mla_layout else None,
|
||||||
model_version="deepseek_v4",
|
model_version="deepseek_v4",
|
||||||
|
kv_quant_mode=get_kv_quant_mode(self.cache_config.cache_dtype),
|
||||||
)
|
)
|
||||||
|
|
||||||
def forward(self): ...
|
def forward(self): ...
|
||||||
|
|||||||
@@ -7164,7 +7164,12 @@ class GPUModelRunner(
|
|||||||
layer_cache_dtype_str = (
|
layer_cache_dtype_str = (
|
||||||
"auto"
|
"auto"
|
||||||
if kv_cache_spec.kv_quant_mode == KVQuantMode.NONE
|
if kv_cache_spec.kv_quant_mode == KVQuantMode.NONE
|
||||||
else self.cache_config.cache_dtype
|
else getattr(
|
||||||
|
kv_cache_spec,
|
||||||
|
"cache_dtype_str",
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
or self.cache_config.cache_dtype
|
||||||
)
|
)
|
||||||
kv_cache_shape = attn_backend.get_kv_cache_shape(
|
kv_cache_shape = attn_backend.get_kv_cache_shape(
|
||||||
kernel_num_blocks,
|
kernel_num_blocks,
|
||||||
|
|||||||
Reference in New Issue
Block a user