[Bugfix][KV Cache][MLA] Align packed block strides for V3.2 sparse MLA (#55528)
Signed-off-by: lz <145014769+200lz@users.noreply.github.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
Robert Shaw
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
3a58aeb1a1
commit
29468dde8b
@@ -735,13 +735,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
self.kv_cache = kv_cache.squeeze(1)
|
||||
if (
|
||||
self._vllm_config.kernel_config.enable_jit_warmup
|
||||
and self.attn_backend.get_name()
|
||||
in (
|
||||
"FLASHMLA_SPARSE",
|
||||
"FLASHINFER_MLA_SPARSE",
|
||||
"FLASHINFER_MLA_SPARSE_SM120",
|
||||
"DEEPSEEK_V32_INDEXER",
|
||||
)
|
||||
and self._uses_flat_kv_cache()
|
||||
):
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
_CONVERT_REQ_INDEX_TO_GLOBAL_INDEX_KERNEL,
|
||||
@@ -754,6 +748,13 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
block_stride_rows=self.kv_cache.stride(0) // row_width,
|
||||
)
|
||||
|
||||
def _uses_flat_kv_cache(self) -> bool:
|
||||
backend = self.attn_backend.get_name()
|
||||
return backend == "FLASHINFER_MLA_SPARSE" or (
|
||||
backend == "FLASHMLA_SPARSE"
|
||||
and self.kv_cache_dtype not in ("fp8_ds_mla", "nvfp4_ds_mla")
|
||||
)
|
||||
|
||||
@property
|
||||
def chunked_prefill_workspace_size(self) -> int:
|
||||
if self._chunked_prefill_workspace_size is None:
|
||||
@@ -1365,11 +1366,20 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
**common_kwargs,
|
||||
sliding_window=self.sliding_window,
|
||||
)
|
||||
return MLAAttentionSpec(
|
||||
spec = MLAAttentionSpec(
|
||||
**common_kwargs,
|
||||
is_index_group_leader=self.indexer is not None,
|
||||
non_causal_multi_token_decode=self.non_causal_multi_token_decode,
|
||||
)
|
||||
# SM100 FlashMLA paged kernels also express TMA coordinates in token rows.
|
||||
uses_tma_rows = (
|
||||
self.attn_backend.get_name() == "FLASHMLA_SPARSE"
|
||||
and self.kv_cache_dtype in ("fp8_ds_mla", "nvfp4_ds_mla")
|
||||
and current_platform.is_device_capability_family(100)
|
||||
)
|
||||
if self._uses_flat_kv_cache() or uses_tma_rows:
|
||||
spec = replace(spec, block_stride_alignment=spec.state_content_size_bytes)
|
||||
return spec
|
||||
|
||||
def _v_up_proj(self, x: torch.Tensor, out: torch.Tensor):
|
||||
# Convert from (B, N, L) to (N, B, L)
|
||||
|
||||
Reference in New Issue
Block a user