[Bugfix][KV Cache][MLA] Align packed block strides for V3.2 sparse MLA (#55528)

Signed-off-by: lz <145014769+200lz@users.noreply.github.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
lz
2026-09-25 05:33:36 +00:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson Robert Shaw mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 3a58aeb1a1
commit 29468dde8b
6 changed files with 187 additions and 27 deletions
@@ -735,13 +735,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
self.kv_cache = kv_cache.squeeze(1)
if (
self._vllm_config.kernel_config.enable_jit_warmup
and self.attn_backend.get_name()
in (
"FLASHMLA_SPARSE",
"FLASHINFER_MLA_SPARSE",
"FLASHINFER_MLA_SPARSE_SM120",
"DEEPSEEK_V32_INDEXER",
)
and self._uses_flat_kv_cache()
):
from vllm.v1.attention.backends.mla.sparse_utils import (
_CONVERT_REQ_INDEX_TO_GLOBAL_INDEX_KERNEL,
@@ -754,6 +748,13 @@ class MLAAttention(nn.Module, AttentionLayerBase):
block_stride_rows=self.kv_cache.stride(0) // row_width,
)
def _uses_flat_kv_cache(self) -> bool:
backend = self.attn_backend.get_name()
return backend == "FLASHINFER_MLA_SPARSE" or (
backend == "FLASHMLA_SPARSE"
and self.kv_cache_dtype not in ("fp8_ds_mla", "nvfp4_ds_mla")
)
@property
def chunked_prefill_workspace_size(self) -> int:
if self._chunked_prefill_workspace_size is None:
@@ -1365,11 +1366,20 @@ class MLAAttention(nn.Module, AttentionLayerBase):
**common_kwargs,
sliding_window=self.sliding_window,
)
return MLAAttentionSpec(
spec = MLAAttentionSpec(
**common_kwargs,
is_index_group_leader=self.indexer is not None,
non_causal_multi_token_decode=self.non_causal_multi_token_decode,
)
# SM100 FlashMLA paged kernels also express TMA coordinates in token rows.
uses_tma_rows = (
self.attn_backend.get_name() == "FLASHMLA_SPARSE"
and self.kv_cache_dtype in ("fp8_ds_mla", "nvfp4_ds_mla")
and current_platform.is_device_capability_family(100)
)
if self._uses_flat_kv_cache() or uses_tma_rows:
spec = replace(spec, block_stride_alignment=spec.state_content_size_bytes)
return spec
def _v_up_proj(self, x: torch.Tensor, out: torch.Tensor):
# Convert from (B, N, L) to (N, B, L)