[Bugfix][KV Cache][MLA] Align packed block strides for V3.2 sparse MLA (#55528)

Signed-off-by: lz <145014769+200lz@users.noreply.github.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
lz
2026-09-25 05:33:36 +00:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson Robert Shaw mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 3a58aeb1a1
commit 29468dde8b
6 changed files with 187 additions and 27 deletions
@@ -735,13 +735,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
self.kv_cache = kv_cache.squeeze(1)
if (
self._vllm_config.kernel_config.enable_jit_warmup
and self.attn_backend.get_name()
in (
"FLASHMLA_SPARSE",
"FLASHINFER_MLA_SPARSE",
"FLASHINFER_MLA_SPARSE_SM120",
"DEEPSEEK_V32_INDEXER",
)
and self._uses_flat_kv_cache()
):
from vllm.v1.attention.backends.mla.sparse_utils import (
_CONVERT_REQ_INDEX_TO_GLOBAL_INDEX_KERNEL,
@@ -754,6 +748,13 @@ class MLAAttention(nn.Module, AttentionLayerBase):
block_stride_rows=self.kv_cache.stride(0) // row_width,
)
def _uses_flat_kv_cache(self) -> bool:
backend = self.attn_backend.get_name()
return backend == "FLASHINFER_MLA_SPARSE" or (
backend == "FLASHMLA_SPARSE"
and self.kv_cache_dtype not in ("fp8_ds_mla", "nvfp4_ds_mla")
)
@property
def chunked_prefill_workspace_size(self) -> int:
if self._chunked_prefill_workspace_size is None:
@@ -1365,11 +1366,20 @@ class MLAAttention(nn.Module, AttentionLayerBase):
**common_kwargs,
sliding_window=self.sliding_window,
)
return MLAAttentionSpec(
spec = MLAAttentionSpec(
**common_kwargs,
is_index_group_leader=self.indexer is not None,
non_causal_multi_token_decode=self.non_causal_multi_token_decode,
)
# SM100 FlashMLA paged kernels also express TMA coordinates in token rows.
uses_tma_rows = (
self.attn_backend.get_name() == "FLASHMLA_SPARSE"
and self.kv_cache_dtype in ("fp8_ds_mla", "nvfp4_ds_mla")
and current_platform.is_device_capability_family(100)
)
if self._uses_flat_kv_cache() or uses_tma_rows:
spec = replace(spec, block_stride_alignment=spec.state_content_size_bytes)
return spec
def _v_up_proj(self, x: torch.Tensor, out: torch.Tensor):
# Convert from (B, N, L) to (N, B, L)
+5 -11
View File
@@ -1608,23 +1608,17 @@ def _get_kv_cache_bytes_per_block(
for group in kv_cache_groups
)
assert bytes_per_block > 0
hot_page_sizes = [
alignments = [
group.kv_cache_spec.page_size_bytes
for group in kv_cache_groups
if isinstance(group.kv_cache_spec, HiSparseHotSpec)
]
if hot_page_sizes:
bytes_per_block = round_up(bytes_per_block, math.lcm(*hot_page_sizes))
stride_alignments = [
spec.block_stride_alignment
alignments.extend(
_get_per_layer_spec(group, layer_name).block_stride_alignment or 1
for group in kv_cache_groups
for layer_name in group.layer_names
if isinstance(spec := _get_per_layer_spec(group, layer_name), MLAAttentionSpec)
and spec.block_stride_alignment
]
if stride_alignments:
bytes_per_block = round_up(bytes_per_block, math.lcm(*stride_alignments))
return bytes_per_block
)
return round_up(bytes_per_block, math.lcm(*alignments))
def validate_kv_cache_layout(
+15 -7
View File
@@ -6,7 +6,7 @@ from __future__ import annotations
import copy
from collections import Counter
from collections.abc import Collection, Sequence
from dataclasses import dataclass, fields, replace
from dataclasses import dataclass, field, fields, replace
from enum import Enum, IntEnum
from fractions import Fraction
from functools import cached_property
@@ -159,6 +159,13 @@ class KVCacheSpec:
# number of tokens in a block
block_size: int
block_stride_alignment: int | None = field(default=None, kw_only=True)
"""Required byte alignment between physical blocks, including packed layers."""
def __post_init__(self):
if self.block_stride_alignment is not None and self.block_stride_alignment <= 0:
raise ValueError("block_stride_alignment must be positive")
@property
def prefix_cacheable(self) -> bool:
"""Whether this spec's group participates in prefix caching."""
@@ -492,6 +499,7 @@ class AttentionSpec(KVCacheSpec):
token (Whisper block pooling: ``Fraction(1, block_pool_size)``)."""
def __post_init__(self):
super().__post_init__()
if self.head_size_v is None:
object.__setattr__(self, "head_size_v", self.head_size)
@@ -598,6 +606,7 @@ class FullAttentionSpec(AttentionSpec):
)
merged_spec = cls(
block_size=specs[0].block_size,
block_stride_alignment=specs[0].block_stride_alignment,
num_kv_heads=specs[0].num_kv_heads,
head_size=specs[0].head_size,
head_size_v=specs[0].head_size_v,
@@ -648,12 +657,6 @@ class MLAAttentionSpec(FullAttentionSpec):
is_index_group_leader: bool = False
storage_block_size: int | None = None
"""Token width used to view storage when it differs from the kernel block."""
block_stride_alignment: int | None = None
"""Required alignment, in bytes, of the distance between consecutive
blocks of this cache. In block-major layouts that distance is the whole
block (all layers' pages), so the allocator rounds the block up to it.
DeepGEMM's paged sparse MQA-logits kernels address pages as
``base + page * stride`` and need it 512B-aligned."""
# Group capability enabled when any member flattens a non-causal query block
# into decode rows. Runtime metadata still selects causal vs. non-causal mode.
non_causal_multi_token_decode: bool = False
@@ -751,6 +754,7 @@ class RSWASpec(FullAttentionSpec):
base = FullAttentionSpec.merge(specs) # type: ignore[arg-type]
return cls(
block_size=base.block_size,
block_stride_alignment=base.block_stride_alignment,
num_kv_heads=base.num_kv_heads,
head_size=base.head_size,
head_size_v=base.head_size_v,
@@ -945,6 +949,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
sliding_window_set = set(spec.sliding_window for spec in specs)
extra_retained_set = set(spec.extra_retained_tokens for spec in specs)
bounded_replay_set = set(spec.bounded_replay for spec in specs)
block_stride_alignment_set = {spec.block_stride_alignment for spec in specs}
assert (
len(cache_dtype_str_set) == 1
and len(tokens_per_state_set) == 1
@@ -952,6 +957,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
and len(sliding_window_set) == 1
and len(extra_retained_set) == 1
and len(bounded_replay_set) == 1
and len(block_stride_alignment_set) == 1
), (
"All attention layers in the same KV cache group must use the same "
"quantization method, tokens per state, model version, sliding "
@@ -959,6 +965,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
)
return cls(
block_size=specs[0].block_size,
block_stride_alignment=block_stride_alignment_set.pop(),
num_kv_heads=specs[0].num_kv_heads,
head_size=specs[0].head_size,
dtype=specs[0].dtype,
@@ -1175,6 +1182,7 @@ class SinkFullAttentionSpec(FullAttentionSpec):
head_size=specs[0].head_size,
head_size_v=specs[0].head_size_v,
sink_len=specs[0].sink_len,
block_stride_alignment=specs[0].block_stride_alignment,
dtype=specs[0].dtype,
kv_quant_mode=specs[0].kv_quant_mode,
page_size_padded=specs[0].page_size_padded,