[Bugfix][KV Cache][MLA] Align packed block strides for V3.2 sparse MLA (#55528)
Signed-off-by: lz <145014769+200lz@users.noreply.github.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
Robert Shaw
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
3a58aeb1a1
commit
29468dde8b
@@ -735,13 +735,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
self.kv_cache = kv_cache.squeeze(1)
|
||||
if (
|
||||
self._vllm_config.kernel_config.enable_jit_warmup
|
||||
and self.attn_backend.get_name()
|
||||
in (
|
||||
"FLASHMLA_SPARSE",
|
||||
"FLASHINFER_MLA_SPARSE",
|
||||
"FLASHINFER_MLA_SPARSE_SM120",
|
||||
"DEEPSEEK_V32_INDEXER",
|
||||
)
|
||||
and self._uses_flat_kv_cache()
|
||||
):
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
_CONVERT_REQ_INDEX_TO_GLOBAL_INDEX_KERNEL,
|
||||
@@ -754,6 +748,13 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
block_stride_rows=self.kv_cache.stride(0) // row_width,
|
||||
)
|
||||
|
||||
def _uses_flat_kv_cache(self) -> bool:
|
||||
backend = self.attn_backend.get_name()
|
||||
return backend == "FLASHINFER_MLA_SPARSE" or (
|
||||
backend == "FLASHMLA_SPARSE"
|
||||
and self.kv_cache_dtype not in ("fp8_ds_mla", "nvfp4_ds_mla")
|
||||
)
|
||||
|
||||
@property
|
||||
def chunked_prefill_workspace_size(self) -> int:
|
||||
if self._chunked_prefill_workspace_size is None:
|
||||
@@ -1365,11 +1366,20 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
**common_kwargs,
|
||||
sliding_window=self.sliding_window,
|
||||
)
|
||||
return MLAAttentionSpec(
|
||||
spec = MLAAttentionSpec(
|
||||
**common_kwargs,
|
||||
is_index_group_leader=self.indexer is not None,
|
||||
non_causal_multi_token_decode=self.non_causal_multi_token_decode,
|
||||
)
|
||||
# SM100 FlashMLA paged kernels also express TMA coordinates in token rows.
|
||||
uses_tma_rows = (
|
||||
self.attn_backend.get_name() == "FLASHMLA_SPARSE"
|
||||
and self.kv_cache_dtype in ("fp8_ds_mla", "nvfp4_ds_mla")
|
||||
and current_platform.is_device_capability_family(100)
|
||||
)
|
||||
if self._uses_flat_kv_cache() or uses_tma_rows:
|
||||
spec = replace(spec, block_stride_alignment=spec.state_content_size_bytes)
|
||||
return spec
|
||||
|
||||
def _v_up_proj(self, x: torch.Tensor, out: torch.Tensor):
|
||||
# Convert from (B, N, L) to (N, B, L)
|
||||
|
||||
@@ -1608,23 +1608,17 @@ def _get_kv_cache_bytes_per_block(
|
||||
for group in kv_cache_groups
|
||||
)
|
||||
assert bytes_per_block > 0
|
||||
hot_page_sizes = [
|
||||
alignments = [
|
||||
group.kv_cache_spec.page_size_bytes
|
||||
for group in kv_cache_groups
|
||||
if isinstance(group.kv_cache_spec, HiSparseHotSpec)
|
||||
]
|
||||
if hot_page_sizes:
|
||||
bytes_per_block = round_up(bytes_per_block, math.lcm(*hot_page_sizes))
|
||||
stride_alignments = [
|
||||
spec.block_stride_alignment
|
||||
alignments.extend(
|
||||
_get_per_layer_spec(group, layer_name).block_stride_alignment or 1
|
||||
for group in kv_cache_groups
|
||||
for layer_name in group.layer_names
|
||||
if isinstance(spec := _get_per_layer_spec(group, layer_name), MLAAttentionSpec)
|
||||
and spec.block_stride_alignment
|
||||
]
|
||||
if stride_alignments:
|
||||
bytes_per_block = round_up(bytes_per_block, math.lcm(*stride_alignments))
|
||||
return bytes_per_block
|
||||
)
|
||||
return round_up(bytes_per_block, math.lcm(*alignments))
|
||||
|
||||
|
||||
def validate_kv_cache_layout(
|
||||
|
||||
@@ -6,7 +6,7 @@ from __future__ import annotations
|
||||
import copy
|
||||
from collections import Counter
|
||||
from collections.abc import Collection, Sequence
|
||||
from dataclasses import dataclass, fields, replace
|
||||
from dataclasses import dataclass, field, fields, replace
|
||||
from enum import Enum, IntEnum
|
||||
from fractions import Fraction
|
||||
from functools import cached_property
|
||||
@@ -159,6 +159,13 @@ class KVCacheSpec:
|
||||
# number of tokens in a block
|
||||
block_size: int
|
||||
|
||||
block_stride_alignment: int | None = field(default=None, kw_only=True)
|
||||
"""Required byte alignment between physical blocks, including packed layers."""
|
||||
|
||||
def __post_init__(self):
|
||||
if self.block_stride_alignment is not None and self.block_stride_alignment <= 0:
|
||||
raise ValueError("block_stride_alignment must be positive")
|
||||
|
||||
@property
|
||||
def prefix_cacheable(self) -> bool:
|
||||
"""Whether this spec's group participates in prefix caching."""
|
||||
@@ -492,6 +499,7 @@ class AttentionSpec(KVCacheSpec):
|
||||
token (Whisper block pooling: ``Fraction(1, block_pool_size)``)."""
|
||||
|
||||
def __post_init__(self):
|
||||
super().__post_init__()
|
||||
if self.head_size_v is None:
|
||||
object.__setattr__(self, "head_size_v", self.head_size)
|
||||
|
||||
@@ -598,6 +606,7 @@ class FullAttentionSpec(AttentionSpec):
|
||||
)
|
||||
merged_spec = cls(
|
||||
block_size=specs[0].block_size,
|
||||
block_stride_alignment=specs[0].block_stride_alignment,
|
||||
num_kv_heads=specs[0].num_kv_heads,
|
||||
head_size=specs[0].head_size,
|
||||
head_size_v=specs[0].head_size_v,
|
||||
@@ -648,12 +657,6 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
is_index_group_leader: bool = False
|
||||
storage_block_size: int | None = None
|
||||
"""Token width used to view storage when it differs from the kernel block."""
|
||||
block_stride_alignment: int | None = None
|
||||
"""Required alignment, in bytes, of the distance between consecutive
|
||||
blocks of this cache. In block-major layouts that distance is the whole
|
||||
block (all layers' pages), so the allocator rounds the block up to it.
|
||||
DeepGEMM's paged sparse MQA-logits kernels address pages as
|
||||
``base + page * stride`` and need it 512B-aligned."""
|
||||
# Group capability enabled when any member flattens a non-causal query block
|
||||
# into decode rows. Runtime metadata still selects causal vs. non-causal mode.
|
||||
non_causal_multi_token_decode: bool = False
|
||||
@@ -751,6 +754,7 @@ class RSWASpec(FullAttentionSpec):
|
||||
base = FullAttentionSpec.merge(specs) # type: ignore[arg-type]
|
||||
return cls(
|
||||
block_size=base.block_size,
|
||||
block_stride_alignment=base.block_stride_alignment,
|
||||
num_kv_heads=base.num_kv_heads,
|
||||
head_size=base.head_size,
|
||||
head_size_v=base.head_size_v,
|
||||
@@ -945,6 +949,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
sliding_window_set = set(spec.sliding_window for spec in specs)
|
||||
extra_retained_set = set(spec.extra_retained_tokens for spec in specs)
|
||||
bounded_replay_set = set(spec.bounded_replay for spec in specs)
|
||||
block_stride_alignment_set = {spec.block_stride_alignment for spec in specs}
|
||||
assert (
|
||||
len(cache_dtype_str_set) == 1
|
||||
and len(tokens_per_state_set) == 1
|
||||
@@ -952,6 +957,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
and len(sliding_window_set) == 1
|
||||
and len(extra_retained_set) == 1
|
||||
and len(bounded_replay_set) == 1
|
||||
and len(block_stride_alignment_set) == 1
|
||||
), (
|
||||
"All attention layers in the same KV cache group must use the same "
|
||||
"quantization method, tokens per state, model version, sliding "
|
||||
@@ -959,6 +965,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
)
|
||||
return cls(
|
||||
block_size=specs[0].block_size,
|
||||
block_stride_alignment=block_stride_alignment_set.pop(),
|
||||
num_kv_heads=specs[0].num_kv_heads,
|
||||
head_size=specs[0].head_size,
|
||||
dtype=specs[0].dtype,
|
||||
@@ -1175,6 +1182,7 @@ class SinkFullAttentionSpec(FullAttentionSpec):
|
||||
head_size=specs[0].head_size,
|
||||
head_size_v=specs[0].head_size_v,
|
||||
sink_len=specs[0].sink_len,
|
||||
block_stride_alignment=specs[0].block_stride_alignment,
|
||||
dtype=specs[0].dtype,
|
||||
kv_quant_mode=specs[0].kv_quant_mode,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
|
||||
Reference in New Issue
Block a user