[Bugfix][KV Cache][MLA] Align packed block strides for V3.2 sparse MLA (#55528)

Signed-off-by: lz <145014769+200lz@users.noreply.github.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
lz
2026-09-25 05:33:36 +00:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson Robert Shaw mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 3a58aeb1a1
commit 29468dde8b
6 changed files with 187 additions and 27 deletions
+15 -7
View File
@@ -6,7 +6,7 @@ from __future__ import annotations
import copy
from collections import Counter
from collections.abc import Collection, Sequence
from dataclasses import dataclass, fields, replace
from dataclasses import dataclass, field, fields, replace
from enum import Enum, IntEnum
from fractions import Fraction
from functools import cached_property
@@ -159,6 +159,13 @@ class KVCacheSpec:
# number of tokens in a block
block_size: int
block_stride_alignment: int | None = field(default=None, kw_only=True)
"""Required byte alignment between physical blocks, including packed layers."""
def __post_init__(self):
if self.block_stride_alignment is not None and self.block_stride_alignment <= 0:
raise ValueError("block_stride_alignment must be positive")
@property
def prefix_cacheable(self) -> bool:
"""Whether this spec's group participates in prefix caching."""
@@ -492,6 +499,7 @@ class AttentionSpec(KVCacheSpec):
token (Whisper block pooling: ``Fraction(1, block_pool_size)``)."""
def __post_init__(self):
super().__post_init__()
if self.head_size_v is None:
object.__setattr__(self, "head_size_v", self.head_size)
@@ -598,6 +606,7 @@ class FullAttentionSpec(AttentionSpec):
)
merged_spec = cls(
block_size=specs[0].block_size,
block_stride_alignment=specs[0].block_stride_alignment,
num_kv_heads=specs[0].num_kv_heads,
head_size=specs[0].head_size,
head_size_v=specs[0].head_size_v,
@@ -648,12 +657,6 @@ class MLAAttentionSpec(FullAttentionSpec):
is_index_group_leader: bool = False
storage_block_size: int | None = None
"""Token width used to view storage when it differs from the kernel block."""
block_stride_alignment: int | None = None
"""Required alignment, in bytes, of the distance between consecutive
blocks of this cache. In block-major layouts that distance is the whole
block (all layers' pages), so the allocator rounds the block up to it.
DeepGEMM's paged sparse MQA-logits kernels address pages as
``base + page * stride`` and need it 512B-aligned."""
# Group capability enabled when any member flattens a non-causal query block
# into decode rows. Runtime metadata still selects causal vs. non-causal mode.
non_causal_multi_token_decode: bool = False
@@ -751,6 +754,7 @@ class RSWASpec(FullAttentionSpec):
base = FullAttentionSpec.merge(specs) # type: ignore[arg-type]
return cls(
block_size=base.block_size,
block_stride_alignment=base.block_stride_alignment,
num_kv_heads=base.num_kv_heads,
head_size=base.head_size,
head_size_v=base.head_size_v,
@@ -945,6 +949,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
sliding_window_set = set(spec.sliding_window for spec in specs)
extra_retained_set = set(spec.extra_retained_tokens for spec in specs)
bounded_replay_set = set(spec.bounded_replay for spec in specs)
block_stride_alignment_set = {spec.block_stride_alignment for spec in specs}
assert (
len(cache_dtype_str_set) == 1
and len(tokens_per_state_set) == 1
@@ -952,6 +957,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
and len(sliding_window_set) == 1
and len(extra_retained_set) == 1
and len(bounded_replay_set) == 1
and len(block_stride_alignment_set) == 1
), (
"All attention layers in the same KV cache group must use the same "
"quantization method, tokens per state, model version, sliding "
@@ -959,6 +965,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
)
return cls(
block_size=specs[0].block_size,
block_stride_alignment=block_stride_alignment_set.pop(),
num_kv_heads=specs[0].num_kv_heads,
head_size=specs[0].head_size,
dtype=specs[0].dtype,
@@ -1175,6 +1182,7 @@ class SinkFullAttentionSpec(FullAttentionSpec):
head_size=specs[0].head_size,
head_size_v=specs[0].head_size_v,
sink_len=specs[0].sink_len,
block_stride_alignment=specs[0].block_stride_alignment,
dtype=specs[0].dtype,
kv_quant_mode=specs[0].kv_quant_mode,
page_size_padded=specs[0].page_size_padded,