[Bugfix][KV Cache][MLA] Align packed block strides for V3.2 sparse MLA (#55528)
Signed-off-by: lz <145014769+200lz@users.noreply.github.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
Robert Shaw
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
3a58aeb1a1
commit
29468dde8b
@@ -6,7 +6,7 @@ from __future__ import annotations
|
||||
import copy
|
||||
from collections import Counter
|
||||
from collections.abc import Collection, Sequence
|
||||
from dataclasses import dataclass, fields, replace
|
||||
from dataclasses import dataclass, field, fields, replace
|
||||
from enum import Enum, IntEnum
|
||||
from fractions import Fraction
|
||||
from functools import cached_property
|
||||
@@ -159,6 +159,13 @@ class KVCacheSpec:
|
||||
# number of tokens in a block
|
||||
block_size: int
|
||||
|
||||
block_stride_alignment: int | None = field(default=None, kw_only=True)
|
||||
"""Required byte alignment between physical blocks, including packed layers."""
|
||||
|
||||
def __post_init__(self):
|
||||
if self.block_stride_alignment is not None and self.block_stride_alignment <= 0:
|
||||
raise ValueError("block_stride_alignment must be positive")
|
||||
|
||||
@property
|
||||
def prefix_cacheable(self) -> bool:
|
||||
"""Whether this spec's group participates in prefix caching."""
|
||||
@@ -492,6 +499,7 @@ class AttentionSpec(KVCacheSpec):
|
||||
token (Whisper block pooling: ``Fraction(1, block_pool_size)``)."""
|
||||
|
||||
def __post_init__(self):
|
||||
super().__post_init__()
|
||||
if self.head_size_v is None:
|
||||
object.__setattr__(self, "head_size_v", self.head_size)
|
||||
|
||||
@@ -598,6 +606,7 @@ class FullAttentionSpec(AttentionSpec):
|
||||
)
|
||||
merged_spec = cls(
|
||||
block_size=specs[0].block_size,
|
||||
block_stride_alignment=specs[0].block_stride_alignment,
|
||||
num_kv_heads=specs[0].num_kv_heads,
|
||||
head_size=specs[0].head_size,
|
||||
head_size_v=specs[0].head_size_v,
|
||||
@@ -648,12 +657,6 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
is_index_group_leader: bool = False
|
||||
storage_block_size: int | None = None
|
||||
"""Token width used to view storage when it differs from the kernel block."""
|
||||
block_stride_alignment: int | None = None
|
||||
"""Required alignment, in bytes, of the distance between consecutive
|
||||
blocks of this cache. In block-major layouts that distance is the whole
|
||||
block (all layers' pages), so the allocator rounds the block up to it.
|
||||
DeepGEMM's paged sparse MQA-logits kernels address pages as
|
||||
``base + page * stride`` and need it 512B-aligned."""
|
||||
# Group capability enabled when any member flattens a non-causal query block
|
||||
# into decode rows. Runtime metadata still selects causal vs. non-causal mode.
|
||||
non_causal_multi_token_decode: bool = False
|
||||
@@ -751,6 +754,7 @@ class RSWASpec(FullAttentionSpec):
|
||||
base = FullAttentionSpec.merge(specs) # type: ignore[arg-type]
|
||||
return cls(
|
||||
block_size=base.block_size,
|
||||
block_stride_alignment=base.block_stride_alignment,
|
||||
num_kv_heads=base.num_kv_heads,
|
||||
head_size=base.head_size,
|
||||
head_size_v=base.head_size_v,
|
||||
@@ -945,6 +949,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
sliding_window_set = set(spec.sliding_window for spec in specs)
|
||||
extra_retained_set = set(spec.extra_retained_tokens for spec in specs)
|
||||
bounded_replay_set = set(spec.bounded_replay for spec in specs)
|
||||
block_stride_alignment_set = {spec.block_stride_alignment for spec in specs}
|
||||
assert (
|
||||
len(cache_dtype_str_set) == 1
|
||||
and len(tokens_per_state_set) == 1
|
||||
@@ -952,6 +957,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
and len(sliding_window_set) == 1
|
||||
and len(extra_retained_set) == 1
|
||||
and len(bounded_replay_set) == 1
|
||||
and len(block_stride_alignment_set) == 1
|
||||
), (
|
||||
"All attention layers in the same KV cache group must use the same "
|
||||
"quantization method, tokens per state, model version, sliding "
|
||||
@@ -959,6 +965,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
)
|
||||
return cls(
|
||||
block_size=specs[0].block_size,
|
||||
block_stride_alignment=block_stride_alignment_set.pop(),
|
||||
num_kv_heads=specs[0].num_kv_heads,
|
||||
head_size=specs[0].head_size,
|
||||
dtype=specs[0].dtype,
|
||||
@@ -1175,6 +1182,7 @@ class SinkFullAttentionSpec(FullAttentionSpec):
|
||||
head_size=specs[0].head_size,
|
||||
head_size_v=specs[0].head_size_v,
|
||||
sink_len=specs[0].sink_len,
|
||||
block_stride_alignment=specs[0].block_stride_alignment,
|
||||
dtype=specs[0].dtype,
|
||||
kv_quant_mode=specs[0].kv_quant_mode,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
|
||||
Reference in New Issue
Block a user