[Bugfix][KV Cache][MLA] Align packed block strides for V3.2 sparse MLA (#55528)
Signed-off-by: lz <145014769+200lz@users.noreply.github.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
Robert Shaw
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
3a58aeb1a1
commit
29468dde8b
@@ -1608,23 +1608,17 @@ def _get_kv_cache_bytes_per_block(
|
||||
for group in kv_cache_groups
|
||||
)
|
||||
assert bytes_per_block > 0
|
||||
hot_page_sizes = [
|
||||
alignments = [
|
||||
group.kv_cache_spec.page_size_bytes
|
||||
for group in kv_cache_groups
|
||||
if isinstance(group.kv_cache_spec, HiSparseHotSpec)
|
||||
]
|
||||
if hot_page_sizes:
|
||||
bytes_per_block = round_up(bytes_per_block, math.lcm(*hot_page_sizes))
|
||||
stride_alignments = [
|
||||
spec.block_stride_alignment
|
||||
alignments.extend(
|
||||
_get_per_layer_spec(group, layer_name).block_stride_alignment or 1
|
||||
for group in kv_cache_groups
|
||||
for layer_name in group.layer_names
|
||||
if isinstance(spec := _get_per_layer_spec(group, layer_name), MLAAttentionSpec)
|
||||
and spec.block_stride_alignment
|
||||
]
|
||||
if stride_alignments:
|
||||
bytes_per_block = round_up(bytes_per_block, math.lcm(*stride_alignments))
|
||||
return bytes_per_block
|
||||
)
|
||||
return round_up(bytes_per_block, math.lcm(*alignments))
|
||||
|
||||
|
||||
def validate_kv_cache_layout(
|
||||
|
||||
Reference in New Issue
Block a user