[Attention] Make FlexAttention and FlashAttention use num-blocks first layouts (#42095)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
Lucas Wilkinson
2026-05-26 19:55:56 -07:00
committed by GitHub
co-authored by Matthew Bonanni Nicolò Lucchesi
parent d8eebe6d97
commit 7e33081cee
20 changed files with 209 additions and 389 deletions
@@ -420,7 +420,7 @@ class TransferTopology:
head_size=1,
)
logger.debug("Test kv_cache_shape: %s", kv_cache_shape)
# Non-MLA backends caches have 5 dims [2, num_blocks, H,N,D],
# Non-MLA backends caches have 5 dims [num_blocks, 2, H,N,D],
# we just mock num_blocks to 1 for the dimension check below.
# Hybrid SSM models assume a single blocks_first layout
self._is_kv_layout_blocks_first = self.is_mamba or (
@@ -484,6 +484,15 @@ class TransferTopology:
def cross_layers_blocks(self) -> bool:
return self._cross_layers_blocks
@property
def virtually_split_kv_in_blocks(self) -> bool:
# Whether to logically split each block into K and V halves.
# Applies when K/V are interleaved within each block (blocks-first),
# but NOT when cross-layer blocks are used — cross-layer blocks have
# per-layer K/V interleaving (L0_K, L0_V, L1_K, L1_V, ...) so a
# simple half-split does not separate K from V.
return self._is_kv_layout_blocks_first and not self._cross_layers_blocks
@property
def split_k_and_v(self) -> bool:
# Whether to register regions for K and V separately (when present).