[Attention] Make FlexAttention and FlashAttention use num-blocks first layouts (#42095)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
co-authored by
Matthew Bonanni
Nicolò Lucchesi
parent
d8eebe6d97
commit
7e33081cee
@@ -420,7 +420,7 @@ class TransferTopology:
|
||||
head_size=1,
|
||||
)
|
||||
logger.debug("Test kv_cache_shape: %s", kv_cache_shape)
|
||||
# Non-MLA backends caches have 5 dims [2, num_blocks, H,N,D],
|
||||
# Non-MLA backends caches have 5 dims [num_blocks, 2, H,N,D],
|
||||
# we just mock num_blocks to 1 for the dimension check below.
|
||||
# Hybrid SSM models assume a single blocks_first layout
|
||||
self._is_kv_layout_blocks_first = self.is_mamba or (
|
||||
@@ -484,6 +484,15 @@ class TransferTopology:
|
||||
def cross_layers_blocks(self) -> bool:
|
||||
return self._cross_layers_blocks
|
||||
|
||||
@property
|
||||
def virtually_split_kv_in_blocks(self) -> bool:
|
||||
# Whether to logically split each block into K and V halves.
|
||||
# Applies when K/V are interleaved within each block (blocks-first),
|
||||
# but NOT when cross-layer blocks are used — cross-layer blocks have
|
||||
# per-layer K/V interleaving (L0_K, L0_V, L1_K, L1_V, ...) so a
|
||||
# simple half-split does not separate K from V.
|
||||
return self._is_kv_layout_blocks_first and not self._cross_layers_blocks
|
||||
|
||||
@property
|
||||
def split_k_and_v(self) -> bool:
|
||||
# Whether to register regions for K and V separately (when present).
|
||||
|
||||
Reference in New Issue
Block a user