[Attention] Make FlexAttention and FlashAttention use num-blocks first layouts (#42095)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
Lucas Wilkinson
2026-05-26 19:55:56 -07:00
committed by GitHub
co-authored by Matthew Bonanni Nicolò Lucchesi
parent d8eebe6d97
commit 7e33081cee
20 changed files with 209 additions and 389 deletions
+4 -4
View File
@@ -530,8 +530,8 @@ class CudaPlatformBase(Platform):
dst_block_indices: torch.Tensor,
) -> None:
"""Copy blocks from src_cache to dst_cache on GPU."""
_src_cache = src_cache[:, src_block_indices]
dst_cache[:, dst_block_indices] = _src_cache.to(dst_cache.device)
_src_cache = src_cache[src_block_indices]
dst_cache[dst_block_indices] = _src_cache.to(dst_cache.device)
@classmethod
def swap_out_blocks_to_host(
@@ -542,8 +542,8 @@ class CudaPlatformBase(Platform):
dst_block_indices: torch.Tensor,
) -> None:
"""Copy blocks from GPU to host (CPU)."""
_src_cache = src_cache[:, src_block_indices]
dst_cache[:, dst_block_indices] = _src_cache.cpu()
_src_cache = src_cache[src_block_indices]
dst_cache[dst_block_indices] = _src_cache.cpu()
@classmethod
def support_hybrid_kv_cache(cls) -> bool:
+4 -4
View File
@@ -901,8 +901,8 @@ class RocmPlatform(Platform):
dst_block_indices: torch.Tensor,
) -> None:
"""Copy blocks from src_cache to dst_cache on GPU."""
_src_cache = src_cache[:, src_block_indices]
dst_cache[:, dst_block_indices] = _src_cache.to(dst_cache.device)
_src_cache = src_cache[src_block_indices]
dst_cache[dst_block_indices] = _src_cache.to(dst_cache.device)
@classmethod
def swap_out_blocks_to_host(
@@ -913,8 +913,8 @@ class RocmPlatform(Platform):
dst_block_indices: torch.Tensor,
) -> None:
"""Copy blocks from GPU to host (CPU)."""
_src_cache = src_cache[:, src_block_indices]
dst_cache[:, dst_block_indices] = _src_cache.cpu()
_src_cache = src_cache[src_block_indices]
dst_cache[dst_block_indices] = _src_cache.cpu()
@classmethod
def support_hybrid_kv_cache(cls) -> bool:
+4 -4
View File
@@ -374,8 +374,8 @@ class XPUPlatform(Platform):
dst_block_indices: torch.Tensor,
) -> None:
"""Copy blocks from src_cache to dst_cache on XPU."""
_src_cache = src_cache[:, src_block_indices]
dst_cache[:, dst_block_indices] = _src_cache.to(dst_cache.device)
_src_cache = src_cache[src_block_indices]
dst_cache[dst_block_indices] = _src_cache.to(dst_cache.device)
@classmethod
def swap_out_blocks_to_host(
@@ -386,8 +386,8 @@ class XPUPlatform(Platform):
dst_block_indices: torch.Tensor,
) -> None:
"""Copy blocks from XPU to host (CPU)."""
_src_cache = src_cache[:, src_block_indices]
dst_cache[:, dst_block_indices] = _src_cache.cpu()
_src_cache = src_cache[src_block_indices]
dst_cache[dst_block_indices] = _src_cache.cpu()
@classmethod
def num_compute_units(cls, device_id: int = 0) -> int: