[Attention] Make FlexAttention and FlashAttention use num-blocks first layouts (#42095)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
co-authored by
Matthew Bonanni
Nicolò Lucchesi
parent
d8eebe6d97
commit
7e33081cee
@@ -530,8 +530,8 @@ class CudaPlatformBase(Platform):
|
||||
dst_block_indices: torch.Tensor,
|
||||
) -> None:
|
||||
"""Copy blocks from src_cache to dst_cache on GPU."""
|
||||
_src_cache = src_cache[:, src_block_indices]
|
||||
dst_cache[:, dst_block_indices] = _src_cache.to(dst_cache.device)
|
||||
_src_cache = src_cache[src_block_indices]
|
||||
dst_cache[dst_block_indices] = _src_cache.to(dst_cache.device)
|
||||
|
||||
@classmethod
|
||||
def swap_out_blocks_to_host(
|
||||
@@ -542,8 +542,8 @@ class CudaPlatformBase(Platform):
|
||||
dst_block_indices: torch.Tensor,
|
||||
) -> None:
|
||||
"""Copy blocks from GPU to host (CPU)."""
|
||||
_src_cache = src_cache[:, src_block_indices]
|
||||
dst_cache[:, dst_block_indices] = _src_cache.cpu()
|
||||
_src_cache = src_cache[src_block_indices]
|
||||
dst_cache[dst_block_indices] = _src_cache.cpu()
|
||||
|
||||
@classmethod
|
||||
def support_hybrid_kv_cache(cls) -> bool:
|
||||
|
||||
@@ -901,8 +901,8 @@ class RocmPlatform(Platform):
|
||||
dst_block_indices: torch.Tensor,
|
||||
) -> None:
|
||||
"""Copy blocks from src_cache to dst_cache on GPU."""
|
||||
_src_cache = src_cache[:, src_block_indices]
|
||||
dst_cache[:, dst_block_indices] = _src_cache.to(dst_cache.device)
|
||||
_src_cache = src_cache[src_block_indices]
|
||||
dst_cache[dst_block_indices] = _src_cache.to(dst_cache.device)
|
||||
|
||||
@classmethod
|
||||
def swap_out_blocks_to_host(
|
||||
@@ -913,8 +913,8 @@ class RocmPlatform(Platform):
|
||||
dst_block_indices: torch.Tensor,
|
||||
) -> None:
|
||||
"""Copy blocks from GPU to host (CPU)."""
|
||||
_src_cache = src_cache[:, src_block_indices]
|
||||
dst_cache[:, dst_block_indices] = _src_cache.cpu()
|
||||
_src_cache = src_cache[src_block_indices]
|
||||
dst_cache[dst_block_indices] = _src_cache.cpu()
|
||||
|
||||
@classmethod
|
||||
def support_hybrid_kv_cache(cls) -> bool:
|
||||
|
||||
@@ -374,8 +374,8 @@ class XPUPlatform(Platform):
|
||||
dst_block_indices: torch.Tensor,
|
||||
) -> None:
|
||||
"""Copy blocks from src_cache to dst_cache on XPU."""
|
||||
_src_cache = src_cache[:, src_block_indices]
|
||||
dst_cache[:, dst_block_indices] = _src_cache.to(dst_cache.device)
|
||||
_src_cache = src_cache[src_block_indices]
|
||||
dst_cache[dst_block_indices] = _src_cache.to(dst_cache.device)
|
||||
|
||||
@classmethod
|
||||
def swap_out_blocks_to_host(
|
||||
@@ -386,8 +386,8 @@ class XPUPlatform(Platform):
|
||||
dst_block_indices: torch.Tensor,
|
||||
) -> None:
|
||||
"""Copy blocks from XPU to host (CPU)."""
|
||||
_src_cache = src_cache[:, src_block_indices]
|
||||
dst_cache[:, dst_block_indices] = _src_cache.cpu()
|
||||
_src_cache = src_cache[src_block_indices]
|
||||
dst_cache[dst_block_indices] = _src_cache.cpu()
|
||||
|
||||
@classmethod
|
||||
def num_compute_units(cls, device_id: int = 0) -> int:
|
||||
|
||||
Reference in New Issue
Block a user