[CPU] fixes heterogeneous NIXL KV transfer into CPU_ATTN decode workers (#47871)
Signed-off-by: Spycsh <sihan.chen@intel.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Li, Jiang <jiang1.li@intel.com>
parent
37bf988c2f
commit
4938d44a3b
@@ -1935,13 +1935,8 @@ class NixlBaseConnectorWorker:
|
||||
indices = torch.tensor(block_ids, device=self.device_type, dtype=torch.long)
|
||||
|
||||
for _, cache_or_caches in self.device_kv_caches.items():
|
||||
blocks_to_update = cache_or_caches.index_select(1, indices)
|
||||
current_platform.pack_kv_cache(
|
||||
key=blocks_to_update[0],
|
||||
value=blocks_to_update[1],
|
||||
key_cache=cache_or_caches[0],
|
||||
value_cache=cache_or_caches[1],
|
||||
block_ids=block_ids,
|
||||
kv_cache=cache_or_caches,
|
||||
indices=indices,
|
||||
)
|
||||
|
||||
|
||||
+14
-7
@@ -461,11 +461,7 @@ class CpuPlatform(Platform):
|
||||
@classmethod
|
||||
def pack_kv_cache(
|
||||
cls,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
block_ids: list[int],
|
||||
kv_cache: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
) -> None:
|
||||
"""
|
||||
@@ -476,15 +472,26 @@ class CpuPlatform(Platform):
|
||||
from vllm._custom_ops import cpu_attn_reshape_and_cache
|
||||
from vllm.v1.attention.backends.cpu_attn import _get_attn_isa
|
||||
|
||||
num_blocks, num_kv_heads, block_size, fused_head_size = kv_cache.shape
|
||||
head_size = fused_head_size // 2
|
||||
|
||||
# Fused path used by heterogeneous NIXL CPU_ATTN post-processing.
|
||||
blocks_to_update = kv_cache.index_select(0, indices)
|
||||
key = blocks_to_update[..., :head_size]
|
||||
value = blocks_to_update[..., head_size:]
|
||||
|
||||
key_cache, value_cache = kv_cache.view(
|
||||
num_blocks, num_kv_heads, block_size * 2, head_size
|
||||
).chunk(2, dim=2)
|
||||
|
||||
dtype = key.dtype
|
||||
# For CPU_ATTN, the shape is [N, num_kv_heads, block_size, head_size]
|
||||
_, _, block_size, head_size = key_cache.shape
|
||||
key = key.permute(0, 2, 1, 3).flatten(0, 1)
|
||||
value = value.permute(0, 2, 1, 3).flatten(0, 1)
|
||||
|
||||
isa = _get_attn_isa(dtype, block_size, head_size)
|
||||
block_offsets = torch.arange(block_size, device="cpu", dtype=torch.long)
|
||||
num_blocks = len(block_ids)
|
||||
num_blocks = indices.numel()
|
||||
slot_mapping = (
|
||||
block_offsets.reshape(1, block_size)
|
||||
+ indices.reshape(num_blocks, 1) * block_size
|
||||
|
||||
Reference in New Issue
Block a user