[CPU] fixes heterogeneous NIXL KV transfer into CPU_ATTN decode workers (#47871)

Signed-off-by: Spycsh <sihan.chen@intel.com>
Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
Sihan Chen
2026-07-20 07:33:13 +00:00
committed by GitHub
co-authored by Li, Jiang <jiang1.li@intel.com>
parent 37bf988c2f
commit 4938d44a3b
2 changed files with 15 additions and 13 deletions
@@ -1935,13 +1935,8 @@ class NixlBaseConnectorWorker:
indices = torch.tensor(block_ids, device=self.device_type, dtype=torch.long)
for _, cache_or_caches in self.device_kv_caches.items():
blocks_to_update = cache_or_caches.index_select(1, indices)
current_platform.pack_kv_cache(
key=blocks_to_update[0],
value=blocks_to_update[1],
key_cache=cache_or_caches[0],
value_cache=cache_or_caches[1],
block_ids=block_ids,
kv_cache=cache_or_caches,
indices=indices,
)
+14 -7
View File
@@ -461,11 +461,7 @@ class CpuPlatform(Platform):
@classmethod
def pack_kv_cache(
cls,
key: torch.Tensor,
value: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
block_ids: list[int],
kv_cache: torch.Tensor,
indices: torch.Tensor,
) -> None:
"""
@@ -476,15 +472,26 @@ class CpuPlatform(Platform):
from vllm._custom_ops import cpu_attn_reshape_and_cache
from vllm.v1.attention.backends.cpu_attn import _get_attn_isa
num_blocks, num_kv_heads, block_size, fused_head_size = kv_cache.shape
head_size = fused_head_size // 2
# Fused path used by heterogeneous NIXL CPU_ATTN post-processing.
blocks_to_update = kv_cache.index_select(0, indices)
key = blocks_to_update[..., :head_size]
value = blocks_to_update[..., head_size:]
key_cache, value_cache = kv_cache.view(
num_blocks, num_kv_heads, block_size * 2, head_size
).chunk(2, dim=2)
dtype = key.dtype
# For CPU_ATTN, the shape is [N, num_kv_heads, block_size, head_size]
_, _, block_size, head_size = key_cache.shape
key = key.permute(0, 2, 1, 3).flatten(0, 1)
value = value.permute(0, 2, 1, 3).flatten(0, 1)
isa = _get_attn_isa(dtype, block_size, head_size)
block_offsets = torch.arange(block_size, device="cpu", dtype=torch.long)
num_blocks = len(block_ids)
num_blocks = indices.numel()
slot_mapping = (
block_offsets.reshape(1, block_size)
+ indices.reshape(num_blocks, 1) * block_size