From 4938d44a3b818fd443cf45aea92e670f16973ba0 Mon Sep 17 00:00:00 2001 From: Sihan Chen <757407490@qq.com> Date: Mon, 20 Jul 2026 15:33:13 +0800 Subject: [PATCH] [CPU] fixes heterogeneous NIXL KV transfer into CPU_ATTN decode workers (#47871) Signed-off-by: Spycsh Co-authored-by: Li, Jiang --- .../kv_connector/v1/nixl/base_worker.py | 7 +------ vllm/platforms/cpu.py | 21 ++++++++++++------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index 834c5fccc7..4cdc54f7e7 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -1935,13 +1935,8 @@ class NixlBaseConnectorWorker: indices = torch.tensor(block_ids, device=self.device_type, dtype=torch.long) for _, cache_or_caches in self.device_kv_caches.items(): - blocks_to_update = cache_or_caches.index_select(1, indices) current_platform.pack_kv_cache( - key=blocks_to_update[0], - value=blocks_to_update[1], - key_cache=cache_or_caches[0], - value_cache=cache_or_caches[1], - block_ids=block_ids, + kv_cache=cache_or_caches, indices=indices, ) diff --git a/vllm/platforms/cpu.py b/vllm/platforms/cpu.py index 0e805794f5..698e83a8ab 100644 --- a/vllm/platforms/cpu.py +++ b/vllm/platforms/cpu.py @@ -461,11 +461,7 @@ class CpuPlatform(Platform): @classmethod def pack_kv_cache( cls, - key: torch.Tensor, - value: torch.Tensor, - key_cache: torch.Tensor, - value_cache: torch.Tensor, - block_ids: list[int], + kv_cache: torch.Tensor, indices: torch.Tensor, ) -> None: """ @@ -476,15 +472,26 @@ class CpuPlatform(Platform): from vllm._custom_ops import cpu_attn_reshape_and_cache from vllm.v1.attention.backends.cpu_attn import _get_attn_isa + num_blocks, num_kv_heads, block_size, fused_head_size = kv_cache.shape + head_size = fused_head_size // 2 + + # Fused path used by heterogeneous NIXL CPU_ATTN post-processing. + blocks_to_update = kv_cache.index_select(0, indices) + key = blocks_to_update[..., :head_size] + value = blocks_to_update[..., head_size:] + + key_cache, value_cache = kv_cache.view( + num_blocks, num_kv_heads, block_size * 2, head_size + ).chunk(2, dim=2) + dtype = key.dtype # For CPU_ATTN, the shape is [N, num_kv_heads, block_size, head_size] - _, _, block_size, head_size = key_cache.shape key = key.permute(0, 2, 1, 3).flatten(0, 1) value = value.permute(0, 2, 1, 3).flatten(0, 1) isa = _get_attn_isa(dtype, block_size, head_size) block_offsets = torch.arange(block_size, device="cpu", dtype=torch.long) - num_blocks = len(block_ids) + num_blocks = indices.numel() slot_mapping = ( block_offsets.reshape(1, block_size) + indices.reshape(num_blocks, 1) * block_size