[Bugfix][NIXL] Fix receive post-process for different P/D block sizes (#58860)

Signed-off-by: Luca Motz <luca.motz@icloud.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Luca Motz
2026-10-09 19:30:11 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 28d76f3bad
commit 83ed0c7c09
4 changed files with 83 additions and 97 deletions
@@ -256,8 +256,6 @@ def kv_postprocess_blksize_on_receive(cache, indices, block_size_ratio):
"""
blocks_to_update = cache.index_select(0, indices)
# use physical order
blocks_to_update = blocks_to_update.permute(0, 2, 1, 3)
n_kv_heads, block_size, head_size = blocks_to_update.shape[1:]
remote_block_size = block_size // block_size_ratio
n_blocks = block_size_ratio
@@ -267,56 +265,28 @@ def kv_postprocess_blksize_on_receive(cache, indices, block_size_ratio):
.permute(0, 2, 1, 3, 4)
.flatten(2, 3)
)
permuted_blocks = permuted_blocks.permute(0, 2, 1, 3)
cache.index_copy_(0, indices, permuted_blocks)
def kv_postprocess_layout_on_receive(cache, indices):
"""Transforms the layout of received KV cache blocks to the local format.
This method corrects layout mismatches from direct memory copies by
permuting the tensor dimensions.
4D cache:
- **Source Layout:** `[num_blocks, n_kv_head, block_size, head_dim]`
- **Target Layout:** `[num_blocks, block_size, n_kv_head, head_dim]`
5D cache:
- **Source Layout:** `[num_blocks, kv_dim, n_kv_head, block_size, head_dim]`
- **Target Layout:** `[num_blocks, kv_dim, block_size, n_kv_head, head_dim]`
Implementation:
- x = blocks_to_update.reshape(src_shape) # view local kv with sender layout
- permuted_blocks = x.permute(*inv_order) # transpose n_kv_heads, block_size
- cache.index_copy_(0, indices, permuted_blocks) # copy permuted kv back
"""
blocks_to_update = cache.index_select(0, indices)
target_shape = list(blocks_to_update.shape)
target_shape[0] = -1
inv_order = [0, 1, 3, 2, 4] if blocks_to_update.ndim == 5 else [0, 2, 1, 3]
src_shape = tuple(target_shape[i] for i in inv_order)
blocks_to_update = cache.index_select(0, indices)
permuted_blocks = blocks_to_update.reshape(src_shape).permute(*inv_order)
cache.index_copy_(0, indices, permuted_blocks)
def kv_postprocess_blksize_and_layout_on_receive(cache, indices, block_size_ratio):
"""Transforms the layout of received KV cache to the local block_size and LBHNC.
(Only works for local blocksize > remote blocksize)
(Only works for local blocksize >= remote blocksize)
prefill is LBHNC, smaller block_size
decode(local) is LBNHC, larger block_size
"""
blocks_to_update = cache.index_select(0, indices)
block_size, n_kv_heads, head_size = blocks_to_update.shape[1:]
n_kv_heads, block_size, head_size = blocks_to_update.shape[1:]
remote_block_size = block_size // block_size_ratio
n_blocks = block_size_ratio
# View the received bytes in memory order as the remote head-major blocks.
permuted_blocks = (
blocks_to_update.reshape(-1, n_blocks, n_kv_heads, remote_block_size, head_size)
.permute(0, 1, 3, 2, 4)
.flatten(1, 2)
blocks_to_update.transpose(1, 2)
.reshape(-1, n_blocks, n_kv_heads, remote_block_size, head_size)
.permute(0, 2, 1, 3, 4)
.flatten(2, 3)
)
cache.index_copy_(0, indices, permuted_blocks)
@@ -14,7 +14,6 @@ import uuid
from collections import defaultdict
from collections.abc import Iterator
from concurrent.futures import Future, ThreadPoolExecutor
from functools import cached_property
from typing import TYPE_CHECKING, Any, cast
import msgspec
@@ -31,7 +30,6 @@ from vllm.distributed.kv_transfer.kv_connector.utils import (
get_current_attn_backends,
kv_postprocess_blksize_and_layout_on_receive,
kv_postprocess_blksize_on_receive,
kv_postprocess_layout_on_receive,
)
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
CopyBlocksOp,
@@ -2596,6 +2594,7 @@ class NixlBaseConnectorWorker:
if (
self.kv_transfer_config.enable_permute_local_kv
and nixl_agent_meta.kv_cache_layout == "LBHNC"
and kv_cache_layout == "LBNHC"
):
logger.info(
"Remote is LBHNC and local is LBNHC, enabled additional permute "
@@ -2607,9 +2606,11 @@ class NixlBaseConnectorWorker:
self.enable_permute_local_kv = True
else:
raise RuntimeError(
"Heterogeneous TP expects same kv_cache_layout. "
"Or enable experimental feature to use HND to NHD support by "
"setting 'enable_permute_local_kv'=True in --kv-transfer-config."
"Heterogeneous TP expects same kv_cache_layout (local "
f"{kv_cache_layout}, remote {nixl_agent_meta.kv_cache_layout}). "
"Or enable experimental feature to receive LBHNC into a local "
"LBNHC cache by setting 'enable_permute_local_kv'=True "
"in --kv-transfer-config."
)
# if remote_agent used attn is not same as local,
# hint heterogenuous attn post process
@@ -2795,26 +2796,10 @@ class NixlBaseConnectorWorker:
"d2h",
)
@cached_property
def _attention_kv_caches(self) -> list[torch.Tensor]:
"""Device KV caches of attention layers (mamba states excluded),
as consumed by the receive post-process."""
assert self.device_kv_caches, (
"_attention_kv_caches accessed before register_kv_caches"
)
mamba_layers = {
name
for g, group in enumerate(self.kv_cache_config.transfer_groups)
if _is_ssm_spec(self._group_spec_types[g])
for name in group.layer_names
}
kv_caches = self.device_kv_caches
return [cache for name, cache in kv_caches.items() if name not in mamba_layers]
def post_process_device_kv_on_receive(
self,
block_size_ratio: int,
block_ids_list: list[tuple[list[int], int]],
block_ids_list: list[tuple[int, list[int], int]],
convert: bool = True,
):
"""Post process device kv cache after receiving from remote.
@@ -2863,9 +2848,11 @@ class NixlBaseConnectorWorker:
block_size_ratio,
)
attn_caches = self._attention_kv_caches
device = attn_caches[0].device
for block_ids, covered_sub_blocks in block_ids_list:
for group, block_ids, covered_sub_blocks in block_ids_list:
attn_caches = [
self.device_kv_caches[name]
for name in self.kv_cache_config.transfer_groups[group].layer_names
]
# Blocks the transfer didn't write: the token tail of the last
# partially covered block, then everything beyond it.
covered_blocks, sub_blocks_in_last = divmod(
@@ -2875,17 +2862,16 @@ class NixlBaseConnectorWorker:
has_stale = first_stale < len(block_ids)
indices = None
if convert or has_stale:
indices = async_tensor_h2d(block_ids, device, torch.long)
indices = async_tensor_h2d(block_ids, attn_caches[0].device, torch.long)
if convert:
for cache in attn_caches:
if self.enable_permute_local_kv and block_size_ratio > 1:
if self.enable_permute_local_kv:
kv_postprocess_blksize_and_layout_on_receive(
cache, indices, block_size_ratio
)
elif self.enable_permute_local_kv:
kv_postprocess_layout_on_receive(cache, indices)
else:
elif KVCacheLayout[self.kv_cache_layout].is_block_contiguous:
# Token-major blocks get the sub-blocks in token order.
kv_postprocess_blksize_on_receive(
cache, indices, block_size_ratio
)
@@ -2893,10 +2879,10 @@ class NixlBaseConnectorWorker:
if sub_blocks_in_last:
last_block_id = block_ids[covered_blocks]
for cache in attn_caches:
# Both post-processed layouts leave tokens on dim 1.
sub_block_tokens = cache.shape[1] // block_size_ratio
# Attention caches are [B, H, N, C]: tokens on dim 2.
sub_block_tokens = cache.shape[2] // block_size_ratio
zero_from = sub_blocks_in_last * sub_block_tokens
cache[last_block_id, zero_from:].zero_()
cache[last_block_id, :, zero_from:].zero_()
if has_stale:
assert indices is not None
stale_ids = indices[first_stale:]
@@ -3068,7 +3054,7 @@ class NixlBaseConnectorWorker:
len(meta.remote.block_ids[g]),
)
block_ids_for_blocksize_post_process[block_size_ratio].append(
(local_group, covered_sub_blocks)
(g, local_group, covered_sub_blocks)
)
# post processing for heterogeneous attention
if self.enable_heterogeneous_attn_post_process: