From 83ed0c7c0944e9d124589d8fa3f82a019343a726 Mon Sep 17 00:00:00 2001 From: Luca Motz Date: Fri, 9 Oct 2026 13:30:11 +0200 Subject: [PATCH] [Bugfix][NIXL] Fix receive post-process for different P/D block sizes (#58860) Signed-off-by: Luca Motz Co-authored-by: Claude Opus 5.5 --- .../kv_connector/unit/test_nixl_connector.py | 16 ++++- .../unit/test_nixl_connector_hma.py | 66 ++++++++++++------- .../kv_transfer/kv_connector/utils.py | 44 ++----------- .../kv_connector/v1/nixl/base_worker.py | 54 ++++++--------- 4 files changed, 83 insertions(+), 97 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index 52ac746391..d918c38fbf 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -1128,17 +1128,24 @@ class TestNixlHandshake: return raise TimeoutError("Took too long to complete async handshake.") + @pytest.mark.parametrize( + ("kv_cache_layout", "enable_permute_local_kv"), + # enable_permute_local_kv only covers a local LBNHC cache. + [("LBHNC", False), ("BLHNC", True)], + ) @patch( "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, ) def test_handshake_fails_on_kv_cache_layout_mismatch( - self, default_vllm_config, dist_init + self, default_vllm_config, dist_init, kv_cache_layout, enable_permute_local_kv ): """Verify that adding a remote agent fails if kv_cache_layout differs. This test is only relevant for heterogeneous TP. """ - vllm_config = create_vllm_config() + vllm_config = create_vllm_config( + enable_permute_local_kv=enable_permute_local_kv + ) # Mock TP world size to 2 to force heterogeneous TP when # remote_tp_size=1 @@ -1151,7 +1158,10 @@ class TestNixlHandshake: vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16) ) connector.connector_worker = FakeNixlConnectorWorker( - vllm_config, connector.engine_id, hand_shake_latency=0 + vllm_config, + connector.engine_id, + hand_shake_latency=0, + kv_cache_layout=kv_cache_layout, ) worker = connector.connector_worker diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py index f016b26475..c665ce0c5b 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py @@ -1404,46 +1404,66 @@ def test_map_block_ids_for_block_size_ratio_hybrid(): @pytest.mark.cpu_test -def test_post_process_zeroes_untransferred_tail(): - """The untransferred sub-blocks of the last local block are zeroed on - receive; mamba state caches are untouched by the attention permute.""" +@pytest.mark.parametrize( + ("kv_cache_layout", "enable_permute_local_kv"), + [("LBHNC", False), ("LBNHC", False), ("LBNHC", True)], +) +def test_post_process_zeroes_untransferred_tail( + kv_cache_layout, enable_permute_local_kv +): + """Received remote sub-blocks are regrouped per head and the untransferred + sub-blocks of the last local block are zeroed on receive, once per block + although two attention groups alias the tensor.""" from unittest.mock import MagicMock from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import ( NixlConnectorWorker, ) - from vllm.v1.kv_cache_interface import FullAttentionSpec, MambaSpec ratio = 4 block_tokens = 8 # 2 tokens per remote sub-block + num_kv_heads = 2 # fewer than ratio worker = MagicMock(spec=NixlConnectorWorker) - worker._group_spec_types = (FullAttentionSpec, MambaSpec) worker.transfer_topo = MagicMock() worker.device_type = "cpu" - worker.enable_permute_local_kv = False - attn_cache = torch.ones(6, block_tokens, 2, 4) - mamba_cache = torch.ones(6, 16) - worker.device_kv_caches = {"attn.0": attn_cache, "mamba.0": mamba_cache} + worker.enable_permute_local_kv = enable_permute_local_kv + worker.kv_cache_layout = kv_cache_layout + # Attention caches are [B, H, N, C]; distinct values per head and token. + expected = torch.arange(6 * num_kv_heads * block_tokens * 4).view( + 6, num_kv_heads, block_tokens, 4 + ) + # Blocks [2, 3] as received: `ratio` head-major remote sub-blocks each. + received = ( + expected[2:4] + .unflatten(2, (ratio, -1)) + .transpose(1, 2) + .reshape(2, num_kv_heads, block_tokens, 4) + ) + if kv_cache_layout == "LBNHC": + attn_cache = expected.transpose(1, 2).contiguous().transpose(1, 2) + if enable_permute_local_kv: + # The remote is LBHNC: its bytes land in token-major memory. + attn_cache.transpose(1, 2)[2:4] = received.view(2, block_tokens, -1, 4) + # Otherwise token-major blocks receive the sub-blocks in token order. + else: + attn_cache = expected.clone() + attn_cache[2:4] = received + worker.device_kv_caches = {"attn.0": attn_cache, "swa.0": attn_cache} fa_group = MagicMock(layer_names=["attn.0"]) - ssm_group = MagicMock(layer_names=["mamba.0"]) - worker.kv_cache_config = MagicMock(transfer_groups=[fa_group, ssm_group]) - # The cached property filters mamba layers out of the permuted caches. - attn_caches = NixlConnectorWorker._attention_kv_caches.func(worker) - assert len(attn_caches) == 1 and attn_caches[0] is attn_cache - worker._attention_kv_caches = attn_caches + swa_group = MagicMock(layer_names=["swa.0"]) + worker.kv_cache_config = MagicMock(transfer_groups=[fa_group, swa_group]) _bind_worker_method(worker, "post_process_device_kv_on_receive") - # Request occupies blocks [2, 3]; only 6 of 8 sub-blocks were received. - worker.post_process_device_kv_on_receive(ratio, [([2, 3], 6)]) + # Group 0 request in blocks [2, 3]; only 6 of 8 sub-blocks were received. + worker.post_process_device_kv_on_receive(ratio, [(0, [2, 3], 6)]) # Block 2 fully covered; block 3 covered for 2 sub-blocks (4 tokens). - assert torch.all(attn_cache[2] == 1) - assert torch.all(attn_cache[3, :4] == 1) - assert torch.all(attn_cache[3, 4:] == 0) - # Untouched blocks and the mamba cache keep their content. - assert torch.all(attn_cache[4] == 1) - assert torch.all(mamba_cache == 1) + assert torch.equal(attn_cache[2], expected[2]) + assert torch.equal(attn_cache[3, :, :4], expected[3, :, :4]) + assert torch.all(attn_cache[3, :, 4:] == 0) + # Untouched blocks keep their content. + assert torch.equal(attn_cache[4], expected[4]) @pytest.mark.cpu_test diff --git a/vllm/distributed/kv_transfer/kv_connector/utils.py b/vllm/distributed/kv_transfer/kv_connector/utils.py index a0ead27c2f..d314d6107a 100644 --- a/vllm/distributed/kv_transfer/kv_connector/utils.py +++ b/vllm/distributed/kv_transfer/kv_connector/utils.py @@ -256,8 +256,6 @@ def kv_postprocess_blksize_on_receive(cache, indices, block_size_ratio): """ blocks_to_update = cache.index_select(0, indices) - # use physical order - blocks_to_update = blocks_to_update.permute(0, 2, 1, 3) n_kv_heads, block_size, head_size = blocks_to_update.shape[1:] remote_block_size = block_size // block_size_ratio n_blocks = block_size_ratio @@ -267,56 +265,28 @@ def kv_postprocess_blksize_on_receive(cache, indices, block_size_ratio): .permute(0, 2, 1, 3, 4) .flatten(2, 3) ) - permuted_blocks = permuted_blocks.permute(0, 2, 1, 3) - cache.index_copy_(0, indices, permuted_blocks) - - -def kv_postprocess_layout_on_receive(cache, indices): - """Transforms the layout of received KV cache blocks to the local format. - - This method corrects layout mismatches from direct memory copies by - permuting the tensor dimensions. - - 4D cache: - - **Source Layout:** `[num_blocks, n_kv_head, block_size, head_dim]` - - **Target Layout:** `[num_blocks, block_size, n_kv_head, head_dim]` - 5D cache: - - **Source Layout:** `[num_blocks, kv_dim, n_kv_head, block_size, head_dim]` - - **Target Layout:** `[num_blocks, kv_dim, block_size, n_kv_head, head_dim]` - - Implementation: - - x = blocks_to_update.reshape(src_shape) # view local kv with sender layout - - permuted_blocks = x.permute(*inv_order) # transpose n_kv_heads, block_size - - cache.index_copy_(0, indices, permuted_blocks) # copy permuted kv back - - """ - blocks_to_update = cache.index_select(0, indices) - target_shape = list(blocks_to_update.shape) - target_shape[0] = -1 - inv_order = [0, 1, 3, 2, 4] if blocks_to_update.ndim == 5 else [0, 2, 1, 3] - src_shape = tuple(target_shape[i] for i in inv_order) - blocks_to_update = cache.index_select(0, indices) - permuted_blocks = blocks_to_update.reshape(src_shape).permute(*inv_order) cache.index_copy_(0, indices, permuted_blocks) def kv_postprocess_blksize_and_layout_on_receive(cache, indices, block_size_ratio): """Transforms the layout of received KV cache to the local block_size and LBHNC. - (Only works for local blocksize > remote blocksize) + (Only works for local blocksize >= remote blocksize) prefill is LBHNC, smaller block_size decode(local) is LBNHC, larger block_size """ blocks_to_update = cache.index_select(0, indices) - block_size, n_kv_heads, head_size = blocks_to_update.shape[1:] + n_kv_heads, block_size, head_size = blocks_to_update.shape[1:] remote_block_size = block_size // block_size_ratio n_blocks = block_size_ratio + # View the received bytes in memory order as the remote head-major blocks. permuted_blocks = ( - blocks_to_update.reshape(-1, n_blocks, n_kv_heads, remote_block_size, head_size) - .permute(0, 1, 3, 2, 4) - .flatten(1, 2) + blocks_to_update.transpose(1, 2) + .reshape(-1, n_blocks, n_kv_heads, remote_block_size, head_size) + .permute(0, 2, 1, 3, 4) + .flatten(2, 3) ) cache.index_copy_(0, indices, permuted_blocks) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index 7ce6e93ebb..38a656954e 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -14,7 +14,6 @@ import uuid from collections import defaultdict from collections.abc import Iterator from concurrent.futures import Future, ThreadPoolExecutor -from functools import cached_property from typing import TYPE_CHECKING, Any, cast import msgspec @@ -31,7 +30,6 @@ from vllm.distributed.kv_transfer.kv_connector.utils import ( get_current_attn_backends, kv_postprocess_blksize_and_layout_on_receive, kv_postprocess_blksize_on_receive, - kv_postprocess_layout_on_receive, ) from vllm.distributed.kv_transfer.kv_connector.v1.base import ( CopyBlocksOp, @@ -2596,6 +2594,7 @@ class NixlBaseConnectorWorker: if ( self.kv_transfer_config.enable_permute_local_kv and nixl_agent_meta.kv_cache_layout == "LBHNC" + and kv_cache_layout == "LBNHC" ): logger.info( "Remote is LBHNC and local is LBNHC, enabled additional permute " @@ -2607,9 +2606,11 @@ class NixlBaseConnectorWorker: self.enable_permute_local_kv = True else: raise RuntimeError( - "Heterogeneous TP expects same kv_cache_layout. " - "Or enable experimental feature to use HND to NHD support by " - "setting 'enable_permute_local_kv'=True in --kv-transfer-config." + "Heterogeneous TP expects same kv_cache_layout (local " + f"{kv_cache_layout}, remote {nixl_agent_meta.kv_cache_layout}). " + "Or enable experimental feature to receive LBHNC into a local " + "LBNHC cache by setting 'enable_permute_local_kv'=True " + "in --kv-transfer-config." ) # if remote_agent used attn is not same as local, # hint heterogenuous attn post process @@ -2795,26 +2796,10 @@ class NixlBaseConnectorWorker: "d2h", ) - @cached_property - def _attention_kv_caches(self) -> list[torch.Tensor]: - """Device KV caches of attention layers (mamba states excluded), - as consumed by the receive post-process.""" - assert self.device_kv_caches, ( - "_attention_kv_caches accessed before register_kv_caches" - ) - mamba_layers = { - name - for g, group in enumerate(self.kv_cache_config.transfer_groups) - if _is_ssm_spec(self._group_spec_types[g]) - for name in group.layer_names - } - kv_caches = self.device_kv_caches - return [cache for name, cache in kv_caches.items() if name not in mamba_layers] - def post_process_device_kv_on_receive( self, block_size_ratio: int, - block_ids_list: list[tuple[list[int], int]], + block_ids_list: list[tuple[int, list[int], int]], convert: bool = True, ): """Post process device kv cache after receiving from remote. @@ -2863,9 +2848,11 @@ class NixlBaseConnectorWorker: block_size_ratio, ) - attn_caches = self._attention_kv_caches - device = attn_caches[0].device - for block_ids, covered_sub_blocks in block_ids_list: + for group, block_ids, covered_sub_blocks in block_ids_list: + attn_caches = [ + self.device_kv_caches[name] + for name in self.kv_cache_config.transfer_groups[group].layer_names + ] # Blocks the transfer didn't write: the token tail of the last # partially covered block, then everything beyond it. covered_blocks, sub_blocks_in_last = divmod( @@ -2875,17 +2862,16 @@ class NixlBaseConnectorWorker: has_stale = first_stale < len(block_ids) indices = None if convert or has_stale: - indices = async_tensor_h2d(block_ids, device, torch.long) + indices = async_tensor_h2d(block_ids, attn_caches[0].device, torch.long) if convert: for cache in attn_caches: - if self.enable_permute_local_kv and block_size_ratio > 1: + if self.enable_permute_local_kv: kv_postprocess_blksize_and_layout_on_receive( cache, indices, block_size_ratio ) - elif self.enable_permute_local_kv: - kv_postprocess_layout_on_receive(cache, indices) - else: + elif KVCacheLayout[self.kv_cache_layout].is_block_contiguous: + # Token-major blocks get the sub-blocks in token order. kv_postprocess_blksize_on_receive( cache, indices, block_size_ratio ) @@ -2893,10 +2879,10 @@ class NixlBaseConnectorWorker: if sub_blocks_in_last: last_block_id = block_ids[covered_blocks] for cache in attn_caches: - # Both post-processed layouts leave tokens on dim 1. - sub_block_tokens = cache.shape[1] // block_size_ratio + # Attention caches are [B, H, N, C]: tokens on dim 2. + sub_block_tokens = cache.shape[2] // block_size_ratio zero_from = sub_blocks_in_last * sub_block_tokens - cache[last_block_id, zero_from:].zero_() + cache[last_block_id, :, zero_from:].zero_() if has_stale: assert indices is not None stale_ids = indices[first_stale:] @@ -3068,7 +3054,7 @@ class NixlBaseConnectorWorker: len(meta.remote.block_ids[g]), ) block_ids_for_blocksize_post_process[block_size_ratio].append( - (local_group, covered_sub_blocks) + (g, local_group, covered_sub_blocks) ) # post processing for heterogeneous attention if self.enable_heterogeneous_attn_post_process: