[Bugfix][NIXL] Fix receive post-process for different P/D block sizes (#58860)
Signed-off-by: Luca Motz <luca.motz@icloud.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
28d76f3bad
commit
83ed0c7c09
@@ -1128,17 +1128,24 @@ class TestNixlHandshake:
|
||||
return
|
||||
raise TimeoutError("Took too long to complete async handshake.")
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kv_cache_layout", "enable_permute_local_kv"),
|
||||
# enable_permute_local_kv only covers a local LBNHC cache.
|
||||
[("LBHNC", False), ("BLHNC", True)],
|
||||
)
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||
FakeNixlWrapper,
|
||||
)
|
||||
def test_handshake_fails_on_kv_cache_layout_mismatch(
|
||||
self, default_vllm_config, dist_init
|
||||
self, default_vllm_config, dist_init, kv_cache_layout, enable_permute_local_kv
|
||||
):
|
||||
"""Verify that adding a remote agent fails if kv_cache_layout differs.
|
||||
This test is only relevant for heterogeneous TP.
|
||||
"""
|
||||
vllm_config = create_vllm_config()
|
||||
vllm_config = create_vllm_config(
|
||||
enable_permute_local_kv=enable_permute_local_kv
|
||||
)
|
||||
|
||||
# Mock TP world size to 2 to force heterogeneous TP when
|
||||
# remote_tp_size=1
|
||||
@@ -1151,7 +1158,10 @@ class TestNixlHandshake:
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config, connector.engine_id, hand_shake_latency=0
|
||||
vllm_config,
|
||||
connector.engine_id,
|
||||
hand_shake_latency=0,
|
||||
kv_cache_layout=kv_cache_layout,
|
||||
)
|
||||
worker = connector.connector_worker
|
||||
|
||||
|
||||
@@ -1404,46 +1404,66 @@ def test_map_block_ids_for_block_size_ratio_hybrid():
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_post_process_zeroes_untransferred_tail():
|
||||
"""The untransferred sub-blocks of the last local block are zeroed on
|
||||
receive; mamba state caches are untouched by the attention permute."""
|
||||
@pytest.mark.parametrize(
|
||||
("kv_cache_layout", "enable_permute_local_kv"),
|
||||
[("LBHNC", False), ("LBNHC", False), ("LBNHC", True)],
|
||||
)
|
||||
def test_post_process_zeroes_untransferred_tail(
|
||||
kv_cache_layout, enable_permute_local_kv
|
||||
):
|
||||
"""Received remote sub-blocks are regrouped per head and the untransferred
|
||||
sub-blocks of the last local block are zeroed on receive, once per block
|
||||
although two attention groups alias the tensor."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
NixlConnectorWorker,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, MambaSpec
|
||||
|
||||
ratio = 4
|
||||
block_tokens = 8 # 2 tokens per remote sub-block
|
||||
num_kv_heads = 2 # fewer than ratio
|
||||
|
||||
worker = MagicMock(spec=NixlConnectorWorker)
|
||||
worker._group_spec_types = (FullAttentionSpec, MambaSpec)
|
||||
worker.transfer_topo = MagicMock()
|
||||
worker.device_type = "cpu"
|
||||
worker.enable_permute_local_kv = False
|
||||
attn_cache = torch.ones(6, block_tokens, 2, 4)
|
||||
mamba_cache = torch.ones(6, 16)
|
||||
worker.device_kv_caches = {"attn.0": attn_cache, "mamba.0": mamba_cache}
|
||||
worker.enable_permute_local_kv = enable_permute_local_kv
|
||||
worker.kv_cache_layout = kv_cache_layout
|
||||
# Attention caches are [B, H, N, C]; distinct values per head and token.
|
||||
expected = torch.arange(6 * num_kv_heads * block_tokens * 4).view(
|
||||
6, num_kv_heads, block_tokens, 4
|
||||
)
|
||||
# Blocks [2, 3] as received: `ratio` head-major remote sub-blocks each.
|
||||
received = (
|
||||
expected[2:4]
|
||||
.unflatten(2, (ratio, -1))
|
||||
.transpose(1, 2)
|
||||
.reshape(2, num_kv_heads, block_tokens, 4)
|
||||
)
|
||||
if kv_cache_layout == "LBNHC":
|
||||
attn_cache = expected.transpose(1, 2).contiguous().transpose(1, 2)
|
||||
if enable_permute_local_kv:
|
||||
# The remote is LBHNC: its bytes land in token-major memory.
|
||||
attn_cache.transpose(1, 2)[2:4] = received.view(2, block_tokens, -1, 4)
|
||||
# Otherwise token-major blocks receive the sub-blocks in token order.
|
||||
else:
|
||||
attn_cache = expected.clone()
|
||||
attn_cache[2:4] = received
|
||||
worker.device_kv_caches = {"attn.0": attn_cache, "swa.0": attn_cache}
|
||||
fa_group = MagicMock(layer_names=["attn.0"])
|
||||
ssm_group = MagicMock(layer_names=["mamba.0"])
|
||||
worker.kv_cache_config = MagicMock(transfer_groups=[fa_group, ssm_group])
|
||||
# The cached property filters mamba layers out of the permuted caches.
|
||||
attn_caches = NixlConnectorWorker._attention_kv_caches.func(worker)
|
||||
assert len(attn_caches) == 1 and attn_caches[0] is attn_cache
|
||||
worker._attention_kv_caches = attn_caches
|
||||
swa_group = MagicMock(layer_names=["swa.0"])
|
||||
worker.kv_cache_config = MagicMock(transfer_groups=[fa_group, swa_group])
|
||||
_bind_worker_method(worker, "post_process_device_kv_on_receive")
|
||||
|
||||
# Request occupies blocks [2, 3]; only 6 of 8 sub-blocks were received.
|
||||
worker.post_process_device_kv_on_receive(ratio, [([2, 3], 6)])
|
||||
# Group 0 request in blocks [2, 3]; only 6 of 8 sub-blocks were received.
|
||||
worker.post_process_device_kv_on_receive(ratio, [(0, [2, 3], 6)])
|
||||
|
||||
# Block 2 fully covered; block 3 covered for 2 sub-blocks (4 tokens).
|
||||
assert torch.all(attn_cache[2] == 1)
|
||||
assert torch.all(attn_cache[3, :4] == 1)
|
||||
assert torch.all(attn_cache[3, 4:] == 0)
|
||||
# Untouched blocks and the mamba cache keep their content.
|
||||
assert torch.all(attn_cache[4] == 1)
|
||||
assert torch.all(mamba_cache == 1)
|
||||
assert torch.equal(attn_cache[2], expected[2])
|
||||
assert torch.equal(attn_cache[3, :, :4], expected[3, :, :4])
|
||||
assert torch.all(attn_cache[3, :, 4:] == 0)
|
||||
# Untouched blocks keep their content.
|
||||
assert torch.equal(attn_cache[4], expected[4])
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
|
||||
@@ -256,8 +256,6 @@ def kv_postprocess_blksize_on_receive(cache, indices, block_size_ratio):
|
||||
|
||||
"""
|
||||
blocks_to_update = cache.index_select(0, indices)
|
||||
# use physical order
|
||||
blocks_to_update = blocks_to_update.permute(0, 2, 1, 3)
|
||||
n_kv_heads, block_size, head_size = blocks_to_update.shape[1:]
|
||||
remote_block_size = block_size // block_size_ratio
|
||||
n_blocks = block_size_ratio
|
||||
@@ -267,56 +265,28 @@ def kv_postprocess_blksize_on_receive(cache, indices, block_size_ratio):
|
||||
.permute(0, 2, 1, 3, 4)
|
||||
.flatten(2, 3)
|
||||
)
|
||||
permuted_blocks = permuted_blocks.permute(0, 2, 1, 3)
|
||||
cache.index_copy_(0, indices, permuted_blocks)
|
||||
|
||||
|
||||
def kv_postprocess_layout_on_receive(cache, indices):
|
||||
"""Transforms the layout of received KV cache blocks to the local format.
|
||||
|
||||
This method corrects layout mismatches from direct memory copies by
|
||||
permuting the tensor dimensions.
|
||||
|
||||
4D cache:
|
||||
- **Source Layout:** `[num_blocks, n_kv_head, block_size, head_dim]`
|
||||
- **Target Layout:** `[num_blocks, block_size, n_kv_head, head_dim]`
|
||||
5D cache:
|
||||
- **Source Layout:** `[num_blocks, kv_dim, n_kv_head, block_size, head_dim]`
|
||||
- **Target Layout:** `[num_blocks, kv_dim, block_size, n_kv_head, head_dim]`
|
||||
|
||||
Implementation:
|
||||
- x = blocks_to_update.reshape(src_shape) # view local kv with sender layout
|
||||
- permuted_blocks = x.permute(*inv_order) # transpose n_kv_heads, block_size
|
||||
- cache.index_copy_(0, indices, permuted_blocks) # copy permuted kv back
|
||||
|
||||
"""
|
||||
blocks_to_update = cache.index_select(0, indices)
|
||||
target_shape = list(blocks_to_update.shape)
|
||||
target_shape[0] = -1
|
||||
inv_order = [0, 1, 3, 2, 4] if blocks_to_update.ndim == 5 else [0, 2, 1, 3]
|
||||
src_shape = tuple(target_shape[i] for i in inv_order)
|
||||
blocks_to_update = cache.index_select(0, indices)
|
||||
permuted_blocks = blocks_to_update.reshape(src_shape).permute(*inv_order)
|
||||
cache.index_copy_(0, indices, permuted_blocks)
|
||||
|
||||
|
||||
def kv_postprocess_blksize_and_layout_on_receive(cache, indices, block_size_ratio):
|
||||
"""Transforms the layout of received KV cache to the local block_size and LBHNC.
|
||||
(Only works for local blocksize > remote blocksize)
|
||||
(Only works for local blocksize >= remote blocksize)
|
||||
|
||||
prefill is LBHNC, smaller block_size
|
||||
decode(local) is LBNHC, larger block_size
|
||||
"""
|
||||
blocks_to_update = cache.index_select(0, indices)
|
||||
|
||||
block_size, n_kv_heads, head_size = blocks_to_update.shape[1:]
|
||||
n_kv_heads, block_size, head_size = blocks_to_update.shape[1:]
|
||||
remote_block_size = block_size // block_size_ratio
|
||||
n_blocks = block_size_ratio
|
||||
|
||||
# View the received bytes in memory order as the remote head-major blocks.
|
||||
permuted_blocks = (
|
||||
blocks_to_update.reshape(-1, n_blocks, n_kv_heads, remote_block_size, head_size)
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.flatten(1, 2)
|
||||
blocks_to_update.transpose(1, 2)
|
||||
.reshape(-1, n_blocks, n_kv_heads, remote_block_size, head_size)
|
||||
.permute(0, 2, 1, 3, 4)
|
||||
.flatten(2, 3)
|
||||
)
|
||||
cache.index_copy_(0, indices, permuted_blocks)
|
||||
|
||||
|
||||
@@ -14,7 +14,6 @@ import uuid
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterator
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from functools import cached_property
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import msgspec
|
||||
@@ -31,7 +30,6 @@ from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
get_current_attn_backends,
|
||||
kv_postprocess_blksize_and_layout_on_receive,
|
||||
kv_postprocess_blksize_on_receive,
|
||||
kv_postprocess_layout_on_receive,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
CopyBlocksOp,
|
||||
@@ -2596,6 +2594,7 @@ class NixlBaseConnectorWorker:
|
||||
if (
|
||||
self.kv_transfer_config.enable_permute_local_kv
|
||||
and nixl_agent_meta.kv_cache_layout == "LBHNC"
|
||||
and kv_cache_layout == "LBNHC"
|
||||
):
|
||||
logger.info(
|
||||
"Remote is LBHNC and local is LBNHC, enabled additional permute "
|
||||
@@ -2607,9 +2606,11 @@ class NixlBaseConnectorWorker:
|
||||
self.enable_permute_local_kv = True
|
||||
else:
|
||||
raise RuntimeError(
|
||||
"Heterogeneous TP expects same kv_cache_layout. "
|
||||
"Or enable experimental feature to use HND to NHD support by "
|
||||
"setting 'enable_permute_local_kv'=True in --kv-transfer-config."
|
||||
"Heterogeneous TP expects same kv_cache_layout (local "
|
||||
f"{kv_cache_layout}, remote {nixl_agent_meta.kv_cache_layout}). "
|
||||
"Or enable experimental feature to receive LBHNC into a local "
|
||||
"LBNHC cache by setting 'enable_permute_local_kv'=True "
|
||||
"in --kv-transfer-config."
|
||||
)
|
||||
# if remote_agent used attn is not same as local,
|
||||
# hint heterogenuous attn post process
|
||||
@@ -2795,26 +2796,10 @@ class NixlBaseConnectorWorker:
|
||||
"d2h",
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def _attention_kv_caches(self) -> list[torch.Tensor]:
|
||||
"""Device KV caches of attention layers (mamba states excluded),
|
||||
as consumed by the receive post-process."""
|
||||
assert self.device_kv_caches, (
|
||||
"_attention_kv_caches accessed before register_kv_caches"
|
||||
)
|
||||
mamba_layers = {
|
||||
name
|
||||
for g, group in enumerate(self.kv_cache_config.transfer_groups)
|
||||
if _is_ssm_spec(self._group_spec_types[g])
|
||||
for name in group.layer_names
|
||||
}
|
||||
kv_caches = self.device_kv_caches
|
||||
return [cache for name, cache in kv_caches.items() if name not in mamba_layers]
|
||||
|
||||
def post_process_device_kv_on_receive(
|
||||
self,
|
||||
block_size_ratio: int,
|
||||
block_ids_list: list[tuple[list[int], int]],
|
||||
block_ids_list: list[tuple[int, list[int], int]],
|
||||
convert: bool = True,
|
||||
):
|
||||
"""Post process device kv cache after receiving from remote.
|
||||
@@ -2863,9 +2848,11 @@ class NixlBaseConnectorWorker:
|
||||
block_size_ratio,
|
||||
)
|
||||
|
||||
attn_caches = self._attention_kv_caches
|
||||
device = attn_caches[0].device
|
||||
for block_ids, covered_sub_blocks in block_ids_list:
|
||||
for group, block_ids, covered_sub_blocks in block_ids_list:
|
||||
attn_caches = [
|
||||
self.device_kv_caches[name]
|
||||
for name in self.kv_cache_config.transfer_groups[group].layer_names
|
||||
]
|
||||
# Blocks the transfer didn't write: the token tail of the last
|
||||
# partially covered block, then everything beyond it.
|
||||
covered_blocks, sub_blocks_in_last = divmod(
|
||||
@@ -2875,17 +2862,16 @@ class NixlBaseConnectorWorker:
|
||||
has_stale = first_stale < len(block_ids)
|
||||
indices = None
|
||||
if convert or has_stale:
|
||||
indices = async_tensor_h2d(block_ids, device, torch.long)
|
||||
indices = async_tensor_h2d(block_ids, attn_caches[0].device, torch.long)
|
||||
|
||||
if convert:
|
||||
for cache in attn_caches:
|
||||
if self.enable_permute_local_kv and block_size_ratio > 1:
|
||||
if self.enable_permute_local_kv:
|
||||
kv_postprocess_blksize_and_layout_on_receive(
|
||||
cache, indices, block_size_ratio
|
||||
)
|
||||
elif self.enable_permute_local_kv:
|
||||
kv_postprocess_layout_on_receive(cache, indices)
|
||||
else:
|
||||
elif KVCacheLayout[self.kv_cache_layout].is_block_contiguous:
|
||||
# Token-major blocks get the sub-blocks in token order.
|
||||
kv_postprocess_blksize_on_receive(
|
||||
cache, indices, block_size_ratio
|
||||
)
|
||||
@@ -2893,10 +2879,10 @@ class NixlBaseConnectorWorker:
|
||||
if sub_blocks_in_last:
|
||||
last_block_id = block_ids[covered_blocks]
|
||||
for cache in attn_caches:
|
||||
# Both post-processed layouts leave tokens on dim 1.
|
||||
sub_block_tokens = cache.shape[1] // block_size_ratio
|
||||
# Attention caches are [B, H, N, C]: tokens on dim 2.
|
||||
sub_block_tokens = cache.shape[2] // block_size_ratio
|
||||
zero_from = sub_blocks_in_last * sub_block_tokens
|
||||
cache[last_block_id, zero_from:].zero_()
|
||||
cache[last_block_id, :, zero_from:].zero_()
|
||||
if has_stale:
|
||||
assert indices is not None
|
||||
stale_ids = indices[first_stale:]
|
||||
@@ -3068,7 +3054,7 @@ class NixlBaseConnectorWorker:
|
||||
len(meta.remote.block_ids[g]),
|
||||
)
|
||||
block_ids_for_blocksize_post_process[block_size_ratio].append(
|
||||
(local_group, covered_sub_blocks)
|
||||
(g, local_group, covered_sub_blocks)
|
||||
)
|
||||
# post processing for heterogeneous attention
|
||||
if self.enable_heterogeneous_attn_post_process:
|
||||
|
||||
Reference in New Issue
Block a user