[Bugfix][NIXL] Fix receive post-process for different P/D block sizes (#58860)

Signed-off-by: Luca Motz <luca.motz@icloud.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Luca Motz
2026-10-09 19:30:11 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 28d76f3bad
commit 83ed0c7c09
4 changed files with 83 additions and 97 deletions
@@ -1128,17 +1128,24 @@ class TestNixlHandshake:
return
raise TimeoutError("Took too long to complete async handshake.")
@pytest.mark.parametrize(
("kv_cache_layout", "enable_permute_local_kv"),
# enable_permute_local_kv only covers a local LBNHC cache.
[("LBHNC", False), ("BLHNC", True)],
)
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
FakeNixlWrapper,
)
def test_handshake_fails_on_kv_cache_layout_mismatch(
self, default_vllm_config, dist_init
self, default_vllm_config, dist_init, kv_cache_layout, enable_permute_local_kv
):
"""Verify that adding a remote agent fails if kv_cache_layout differs.
This test is only relevant for heterogeneous TP.
"""
vllm_config = create_vllm_config()
vllm_config = create_vllm_config(
enable_permute_local_kv=enable_permute_local_kv
)
# Mock TP world size to 2 to force heterogeneous TP when
# remote_tp_size=1
@@ -1151,7 +1158,10 @@ class TestNixlHandshake:
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config, connector.engine_id, hand_shake_latency=0
vllm_config,
connector.engine_id,
hand_shake_latency=0,
kv_cache_layout=kv_cache_layout,
)
worker = connector.connector_worker
@@ -1404,46 +1404,66 @@ def test_map_block_ids_for_block_size_ratio_hybrid():
@pytest.mark.cpu_test
def test_post_process_zeroes_untransferred_tail():
"""The untransferred sub-blocks of the last local block are zeroed on
receive; mamba state caches are untouched by the attention permute."""
@pytest.mark.parametrize(
("kv_cache_layout", "enable_permute_local_kv"),
[("LBHNC", False), ("LBNHC", False), ("LBNHC", True)],
)
def test_post_process_zeroes_untransferred_tail(
kv_cache_layout, enable_permute_local_kv
):
"""Received remote sub-blocks are regrouped per head and the untransferred
sub-blocks of the last local block are zeroed on receive, once per block
although two attention groups alias the tensor."""
from unittest.mock import MagicMock
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
NixlConnectorWorker,
)
from vllm.v1.kv_cache_interface import FullAttentionSpec, MambaSpec
ratio = 4
block_tokens = 8 # 2 tokens per remote sub-block
num_kv_heads = 2 # fewer than ratio
worker = MagicMock(spec=NixlConnectorWorker)
worker._group_spec_types = (FullAttentionSpec, MambaSpec)
worker.transfer_topo = MagicMock()
worker.device_type = "cpu"
worker.enable_permute_local_kv = False
attn_cache = torch.ones(6, block_tokens, 2, 4)
mamba_cache = torch.ones(6, 16)
worker.device_kv_caches = {"attn.0": attn_cache, "mamba.0": mamba_cache}
worker.enable_permute_local_kv = enable_permute_local_kv
worker.kv_cache_layout = kv_cache_layout
# Attention caches are [B, H, N, C]; distinct values per head and token.
expected = torch.arange(6 * num_kv_heads * block_tokens * 4).view(
6, num_kv_heads, block_tokens, 4
)
# Blocks [2, 3] as received: `ratio` head-major remote sub-blocks each.
received = (
expected[2:4]
.unflatten(2, (ratio, -1))
.transpose(1, 2)
.reshape(2, num_kv_heads, block_tokens, 4)
)
if kv_cache_layout == "LBNHC":
attn_cache = expected.transpose(1, 2).contiguous().transpose(1, 2)
if enable_permute_local_kv:
# The remote is LBHNC: its bytes land in token-major memory.
attn_cache.transpose(1, 2)[2:4] = received.view(2, block_tokens, -1, 4)
# Otherwise token-major blocks receive the sub-blocks in token order.
else:
attn_cache = expected.clone()
attn_cache[2:4] = received
worker.device_kv_caches = {"attn.0": attn_cache, "swa.0": attn_cache}
fa_group = MagicMock(layer_names=["attn.0"])
ssm_group = MagicMock(layer_names=["mamba.0"])
worker.kv_cache_config = MagicMock(transfer_groups=[fa_group, ssm_group])
# The cached property filters mamba layers out of the permuted caches.
attn_caches = NixlConnectorWorker._attention_kv_caches.func(worker)
assert len(attn_caches) == 1 and attn_caches[0] is attn_cache
worker._attention_kv_caches = attn_caches
swa_group = MagicMock(layer_names=["swa.0"])
worker.kv_cache_config = MagicMock(transfer_groups=[fa_group, swa_group])
_bind_worker_method(worker, "post_process_device_kv_on_receive")
# Request occupies blocks [2, 3]; only 6 of 8 sub-blocks were received.
worker.post_process_device_kv_on_receive(ratio, [([2, 3], 6)])
# Group 0 request in blocks [2, 3]; only 6 of 8 sub-blocks were received.
worker.post_process_device_kv_on_receive(ratio, [(0, [2, 3], 6)])
# Block 2 fully covered; block 3 covered for 2 sub-blocks (4 tokens).
assert torch.all(attn_cache[2] == 1)
assert torch.all(attn_cache[3, :4] == 1)
assert torch.all(attn_cache[3, 4:] == 0)
# Untouched blocks and the mamba cache keep their content.
assert torch.all(attn_cache[4] == 1)
assert torch.all(mamba_cache == 1)
assert torch.equal(attn_cache[2], expected[2])
assert torch.equal(attn_cache[3, :, :4], expected[3, :, :4])
assert torch.all(attn_cache[3, :, 4:] == 0)
# Untouched blocks keep their content.
assert torch.equal(attn_cache[4], expected[4])
@pytest.mark.cpu_test