[KVConnector][NIXL] Support attention-HMA layouts in pipeline-parallel push prefill (#50494)

Signed-off-by: zixi-qi <zixi@inferact.ai>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Yiliu Dong <91178480+qianlihuang@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
qizixi
2026-09-15 15:43:56 -07:00
committed by GitHub
co-authored by Claude Yiliu Dong Codex
parent c6fa1f05d1
commit 0fefffc934
8 changed files with 713 additions and 49 deletions
@@ -918,6 +918,7 @@ def _make_mock_worker_for_desc_ids(
worker = MagicMock(spec=NixlConnectorWorker)
worker.num_regions = num_regions
worker._uses_region_group_mapping = False
worker._transfer_layer_group_ids = ()
worker._has_mamba = has_mamba
worker._group_spec_types = group_spec_types
worker.block_len_per_layer = block_len_per_layer or [100]
@@ -12,6 +12,7 @@ mid-decode (silent corruption of an unrelated request).
"""
from collections import defaultdict
from threading import Event, Lock
from unittest.mock import MagicMock, patch
import numpy as np
@@ -112,6 +113,7 @@ def test_local_descriptors_follow_each_region_pool_capacity():
worker.block_len_per_layer = [16, 16]
worker.block_stride_per_layer = [16, 16]
worker.region_num_blocks = [2, 3]
worker._transfer_layer_region_indices = ()
descriptors = worker._build_fa_local([100, 1000], block_size_ratio=1)
@@ -119,7 +121,8 @@ def test_local_descriptors_follow_each_region_pool_capacity():
@pytest.mark.cpu_test
def test_overlaid_transfer_groups_share_region_geometry():
@pytest.mark.parametrize("push_pp", [False, True])
def test_overlaid_transfer_groups_share_region_geometry(push_pp):
"""Groups overlaid on one allocation share its transfer region."""
import msgspec
@@ -127,6 +130,9 @@ def test_overlaid_transfer_groups_share_region_geometry():
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
NixlAgentMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
NixlPushConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
NixlConnectorWorker,
)
@@ -153,7 +159,14 @@ def test_overlaid_transfer_groups_share_region_geometry():
}
groups = [KVCacheGroupSpec([layer_name], spec) for layer_name in caches]
worker = object.__new__(NixlConnectorWorker)
worker_cls = NixlPushConnectorWorker if push_pp else NixlConnectorWorker
worker = object.__new__(worker_cls)
if push_pp:
worker._push_writer_stop = Event()
worker._push_writer_wake = Event()
worker._push_writer_thread = MagicMock()
worker._sending_transfers_lock = Lock()
worker._sending_transfers = defaultdict(list)
worker.tp_rank = 0
worker.world_size = 1
worker.transfer_tp_rank = 0
@@ -198,6 +211,9 @@ def test_overlaid_transfer_groups_share_region_geometry():
worker._desc_is_dram_by_block_size = {}
worker._desc_pos_by_block_size = {}
worker._dram_src_handles_by_block_size = {}
worker._transfer_layer_names = ()
worker._transfer_layer_region_indices = ()
worker._transfer_layer_group_ids = ()
worker._region_is_mla = []
worker.block_len_per_layer = []
worker.block_stride_per_layer = []
@@ -205,7 +221,8 @@ def test_overlaid_transfer_groups_share_region_geometry():
worker.use_host_buffer = False
worker.host_xfer_buffers = {}
worker.device_kv_caches = {}
worker.pp_size = 1
worker.pp_size = 2 if push_pp else 1
worker._is_hma_required = True
worker.dcp_size = 1
worker.pcp_size = 1
worker.kv_buffer_device = "cuda"
@@ -240,7 +257,13 @@ def test_overlaid_transfer_groups_share_region_geometry():
expected_addrs = [
backing.data_ptr() + block * block_stride for block in range(num_blocks)
]
assert worker.src_blocks_data[:, 0].tolist() == expected_addrs
num_desc_regions = 2 if push_pp else 1
assert worker.src_blocks_data[:, 0].tolist() == expected_addrs * num_desc_regions
assert worker.num_descs == num_blocks * num_desc_regions
assert (
worker.dst_region_num_blocks[worker.engine_id]
== [num_blocks] * num_desc_regions
)
metadata = msgspec.msgpack.decode(
worker.xfer_handshake_metadata.agent_metadata_bytes,
@@ -248,6 +271,7 @@ def test_overlaid_transfer_groups_share_region_geometry():
)
assert metadata.region_group_ids == [-1]
assert metadata.region_num_blocks == [num_blocks]
assert metadata.region_members == ([["layer.0", "layer.1"]] if push_pp else [])
assert worker._block_ids_by_region(([0], [2]), worker.region_group_ids) == [[0, 2]]
@@ -32,9 +32,13 @@ from unittest.mock import MagicMock, patch
import msgspec
import pytest
from vllm.distributed.kv_transfer.kv_connector.utils import TransferTopology
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorTransferResults,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import (
NixlBaseConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
PUSH_REG_NOTIF_PREFIX,
NixlAgentMetadata,
@@ -43,6 +47,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
NixlPushConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import TPMapping
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import (
get_base_request_id,
)
@@ -354,6 +359,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
w._failed_recv_reqs = queue.Queue()
w._recv_failures = set()
w._recving_transfers = defaultdict(list)
w._is_hma_required = False
w._reqs_to_process = set()
w._reqs_to_send = {}
w.consumer_notification_counts_by_req = defaultdict(int)
@@ -361,12 +367,16 @@ class _StubWriterWorker(NixlPushConnectorWorker):
w.pcp_rank = 0
w.pcp_dcp_sharded = False
w.world_size = 1
w.pp_size = 1
w.engine_id = "test-decode-engine"
w._remote_agents = {}
w._handshake_lock = threading.RLock()
w._physical_blocks_per_logical_kv_block = 1
w._uses_region_group_mapping = False
w.region_group_ids = [0]
w._transfer_layer_names = ()
w._transfer_layer_region_indices = ()
w._transfer_layer_group_ids = ()
w.dst_region_num_blocks = {}
w.dst_region_group_ids = {}
w.dst_uses_region_group_mapping = {}
@@ -922,12 +932,20 @@ class TestPushWriterNotifs:
# Both handles were still cleaned up.
assert request_id not in w._sending_transfers
def test_completed_push_send_waits_for_consumer_notif(self):
@pytest.mark.parametrize(("pp_size", "is_hma"), [(1, False), (2, True)])
def test_completed_push_send_waits_for_consumer_notif(self, pp_size, is_hma):
"""P-side sends are completed by consumer notifs / lease expiry,
not by local WRITE-handle completion."""
w = self._pollable_worker()
w.pp_size = pp_size
w._is_hma_required = is_hma
request_id = self._make_sending_req(w)
w.nixl_wrapper.check_xfer_state.return_value = "DONE"
w.nixl_wrapper.check_xfer_state.side_effect = ["DONE", "PROC", "DONE"]
results = w.get_transfer_results()
assert request_id not in results.finished_sending
assert w._sending_transfers[request_id] == [102]
assert request_id in w._reqs_to_send
results = w.get_transfer_results()
@@ -940,6 +958,12 @@ class TestPushWriterNotifs:
results = w.get_transfer_results()
assert request_id in results.finished_sending
assert request_id not in w._reqs_to_send
assert request_id not in w._reqs_to_process
assert w._evict_finished_inbox.get_nowait() == request_id
assert w.get_transfer_results().finished_sending == set()
assert w._evict_finished_inbox.empty()
assert w.nixl_wrapper.release_xfer_handle.call_count == 2
w.xfer_stats.record_kv_expired_req.assert_not_called()
# ----------------------------------------------------------------- #
@@ -1576,3 +1600,375 @@ class TestPushPrefixCaching:
local, remote = self._written_block_ids(w)
assert local == [10, 11, 12]
assert remote == [500, 501, 502]
@pytest.mark.parametrize("layer_name_routing", [False, True])
def test_different_region_groups_require_layer_routing(self, layer_name_routing):
"""A layer mapped to a pooled remote region can still receive its blocks."""
w, engine_id = self._worker_driving_xfer()
w.dst_region_group_ids[engine_id] = [-1]
if layer_name_routing:
w._transfer_layer_group_ids = (0,)
reg = _registration_data("req-groups", local_block_ids=([500, 501],))
if not layer_name_routing:
with pytest.raises(NotImplementedError, match="cache-group layouts"):
NixlPushConnectorWorker._do_start_push_kv(
w, "req-groups", ([10, 11],), reg
)
w.nixl_wrapper.make_prepped_xfer.assert_not_called()
return
NixlPushConnectorWorker._do_start_push_kv(w, "req-groups", ([10, 11],), reg)
local, remote = self._written_block_ids(w)
assert local == [10, 11]
assert remote == [500, 501]
def _agent_metadata(
region_layers: list[list[str]],
base_addresses: list[int],
block_lens: list[int],
block_strides: list[int] | None = None,
) -> NixlAgentMetadata:
return NixlAgentMetadata(
engine_id="remote-engine",
agent_metadata=b"agent",
kv_caches_base_addr=base_addresses,
device_id=7,
num_blocks=2,
block_lens=block_lens,
block_strides=list(block_lens if block_strides is None else block_strides),
kv_cache_layout="LBHNC",
block_size=16,
ssm_sizes=(0, 0),
attn_backend_name="FLASH_ATTN",
physical_blocks_per_logical_kv_block=1,
region_members=region_layers,
)
def _layer_routing_worker(
region_layers: list[list[str]],
group_by_layer: dict[str, int],
pp_size: int = 2,
is_hma: bool = True,
) -> _StubWriterWorker:
worker = _StubWriterWorker.fresh()
worker.pp_size = pp_size
worker.dcp_size = 1
worker.block_size = 16
worker._has_mamba = False
worker._is_hma_required = is_hma
worker.kv_cache_config = MagicMock(transfer_group_index_by_layer=group_by_layer)
worker.transfer_topo = MagicMock()
worker._set_region_layers(region_layers)
return worker
@pytest.mark.parametrize(
("region_num_blocks", "expected"),
[(None, [1, 2, 15, 21, 22]), ([4, 7, 5], [1, 2, 9, 12, 13])],
)
def test_layer_group_ids_route_descriptor_blocks(region_num_blocks, expected):
worker = _layer_routing_worker(
[["a", "a.swa"], ["b"]], {"a": 0, "a.swa": 1, "b": 0}
)
desc_ids = worker._compute_desc_ids(
block_ids=[[1, 2], [5]],
dst_num_blocks=10,
block_size_ratio=None,
physical_blocks_per_logical=1,
region_num_blocks=region_num_blocks,
)
assert desc_ids.tolist() == expected
def test_layer_metadata_round_trip():
metadata = _agent_metadata([["L0", "L1"]], [0x10000], [256])
metadata.region_num_blocks = [4]
metadata.region_group_ids = [-1]
metadata.region_names = ["L0"]
metadata.region_mem_types = ["VRAM"]
encoded = msgspec.msgpack.encode(metadata)
assert msgspec.msgpack.Decoder(NixlAgentMetadata).decode(encoded) == metadata
def test_layer_identity_gate_preserves_the_non_hma_path():
assert _layer_routing_worker([["a"]], {"a": 0})._transfer_layer_region_indices == (
0,
)
# A base worker (pull) never routes by layer name.
pull = object.__new__(NixlBaseConnectorWorker)
pull._transfer_layer_names = ()
pull._transfer_layer_region_indices = ()
pull._transfer_layer_group_ids = ()
pull._set_region_layers([["a"]])
assert pull._transfer_layer_region_indices == ()
# A non-HMA local layout does not require layer-name routing.
assert (
_layer_routing_worker(
[["a"]], {"a": 0}, is_hma=False
)._transfer_layer_region_indices
== ()
)
# PP=1 has congruent local/remote regions and keeps the region route even
# when its allocator is hybrid.
assert (
_layer_routing_worker(
[["a"]], {"a": 0}, pp_size=1
)._transfer_layer_region_indices
== ()
)
def test_layer_alignment_fails_loud_when_remote_omits_layers():
# Falling back to region-index routing here would silently transfer stale
# KV, so an unannounced peer must fail the handshake instead.
worker = _layer_routing_worker([["a"]], {"a": 0})
with pytest.raises(AssertionError, match="no region_members"):
worker._align_remote_regions_by_layer(_agent_metadata([], [0xA000], [128]))
def test_layer_alignment_expands_pooled_regions():
worker = _layer_routing_worker(
[["a", "a.swa"], ["b"]], {"a": 0, "a.swa": 1, "b": 0}
)
assert worker._transfer_layer_names == ("a", "a.swa", "b")
assert worker._transfer_layer_region_indices == (0, 0, 1)
assert worker._transfer_layer_group_ids == (0, 1, 0)
metadata = _agent_metadata([["a", "a.swa"], ["b"]], [0xA000, 0xB000], [128, 128])
metadata.region_num_blocks = [4, 6]
metadata.region_group_ids = [-1, 0]
metadata.region_names = ["a", "b"]
metadata.region_mem_types = ["VRAM", "VRAM"]
worker._align_remote_regions_by_layer(metadata)
assert metadata.kv_caches_base_addr == [0xA000, 0xA000, 0xB000]
assert metadata.block_lens == [128, 128, 128]
assert metadata.block_strides == [128, 128, 128]
assert metadata.region_members == [["a"], ["a.swa"], ["b"]]
assert metadata.region_num_blocks == [4, 4, 6]
assert metadata.region_group_ids == [-1, -1, 0]
assert metadata.region_names == ["a", "a.swa", "b"]
assert metadata.region_mem_types == ["VRAM", "VRAM", "VRAM"]
def test_layer_alignment_filters_and_reorders_a_pp_stage():
worker = _layer_routing_worker([["l2"], ["l3"]], {"l2": 0, "l3": 1})
metadata = _agent_metadata(
[["l2"], ["l0"], ["l3"], ["l1"]],
[0xC000, 0xA000, 0xD000, 0xB000],
[65536, 65536, 32768, 32768],
[131072, 131072, 65536, 65536],
)
metadata.region_num_blocks = [4, 6, 8, 10]
metadata.region_group_ids = [0, 0, 1, 1]
metadata.region_names = ["l2", "l0", "l3", "l1"]
metadata.region_mem_types = ["VRAM"] * 4
worker._align_remote_regions_by_layer(metadata)
assert worker._transfer_layer_region_indices == (0, 1)
assert worker._transfer_layer_group_ids == (0, 1)
assert metadata.kv_caches_base_addr == [0xC000, 0xD000]
assert metadata.block_lens == [65536, 32768]
assert metadata.block_strides == [131072, 65536]
assert metadata.region_num_blocks == [4, 8]
assert metadata.region_group_ids == [0, 1]
assert metadata.region_names == ["l2", "l3"]
assert metadata.region_mem_types == ["VRAM", "VRAM"]
@pytest.mark.parametrize(
("local_num_blocks", "remote_num_blocks"),
[([4, 4], [4, 4, 4]), ([4, 6], [3, 5, 7])],
)
def test_layer_descriptors_pair_layers_across_asymmetric_pp_split(
local_num_blocks, remote_num_blocks
):
"""A PP split can leave each stage a different mix of attention types.
Stage 1 owns L3 (full) pooled with L4 (sliding) in region 0, and L5 (full)
alone in region 1. The consumer holds every layer and pools them
differently, so region indices cannot be paired positionally.
"""
worker = _layer_routing_worker([["L3", "L4"], ["L5"]], {"L3": 0, "L4": 1, "L5": 0})
worker.block_len_per_layer = [128, 128]
worker.block_stride_per_layer = [256, 512]
worker._region_is_mla = [False, False]
worker.num_blocks = 4
worker.region_num_blocks = local_num_blocks
worker.device_id = 0
consumer = _agent_metadata(
[["L0", "L1"], ["L3", "L2"], ["L5", "L4"]],
[0xA000, 0xB000, 0xC000],
[128, 128, 128],
[256, 512, 1024],
)
consumer.num_blocks = 4
consumer.region_num_blocks = remote_num_blocks
worker._align_remote_regions_by_layer(consumer)
# L3 -> remote region 1, L4 -> 2, L5 -> 2. Pairing by index would have sent
# L5 to region 1 and L4 to region 2's sibling.
assert consumer.kv_caches_base_addr == [0xB000, 0xC000, 0xC000]
plan = TPMapping(((0,), (0,)), (0,), {0: 0}, 0)
local_descs = worker._build_fa_local([0x1000, 0x2000], block_size_ratio=1)
remote_descs = worker._build_fa_remote(plan, consumer, block_size_ratio=1)
local_counts = [local_num_blocks[i] for i in worker._transfer_layer_region_indices]
local_ids = worker._compute_desc_ids(
[[1, 2], [3]], 4, None, 1, region_num_blocks=local_counts
)
remote_ids = worker._compute_desc_ids(
[[0, 1], [2]], 4, None, 1, region_num_blocks=consumer.region_num_blocks
)
assert len(local_descs) == sum(local_counts)
assert len(remote_descs) == sum(consumer.region_num_blocks)
# Each pair addresses the same layer's blocks despite different pooling
# and strides. L3 and L5 use group 0; L4 uses group 1.
assert local_descs[local_ids].tolist() == [
[0x1100, 128, 0],
[0x1200, 128, 0],
[0x1300, 128, 0],
[0x2200, 128, 0],
[0x2400, 128, 0],
]
assert remote_descs[remote_ids].tolist() == [
[0xB000, 128, 7],
[0xB200, 128, 7],
[0xC800, 128, 7],
[0xC000, 128, 7],
[0xC400, 128, 7],
]
def test_layer_alignment_is_canonical_across_remote_orderings():
local, groups = [["x"], ["y"]], {"x": 0, "y": 1}
rank0 = _agent_metadata([["x"], ["y"]], [0x1000, 0x2000], [64, 128], [256, 512])
rank1 = _agent_metadata([["y"], ["x"]], [0x2000, 0x1000], [128, 64], [512, 256])
_layer_routing_worker(local, groups)._align_remote_regions_by_layer(rank0)
_layer_routing_worker(local, groups)._align_remote_regions_by_layer(rank1)
assert rank0.kv_caches_base_addr == rank1.kv_caches_base_addr == [0x1000, 0x2000]
assert rank0.block_lens == rank1.block_lens == [64, 128]
assert rank0.block_strides == rank1.block_strides == [256, 512]
def test_layer_alignment_is_idempotent():
worker = _layer_routing_worker(
[["a", "a.swa"], ["b"]], {"a": 0, "a.swa": 1, "b": 0}
)
metadata = _agent_metadata([["a", "a.swa"], ["b"]], [0xA000, 0xB000], [128, 128])
metadata.region_num_blocks = [4, 6]
metadata.region_group_ids = [-1, 0]
metadata.region_names = ["a", "b"]
metadata.region_mem_types = ["VRAM", "VRAM"]
worker._align_remote_regions_by_layer(metadata)
aligned = msgspec.msgpack.encode(metadata)
worker._align_remote_regions_by_layer(metadata)
assert msgspec.msgpack.encode(metadata) == aligned
def test_layer_alignment_rejects_missing_local_layer():
worker = _layer_routing_worker([["l0"], ["l1"]], {"l0": 0, "l1": 1})
metadata = _agent_metadata([["l0"]], [0xA000], [128])
with pytest.raises(AssertionError, match="missing locally owned layers"):
worker._align_remote_regions_by_layer(metadata)
def test_layer_alignment_rejects_duplicate_remote_layer():
worker = _layer_routing_worker([["a"]], {"a": 0})
metadata = _agent_metadata([["a"], ["a"]], [0xA000, 0xB000], [128, 128])
with pytest.raises(AssertionError, match="in multiple regions"):
worker._align_remote_regions_by_layer(metadata)
def test_layer_alignment_rejects_inconsistent_remote_metadata():
worker = _layer_routing_worker([["a"]], {"a": 0})
metadata = _agent_metadata([["a"], ["b"]], [0xA000], [128])
with pytest.raises(AssertionError, match="lengths disagree"):
worker._align_remote_regions_by_layer(metadata)
@pytest.mark.parametrize(
"field",
["region_num_blocks", "region_group_ids", "region_names", "region_mem_types"],
)
def test_layer_alignment_rejects_inconsistent_region_geometry(field):
worker = _layer_routing_worker([["a"]], {"a": 0})
metadata = _agent_metadata([["a"], ["b"]], [0xA000, 0xB000], [128, 128])
setattr(metadata, field, [])
original = msgspec.msgpack.encode(metadata)
with pytest.raises(AssertionError, match="lengths disagree"):
worker._align_remote_regions_by_layer(metadata)
assert msgspec.msgpack.encode(metadata) == original
def test_set_region_layers_rejects_duplicate_local_layer():
with pytest.raises(AssertionError, match="spans multiple NIXL regions"):
_layer_routing_worker([["a"], ["a"]], {"a": 0})
def test_set_region_layers_rejects_layer_outside_any_kv_group():
with pytest.raises(AssertionError, match="outside any local group"):
_layer_routing_worker([["a"], ["b"]], {"a": 0})
@pytest.mark.parametrize(
("local_block_size", "remote_block_size", "remote_tp_size", "error"),
[
(32, 16, 1, "identical P/D block sizes"),
(16, 32, 1, "identical P/D block sizes"),
(16, 16, 2, "decode TP greater"),
],
)
def test_layer_handshake_rejects_unsupported_geometry(
local_block_size: int, remote_block_size: int, remote_tp_size: int, error: str
):
"""Reject unsupported peers without registering agents or transfer state."""
metadata = _agent_metadata([["a"]], [0xA000], [128])
metadata.block_size = remote_block_size
worker = _layer_routing_worker([["a"]], {"a": 0})
worker.block_size = local_block_size
worker.block_len_per_layer = [128]
worker.use_mla = False
worker.nixl_wrapper = MagicMock()
worker.kv_caches_base_addr = defaultdict(dict)
worker.dst_num_blocks = {}
worker.tp_mappings = {}
worker.transfer_topo = TransferTopology(
tp_rank=0,
tp_size=1,
block_size=local_block_size,
engine_id=worker.engine_id,
is_mla=False,
is_mamba=False,
total_num_kv_heads=8,
attn_backends=[],
)
with pytest.raises(NotImplementedError, match=error):
worker.add_remote_agent(metadata, remote_tp_size=remote_tp_size)
worker.nixl_wrapper.add_remote_agent.assert_not_called()
worker.nixl_wrapper.prep_xfer_dlist.assert_not_called()
assert not worker.tp_mappings
assert not worker.dst_num_blocks
assert not worker.kv_caches_base_addr
with pytest.raises(KeyError):
worker.transfer_topo.get_engine_info(metadata.engine_id)