diff --git a/docs/design/nixl_kv_push_connector.md b/docs/design/nixl_kv_push_connector.md index 6f66be5eb9..a6debd7dba 100644 --- a/docs/design/nixl_kv_push_connector.md +++ b/docs/design/nixl_kv_push_connector.md @@ -181,6 +181,50 @@ The completion notif sent from P to D after a WRITE is the existing is D's own request id, taken from the registration), so the D-side accounting code is unchanged. +## Pipeline parallelism and hybrid KV caches + +The pull connector requires the local and remote workers to expose a +congruent list of KV regions — region *i* here corresponds to region +*i* there. That assumption breaks under pipeline parallelism (PP) +combined with a hybrid (HMA) KV layout: + +* a PP-sharded prefiller (**P**) holds only a slice of the model's + layers while the `PP=1` decoder (**D**) holds them all, so region + counts differ; +* with HMA, several layer names are pooled into one region and the layer + that represents a pooled region can differ between P and D. + +`NixlPushConnector` handles this for PP-sharded producers by routing +**by layer-name (member) identity** instead of by region index. Each +worker advertises which layer names back each of its NIXL regions in the +handshake metadata (`NixlAgentMetadata.region_members`). A producer that +needs member routing derives its member-major layout once, when it +registers its KV caches, and every transfer it issues uses that order. +`add_remote_agent` then selects exactly the remote regions this stage +owns and reorders them to match, so both sides stay paired regardless of +how each remote rank happens to order its metadata. + +Addresses, block lengths, strides, and per-region capacities follow the same +member order. Descriptor offsets use each member's region capacity, so P and +D need not allocate the same number of blocks. Physical allocations are still +registered once, even when multiple members share them. + +Invariants enforced when the remote regions are aligned: + +* every locally owned member must be advertised exactly once by the + remote; a missing member fails the handshake rather than silently + leaving that layer's KV stale, and remote-only members (owned by other + PP stages) are ignored; +* a remote that omits member metadata while the local layout requires + member routing fails the handshake instead of falling back to + region-index routing; +* member order is a property of the local layout alone, so the same local + source descriptors serve every remote engine and TP rank; only the + remote descriptor list is rebuilt per rank. + +Decode-side PP is unsupported because completions are counted per +consumer rank. Mamba/SSM hybrids are unsupported under PP. + ## Scheduler-side responsibilities `NixlPushConnectorScheduler` extends the base scheduler with: diff --git a/docs/features/nixl_connector_compatibility.md b/docs/features/nixl_connector_compatibility.md index 031fba5b2f..a41d5868b1 100644 --- a/docs/features/nixl_connector_compatibility.md +++ b/docs/features/nixl_connector_compatibility.md @@ -104,6 +104,24 @@ By default, a **compatibility hash** is checked during handshake. P and D instan - `LBNHC` (token-major, formerly `NHD`) layout is supported but does **not** allow heterogeneous TP head splitting. - Experimental `LBHNC` ↔ `LBNHC` permute: enable via `--kv-transfer-config '{"enable_permute_local_kv": true}'`. Not supported with HMA. +### Pipeline parallelism + +The default (pull) NixlConnector does **not** support +`pipeline-parallel-size > 1` together with a hybrid KV cache (HMA) +layout: region indices are not a stable identity across the +prefill/decode layer split, so the connector raises at startup. Use the +push connector (`NixlPushConnector`) for pipeline parallelism with +hybrid KV caches — it routes transfers by layer-name (member) identity, +so a PP-sharded prefiller can write into a `PP=1` decoder. See +[NIXL push-mode KV transfer](../design/nixl_kv_push_connector.md). + +Current push PP + HMA limitations: + +- Only the prefiller (producer) may be PP-sharded; decode-side PP is not supported. +- Hybrid SSM/Mamba layouts are not supported under PP. +- HMA requires the same block size on P and D. +- Attention-HMA member routing requires decode TP to be no greater than prefill TP. + ### Quantized KV cache [Quantized KV cache](quantization/quantized_kvcache.md) (e.g., FP8) requires both P and D instances to use the **same** `cache_dtype`. Mismatched cache dtypes will fail the compatibility hash check during handshake. diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py index 60eb2d6614..ae2ddece7f 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py @@ -918,6 +918,7 @@ def _make_mock_worker_for_desc_ids( worker = MagicMock(spec=NixlConnectorWorker) worker.num_regions = num_regions worker._uses_region_group_mapping = False + worker._transfer_layer_group_ids = () worker._has_mamba = has_mamba worker._group_spec_types = group_spec_types worker.block_len_per_layer = block_len_per_layer or [100] diff --git a/tests/v1/kv_connector/unit/test_nixl_desc_geometry.py b/tests/v1/kv_connector/unit/test_nixl_desc_geometry.py index bfec51da69..a4bfa9ab83 100644 --- a/tests/v1/kv_connector/unit/test_nixl_desc_geometry.py +++ b/tests/v1/kv_connector/unit/test_nixl_desc_geometry.py @@ -12,6 +12,7 @@ mid-decode (silent corruption of an unrelated request). """ from collections import defaultdict +from threading import Event, Lock from unittest.mock import MagicMock, patch import numpy as np @@ -112,6 +113,7 @@ def test_local_descriptors_follow_each_region_pool_capacity(): worker.block_len_per_layer = [16, 16] worker.block_stride_per_layer = [16, 16] worker.region_num_blocks = [2, 3] + worker._transfer_layer_region_indices = () descriptors = worker._build_fa_local([100, 1000], block_size_ratio=1) @@ -119,7 +121,8 @@ def test_local_descriptors_follow_each_region_pool_capacity(): @pytest.mark.cpu_test -def test_overlaid_transfer_groups_share_region_geometry(): +@pytest.mark.parametrize("push_pp", [False, True]) +def test_overlaid_transfer_groups_share_region_geometry(push_pp): """Groups overlaid on one allocation share its transfer region.""" import msgspec @@ -127,6 +130,9 @@ def test_overlaid_transfer_groups_share_region_geometry(): from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( NixlAgentMetadata, ) + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import ( + NixlPushConnectorWorker, + ) from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import ( NixlConnectorWorker, ) @@ -153,7 +159,14 @@ def test_overlaid_transfer_groups_share_region_geometry(): } groups = [KVCacheGroupSpec([layer_name], spec) for layer_name in caches] - worker = object.__new__(NixlConnectorWorker) + worker_cls = NixlPushConnectorWorker if push_pp else NixlConnectorWorker + worker = object.__new__(worker_cls) + if push_pp: + worker._push_writer_stop = Event() + worker._push_writer_wake = Event() + worker._push_writer_thread = MagicMock() + worker._sending_transfers_lock = Lock() + worker._sending_transfers = defaultdict(list) worker.tp_rank = 0 worker.world_size = 1 worker.transfer_tp_rank = 0 @@ -198,6 +211,9 @@ def test_overlaid_transfer_groups_share_region_geometry(): worker._desc_is_dram_by_block_size = {} worker._desc_pos_by_block_size = {} worker._dram_src_handles_by_block_size = {} + worker._transfer_layer_names = () + worker._transfer_layer_region_indices = () + worker._transfer_layer_group_ids = () worker._region_is_mla = [] worker.block_len_per_layer = [] worker.block_stride_per_layer = [] @@ -205,7 +221,8 @@ def test_overlaid_transfer_groups_share_region_geometry(): worker.use_host_buffer = False worker.host_xfer_buffers = {} worker.device_kv_caches = {} - worker.pp_size = 1 + worker.pp_size = 2 if push_pp else 1 + worker._is_hma_required = True worker.dcp_size = 1 worker.pcp_size = 1 worker.kv_buffer_device = "cuda" @@ -240,7 +257,13 @@ def test_overlaid_transfer_groups_share_region_geometry(): expected_addrs = [ backing.data_ptr() + block * block_stride for block in range(num_blocks) ] - assert worker.src_blocks_data[:, 0].tolist() == expected_addrs + num_desc_regions = 2 if push_pp else 1 + assert worker.src_blocks_data[:, 0].tolist() == expected_addrs * num_desc_regions + assert worker.num_descs == num_blocks * num_desc_regions + assert ( + worker.dst_region_num_blocks[worker.engine_id] + == [num_blocks] * num_desc_regions + ) metadata = msgspec.msgpack.decode( worker.xfer_handshake_metadata.agent_metadata_bytes, @@ -248,6 +271,7 @@ def test_overlaid_transfer_groups_share_region_geometry(): ) assert metadata.region_group_ids == [-1] assert metadata.region_num_blocks == [num_blocks] + assert metadata.region_members == ([["layer.0", "layer.1"]] if push_pp else []) assert worker._block_ids_by_region(([0], [2]), worker.region_group_ids) == [[0, 2]] diff --git a/tests/v1/kv_connector/unit/test_nixl_push_connector.py b/tests/v1/kv_connector/unit/test_nixl_push_connector.py index 913ed0dcba..a908d32462 100644 --- a/tests/v1/kv_connector/unit/test_nixl_push_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_push_connector.py @@ -32,9 +32,13 @@ from unittest.mock import MagicMock, patch import msgspec import pytest +from vllm.distributed.kv_transfer.kv_connector.utils import TransferTopology from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorTransferResults, ) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import ( + NixlBaseConnectorWorker, +) from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( PUSH_REG_NOTIF_PREFIX, NixlAgentMetadata, @@ -43,6 +47,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import ( NixlPushConnectorWorker, ) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import TPMapping from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import ( get_base_request_id, ) @@ -354,6 +359,7 @@ class _StubWriterWorker(NixlPushConnectorWorker): w._failed_recv_reqs = queue.Queue() w._recv_failures = set() w._recving_transfers = defaultdict(list) + w._is_hma_required = False w._reqs_to_process = set() w._reqs_to_send = {} w.consumer_notification_counts_by_req = defaultdict(int) @@ -361,12 +367,16 @@ class _StubWriterWorker(NixlPushConnectorWorker): w.pcp_rank = 0 w.pcp_dcp_sharded = False w.world_size = 1 + w.pp_size = 1 w.engine_id = "test-decode-engine" w._remote_agents = {} w._handshake_lock = threading.RLock() w._physical_blocks_per_logical_kv_block = 1 w._uses_region_group_mapping = False w.region_group_ids = [0] + w._transfer_layer_names = () + w._transfer_layer_region_indices = () + w._transfer_layer_group_ids = () w.dst_region_num_blocks = {} w.dst_region_group_ids = {} w.dst_uses_region_group_mapping = {} @@ -922,12 +932,20 @@ class TestPushWriterNotifs: # Both handles were still cleaned up. assert request_id not in w._sending_transfers - def test_completed_push_send_waits_for_consumer_notif(self): + @pytest.mark.parametrize(("pp_size", "is_hma"), [(1, False), (2, True)]) + def test_completed_push_send_waits_for_consumer_notif(self, pp_size, is_hma): """P-side sends are completed by consumer notifs / lease expiry, not by local WRITE-handle completion.""" w = self._pollable_worker() + w.pp_size = pp_size + w._is_hma_required = is_hma request_id = self._make_sending_req(w) - w.nixl_wrapper.check_xfer_state.return_value = "DONE" + w.nixl_wrapper.check_xfer_state.side_effect = ["DONE", "PROC", "DONE"] + + results = w.get_transfer_results() + assert request_id not in results.finished_sending + assert w._sending_transfers[request_id] == [102] + assert request_id in w._reqs_to_send results = w.get_transfer_results() @@ -940,6 +958,12 @@ class TestPushWriterNotifs: results = w.get_transfer_results() assert request_id in results.finished_sending assert request_id not in w._reqs_to_send + assert request_id not in w._reqs_to_process + assert w._evict_finished_inbox.get_nowait() == request_id + assert w.get_transfer_results().finished_sending == set() + assert w._evict_finished_inbox.empty() + assert w.nixl_wrapper.release_xfer_handle.call_count == 2 + w.xfer_stats.record_kv_expired_req.assert_not_called() # ----------------------------------------------------------------- # @@ -1576,3 +1600,375 @@ class TestPushPrefixCaching: local, remote = self._written_block_ids(w) assert local == [10, 11, 12] assert remote == [500, 501, 502] + + @pytest.mark.parametrize("layer_name_routing", [False, True]) + def test_different_region_groups_require_layer_routing(self, layer_name_routing): + """A layer mapped to a pooled remote region can still receive its blocks.""" + w, engine_id = self._worker_driving_xfer() + w.dst_region_group_ids[engine_id] = [-1] + if layer_name_routing: + w._transfer_layer_group_ids = (0,) + reg = _registration_data("req-groups", local_block_ids=([500, 501],)) + + if not layer_name_routing: + with pytest.raises(NotImplementedError, match="cache-group layouts"): + NixlPushConnectorWorker._do_start_push_kv( + w, "req-groups", ([10, 11],), reg + ) + w.nixl_wrapper.make_prepped_xfer.assert_not_called() + return + + NixlPushConnectorWorker._do_start_push_kv(w, "req-groups", ([10, 11],), reg) + + local, remote = self._written_block_ids(w) + assert local == [10, 11] + assert remote == [500, 501] + + +def _agent_metadata( + region_layers: list[list[str]], + base_addresses: list[int], + block_lens: list[int], + block_strides: list[int] | None = None, +) -> NixlAgentMetadata: + return NixlAgentMetadata( + engine_id="remote-engine", + agent_metadata=b"agent", + kv_caches_base_addr=base_addresses, + device_id=7, + num_blocks=2, + block_lens=block_lens, + block_strides=list(block_lens if block_strides is None else block_strides), + kv_cache_layout="LBHNC", + block_size=16, + ssm_sizes=(0, 0), + attn_backend_name="FLASH_ATTN", + physical_blocks_per_logical_kv_block=1, + region_members=region_layers, + ) + + +def _layer_routing_worker( + region_layers: list[list[str]], + group_by_layer: dict[str, int], + pp_size: int = 2, + is_hma: bool = True, +) -> _StubWriterWorker: + worker = _StubWriterWorker.fresh() + worker.pp_size = pp_size + worker.dcp_size = 1 + worker.block_size = 16 + worker._has_mamba = False + worker._is_hma_required = is_hma + worker.kv_cache_config = MagicMock(transfer_group_index_by_layer=group_by_layer) + worker.transfer_topo = MagicMock() + worker._set_region_layers(region_layers) + return worker + + +@pytest.mark.parametrize( + ("region_num_blocks", "expected"), + [(None, [1, 2, 15, 21, 22]), ([4, 7, 5], [1, 2, 9, 12, 13])], +) +def test_layer_group_ids_route_descriptor_blocks(region_num_blocks, expected): + worker = _layer_routing_worker( + [["a", "a.swa"], ["b"]], {"a": 0, "a.swa": 1, "b": 0} + ) + desc_ids = worker._compute_desc_ids( + block_ids=[[1, 2], [5]], + dst_num_blocks=10, + block_size_ratio=None, + physical_blocks_per_logical=1, + region_num_blocks=region_num_blocks, + ) + assert desc_ids.tolist() == expected + + +def test_layer_metadata_round_trip(): + metadata = _agent_metadata([["L0", "L1"]], [0x10000], [256]) + metadata.region_num_blocks = [4] + metadata.region_group_ids = [-1] + metadata.region_names = ["L0"] + metadata.region_mem_types = ["VRAM"] + + encoded = msgspec.msgpack.encode(metadata) + assert msgspec.msgpack.Decoder(NixlAgentMetadata).decode(encoded) == metadata + + +def test_layer_identity_gate_preserves_the_non_hma_path(): + assert _layer_routing_worker([["a"]], {"a": 0})._transfer_layer_region_indices == ( + 0, + ) + + # A base worker (pull) never routes by layer name. + pull = object.__new__(NixlBaseConnectorWorker) + pull._transfer_layer_names = () + pull._transfer_layer_region_indices = () + pull._transfer_layer_group_ids = () + pull._set_region_layers([["a"]]) + assert pull._transfer_layer_region_indices == () + + # A non-HMA local layout does not require layer-name routing. + assert ( + _layer_routing_worker( + [["a"]], {"a": 0}, is_hma=False + )._transfer_layer_region_indices + == () + ) + + # PP=1 has congruent local/remote regions and keeps the region route even + # when its allocator is hybrid. + assert ( + _layer_routing_worker( + [["a"]], {"a": 0}, pp_size=1 + )._transfer_layer_region_indices + == () + ) + + +def test_layer_alignment_fails_loud_when_remote_omits_layers(): + # Falling back to region-index routing here would silently transfer stale + # KV, so an unannounced peer must fail the handshake instead. + worker = _layer_routing_worker([["a"]], {"a": 0}) + with pytest.raises(AssertionError, match="no region_members"): + worker._align_remote_regions_by_layer(_agent_metadata([], [0xA000], [128])) + + +def test_layer_alignment_expands_pooled_regions(): + worker = _layer_routing_worker( + [["a", "a.swa"], ["b"]], {"a": 0, "a.swa": 1, "b": 0} + ) + assert worker._transfer_layer_names == ("a", "a.swa", "b") + assert worker._transfer_layer_region_indices == (0, 0, 1) + assert worker._transfer_layer_group_ids == (0, 1, 0) + + metadata = _agent_metadata([["a", "a.swa"], ["b"]], [0xA000, 0xB000], [128, 128]) + metadata.region_num_blocks = [4, 6] + metadata.region_group_ids = [-1, 0] + metadata.region_names = ["a", "b"] + metadata.region_mem_types = ["VRAM", "VRAM"] + worker._align_remote_regions_by_layer(metadata) + + assert metadata.kv_caches_base_addr == [0xA000, 0xA000, 0xB000] + assert metadata.block_lens == [128, 128, 128] + assert metadata.block_strides == [128, 128, 128] + assert metadata.region_members == [["a"], ["a.swa"], ["b"]] + assert metadata.region_num_blocks == [4, 4, 6] + assert metadata.region_group_ids == [-1, -1, 0] + assert metadata.region_names == ["a", "a.swa", "b"] + assert metadata.region_mem_types == ["VRAM", "VRAM", "VRAM"] + + +def test_layer_alignment_filters_and_reorders_a_pp_stage(): + worker = _layer_routing_worker([["l2"], ["l3"]], {"l2": 0, "l3": 1}) + metadata = _agent_metadata( + [["l2"], ["l0"], ["l3"], ["l1"]], + [0xC000, 0xA000, 0xD000, 0xB000], + [65536, 65536, 32768, 32768], + [131072, 131072, 65536, 65536], + ) + metadata.region_num_blocks = [4, 6, 8, 10] + metadata.region_group_ids = [0, 0, 1, 1] + metadata.region_names = ["l2", "l0", "l3", "l1"] + metadata.region_mem_types = ["VRAM"] * 4 + + worker._align_remote_regions_by_layer(metadata) + + assert worker._transfer_layer_region_indices == (0, 1) + assert worker._transfer_layer_group_ids == (0, 1) + assert metadata.kv_caches_base_addr == [0xC000, 0xD000] + assert metadata.block_lens == [65536, 32768] + assert metadata.block_strides == [131072, 65536] + assert metadata.region_num_blocks == [4, 8] + assert metadata.region_group_ids == [0, 1] + assert metadata.region_names == ["l2", "l3"] + assert metadata.region_mem_types == ["VRAM", "VRAM"] + + +@pytest.mark.parametrize( + ("local_num_blocks", "remote_num_blocks"), + [([4, 4], [4, 4, 4]), ([4, 6], [3, 5, 7])], +) +def test_layer_descriptors_pair_layers_across_asymmetric_pp_split( + local_num_blocks, remote_num_blocks +): + """A PP split can leave each stage a different mix of attention types. + + Stage 1 owns L3 (full) pooled with L4 (sliding) in region 0, and L5 (full) + alone in region 1. The consumer holds every layer and pools them + differently, so region indices cannot be paired positionally. + """ + worker = _layer_routing_worker([["L3", "L4"], ["L5"]], {"L3": 0, "L4": 1, "L5": 0}) + worker.block_len_per_layer = [128, 128] + worker.block_stride_per_layer = [256, 512] + worker._region_is_mla = [False, False] + worker.num_blocks = 4 + worker.region_num_blocks = local_num_blocks + worker.device_id = 0 + + consumer = _agent_metadata( + [["L0", "L1"], ["L3", "L2"], ["L5", "L4"]], + [0xA000, 0xB000, 0xC000], + [128, 128, 128], + [256, 512, 1024], + ) + consumer.num_blocks = 4 + consumer.region_num_blocks = remote_num_blocks + worker._align_remote_regions_by_layer(consumer) + + # L3 -> remote region 1, L4 -> 2, L5 -> 2. Pairing by index would have sent + # L5 to region 1 and L4 to region 2's sibling. + assert consumer.kv_caches_base_addr == [0xB000, 0xC000, 0xC000] + + plan = TPMapping(((0,), (0,)), (0,), {0: 0}, 0) + local_descs = worker._build_fa_local([0x1000, 0x2000], block_size_ratio=1) + remote_descs = worker._build_fa_remote(plan, consumer, block_size_ratio=1) + local_counts = [local_num_blocks[i] for i in worker._transfer_layer_region_indices] + local_ids = worker._compute_desc_ids( + [[1, 2], [3]], 4, None, 1, region_num_blocks=local_counts + ) + remote_ids = worker._compute_desc_ids( + [[0, 1], [2]], 4, None, 1, region_num_blocks=consumer.region_num_blocks + ) + assert len(local_descs) == sum(local_counts) + assert len(remote_descs) == sum(consumer.region_num_blocks) + + # Each pair addresses the same layer's blocks despite different pooling + # and strides. L3 and L5 use group 0; L4 uses group 1. + assert local_descs[local_ids].tolist() == [ + [0x1100, 128, 0], + [0x1200, 128, 0], + [0x1300, 128, 0], + [0x2200, 128, 0], + [0x2400, 128, 0], + ] + assert remote_descs[remote_ids].tolist() == [ + [0xB000, 128, 7], + [0xB200, 128, 7], + [0xC800, 128, 7], + [0xC000, 128, 7], + [0xC400, 128, 7], + ] + + +def test_layer_alignment_is_canonical_across_remote_orderings(): + local, groups = [["x"], ["y"]], {"x": 0, "y": 1} + rank0 = _agent_metadata([["x"], ["y"]], [0x1000, 0x2000], [64, 128], [256, 512]) + rank1 = _agent_metadata([["y"], ["x"]], [0x2000, 0x1000], [128, 64], [512, 256]) + + _layer_routing_worker(local, groups)._align_remote_regions_by_layer(rank0) + _layer_routing_worker(local, groups)._align_remote_regions_by_layer(rank1) + + assert rank0.kv_caches_base_addr == rank1.kv_caches_base_addr == [0x1000, 0x2000] + assert rank0.block_lens == rank1.block_lens == [64, 128] + assert rank0.block_strides == rank1.block_strides == [256, 512] + + +def test_layer_alignment_is_idempotent(): + worker = _layer_routing_worker( + [["a", "a.swa"], ["b"]], {"a": 0, "a.swa": 1, "b": 0} + ) + metadata = _agent_metadata([["a", "a.swa"], ["b"]], [0xA000, 0xB000], [128, 128]) + metadata.region_num_blocks = [4, 6] + metadata.region_group_ids = [-1, 0] + metadata.region_names = ["a", "b"] + metadata.region_mem_types = ["VRAM", "VRAM"] + + worker._align_remote_regions_by_layer(metadata) + aligned = msgspec.msgpack.encode(metadata) + worker._align_remote_regions_by_layer(metadata) + + assert msgspec.msgpack.encode(metadata) == aligned + + +def test_layer_alignment_rejects_missing_local_layer(): + worker = _layer_routing_worker([["l0"], ["l1"]], {"l0": 0, "l1": 1}) + metadata = _agent_metadata([["l0"]], [0xA000], [128]) + + with pytest.raises(AssertionError, match="missing locally owned layers"): + worker._align_remote_regions_by_layer(metadata) + + +def test_layer_alignment_rejects_duplicate_remote_layer(): + worker = _layer_routing_worker([["a"]], {"a": 0}) + metadata = _agent_metadata([["a"], ["a"]], [0xA000, 0xB000], [128, 128]) + + with pytest.raises(AssertionError, match="in multiple regions"): + worker._align_remote_regions_by_layer(metadata) + + +def test_layer_alignment_rejects_inconsistent_remote_metadata(): + worker = _layer_routing_worker([["a"]], {"a": 0}) + metadata = _agent_metadata([["a"], ["b"]], [0xA000], [128]) + + with pytest.raises(AssertionError, match="lengths disagree"): + worker._align_remote_regions_by_layer(metadata) + + +@pytest.mark.parametrize( + "field", + ["region_num_blocks", "region_group_ids", "region_names", "region_mem_types"], +) +def test_layer_alignment_rejects_inconsistent_region_geometry(field): + worker = _layer_routing_worker([["a"]], {"a": 0}) + metadata = _agent_metadata([["a"], ["b"]], [0xA000, 0xB000], [128, 128]) + setattr(metadata, field, []) + original = msgspec.msgpack.encode(metadata) + + with pytest.raises(AssertionError, match="lengths disagree"): + worker._align_remote_regions_by_layer(metadata) + assert msgspec.msgpack.encode(metadata) == original + + +def test_set_region_layers_rejects_duplicate_local_layer(): + with pytest.raises(AssertionError, match="spans multiple NIXL regions"): + _layer_routing_worker([["a"], ["a"]], {"a": 0}) + + +def test_set_region_layers_rejects_layer_outside_any_kv_group(): + with pytest.raises(AssertionError, match="outside any local group"): + _layer_routing_worker([["a"], ["b"]], {"a": 0}) + + +@pytest.mark.parametrize( + ("local_block_size", "remote_block_size", "remote_tp_size", "error"), + [ + (32, 16, 1, "identical P/D block sizes"), + (16, 32, 1, "identical P/D block sizes"), + (16, 16, 2, "decode TP greater"), + ], +) +def test_layer_handshake_rejects_unsupported_geometry( + local_block_size: int, remote_block_size: int, remote_tp_size: int, error: str +): + """Reject unsupported peers without registering agents or transfer state.""" + metadata = _agent_metadata([["a"]], [0xA000], [128]) + metadata.block_size = remote_block_size + worker = _layer_routing_worker([["a"]], {"a": 0}) + worker.block_size = local_block_size + worker.block_len_per_layer = [128] + worker.use_mla = False + worker.nixl_wrapper = MagicMock() + worker.kv_caches_base_addr = defaultdict(dict) + worker.dst_num_blocks = {} + worker.tp_mappings = {} + worker.transfer_topo = TransferTopology( + tp_rank=0, + tp_size=1, + block_size=local_block_size, + engine_id=worker.engine_id, + is_mla=False, + is_mamba=False, + total_num_kv_heads=8, + attn_backends=[], + ) + + with pytest.raises(NotImplementedError, match=error): + worker.add_remote_agent(metadata, remote_tp_size=remote_tp_size) + worker.nixl_wrapper.add_remote_agent.assert_not_called() + worker.nixl_wrapper.prep_xfer_dlist.assert_not_called() + assert not worker.tp_mappings + assert not worker.dst_num_blocks + assert not worker.kv_caches_base_addr + with pytest.raises(KeyError): + worker.transfer_topo.get_engine_info(metadata.engine_id) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index 6f8f1e55dc..2704de589f 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -164,6 +164,10 @@ class NixlBaseConnectorWorker: # Overridden by NixlPushConnectorWorker. _TRANSFER_MODE: str = "pull" + # Layer-name routing is supported only by NixlPushConnector for HMA KV + # caches under PP. + _supports_pp_hma = False + def _compute_desc_ids( self, block_ids: BlockIds, @@ -191,28 +195,42 @@ class NixlBaseConnectorWorker: num_blocks = dst_num_blocks if block_size_ratio is not None: num_blocks = int(num_blocks * block_size_ratio) - num_regions = self.num_regions or len(self.block_len_per_layer) + use_layer_name_routing = bool(self._transfer_layer_group_ids) + num_desc_regions = ( + len(self._transfer_layer_group_ids) + if use_layer_name_routing + else self.num_regions or len(self.block_len_per_layer) + ) if region_num_blocks is None: - region_num_blocks = [num_blocks] * num_regions + region_num_blocks = [num_blocks] * num_desc_regions elif block_size_ratio is not None: region_num_blocks = [ int(count * block_size_ratio) for count in region_num_blocks ] - if region_num_blocks: - num_regions = len(region_num_blocks) - assert len(region_num_blocks) == num_regions + if region_num_blocks and not use_layer_name_routing: + num_desc_regions = len(region_num_blocks) + assert len(region_num_blocks) == num_desc_regions region_offsets = np.cumsum([0, *region_num_blocks[:-1]]) num_fa_descs = sum(region_num_blocks) + if use_layer_name_routing: + # Descriptors in layer order, using each layer's KV-cache group. + return np.concatenate( + [ + np.asarray(block_ids[g], dtype=np.int32) + region_offsets[k] + for k, g in enumerate(self._transfer_layer_group_ids) + ] + ) + # All-attention fast path. if num_ssm_regions == 0: if region_group_ids is None: region_group_ids = self.region_group_ids if uses_region_group_mapping is None: uses_region_group_mapping = self._uses_region_group_mapping - if not region_group_ids and num_regions == 1: + if not region_group_ids and num_desc_regions == 1: region_group_ids = [0] - assert len(region_group_ids) == num_regions + assert len(region_group_ids) == num_desc_regions if uses_region_group_mapping is None: uses_region_group_mapping = len(set(region_group_ids)) > 1 region_group_ids_array = np.asarray(region_group_ids, dtype=np.int32) @@ -392,6 +410,103 @@ class NixlBaseConnectorWorker: """ return region_idx < len(self._region_is_mla) and self._region_is_mla[region_idx] + def _set_region_layers(self, region_layers: list[list[str]]) -> None: + layer_names = [layer_name for region in region_layers for layer_name in region] + assert len(layer_names) == len(set(layer_names)), ( + "A KV cache layer spans multiple NIXL regions" + ) + self.region_members: list[list[str]] = region_layers + if not self._requires_layer_name_routing(): + return + group_by_layer = self.kv_cache_config.transfer_group_index_by_layer + ungrouped = [name for name in layer_names if name not in group_by_layer] + assert not ungrouped, f"KV cache layers outside any local group: {ungrouped}" + self._transfer_layer_names = tuple(layer_names) + self._transfer_layer_region_indices = tuple( + region_idx + for region_idx, region in enumerate(region_layers) + for _ in region + ) + self._transfer_layer_group_ids = tuple( + group_by_layer[name] for name in layer_names + ) + + def _requires_layer_name_routing(self) -> bool: + """Whether PP push must match HMA layers by name, not region index.""" + return ( + self._supports_pp_hma + and self.pp_size > 1 + and self._is_hma_required + and not self._has_mamba + ) + + def _align_remote_regions_by_layer( + self, nixl_agent_meta: NixlAgentMetadata + ) -> None: + """Select remote regions for this PP stage in local layer order.""" + remote_region_layers = nixl_agent_meta.region_members + # Region-index routing here would silently transfer stale KV. + assert remote_region_layers, "Remote advertised no region_members" + assert ( + len(nixl_agent_meta.kv_caches_base_addr) + == len(nixl_agent_meta.block_lens) + == len(nixl_agent_meta.block_strides) + == len(remote_region_layers) + ), "Remote region metadata lengths disagree" + assert all( + values is None or len(values) == len(remote_region_layers) + for values in ( + nixl_agent_meta.region_num_blocks, + nixl_agent_meta.region_group_ids, + nixl_agent_meta.region_names, + nixl_agent_meta.region_mem_types, + ) + ), "Remote region metadata lengths disagree" + + remote_region_by_layer: dict[str, int] = {} + for region_idx, layer_names in enumerate(remote_region_layers): + for layer_name in layer_names: + assert layer_name not in remote_region_by_layer, ( + f"Remote advertised layer {layer_name!r} in multiple regions" + ) + remote_region_by_layer[layer_name] = region_idx + + missing = [ + name + for name in self._transfer_layer_names + if name not in remote_region_by_layer + ] + assert not missing, f"Remote is missing locally owned layers: {missing}" + + remote_regions = [ + remote_region_by_layer[name] for name in self._transfer_layer_names + ] + nixl_agent_meta.kv_caches_base_addr = [ + nixl_agent_meta.kv_caches_base_addr[i] for i in remote_regions + ] + nixl_agent_meta.block_lens = [ + nixl_agent_meta.block_lens[i] for i in remote_regions + ] + nixl_agent_meta.block_strides = [ + nixl_agent_meta.block_strides[i] for i in remote_regions + ] + if nixl_agent_meta.region_num_blocks is not None: + nixl_agent_meta.region_num_blocks = [ + nixl_agent_meta.region_num_blocks[i] for i in remote_regions + ] + if nixl_agent_meta.region_group_ids is not None: + nixl_agent_meta.region_group_ids = [ + nixl_agent_meta.region_group_ids[i] for i in remote_regions + ] + if nixl_agent_meta.region_mem_types is not None: + nixl_agent_meta.region_mem_types = [ + nixl_agent_meta.region_mem_types[i] for i in remote_regions + ] + if nixl_agent_meta.region_names is not None: + nixl_agent_meta.region_names = list(self._transfer_layer_names) + # One layer per region keeps a second alignment pass a no-op. + nixl_agent_meta.region_members = [[name] for name in self._transfer_layer_names] + def __init__( self, vllm_config: "VllmConfig", @@ -638,18 +753,26 @@ class NixlBaseConnectorWorker: # transfers into the matching sub-range of a PP=1 remote's regions. self.pp_size = vllm_config.parallel_config.pipeline_parallel_size self._remote_region_offset = 0 - # PP push slices regions per layer (uniform count); HMA breaks that. - if self.pp_size > 1 and self._is_hma_required: - raise NotImplementedError( - "NixlPushConnector does not support pipeline_parallel_size > 1 " - "with hybrid KV cache layouts (HMA) yet." - ) - # Decode-side PP is unsupported (completions counted per consumer rank). - if vllm_config.kv_transfer_config.kv_role == "kv_consumer" and self.pp_size > 1: - raise NotImplementedError( - "NixlPushConnector consumer (decode) does not support " - "pipeline_parallel_size > 1." - ) + if self.pp_size > 1: + if self._is_hma_required and not self._supports_pp_hma: + raise NotImplementedError( + "NixlConnector (pull) does not support pipeline_parallel_size " + "> 1 with hybrid KV cache layouts (HMA); use NixlPushConnector " + "for PP + HMA." + ) + # PP push routes HMA (hybrid) attention layouts by layer name; + # Mamba/SSM hybrids are not yet supported under PP. + if self._has_mamba: + raise NotImplementedError( + "NixlPushConnector does not support pipeline_parallel_size > 1 " + "with Mamba/SSM hybrid KV cache layouts yet." + ) + # Decode-side PP is unsupported (completions counted per consumer rank). + if vllm_config.kv_transfer_config.kv_role == "kv_consumer": + raise NotImplementedError( + "NixlPushConnector consumer (decode) does not support " + "pipeline_parallel_size > 1." + ) # Keep heartbeat handshakes to a PP-sharded producer notif-only. self._hb_handshake_notif_only = False @@ -773,6 +896,13 @@ class NixlBaseConnectorWorker: # within a block (BLHNC/BHLNC), where stride > block_len. self.block_stride_per_layer = list[int]() + # Region layer membership is populated for HMA layouts. + self.region_members = [] + # Local layer order, set by ``_set_region_layers`` only when + # routing by layer name. Empty means region-index routing. + self._transfer_layer_names = tuple[str, ...]() + self._transfer_layer_region_indices = tuple[int, ...]() + self._transfer_layer_group_ids = tuple[int, ...]() # Per-engine TP mappings. Generated during handshake. self.tp_mappings: dict[EngineId, TPMapping] = {} @@ -1306,6 +1436,11 @@ class NixlBaseConnectorWorker: ): compressed_region_owners.setdefault(cache.data_ptr(), cache) + track_region_layers = ( + self._supports_pp_hma and self._is_hma_required and not self._has_mamba + ) + region_layers: list[list[str]] = [] + # K and V are packed into the content dim, so each attention layer is a # single NIXL region whose block transfers as one unit. Mamba layers instead # register separate conv/ssm sub-regions (see `_build_mamba_local`). @@ -1508,6 +1643,12 @@ class NixlBaseConnectorWorker: self._region_is_mla.append(is_mla_region) region_mem_types.append(mem_type) + if track_region_layers: + if region_index == len(region_layers): + region_layers.append([layer_name]) + else: + region_layers[region_index].append(layer_name) + if isinstance(layer_spec, MambaSpec): if layer_spec.tp_replicated: assert self._is_csa_linear @@ -1563,7 +1704,9 @@ class NixlBaseConnectorWorker: if self._has_mamba: self.region_num_blocks = [self.num_blocks] * self.num_regions - if self.pp_size > 1: + self._set_region_layers(region_layers) + + if self.pp_size > 1 and not self._is_hma_required: start_layer, end_layer = self.model_config.get_layers_start_end_indices( self.vllm_config.parallel_config ) @@ -1573,7 +1716,12 @@ class NixlBaseConnectorWorker: self._remote_region_offset = regions_per_layer * start_layer # Total local FA descriptors (boundary between FA and mamba descs). - self.num_descs = sum(self.region_num_blocks) + xfer_region_num_blocks = ( + [self.region_num_blocks[i] for i in self._transfer_layer_region_indices] + if self._transfer_layer_region_indices + else self.region_num_blocks + ) + self.num_descs = sum(xfer_region_num_blocks) self._mixed_mem_types = len(set(region_mem_types)) > 1 if self._mixed_mem_types: @@ -1597,7 +1745,7 @@ class NixlBaseConnectorWorker: self.device_kv_caches = kv_caches self.dst_num_blocks[self.engine_id] = self.num_blocks - self.dst_region_num_blocks[self.engine_id] = self.region_num_blocks + self.dst_region_num_blocks[self.engine_id] = xfer_region_num_blocks self.dst_region_group_ids[self.engine_id] = self.region_group_ids self.dst_uses_region_group_mapping[self.engine_id] = ( self._uses_region_group_mapping @@ -1647,6 +1795,7 @@ class NixlBaseConnectorWorker: region_mem_types=self.region_mem_types, dcp_size=self.dcp_size, pcp_size=self.pcp_size, + region_members=self.region_members, ) # Wrap metadata in payload with hash for defensive decoding assert self.compat_hash is not None @@ -1805,12 +1954,16 @@ class NixlBaseConnectorWorker: base_addresses: list[int], block_size_ratio: int, ) -> np.ndarray: - """Build local FA descriptors for all layers as an Nx3 uint64 array.""" + """Build local FA descriptors as an Nx3 uint64 array.""" assert self.transfer_topo is not None assert base_addresses, "Local KV cache base addresses must not be empty." device_id = self.device_id + region_indices = self._transfer_layer_region_indices or range( + len(base_addresses) + ) parts: list[np.ndarray] = [] - for i, base_addr in enumerate(base_addresses): + for i in region_indices: + base_addr = base_addresses[i] block_len = self.block_len_per_layer[i] // block_size_ratio logical_blocks = np.repeat( np.arange(self.region_num_blocks[i], dtype=np.uint64), @@ -1834,7 +1987,7 @@ class NixlBaseConnectorWorker: nixl_agent_meta: NixlAgentMetadata, block_size_ratio: int, ) -> np.ndarray: - """Build remote FA descriptors for all layers as an Nx3 uint64 array.""" + """Build remote FA descriptors as an Nx3 uint64 array.""" assert self.transfer_topo is not None assert nixl_agent_meta.kv_caches_base_addr, ( "Remote KV cache base addresses must not be empty." @@ -1851,10 +2004,14 @@ class NixlBaseConnectorWorker: device_id = nixl_agent_meta.device_id block_strides = nixl_agent_meta.block_strides parts: list[np.ndarray] = [] + use_layer_name_routing = bool(self._transfer_layer_region_indices) for i, base_addr in enumerate(nixl_agent_meta.kv_caches_base_addr): - replicated = self._is_region_replicated(i) + local_region = ( + self._transfer_layer_region_indices[i] if use_layer_name_routing else i + ) + replicated = self._is_region_replicated(local_region) # Read our whole local region size from remote.. - local_block_len = self.block_len_per_layer[i] + local_block_len = self.block_len_per_layer[local_region] remote_kv_block_len = local_block_len // block_size_ratio if block_size_ratio > 1: # ..using remote kv_block_len as transfer unit @@ -2021,9 +2178,22 @@ class NixlBaseConnectorWorker: ) return self._remote_agents[engine_id][(0, remote_tp_rank)] + assert self.transfer_topo is not None + transfer_topo = self.transfer_topo # Number of physical regions registered locally (one per layer/tensor). num_local_regions = len(self.block_len_per_layer) - if ( + if self._transfer_layer_region_indices: + if self.block_size != nixl_agent_meta.block_size: + raise NotImplementedError( + "Attention-HMA push requires identical P/D block sizes." + ) + if remote_tp_size > transfer_topo.tp_size and not self.use_mla: + raise NotImplementedError( + "Attention-HMA push does not support decode TP greater than " + "prefill TP yet" + ) + self._align_remote_regions_by_layer(nixl_agent_meta) + elif ( self.pp_size > 1 and len(nixl_agent_meta.kv_caches_base_addr) > num_local_regions ): @@ -2038,6 +2208,10 @@ class NixlBaseConnectorWorker: ] nixl_agent_meta.block_lens = nixl_agent_meta.block_lens[start:end] nixl_agent_meta.block_strides = nixl_agent_meta.block_strides[start:end] + if nixl_agent_meta.region_num_blocks is not None: + nixl_agent_meta.region_num_blocks = nixl_agent_meta.region_num_blocks[ + start:end + ] if nixl_agent_meta.region_group_ids is not None: nixl_agent_meta.region_group_ids = nixl_agent_meta.region_group_ids[ start:end @@ -2048,14 +2222,8 @@ class NixlBaseConnectorWorker: nixl_agent_meta.region_mem_types = nixl_agent_meta.region_mem_types[ start:end ] - if nixl_agent_meta.region_num_blocks is not None: - nixl_agent_meta.region_num_blocks = nixl_agent_meta.region_num_blocks[ - start:end - ] ### Register remote engine in TransferTopology (idempotent). - assert self.transfer_topo is not None - transfer_topo = self.transfer_topo physical_blocks_per_logical = ( nixl_agent_meta.physical_blocks_per_logical_kv_block ) @@ -2349,6 +2517,9 @@ class NixlBaseConnectorWorker: # the per-rank KV head ratio rather than the raw tp_ratio, because GQA # replication caps per-rank heads at 1 when tp > total_kv_heads # (issue #45330). Mamba uses the ssm_sizes counterpart, so skip here. + local_regions = self._transfer_layer_region_indices or range( + len(self.block_len_per_layer) + ) if self._has_mamba and self.use_mla: # Hybrid MLA+SSM (e.g. KimiLinear's KDA+MLA): regions are # kernel-granularity views of the mamba-unified page. The MLA @@ -2365,7 +2536,7 @@ class NixlBaseConnectorWorker: f"remote={nixl_agent_meta.block_lens}." ) elif not self._has_mamba: - assert len(self.block_len_per_layer) == len(nixl_agent_meta.block_lens), ( + assert len(local_regions) == len(nixl_agent_meta.block_lens), ( "Number of KV layers must match between prefill and decode" ) model_replicated = self.use_mla or self.transfer_topo.is_kv_replicated( @@ -2374,8 +2545,9 @@ class NixlBaseConnectorWorker: total_kv_heads = self.transfer_topo.total_num_kv_heads local_heads = self.transfer_topo.local_physical_heads remote_heads = max(1, total_kv_heads // remote_tp_size) - for i, local_len in enumerate(self.block_len_per_layer): - replicated = model_replicated or self._is_region_replicated(i) + for i, region_idx in enumerate(local_regions): + local_len = self.block_len_per_layer[region_idx] + replicated = model_replicated or self._is_region_replicated(region_idx) remote_len = nixl_agent_meta.block_lens[i] if replicated: assert local_len // block_size_ratio == remote_len, ( @@ -2406,8 +2578,7 @@ class NixlBaseConnectorWorker: # TP workers that handhshake with same remote have same #blocks. assert self.dst_num_blocks[remote_engine_id] == nixl_agent_meta.num_blocks - # Same number of regions/~layers. - assert len(nixl_agent_meta.kv_caches_base_addr) == len(self.block_len_per_layer) + assert len(nixl_agent_meta.kv_caches_base_addr) == len(local_regions) num_remote_regions = len(nixl_agent_meta.kv_caches_base_addr) if nixl_agent_meta.region_num_blocks is not None: assert len(nixl_agent_meta.region_num_blocks) == num_remote_regions @@ -2423,7 +2594,12 @@ class NixlBaseConnectorWorker: if nixl_agent_meta.region_group_ids is not None: assert len(nixl_agent_meta.region_group_ids) == num_remote_regions if nixl_agent_meta.region_names is not None: - assert nixl_agent_meta.region_names == self.region_names, ( + expected_names = ( + list(self._transfer_layer_names) + if self._transfer_layer_names + else self.region_names + ) + assert nixl_agent_meta.region_names == expected_names, ( "NIXL transfer regions must name the same model caches in the " "same order" ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py index be32a6969d..c2b4d9dce7 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Metadata dataclasses and helpers for the NIXL connector.""" -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any from vllm.config import VllmConfig @@ -47,8 +47,9 @@ PUSH_REG_NOTIF_PREFIX = b"PUSH_REG:" # 9: Add block_strides # 10: Add dense virtual transfer pages for compressed MLA caches # 11: Add per-region transfer geometry and memory types to NixlAgentMetadata +# 12: Add per-region member names for PP push # -NIXL_CONNECTOR_VERSION: int = 11 +NIXL_CONNECTOR_VERSION: int = 12 @dataclass @@ -71,6 +72,8 @@ class NixlAgentMetadata: region_mem_types: list[str] | None = None dcp_size: int = 1 pcp_size: int = 1 + # Layer names sharing each advertised region, in region order. + region_members: list[list[str]] = field(default_factory=list) @dataclass diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py index 8b34e7ffbb..01c956a525 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py @@ -83,6 +83,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): # Distinguishes push from pull in the NIXL compatibility hash. _TRANSFER_MODE: str = "push" + _supports_pp_hma = True + def __init__( self, vllm_config: "VllmConfig", @@ -530,7 +532,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): local_region_groups = self.region_group_ids remote_region_groups = self.dst_region_group_ids[engine_id] groups_differ = local_region_groups != remote_region_groups - if groups_differ: + if groups_differ and not self._transfer_layer_group_ids: raise NotImplementedError( "NixlPushConnector does not support different producer and " "consumer cache-group layouts"