From daa9085143d65896ba3fa77635b0b110aab3e585 Mon Sep 17 00:00:00 2001 From: Pravein Govindan Kannan Date: Thu, 8 Oct 2026 19:55:02 +0530 Subject: [PATCH] [KV Connector][NIXL] Register HiSparse host pool under single TP Rank (#60161) Signed-off-by: Pravein Govindan Kannan Signed-off-by: Matthew Bonanni Co-authored-by: Matthew Bonanni Co-authored-by: Claude Opus 5.5 --- .../kv_connector/unit/test_nixl_connector.py | 62 ++++++++++++++++ .../unit/test_nixl_connector_hma.py | 1 + .../kv_connector/v1/nixl/base_worker.py | 72 ++++++++++++++----- .../kv_connector/v1/nixl/pull_worker.py | 11 ++- 4 files changed, 128 insertions(+), 18 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index 18097d220e..52ac746391 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -2083,6 +2083,7 @@ def test_mixed_memory_local_descriptors_split_by_memory_type(): worker.kv_caches_base_addr = {"local": {0: [100, 200]}} worker._has_mamba = False worker._mixed_mem_types = True + worker._skip_dram_xfer = False worker.region_mem_types = ["DRAM", "VRAM"] worker.region_num_blocks = [2, 2] worker._desc_is_dram_by_block_size = {} @@ -2150,6 +2151,7 @@ def recv_worker(): worker.enable_permute_local_kv = False worker.enable_heterogeneous_attn_post_process = False worker.tp_rank = 0 + worker._skip_dram_xfer = False worker._log_failure = MagicMock() # type: ignore[method-assign] worker.xfer_stats = NixlKVConnectorStats() worker.nixl_wrapper = MagicMock() @@ -2207,6 +2209,66 @@ def test_mixed_memory_read_failure_does_not_notify_producer(recv_worker): worker.nixl_wrapper.send_notif.assert_not_called() +def test_hisparse_dram_rank0_only_read_keeps_device_part(recv_worker): + worker = recv_worker + worker.tp_rank = 1 + worker._skip_dram_xfer = True + worker._desc_is_dram_by_block_size = {16: np.array([True, True, False, False])} + worker._desc_pos_by_block_size = {16: np.array([0, 1, 0, 1])} + worker._dram_src_handles_by_block_size = {16: None} + worker.nixl_wrapper.make_prepped_xfer.side_effect = [102] + worker.nixl_wrapper.check_xfer_state.return_value = "DONE" + + worker._read_blocks_mixed( + request_id="request", + local_block_size_key=16, + local_device_handle=20, + local_dram_handle=None, + remote_xfer_side_handle=30, + local_block_descs_ids=np.array([0, 2]), + remote_block_descs_ids=np.array([5, 7]), + notif_agent="prefill", + notif_id=b"request:1", + ) + + (device_read,) = worker.nixl_wrapper.make_prepped_xfer.call_args_list + assert device_read.args[:2] == ("READ", 20) + np.testing.assert_array_equal(device_read.args[2], [0]) + np.testing.assert_array_equal(device_read.args[4], [7]) + + assert worker.get_finished() == (set(), {"request"}) + worker.nixl_wrapper.send_notif.assert_called_once_with( + "prefill", notif_msg=b"request:1" + ) + + +def test_hisparse_dram_rank0_only_dram_only_read_completes(recv_worker): + """With nothing left to read, the request still completes and notifies.""" + worker = recv_worker + worker.tp_rank = 1 + worker._skip_dram_xfer = True + worker._desc_is_dram_by_block_size = {16: np.array([True, True, False, False])} + worker._desc_pos_by_block_size = {16: np.array([0, 1, 0, 1])} + + worker._read_blocks_mixed( + request_id="request", + local_block_size_key=16, + local_device_handle=20, + local_dram_handle=None, + remote_xfer_side_handle=30, + local_block_descs_ids=np.array([0, 1]), + remote_block_descs_ids=np.array([5, 6]), + notif_agent="prefill", + notif_id=b"request:1", + ) + + worker.nixl_wrapper.make_prepped_xfer.assert_not_called() + assert worker.get_finished() == (set(), {"request"}) + worker.nixl_wrapper.send_notif.assert_called_once_with( + "prefill", notif_msg=b"request:1" + ) + + def element_byte_addrs(view: torch.Tensor) -> list[int]: """Absolute byte addresses of every element of a (possibly strided) view.""" offsets = torch.zeros(view.shape, dtype=torch.int64) diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py index b4b5e06344..f016b26475 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py @@ -267,6 +267,7 @@ def test_region_pull_completion_zeros_only_own_padding(region_pull_worker): worker.region_num_blocks = [4, 4] worker.block_len_per_layer = [8, 8] worker.block_stride_per_layer = [24, 24] + worker._skip_dram_xfer = False worker._recving_metadata = { "request": ReqMeta( local_block_ids=[[1, 2], [0, 1]], diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index fa29d41a82..2a2f660cf1 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -789,9 +789,12 @@ class NixlBaseConnectorWorker: self.region_names: list[str] = [] self.region_num_blocks: list[int] = [] self._mixed_mem_types = False + # True on TP ranks that share a HiSparse host pool with TP rank 0 and + # leave its NIXL registration and DRAM reads to rank 0. + self._skip_dram_xfer = False self._desc_is_dram_by_block_size: dict[int, np.ndarray] = {} self._desc_pos_by_block_size: dict[int, np.ndarray] = {} - self._dram_src_handles_by_block_size: dict[int, int] = {} + self._dram_src_handles_by_block_size: dict[int, int | None] = {} # PP>1 (push mode): this worker holds a contiguous layer slice and # transfers into the matching sub-range of a PP=1 remote's regions. @@ -828,7 +831,7 @@ class NixlBaseConnectorWorker: # Populated dynamically during handshake based on remote configuration. # Per-source split handles, keyed by (tp_ratio, remote_block_size). self.src_xfer_handles_by_tp_ratio: dict[tuple[int, int], list[int]] = {} - self._dram_src_handles_by_tp_ratio: dict[tuple[int, int], list[int]] = {} + self._dram_src_handles_by_tp_ratio: dict[tuple[int, int], list[int | None]] = {} # Map of engine_id -> {tp_rank: nixl_prepped_dlist_handle (int)}. self.dst_xfer_side_handles = defaultdict[EngineId, dict[int, int]](dict) @@ -1818,12 +1821,15 @@ class NixlBaseConnectorWorker: self.num_descs = sum(xfer_region_num_blocks) self._mixed_mem_types = len(set(region_mem_types)) > 1 + self._skip_dram_xfer = self._leaves_dram_xfer_to_rank0() if self._mixed_mem_types: assert self.use_mla and not self._has_mamba, ( "Mixed-device KV registration is only supported for MLA " "models without Mamba layers." ) for mem_type in sorted(set(region_mem_types)): + if mem_type == "DRAM" and self._skip_dram_xfer: + continue ranges_for_mem_type = [ (start, end - start, device_id, "") for (_, cache_mem_type), ( @@ -2191,10 +2197,13 @@ class NixlBaseConnectorWorker: blocks_data[desc_is_dram, 2] = 0 dram_blocks = blocks_data[dram_idx] vram_blocks = blocks_data[vram_idx] - dram_descs = self.nixl_wrapper.get_xfer_descs(dram_blocks, "DRAM") - self._dram_src_handles_by_block_size[block_size] = ( - self.nixl_wrapper.prep_xfer_dlist("NIXL_INIT_AGENT", dram_descs) - ) + if self._skip_dram_xfer: + self._dram_src_handles_by_block_size[block_size] = None + else: + dram_descs = self.nixl_wrapper.get_xfer_descs(dram_blocks, "DRAM") + self._dram_src_handles_by_block_size[block_size] = ( + self.nixl_wrapper.prep_xfer_dlist("NIXL_INIT_AGENT", dram_descs) + ) descs = self.nixl_wrapper.get_xfer_descs(vram_blocks, self.nixl_memory_type) return ( self.nixl_wrapper.prep_xfer_dlist("NIXL_INIT_AGENT", descs), @@ -2423,12 +2432,15 @@ class NixlBaseConnectorWorker: if self._mixed_mem_types: handle_data = np.asarray(handle_data, dtype=np.uint64) desc_is_dram = self._desc_is_dram_by_block_size[remote_block_size] - dram_descs = self.nixl_wrapper.get_xfer_descs( - handle_data[desc_is_dram], "DRAM" - ) - dram_handle = self.nixl_wrapper.prep_xfer_dlist( - "NIXL_INIT_AGENT", dram_descs - ) + if self._skip_dram_xfer: + dram_handle = None + else: + dram_descs = self.nixl_wrapper.get_xfer_descs( + handle_data[desc_is_dram], "DRAM" + ) + dram_handle = self.nixl_wrapper.prep_xfer_dlist( + "NIXL_INIT_AGENT", dram_descs + ) self._dram_src_handles_by_tp_ratio[split_key].append(dram_handle) handle_data = handle_data[~desc_is_dram] descs = self.nixl_wrapper.get_xfer_descs( @@ -2887,6 +2899,27 @@ class NixlBaseConnectorWorker: indices=indices, ) + def _leaves_dram_xfer_to_rank0(self) -> bool: + """Whether this rank leaves the DRAM regions' NIXL work to TP rank 0. + + A HiSparse host pool shared by the local TP ranks (one mmap per DP + rank) is registered with NIXL and filled by TP rank 0 alone: every + rank would otherwise register the same pages on every NIC and READ the + same host KV into them. Private pools are unaffected. + """ + if not ( + self._mixed_mem_types and self.kv_cache_config.hisparse_shared_host_pool + ): + return False + skip = self.tp_rank != 0 + logger.info( + "HiSparse host pool: NIXL DRAM registration and reads on TP rank 0 " + "only (tp_rank=%d, skip=%s)", + self.tp_rank, + skip, + ) + return skip + def _zero_region_blocks(self, block_ids: BlockIds) -> None: """Clear clipped physical pages in their owning region only.""" if not any(block_ids): @@ -2895,6 +2928,9 @@ class NixlBaseConnectorWorker: for region, blocks in enumerate(block_ids): if not blocks: continue + if self._skip_dram_xfer and self.region_mem_types[region] == "DRAM": + # The shared pool is rank 0's to write. + continue cache = self.device_kv_caches[self.region_names[region]] storage = cache.untyped_storage() pages = torch.empty(0, dtype=torch.uint8, device=cache.device).set_( @@ -3795,11 +3831,13 @@ class NixlBaseConnectorWorker: for handles in self.src_xfer_handles_by_tp_ratio.values(): for handle in handles: self.nixl_wrapper.release_dlist_handle(handle) - for handles in self._dram_src_handles_by_tp_ratio.values(): - for handle in handles: - self.nixl_wrapper.release_dlist_handle(handle) - for handle in self._dram_src_handles_by_block_size.values(): - self.nixl_wrapper.release_dlist_handle(handle) + for dram_handles in self._dram_src_handles_by_tp_ratio.values(): + for dram_handle in dram_handles: + if dram_handle is not None: + self.nixl_wrapper.release_dlist_handle(dram_handle) + for dram_handle in self._dram_src_handles_by_block_size.values(): + if dram_handle is not None: + self.nixl_wrapper.release_dlist_handle(dram_handle) except Exception: logger.exception("NIXL dlist-handle release failed at shutdown.") self.src_xfer_handles_by_block_size.clear() diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py index 5bcfdb9560..16a25149ea 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py @@ -593,7 +593,16 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): remote_ids = np.asarray(remote_block_descs_ids) is_dram = desc_is_dram[local_ids] - assert local_dram_handle is not None + if self._skip_dram_xfer: + # TP rank 0 reads the DRAM (shared host pool) part for every + # local rank; keep only the device part. Register the request + # even if nothing is left, so it still completes and notifies. + keep = ~is_dram + local_ids = local_ids[keep] + remote_ids = remote_ids[keep] + is_dram = is_dram[keep] + self._recving_transfers.setdefault(request_id, []) + assert local_dram_handle is not None or not is_dram.any() reads = ( (is_dram, local_dram_handle), (~is_dram, local_device_handle),