[KV Connector][NIXL] Register HiSparse host pool under single TP Rank (#60161)

Signed-off-by: Pravein Govindan Kannan <pravein.govindan.kannan@ibm.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Pravein Govindan Kannan
2026-10-08 07:25:02 -07:00
committed by GitHub
co-authored by Matthew Bonanni Claude Opus 5.5
parent 4ed4c4433b
commit daa9085143
4 changed files with 128 additions and 18 deletions
@@ -2083,6 +2083,7 @@ def test_mixed_memory_local_descriptors_split_by_memory_type():
worker.kv_caches_base_addr = {"local": {0: [100, 200]}}
worker._has_mamba = False
worker._mixed_mem_types = True
worker._skip_dram_xfer = False
worker.region_mem_types = ["DRAM", "VRAM"]
worker.region_num_blocks = [2, 2]
worker._desc_is_dram_by_block_size = {}
@@ -2150,6 +2151,7 @@ def recv_worker():
worker.enable_permute_local_kv = False
worker.enable_heterogeneous_attn_post_process = False
worker.tp_rank = 0
worker._skip_dram_xfer = False
worker._log_failure = MagicMock() # type: ignore[method-assign]
worker.xfer_stats = NixlKVConnectorStats()
worker.nixl_wrapper = MagicMock()
@@ -2207,6 +2209,66 @@ def test_mixed_memory_read_failure_does_not_notify_producer(recv_worker):
worker.nixl_wrapper.send_notif.assert_not_called()
def test_hisparse_dram_rank0_only_read_keeps_device_part(recv_worker):
worker = recv_worker
worker.tp_rank = 1
worker._skip_dram_xfer = True
worker._desc_is_dram_by_block_size = {16: np.array([True, True, False, False])}
worker._desc_pos_by_block_size = {16: np.array([0, 1, 0, 1])}
worker._dram_src_handles_by_block_size = {16: None}
worker.nixl_wrapper.make_prepped_xfer.side_effect = [102]
worker.nixl_wrapper.check_xfer_state.return_value = "DONE"
worker._read_blocks_mixed(
request_id="request",
local_block_size_key=16,
local_device_handle=20,
local_dram_handle=None,
remote_xfer_side_handle=30,
local_block_descs_ids=np.array([0, 2]),
remote_block_descs_ids=np.array([5, 7]),
notif_agent="prefill",
notif_id=b"request:1",
)
(device_read,) = worker.nixl_wrapper.make_prepped_xfer.call_args_list
assert device_read.args[:2] == ("READ", 20)
np.testing.assert_array_equal(device_read.args[2], [0])
np.testing.assert_array_equal(device_read.args[4], [7])
assert worker.get_finished() == (set(), {"request"})
worker.nixl_wrapper.send_notif.assert_called_once_with(
"prefill", notif_msg=b"request:1"
)
def test_hisparse_dram_rank0_only_dram_only_read_completes(recv_worker):
"""With nothing left to read, the request still completes and notifies."""
worker = recv_worker
worker.tp_rank = 1
worker._skip_dram_xfer = True
worker._desc_is_dram_by_block_size = {16: np.array([True, True, False, False])}
worker._desc_pos_by_block_size = {16: np.array([0, 1, 0, 1])}
worker._read_blocks_mixed(
request_id="request",
local_block_size_key=16,
local_device_handle=20,
local_dram_handle=None,
remote_xfer_side_handle=30,
local_block_descs_ids=np.array([0, 1]),
remote_block_descs_ids=np.array([5, 6]),
notif_agent="prefill",
notif_id=b"request:1",
)
worker.nixl_wrapper.make_prepped_xfer.assert_not_called()
assert worker.get_finished() == (set(), {"request"})
worker.nixl_wrapper.send_notif.assert_called_once_with(
"prefill", notif_msg=b"request:1"
)
def element_byte_addrs(view: torch.Tensor) -> list[int]:
"""Absolute byte addresses of every element of a (possibly strided) view."""
offsets = torch.zeros(view.shape, dtype=torch.int64)
@@ -267,6 +267,7 @@ def test_region_pull_completion_zeros_only_own_padding(region_pull_worker):
worker.region_num_blocks = [4, 4]
worker.block_len_per_layer = [8, 8]
worker.block_stride_per_layer = [24, 24]
worker._skip_dram_xfer = False
worker._recving_metadata = {
"request": ReqMeta(
local_block_ids=[[1, 2], [0, 1]],
@@ -789,9 +789,12 @@ class NixlBaseConnectorWorker:
self.region_names: list[str] = []
self.region_num_blocks: list[int] = []
self._mixed_mem_types = False
# True on TP ranks that share a HiSparse host pool with TP rank 0 and
# leave its NIXL registration and DRAM reads to rank 0.
self._skip_dram_xfer = False
self._desc_is_dram_by_block_size: dict[int, np.ndarray] = {}
self._desc_pos_by_block_size: dict[int, np.ndarray] = {}
self._dram_src_handles_by_block_size: dict[int, int] = {}
self._dram_src_handles_by_block_size: dict[int, int | None] = {}
# PP>1 (push mode): this worker holds a contiguous layer slice and
# transfers into the matching sub-range of a PP=1 remote's regions.
@@ -828,7 +831,7 @@ class NixlBaseConnectorWorker:
# Populated dynamically during handshake based on remote configuration.
# Per-source split handles, keyed by (tp_ratio, remote_block_size).
self.src_xfer_handles_by_tp_ratio: dict[tuple[int, int], list[int]] = {}
self._dram_src_handles_by_tp_ratio: dict[tuple[int, int], list[int]] = {}
self._dram_src_handles_by_tp_ratio: dict[tuple[int, int], list[int | None]] = {}
# Map of engine_id -> {tp_rank: nixl_prepped_dlist_handle (int)}.
self.dst_xfer_side_handles = defaultdict[EngineId, dict[int, int]](dict)
@@ -1818,12 +1821,15 @@ class NixlBaseConnectorWorker:
self.num_descs = sum(xfer_region_num_blocks)
self._mixed_mem_types = len(set(region_mem_types)) > 1
self._skip_dram_xfer = self._leaves_dram_xfer_to_rank0()
if self._mixed_mem_types:
assert self.use_mla and not self._has_mamba, (
"Mixed-device KV registration is only supported for MLA "
"models without Mamba layers."
)
for mem_type in sorted(set(region_mem_types)):
if mem_type == "DRAM" and self._skip_dram_xfer:
continue
ranges_for_mem_type = [
(start, end - start, device_id, "")
for (_, cache_mem_type), (
@@ -2191,10 +2197,13 @@ class NixlBaseConnectorWorker:
blocks_data[desc_is_dram, 2] = 0
dram_blocks = blocks_data[dram_idx]
vram_blocks = blocks_data[vram_idx]
dram_descs = self.nixl_wrapper.get_xfer_descs(dram_blocks, "DRAM")
self._dram_src_handles_by_block_size[block_size] = (
self.nixl_wrapper.prep_xfer_dlist("NIXL_INIT_AGENT", dram_descs)
)
if self._skip_dram_xfer:
self._dram_src_handles_by_block_size[block_size] = None
else:
dram_descs = self.nixl_wrapper.get_xfer_descs(dram_blocks, "DRAM")
self._dram_src_handles_by_block_size[block_size] = (
self.nixl_wrapper.prep_xfer_dlist("NIXL_INIT_AGENT", dram_descs)
)
descs = self.nixl_wrapper.get_xfer_descs(vram_blocks, self.nixl_memory_type)
return (
self.nixl_wrapper.prep_xfer_dlist("NIXL_INIT_AGENT", descs),
@@ -2423,12 +2432,15 @@ class NixlBaseConnectorWorker:
if self._mixed_mem_types:
handle_data = np.asarray(handle_data, dtype=np.uint64)
desc_is_dram = self._desc_is_dram_by_block_size[remote_block_size]
dram_descs = self.nixl_wrapper.get_xfer_descs(
handle_data[desc_is_dram], "DRAM"
)
dram_handle = self.nixl_wrapper.prep_xfer_dlist(
"NIXL_INIT_AGENT", dram_descs
)
if self._skip_dram_xfer:
dram_handle = None
else:
dram_descs = self.nixl_wrapper.get_xfer_descs(
handle_data[desc_is_dram], "DRAM"
)
dram_handle = self.nixl_wrapper.prep_xfer_dlist(
"NIXL_INIT_AGENT", dram_descs
)
self._dram_src_handles_by_tp_ratio[split_key].append(dram_handle)
handle_data = handle_data[~desc_is_dram]
descs = self.nixl_wrapper.get_xfer_descs(
@@ -2887,6 +2899,27 @@ class NixlBaseConnectorWorker:
indices=indices,
)
def _leaves_dram_xfer_to_rank0(self) -> bool:
"""Whether this rank leaves the DRAM regions' NIXL work to TP rank 0.
A HiSparse host pool shared by the local TP ranks (one mmap per DP
rank) is registered with NIXL and filled by TP rank 0 alone: every
rank would otherwise register the same pages on every NIC and READ the
same host KV into them. Private pools are unaffected.
"""
if not (
self._mixed_mem_types and self.kv_cache_config.hisparse_shared_host_pool
):
return False
skip = self.tp_rank != 0
logger.info(
"HiSparse host pool: NIXL DRAM registration and reads on TP rank 0 "
"only (tp_rank=%d, skip=%s)",
self.tp_rank,
skip,
)
return skip
def _zero_region_blocks(self, block_ids: BlockIds) -> None:
"""Clear clipped physical pages in their owning region only."""
if not any(block_ids):
@@ -2895,6 +2928,9 @@ class NixlBaseConnectorWorker:
for region, blocks in enumerate(block_ids):
if not blocks:
continue
if self._skip_dram_xfer and self.region_mem_types[region] == "DRAM":
# The shared pool is rank 0's to write.
continue
cache = self.device_kv_caches[self.region_names[region]]
storage = cache.untyped_storage()
pages = torch.empty(0, dtype=torch.uint8, device=cache.device).set_(
@@ -3795,11 +3831,13 @@ class NixlBaseConnectorWorker:
for handles in self.src_xfer_handles_by_tp_ratio.values():
for handle in handles:
self.nixl_wrapper.release_dlist_handle(handle)
for handles in self._dram_src_handles_by_tp_ratio.values():
for handle in handles:
self.nixl_wrapper.release_dlist_handle(handle)
for handle in self._dram_src_handles_by_block_size.values():
self.nixl_wrapper.release_dlist_handle(handle)
for dram_handles in self._dram_src_handles_by_tp_ratio.values():
for dram_handle in dram_handles:
if dram_handle is not None:
self.nixl_wrapper.release_dlist_handle(dram_handle)
for dram_handle in self._dram_src_handles_by_block_size.values():
if dram_handle is not None:
self.nixl_wrapper.release_dlist_handle(dram_handle)
except Exception:
logger.exception("NIXL dlist-handle release failed at shutdown.")
self.src_xfer_handles_by_block_size.clear()
@@ -593,7 +593,16 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
remote_ids = np.asarray(remote_block_descs_ids)
is_dram = desc_is_dram[local_ids]
assert local_dram_handle is not None
if self._skip_dram_xfer:
# TP rank 0 reads the DRAM (shared host pool) part for every
# local rank; keep only the device part. Register the request
# even if nothing is left, so it still completes and notifies.
keep = ~is_dram
local_ids = local_ids[keep]
remote_ids = remote_ids[keep]
is_dram = is_dram[keep]
self._recving_transfers.setdefault(request_id, [])
assert local_dram_handle is not None or not is_dram.any()
reads = (
(is_dram, local_dram_handle),
(~is_dram, local_device_handle),