[KV Connector][NIXL] Register HiSparse host pool under single TP Rank (#60161)
Signed-off-by: Pravein Govindan Kannan <pravein.govindan.kannan@ibm.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Matthew Bonanni
Claude Opus 5.5
parent
4ed4c4433b
commit
daa9085143
@@ -2083,6 +2083,7 @@ def test_mixed_memory_local_descriptors_split_by_memory_type():
|
||||
worker.kv_caches_base_addr = {"local": {0: [100, 200]}}
|
||||
worker._has_mamba = False
|
||||
worker._mixed_mem_types = True
|
||||
worker._skip_dram_xfer = False
|
||||
worker.region_mem_types = ["DRAM", "VRAM"]
|
||||
worker.region_num_blocks = [2, 2]
|
||||
worker._desc_is_dram_by_block_size = {}
|
||||
@@ -2150,6 +2151,7 @@ def recv_worker():
|
||||
worker.enable_permute_local_kv = False
|
||||
worker.enable_heterogeneous_attn_post_process = False
|
||||
worker.tp_rank = 0
|
||||
worker._skip_dram_xfer = False
|
||||
worker._log_failure = MagicMock() # type: ignore[method-assign]
|
||||
worker.xfer_stats = NixlKVConnectorStats()
|
||||
worker.nixl_wrapper = MagicMock()
|
||||
@@ -2207,6 +2209,66 @@ def test_mixed_memory_read_failure_does_not_notify_producer(recv_worker):
|
||||
worker.nixl_wrapper.send_notif.assert_not_called()
|
||||
|
||||
|
||||
def test_hisparse_dram_rank0_only_read_keeps_device_part(recv_worker):
|
||||
worker = recv_worker
|
||||
worker.tp_rank = 1
|
||||
worker._skip_dram_xfer = True
|
||||
worker._desc_is_dram_by_block_size = {16: np.array([True, True, False, False])}
|
||||
worker._desc_pos_by_block_size = {16: np.array([0, 1, 0, 1])}
|
||||
worker._dram_src_handles_by_block_size = {16: None}
|
||||
worker.nixl_wrapper.make_prepped_xfer.side_effect = [102]
|
||||
worker.nixl_wrapper.check_xfer_state.return_value = "DONE"
|
||||
|
||||
worker._read_blocks_mixed(
|
||||
request_id="request",
|
||||
local_block_size_key=16,
|
||||
local_device_handle=20,
|
||||
local_dram_handle=None,
|
||||
remote_xfer_side_handle=30,
|
||||
local_block_descs_ids=np.array([0, 2]),
|
||||
remote_block_descs_ids=np.array([5, 7]),
|
||||
notif_agent="prefill",
|
||||
notif_id=b"request:1",
|
||||
)
|
||||
|
||||
(device_read,) = worker.nixl_wrapper.make_prepped_xfer.call_args_list
|
||||
assert device_read.args[:2] == ("READ", 20)
|
||||
np.testing.assert_array_equal(device_read.args[2], [0])
|
||||
np.testing.assert_array_equal(device_read.args[4], [7])
|
||||
|
||||
assert worker.get_finished() == (set(), {"request"})
|
||||
worker.nixl_wrapper.send_notif.assert_called_once_with(
|
||||
"prefill", notif_msg=b"request:1"
|
||||
)
|
||||
|
||||
|
||||
def test_hisparse_dram_rank0_only_dram_only_read_completes(recv_worker):
|
||||
"""With nothing left to read, the request still completes and notifies."""
|
||||
worker = recv_worker
|
||||
worker.tp_rank = 1
|
||||
worker._skip_dram_xfer = True
|
||||
worker._desc_is_dram_by_block_size = {16: np.array([True, True, False, False])}
|
||||
worker._desc_pos_by_block_size = {16: np.array([0, 1, 0, 1])}
|
||||
|
||||
worker._read_blocks_mixed(
|
||||
request_id="request",
|
||||
local_block_size_key=16,
|
||||
local_device_handle=20,
|
||||
local_dram_handle=None,
|
||||
remote_xfer_side_handle=30,
|
||||
local_block_descs_ids=np.array([0, 1]),
|
||||
remote_block_descs_ids=np.array([5, 6]),
|
||||
notif_agent="prefill",
|
||||
notif_id=b"request:1",
|
||||
)
|
||||
|
||||
worker.nixl_wrapper.make_prepped_xfer.assert_not_called()
|
||||
assert worker.get_finished() == (set(), {"request"})
|
||||
worker.nixl_wrapper.send_notif.assert_called_once_with(
|
||||
"prefill", notif_msg=b"request:1"
|
||||
)
|
||||
|
||||
|
||||
def element_byte_addrs(view: torch.Tensor) -> list[int]:
|
||||
"""Absolute byte addresses of every element of a (possibly strided) view."""
|
||||
offsets = torch.zeros(view.shape, dtype=torch.int64)
|
||||
|
||||
@@ -267,6 +267,7 @@ def test_region_pull_completion_zeros_only_own_padding(region_pull_worker):
|
||||
worker.region_num_blocks = [4, 4]
|
||||
worker.block_len_per_layer = [8, 8]
|
||||
worker.block_stride_per_layer = [24, 24]
|
||||
worker._skip_dram_xfer = False
|
||||
worker._recving_metadata = {
|
||||
"request": ReqMeta(
|
||||
local_block_ids=[[1, 2], [0, 1]],
|
||||
|
||||
Reference in New Issue
Block a user