[NIXL] Prepare KV transfer descriptors as NIXL 1.5 strided lists (#61032)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Ilia Yastrebov <iyastrebov@nvidia.com>
This commit is contained in:
co-authored by
Ilia Yastrebov
parent
9e2b579aab
commit
eaf25f675a
@@ -83,6 +83,7 @@ from .utils import (
|
||||
create_request,
|
||||
create_scheduler,
|
||||
create_vllm_config,
|
||||
expand_strided_descs,
|
||||
make_kv_cache_config,
|
||||
maybe_update_block_size,
|
||||
)
|
||||
@@ -159,7 +160,9 @@ class FakeNixlWrapper:
|
||||
def get_xfer_descs(self, blocks_data, memory_type: str) -> list:
|
||||
return [str(uuid.uuid4()) for _ in blocks_data]
|
||||
|
||||
def prep_xfer_dlist(self, agent_name: str, descs: list) -> int:
|
||||
def prep_xfer_dlist(
|
||||
self, agent_name: str, descs, mem_type: str | None = None
|
||||
) -> int:
|
||||
return uuid.uuid4().int
|
||||
|
||||
def get_agent_metadata(self) -> bytes:
|
||||
@@ -554,7 +557,7 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
|
||||
self.kv_cache_layout = kv_cache_layout
|
||||
# Mock register_kv_caches attributes needed for tests that do not call it.
|
||||
self.src_xfer_handles_by_block_size = {self.block_size: 1}
|
||||
self.src_blocks_data = np.empty((0, 3), dtype=np.uint64)
|
||||
self.src_blocks_data = np.empty((0, 5), dtype=np.uint64)
|
||||
rep_spec = self.kv_cache_config.kv_cache_groups[0].kv_cache_spec
|
||||
test_shape = compute_layer_kv_cache_shape_bytes(rep_spec, 1)
|
||||
self.transfer_topo = TransferTopology(
|
||||
@@ -929,11 +932,12 @@ class TestNixlHandshake:
|
||||
worker.block_len_per_layer = [4096 * worker.block_size]
|
||||
worker.num_blocks = 1
|
||||
worker.dst_num_blocks[worker.engine_id] = worker.num_blocks
|
||||
block_len = worker.block_len_per_layer[0]
|
||||
worker.src_blocks_data = np.array(
|
||||
[(0, worker.block_len_per_layer[0], worker.tp_rank)],
|
||||
[(0, block_len, worker.tp_rank, block_len, worker.num_blocks)],
|
||||
dtype=np.uint64,
|
||||
)
|
||||
worker.num_descs = len(worker.src_blocks_data)
|
||||
worker.num_descs = int(worker.src_blocks_data[:, 4].sum())
|
||||
|
||||
def check_handshake(remote_tp_size: int):
|
||||
tp_ratio = remote_tp_size // local_tp_size
|
||||
@@ -1333,7 +1337,7 @@ class TestNixlHandshake:
|
||||
)
|
||||
|
||||
assert worker._build_fa_remote(plan, meta, block_size_ratio=1).tolist() == [
|
||||
[0x1000 + local_block_len, local_block_len, 0]
|
||||
[0x1000 + local_block_len, local_block_len, 0, remote_block_len, 1]
|
||||
]
|
||||
|
||||
@patch(
|
||||
@@ -1371,12 +1375,12 @@ class TestNixlHandshake:
|
||||
worker.dst_num_blocks[worker.engine_id] = worker.num_blocks
|
||||
worker.src_blocks_data = np.array(
|
||||
[
|
||||
(0, fa_len, worker.tp_rank),
|
||||
(0, idx_len, worker.tp_rank),
|
||||
(0, fa_len, worker.tp_rank, fa_len, worker.num_blocks),
|
||||
(0, idx_len, worker.tp_rank, idx_len, worker.num_blocks),
|
||||
],
|
||||
dtype=np.uint64,
|
||||
)
|
||||
worker.num_descs = len(worker.src_blocks_data)
|
||||
worker.num_descs = int(worker.src_blocks_data[:, 4].sum())
|
||||
|
||||
# D_TP=2, P_TP=1 -> tp_ratio=2. SPLIT region scales by tp_ratio;
|
||||
# REPLICATE region is unchanged.
|
||||
@@ -2142,21 +2146,11 @@ def test_mixed_memory_local_descriptors_split_by_memory_type():
|
||||
worker._desc_pos_by_block_size = {}
|
||||
worker._dram_src_handles_by_block_size = {}
|
||||
worker.nixl_memory_type = "VRAM"
|
||||
# One dense run of two 10-byte blocks per region: (addr, len, dev, stride, count).
|
||||
worker._build_fa_local = MagicMock( # type: ignore[method-assign]
|
||||
return_value=np.array(
|
||||
[
|
||||
[100, 10, 3],
|
||||
[110, 10, 3],
|
||||
[200, 10, 3],
|
||||
[210, 10, 3],
|
||||
]
|
||||
)
|
||||
return_value=np.array([[100, 10, 3, 10, 2], [200, 10, 3, 10, 2]])
|
||||
)
|
||||
worker.nixl_wrapper = MagicMock()
|
||||
worker.nixl_wrapper.get_xfer_descs.side_effect = lambda blocks, memory_type: (
|
||||
memory_type,
|
||||
blocks,
|
||||
)
|
||||
worker.nixl_wrapper.prep_xfer_dlist.side_effect = [11, 22]
|
||||
|
||||
handle, blocks = worker.register_local_xfer_handler(worker.block_size)
|
||||
@@ -2165,11 +2159,17 @@ def test_mixed_memory_local_descriptors_split_by_memory_type():
|
||||
assert worker._dram_src_handles_by_block_size[worker.block_size] == 11
|
||||
# Must stay an array: P_TP > D_TP splits call .tolist() on it.
|
||||
assert isinstance(blocks, np.ndarray)
|
||||
assert blocks[:, 2].tolist() == [0, 0, 3, 3]
|
||||
memory_types = [
|
||||
call.args[1] for call in worker.nixl_wrapper.get_xfer_descs.call_args_list
|
||||
# DRAM runs are re-homed to device 0; the per-descriptor maps consumed by
|
||||
# the mixed READ path still cover every block.
|
||||
assert blocks[:, 2].tolist() == [0, 3]
|
||||
assert worker._desc_is_dram_by_block_size[16].tolist() == [True, True, False, False]
|
||||
assert worker._desc_pos_by_block_size[16].tolist() == [0, 1, 0, 1]
|
||||
dlist_calls = worker.nixl_wrapper.prep_xfer_dlist.call_args_list
|
||||
assert [call.kwargs["mem_type"] for call in dlist_calls] == ["DRAM", "VRAM"]
|
||||
assert [call.args[1].tolist() for call in dlist_calls] == [
|
||||
[[100, 10, 0, 10, 2]],
|
||||
[[200, 10, 3, 10, 2]],
|
||||
]
|
||||
assert memory_types == ["DRAM", "VRAM"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -2362,8 +2362,8 @@ def test_register_kv_caches(
|
||||
This test verifies:
|
||||
1. nixl_wrapper.get_reg_descs() is called with caches_data containing
|
||||
tensor metadata
|
||||
2. nixl_wrapper.get_xfer_descs() is called with blocks_data containing
|
||||
block layout info
|
||||
2. nixl_wrapper.prep_xfer_dlist() is called with strided block runs whose
|
||||
expansion covers exactly the registered blocks
|
||||
"""
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(
|
||||
@@ -2479,9 +2479,12 @@ def test_register_kv_caches(
|
||||
assert size == raw.nbytes
|
||||
assert base_addr == raw.data_ptr()
|
||||
|
||||
# Verify get_xfer_descs was called with blocks_data
|
||||
assert mock_wrapper_instance.get_xfer_descs.called
|
||||
blocks_data, _ = mock_wrapper_instance.get_xfer_descs.call_args[0]
|
||||
# Verify the local dlist was prepared from one strided run per region;
|
||||
# expand it to per-block windows for the layout-blind checks below.
|
||||
assert mock_wrapper_instance.prep_xfer_dlist.called
|
||||
_agent, block_runs = mock_wrapper_instance.prep_xfer_dlist.call_args[0]
|
||||
assert block_runs.shape[1] == 5
|
||||
blocks_data = expand_strided_descs(block_runs)
|
||||
|
||||
# Layout-blind contract: whatever regions the worker carves out,
|
||||
# transferring "block b" must move exactly logical block b's bytes for
|
||||
@@ -2515,6 +2518,7 @@ def test_register_kv_caches(
|
||||
start for start, _len, _tp in blocks_data if owner[start] == 0
|
||||
}
|
||||
assert len(blocks_data) == num_blocks * len(base_addrs)
|
||||
assert len(block_runs) == len(base_addrs)
|
||||
|
||||
assert connector.connector_worker.block_size == block_size
|
||||
|
||||
@@ -2578,14 +2582,14 @@ def test_register_packed_dsv4_mla_cache_as_single_region(
|
||||
)
|
||||
connector.register_kv_caches(dict(zip(layer_names, views)))
|
||||
|
||||
blocks_data, _ = wrapper.get_xfer_descs.call_args[0]
|
||||
_agent, block_runs = wrapper.prep_xfer_dlist.call_args[0]
|
||||
packed_block_len = num_layers * spec.page_size_bytes
|
||||
assert connector.connector_worker.kv_caches_base_addr[
|
||||
connector.connector_worker.engine_id
|
||||
][0] == [raw.data_ptr()]
|
||||
assert blocks_data.tolist() == [
|
||||
[raw.data_ptr() + block_idx * packed_block_len, packed_block_len, 0]
|
||||
for block_idx in range(num_blocks)
|
||||
# One packed region -> one run of num_blocks packed rows.
|
||||
assert block_runs.tolist() == [
|
||||
[raw.data_ptr(), packed_block_len, 0, packed_block_len, num_blocks]
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
from .utils import (
|
||||
create_request,
|
||||
create_vllm_config,
|
||||
expand_strided_descs,
|
||||
make_kv_cache_config,
|
||||
make_nixl_scheduler,
|
||||
maybe_update_block_size,
|
||||
@@ -2201,7 +2202,8 @@ def test_nixl_keeps_device_block_count_with_hisparse_host_pool(kernel_block_size
|
||||
)
|
||||
for block in range(count)
|
||||
]
|
||||
assert worker.src_blocks_data[:, 0].tolist() == expected_addrs
|
||||
local_descs = expand_strided_descs(worker.src_blocks_data)
|
||||
assert local_descs[:, 0].tolist() == expected_addrs
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@@ -2284,19 +2286,21 @@ def test_register_kv_caches_hybrid_mla_dual_purpose_regions():
|
||||
assert worker.num_regions == 2 and worker.num_descs == 24
|
||||
# Kernel-granularity block lens; TP-independent for MLA hybrids.
|
||||
assert worker.block_len_per_layer == [unified_page // 3] * 2
|
||||
# Split handles must replicate every FA descriptor (MLA isn't head-sharded).
|
||||
assert worker._fa_desc_replicated(worker.num_descs) == [True] * 24
|
||||
# FA descs: 2 regions x 12 kernel blocks, page stride = kernel page.
|
||||
# Mamba descs: 2 regions x (3 conv sub-projections + 1 ssm) x 4 blocks.
|
||||
assert worker.src_blocks_data.shape == (24 + 32, 3)
|
||||
fa_descs = worker.src_blocks_data[:24]
|
||||
# FA runs: one per region x 12 kernel blocks, page stride = block_stride.
|
||||
# Mamba runs: 2 regions x (3 conv sub-projections + 1 ssm), 4 blocks each.
|
||||
assert worker.src_blocks_data.shape == (2 + 8, 5)
|
||||
fa_runs = worker.src_blocks_data[:2]
|
||||
assert fa_runs[:, 1].tolist() == [unified_page // 3] * 2
|
||||
assert fa_runs[:, 3].tolist() == [block_stride] * 2
|
||||
assert fa_runs[:, 4].tolist() == [12] * 2
|
||||
assert worker.src_blocks_data[2:, 4].tolist() == [4] * 8
|
||||
expected_addrs = [
|
||||
tensor.data_ptr() + block * block_stride
|
||||
for tensor in tensors
|
||||
for block in range(12)
|
||||
]
|
||||
fa_descs = expand_strided_descs(fa_runs)
|
||||
assert fa_descs[:, 0].tolist() == expected_addrs
|
||||
assert all(size == unified_page // 3 for size in fa_descs[:, 1])
|
||||
worker.nixl_wrapper.register_memory.assert_called_once()
|
||||
metadata = msgspec.msgpack.decode(
|
||||
worker.xfer_handshake_metadata.agent_metadata_bytes,
|
||||
|
||||
@@ -37,7 +37,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
create_kv_cache_views,
|
||||
)
|
||||
|
||||
from .utils import create_vllm_config
|
||||
from .utils import create_vllm_config, expand_strided_descs
|
||||
|
||||
|
||||
class _RecordingNixl:
|
||||
@@ -65,10 +65,11 @@ class _RecordingNixl:
|
||||
def get_xfer_descs(self, blocks_data, mem_type):
|
||||
return blocks_data
|
||||
|
||||
def prep_xfer_dlist(self, agent, descs):
|
||||
def prep_xfer_dlist(self, agent, descs, mem_type=None):
|
||||
handle = self._next_handle
|
||||
self._next_handle += 1
|
||||
self.dlists[handle] = np.asarray(descs, dtype=np.uint64).reshape(-1, 3)
|
||||
# Store per-block (addr, len, dev) rows so desc ids index directly.
|
||||
self.dlists[handle] = expand_strided_descs(np.asarray(descs, dtype=np.uint64))
|
||||
return handle
|
||||
|
||||
def add_remote_agent(self, metadata):
|
||||
@@ -186,9 +187,8 @@ def test_packed_mla_pp1_push_peer_transfers_whole_rows(num_layers):
|
||||
stride = num_layers * 128
|
||||
worker, raw = _make_packed_mla_view_worker(layouts, stride)
|
||||
assert worker._registered_descs == [[(raw.data_ptr(), raw.nbytes, 0, "")]]
|
||||
assert worker.src_blocks_data.tolist() == [
|
||||
[raw.data_ptr() + block * stride, stride, 0] for block in range(4)
|
||||
]
|
||||
# One packed region -> one run of 4 whole rows.
|
||||
assert worker.src_blocks_data.tolist() == [[raw.data_ptr(), stride, 0, stride, 4]]
|
||||
assert worker._transfer_layer_group_ids == ()
|
||||
metadata = msgspec.msgpack.decode(
|
||||
worker.xfer_handshake_metadata.agent_metadata_bytes, type=NixlAgentMetadata
|
||||
@@ -248,7 +248,8 @@ def test_packed_mla_pp_pairs_asymmetric_strides_and_overlapping_layers(
|
||||
1,
|
||||
region_num_blocks=producer.dst_region_num_blocks[producer.engine_id],
|
||||
)
|
||||
assert producer.src_blocks_data[local_ids].tolist() == [
|
||||
local_descs = expand_strided_descs(producer.src_blocks_data)
|
||||
assert local_descs[local_ids].tolist() == [
|
||||
[p_raw.data_ptr() + 192, 128, 0],
|
||||
[p_raw.data_ptr() + 2 * 192, 64, 0],
|
||||
[p_raw.data_ptr() + 128 + 3 * 192, 64, 0],
|
||||
@@ -318,7 +319,8 @@ def test_local_descriptors_follow_each_region_pool_capacity():
|
||||
|
||||
descriptors = worker._build_fa_local([100, 1000], block_size_ratio=1)
|
||||
|
||||
assert descriptors[:, 0].tolist() == [100, 116, 1000, 1016, 1032]
|
||||
# One (addr, len, dev, stride, count) run per region, sized to its pool.
|
||||
assert descriptors.tolist() == [[100, 16, 0, 16, 2], [1000, 16, 0, 16, 3]]
|
||||
|
||||
|
||||
def _register_overlaid_mla_worker(
|
||||
@@ -479,7 +481,8 @@ def test_overlaid_transfer_groups_share_region_geometry(push_pp):
|
||||
allocation.data_ptr() + block * block_stride for block in range(num_blocks)
|
||||
]
|
||||
num_desc_regions = 2 if push_pp else 1
|
||||
assert worker.src_blocks_data[:, 0].tolist() == expected_addrs * num_desc_regions
|
||||
local_descs = expand_strided_descs(worker.src_blocks_data)
|
||||
assert local_descs[:, 0].tolist() == expected_addrs * num_desc_regions
|
||||
assert worker.num_descs == num_blocks * num_desc_regions
|
||||
assert (
|
||||
worker.dst_region_num_blocks[worker.engine_id]
|
||||
@@ -500,11 +503,12 @@ def _descriptor_geometry(registered) -> dict:
|
||||
"""The geometry handed to NIXL, addressed relative to the allocation."""
|
||||
worker = registered.worker
|
||||
base = registered.allocation.data_ptr()
|
||||
descs = expand_strided_descs(worker.src_blocks_data)
|
||||
return {
|
||||
"block_len_per_layer": list(worker.block_len_per_layer),
|
||||
"block_stride_per_layer": list(worker.block_stride_per_layer),
|
||||
"desc_offsets": [addr - base for addr in worker.src_blocks_data[:, 0].tolist()],
|
||||
"desc_lens": worker.src_blocks_data[:, 1].tolist(),
|
||||
"desc_offsets": [addr - base for addr in descs[:, 0].tolist()],
|
||||
"desc_lens": descs[:, 1].tolist(),
|
||||
}
|
||||
|
||||
|
||||
@@ -1365,10 +1369,11 @@ def test_csa_linear_ple_descriptor_is_not_split():
|
||||
bases = [0x10000, 0x20000, 0x30000, 0x40000]
|
||||
attention = worker._build_fa_local(bases, block_size_ratio=1)
|
||||
mamba = worker._build_mamba_local(bases)
|
||||
assert attention.shape == (8, 3)
|
||||
assert mamba.shape == (10, 3)
|
||||
assert mamba[-2:, 0].tolist() == [0x10000, 0x10100]
|
||||
assert mamba[-2:, 1].tolist() == [256, 256]
|
||||
# One (addr, len, dev, stride, count) run per region / sub-region.
|
||||
assert attention.shape == (4, 5)
|
||||
assert mamba.shape == (5, 5)
|
||||
# PLE run: 2 whole 256-byte pages at 0x10000, 0x10100.
|
||||
assert mamba[-1].tolist() == [0x10000, 256, 0, 256, 2]
|
||||
|
||||
mapping = TPMapping(
|
||||
source_ranks_per_group=((0,), (0,), (0, 1), (0, 1)),
|
||||
@@ -1380,14 +1385,15 @@ def test_csa_linear_ple_descriptor_is_not_split():
|
||||
worker._build_local_splits_from_plan(
|
||||
mapping,
|
||||
np.concatenate([attention, mamba]),
|
||||
num_fa_descs=len(attention),
|
||||
num_fa_descs=int(attention[:, 4].sum()),
|
||||
)
|
||||
)
|
||||
assert len(splits) == 2
|
||||
assert [row[1] for row in splits[0][-2:]] == [256, 256]
|
||||
assert [row[1] for row in splits[1][-2:]] == [256, 256]
|
||||
assert splits[0][-3][1] * 2 == mamba[-3, 1]
|
||||
assert splits[1][-3][1] * 2 == mamba[-3, 1]
|
||||
for split in splits:
|
||||
# The PLE run is copied whole to every source rank, while the sharded
|
||||
# SSM run right before it is halved per source.
|
||||
assert split[-1].tolist() == mamba[-1].tolist()
|
||||
assert split[-2, 1] * 2 == mamba[-2, 1]
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@@ -1424,9 +1430,9 @@ def test_csa_linear_remote_ple_is_copied_whole():
|
||||
tp_ratio=-2,
|
||||
transfer_info=SimpleNamespace(remote_physical_blocks_per_logical=1),
|
||||
)
|
||||
assert descriptors.shape == (10, 3)
|
||||
assert descriptors[-2:, 0].tolist() == [0x10000, 0x10100]
|
||||
assert descriptors[-2:, 1].tolist() == [256, 256]
|
||||
assert descriptors.shape == (5, 5)
|
||||
# PLE run: 2 whole 256-byte pages at 0x10000, 0x10100.
|
||||
assert descriptors[-1].tolist() == [0x10000, 256, 0, 256, 2]
|
||||
|
||||
metadata.ple_block_len = 128
|
||||
with pytest.raises(ValueError, match="PLE pages require identical"):
|
||||
@@ -1451,8 +1457,8 @@ def test_csa_linear_ple_page_wider_than_the_shared_region_page():
|
||||
|
||||
bases = [0x10000, 0x20000, 0x30000, 0x40000]
|
||||
mamba = worker._build_mamba_local(bases)
|
||||
assert mamba[-2:, 0].tolist() == [0x10000, 0x10000 + 384]
|
||||
assert mamba[-2:, 1].tolist() == [384, 384]
|
||||
# PLE run: 2 whole 384-byte pages at 0x10000, 0x10000 + 384.
|
||||
assert mamba[-1].tolist() == [0x10000, 384, 0, 384, 2]
|
||||
|
||||
metadata = msgspec.msgpack.decode(
|
||||
worker.xfer_handshake_metadata.agent_metadata_bytes, type=NixlAgentMetadata
|
||||
|
||||
@@ -60,7 +60,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import (
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
from .utils import create_request, make_nixl_push_scheduler
|
||||
from .utils import create_request, expand_strided_descs, make_nixl_push_scheduler
|
||||
|
||||
# ----------------------------------------------------------------- #
|
||||
# Helpers / fakes #
|
||||
@@ -2008,8 +2008,13 @@ def test_layer_descriptors_pair_layers_across_asymmetric_pp_split(
|
||||
assert consumer.kv_caches_base_addr == [0xB000, 0xC000, 0xC000]
|
||||
|
||||
plan = TPMapping(((0,), (0,)), (0,), {0: 0}, 0)
|
||||
local_descs = worker._build_fa_local([0x1000, 0x2000], block_size_ratio=1)
|
||||
remote_descs = worker._build_fa_remote(plan, consumer, block_size_ratio=1)
|
||||
# Builders emit one strided run per region; expand to per-block descriptors.
|
||||
local_descs = expand_strided_descs(
|
||||
worker._build_fa_local([0x1000, 0x2000], block_size_ratio=1)
|
||||
)
|
||||
remote_descs = expand_strided_descs(
|
||||
worker._build_fa_remote(plan, consumer, block_size_ratio=1)
|
||||
)
|
||||
local_counts = [local_num_blocks[i] for i in worker._transfer_layer_region_indices]
|
||||
local_ids = worker._compute_desc_ids(
|
||||
[[1, 2], [3]], 4, None, 1, region_num_blocks=local_counts
|
||||
|
||||
@@ -75,7 +75,9 @@ class FakeNixlWrapper:
|
||||
def get_xfer_descs(self, blocks_data, memory_type: str) -> list:
|
||||
return [str(uuid.uuid4()) for _ in blocks_data]
|
||||
|
||||
def prep_xfer_dlist(self, agent_name: str, descs: list) -> int:
|
||||
def prep_xfer_dlist(
|
||||
self, agent_name: str, descs, mem_type: str | None = None
|
||||
) -> int:
|
||||
return uuid.uuid4().int
|
||||
|
||||
def get_agent_metadata(self) -> bytes:
|
||||
|
||||
@@ -145,9 +145,8 @@ def _make_mock_worker_for_splits(group_spec_types):
|
||||
"""Build a mock NixlConnectorWorker with _group_spec_types for split tests.
|
||||
|
||||
No per-region replicate flags are configured (``block_len_per_layer`` empty
|
||||
and ``num_regions == 0``), so ``_fa_desc_replicated`` takes its early-return
|
||||
path and treats every FA descriptor as SPLIT, matching the legacy behavior
|
||||
these tests assert.
|
||||
and ``num_regions == 0``), so ``_build_local_splits_from_plan`` treats every
|
||||
FA descriptor as SPLIT, matching the legacy behavior these tests assert.
|
||||
"""
|
||||
worker = object.__new__(NixlConnectorWorker)
|
||||
worker._group_spec_types = group_spec_types
|
||||
@@ -174,11 +173,9 @@ class TestBuildSrcSplitHandles:
|
||||
)
|
||||
|
||||
worker = _make_mock_worker_for_splits((FullAttentionSpec,))
|
||||
src_blocks_data = np.array(
|
||||
[(0x2000 + i * 1024, 1024, 0) for i in range(8)],
|
||||
dtype=np.uint64,
|
||||
)
|
||||
num_descs = len(src_blocks_data)
|
||||
# One region of 8 dense 1024-byte blocks as a single strided run.
|
||||
src_blocks_data = np.array([(0x2000, 1024, 0, 1024, 8)], dtype=np.uint64)
|
||||
num_descs = 8
|
||||
splits = list(
|
||||
worker._build_local_splits_from_plan(
|
||||
plan,
|
||||
@@ -189,9 +186,10 @@ class TestBuildSrcSplitHandles:
|
||||
|
||||
assert len(splits) == remote_tp_size
|
||||
for handle in splits:
|
||||
assert len(handle) == len(src_blocks_data)
|
||||
for _, length, _ in handle:
|
||||
assert length == 1024 // remote_tp_size
|
||||
assert handle.shape == src_blocks_data.shape
|
||||
# The head slice shrinks the length; block stride and count are kept.
|
||||
assert handle[:, 1].tolist() == [1024 // remote_tp_size]
|
||||
assert handle[:, 3:].tolist() == [[1024, 8]]
|
||||
|
||||
|
||||
class TestMambaPlanSplitHandles:
|
||||
@@ -209,12 +207,12 @@ class TestMambaPlanSplitHandles:
|
||||
)
|
||||
|
||||
worker = _make_mock_worker_for_splits((FullAttentionSpec, MambaSpec))
|
||||
# 2 FA descs + 1 SSM desc
|
||||
# 2 FA runs + 1 SSM run of one block each: (addr, len, dev, stride, count)
|
||||
src_blocks_data = np.array(
|
||||
[
|
||||
(1000, 200, 0), # FA desc 0
|
||||
(2000, 200, 0), # FA desc 1
|
||||
(3000, 400, 0), # SSM desc 0
|
||||
(1000, 200, 0, 200, 1), # FA desc 0
|
||||
(2000, 200, 0, 200, 1), # FA desc 1
|
||||
(3000, 400, 0, 400, 1), # SSM desc 0
|
||||
],
|
||||
dtype=np.uint64,
|
||||
)
|
||||
@@ -224,14 +222,22 @@ class TestMambaPlanSplitHandles:
|
||||
assert len(splits) == 2 # 2 source ranks
|
||||
|
||||
# Rank 0 (FA source, p_idx=0):
|
||||
# FA: chunk=200//1=200, slot=0 → (1000, 200, 0), (2000, 200, 0)
|
||||
# SSM: chunk=400//2=200, idx=0 → (3000, 200, 0)
|
||||
assert splits[0] == [(1000, 200, 0), (2000, 200, 0), (3000, 200, 0)]
|
||||
# FA: chunk=200//1=200, slot=0 → (1000, 200), (2000, 200)
|
||||
# SSM: chunk=400//2=200, idx=0 → (3000, 200); block stride stays 400
|
||||
assert splits[0].tolist() == [
|
||||
[1000, 200, 0, 200, 1],
|
||||
[2000, 200, 0, 200, 1],
|
||||
[3000, 200, 0, 400, 1],
|
||||
]
|
||||
|
||||
# Rank 1 (not FA source, p_idx=1):
|
||||
# FA: chunk=200//1=200, slot=0 (skip_fa) → (1000, 200, 0), (2000, 200, 0)
|
||||
# SSM: chunk=400//2=200, idx=1 → (3200, 200, 0)
|
||||
assert splits[1] == [(1000, 200, 0), (2000, 200, 0), (3200, 200, 0)]
|
||||
# FA: chunk=200//1=200, slot=0 (skip_fa) → (1000, 200), (2000, 200)
|
||||
# SSM: chunk=400//2=200, idx=1 → (3200, 200); block stride stays 400
|
||||
assert splits[1].tolist() == [
|
||||
[1000, 200, 0, 200, 1],
|
||||
[2000, 200, 0, 200, 1],
|
||||
[3200, 200, 0, 400, 1],
|
||||
]
|
||||
|
||||
def test_hetero_block_size_splits(self):
|
||||
"""With a block-size ratio, single-source FA sub-block descs pass
|
||||
@@ -244,14 +250,13 @@ class TestMambaPlanSplitHandles:
|
||||
)
|
||||
|
||||
worker = _make_mock_worker_for_splits((FullAttentionSpec, MambaSpec))
|
||||
# 2 FA blocks x ratio 2 sub-blocks + 1 SSM desc (never expanded).
|
||||
# 2 FA blocks x ratio 2 sub-blocks (one run of 2 per block) + 1 SSM desc
|
||||
# (never expanded).
|
||||
src_blocks_data = np.array(
|
||||
[
|
||||
(1000, 100, 0),
|
||||
(1100, 100, 0),
|
||||
(2000, 100, 0),
|
||||
(2100, 100, 0),
|
||||
(3000, 400, 0),
|
||||
(1000, 100, 0, 100, 2),
|
||||
(2000, 100, 0, 100, 2),
|
||||
(3000, 400, 0, 400, 1),
|
||||
],
|
||||
dtype=np.uint64,
|
||||
)
|
||||
@@ -259,14 +264,9 @@ class TestMambaPlanSplitHandles:
|
||||
splits = list(worker._build_local_splits_from_plan(plan, src_blocks_data, 4, 2))
|
||||
|
||||
assert len(splits) == 2
|
||||
fa_passthrough = [
|
||||
(1000, 100, 0),
|
||||
(1100, 100, 0),
|
||||
(2000, 100, 0),
|
||||
(2100, 100, 0),
|
||||
]
|
||||
assert splits[0] == fa_passthrough + [(3000, 200, 0)]
|
||||
assert splits[1] == fa_passthrough + [(3200, 200, 0)]
|
||||
fa_passthrough = [[1000, 100, 0, 100, 2], [2000, 100, 0, 100, 2]]
|
||||
assert splits[0].tolist() == fa_passthrough + [[3000, 200, 0, 400, 1]]
|
||||
assert splits[1].tolist() == fa_passthrough + [[3200, 200, 0, 400, 1]]
|
||||
|
||||
def test_hetero_block_size_head_sharded_asserts(self):
|
||||
"""Head-sharded FA reads (multiple FA sources) are incompatible with
|
||||
@@ -280,7 +280,7 @@ class TestMambaPlanSplitHandles:
|
||||
|
||||
worker = _make_mock_worker_for_splits((FullAttentionSpec, MambaSpec))
|
||||
src_blocks_data = np.array(
|
||||
[(1000, 100, 0), (1100, 100, 0), (3000, 400, 0)],
|
||||
[(1000, 100, 0, 100, 2), (3000, 400, 0, 400, 1)],
|
||||
dtype=np.uint64,
|
||||
)
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ from itertools import chain, count
|
||||
from typing import Any, Literal
|
||||
from unittest.mock import Mock
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm import SamplingParams
|
||||
@@ -682,3 +683,18 @@ def make_moriio_writer(fake_worker: Any) -> Any:
|
||||
writer._defer_timeout = 60.0
|
||||
writer.ensure_worker_started = lambda: None
|
||||
return writer
|
||||
|
||||
|
||||
def expand_strided_descs(strided_descs: np.ndarray) -> np.ndarray:
|
||||
"""Expand Nx5 (addr, len, dev, stride, count) rows to one row per block.
|
||||
|
||||
Mirrors how NIXL indexes a strided descriptor list, so desc ids computed
|
||||
by the connector can be resolved to (addr, len, dev) byte ranges.
|
||||
"""
|
||||
strided_descs = np.asarray(strided_descs, dtype=np.uint64).reshape(-1, 5)
|
||||
counts = strided_descs[:, 4].astype(np.intp)
|
||||
starts = np.cumsum(strided_descs[:, 4]) - strided_descs[:, 4]
|
||||
out = np.repeat(strided_descs[:, :3], counts, axis=0)
|
||||
offsets = np.arange(len(out), dtype=np.uint64) - np.repeat(starts, counts)
|
||||
out[:, 0] += offsets * np.repeat(strided_descs[:, 3], counts)
|
||||
return out
|
||||
|
||||
Reference in New Issue
Block a user