[NIXL] Prepare KV transfer descriptors as NIXL 1.5 strided lists (#61032)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Ilia Yastrebov <iyastrebov@nvidia.com>
This commit is contained in:
Nick Hill
2026-10-14 18:07:24 +02:00
committed by GitHub
co-authored by Ilia Yastrebov
parent 9e2b579aab
commit eaf25f675a
11 changed files with 343 additions and 227 deletions
@@ -83,6 +83,7 @@ from .utils import (
create_request,
create_scheduler,
create_vllm_config,
expand_strided_descs,
make_kv_cache_config,
maybe_update_block_size,
)
@@ -159,7 +160,9 @@ class FakeNixlWrapper:
def get_xfer_descs(self, blocks_data, memory_type: str) -> list:
return [str(uuid.uuid4()) for _ in blocks_data]
def prep_xfer_dlist(self, agent_name: str, descs: list) -> int:
def prep_xfer_dlist(
self, agent_name: str, descs, mem_type: str | None = None
) -> int:
return uuid.uuid4().int
def get_agent_metadata(self) -> bytes:
@@ -554,7 +557,7 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
self.kv_cache_layout = kv_cache_layout
# Mock register_kv_caches attributes needed for tests that do not call it.
self.src_xfer_handles_by_block_size = {self.block_size: 1}
self.src_blocks_data = np.empty((0, 3), dtype=np.uint64)
self.src_blocks_data = np.empty((0, 5), dtype=np.uint64)
rep_spec = self.kv_cache_config.kv_cache_groups[0].kv_cache_spec
test_shape = compute_layer_kv_cache_shape_bytes(rep_spec, 1)
self.transfer_topo = TransferTopology(
@@ -929,11 +932,12 @@ class TestNixlHandshake:
worker.block_len_per_layer = [4096 * worker.block_size]
worker.num_blocks = 1
worker.dst_num_blocks[worker.engine_id] = worker.num_blocks
block_len = worker.block_len_per_layer[0]
worker.src_blocks_data = np.array(
[(0, worker.block_len_per_layer[0], worker.tp_rank)],
[(0, block_len, worker.tp_rank, block_len, worker.num_blocks)],
dtype=np.uint64,
)
worker.num_descs = len(worker.src_blocks_data)
worker.num_descs = int(worker.src_blocks_data[:, 4].sum())
def check_handshake(remote_tp_size: int):
tp_ratio = remote_tp_size // local_tp_size
@@ -1333,7 +1337,7 @@ class TestNixlHandshake:
)
assert worker._build_fa_remote(plan, meta, block_size_ratio=1).tolist() == [
[0x1000 + local_block_len, local_block_len, 0]
[0x1000 + local_block_len, local_block_len, 0, remote_block_len, 1]
]
@patch(
@@ -1371,12 +1375,12 @@ class TestNixlHandshake:
worker.dst_num_blocks[worker.engine_id] = worker.num_blocks
worker.src_blocks_data = np.array(
[
(0, fa_len, worker.tp_rank),
(0, idx_len, worker.tp_rank),
(0, fa_len, worker.tp_rank, fa_len, worker.num_blocks),
(0, idx_len, worker.tp_rank, idx_len, worker.num_blocks),
],
dtype=np.uint64,
)
worker.num_descs = len(worker.src_blocks_data)
worker.num_descs = int(worker.src_blocks_data[:, 4].sum())
# D_TP=2, P_TP=1 -> tp_ratio=2. SPLIT region scales by tp_ratio;
# REPLICATE region is unchanged.
@@ -2142,21 +2146,11 @@ def test_mixed_memory_local_descriptors_split_by_memory_type():
worker._desc_pos_by_block_size = {}
worker._dram_src_handles_by_block_size = {}
worker.nixl_memory_type = "VRAM"
# One dense run of two 10-byte blocks per region: (addr, len, dev, stride, count).
worker._build_fa_local = MagicMock( # type: ignore[method-assign]
return_value=np.array(
[
[100, 10, 3],
[110, 10, 3],
[200, 10, 3],
[210, 10, 3],
]
)
return_value=np.array([[100, 10, 3, 10, 2], [200, 10, 3, 10, 2]])
)
worker.nixl_wrapper = MagicMock()
worker.nixl_wrapper.get_xfer_descs.side_effect = lambda blocks, memory_type: (
memory_type,
blocks,
)
worker.nixl_wrapper.prep_xfer_dlist.side_effect = [11, 22]
handle, blocks = worker.register_local_xfer_handler(worker.block_size)
@@ -2165,11 +2159,17 @@ def test_mixed_memory_local_descriptors_split_by_memory_type():
assert worker._dram_src_handles_by_block_size[worker.block_size] == 11
# Must stay an array: P_TP > D_TP splits call .tolist() on it.
assert isinstance(blocks, np.ndarray)
assert blocks[:, 2].tolist() == [0, 0, 3, 3]
memory_types = [
call.args[1] for call in worker.nixl_wrapper.get_xfer_descs.call_args_list
# DRAM runs are re-homed to device 0; the per-descriptor maps consumed by
# the mixed READ path still cover every block.
assert blocks[:, 2].tolist() == [0, 3]
assert worker._desc_is_dram_by_block_size[16].tolist() == [True, True, False, False]
assert worker._desc_pos_by_block_size[16].tolist() == [0, 1, 0, 1]
dlist_calls = worker.nixl_wrapper.prep_xfer_dlist.call_args_list
assert [call.kwargs["mem_type"] for call in dlist_calls] == ["DRAM", "VRAM"]
assert [call.args[1].tolist() for call in dlist_calls] == [
[[100, 10, 0, 10, 2]],
[[200, 10, 3, 10, 2]],
]
assert memory_types == ["DRAM", "VRAM"]
@pytest.fixture
@@ -2362,8 +2362,8 @@ def test_register_kv_caches(
This test verifies:
1. nixl_wrapper.get_reg_descs() is called with caches_data containing
tensor metadata
2. nixl_wrapper.get_xfer_descs() is called with blocks_data containing
block layout info
2. nixl_wrapper.prep_xfer_dlist() is called with strided block runs whose
expansion covers exactly the registered blocks
"""
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(
@@ -2479,9 +2479,12 @@ def test_register_kv_caches(
assert size == raw.nbytes
assert base_addr == raw.data_ptr()
# Verify get_xfer_descs was called with blocks_data
assert mock_wrapper_instance.get_xfer_descs.called
blocks_data, _ = mock_wrapper_instance.get_xfer_descs.call_args[0]
# Verify the local dlist was prepared from one strided run per region;
# expand it to per-block windows for the layout-blind checks below.
assert mock_wrapper_instance.prep_xfer_dlist.called
_agent, block_runs = mock_wrapper_instance.prep_xfer_dlist.call_args[0]
assert block_runs.shape[1] == 5
blocks_data = expand_strided_descs(block_runs)
# Layout-blind contract: whatever regions the worker carves out,
# transferring "block b" must move exactly logical block b's bytes for
@@ -2515,6 +2518,7 @@ def test_register_kv_caches(
start for start, _len, _tp in blocks_data if owner[start] == 0
}
assert len(blocks_data) == num_blocks * len(base_addrs)
assert len(block_runs) == len(base_addrs)
assert connector.connector_worker.block_size == block_size
@@ -2578,14 +2582,14 @@ def test_register_packed_dsv4_mla_cache_as_single_region(
)
connector.register_kv_caches(dict(zip(layer_names, views)))
blocks_data, _ = wrapper.get_xfer_descs.call_args[0]
_agent, block_runs = wrapper.prep_xfer_dlist.call_args[0]
packed_block_len = num_layers * spec.page_size_bytes
assert connector.connector_worker.kv_caches_base_addr[
connector.connector_worker.engine_id
][0] == [raw.data_ptr()]
assert blocks_data.tolist() == [
[raw.data_ptr() + block_idx * packed_block_len, packed_block_len, 0]
for block_idx in range(num_blocks)
# One packed region -> one run of num_blocks packed rows.
assert block_runs.tolist() == [
[raw.data_ptr(), packed_block_len, 0, packed_block_len, num_blocks]
]
@@ -37,6 +37,7 @@ from vllm.v1.kv_cache_interface import (
from .utils import (
create_request,
create_vllm_config,
expand_strided_descs,
make_kv_cache_config,
make_nixl_scheduler,
maybe_update_block_size,
@@ -2201,7 +2202,8 @@ def test_nixl_keeps_device_block_count_with_hisparse_host_pool(kernel_block_size
)
for block in range(count)
]
assert worker.src_blocks_data[:, 0].tolist() == expected_addrs
local_descs = expand_strided_descs(worker.src_blocks_data)
assert local_descs[:, 0].tolist() == expected_addrs
@pytest.mark.cpu_test
@@ -2284,19 +2286,21 @@ def test_register_kv_caches_hybrid_mla_dual_purpose_regions():
assert worker.num_regions == 2 and worker.num_descs == 24
# Kernel-granularity block lens; TP-independent for MLA hybrids.
assert worker.block_len_per_layer == [unified_page // 3] * 2
# Split handles must replicate every FA descriptor (MLA isn't head-sharded).
assert worker._fa_desc_replicated(worker.num_descs) == [True] * 24
# FA descs: 2 regions x 12 kernel blocks, page stride = kernel page.
# Mamba descs: 2 regions x (3 conv sub-projections + 1 ssm) x 4 blocks.
assert worker.src_blocks_data.shape == (24 + 32, 3)
fa_descs = worker.src_blocks_data[:24]
# FA runs: one per region x 12 kernel blocks, page stride = block_stride.
# Mamba runs: 2 regions x (3 conv sub-projections + 1 ssm), 4 blocks each.
assert worker.src_blocks_data.shape == (2 + 8, 5)
fa_runs = worker.src_blocks_data[:2]
assert fa_runs[:, 1].tolist() == [unified_page // 3] * 2
assert fa_runs[:, 3].tolist() == [block_stride] * 2
assert fa_runs[:, 4].tolist() == [12] * 2
assert worker.src_blocks_data[2:, 4].tolist() == [4] * 8
expected_addrs = [
tensor.data_ptr() + block * block_stride
for tensor in tensors
for block in range(12)
]
fa_descs = expand_strided_descs(fa_runs)
assert fa_descs[:, 0].tolist() == expected_addrs
assert all(size == unified_page // 3 for size in fa_descs[:, 1])
worker.nixl_wrapper.register_memory.assert_called_once()
metadata = msgspec.msgpack.decode(
worker.xfer_handshake_metadata.agent_metadata_bytes,
@@ -37,7 +37,7 @@ from vllm.v1.kv_cache_interface import (
create_kv_cache_views,
)
from .utils import create_vllm_config
from .utils import create_vllm_config, expand_strided_descs
class _RecordingNixl:
@@ -65,10 +65,11 @@ class _RecordingNixl:
def get_xfer_descs(self, blocks_data, mem_type):
return blocks_data
def prep_xfer_dlist(self, agent, descs):
def prep_xfer_dlist(self, agent, descs, mem_type=None):
handle = self._next_handle
self._next_handle += 1
self.dlists[handle] = np.asarray(descs, dtype=np.uint64).reshape(-1, 3)
# Store per-block (addr, len, dev) rows so desc ids index directly.
self.dlists[handle] = expand_strided_descs(np.asarray(descs, dtype=np.uint64))
return handle
def add_remote_agent(self, metadata):
@@ -186,9 +187,8 @@ def test_packed_mla_pp1_push_peer_transfers_whole_rows(num_layers):
stride = num_layers * 128
worker, raw = _make_packed_mla_view_worker(layouts, stride)
assert worker._registered_descs == [[(raw.data_ptr(), raw.nbytes, 0, "")]]
assert worker.src_blocks_data.tolist() == [
[raw.data_ptr() + block * stride, stride, 0] for block in range(4)
]
# One packed region -> one run of 4 whole rows.
assert worker.src_blocks_data.tolist() == [[raw.data_ptr(), stride, 0, stride, 4]]
assert worker._transfer_layer_group_ids == ()
metadata = msgspec.msgpack.decode(
worker.xfer_handshake_metadata.agent_metadata_bytes, type=NixlAgentMetadata
@@ -248,7 +248,8 @@ def test_packed_mla_pp_pairs_asymmetric_strides_and_overlapping_layers(
1,
region_num_blocks=producer.dst_region_num_blocks[producer.engine_id],
)
assert producer.src_blocks_data[local_ids].tolist() == [
local_descs = expand_strided_descs(producer.src_blocks_data)
assert local_descs[local_ids].tolist() == [
[p_raw.data_ptr() + 192, 128, 0],
[p_raw.data_ptr() + 2 * 192, 64, 0],
[p_raw.data_ptr() + 128 + 3 * 192, 64, 0],
@@ -318,7 +319,8 @@ def test_local_descriptors_follow_each_region_pool_capacity():
descriptors = worker._build_fa_local([100, 1000], block_size_ratio=1)
assert descriptors[:, 0].tolist() == [100, 116, 1000, 1016, 1032]
# One (addr, len, dev, stride, count) run per region, sized to its pool.
assert descriptors.tolist() == [[100, 16, 0, 16, 2], [1000, 16, 0, 16, 3]]
def _register_overlaid_mla_worker(
@@ -479,7 +481,8 @@ def test_overlaid_transfer_groups_share_region_geometry(push_pp):
allocation.data_ptr() + block * block_stride for block in range(num_blocks)
]
num_desc_regions = 2 if push_pp else 1
assert worker.src_blocks_data[:, 0].tolist() == expected_addrs * num_desc_regions
local_descs = expand_strided_descs(worker.src_blocks_data)
assert local_descs[:, 0].tolist() == expected_addrs * num_desc_regions
assert worker.num_descs == num_blocks * num_desc_regions
assert (
worker.dst_region_num_blocks[worker.engine_id]
@@ -500,11 +503,12 @@ def _descriptor_geometry(registered) -> dict:
"""The geometry handed to NIXL, addressed relative to the allocation."""
worker = registered.worker
base = registered.allocation.data_ptr()
descs = expand_strided_descs(worker.src_blocks_data)
return {
"block_len_per_layer": list(worker.block_len_per_layer),
"block_stride_per_layer": list(worker.block_stride_per_layer),
"desc_offsets": [addr - base for addr in worker.src_blocks_data[:, 0].tolist()],
"desc_lens": worker.src_blocks_data[:, 1].tolist(),
"desc_offsets": [addr - base for addr in descs[:, 0].tolist()],
"desc_lens": descs[:, 1].tolist(),
}
@@ -1365,10 +1369,11 @@ def test_csa_linear_ple_descriptor_is_not_split():
bases = [0x10000, 0x20000, 0x30000, 0x40000]
attention = worker._build_fa_local(bases, block_size_ratio=1)
mamba = worker._build_mamba_local(bases)
assert attention.shape == (8, 3)
assert mamba.shape == (10, 3)
assert mamba[-2:, 0].tolist() == [0x10000, 0x10100]
assert mamba[-2:, 1].tolist() == [256, 256]
# One (addr, len, dev, stride, count) run per region / sub-region.
assert attention.shape == (4, 5)
assert mamba.shape == (5, 5)
# PLE run: 2 whole 256-byte pages at 0x10000, 0x10100.
assert mamba[-1].tolist() == [0x10000, 256, 0, 256, 2]
mapping = TPMapping(
source_ranks_per_group=((0,), (0,), (0, 1), (0, 1)),
@@ -1380,14 +1385,15 @@ def test_csa_linear_ple_descriptor_is_not_split():
worker._build_local_splits_from_plan(
mapping,
np.concatenate([attention, mamba]),
num_fa_descs=len(attention),
num_fa_descs=int(attention[:, 4].sum()),
)
)
assert len(splits) == 2
assert [row[1] for row in splits[0][-2:]] == [256, 256]
assert [row[1] for row in splits[1][-2:]] == [256, 256]
assert splits[0][-3][1] * 2 == mamba[-3, 1]
assert splits[1][-3][1] * 2 == mamba[-3, 1]
for split in splits:
# The PLE run is copied whole to every source rank, while the sharded
# SSM run right before it is halved per source.
assert split[-1].tolist() == mamba[-1].tolist()
assert split[-2, 1] * 2 == mamba[-2, 1]
@pytest.mark.cpu_test
@@ -1424,9 +1430,9 @@ def test_csa_linear_remote_ple_is_copied_whole():
tp_ratio=-2,
transfer_info=SimpleNamespace(remote_physical_blocks_per_logical=1),
)
assert descriptors.shape == (10, 3)
assert descriptors[-2:, 0].tolist() == [0x10000, 0x10100]
assert descriptors[-2:, 1].tolist() == [256, 256]
assert descriptors.shape == (5, 5)
# PLE run: 2 whole 256-byte pages at 0x10000, 0x10100.
assert descriptors[-1].tolist() == [0x10000, 256, 0, 256, 2]
metadata.ple_block_len = 128
with pytest.raises(ValueError, match="PLE pages require identical"):
@@ -1451,8 +1457,8 @@ def test_csa_linear_ple_page_wider_than_the_shared_region_page():
bases = [0x10000, 0x20000, 0x30000, 0x40000]
mamba = worker._build_mamba_local(bases)
assert mamba[-2:, 0].tolist() == [0x10000, 0x10000 + 384]
assert mamba[-2:, 1].tolist() == [384, 384]
# PLE run: 2 whole 384-byte pages at 0x10000, 0x10000 + 384.
assert mamba[-1].tolist() == [0x10000, 384, 0, 384, 2]
metadata = msgspec.msgpack.decode(
worker.xfer_handshake_metadata.agent_metadata_bytes, type=NixlAgentMetadata
@@ -60,7 +60,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import (
from vllm.v1.kv_cache_interface import FullAttentionSpec
from vllm.v1.outputs import KVConnectorOutput
from .utils import create_request, make_nixl_push_scheduler
from .utils import create_request, expand_strided_descs, make_nixl_push_scheduler
# ----------------------------------------------------------------- #
# Helpers / fakes #
@@ -2008,8 +2008,13 @@ def test_layer_descriptors_pair_layers_across_asymmetric_pp_split(
assert consumer.kv_caches_base_addr == [0xB000, 0xC000, 0xC000]
plan = TPMapping(((0,), (0,)), (0,), {0: 0}, 0)
local_descs = worker._build_fa_local([0x1000, 0x2000], block_size_ratio=1)
remote_descs = worker._build_fa_remote(plan, consumer, block_size_ratio=1)
# Builders emit one strided run per region; expand to per-block descriptors.
local_descs = expand_strided_descs(
worker._build_fa_local([0x1000, 0x2000], block_size_ratio=1)
)
remote_descs = expand_strided_descs(
worker._build_fa_remote(plan, consumer, block_size_ratio=1)
)
local_counts = [local_num_blocks[i] for i in worker._transfer_layer_region_indices]
local_ids = worker._compute_desc_ids(
[[1, 2], [3]], 4, None, 1, region_num_blocks=local_counts
@@ -75,7 +75,9 @@ class FakeNixlWrapper:
def get_xfer_descs(self, blocks_data, memory_type: str) -> list:
return [str(uuid.uuid4()) for _ in blocks_data]
def prep_xfer_dlist(self, agent_name: str, descs: list) -> int:
def prep_xfer_dlist(
self, agent_name: str, descs, mem_type: str | None = None
) -> int:
return uuid.uuid4().int
def get_agent_metadata(self) -> bytes:
+36 -36
View File
@@ -145,9 +145,8 @@ def _make_mock_worker_for_splits(group_spec_types):
"""Build a mock NixlConnectorWorker with _group_spec_types for split tests.
No per-region replicate flags are configured (``block_len_per_layer`` empty
and ``num_regions == 0``), so ``_fa_desc_replicated`` takes its early-return
path and treats every FA descriptor as SPLIT, matching the legacy behavior
these tests assert.
and ``num_regions == 0``), so ``_build_local_splits_from_plan`` treats every
FA descriptor as SPLIT, matching the legacy behavior these tests assert.
"""
worker = object.__new__(NixlConnectorWorker)
worker._group_spec_types = group_spec_types
@@ -174,11 +173,9 @@ class TestBuildSrcSplitHandles:
)
worker = _make_mock_worker_for_splits((FullAttentionSpec,))
src_blocks_data = np.array(
[(0x2000 + i * 1024, 1024, 0) for i in range(8)],
dtype=np.uint64,
)
num_descs = len(src_blocks_data)
# One region of 8 dense 1024-byte blocks as a single strided run.
src_blocks_data = np.array([(0x2000, 1024, 0, 1024, 8)], dtype=np.uint64)
num_descs = 8
splits = list(
worker._build_local_splits_from_plan(
plan,
@@ -189,9 +186,10 @@ class TestBuildSrcSplitHandles:
assert len(splits) == remote_tp_size
for handle in splits:
assert len(handle) == len(src_blocks_data)
for _, length, _ in handle:
assert length == 1024 // remote_tp_size
assert handle.shape == src_blocks_data.shape
# The head slice shrinks the length; block stride and count are kept.
assert handle[:, 1].tolist() == [1024 // remote_tp_size]
assert handle[:, 3:].tolist() == [[1024, 8]]
class TestMambaPlanSplitHandles:
@@ -209,12 +207,12 @@ class TestMambaPlanSplitHandles:
)
worker = _make_mock_worker_for_splits((FullAttentionSpec, MambaSpec))
# 2 FA descs + 1 SSM desc
# 2 FA runs + 1 SSM run of one block each: (addr, len, dev, stride, count)
src_blocks_data = np.array(
[
(1000, 200, 0), # FA desc 0
(2000, 200, 0), # FA desc 1
(3000, 400, 0), # SSM desc 0
(1000, 200, 0, 200, 1), # FA desc 0
(2000, 200, 0, 200, 1), # FA desc 1
(3000, 400, 0, 400, 1), # SSM desc 0
],
dtype=np.uint64,
)
@@ -224,14 +222,22 @@ class TestMambaPlanSplitHandles:
assert len(splits) == 2 # 2 source ranks
# Rank 0 (FA source, p_idx=0):
# FA: chunk=200//1=200, slot=0 → (1000, 200, 0), (2000, 200, 0)
# SSM: chunk=400//2=200, idx=0 → (3000, 200, 0)
assert splits[0] == [(1000, 200, 0), (2000, 200, 0), (3000, 200, 0)]
# FA: chunk=200//1=200, slot=0 → (1000, 200), (2000, 200)
# SSM: chunk=400//2=200, idx=0 → (3000, 200); block stride stays 400
assert splits[0].tolist() == [
[1000, 200, 0, 200, 1],
[2000, 200, 0, 200, 1],
[3000, 200, 0, 400, 1],
]
# Rank 1 (not FA source, p_idx=1):
# FA: chunk=200//1=200, slot=0 (skip_fa) → (1000, 200, 0), (2000, 200, 0)
# SSM: chunk=400//2=200, idx=1 → (3200, 200, 0)
assert splits[1] == [(1000, 200, 0), (2000, 200, 0), (3200, 200, 0)]
# FA: chunk=200//1=200, slot=0 (skip_fa) → (1000, 200), (2000, 200)
# SSM: chunk=400//2=200, idx=1 → (3200, 200); block stride stays 400
assert splits[1].tolist() == [
[1000, 200, 0, 200, 1],
[2000, 200, 0, 200, 1],
[3200, 200, 0, 400, 1],
]
def test_hetero_block_size_splits(self):
"""With a block-size ratio, single-source FA sub-block descs pass
@@ -244,14 +250,13 @@ class TestMambaPlanSplitHandles:
)
worker = _make_mock_worker_for_splits((FullAttentionSpec, MambaSpec))
# 2 FA blocks x ratio 2 sub-blocks + 1 SSM desc (never expanded).
# 2 FA blocks x ratio 2 sub-blocks (one run of 2 per block) + 1 SSM desc
# (never expanded).
src_blocks_data = np.array(
[
(1000, 100, 0),
(1100, 100, 0),
(2000, 100, 0),
(2100, 100, 0),
(3000, 400, 0),
(1000, 100, 0, 100, 2),
(2000, 100, 0, 100, 2),
(3000, 400, 0, 400, 1),
],
dtype=np.uint64,
)
@@ -259,14 +264,9 @@ class TestMambaPlanSplitHandles:
splits = list(worker._build_local_splits_from_plan(plan, src_blocks_data, 4, 2))
assert len(splits) == 2
fa_passthrough = [
(1000, 100, 0),
(1100, 100, 0),
(2000, 100, 0),
(2100, 100, 0),
]
assert splits[0] == fa_passthrough + [(3000, 200, 0)]
assert splits[1] == fa_passthrough + [(3200, 200, 0)]
fa_passthrough = [[1000, 100, 0, 100, 2], [2000, 100, 0, 100, 2]]
assert splits[0].tolist() == fa_passthrough + [[3000, 200, 0, 400, 1]]
assert splits[1].tolist() == fa_passthrough + [[3200, 200, 0, 400, 1]]
def test_hetero_block_size_head_sharded_asserts(self):
"""Head-sharded FA reads (multiple FA sources) are incompatible with
@@ -280,7 +280,7 @@ class TestMambaPlanSplitHandles:
worker = _make_mock_worker_for_splits((FullAttentionSpec, MambaSpec))
src_blocks_data = np.array(
[(1000, 100, 0), (1100, 100, 0), (3000, 400, 0)],
[(1000, 100, 0, 100, 2), (3000, 400, 0, 400, 1)],
dtype=np.uint64,
)
+16
View File
@@ -8,6 +8,7 @@ from itertools import chain, count
from typing import Any, Literal
from unittest.mock import Mock
import numpy as np
import torch
from vllm import SamplingParams
@@ -682,3 +683,18 @@ def make_moriio_writer(fake_worker: Any) -> Any:
writer._defer_timeout = 60.0
writer.ensure_worker_started = lambda: None
return writer
def expand_strided_descs(strided_descs: np.ndarray) -> np.ndarray:
"""Expand Nx5 (addr, len, dev, stride, count) rows to one row per block.
Mirrors how NIXL indexes a strided descriptor list, so desc ids computed
by the connector can be resolved to (addr, len, dev) byte ranges.
"""
strided_descs = np.asarray(strided_descs, dtype=np.uint64).reshape(-1, 5)
counts = strided_descs[:, 4].astype(np.intp)
starts = np.cumsum(strided_descs[:, 4]) - strided_descs[:, 4]
out = np.repeat(strided_descs[:, :3], counts, axis=0)
offsets = np.arange(len(out), dtype=np.uint64) - np.repeat(starts, counts)
out[:, 0] += offsets * np.repeat(strided_descs[:, 3], counts)
return out