[2/N][KV Connector][NIXL] Support per-region transfer geometry (#53780)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Matthew Bonanni
2026-09-08 16:01:12 -04:00
committed by GitHub
co-authored by OpenAI Codex OpenAI Codex
parent 2c9d68f80b
commit e0aaef85f3
9 changed files with 894 additions and 112 deletions
@@ -518,8 +518,15 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
slot_size_bytes = 4096
self.slot_size_per_layer = [slot_size_bytes]
self.block_len_per_layer = [slot_size_bytes * self.block_size]
self.num_blocks = 1
self.num_regions = 1
self.block_stride_per_layer = list(self.block_len_per_layer)
self.region_num_blocks = [self.num_blocks]
self.region_group_ids = [0]
self.region_mem_types = [self.nixl_memory_type]
self.dst_num_blocks[self.engine_id] = self.num_blocks
self.dst_region_num_blocks[self.engine_id] = self.region_num_blocks
self.dst_region_group_ids[self.engine_id] = self.region_group_ids
self.dst_region_mem_types[self.engine_id] = self.region_mem_types
assert expected_engine_id == self.REMOTE_ENGINE_ID
@@ -548,7 +555,7 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
agent_metadata=FakeNixlWrapper.AGENT_METADATA,
kv_caches_base_addr=[0],
device_id=remote_tp_rank,
num_blocks=1,
num_blocks=self.num_blocks,
block_lens=remote_block_lens,
block_strides=remote_block_lens,
kv_cache_layout="LBHNC",
@@ -556,6 +563,9 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
ssm_sizes=(0, 0),
attn_backend_name=self.backend_name,
physical_blocks_per_logical_kv_block=1,
region_num_blocks=self.region_num_blocks,
region_group_ids=self.region_group_ids,
region_mem_types=self.region_mem_types,
),
remote_tp_rank=remote_tp_rank,
remote_tp_size=remote_tp_size,
@@ -640,7 +650,7 @@ class TestNixlHandshake:
request_id = "req_id"
# Test worker role in decode server.
kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=2)
kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=10)
connector = NixlConnector(vllm_config, KVConnectorRole.WORKER, kv_cache_config)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config,
@@ -1167,12 +1177,19 @@ class TestNixlHandshake:
source_ranks_per_group=((0,), (0,)),
rank_offset_factor=1,
)
meta = MagicMock(
meta = NixlAgentMetadata(
engine_id=FakeNixlConnectorWorker.REMOTE_ENGINE_ID,
agent_metadata=FakeNixlWrapper.AGENT_METADATA,
kv_caches_base_addr=[0x1000],
device_id=0,
num_blocks=1,
block_lens=[remote_block_len],
block_strides=[remote_block_len],
kv_cache_layout="HND",
block_size=worker.block_size,
ssm_sizes=(0, 0),
attn_backend_name=worker.backend_name,
physical_blocks_per_logical_kv_block=1,
)
assert worker._build_fa_remote(plan, meta, block_size_ratio=1).tolist() == [
@@ -1846,6 +1863,110 @@ def _run_abort_timeout_test(llm: LLM, timeout: int):
llm.llm_engine.engine_core.shutdown()
def test_mixed_memory_local_descriptors_split_by_memory_type():
worker = object.__new__(NixlConnectorWorker)
worker.transfer_topo = MagicMock()
worker.block_size = 16
worker.engine_id = "local"
worker.tp_rank = 0
worker.device_id = 3
worker.kv_caches_base_addr = {"local": {0: [100, 200]}}
worker._has_mamba = False
worker._mixed_mem_types = True
worker.region_mem_types = ["DRAM", "VRAM"]
worker.region_num_blocks = [2, 2]
worker._desc_is_dram_by_block_size = {}
worker._desc_pos_by_block_size = {}
worker._dram_src_handles_by_block_size = {}
worker.nixl_memory_type = "VRAM"
worker._build_fa_local = MagicMock( # type: ignore[method-assign]
return_value=np.array(
[
[100, 10, 3],
[110, 10, 3],
[200, 10, 3],
[210, 10, 3],
]
)
)
worker.nixl_wrapper = MagicMock()
worker.nixl_wrapper.get_xfer_descs.side_effect = lambda blocks, memory_type: (
memory_type,
blocks,
)
worker.nixl_wrapper.prep_xfer_dlist.side_effect = [11, 22]
handle, blocks = worker.register_local_xfer_handler(worker.block_size)
assert handle == 22
assert worker._dram_src_handles_by_block_size[worker.block_size] == 11
assert [block[2] for block in blocks] == [0, 0, 3, 3]
assert [
call.args[1] for call in worker.nixl_wrapper.get_xfer_descs.call_args_list
] == ["DRAM", "VRAM"]
def test_mixed_memory_read_notifies_after_both_transfers_finish():
worker = object.__new__(NixlConnectorWorker)
worker._desc_is_dram_by_block_size = {16: np.array([True, True, False, False])}
worker._desc_pos_by_block_size = {16: np.array([0, 1, 0, 1])}
worker._dram_src_handles_by_block_size = {16: 10}
worker._recving_metadata = {"request": MagicMock()}
worker._recving_transfers = defaultdict(list)
worker._pending_recv_notifs = {}
worker.xfer_stats = MagicMock()
worker.nixl_wrapper = MagicMock()
worker.nixl_wrapper.make_prepped_xfer.side_effect = [101, 102]
worker.nixl_wrapper.check_xfer_state.return_value = "DONE"
worker._read_blocks_mixed(
request_id="request",
local_block_size_key=16,
local_device_handle=20,
local_dram_handle=10,
remote_xfer_side_handle=30,
local_block_descs_ids=np.array([0, 2]),
remote_block_descs_ids=np.array([5, 7]),
notif_agent="prefill",
notif_id=b"request:1",
)
dram_read, device_read = worker.nixl_wrapper.make_prepped_xfer.call_args_list
assert dram_read.args[:2] == ("READ", 10)
assert dram_read.args[3] == 30
np.testing.assert_array_equal(dram_read.args[2], [0])
np.testing.assert_array_equal(dram_read.args[4], [5])
assert device_read.args[:2] == ("READ", 20)
assert device_read.args[3] == 30
np.testing.assert_array_equal(device_read.args[2], [0])
np.testing.assert_array_equal(device_read.args[4], [7])
worker.nixl_wrapper.send_notif.assert_not_called()
assert worker._pop_done_transfers(worker._recving_transfers) == {"request"}
worker.nixl_wrapper.send_notif.assert_called_once_with(
"prefill", notif_msg=b"request:1"
)
def test_mixed_memory_read_failure_does_not_notify_producer():
"""A failed half of a split READ must suppress its success notification."""
worker = object.__new__(NixlConnectorWorker)
worker._recving_metadata = {"request": MagicMock()}
worker._recving_transfers = {"request": [101, 102]}
worker._pending_recv_notifs = {"request": [("prefill", b"request:1")]}
worker._is_hma_required = True
worker._failed_recv_reqs = MagicMock()
worker._log_failure = MagicMock() # type: ignore[method-assign]
worker.xfer_stats = MagicMock()
worker.nixl_wrapper = MagicMock()
worker.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "DONE"]
worker._pop_done_transfers(worker._recving_transfers)
assert "request" not in worker._pending_recv_notifs
worker.nixl_wrapper.send_notif.assert_not_called()
def element_byte_addrs(view: torch.Tensor) -> list[int]:
"""Absolute byte addresses of every element of a (possibly strided) view."""
offsets = torch.zeros(view.shape, dtype=torch.int64)
@@ -861,6 +861,7 @@ def _make_mock_worker_for_desc_ids(
worker = MagicMock(spec=NixlConnectorWorker)
worker.num_regions = num_regions
worker._uses_region_group_mapping = False
worker._has_mamba = has_mamba
worker._group_spec_types = group_spec_types
worker.block_len_per_layer = block_len_per_layer or [100]
@@ -913,6 +914,64 @@ def test_get_block_descs_ids_hybrid_ssm():
assert list(result) == expected, f"Expected {expected}, got {list(result)}"
@pytest.mark.cpu_test
def test_get_block_descs_ids_selects_attention_regions_by_group():
"""Each attention group's blocks address only that group's regions."""
from vllm.v1.kv_cache_interface import FullAttentionSpec
worker = _make_mock_worker_for_desc_ids(
num_regions=3,
has_mamba=False,
group_spec_types=(FullAttentionSpec, FullAttentionSpec),
block_len_per_layer=[100, 100, 100],
)
worker.region_group_ids = [0, 0, 1]
worker._uses_region_group_mapping = True
result = worker._compute_desc_ids(
block_ids=([1, 2], [7]),
dst_num_blocks=10,
block_size_ratio=None,
physical_blocks_per_logical=1,
)
assert result.tolist() == [1, 2, 11, 12, 27]
remapped = worker._compute_desc_ids(
block_ids=([1, 2], [7]),
dst_num_blocks=10,
block_size_ratio=None,
physical_blocks_per_logical=1,
region_group_ids=[1, 1, 0],
)
assert remapped.tolist() == [21, 22, 7, 17]
@pytest.mark.cpu_test
def test_get_block_descs_ids_uses_per_region_pool_capacity():
"""Independent host and device pools use cumulative descriptor offsets."""
from vllm.v1.kv_cache_interface import FullAttentionSpec
worker = _make_mock_worker_for_desc_ids(
num_regions=2,
has_mamba=False,
group_spec_types=(FullAttentionSpec, FullAttentionSpec),
block_len_per_layer=[100, 100],
)
worker.region_group_ids = [0, 1]
worker._uses_region_group_mapping = True
result = worker._compute_desc_ids(
block_ids=([4], [8]),
dst_num_blocks=10,
block_size_ratio=None,
physical_blocks_per_logical=1,
region_num_blocks=[5, 10],
)
assert result.tolist() == [4, 13]
@pytest.mark.cpu_test
def test_get_block_descs_ids_kernel_block_mismatch():
"""Test _compute_desc_ids uses different strides for FA
@@ -11,7 +11,8 @@ incoming transfer can overwrite a co-resident request's KV or mamba state
mid-decode (silent corruption of an unrelated request).
"""
from unittest.mock import patch
from collections import defaultdict
from unittest.mock import MagicMock, patch
import numpy as np
import pytest
@@ -27,13 +28,14 @@ class _RecordingNixl:
def __init__(self, *args, **kwargs):
self.dlists: dict[int, np.ndarray] = {}
self.xfers: list[tuple] = []
self.registered: list[tuple[list[tuple], object]] = []
self._next_handle = 1
def get_reg_descs(self, caches_data, mem_type):
return caches_data
def register_memory(self, descs, backends=None):
pass
self.registered.append((descs, backends))
def deregister_memory(self, descs):
pass
@@ -98,6 +100,150 @@ class _RecordingNixl:
pass
@pytest.mark.cpu_test
def test_local_descriptors_follow_each_region_pool_capacity():
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
NixlConnectorWorker,
)
worker = object.__new__(NixlConnectorWorker)
worker.transfer_topo = MagicMock()
worker.device_id = 0
worker.block_len_per_layer = [16, 16]
worker.block_stride_per_layer = [16, 16]
worker.region_num_blocks = [2, 3]
descriptors = worker._build_fa_local([100, 1000], block_size_ratio=1)
assert descriptors[:, 0].tolist() == [100, 116, 1000, 1016, 1032]
@pytest.mark.cpu_test
def test_overlaid_transfer_groups_share_region_geometry():
"""Groups overlaid on one allocation share its transfer region."""
import msgspec
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
NixlAgentMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
NixlConnectorWorker,
)
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
KVCacheTensor,
MLAAttentionSpec,
)
num_blocks = 4
spec = MLAAttentionSpec(
block_size=4,
num_kv_heads=1,
head_size=8,
dtype=torch.uint8,
)
page_size = spec.page_size_bytes
block_stride = 2 * page_size
backing = torch.zeros(num_blocks, block_stride, dtype=torch.uint8)
caches = {
"layer.0": backing[:, :page_size],
"layer.1": backing[:, :page_size],
}
groups = [KVCacheGroupSpec([layer_name], spec) for layer_name in caches]
worker = object.__new__(NixlConnectorWorker)
worker.tp_rank = 0
worker.world_size = 1
worker.block_size = 4
worker.engine_id = "local-engine"
worker.use_mla = True
worker.model_config = MagicMock()
worker.model_config.get_total_num_kv_heads.return_value = 1
worker.attn_backends = []
worker._has_mamba = False
worker._is_csa_linear = False
worker.vllm_config = MagicMock()
worker.backend_name = "FLASHMLA"
worker.num_blocks = num_blocks
worker.nixl_memory_type = "VRAM"
worker.nixl_backends = None
worker.nixl_wrapper = _RecordingNixl()
worker._registered_descs = []
worker.dst_num_blocks = {}
worker.dst_region_num_blocks = {}
worker.dst_region_group_ids = {}
worker.dst_uses_region_group_mapping = {}
worker.dst_region_mem_types = {}
worker._desc_is_dram_by_block_size = {}
worker._desc_pos_by_block_size = {}
worker._dram_src_handles_by_block_size = {}
worker.src_xfer_handles_by_block_size = {}
worker.kv_caches_base_addr = defaultdict(dict)
worker._mamba_ssm_size = (0, 0)
worker.kv_cache_layout = "NHD"
worker.host_buffer_kv_cache_layout = "NHD"
worker._physical_blocks_per_logical_kv_block = 1
worker._logical_num_blocks = num_blocks
worker.region_group_ids = []
worker.region_mem_types = []
worker._mixed_mem_types = False
worker.region_names = []
worker.region_num_blocks = []
worker._region_is_mla = []
worker.block_len_per_layer = []
worker.block_stride_per_layer = []
worker.device_id = 0
worker.use_host_buffer = False
worker.host_xfer_buffers = {}
worker.device_kv_caches = {}
worker.pp_size = 1
worker.dcp_size = 1
worker.pcp_size = 1
worker.kv_buffer_device = "cuda"
worker._layer_specs = {name: spec for name in caches}
worker.kv_cache_config = KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=[
KVCacheTensor(
size=backing.nbytes,
layers=[name],
layer_stride=page_size,
block_stride=block_stride,
)
for name in caches
],
kv_cache_groups=groups,
)
transfer_topology = MagicMock()
with (
patch.object(bw, "TransferTopology", return_value=transfer_topology),
patch.object(bw, "compute_nixl_compatibility_hash", return_value="hash"),
):
worker.register_kv_caches(caches)
assert worker.region_group_ids == [-1]
assert worker.block_stride_per_layer == [block_stride]
assert worker.nixl_wrapper.registered[0][0] == [
(backing.data_ptr(), backing.nbytes, 0, "")
]
expected_addrs = [
backing.data_ptr() + block * block_stride for block in range(num_blocks)
]
assert worker.src_blocks_data[:, 0].tolist() == expected_addrs
metadata = msgspec.msgpack.decode(
worker.xfer_handshake_metadata.agent_metadata_bytes,
type=NixlAgentMetadata,
)
assert metadata.region_group_ids == [-1]
assert metadata.region_num_blocks == [num_blocks]
assert worker._block_ids_by_region(([0], [2]), worker.region_group_ids) == [[0, 2]]
def _make_mla_hybrid_worker(local_block_size, kernel_block_size, num_logical_blocks):
"""Build a real pull worker with a hybrid MLA + 2xKDA HMA layout."""
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import (
@@ -357,6 +357,12 @@ class _StubWriterWorker(NixlPushConnectorWorker):
w.engine_id = "test-decode-engine"
w._remote_agents = {}
w._physical_blocks_per_logical_kv_block = 1
w._uses_region_group_mapping = False
w.region_group_ids = [0]
w.dst_region_num_blocks = {}
w.dst_region_group_ids = {}
w.dst_uses_region_group_mapping = {}
w.dst_region_mem_types = {}
# Single non-hybrid attention group, matching the stub block id lists.
w._has_mamba = False
w._is_csa_linear = False
@@ -1314,6 +1320,15 @@ class TestPushPrefixCaching:
)
}
w.dst_num_blocks = {engine_id: 10_000, w.engine_id: 10_000}
w.dst_region_num_blocks = {
engine_id: [10_000],
w.engine_id: [10_000],
}
w.dst_region_group_ids = {engine_id: [0], w.engine_id: [0]}
w.dst_uses_region_group_mapping = {
engine_id: False,
w.engine_id: False,
}
w.dst_xfer_side_handles = {engine_id: {0: 5000}}
w.src_xfer_handles_by_block_size = {16: 2000}