[2/N][KV Connector][NIXL] Support per-region transfer geometry (#53780)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
OpenAI Codex
parent
2c9d68f80b
commit
e0aaef85f3
@@ -518,8 +518,15 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
|
||||
slot_size_bytes = 4096
|
||||
self.slot_size_per_layer = [slot_size_bytes]
|
||||
self.block_len_per_layer = [slot_size_bytes * self.block_size]
|
||||
self.num_blocks = 1
|
||||
self.num_regions = 1
|
||||
self.block_stride_per_layer = list(self.block_len_per_layer)
|
||||
self.region_num_blocks = [self.num_blocks]
|
||||
self.region_group_ids = [0]
|
||||
self.region_mem_types = [self.nixl_memory_type]
|
||||
self.dst_num_blocks[self.engine_id] = self.num_blocks
|
||||
self.dst_region_num_blocks[self.engine_id] = self.region_num_blocks
|
||||
self.dst_region_group_ids[self.engine_id] = self.region_group_ids
|
||||
self.dst_region_mem_types[self.engine_id] = self.region_mem_types
|
||||
|
||||
assert expected_engine_id == self.REMOTE_ENGINE_ID
|
||||
|
||||
@@ -548,7 +555,7 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
|
||||
agent_metadata=FakeNixlWrapper.AGENT_METADATA,
|
||||
kv_caches_base_addr=[0],
|
||||
device_id=remote_tp_rank,
|
||||
num_blocks=1,
|
||||
num_blocks=self.num_blocks,
|
||||
block_lens=remote_block_lens,
|
||||
block_strides=remote_block_lens,
|
||||
kv_cache_layout="LBHNC",
|
||||
@@ -556,6 +563,9 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
|
||||
ssm_sizes=(0, 0),
|
||||
attn_backend_name=self.backend_name,
|
||||
physical_blocks_per_logical_kv_block=1,
|
||||
region_num_blocks=self.region_num_blocks,
|
||||
region_group_ids=self.region_group_ids,
|
||||
region_mem_types=self.region_mem_types,
|
||||
),
|
||||
remote_tp_rank=remote_tp_rank,
|
||||
remote_tp_size=remote_tp_size,
|
||||
@@ -640,7 +650,7 @@ class TestNixlHandshake:
|
||||
request_id = "req_id"
|
||||
|
||||
# Test worker role in decode server.
|
||||
kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=2)
|
||||
kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=10)
|
||||
connector = NixlConnector(vllm_config, KVConnectorRole.WORKER, kv_cache_config)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config,
|
||||
@@ -1167,12 +1177,19 @@ class TestNixlHandshake:
|
||||
source_ranks_per_group=((0,), (0,)),
|
||||
rank_offset_factor=1,
|
||||
)
|
||||
meta = MagicMock(
|
||||
meta = NixlAgentMetadata(
|
||||
engine_id=FakeNixlConnectorWorker.REMOTE_ENGINE_ID,
|
||||
agent_metadata=FakeNixlWrapper.AGENT_METADATA,
|
||||
kv_caches_base_addr=[0x1000],
|
||||
device_id=0,
|
||||
num_blocks=1,
|
||||
block_lens=[remote_block_len],
|
||||
block_strides=[remote_block_len],
|
||||
kv_cache_layout="HND",
|
||||
block_size=worker.block_size,
|
||||
ssm_sizes=(0, 0),
|
||||
attn_backend_name=worker.backend_name,
|
||||
physical_blocks_per_logical_kv_block=1,
|
||||
)
|
||||
|
||||
assert worker._build_fa_remote(plan, meta, block_size_ratio=1).tolist() == [
|
||||
@@ -1846,6 +1863,110 @@ def _run_abort_timeout_test(llm: LLM, timeout: int):
|
||||
llm.llm_engine.engine_core.shutdown()
|
||||
|
||||
|
||||
def test_mixed_memory_local_descriptors_split_by_memory_type():
|
||||
worker = object.__new__(NixlConnectorWorker)
|
||||
worker.transfer_topo = MagicMock()
|
||||
worker.block_size = 16
|
||||
worker.engine_id = "local"
|
||||
worker.tp_rank = 0
|
||||
worker.device_id = 3
|
||||
worker.kv_caches_base_addr = {"local": {0: [100, 200]}}
|
||||
worker._has_mamba = False
|
||||
worker._mixed_mem_types = True
|
||||
worker.region_mem_types = ["DRAM", "VRAM"]
|
||||
worker.region_num_blocks = [2, 2]
|
||||
worker._desc_is_dram_by_block_size = {}
|
||||
worker._desc_pos_by_block_size = {}
|
||||
worker._dram_src_handles_by_block_size = {}
|
||||
worker.nixl_memory_type = "VRAM"
|
||||
worker._build_fa_local = MagicMock( # type: ignore[method-assign]
|
||||
return_value=np.array(
|
||||
[
|
||||
[100, 10, 3],
|
||||
[110, 10, 3],
|
||||
[200, 10, 3],
|
||||
[210, 10, 3],
|
||||
]
|
||||
)
|
||||
)
|
||||
worker.nixl_wrapper = MagicMock()
|
||||
worker.nixl_wrapper.get_xfer_descs.side_effect = lambda blocks, memory_type: (
|
||||
memory_type,
|
||||
blocks,
|
||||
)
|
||||
worker.nixl_wrapper.prep_xfer_dlist.side_effect = [11, 22]
|
||||
|
||||
handle, blocks = worker.register_local_xfer_handler(worker.block_size)
|
||||
|
||||
assert handle == 22
|
||||
assert worker._dram_src_handles_by_block_size[worker.block_size] == 11
|
||||
assert [block[2] for block in blocks] == [0, 0, 3, 3]
|
||||
assert [
|
||||
call.args[1] for call in worker.nixl_wrapper.get_xfer_descs.call_args_list
|
||||
] == ["DRAM", "VRAM"]
|
||||
|
||||
|
||||
def test_mixed_memory_read_notifies_after_both_transfers_finish():
|
||||
worker = object.__new__(NixlConnectorWorker)
|
||||
worker._desc_is_dram_by_block_size = {16: np.array([True, True, False, False])}
|
||||
worker._desc_pos_by_block_size = {16: np.array([0, 1, 0, 1])}
|
||||
worker._dram_src_handles_by_block_size = {16: 10}
|
||||
worker._recving_metadata = {"request": MagicMock()}
|
||||
worker._recving_transfers = defaultdict(list)
|
||||
worker._pending_recv_notifs = {}
|
||||
worker.xfer_stats = MagicMock()
|
||||
worker.nixl_wrapper = MagicMock()
|
||||
worker.nixl_wrapper.make_prepped_xfer.side_effect = [101, 102]
|
||||
worker.nixl_wrapper.check_xfer_state.return_value = "DONE"
|
||||
|
||||
worker._read_blocks_mixed(
|
||||
request_id="request",
|
||||
local_block_size_key=16,
|
||||
local_device_handle=20,
|
||||
local_dram_handle=10,
|
||||
remote_xfer_side_handle=30,
|
||||
local_block_descs_ids=np.array([0, 2]),
|
||||
remote_block_descs_ids=np.array([5, 7]),
|
||||
notif_agent="prefill",
|
||||
notif_id=b"request:1",
|
||||
)
|
||||
|
||||
dram_read, device_read = worker.nixl_wrapper.make_prepped_xfer.call_args_list
|
||||
assert dram_read.args[:2] == ("READ", 10)
|
||||
assert dram_read.args[3] == 30
|
||||
np.testing.assert_array_equal(dram_read.args[2], [0])
|
||||
np.testing.assert_array_equal(dram_read.args[4], [5])
|
||||
assert device_read.args[:2] == ("READ", 20)
|
||||
assert device_read.args[3] == 30
|
||||
np.testing.assert_array_equal(device_read.args[2], [0])
|
||||
np.testing.assert_array_equal(device_read.args[4], [7])
|
||||
worker.nixl_wrapper.send_notif.assert_not_called()
|
||||
|
||||
assert worker._pop_done_transfers(worker._recving_transfers) == {"request"}
|
||||
worker.nixl_wrapper.send_notif.assert_called_once_with(
|
||||
"prefill", notif_msg=b"request:1"
|
||||
)
|
||||
|
||||
|
||||
def test_mixed_memory_read_failure_does_not_notify_producer():
|
||||
"""A failed half of a split READ must suppress its success notification."""
|
||||
worker = object.__new__(NixlConnectorWorker)
|
||||
worker._recving_metadata = {"request": MagicMock()}
|
||||
worker._recving_transfers = {"request": [101, 102]}
|
||||
worker._pending_recv_notifs = {"request": [("prefill", b"request:1")]}
|
||||
worker._is_hma_required = True
|
||||
worker._failed_recv_reqs = MagicMock()
|
||||
worker._log_failure = MagicMock() # type: ignore[method-assign]
|
||||
worker.xfer_stats = MagicMock()
|
||||
worker.nixl_wrapper = MagicMock()
|
||||
worker.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "DONE"]
|
||||
|
||||
worker._pop_done_transfers(worker._recving_transfers)
|
||||
|
||||
assert "request" not in worker._pending_recv_notifs
|
||||
worker.nixl_wrapper.send_notif.assert_not_called()
|
||||
|
||||
|
||||
def element_byte_addrs(view: torch.Tensor) -> list[int]:
|
||||
"""Absolute byte addresses of every element of a (possibly strided) view."""
|
||||
offsets = torch.zeros(view.shape, dtype=torch.int64)
|
||||
|
||||
@@ -861,6 +861,7 @@ def _make_mock_worker_for_desc_ids(
|
||||
|
||||
worker = MagicMock(spec=NixlConnectorWorker)
|
||||
worker.num_regions = num_regions
|
||||
worker._uses_region_group_mapping = False
|
||||
worker._has_mamba = has_mamba
|
||||
worker._group_spec_types = group_spec_types
|
||||
worker.block_len_per_layer = block_len_per_layer or [100]
|
||||
@@ -913,6 +914,64 @@ def test_get_block_descs_ids_hybrid_ssm():
|
||||
assert list(result) == expected, f"Expected {expected}, got {list(result)}"
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_get_block_descs_ids_selects_attention_regions_by_group():
|
||||
"""Each attention group's blocks address only that group's regions."""
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
|
||||
worker = _make_mock_worker_for_desc_ids(
|
||||
num_regions=3,
|
||||
has_mamba=False,
|
||||
group_spec_types=(FullAttentionSpec, FullAttentionSpec),
|
||||
block_len_per_layer=[100, 100, 100],
|
||||
)
|
||||
worker.region_group_ids = [0, 0, 1]
|
||||
worker._uses_region_group_mapping = True
|
||||
|
||||
result = worker._compute_desc_ids(
|
||||
block_ids=([1, 2], [7]),
|
||||
dst_num_blocks=10,
|
||||
block_size_ratio=None,
|
||||
physical_blocks_per_logical=1,
|
||||
)
|
||||
|
||||
assert result.tolist() == [1, 2, 11, 12, 27]
|
||||
|
||||
remapped = worker._compute_desc_ids(
|
||||
block_ids=([1, 2], [7]),
|
||||
dst_num_blocks=10,
|
||||
block_size_ratio=None,
|
||||
physical_blocks_per_logical=1,
|
||||
region_group_ids=[1, 1, 0],
|
||||
)
|
||||
assert remapped.tolist() == [21, 22, 7, 17]
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_get_block_descs_ids_uses_per_region_pool_capacity():
|
||||
"""Independent host and device pools use cumulative descriptor offsets."""
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
|
||||
worker = _make_mock_worker_for_desc_ids(
|
||||
num_regions=2,
|
||||
has_mamba=False,
|
||||
group_spec_types=(FullAttentionSpec, FullAttentionSpec),
|
||||
block_len_per_layer=[100, 100],
|
||||
)
|
||||
worker.region_group_ids = [0, 1]
|
||||
worker._uses_region_group_mapping = True
|
||||
|
||||
result = worker._compute_desc_ids(
|
||||
block_ids=([4], [8]),
|
||||
dst_num_blocks=10,
|
||||
block_size_ratio=None,
|
||||
physical_blocks_per_logical=1,
|
||||
region_num_blocks=[5, 10],
|
||||
)
|
||||
|
||||
assert result.tolist() == [4, 13]
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_get_block_descs_ids_kernel_block_mismatch():
|
||||
"""Test _compute_desc_ids uses different strides for FA
|
||||
|
||||
@@ -11,7 +11,8 @@ incoming transfer can overwrite a co-resident request's KV or mamba state
|
||||
mid-decode (silent corruption of an unrelated request).
|
||||
"""
|
||||
|
||||
from unittest.mock import patch
|
||||
from collections import defaultdict
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
@@ -27,13 +28,14 @@ class _RecordingNixl:
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.dlists: dict[int, np.ndarray] = {}
|
||||
self.xfers: list[tuple] = []
|
||||
self.registered: list[tuple[list[tuple], object]] = []
|
||||
self._next_handle = 1
|
||||
|
||||
def get_reg_descs(self, caches_data, mem_type):
|
||||
return caches_data
|
||||
|
||||
def register_memory(self, descs, backends=None):
|
||||
pass
|
||||
self.registered.append((descs, backends))
|
||||
|
||||
def deregister_memory(self, descs):
|
||||
pass
|
||||
@@ -98,6 +100,150 @@ class _RecordingNixl:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_local_descriptors_follow_each_region_pool_capacity():
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
NixlConnectorWorker,
|
||||
)
|
||||
|
||||
worker = object.__new__(NixlConnectorWorker)
|
||||
worker.transfer_topo = MagicMock()
|
||||
worker.device_id = 0
|
||||
worker.block_len_per_layer = [16, 16]
|
||||
worker.block_stride_per_layer = [16, 16]
|
||||
worker.region_num_blocks = [2, 3]
|
||||
|
||||
descriptors = worker._build_fa_local([100, 1000], block_size_ratio=1)
|
||||
|
||||
assert descriptors[:, 0].tolist() == [100, 116, 1000, 1016, 1032]
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_overlaid_transfer_groups_share_region_geometry():
|
||||
"""Groups overlaid on one allocation share its transfer region."""
|
||||
import msgspec
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
NixlAgentMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
NixlConnectorWorker,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
MLAAttentionSpec,
|
||||
)
|
||||
|
||||
num_blocks = 4
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=8,
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
page_size = spec.page_size_bytes
|
||||
block_stride = 2 * page_size
|
||||
backing = torch.zeros(num_blocks, block_stride, dtype=torch.uint8)
|
||||
caches = {
|
||||
"layer.0": backing[:, :page_size],
|
||||
"layer.1": backing[:, :page_size],
|
||||
}
|
||||
groups = [KVCacheGroupSpec([layer_name], spec) for layer_name in caches]
|
||||
|
||||
worker = object.__new__(NixlConnectorWorker)
|
||||
worker.tp_rank = 0
|
||||
worker.world_size = 1
|
||||
worker.block_size = 4
|
||||
worker.engine_id = "local-engine"
|
||||
worker.use_mla = True
|
||||
worker.model_config = MagicMock()
|
||||
worker.model_config.get_total_num_kv_heads.return_value = 1
|
||||
worker.attn_backends = []
|
||||
worker._has_mamba = False
|
||||
worker._is_csa_linear = False
|
||||
worker.vllm_config = MagicMock()
|
||||
worker.backend_name = "FLASHMLA"
|
||||
worker.num_blocks = num_blocks
|
||||
worker.nixl_memory_type = "VRAM"
|
||||
worker.nixl_backends = None
|
||||
worker.nixl_wrapper = _RecordingNixl()
|
||||
worker._registered_descs = []
|
||||
worker.dst_num_blocks = {}
|
||||
worker.dst_region_num_blocks = {}
|
||||
worker.dst_region_group_ids = {}
|
||||
worker.dst_uses_region_group_mapping = {}
|
||||
worker.dst_region_mem_types = {}
|
||||
worker._desc_is_dram_by_block_size = {}
|
||||
worker._desc_pos_by_block_size = {}
|
||||
worker._dram_src_handles_by_block_size = {}
|
||||
worker.src_xfer_handles_by_block_size = {}
|
||||
worker.kv_caches_base_addr = defaultdict(dict)
|
||||
worker._mamba_ssm_size = (0, 0)
|
||||
worker.kv_cache_layout = "NHD"
|
||||
worker.host_buffer_kv_cache_layout = "NHD"
|
||||
worker._physical_blocks_per_logical_kv_block = 1
|
||||
worker._logical_num_blocks = num_blocks
|
||||
worker.region_group_ids = []
|
||||
worker.region_mem_types = []
|
||||
worker._mixed_mem_types = False
|
||||
worker.region_names = []
|
||||
worker.region_num_blocks = []
|
||||
worker._region_is_mla = []
|
||||
worker.block_len_per_layer = []
|
||||
worker.block_stride_per_layer = []
|
||||
worker.device_id = 0
|
||||
worker.use_host_buffer = False
|
||||
worker.host_xfer_buffers = {}
|
||||
worker.device_kv_caches = {}
|
||||
worker.pp_size = 1
|
||||
worker.dcp_size = 1
|
||||
worker.pcp_size = 1
|
||||
worker.kv_buffer_device = "cuda"
|
||||
worker._layer_specs = {name: spec for name in caches}
|
||||
worker.kv_cache_config = KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=backing.nbytes,
|
||||
layers=[name],
|
||||
layer_stride=page_size,
|
||||
block_stride=block_stride,
|
||||
)
|
||||
for name in caches
|
||||
],
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
|
||||
transfer_topology = MagicMock()
|
||||
|
||||
with (
|
||||
patch.object(bw, "TransferTopology", return_value=transfer_topology),
|
||||
patch.object(bw, "compute_nixl_compatibility_hash", return_value="hash"),
|
||||
):
|
||||
worker.register_kv_caches(caches)
|
||||
|
||||
assert worker.region_group_ids == [-1]
|
||||
assert worker.block_stride_per_layer == [block_stride]
|
||||
assert worker.nixl_wrapper.registered[0][0] == [
|
||||
(backing.data_ptr(), backing.nbytes, 0, "")
|
||||
]
|
||||
expected_addrs = [
|
||||
backing.data_ptr() + block * block_stride for block in range(num_blocks)
|
||||
]
|
||||
assert worker.src_blocks_data[:, 0].tolist() == expected_addrs
|
||||
|
||||
metadata = msgspec.msgpack.decode(
|
||||
worker.xfer_handshake_metadata.agent_metadata_bytes,
|
||||
type=NixlAgentMetadata,
|
||||
)
|
||||
assert metadata.region_group_ids == [-1]
|
||||
assert metadata.region_num_blocks == [num_blocks]
|
||||
assert worker._block_ids_by_region(([0], [2]), worker.region_group_ids) == [[0, 2]]
|
||||
|
||||
|
||||
def _make_mla_hybrid_worker(local_block_size, kernel_block_size, num_logical_blocks):
|
||||
"""Build a real pull worker with a hybrid MLA + 2xKDA HMA layout."""
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import (
|
||||
|
||||
@@ -357,6 +357,12 @@ class _StubWriterWorker(NixlPushConnectorWorker):
|
||||
w.engine_id = "test-decode-engine"
|
||||
w._remote_agents = {}
|
||||
w._physical_blocks_per_logical_kv_block = 1
|
||||
w._uses_region_group_mapping = False
|
||||
w.region_group_ids = [0]
|
||||
w.dst_region_num_blocks = {}
|
||||
w.dst_region_group_ids = {}
|
||||
w.dst_uses_region_group_mapping = {}
|
||||
w.dst_region_mem_types = {}
|
||||
# Single non-hybrid attention group, matching the stub block id lists.
|
||||
w._has_mamba = False
|
||||
w._is_csa_linear = False
|
||||
@@ -1314,6 +1320,15 @@ class TestPushPrefixCaching:
|
||||
)
|
||||
}
|
||||
w.dst_num_blocks = {engine_id: 10_000, w.engine_id: 10_000}
|
||||
w.dst_region_num_blocks = {
|
||||
engine_id: [10_000],
|
||||
w.engine_id: [10_000],
|
||||
}
|
||||
w.dst_region_group_ids = {engine_id: [0], w.engine_id: [0]}
|
||||
w.dst_uses_region_group_mapping = {
|
||||
engine_id: False,
|
||||
w.engine_id: False,
|
||||
}
|
||||
w.dst_xfer_side_handles = {engine_id: {0: 5000}}
|
||||
w.src_xfer_handles_by_block_size = {16: 2000}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user