[KVConnector][NIXL] Support attention-HMA layouts in pipeline-parallel push prefill (#50494)
Signed-off-by: zixi-qi <zixi@inferact.ai> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Yiliu Dong <91178480+qianlihuang@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Claude
Yiliu Dong
Codex
parent
c6fa1f05d1
commit
0fefffc934
@@ -918,6 +918,7 @@ def _make_mock_worker_for_desc_ids(
|
||||
worker = MagicMock(spec=NixlConnectorWorker)
|
||||
worker.num_regions = num_regions
|
||||
worker._uses_region_group_mapping = False
|
||||
worker._transfer_layer_group_ids = ()
|
||||
worker._has_mamba = has_mamba
|
||||
worker._group_spec_types = group_spec_types
|
||||
worker.block_len_per_layer = block_len_per_layer or [100]
|
||||
|
||||
@@ -12,6 +12,7 @@ mid-decode (silent corruption of an unrelated request).
|
||||
"""
|
||||
|
||||
from collections import defaultdict
|
||||
from threading import Event, Lock
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import numpy as np
|
||||
@@ -112,6 +113,7 @@ def test_local_descriptors_follow_each_region_pool_capacity():
|
||||
worker.block_len_per_layer = [16, 16]
|
||||
worker.block_stride_per_layer = [16, 16]
|
||||
worker.region_num_blocks = [2, 3]
|
||||
worker._transfer_layer_region_indices = ()
|
||||
|
||||
descriptors = worker._build_fa_local([100, 1000], block_size_ratio=1)
|
||||
|
||||
@@ -119,7 +121,8 @@ def test_local_descriptors_follow_each_region_pool_capacity():
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_overlaid_transfer_groups_share_region_geometry():
|
||||
@pytest.mark.parametrize("push_pp", [False, True])
|
||||
def test_overlaid_transfer_groups_share_region_geometry(push_pp):
|
||||
"""Groups overlaid on one allocation share its transfer region."""
|
||||
import msgspec
|
||||
|
||||
@@ -127,6 +130,9 @@ def test_overlaid_transfer_groups_share_region_geometry():
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
NixlAgentMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
|
||||
NixlPushConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
NixlConnectorWorker,
|
||||
)
|
||||
@@ -153,7 +159,14 @@ def test_overlaid_transfer_groups_share_region_geometry():
|
||||
}
|
||||
groups = [KVCacheGroupSpec([layer_name], spec) for layer_name in caches]
|
||||
|
||||
worker = object.__new__(NixlConnectorWorker)
|
||||
worker_cls = NixlPushConnectorWorker if push_pp else NixlConnectorWorker
|
||||
worker = object.__new__(worker_cls)
|
||||
if push_pp:
|
||||
worker._push_writer_stop = Event()
|
||||
worker._push_writer_wake = Event()
|
||||
worker._push_writer_thread = MagicMock()
|
||||
worker._sending_transfers_lock = Lock()
|
||||
worker._sending_transfers = defaultdict(list)
|
||||
worker.tp_rank = 0
|
||||
worker.world_size = 1
|
||||
worker.transfer_tp_rank = 0
|
||||
@@ -198,6 +211,9 @@ def test_overlaid_transfer_groups_share_region_geometry():
|
||||
worker._desc_is_dram_by_block_size = {}
|
||||
worker._desc_pos_by_block_size = {}
|
||||
worker._dram_src_handles_by_block_size = {}
|
||||
worker._transfer_layer_names = ()
|
||||
worker._transfer_layer_region_indices = ()
|
||||
worker._transfer_layer_group_ids = ()
|
||||
worker._region_is_mla = []
|
||||
worker.block_len_per_layer = []
|
||||
worker.block_stride_per_layer = []
|
||||
@@ -205,7 +221,8 @@ def test_overlaid_transfer_groups_share_region_geometry():
|
||||
worker.use_host_buffer = False
|
||||
worker.host_xfer_buffers = {}
|
||||
worker.device_kv_caches = {}
|
||||
worker.pp_size = 1
|
||||
worker.pp_size = 2 if push_pp else 1
|
||||
worker._is_hma_required = True
|
||||
worker.dcp_size = 1
|
||||
worker.pcp_size = 1
|
||||
worker.kv_buffer_device = "cuda"
|
||||
@@ -240,7 +257,13 @@ def test_overlaid_transfer_groups_share_region_geometry():
|
||||
expected_addrs = [
|
||||
backing.data_ptr() + block * block_stride for block in range(num_blocks)
|
||||
]
|
||||
assert worker.src_blocks_data[:, 0].tolist() == expected_addrs
|
||||
num_desc_regions = 2 if push_pp else 1
|
||||
assert worker.src_blocks_data[:, 0].tolist() == expected_addrs * num_desc_regions
|
||||
assert worker.num_descs == num_blocks * num_desc_regions
|
||||
assert (
|
||||
worker.dst_region_num_blocks[worker.engine_id]
|
||||
== [num_blocks] * num_desc_regions
|
||||
)
|
||||
|
||||
metadata = msgspec.msgpack.decode(
|
||||
worker.xfer_handshake_metadata.agent_metadata_bytes,
|
||||
@@ -248,6 +271,7 @@ def test_overlaid_transfer_groups_share_region_geometry():
|
||||
)
|
||||
assert metadata.region_group_ids == [-1]
|
||||
assert metadata.region_num_blocks == [num_blocks]
|
||||
assert metadata.region_members == ([["layer.0", "layer.1"]] if push_pp else [])
|
||||
assert worker._block_ids_by_region(([0], [2]), worker.region_group_ids) == [[0, 2]]
|
||||
|
||||
|
||||
|
||||
@@ -32,9 +32,13 @@ from unittest.mock import MagicMock, patch
|
||||
import msgspec
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import TransferTopology
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorTransferResults,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import (
|
||||
NixlBaseConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
PUSH_REG_NOTIF_PREFIX,
|
||||
NixlAgentMetadata,
|
||||
@@ -43,6 +47,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
|
||||
NixlPushConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import TPMapping
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import (
|
||||
get_base_request_id,
|
||||
)
|
||||
@@ -354,6 +359,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
|
||||
w._failed_recv_reqs = queue.Queue()
|
||||
w._recv_failures = set()
|
||||
w._recving_transfers = defaultdict(list)
|
||||
w._is_hma_required = False
|
||||
w._reqs_to_process = set()
|
||||
w._reqs_to_send = {}
|
||||
w.consumer_notification_counts_by_req = defaultdict(int)
|
||||
@@ -361,12 +367,16 @@ class _StubWriterWorker(NixlPushConnectorWorker):
|
||||
w.pcp_rank = 0
|
||||
w.pcp_dcp_sharded = False
|
||||
w.world_size = 1
|
||||
w.pp_size = 1
|
||||
w.engine_id = "test-decode-engine"
|
||||
w._remote_agents = {}
|
||||
w._handshake_lock = threading.RLock()
|
||||
w._physical_blocks_per_logical_kv_block = 1
|
||||
w._uses_region_group_mapping = False
|
||||
w.region_group_ids = [0]
|
||||
w._transfer_layer_names = ()
|
||||
w._transfer_layer_region_indices = ()
|
||||
w._transfer_layer_group_ids = ()
|
||||
w.dst_region_num_blocks = {}
|
||||
w.dst_region_group_ids = {}
|
||||
w.dst_uses_region_group_mapping = {}
|
||||
@@ -922,12 +932,20 @@ class TestPushWriterNotifs:
|
||||
# Both handles were still cleaned up.
|
||||
assert request_id not in w._sending_transfers
|
||||
|
||||
def test_completed_push_send_waits_for_consumer_notif(self):
|
||||
@pytest.mark.parametrize(("pp_size", "is_hma"), [(1, False), (2, True)])
|
||||
def test_completed_push_send_waits_for_consumer_notif(self, pp_size, is_hma):
|
||||
"""P-side sends are completed by consumer notifs / lease expiry,
|
||||
not by local WRITE-handle completion."""
|
||||
w = self._pollable_worker()
|
||||
w.pp_size = pp_size
|
||||
w._is_hma_required = is_hma
|
||||
request_id = self._make_sending_req(w)
|
||||
w.nixl_wrapper.check_xfer_state.return_value = "DONE"
|
||||
w.nixl_wrapper.check_xfer_state.side_effect = ["DONE", "PROC", "DONE"]
|
||||
|
||||
results = w.get_transfer_results()
|
||||
assert request_id not in results.finished_sending
|
||||
assert w._sending_transfers[request_id] == [102]
|
||||
assert request_id in w._reqs_to_send
|
||||
|
||||
results = w.get_transfer_results()
|
||||
|
||||
@@ -940,6 +958,12 @@ class TestPushWriterNotifs:
|
||||
results = w.get_transfer_results()
|
||||
assert request_id in results.finished_sending
|
||||
assert request_id not in w._reqs_to_send
|
||||
assert request_id not in w._reqs_to_process
|
||||
assert w._evict_finished_inbox.get_nowait() == request_id
|
||||
assert w.get_transfer_results().finished_sending == set()
|
||||
assert w._evict_finished_inbox.empty()
|
||||
assert w.nixl_wrapper.release_xfer_handle.call_count == 2
|
||||
w.xfer_stats.record_kv_expired_req.assert_not_called()
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- #
|
||||
@@ -1576,3 +1600,375 @@ class TestPushPrefixCaching:
|
||||
local, remote = self._written_block_ids(w)
|
||||
assert local == [10, 11, 12]
|
||||
assert remote == [500, 501, 502]
|
||||
|
||||
@pytest.mark.parametrize("layer_name_routing", [False, True])
|
||||
def test_different_region_groups_require_layer_routing(self, layer_name_routing):
|
||||
"""A layer mapped to a pooled remote region can still receive its blocks."""
|
||||
w, engine_id = self._worker_driving_xfer()
|
||||
w.dst_region_group_ids[engine_id] = [-1]
|
||||
if layer_name_routing:
|
||||
w._transfer_layer_group_ids = (0,)
|
||||
reg = _registration_data("req-groups", local_block_ids=([500, 501],))
|
||||
|
||||
if not layer_name_routing:
|
||||
with pytest.raises(NotImplementedError, match="cache-group layouts"):
|
||||
NixlPushConnectorWorker._do_start_push_kv(
|
||||
w, "req-groups", ([10, 11],), reg
|
||||
)
|
||||
w.nixl_wrapper.make_prepped_xfer.assert_not_called()
|
||||
return
|
||||
|
||||
NixlPushConnectorWorker._do_start_push_kv(w, "req-groups", ([10, 11],), reg)
|
||||
|
||||
local, remote = self._written_block_ids(w)
|
||||
assert local == [10, 11]
|
||||
assert remote == [500, 501]
|
||||
|
||||
|
||||
def _agent_metadata(
|
||||
region_layers: list[list[str]],
|
||||
base_addresses: list[int],
|
||||
block_lens: list[int],
|
||||
block_strides: list[int] | None = None,
|
||||
) -> NixlAgentMetadata:
|
||||
return NixlAgentMetadata(
|
||||
engine_id="remote-engine",
|
||||
agent_metadata=b"agent",
|
||||
kv_caches_base_addr=base_addresses,
|
||||
device_id=7,
|
||||
num_blocks=2,
|
||||
block_lens=block_lens,
|
||||
block_strides=list(block_lens if block_strides is None else block_strides),
|
||||
kv_cache_layout="LBHNC",
|
||||
block_size=16,
|
||||
ssm_sizes=(0, 0),
|
||||
attn_backend_name="FLASH_ATTN",
|
||||
physical_blocks_per_logical_kv_block=1,
|
||||
region_members=region_layers,
|
||||
)
|
||||
|
||||
|
||||
def _layer_routing_worker(
|
||||
region_layers: list[list[str]],
|
||||
group_by_layer: dict[str, int],
|
||||
pp_size: int = 2,
|
||||
is_hma: bool = True,
|
||||
) -> _StubWriterWorker:
|
||||
worker = _StubWriterWorker.fresh()
|
||||
worker.pp_size = pp_size
|
||||
worker.dcp_size = 1
|
||||
worker.block_size = 16
|
||||
worker._has_mamba = False
|
||||
worker._is_hma_required = is_hma
|
||||
worker.kv_cache_config = MagicMock(transfer_group_index_by_layer=group_by_layer)
|
||||
worker.transfer_topo = MagicMock()
|
||||
worker._set_region_layers(region_layers)
|
||||
return worker
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("region_num_blocks", "expected"),
|
||||
[(None, [1, 2, 15, 21, 22]), ([4, 7, 5], [1, 2, 9, 12, 13])],
|
||||
)
|
||||
def test_layer_group_ids_route_descriptor_blocks(region_num_blocks, expected):
|
||||
worker = _layer_routing_worker(
|
||||
[["a", "a.swa"], ["b"]], {"a": 0, "a.swa": 1, "b": 0}
|
||||
)
|
||||
desc_ids = worker._compute_desc_ids(
|
||||
block_ids=[[1, 2], [5]],
|
||||
dst_num_blocks=10,
|
||||
block_size_ratio=None,
|
||||
physical_blocks_per_logical=1,
|
||||
region_num_blocks=region_num_blocks,
|
||||
)
|
||||
assert desc_ids.tolist() == expected
|
||||
|
||||
|
||||
def test_layer_metadata_round_trip():
|
||||
metadata = _agent_metadata([["L0", "L1"]], [0x10000], [256])
|
||||
metadata.region_num_blocks = [4]
|
||||
metadata.region_group_ids = [-1]
|
||||
metadata.region_names = ["L0"]
|
||||
metadata.region_mem_types = ["VRAM"]
|
||||
|
||||
encoded = msgspec.msgpack.encode(metadata)
|
||||
assert msgspec.msgpack.Decoder(NixlAgentMetadata).decode(encoded) == metadata
|
||||
|
||||
|
||||
def test_layer_identity_gate_preserves_the_non_hma_path():
|
||||
assert _layer_routing_worker([["a"]], {"a": 0})._transfer_layer_region_indices == (
|
||||
0,
|
||||
)
|
||||
|
||||
# A base worker (pull) never routes by layer name.
|
||||
pull = object.__new__(NixlBaseConnectorWorker)
|
||||
pull._transfer_layer_names = ()
|
||||
pull._transfer_layer_region_indices = ()
|
||||
pull._transfer_layer_group_ids = ()
|
||||
pull._set_region_layers([["a"]])
|
||||
assert pull._transfer_layer_region_indices == ()
|
||||
|
||||
# A non-HMA local layout does not require layer-name routing.
|
||||
assert (
|
||||
_layer_routing_worker(
|
||||
[["a"]], {"a": 0}, is_hma=False
|
||||
)._transfer_layer_region_indices
|
||||
== ()
|
||||
)
|
||||
|
||||
# PP=1 has congruent local/remote regions and keeps the region route even
|
||||
# when its allocator is hybrid.
|
||||
assert (
|
||||
_layer_routing_worker(
|
||||
[["a"]], {"a": 0}, pp_size=1
|
||||
)._transfer_layer_region_indices
|
||||
== ()
|
||||
)
|
||||
|
||||
|
||||
def test_layer_alignment_fails_loud_when_remote_omits_layers():
|
||||
# Falling back to region-index routing here would silently transfer stale
|
||||
# KV, so an unannounced peer must fail the handshake instead.
|
||||
worker = _layer_routing_worker([["a"]], {"a": 0})
|
||||
with pytest.raises(AssertionError, match="no region_members"):
|
||||
worker._align_remote_regions_by_layer(_agent_metadata([], [0xA000], [128]))
|
||||
|
||||
|
||||
def test_layer_alignment_expands_pooled_regions():
|
||||
worker = _layer_routing_worker(
|
||||
[["a", "a.swa"], ["b"]], {"a": 0, "a.swa": 1, "b": 0}
|
||||
)
|
||||
assert worker._transfer_layer_names == ("a", "a.swa", "b")
|
||||
assert worker._transfer_layer_region_indices == (0, 0, 1)
|
||||
assert worker._transfer_layer_group_ids == (0, 1, 0)
|
||||
|
||||
metadata = _agent_metadata([["a", "a.swa"], ["b"]], [0xA000, 0xB000], [128, 128])
|
||||
metadata.region_num_blocks = [4, 6]
|
||||
metadata.region_group_ids = [-1, 0]
|
||||
metadata.region_names = ["a", "b"]
|
||||
metadata.region_mem_types = ["VRAM", "VRAM"]
|
||||
worker._align_remote_regions_by_layer(metadata)
|
||||
|
||||
assert metadata.kv_caches_base_addr == [0xA000, 0xA000, 0xB000]
|
||||
assert metadata.block_lens == [128, 128, 128]
|
||||
assert metadata.block_strides == [128, 128, 128]
|
||||
assert metadata.region_members == [["a"], ["a.swa"], ["b"]]
|
||||
assert metadata.region_num_blocks == [4, 4, 6]
|
||||
assert metadata.region_group_ids == [-1, -1, 0]
|
||||
assert metadata.region_names == ["a", "a.swa", "b"]
|
||||
assert metadata.region_mem_types == ["VRAM", "VRAM", "VRAM"]
|
||||
|
||||
|
||||
def test_layer_alignment_filters_and_reorders_a_pp_stage():
|
||||
worker = _layer_routing_worker([["l2"], ["l3"]], {"l2": 0, "l3": 1})
|
||||
metadata = _agent_metadata(
|
||||
[["l2"], ["l0"], ["l3"], ["l1"]],
|
||||
[0xC000, 0xA000, 0xD000, 0xB000],
|
||||
[65536, 65536, 32768, 32768],
|
||||
[131072, 131072, 65536, 65536],
|
||||
)
|
||||
metadata.region_num_blocks = [4, 6, 8, 10]
|
||||
metadata.region_group_ids = [0, 0, 1, 1]
|
||||
metadata.region_names = ["l2", "l0", "l3", "l1"]
|
||||
metadata.region_mem_types = ["VRAM"] * 4
|
||||
|
||||
worker._align_remote_regions_by_layer(metadata)
|
||||
|
||||
assert worker._transfer_layer_region_indices == (0, 1)
|
||||
assert worker._transfer_layer_group_ids == (0, 1)
|
||||
assert metadata.kv_caches_base_addr == [0xC000, 0xD000]
|
||||
assert metadata.block_lens == [65536, 32768]
|
||||
assert metadata.block_strides == [131072, 65536]
|
||||
assert metadata.region_num_blocks == [4, 8]
|
||||
assert metadata.region_group_ids == [0, 1]
|
||||
assert metadata.region_names == ["l2", "l3"]
|
||||
assert metadata.region_mem_types == ["VRAM", "VRAM"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("local_num_blocks", "remote_num_blocks"),
|
||||
[([4, 4], [4, 4, 4]), ([4, 6], [3, 5, 7])],
|
||||
)
|
||||
def test_layer_descriptors_pair_layers_across_asymmetric_pp_split(
|
||||
local_num_blocks, remote_num_blocks
|
||||
):
|
||||
"""A PP split can leave each stage a different mix of attention types.
|
||||
|
||||
Stage 1 owns L3 (full) pooled with L4 (sliding) in region 0, and L5 (full)
|
||||
alone in region 1. The consumer holds every layer and pools them
|
||||
differently, so region indices cannot be paired positionally.
|
||||
"""
|
||||
worker = _layer_routing_worker([["L3", "L4"], ["L5"]], {"L3": 0, "L4": 1, "L5": 0})
|
||||
worker.block_len_per_layer = [128, 128]
|
||||
worker.block_stride_per_layer = [256, 512]
|
||||
worker._region_is_mla = [False, False]
|
||||
worker.num_blocks = 4
|
||||
worker.region_num_blocks = local_num_blocks
|
||||
worker.device_id = 0
|
||||
|
||||
consumer = _agent_metadata(
|
||||
[["L0", "L1"], ["L3", "L2"], ["L5", "L4"]],
|
||||
[0xA000, 0xB000, 0xC000],
|
||||
[128, 128, 128],
|
||||
[256, 512, 1024],
|
||||
)
|
||||
consumer.num_blocks = 4
|
||||
consumer.region_num_blocks = remote_num_blocks
|
||||
worker._align_remote_regions_by_layer(consumer)
|
||||
|
||||
# L3 -> remote region 1, L4 -> 2, L5 -> 2. Pairing by index would have sent
|
||||
# L5 to region 1 and L4 to region 2's sibling.
|
||||
assert consumer.kv_caches_base_addr == [0xB000, 0xC000, 0xC000]
|
||||
|
||||
plan = TPMapping(((0,), (0,)), (0,), {0: 0}, 0)
|
||||
local_descs = worker._build_fa_local([0x1000, 0x2000], block_size_ratio=1)
|
||||
remote_descs = worker._build_fa_remote(plan, consumer, block_size_ratio=1)
|
||||
local_counts = [local_num_blocks[i] for i in worker._transfer_layer_region_indices]
|
||||
local_ids = worker._compute_desc_ids(
|
||||
[[1, 2], [3]], 4, None, 1, region_num_blocks=local_counts
|
||||
)
|
||||
remote_ids = worker._compute_desc_ids(
|
||||
[[0, 1], [2]], 4, None, 1, region_num_blocks=consumer.region_num_blocks
|
||||
)
|
||||
assert len(local_descs) == sum(local_counts)
|
||||
assert len(remote_descs) == sum(consumer.region_num_blocks)
|
||||
|
||||
# Each pair addresses the same layer's blocks despite different pooling
|
||||
# and strides. L3 and L5 use group 0; L4 uses group 1.
|
||||
assert local_descs[local_ids].tolist() == [
|
||||
[0x1100, 128, 0],
|
||||
[0x1200, 128, 0],
|
||||
[0x1300, 128, 0],
|
||||
[0x2200, 128, 0],
|
||||
[0x2400, 128, 0],
|
||||
]
|
||||
assert remote_descs[remote_ids].tolist() == [
|
||||
[0xB000, 128, 7],
|
||||
[0xB200, 128, 7],
|
||||
[0xC800, 128, 7],
|
||||
[0xC000, 128, 7],
|
||||
[0xC400, 128, 7],
|
||||
]
|
||||
|
||||
|
||||
def test_layer_alignment_is_canonical_across_remote_orderings():
|
||||
local, groups = [["x"], ["y"]], {"x": 0, "y": 1}
|
||||
rank0 = _agent_metadata([["x"], ["y"]], [0x1000, 0x2000], [64, 128], [256, 512])
|
||||
rank1 = _agent_metadata([["y"], ["x"]], [0x2000, 0x1000], [128, 64], [512, 256])
|
||||
|
||||
_layer_routing_worker(local, groups)._align_remote_regions_by_layer(rank0)
|
||||
_layer_routing_worker(local, groups)._align_remote_regions_by_layer(rank1)
|
||||
|
||||
assert rank0.kv_caches_base_addr == rank1.kv_caches_base_addr == [0x1000, 0x2000]
|
||||
assert rank0.block_lens == rank1.block_lens == [64, 128]
|
||||
assert rank0.block_strides == rank1.block_strides == [256, 512]
|
||||
|
||||
|
||||
def test_layer_alignment_is_idempotent():
|
||||
worker = _layer_routing_worker(
|
||||
[["a", "a.swa"], ["b"]], {"a": 0, "a.swa": 1, "b": 0}
|
||||
)
|
||||
metadata = _agent_metadata([["a", "a.swa"], ["b"]], [0xA000, 0xB000], [128, 128])
|
||||
metadata.region_num_blocks = [4, 6]
|
||||
metadata.region_group_ids = [-1, 0]
|
||||
metadata.region_names = ["a", "b"]
|
||||
metadata.region_mem_types = ["VRAM", "VRAM"]
|
||||
|
||||
worker._align_remote_regions_by_layer(metadata)
|
||||
aligned = msgspec.msgpack.encode(metadata)
|
||||
worker._align_remote_regions_by_layer(metadata)
|
||||
|
||||
assert msgspec.msgpack.encode(metadata) == aligned
|
||||
|
||||
|
||||
def test_layer_alignment_rejects_missing_local_layer():
|
||||
worker = _layer_routing_worker([["l0"], ["l1"]], {"l0": 0, "l1": 1})
|
||||
metadata = _agent_metadata([["l0"]], [0xA000], [128])
|
||||
|
||||
with pytest.raises(AssertionError, match="missing locally owned layers"):
|
||||
worker._align_remote_regions_by_layer(metadata)
|
||||
|
||||
|
||||
def test_layer_alignment_rejects_duplicate_remote_layer():
|
||||
worker = _layer_routing_worker([["a"]], {"a": 0})
|
||||
metadata = _agent_metadata([["a"], ["a"]], [0xA000, 0xB000], [128, 128])
|
||||
|
||||
with pytest.raises(AssertionError, match="in multiple regions"):
|
||||
worker._align_remote_regions_by_layer(metadata)
|
||||
|
||||
|
||||
def test_layer_alignment_rejects_inconsistent_remote_metadata():
|
||||
worker = _layer_routing_worker([["a"]], {"a": 0})
|
||||
metadata = _agent_metadata([["a"], ["b"]], [0xA000], [128])
|
||||
|
||||
with pytest.raises(AssertionError, match="lengths disagree"):
|
||||
worker._align_remote_regions_by_layer(metadata)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field",
|
||||
["region_num_blocks", "region_group_ids", "region_names", "region_mem_types"],
|
||||
)
|
||||
def test_layer_alignment_rejects_inconsistent_region_geometry(field):
|
||||
worker = _layer_routing_worker([["a"]], {"a": 0})
|
||||
metadata = _agent_metadata([["a"], ["b"]], [0xA000, 0xB000], [128, 128])
|
||||
setattr(metadata, field, [])
|
||||
original = msgspec.msgpack.encode(metadata)
|
||||
|
||||
with pytest.raises(AssertionError, match="lengths disagree"):
|
||||
worker._align_remote_regions_by_layer(metadata)
|
||||
assert msgspec.msgpack.encode(metadata) == original
|
||||
|
||||
|
||||
def test_set_region_layers_rejects_duplicate_local_layer():
|
||||
with pytest.raises(AssertionError, match="spans multiple NIXL regions"):
|
||||
_layer_routing_worker([["a"], ["a"]], {"a": 0})
|
||||
|
||||
|
||||
def test_set_region_layers_rejects_layer_outside_any_kv_group():
|
||||
with pytest.raises(AssertionError, match="outside any local group"):
|
||||
_layer_routing_worker([["a"], ["b"]], {"a": 0})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("local_block_size", "remote_block_size", "remote_tp_size", "error"),
|
||||
[
|
||||
(32, 16, 1, "identical P/D block sizes"),
|
||||
(16, 32, 1, "identical P/D block sizes"),
|
||||
(16, 16, 2, "decode TP greater"),
|
||||
],
|
||||
)
|
||||
def test_layer_handshake_rejects_unsupported_geometry(
|
||||
local_block_size: int, remote_block_size: int, remote_tp_size: int, error: str
|
||||
):
|
||||
"""Reject unsupported peers without registering agents or transfer state."""
|
||||
metadata = _agent_metadata([["a"]], [0xA000], [128])
|
||||
metadata.block_size = remote_block_size
|
||||
worker = _layer_routing_worker([["a"]], {"a": 0})
|
||||
worker.block_size = local_block_size
|
||||
worker.block_len_per_layer = [128]
|
||||
worker.use_mla = False
|
||||
worker.nixl_wrapper = MagicMock()
|
||||
worker.kv_caches_base_addr = defaultdict(dict)
|
||||
worker.dst_num_blocks = {}
|
||||
worker.tp_mappings = {}
|
||||
worker.transfer_topo = TransferTopology(
|
||||
tp_rank=0,
|
||||
tp_size=1,
|
||||
block_size=local_block_size,
|
||||
engine_id=worker.engine_id,
|
||||
is_mla=False,
|
||||
is_mamba=False,
|
||||
total_num_kv_heads=8,
|
||||
attn_backends=[],
|
||||
)
|
||||
|
||||
with pytest.raises(NotImplementedError, match=error):
|
||||
worker.add_remote_agent(metadata, remote_tp_size=remote_tp_size)
|
||||
worker.nixl_wrapper.add_remote_agent.assert_not_called()
|
||||
worker.nixl_wrapper.prep_xfer_dlist.assert_not_called()
|
||||
assert not worker.tp_mappings
|
||||
assert not worker.dst_num_blocks
|
||||
assert not worker.kv_caches_base_addr
|
||||
with pytest.raises(KeyError):
|
||||
worker.transfer_topo.get_engine_info(metadata.engine_id)
|
||||
|
||||
Reference in New Issue
Block a user