[KVConnector][NIXL] Support packed MLA KV layouts in pipeline-parallel push prefill (#50499)
Signed-off-by: zixi-qi <zixi@inferact.ai>
This commit is contained in:
@@ -16,10 +16,27 @@ from threading import Event, Lock
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import msgspec
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
NixlAgentMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
|
||||
NixlPushConnectorWorker,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheLayout,
|
||||
KVCacheTensor,
|
||||
MLAAttentionSpec,
|
||||
create_kv_cache_views,
|
||||
)
|
||||
|
||||
from .utils import create_vllm_config
|
||||
|
||||
|
||||
@@ -102,6 +119,189 @@ class _RecordingNixl:
|
||||
pass
|
||||
|
||||
|
||||
def _make_packed_mla_view_worker(
|
||||
layouts,
|
||||
block_stride,
|
||||
*,
|
||||
num_blocks=4,
|
||||
pp_size=1,
|
||||
backend_names=("FLASHMLA",),
|
||||
):
|
||||
"""Register real strided views; only NIXL and distributed runtime are fake."""
|
||||
block_size = 16
|
||||
raw = torch.zeros(num_blocks * block_stride, dtype=torch.int8)
|
||||
tensors, groups, caches = [], [], {}
|
||||
for name, (offset, page_size) in layouts.items():
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=page_size // block_size,
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
tensor = KVCacheTensor(
|
||||
size=raw.nbytes,
|
||||
layers=[name],
|
||||
layer_stride=page_size,
|
||||
block_stride=block_stride,
|
||||
offset=offset,
|
||||
)
|
||||
tensors.append(tensor)
|
||||
groups.append(KVCacheGroupSpec([name], spec))
|
||||
(caches[name],) = create_kv_cache_views(
|
||||
raw, spec, num_blocks, KVCacheLayout.BLHNC, tensor
|
||||
)
|
||||
|
||||
config = create_vllm_config(block_size=block_size)
|
||||
config.parallel_config.pipeline_parallel_size = pp_size
|
||||
config.kv_transfer_config.kv_role = "kv_producer"
|
||||
config.cache_config.kv_cache_layout = "BLHNC"
|
||||
backends = []
|
||||
for name in backend_names:
|
||||
backend = MagicMock()
|
||||
backend.get_supported_kernel_block_sizes.return_value = [block_size]
|
||||
backend.get_name.return_value = name
|
||||
backend.full_cls_name.return_value = f"fake.{name}"
|
||||
backends.append(backend)
|
||||
with (
|
||||
patch.object(bw, "NixlWrapper", _RecordingNixl),
|
||||
patch.object(bw, "get_tensor_model_parallel_rank", return_value=0),
|
||||
patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1),
|
||||
patch.object(bw, "get_current_attn_backends", return_value=backends),
|
||||
patch("threading.Thread"),
|
||||
):
|
||||
worker = NixlPushConnectorWorker(
|
||||
config,
|
||||
"producer" if pp_size > 1 else "consumer",
|
||||
KVCacheConfig(num_blocks, tensors, groups),
|
||||
)
|
||||
worker.use_mla = True
|
||||
worker.register_kv_caches(caches)
|
||||
return worker, raw
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize("num_layers", [1, 2])
|
||||
def test_packed_mla_pp1_push_peer_transfers_whole_rows(num_layers):
|
||||
layouts = {f"L{i}": (i * 128, 128) for i in range(num_layers)}
|
||||
stride = num_layers * 128
|
||||
worker, raw = _make_packed_mla_view_worker(layouts, stride)
|
||||
assert worker._registered_descs == [[(raw.data_ptr(), raw.nbytes, 0, "")]]
|
||||
assert worker.src_blocks_data.tolist() == [
|
||||
[raw.data_ptr() + block * stride, stride, 0] for block in range(4)
|
||||
]
|
||||
assert worker._transfer_layer_group_ids == ()
|
||||
metadata = msgspec.msgpack.decode(
|
||||
worker.xfer_handshake_metadata.agent_metadata_bytes, type=NixlAgentMetadata
|
||||
)
|
||||
assert metadata.packed_member_layouts == layouts
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
"remote_backends, compatible",
|
||||
[
|
||||
(("INDEXER", "FLASHMLA"), True),
|
||||
(("FLASHMLA", "OTHER"), False),
|
||||
(("FLASHMLA",), False),
|
||||
],
|
||||
)
|
||||
def test_packed_push_compatibility_hash_uses_all_backends_in_stable_order(
|
||||
remote_backends, compatible
|
||||
):
|
||||
producer, _ = _make_packed_mla_view_worker(
|
||||
{"L0": (0, 128)},
|
||||
128,
|
||||
pp_size=2,
|
||||
backend_names=("FLASHMLA", "INDEXER"),
|
||||
)
|
||||
consumer, _ = _make_packed_mla_view_worker(
|
||||
{"L0": (0, 128)}, 128, backend_names=remote_backends
|
||||
)
|
||||
assert (producer.compat_hash == consumer.compat_hash) is compatible
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize("p_num_blocks, d_num_blocks", [(4, 4), (4, 6), (6, 4)])
|
||||
def test_packed_mla_pp_pairs_asymmetric_strides_and_overlapping_layers(
|
||||
p_num_blocks, d_num_blocks
|
||||
):
|
||||
# Different cache groups overlay pages of different sizes at the same address.
|
||||
# PP also changes both the placement and the stride of the matching D pages.
|
||||
producer, p_raw = _make_packed_mla_view_worker(
|
||||
{"L2": (0, 128), "L2.swa": (0, 64), "L3": (128, 64)},
|
||||
192,
|
||||
num_blocks=p_num_blocks,
|
||||
pp_size=2,
|
||||
)
|
||||
consumer, d_raw = _make_packed_mla_view_worker(
|
||||
{"L0": (0, 128), "L2": (128, 128), "L2.swa": (0, 64), "L3": (64, 64)},
|
||||
256,
|
||||
num_blocks=d_num_blocks,
|
||||
)
|
||||
assert producer.block_len_per_layer == [128, 64, 64]
|
||||
assert producer._transfer_layer_group_ids == (0, 1, 2)
|
||||
assert producer.num_descs == 3 * p_num_blocks
|
||||
local_ids = producer._compute_desc_ids(
|
||||
[[1], [2], [3]],
|
||||
producer.num_blocks,
|
||||
None,
|
||||
1,
|
||||
region_num_blocks=producer.dst_region_num_blocks[producer.engine_id],
|
||||
)
|
||||
assert producer.src_blocks_data[local_ids].tolist() == [
|
||||
[p_raw.data_ptr() + 192, 128, 0],
|
||||
[p_raw.data_ptr() + 2 * 192, 64, 0],
|
||||
[p_raw.data_ptr() + 128 + 3 * 192, 64, 0],
|
||||
]
|
||||
# Each decoder TP rank receives the complete MLA page, without head slicing.
|
||||
for rank in (0, 1):
|
||||
metadata = msgspec.msgpack.decode(
|
||||
consumer.xfer_handshake_metadata.agent_metadata_bytes,
|
||||
type=NixlAgentMetadata,
|
||||
)
|
||||
producer.add_remote_agent(metadata, remote_tp_rank=rank, remote_tp_size=2)
|
||||
assert metadata.region_num_blocks == [d_num_blocks] * 3
|
||||
assert metadata.region_names == ["L2", "L2.swa", "L3"]
|
||||
remote_ids = producer._compute_desc_ids(
|
||||
[[3], [1], [2]],
|
||||
metadata.num_blocks,
|
||||
None,
|
||||
1,
|
||||
region_num_blocks=producer.dst_region_num_blocks[consumer.engine_id],
|
||||
)
|
||||
handle = producer.dst_xfer_side_handles[consumer.engine_id][rank]
|
||||
remote_descs = producer.nixl_wrapper.dlists[handle]
|
||||
assert len(remote_descs) == 3 * d_num_blocks
|
||||
assert remote_descs[remote_ids].tolist() == [
|
||||
[d_raw.data_ptr() + 128 + 3 * 256, 128, 0],
|
||||
[d_raw.data_ptr() + 256, 64, 0],
|
||||
[d_raw.data_ptr() + 64 + 2 * 256, 64, 0],
|
||||
]
|
||||
aligned = msgspec.msgpack.encode(metadata)
|
||||
producer._align_remote_regions_by_layer(metadata)
|
||||
assert msgspec.msgpack.encode(metadata) == aligned
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize("remote_block_size", [8, 32])
|
||||
def test_packed_mla_rejects_unequal_block_sizes_before_peer_registration(
|
||||
remote_block_size,
|
||||
):
|
||||
producer, _ = _make_packed_mla_view_worker({"L0": (0, 128)}, 128, pp_size=2)
|
||||
metadata = msgspec.msgpack.decode(
|
||||
producer.xfer_handshake_metadata.agent_metadata_bytes, type=NixlAgentMetadata
|
||||
)
|
||||
metadata.engine_id = "remote"
|
||||
metadata.block_size = remote_block_size
|
||||
with pytest.raises(NotImplementedError, match="identical P/D block sizes"):
|
||||
producer.add_remote_agent(metadata)
|
||||
assert len(producer.nixl_wrapper.dlists) == 1 # Only the local list exists.
|
||||
assert producer.tp_mappings == {}
|
||||
assert "remote" not in producer.dst_num_blocks
|
||||
with pytest.raises(KeyError):
|
||||
producer.transfer_topo.get_engine_info("remote")
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_local_descriptors_follow_each_region_pool_capacity():
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
@@ -200,8 +400,8 @@ def _register_overlaid_mla_worker(
|
||||
worker.src_xfer_handles_by_block_size = {}
|
||||
worker.kv_caches_base_addr = defaultdict(dict)
|
||||
worker._mamba_ssm_size = (0, 0)
|
||||
worker.kv_cache_layout = "NHD"
|
||||
worker.host_buffer_kv_cache_layout = "NHD"
|
||||
worker.kv_cache_layout = "LBNHC"
|
||||
worker.host_buffer_kv_cache_layout = "LBNHC"
|
||||
worker._physical_blocks_per_logical_kv_block = 1
|
||||
worker._logical_num_blocks = num_blocks
|
||||
worker.region_mem_types = []
|
||||
@@ -221,6 +421,7 @@ def _register_overlaid_mla_worker(
|
||||
worker.device_kv_caches = {}
|
||||
worker.pp_size = 2 if push_pp else 1
|
||||
worker._is_hma_required = True
|
||||
worker._has_packed_cache = False
|
||||
worker.dcp_size = 1
|
||||
worker.pcp_size = 1
|
||||
worker.kv_buffer_device = "cuda"
|
||||
|
||||
@@ -368,6 +368,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
|
||||
w._recving_transfers = defaultdict(list)
|
||||
w._is_hma_required = False
|
||||
w.xfer_stats = NixlKVConnectorStats()
|
||||
w._has_packed_cache = False
|
||||
w._reqs_to_process = set()
|
||||
w._reqs_to_send = {}
|
||||
w.consumer_notification_counts_by_req = defaultdict(int)
|
||||
@@ -1843,6 +1844,40 @@ def test_layer_metadata_round_trip():
|
||||
assert msgspec.msgpack.Decoder(NixlAgentMetadata).decode(encoded) == metadata
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"layouts, error",
|
||||
[
|
||||
({"L0": (-1, 64)}, "escapes its block"),
|
||||
({"L0": (0, 0)}, "escapes its block"),
|
||||
({"L0": (200, 64)}, "escapes its block"),
|
||||
({"L1": (0, 64)}, "has no layout"),
|
||||
({"L0": (0, 32)}, "page sizes must match"),
|
||||
],
|
||||
)
|
||||
def test_packed_layer_alignment_rejects_invalid_page_layout(layouts, error):
|
||||
worker = _layer_routing_worker([["L0"]], {"L0": 0})
|
||||
worker.block_len_per_layer = [64]
|
||||
metadata = _agent_metadata([["L0", "L1"]], [0x10000], [256])
|
||||
metadata.packed_member_layouts = layouts
|
||||
with pytest.raises(AssertionError, match=error):
|
||||
worker._align_remote_regions_by_layer(metadata)
|
||||
|
||||
|
||||
def test_layer_alignment_preserves_nonpacked_regions_alongside_packed_layers():
|
||||
worker = _layer_routing_worker([["a"], ["b"]], {"a": 0, "b": 1})
|
||||
worker.block_len_per_layer = [64, 128]
|
||||
metadata = _agent_metadata(
|
||||
[["a", "other"], ["b"]], [0x1000, 0x2000], [192, 128], [256, 128]
|
||||
)
|
||||
metadata.packed_member_layouts = {"a": (64, 64), "other": (128, 64)}
|
||||
|
||||
worker._align_remote_regions_by_layer(metadata)
|
||||
|
||||
assert metadata.kv_caches_base_addr == [0x1040, 0x2000]
|
||||
assert metadata.block_lens == [64, 128]
|
||||
assert metadata.block_strides == [256, 128]
|
||||
|
||||
|
||||
def test_layer_identity_gate_preserves_the_non_hma_path():
|
||||
assert _layer_routing_worker([["a"]], {"a": 0})._transfer_layer_region_indices == (
|
||||
0,
|
||||
@@ -1850,6 +1885,7 @@ def test_layer_identity_gate_preserves_the_non_hma_path():
|
||||
|
||||
# A base worker (pull) never routes by layer name.
|
||||
pull = object.__new__(NixlBaseConnectorWorker)
|
||||
pull.pp_size = 2
|
||||
pull._transfer_layer_names = ()
|
||||
pull._transfer_layer_region_indices = ()
|
||||
pull._transfer_layer_group_ids = ()
|
||||
|
||||
Reference in New Issue
Block a user