[KV Connector][NIXL] Register packed MLA KV rows as a single region (#61074)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Nicolò Lucchesi
2026-10-15 09:18:15 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 Yifan Qiao Codex
parent 80dd705c1d
commit 7020600b1d
4 changed files with 206 additions and 41 deletions
@@ -68,6 +68,7 @@ from vllm.v1.engine import EngineCoreRequest
from vllm.v1.engine.output_processor import OutputProcessor
from vllm.v1.kv_cache_interface import (
AttentionSpec,
CircularBufferSpec,
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
@@ -2523,8 +2524,46 @@ def test_register_kv_caches(
assert connector.connector_worker.block_size == block_size
def _dsv4_mla_spec(record_bytes: int, alignment: int, tokens_per_state: int):
return MLAAttentionSpec(
block_size=256,
num_kv_heads=1,
head_size=512,
dtype=torch.uint8,
cache_dtype_str="fp8_ds_mla",
tokens_per_state=tokens_per_state,
alignment=alignment,
model_version="deepseek_v4",
state_content_bytes=record_bytes,
)
@pytest.mark.parametrize(
"groups",
[
# V4 record: padded pages, so a layer view is not a contiguous page.
[(_dsv4_mla_spec(584, 576, 4), 4)],
# V4.1 MXFP8 record: unpadded pages in rows that the compressed caches
# and a compressor-state ring alias.
[
(_dsv4_mla_spec(528, 512, 2), 2),
(_dsv4_mla_spec(528, 512, 2), 2),
(
CircularBufferSpec(
block_size=8,
num_kv_heads=1,
head_size=8448,
head_size_v=0,
dtype=torch.uint8,
),
2,
),
],
],
ids=["v4-padded-pages", "v41-aliased-groups"],
)
def test_register_packed_dsv4_mla_cache_as_single_region(
default_vllm_config, dist_init
default_vllm_config, dist_init, groups
):
from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend
@@ -2537,27 +2576,21 @@ def test_register_packed_dsv4_mla_cache_as_single_region(
):
mock_backends.return_value = [TritonAttentionBackend]
num_blocks = 2
num_layers = 4
block_size = 256
spec = MLAAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=512,
dtype=torch.uint8,
cache_dtype_str="fp8_ds_mla",
tokens_per_state=4,
alignment=576,
model_version="deepseek_v4",
state_content_bytes=584,
)
layer_names = [f"layer{idx}" for idx in range(num_layers)]
specs = [spec for spec, _ in groups]
group_layers = [
[f"group{group_idx}.layer{idx}" for idx in range(num_layers)]
for group_idx, (_, num_layers) in enumerate(groups)
]
kv_cache_config = KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=[],
kv_cache_groups=[KVCacheGroupSpec(layer_names, spec)],
kv_cache_groups=[
KVCacheGroupSpec(layer_names, spec)
for layer_names, spec in zip(group_layers, specs)
],
)
vllm_config = create_vllm_config(attention_backend="TRITON_ATTN")
vllm_config.cache_config.block_size = block_size
vllm_config.cache_config.block_size = 256
vllm_config.cache_config.kv_cache_layout = "BLHNC"
connector = NixlConnector(vllm_config, KVConnectorRole.WORKER, kv_cache_config)
connector.connector_worker = FakeNixlConnectorWorker(
@@ -2571,25 +2604,44 @@ def test_register_packed_dsv4_mla_cache_as_single_region(
connector.connector_worker.nixl_wrapper = wrapper
wrapper.get_agent_metadata.return_value = b"fake_agent_metadata"
mock_thread.return_value.is_alive.return_value = False
connector.connector_worker.use_mla = True
# Every group packs its layers' pages into the same block rows.
(packed_block_len,) = {
len(layer_names) * spec.page_size_bytes
for layer_names, spec in zip(group_layers, specs)
}
raw = torch.zeros(
spec.page_size_bytes * num_blocks * num_layers,
packed_block_len * num_blocks,
dtype=torch.int8,
device=current_platform.device_type,
)
views = dense_kv_cache_views(
raw, spec, num_blocks, num_layers, KVCacheLayout.BLHNC
)
connector.register_kv_caches(dict(zip(layer_names, views)))
kv_caches: dict[str, torch.Tensor] = {}
for layer_names, spec in zip(group_layers, specs):
views = dense_kv_cache_views(
raw, spec, num_blocks, len(layer_names), KVCacheLayout.BLHNC
)
kv_caches.update(zip(layer_names, views))
connector.register_kv_caches(kv_caches)
# MLA pages transfer as whole rows; a ring page past the row start keeps
# its own region.
ring_pages = [
(cache.data_ptr(), spec.page_size_bytes)
for layer_names, spec in zip(group_layers, specs)
if isinstance(spec, CircularBufferSpec)
for cache in (kv_caches[name] for name in layer_names)
if cache.data_ptr() != raw.data_ptr()
]
_agent, block_runs = wrapper.prep_xfer_dlist.call_args[0]
packed_block_len = num_layers * spec.page_size_bytes
assert connector.connector_worker.kv_caches_base_addr[
connector.connector_worker.engine_id
][0] == [raw.data_ptr()]
# One packed region -> one run of num_blocks packed rows.
][0] == [raw.data_ptr()] + [addr for addr, _ in ring_pages]
assert block_runs.tolist() == [
[raw.data_ptr(), packed_block_len, 0, packed_block_len, num_blocks]
] + [
[addr, page_size, 0, packed_block_len, num_blocks]
for addr, page_size in ring_pages
]
@@ -25,6 +25,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
NixlAgentMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_worker import (
NixlPullConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
NixlPushConnectorWorker,
)
@@ -127,6 +130,7 @@ def _make_packed_mla_view_worker(
num_blocks=4,
pp_size=1,
backend_names=("FLASHMLA",),
worker_cls=NixlPushConnectorWorker,
):
"""Register real strided views; only NIXL and distributed runtime are fake."""
block_size = 16
@@ -170,7 +174,7 @@ def _make_packed_mla_view_worker(
patch.object(bw, "get_current_attn_backends", return_value=backends),
patch("threading.Thread"),
):
worker = NixlPushConnectorWorker(
worker = worker_cls(
config,
"producer" if pp_size > 1 else "consumer",
KVCacheConfig(num_blocks, tensors, groups),
@@ -181,11 +185,18 @@ def _make_packed_mla_view_worker(
@pytest.mark.cpu_test
@pytest.mark.parametrize(
"worker_cls", [NixlPullConnectorWorker, NixlPushConnectorWorker]
)
@pytest.mark.parametrize("num_layers", [1, 2])
def test_packed_mla_pp1_push_peer_transfers_whole_rows(num_layers):
def test_packed_mla_pp1_transfers_whole_rows(num_layers, worker_cls):
# BLHNC packs every layer's page of a block in one row: register the row
# once so a block is one descriptor, not one per layer.
layouts = {f"L{i}": (i * 128, 128) for i in range(num_layers)}
stride = num_layers * 128
worker, raw = _make_packed_mla_view_worker(layouts, stride)
worker, raw = _make_packed_mla_view_worker(layouts, stride, worker_cls=worker_cls)
assert worker.num_regions == 1
assert worker.num_descs == 4
assert worker._registered_descs == [[(raw.data_ptr(), raw.nbytes, 0, "")]]
# One packed region -> one run of 4 whole rows.
assert worker.src_blocks_data.tolist() == [[raw.data_ptr(), stride, 0, stride, 4]]
@@ -196,6 +207,35 @@ def test_packed_mla_pp1_push_peer_transfers_whole_rows(num_layers):
assert metadata.packed_member_layouts == layouts
@pytest.mark.cpu_test
def test_per_layer_peer_addresses_pages_inside_packed_rows():
# E.g. a HiSparse decoder reading a BLHNC prefiller's whole-row region.
remote, raw = _make_packed_mla_view_worker({"L0": (64, 128)}, 192)
metadata = msgspec.msgpack.decode(
remote.xfer_handshake_metadata.agent_metadata_bytes, type=NixlAgentMetadata
)
assert metadata.block_lens == [192]
local = SimpleNamespace(region_names=["L0"])
bw.NixlBaseConnectorWorker._split_remote_packed_rows(local, metadata)
assert metadata.kv_caches_base_addr == [raw.data_ptr() + 64]
assert metadata.block_lens == [128]
assert metadata.block_strides == [192]
assert metadata.region_names == ["L0"]
assert metadata.packed_member_layouts == {}
@pytest.mark.cpu_test
def test_packed_rows_reject_a_peer_packing_rows_differently():
producer, _ = _make_packed_mla_view_worker({"L0": (0, 128), "L1": (128, 64)}, 192)
consumer, _ = _make_packed_mla_view_worker({"L0": (64, 128), "L1": (0, 64)}, 192)
metadata = msgspec.msgpack.decode(
producer.xfer_handshake_metadata.agent_metadata_bytes, type=NixlAgentMetadata
)
metadata.engine_id = "remote"
with pytest.raises(RuntimeError, match="Packed KV block rows differ"):
consumer.add_remote_agent(metadata)
@pytest.mark.cpu_test
@pytest.mark.parametrize(
"remote_backends, compatible",
@@ -387,6 +387,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
w.region_group_ids = [0]
w._transfer_layer_names = ()
w._transfer_layer_region_indices = ()
w._packed_member_layouts = {}
w._transfer_layer_group_ids = ()
w.dst_region_num_blocks = {}
w.dst_region_group_ids = {}