[Bugfix][KV Connector][NIXL] Transfer the whole PLE short-conv page in disaggregated P/D (#59997)

Signed-off-by: Alberto Massidda <amassidda@nvidia.com>
Co-authored-by: Alberto Massidda <amassidda@nvidia.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Artem Perevedentsev <aperevedents@nvidia.com>
This commit is contained in:
Alberto Massidda
2026-10-08 23:58:13 +04:00
committed by GitHub
co-authored by Alberto Massidda Claude Fable 5.1 Artem Perevedentsev
parent 1afb029839
commit 72d59adc2c
3 changed files with 77 additions and 16 deletions
@@ -1245,8 +1245,12 @@ def test_mismatched_mla_kernel_page_rejected_for_mla_hybrid():
worker.add_remote_agent(meta_r, remote_tp_rank=0, remote_tp_size=2)
def _make_csa_linear_ple_worker(scratch_aliases: str = "compressed"):
"""``scratch_aliases`` selects which pages the compressor ring overlays."""
def _make_csa_linear_ple_worker(
scratch_aliases: str = "compressed", ple_page_size: int = 256
):
"""``scratch_aliases`` selects which pages the compressor ring overlays;
``ple_page_size`` widens the PLE page beyond the page it shares a region
with (block-outer layout: both start at byte 0 of the block)."""
from unittest.mock import MagicMock
from vllm.config import set_current_vllm_config
@@ -1305,7 +1309,7 @@ def _make_csa_linear_ple_worker(scratch_aliases: str = "compressed"):
block_size=1,
shapes=((6, 3),),
dtypes=(torch.float16,),
page_size_padded=256,
page_size_padded=ple_page_size,
mamba_type=MambaAttentionBackendEnum.SHORT_CONV,
tp_replicated=True,
)
@@ -1339,17 +1343,22 @@ def _make_csa_linear_ple_worker(scratch_aliases: str = "compressed"):
("compressed.0",),
("compressed.1",),
)
region_size = 512
page_size = 256
# Region 0 hosts the PLE page: its blocks are as wide as the widest page.
block_strides = [max(page_size, ple_page_size)] + [page_size] * (
len(tensor_regions) - 1
)
region_sizes = [2 * stride for stride in block_strides]
region_offsets = [sum(region_sizes[:index]) for index in range(len(region_sizes))]
kv_cache_config = KVCacheConfig(
num_blocks=2,
kv_cache_tensors=[
KVCacheTensor(
size=len(tensor_regions) * region_size,
size=sum(region_sizes),
layers=[layer_name],
layer_stride=region_size,
block_stride=page_size,
offset=region_index * region_size,
layer_stride=region_sizes[region_index],
block_stride=block_strides[region_index],
offset=region_offsets[region_index],
)
for region_index, layer_names in enumerate(tensor_regions)
for layer_name in layer_names
@@ -1386,10 +1395,14 @@ def _make_csa_linear_ple_worker(scratch_aliases: str = "compressed"):
set_current_vllm_config(vllm_config),
):
worker = NixlConnectorWorker(vllm_config, "local-engine", kv_cache_config)
tensors = [torch.zeros((2, 256), dtype=torch.uint8) for _ in range(4)]
tensors = [
torch.zeros((2, stride), dtype=torch.uint8) for stride in block_strides
]
worker.register_kv_caches(
{
layer_name: tensors[region_index]
layer_name: tensors[region_index][
:, : (ple_page_size if layer_name == "mamba.ple" else page_size)
]
for region_index, layer_names in enumerate(tensor_regions)
for layer_name in layer_names
}
@@ -1523,6 +1536,7 @@ def test_csa_linear_remote_ple_is_copied_whole():
kv_cache_layout="HND",
block_size=4,
ssm_sizes=(24, 32),
ple_block_len=256,
attn_backend_name="test",
physical_blocks_per_logical_kv_block=1,
)
@@ -1535,7 +1549,7 @@ def test_csa_linear_remote_ple_is_copied_whole():
assert descriptors[-2:, 0].tolist() == [0x10000, 0x10100]
assert descriptors[-2:, 1].tolist() == [256, 256]
metadata.block_lens[0] = 128
metadata.ple_block_len = 128
with pytest.raises(ValueError, match="PLE pages require identical"):
worker._build_mamba_remote(
metadata,
@@ -1544,6 +1558,30 @@ def test_csa_linear_remote_ple_is_copied_whole():
)
@pytest.mark.cpu_test
def test_csa_linear_ple_page_wider_than_the_shared_region_page():
"""The PLE page is discovered in the region of the main KV page registered
first (both start at byte 0 of the block); its descriptors must still cover
the whole PLE page, not the region's block_len."""
worker = _make_csa_linear_ple_worker(ple_page_size=384)
assert worker._ple_region_index == 0
assert worker.block_len_per_layer[0] == 256
assert worker.block_stride_per_layer[0] == 384
assert worker._ple_block_len == 384
bases = [0x10000, 0x20000, 0x30000, 0x40000]
mamba = worker._build_mamba_local(bases)
assert mamba[-2:, 0].tolist() == [0x10000, 0x10000 + 384]
assert mamba[-2:, 1].tolist() == [384, 384]
metadata = msgspec.msgpack.decode(
worker.xfer_handshake_metadata.agent_metadata_bytes, type=NixlAgentMetadata
)
assert metadata.ple_block_len == 384
assert metadata.block_lens[0] == 256
def _make_ring_worker():
"""Paged MLA group plus a per-layer ring group, no Mamba: the shape of a
model whose sliding-window KV lives in per-request rings."""
@@ -935,6 +935,9 @@ class NixlBaseConnectorWorker:
# page it overlays, so its regions cannot be recovered from spec type.
self._scratch_region_indices = list[int]()
self._ple_region_index: int | None = None
# Page length of the PLE short-conv state; its region may be shared with
# pages of a different (smaller) size, so it cannot use the region's.
self._ple_block_len: int | None = None
# Enable different block lengths for different layers *only* when MLA is used.
# This is not used for SSM layers, which use the counterpart `mamba_ssm_size`.
@@ -1464,6 +1467,7 @@ class NixlBaseConnectorWorker:
seen_base_addresses: list[int] = []
self._ssm_region_indices = []
self._scratch_region_indices = []
self._ple_block_len = None
self._ple_region_index = None
packed_storage = _share_storage_and_block_stride(list(xfer_buffers.values()))
@@ -1751,6 +1755,7 @@ class NixlBaseConnectorWorker:
assert self._is_csa_linear
assert self._ple_region_index in (None, region_index)
self._ple_region_index = region_index
self._ple_block_len = block_len
elif region_index not in self._ssm_region_indices:
self._ssm_region_indices.append(region_index)
elif (
@@ -1885,6 +1890,7 @@ class NixlBaseConnectorWorker:
else self.host_buffer_kv_cache_layout,
block_size=self.block_size,
ssm_sizes=self._mamba_ssm_size,
ple_block_len=self._ple_block_len,
attn_backend_name=self.backend_name,
physical_blocks_per_logical_kv_block=(
self._physical_blocks_per_logical_kv_block
@@ -1906,6 +1912,19 @@ class NixlBaseConnectorWorker:
agent_metadata_bytes=encoder.encode(agent_metadata),
)
def _ple_page_len(self) -> int:
"""Transfer length of one PLE state page.
The PLE page shares its region with the first page of other cache
groups (block-outer layout), whose block_len may be smaller; the
descriptors must cover the whole PLE page or part of the conv state
never reaches the decode side."""
assert self._ple_region_index is not None
assert self._ple_block_len is not None, (
"PLE region registered without its page length"
)
return self._ple_block_len
def _build_mamba_local(self, base_addresses: list[int]) -> np.ndarray:
"""Build desc regions (conv sub-projections + ssm) per layer for
local mamba blocks with DS conv layout, as an Nx3 uint64 array.
@@ -1960,7 +1979,7 @@ class NixlBaseConnectorWorker:
parts.append(self._stack_descs(blk_addrs + conv_size, ssm_size, device_id))
if (region_index := self._ple_region_index) is not None:
block_len = self.block_len_per_layer[region_index] * physical_per_logical
block_len = self._ple_page_len() * physical_per_logical
block_stride = (
self.block_stride_per_layer[region_index] * physical_per_logical
)
@@ -2019,11 +2038,12 @@ class NixlBaseConnectorWorker:
if (region_index := self._ple_region_index) is not None:
local_block_len = (
self.block_len_per_layer[region_index]
* self._physical_blocks_per_logical_kv_block
self._ple_page_len() * self._physical_blocks_per_logical_kv_block
)
# Same NIXL_CONNECTOR_VERSION on both sides, so the field is set.
assert nixl_agent_meta.ple_block_len is not None
remote_block_len = (
nixl_agent_meta.block_lens[region_index] * remote_physical_per_logical
nixl_agent_meta.ple_block_len * remote_physical_per_logical
)
if local_block_len != remote_block_len:
raise ValueError(
@@ -49,8 +49,9 @@ PUSH_REG_NOTIF_PREFIX = b"PUSH_REG:"
# 11: Add per-region transfer geometry and memory types to NixlAgentMetadata
# 12: Add per-region member names for PP push
# 13: Add packed-member layouts and order-independent packed-push backend hashes
# 14: Add Qwen3.8-Flash-Next PLE page length - #59997
#
NIXL_CONNECTOR_VERSION: int = 13
NIXL_CONNECTOR_VERSION: int = 14
@dataclass
@@ -73,6 +74,8 @@ class NixlAgentMetadata:
region_mem_types: list[str] | None = None
dcp_size: int = 1
pcp_size: int = 1
# Page length of the PLE short-conv state (None when the model has none).
ple_block_len: int | None = None
# Layer names sharing each advertised region, in region order.
region_members: list[list[str]] = field(default_factory=list)
# Packed member -> (byte offset in its region's block, bytes per page).