From 72d59adc2c76de8f77aad640c7a04d663a970735 Mon Sep 17 00:00:00 2001 From: Alberto Massidda Date: Thu, 8 Oct 2026 21:58:13 +0200 Subject: [PATCH] [Bugfix][KV Connector][NIXL] Transfer the whole PLE short-conv page in disaggregated P/D (#59997) Signed-off-by: Alberto Massidda Co-authored-by: Alberto Massidda Co-authored-by: Claude Fable 5.1 Co-authored-by: Artem Perevedentsev --- .../unit/test_nixl_desc_geometry.py | 60 +++++++++++++++---- .../kv_connector/v1/nixl/base_worker.py | 28 +++++++-- .../kv_connector/v1/nixl/metadata.py | 5 +- 3 files changed, 77 insertions(+), 16 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_nixl_desc_geometry.py b/tests/v1/kv_connector/unit/test_nixl_desc_geometry.py index 029ffafd3c..9f6bc56dee 100644 --- a/tests/v1/kv_connector/unit/test_nixl_desc_geometry.py +++ b/tests/v1/kv_connector/unit/test_nixl_desc_geometry.py @@ -1245,8 +1245,12 @@ def test_mismatched_mla_kernel_page_rejected_for_mla_hybrid(): worker.add_remote_agent(meta_r, remote_tp_rank=0, remote_tp_size=2) -def _make_csa_linear_ple_worker(scratch_aliases: str = "compressed"): - """``scratch_aliases`` selects which pages the compressor ring overlays.""" +def _make_csa_linear_ple_worker( + scratch_aliases: str = "compressed", ple_page_size: int = 256 +): + """``scratch_aliases`` selects which pages the compressor ring overlays; + ``ple_page_size`` widens the PLE page beyond the page it shares a region + with (block-outer layout: both start at byte 0 of the block).""" from unittest.mock import MagicMock from vllm.config import set_current_vllm_config @@ -1305,7 +1309,7 @@ def _make_csa_linear_ple_worker(scratch_aliases: str = "compressed"): block_size=1, shapes=((6, 3),), dtypes=(torch.float16,), - page_size_padded=256, + page_size_padded=ple_page_size, mamba_type=MambaAttentionBackendEnum.SHORT_CONV, tp_replicated=True, ) @@ -1339,17 +1343,22 @@ def _make_csa_linear_ple_worker(scratch_aliases: str = "compressed"): ("compressed.0",), ("compressed.1",), ) - region_size = 512 page_size = 256 + # Region 0 hosts the PLE page: its blocks are as wide as the widest page. + block_strides = [max(page_size, ple_page_size)] + [page_size] * ( + len(tensor_regions) - 1 + ) + region_sizes = [2 * stride for stride in block_strides] + region_offsets = [sum(region_sizes[:index]) for index in range(len(region_sizes))] kv_cache_config = KVCacheConfig( num_blocks=2, kv_cache_tensors=[ KVCacheTensor( - size=len(tensor_regions) * region_size, + size=sum(region_sizes), layers=[layer_name], - layer_stride=region_size, - block_stride=page_size, - offset=region_index * region_size, + layer_stride=region_sizes[region_index], + block_stride=block_strides[region_index], + offset=region_offsets[region_index], ) for region_index, layer_names in enumerate(tensor_regions) for layer_name in layer_names @@ -1386,10 +1395,14 @@ def _make_csa_linear_ple_worker(scratch_aliases: str = "compressed"): set_current_vllm_config(vllm_config), ): worker = NixlConnectorWorker(vllm_config, "local-engine", kv_cache_config) - tensors = [torch.zeros((2, 256), dtype=torch.uint8) for _ in range(4)] + tensors = [ + torch.zeros((2, stride), dtype=torch.uint8) for stride in block_strides + ] worker.register_kv_caches( { - layer_name: tensors[region_index] + layer_name: tensors[region_index][ + :, : (ple_page_size if layer_name == "mamba.ple" else page_size) + ] for region_index, layer_names in enumerate(tensor_regions) for layer_name in layer_names } @@ -1523,6 +1536,7 @@ def test_csa_linear_remote_ple_is_copied_whole(): kv_cache_layout="HND", block_size=4, ssm_sizes=(24, 32), + ple_block_len=256, attn_backend_name="test", physical_blocks_per_logical_kv_block=1, ) @@ -1535,7 +1549,7 @@ def test_csa_linear_remote_ple_is_copied_whole(): assert descriptors[-2:, 0].tolist() == [0x10000, 0x10100] assert descriptors[-2:, 1].tolist() == [256, 256] - metadata.block_lens[0] = 128 + metadata.ple_block_len = 128 with pytest.raises(ValueError, match="PLE pages require identical"): worker._build_mamba_remote( metadata, @@ -1544,6 +1558,30 @@ def test_csa_linear_remote_ple_is_copied_whole(): ) +@pytest.mark.cpu_test +def test_csa_linear_ple_page_wider_than_the_shared_region_page(): + """The PLE page is discovered in the region of the main KV page registered + first (both start at byte 0 of the block); its descriptors must still cover + the whole PLE page, not the region's block_len.""" + worker = _make_csa_linear_ple_worker(ple_page_size=384) + + assert worker._ple_region_index == 0 + assert worker.block_len_per_layer[0] == 256 + assert worker.block_stride_per_layer[0] == 384 + assert worker._ple_block_len == 384 + + bases = [0x10000, 0x20000, 0x30000, 0x40000] + mamba = worker._build_mamba_local(bases) + assert mamba[-2:, 0].tolist() == [0x10000, 0x10000 + 384] + assert mamba[-2:, 1].tolist() == [384, 384] + + metadata = msgspec.msgpack.decode( + worker.xfer_handshake_metadata.agent_metadata_bytes, type=NixlAgentMetadata + ) + assert metadata.ple_block_len == 384 + assert metadata.block_lens[0] == 256 + + def _make_ring_worker(): """Paged MLA group plus a per-layer ring group, no Mamba: the shape of a model whose sliding-window KV lives in per-request rings.""" diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index 2a2f660cf1..7ce6e93ebb 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -935,6 +935,9 @@ class NixlBaseConnectorWorker: # page it overlays, so its regions cannot be recovered from spec type. self._scratch_region_indices = list[int]() self._ple_region_index: int | None = None + # Page length of the PLE short-conv state; its region may be shared with + # pages of a different (smaller) size, so it cannot use the region's. + self._ple_block_len: int | None = None # Enable different block lengths for different layers *only* when MLA is used. # This is not used for SSM layers, which use the counterpart `mamba_ssm_size`. @@ -1464,6 +1467,7 @@ class NixlBaseConnectorWorker: seen_base_addresses: list[int] = [] self._ssm_region_indices = [] self._scratch_region_indices = [] + self._ple_block_len = None self._ple_region_index = None packed_storage = _share_storage_and_block_stride(list(xfer_buffers.values())) @@ -1751,6 +1755,7 @@ class NixlBaseConnectorWorker: assert self._is_csa_linear assert self._ple_region_index in (None, region_index) self._ple_region_index = region_index + self._ple_block_len = block_len elif region_index not in self._ssm_region_indices: self._ssm_region_indices.append(region_index) elif ( @@ -1885,6 +1890,7 @@ class NixlBaseConnectorWorker: else self.host_buffer_kv_cache_layout, block_size=self.block_size, ssm_sizes=self._mamba_ssm_size, + ple_block_len=self._ple_block_len, attn_backend_name=self.backend_name, physical_blocks_per_logical_kv_block=( self._physical_blocks_per_logical_kv_block @@ -1906,6 +1912,19 @@ class NixlBaseConnectorWorker: agent_metadata_bytes=encoder.encode(agent_metadata), ) + def _ple_page_len(self) -> int: + """Transfer length of one PLE state page. + + The PLE page shares its region with the first page of other cache + groups (block-outer layout), whose block_len may be smaller; the + descriptors must cover the whole PLE page or part of the conv state + never reaches the decode side.""" + assert self._ple_region_index is not None + assert self._ple_block_len is not None, ( + "PLE region registered without its page length" + ) + return self._ple_block_len + def _build_mamba_local(self, base_addresses: list[int]) -> np.ndarray: """Build desc regions (conv sub-projections + ssm) per layer for local mamba blocks with DS conv layout, as an Nx3 uint64 array. @@ -1960,7 +1979,7 @@ class NixlBaseConnectorWorker: parts.append(self._stack_descs(blk_addrs + conv_size, ssm_size, device_id)) if (region_index := self._ple_region_index) is not None: - block_len = self.block_len_per_layer[region_index] * physical_per_logical + block_len = self._ple_page_len() * physical_per_logical block_stride = ( self.block_stride_per_layer[region_index] * physical_per_logical ) @@ -2019,11 +2038,12 @@ class NixlBaseConnectorWorker: if (region_index := self._ple_region_index) is not None: local_block_len = ( - self.block_len_per_layer[region_index] - * self._physical_blocks_per_logical_kv_block + self._ple_page_len() * self._physical_blocks_per_logical_kv_block ) + # Same NIXL_CONNECTOR_VERSION on both sides, so the field is set. + assert nixl_agent_meta.ple_block_len is not None remote_block_len = ( - nixl_agent_meta.block_lens[region_index] * remote_physical_per_logical + nixl_agent_meta.ple_block_len * remote_physical_per_logical ) if local_block_len != remote_block_len: raise ValueError( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py index 8d0aad7633..a33f620a0a 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py @@ -49,8 +49,9 @@ PUSH_REG_NOTIF_PREFIX = b"PUSH_REG:" # 11: Add per-region transfer geometry and memory types to NixlAgentMetadata # 12: Add per-region member names for PP push # 13: Add packed-member layouts and order-independent packed-push backend hashes +# 14: Add Qwen3.8-Flash-Next PLE page length - #59997 # -NIXL_CONNECTOR_VERSION: int = 13 +NIXL_CONNECTOR_VERSION: int = 14 @dataclass @@ -73,6 +74,8 @@ class NixlAgentMetadata: region_mem_types: list[str] | None = None dcp_size: int = 1 pcp_size: int = 1 + # Page length of the PLE short-conv state (None when the model has none). + ple_block_len: int | None = None # Layer names sharing each advertised region, in region order. region_members: list[list[str]] = field(default_factory=list) # Packed member -> (byte offset in its region's block, bytes per page).