[Bugfix][KV Connector][NIXL] Transfer the whole PLE short-conv page in disaggregated P/D (#59997)
Signed-off-by: Alberto Massidda <amassidda@nvidia.com> Co-authored-by: Alberto Massidda <amassidda@nvidia.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Artem Perevedentsev <aperevedents@nvidia.com>
This commit is contained in:
co-authored by
Alberto Massidda
Claude Fable 5.1
Artem Perevedentsev
parent
1afb029839
commit
72d59adc2c
@@ -1245,8 +1245,12 @@ def test_mismatched_mla_kernel_page_rejected_for_mla_hybrid():
|
||||
worker.add_remote_agent(meta_r, remote_tp_rank=0, remote_tp_size=2)
|
||||
|
||||
|
||||
def _make_csa_linear_ple_worker(scratch_aliases: str = "compressed"):
|
||||
"""``scratch_aliases`` selects which pages the compressor ring overlays."""
|
||||
def _make_csa_linear_ple_worker(
|
||||
scratch_aliases: str = "compressed", ple_page_size: int = 256
|
||||
):
|
||||
"""``scratch_aliases`` selects which pages the compressor ring overlays;
|
||||
``ple_page_size`` widens the PLE page beyond the page it shares a region
|
||||
with (block-outer layout: both start at byte 0 of the block)."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
@@ -1305,7 +1309,7 @@ def _make_csa_linear_ple_worker(scratch_aliases: str = "compressed"):
|
||||
block_size=1,
|
||||
shapes=((6, 3),),
|
||||
dtypes=(torch.float16,),
|
||||
page_size_padded=256,
|
||||
page_size_padded=ple_page_size,
|
||||
mamba_type=MambaAttentionBackendEnum.SHORT_CONV,
|
||||
tp_replicated=True,
|
||||
)
|
||||
@@ -1339,17 +1343,22 @@ def _make_csa_linear_ple_worker(scratch_aliases: str = "compressed"):
|
||||
("compressed.0",),
|
||||
("compressed.1",),
|
||||
)
|
||||
region_size = 512
|
||||
page_size = 256
|
||||
# Region 0 hosts the PLE page: its blocks are as wide as the widest page.
|
||||
block_strides = [max(page_size, ple_page_size)] + [page_size] * (
|
||||
len(tensor_regions) - 1
|
||||
)
|
||||
region_sizes = [2 * stride for stride in block_strides]
|
||||
region_offsets = [sum(region_sizes[:index]) for index in range(len(region_sizes))]
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=2,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=len(tensor_regions) * region_size,
|
||||
size=sum(region_sizes),
|
||||
layers=[layer_name],
|
||||
layer_stride=region_size,
|
||||
block_stride=page_size,
|
||||
offset=region_index * region_size,
|
||||
layer_stride=region_sizes[region_index],
|
||||
block_stride=block_strides[region_index],
|
||||
offset=region_offsets[region_index],
|
||||
)
|
||||
for region_index, layer_names in enumerate(tensor_regions)
|
||||
for layer_name in layer_names
|
||||
@@ -1386,10 +1395,14 @@ def _make_csa_linear_ple_worker(scratch_aliases: str = "compressed"):
|
||||
set_current_vllm_config(vllm_config),
|
||||
):
|
||||
worker = NixlConnectorWorker(vllm_config, "local-engine", kv_cache_config)
|
||||
tensors = [torch.zeros((2, 256), dtype=torch.uint8) for _ in range(4)]
|
||||
tensors = [
|
||||
torch.zeros((2, stride), dtype=torch.uint8) for stride in block_strides
|
||||
]
|
||||
worker.register_kv_caches(
|
||||
{
|
||||
layer_name: tensors[region_index]
|
||||
layer_name: tensors[region_index][
|
||||
:, : (ple_page_size if layer_name == "mamba.ple" else page_size)
|
||||
]
|
||||
for region_index, layer_names in enumerate(tensor_regions)
|
||||
for layer_name in layer_names
|
||||
}
|
||||
@@ -1523,6 +1536,7 @@ def test_csa_linear_remote_ple_is_copied_whole():
|
||||
kv_cache_layout="HND",
|
||||
block_size=4,
|
||||
ssm_sizes=(24, 32),
|
||||
ple_block_len=256,
|
||||
attn_backend_name="test",
|
||||
physical_blocks_per_logical_kv_block=1,
|
||||
)
|
||||
@@ -1535,7 +1549,7 @@ def test_csa_linear_remote_ple_is_copied_whole():
|
||||
assert descriptors[-2:, 0].tolist() == [0x10000, 0x10100]
|
||||
assert descriptors[-2:, 1].tolist() == [256, 256]
|
||||
|
||||
metadata.block_lens[0] = 128
|
||||
metadata.ple_block_len = 128
|
||||
with pytest.raises(ValueError, match="PLE pages require identical"):
|
||||
worker._build_mamba_remote(
|
||||
metadata,
|
||||
@@ -1544,6 +1558,30 @@ def test_csa_linear_remote_ple_is_copied_whole():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_csa_linear_ple_page_wider_than_the_shared_region_page():
|
||||
"""The PLE page is discovered in the region of the main KV page registered
|
||||
first (both start at byte 0 of the block); its descriptors must still cover
|
||||
the whole PLE page, not the region's block_len."""
|
||||
worker = _make_csa_linear_ple_worker(ple_page_size=384)
|
||||
|
||||
assert worker._ple_region_index == 0
|
||||
assert worker.block_len_per_layer[0] == 256
|
||||
assert worker.block_stride_per_layer[0] == 384
|
||||
assert worker._ple_block_len == 384
|
||||
|
||||
bases = [0x10000, 0x20000, 0x30000, 0x40000]
|
||||
mamba = worker._build_mamba_local(bases)
|
||||
assert mamba[-2:, 0].tolist() == [0x10000, 0x10000 + 384]
|
||||
assert mamba[-2:, 1].tolist() == [384, 384]
|
||||
|
||||
metadata = msgspec.msgpack.decode(
|
||||
worker.xfer_handshake_metadata.agent_metadata_bytes, type=NixlAgentMetadata
|
||||
)
|
||||
assert metadata.ple_block_len == 384
|
||||
assert metadata.block_lens[0] == 256
|
||||
|
||||
|
||||
def _make_ring_worker():
|
||||
"""Paged MLA group plus a per-layer ring group, no Mamba: the shape of a
|
||||
model whose sliding-window KV lives in per-request rings."""
|
||||
|
||||
@@ -935,6 +935,9 @@ class NixlBaseConnectorWorker:
|
||||
# page it overlays, so its regions cannot be recovered from spec type.
|
||||
self._scratch_region_indices = list[int]()
|
||||
self._ple_region_index: int | None = None
|
||||
# Page length of the PLE short-conv state; its region may be shared with
|
||||
# pages of a different (smaller) size, so it cannot use the region's.
|
||||
self._ple_block_len: int | None = None
|
||||
|
||||
# Enable different block lengths for different layers *only* when MLA is used.
|
||||
# This is not used for SSM layers, which use the counterpart `mamba_ssm_size`.
|
||||
@@ -1464,6 +1467,7 @@ class NixlBaseConnectorWorker:
|
||||
seen_base_addresses: list[int] = []
|
||||
self._ssm_region_indices = []
|
||||
self._scratch_region_indices = []
|
||||
self._ple_block_len = None
|
||||
self._ple_region_index = None
|
||||
|
||||
packed_storage = _share_storage_and_block_stride(list(xfer_buffers.values()))
|
||||
@@ -1751,6 +1755,7 @@ class NixlBaseConnectorWorker:
|
||||
assert self._is_csa_linear
|
||||
assert self._ple_region_index in (None, region_index)
|
||||
self._ple_region_index = region_index
|
||||
self._ple_block_len = block_len
|
||||
elif region_index not in self._ssm_region_indices:
|
||||
self._ssm_region_indices.append(region_index)
|
||||
elif (
|
||||
@@ -1885,6 +1890,7 @@ class NixlBaseConnectorWorker:
|
||||
else self.host_buffer_kv_cache_layout,
|
||||
block_size=self.block_size,
|
||||
ssm_sizes=self._mamba_ssm_size,
|
||||
ple_block_len=self._ple_block_len,
|
||||
attn_backend_name=self.backend_name,
|
||||
physical_blocks_per_logical_kv_block=(
|
||||
self._physical_blocks_per_logical_kv_block
|
||||
@@ -1906,6 +1912,19 @@ class NixlBaseConnectorWorker:
|
||||
agent_metadata_bytes=encoder.encode(agent_metadata),
|
||||
)
|
||||
|
||||
def _ple_page_len(self) -> int:
|
||||
"""Transfer length of one PLE state page.
|
||||
|
||||
The PLE page shares its region with the first page of other cache
|
||||
groups (block-outer layout), whose block_len may be smaller; the
|
||||
descriptors must cover the whole PLE page or part of the conv state
|
||||
never reaches the decode side."""
|
||||
assert self._ple_region_index is not None
|
||||
assert self._ple_block_len is not None, (
|
||||
"PLE region registered without its page length"
|
||||
)
|
||||
return self._ple_block_len
|
||||
|
||||
def _build_mamba_local(self, base_addresses: list[int]) -> np.ndarray:
|
||||
"""Build desc regions (conv sub-projections + ssm) per layer for
|
||||
local mamba blocks with DS conv layout, as an Nx3 uint64 array.
|
||||
@@ -1960,7 +1979,7 @@ class NixlBaseConnectorWorker:
|
||||
parts.append(self._stack_descs(blk_addrs + conv_size, ssm_size, device_id))
|
||||
|
||||
if (region_index := self._ple_region_index) is not None:
|
||||
block_len = self.block_len_per_layer[region_index] * physical_per_logical
|
||||
block_len = self._ple_page_len() * physical_per_logical
|
||||
block_stride = (
|
||||
self.block_stride_per_layer[region_index] * physical_per_logical
|
||||
)
|
||||
@@ -2019,11 +2038,12 @@ class NixlBaseConnectorWorker:
|
||||
|
||||
if (region_index := self._ple_region_index) is not None:
|
||||
local_block_len = (
|
||||
self.block_len_per_layer[region_index]
|
||||
* self._physical_blocks_per_logical_kv_block
|
||||
self._ple_page_len() * self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
# Same NIXL_CONNECTOR_VERSION on both sides, so the field is set.
|
||||
assert nixl_agent_meta.ple_block_len is not None
|
||||
remote_block_len = (
|
||||
nixl_agent_meta.block_lens[region_index] * remote_physical_per_logical
|
||||
nixl_agent_meta.ple_block_len * remote_physical_per_logical
|
||||
)
|
||||
if local_block_len != remote_block_len:
|
||||
raise ValueError(
|
||||
|
||||
@@ -49,8 +49,9 @@ PUSH_REG_NOTIF_PREFIX = b"PUSH_REG:"
|
||||
# 11: Add per-region transfer geometry and memory types to NixlAgentMetadata
|
||||
# 12: Add per-region member names for PP push
|
||||
# 13: Add packed-member layouts and order-independent packed-push backend hashes
|
||||
# 14: Add Qwen3.8-Flash-Next PLE page length - #59997
|
||||
#
|
||||
NIXL_CONNECTOR_VERSION: int = 13
|
||||
NIXL_CONNECTOR_VERSION: int = 14
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -73,6 +74,8 @@ class NixlAgentMetadata:
|
||||
region_mem_types: list[str] | None = None
|
||||
dcp_size: int = 1
|
||||
pcp_size: int = 1
|
||||
# Page length of the PLE short-conv state (None when the model has none).
|
||||
ple_block_len: int | None = None
|
||||
# Layer names sharing each advertised region, in region order.
|
||||
region_members: list[list[str]] = field(default_factory=list)
|
||||
# Packed member -> (byte offset in its region's block, bytes per page).
|
||||
|
||||
Reference in New Issue
Block a user