[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Dakai An
2026-10-09 15:48:37 -04:00
committed by GitHub
co-authored by Yifan Qiao OpenAI Codex OpenAI Codex Lucas Wilkinson Claude Opus 5.5 Lucas Wilkinson
parent 3766a67a32
commit 02e55d2af3
83 changed files with 1280 additions and 1986 deletions
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.platforms import current_platform
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backend import AttentionBackend
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheConfig
from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
if TYPE_CHECKING:
@@ -351,6 +352,35 @@ def get_current_attn_backends(
]
def get_current_attn_backends_and_specs(
vllm_config: VllmConfig,
kv_cache_config: KVCacheConfig,
fallback: list[type[AttentionBackend]],
) -> tuple[list[type[AttentionBackend]], list[AttentionSpec | None]]:
"""Distinct (backend, spec) pairs of the transfer layers, else ``fallback``.
Compressed caches are skipped: their kernel pages never split transferred
blocks.
"""
pairs: dict[tuple[type[AttentionBackend], AttentionSpec | None], None] = {}
layer_type = cast(type[Any], AttentionLayerBase)
for group in kv_cache_config.transfer_groups:
specs = getattr(group.kv_cache_spec, "kv_cache_specs", {})
layers = get_layers_from_vllm_config(vllm_config, layer_type, group.layer_names)
for name, layer in layers.items():
spec = specs.get(name, group.kv_cache_spec)
if isinstance(spec, AttentionSpec) and spec.tokens_per_state > 1:
layout = vllm_config.cache_config.get_resolved_kv_cache_layout()
assert layout.is_block_outermost, "Compressed caches must be packed"
continue
if not isinstance(spec, AttentionSpec):
spec = None
pairs[layer.get_attn_backend(), spec] = None
if not pairs:
return fallback, [None] * len(fallback)
return [backend for backend, _ in pairs], [spec for _, spec in pairs]
def get_current_attn_backend(
vllm_config: VllmConfig, layer_names: list[str] | None = None
) -> type[AttentionBackend]:
@@ -26,6 +26,7 @@ from vllm.distributed.kv_transfer.kv_connector.utils import (
EngineId,
TransferTopology,
get_current_attn_backends,
get_current_attn_backends_and_specs,
)
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
@@ -59,8 +60,8 @@ from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import (
AttentionSpec,
CircularBufferSpec,
FullAttentionSpec,
KpoolTailSpec,
KVCacheSpec,
MambaSpec,
MLAAttentionSpec,
@@ -69,7 +70,6 @@ from vllm.v1.kv_cache_interface import (
)
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.request import RequestStatus
from vllm.v1.worker.block_table import BlockTable
from vllm.v1.worker.utils import select_common_block_size
logger = init_logger(__name__)
@@ -1350,7 +1350,10 @@ class MooncakeConnectorWorker:
# physical block sizes. Pick the common (smallest) block size so that
# KV-cache registration and transfer work correctly for both models.
backends = get_current_attn_backends(self.vllm_config)
kernel_block_size = select_common_block_size(self.block_size, backends)
backends, specs = get_current_attn_backends_and_specs(
self.vllm_config, self.kv_cache_config, backends
)
kernel_block_size = select_common_block_size(self.block_size, backends, specs)
if self.block_size != kernel_block_size:
logger.info_once(
"User-specified logical block size (%s) does not match"
@@ -1727,11 +1730,12 @@ class MooncakeConnectorWorker:
)
group_specs = self.kv_cache_config.transfer_groups
return [
BlockTable.map_to_kernel_blocks(
np.array(group),
self._physical_blocks_per_logical_kv_block,
block_arange,
).tolist()
(
np.array(group)[:, None] * self._physical_blocks_per_logical_kv_block
+ block_arange
)
.reshape(-1)
.tolist()
if not isinstance(group_specs[i].kv_cache_spec, MambaSpec)
else group
for i, group in enumerate(block_ids)
@@ -2054,7 +2058,7 @@ class MooncakeConnectorWorker:
# hetero PP can align by name (#56033). A non-contiguous row cannot
# name a sub-span, so it is one region anchored at the allocation.
use_packed_row = (
not isinstance(layer_spec, (MambaSpec, KpoolTailSpec))
not isinstance(layer_spec, (MambaSpec, CircularBufferSpec))
and storage_is_block_major
and block_stride > 0
and (is_mla_region or not hnc_contiguous)
@@ -2121,9 +2125,7 @@ class MooncakeConnectorWorker:
block_len = region_cache.stride(0) * region_cache.element_size()
region_base_addresses.append(base_addr)
if isinstance(layer_spec, KpoolTailSpec):
kv_block_len = layer_spec.unpadded_page_size_bytes // 2
elif isinstance(layer_spec, AttentionSpec) and block_is_contiguous:
if isinstance(layer_spec, AttentionSpec) and block_is_contiguous:
assert (
layer_spec.page_size_bytes
% self._physical_blocks_per_logical_kv_block
@@ -2133,6 +2135,8 @@ class MooncakeConnectorWorker:
layer_spec.page_size_bytes
// self._physical_blocks_per_logical_kv_block
)
elif isinstance(layer_spec, MambaSpec) and block_is_contiguous:
kv_block_len = layer_spec.page_size_bytes
else:
kv_block_len = block_len
if kv_block_len > block_len:
@@ -28,6 +28,7 @@ from vllm.distributed.kv_transfer.kv_connector.utils import (
EngineTransferInfo,
TransferTopology,
get_current_attn_backends,
get_current_attn_backends_and_specs,
kv_postprocess_blksize_and_layout_on_receive,
kv_postprocess_blksize_on_receive,
)
@@ -79,7 +80,6 @@ from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
KpoolTailSpec,
KVCacheLayout,
KVCacheSpec,
MambaSpec,
@@ -88,7 +88,6 @@ from vllm.v1.kv_cache_interface import (
UniformTypeKVCacheSpecs,
iter_layer_specs,
)
from vllm.v1.worker.block_table import BlockTable
from vllm.v1.worker.utils import select_common_block_size
if TYPE_CHECKING:
@@ -120,46 +119,6 @@ def _share_storage_and_block_stride(caches: list[torch.Tensor]) -> bool:
return len(block_strides) == len(storage_ptrs) == 1
def _tensor_byte_span_end(cache: torch.Tensor) -> int:
"""Return the exclusive end address touched by a nonnegative-stride view."""
if cache.numel() == 0:
return cache.data_ptr()
if any(stride < 0 for stride in cache.stride()):
raise ValueError("NIXL cache views must have nonnegative strides")
max_element_offset = sum(
(size - 1) * stride for size, stride in zip(cache.shape, cache.stride())
)
return cache.data_ptr() + (max_element_offset + 1) * cache.element_size()
def _uses_dense_virtual_transfer_pages(
layer_spec: KVCacheSpec,
cache: torch.Tensor,
physical_page_size: int,
num_blocks: int,
) -> bool:
"""Return whether a compressed kernel view can be split into NIXL pages."""
if not (
isinstance(layer_spec, MLAAttentionSpec)
and layer_spec.tokens_per_state > 1
and cache.ndim == 4
and cache.shape[1] == 1
and cache.is_contiguous()
and physical_page_size > 0
and layer_spec.state_content_size_bytes > 0
):
return False
block_stride = cache.stride(0) * cache.element_size()
return (
block_stride > physical_page_size
and block_stride % physical_page_size == 0
and physical_page_size % layer_spec.state_content_size_bytes == 0
and cache.shape[0] * (block_stride // physical_page_size) == num_blocks
and cache.nbytes == num_blocks * physical_page_size
)
class NixlBaseConnectorWorker:
"""Base implementation of Worker side methods shared by pull and push."""
@@ -985,8 +944,10 @@ class NixlBaseConnectorWorker:
)
def _sync_block_size_with_kernel(self) -> None:
backends = get_current_attn_backends(self.vllm_config)
kernel_block_size = select_common_block_size(self.block_size, backends)
backends, specs = get_current_attn_backends_and_specs(
self.vllm_config, self.kv_cache_config, self.attn_backends
)
kernel_block_size = select_common_block_size(self.block_size, backends, specs)
# Number of blocks not accounting for kernel block mismatches
self._logical_num_blocks = self.num_blocks
if self.block_size != kernel_block_size:
@@ -1425,7 +1386,7 @@ class NixlBaseConnectorWorker:
if self._supports_pp_hma and self._has_packed_cache:
# PP can reorder the same backends across stages of a packed model.
backend_name = ",".join(
sorted(backend.get_name() for backend in self.attn_backends)
sorted({backend.get_name() for backend in self.attn_backends})
)
self.compat_hash = compute_nixl_compatibility_hash(
self.vllm_config,
@@ -1478,31 +1439,6 @@ class NixlBaseConnectorWorker:
):
raise NotImplementedError("PP push with packed KV caches requires MLA")
layer_specs: dict[str, KVCacheSpec] = {}
compressed_region_owners: dict[int, torch.Tensor] = {}
for layer_name, cache in xfer_buffers.items():
layer_spec = self._layer_specs.get(layer_name)
if isinstance(layer_spec, UniformTypeKVCacheSpecs):
layer_spec = layer_spec.kv_cache_specs[layer_name]
if layer_spec is None:
continue
layer_specs[layer_name] = layer_spec
physical_page_size = (
layer_spec.page_size_bytes
if isinstance(layer_spec, MambaSpec)
else layer_spec.page_size_bytes
// self._physical_blocks_per_logical_kv_block
)
num_blocks = (
self._logical_num_blocks
if isinstance(layer_spec, MambaSpec)
else self.num_blocks
)
if _uses_dense_virtual_transfer_pages(
layer_spec, cache, physical_page_size, num_blocks
):
compressed_region_owners.setdefault(cache.data_ptr(), cache)
track_region_layers = self._tracks_region_layers()
region_layers: list[list[str]] = []
packed_member_layouts: dict[str, tuple[int, int]] = {}
@@ -1527,7 +1463,7 @@ class NixlBaseConnectorWorker:
# kernel requires a specific block size. This leads to SSM and FA layers
# having different num_blocks.
# `_physical_blocks_per_logical_kv_block` ratio is used to adjust for this.
layer_spec = layer_specs.get(layer_name)
layer_spec = self._layer_specs.get(layer_name)
if layer_spec is None:
logger.debug(
"Skipping layer %s as no KVCache spec is present. "
@@ -1587,32 +1523,6 @@ class NixlBaseConnectorWorker:
region_device_id,
)
if isinstance(layer_spec, KpoolTailSpec):
compressed_owner = compressed_region_owners.get(cache.data_ptr())
if compressed_owner is not None:
owner_storage = compressed_owner.untyped_storage()
owner_end = compressed_owner.data_ptr() + compressed_owner.nbytes
tail_is_covered = (
compressed_owner.is_contiguous()
and owner_storage.data_ptr() == storage_addr
and _tensor_byte_span_end(cache) <= owner_end
)
if not tail_is_covered:
raise AssertionError(
"Kpool tail cache is not fully covered by its compressed "
f"indexer region: layer={layer_name}, "
f"tail_shape={tuple(cache.shape)}, "
f"tail_stride={tuple(cache.stride())}, "
f"owner_shape={tuple(compressed_owner.shape)}, "
f"owner_stride={tuple(compressed_owner.stride())}"
)
logger.debug(
"Skipping layer %s because its compressed indexer region "
"covers the same storage",
layer_name,
)
continue
if isinstance(layer_spec, MambaSpec):
physical_ratio = self._physical_blocks_per_logical_kv_block
block_len = physical_page_size // physical_ratio
@@ -1647,9 +1557,6 @@ class NixlBaseConnectorWorker:
and cache[0].is_contiguous()
and cache[0].nbytes == physical_page_size
)
virtual_transfer_pages = _uses_dense_virtual_transfer_pages(
layer_spec, cache, physical_page_size, num_blocks
)
# Push workers address the pages of packed MLA rows by layer.
packed_mla_push = (
track_region_layers
@@ -1658,12 +1565,7 @@ class NixlBaseConnectorWorker:
and storage_is_block_major
and is_mla_region
)
if virtual_transfer_pages:
# A compressed kernel row can contain multiple NIXL transfer pages.
region_specs = [
(cache.data_ptr(), physical_page_size, physical_page_size)
]
elif packed_mla_push and use_layer_name_routing:
if packed_mla_push and use_layer_name_routing:
# PP>1 prefill stage: register only this layer's page of each row.
region_specs = [
(cache.data_ptr(), physical_page_size, block_stride)
@@ -1671,7 +1573,7 @@ class NixlBaseConnectorWorker:
elif packed_mla_push:
# PP=1 decode or prefill: register whole rows, and advertise where
# this layer's page sits so a PP>1 prefill can write into it.
storage_block_len = registration_len // num_blocks
storage_block_len = block_stride
region_specs = [
(registration_base, storage_block_len, storage_block_len)
]
@@ -3388,11 +3290,9 @@ class NixlBaseConnectorWorker:
physical_block_ids.append(group)
else:
physical_block_ids.append(
BlockTable.map_to_kernel_blocks(
np.array(group),
ratio,
block_arange,
).tolist()
(np.array(group)[:, None] * ratio + block_arange)
.reshape(-1)
.tolist()
)
return physical_block_ids