[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
OpenAI Codex
OpenAI Codex
Lucas Wilkinson
Claude Opus 5.5
Lucas Wilkinson
parent
3766a67a32
commit
02e55d2af3
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheConfig
|
||||
from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -351,6 +352,35 @@ def get_current_attn_backends(
|
||||
]
|
||||
|
||||
|
||||
def get_current_attn_backends_and_specs(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
fallback: list[type[AttentionBackend]],
|
||||
) -> tuple[list[type[AttentionBackend]], list[AttentionSpec | None]]:
|
||||
"""Distinct (backend, spec) pairs of the transfer layers, else ``fallback``.
|
||||
|
||||
Compressed caches are skipped: their kernel pages never split transferred
|
||||
blocks.
|
||||
"""
|
||||
pairs: dict[tuple[type[AttentionBackend], AttentionSpec | None], None] = {}
|
||||
layer_type = cast(type[Any], AttentionLayerBase)
|
||||
for group in kv_cache_config.transfer_groups:
|
||||
specs = getattr(group.kv_cache_spec, "kv_cache_specs", {})
|
||||
layers = get_layers_from_vllm_config(vllm_config, layer_type, group.layer_names)
|
||||
for name, layer in layers.items():
|
||||
spec = specs.get(name, group.kv_cache_spec)
|
||||
if isinstance(spec, AttentionSpec) and spec.tokens_per_state > 1:
|
||||
layout = vllm_config.cache_config.get_resolved_kv_cache_layout()
|
||||
assert layout.is_block_outermost, "Compressed caches must be packed"
|
||||
continue
|
||||
if not isinstance(spec, AttentionSpec):
|
||||
spec = None
|
||||
pairs[layer.get_attn_backend(), spec] = None
|
||||
if not pairs:
|
||||
return fallback, [None] * len(fallback)
|
||||
return [backend for backend, _ in pairs], [spec for _, spec in pairs]
|
||||
|
||||
|
||||
def get_current_attn_backend(
|
||||
vllm_config: VllmConfig, layer_names: list[str] | None = None
|
||||
) -> type[AttentionBackend]:
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
EngineId,
|
||||
TransferTopology,
|
||||
get_current_attn_backends,
|
||||
get_current_attn_backends_and_specs,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
@@ -59,8 +60,8 @@ from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
@@ -69,7 +70,6 @@ from vllm.v1.kv_cache_interface import (
|
||||
)
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.request import RequestStatus
|
||||
from vllm.v1.worker.block_table import BlockTable
|
||||
from vllm.v1.worker.utils import select_common_block_size
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -1350,7 +1350,10 @@ class MooncakeConnectorWorker:
|
||||
# physical block sizes. Pick the common (smallest) block size so that
|
||||
# KV-cache registration and transfer work correctly for both models.
|
||||
backends = get_current_attn_backends(self.vllm_config)
|
||||
kernel_block_size = select_common_block_size(self.block_size, backends)
|
||||
backends, specs = get_current_attn_backends_and_specs(
|
||||
self.vllm_config, self.kv_cache_config, backends
|
||||
)
|
||||
kernel_block_size = select_common_block_size(self.block_size, backends, specs)
|
||||
if self.block_size != kernel_block_size:
|
||||
logger.info_once(
|
||||
"User-specified logical block size (%s) does not match"
|
||||
@@ -1727,11 +1730,12 @@ class MooncakeConnectorWorker:
|
||||
)
|
||||
group_specs = self.kv_cache_config.transfer_groups
|
||||
return [
|
||||
BlockTable.map_to_kernel_blocks(
|
||||
np.array(group),
|
||||
self._physical_blocks_per_logical_kv_block,
|
||||
block_arange,
|
||||
).tolist()
|
||||
(
|
||||
np.array(group)[:, None] * self._physical_blocks_per_logical_kv_block
|
||||
+ block_arange
|
||||
)
|
||||
.reshape(-1)
|
||||
.tolist()
|
||||
if not isinstance(group_specs[i].kv_cache_spec, MambaSpec)
|
||||
else group
|
||||
for i, group in enumerate(block_ids)
|
||||
@@ -2054,7 +2058,7 @@ class MooncakeConnectorWorker:
|
||||
# hetero PP can align by name (#56033). A non-contiguous row cannot
|
||||
# name a sub-span, so it is one region anchored at the allocation.
|
||||
use_packed_row = (
|
||||
not isinstance(layer_spec, (MambaSpec, KpoolTailSpec))
|
||||
not isinstance(layer_spec, (MambaSpec, CircularBufferSpec))
|
||||
and storage_is_block_major
|
||||
and block_stride > 0
|
||||
and (is_mla_region or not hnc_contiguous)
|
||||
@@ -2121,9 +2125,7 @@ class MooncakeConnectorWorker:
|
||||
block_len = region_cache.stride(0) * region_cache.element_size()
|
||||
region_base_addresses.append(base_addr)
|
||||
|
||||
if isinstance(layer_spec, KpoolTailSpec):
|
||||
kv_block_len = layer_spec.unpadded_page_size_bytes // 2
|
||||
elif isinstance(layer_spec, AttentionSpec) and block_is_contiguous:
|
||||
if isinstance(layer_spec, AttentionSpec) and block_is_contiguous:
|
||||
assert (
|
||||
layer_spec.page_size_bytes
|
||||
% self._physical_blocks_per_logical_kv_block
|
||||
@@ -2133,6 +2135,8 @@ class MooncakeConnectorWorker:
|
||||
layer_spec.page_size_bytes
|
||||
// self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
elif isinstance(layer_spec, MambaSpec) and block_is_contiguous:
|
||||
kv_block_len = layer_spec.page_size_bytes
|
||||
else:
|
||||
kv_block_len = block_len
|
||||
if kv_block_len > block_len:
|
||||
|
||||
@@ -28,6 +28,7 @@ from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
EngineTransferInfo,
|
||||
TransferTopology,
|
||||
get_current_attn_backends,
|
||||
get_current_attn_backends_and_specs,
|
||||
kv_postprocess_blksize_and_layout_on_receive,
|
||||
kv_postprocess_blksize_on_receive,
|
||||
)
|
||||
@@ -79,7 +80,6 @@ from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheLayout,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
@@ -88,7 +88,6 @@ from vllm.v1.kv_cache_interface import (
|
||||
UniformTypeKVCacheSpecs,
|
||||
iter_layer_specs,
|
||||
)
|
||||
from vllm.v1.worker.block_table import BlockTable
|
||||
from vllm.v1.worker.utils import select_common_block_size
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -120,46 +119,6 @@ def _share_storage_and_block_stride(caches: list[torch.Tensor]) -> bool:
|
||||
return len(block_strides) == len(storage_ptrs) == 1
|
||||
|
||||
|
||||
def _tensor_byte_span_end(cache: torch.Tensor) -> int:
|
||||
"""Return the exclusive end address touched by a nonnegative-stride view."""
|
||||
if cache.numel() == 0:
|
||||
return cache.data_ptr()
|
||||
if any(stride < 0 for stride in cache.stride()):
|
||||
raise ValueError("NIXL cache views must have nonnegative strides")
|
||||
max_element_offset = sum(
|
||||
(size - 1) * stride for size, stride in zip(cache.shape, cache.stride())
|
||||
)
|
||||
return cache.data_ptr() + (max_element_offset + 1) * cache.element_size()
|
||||
|
||||
|
||||
def _uses_dense_virtual_transfer_pages(
|
||||
layer_spec: KVCacheSpec,
|
||||
cache: torch.Tensor,
|
||||
physical_page_size: int,
|
||||
num_blocks: int,
|
||||
) -> bool:
|
||||
"""Return whether a compressed kernel view can be split into NIXL pages."""
|
||||
if not (
|
||||
isinstance(layer_spec, MLAAttentionSpec)
|
||||
and layer_spec.tokens_per_state > 1
|
||||
and cache.ndim == 4
|
||||
and cache.shape[1] == 1
|
||||
and cache.is_contiguous()
|
||||
and physical_page_size > 0
|
||||
and layer_spec.state_content_size_bytes > 0
|
||||
):
|
||||
return False
|
||||
|
||||
block_stride = cache.stride(0) * cache.element_size()
|
||||
return (
|
||||
block_stride > physical_page_size
|
||||
and block_stride % physical_page_size == 0
|
||||
and physical_page_size % layer_spec.state_content_size_bytes == 0
|
||||
and cache.shape[0] * (block_stride // physical_page_size) == num_blocks
|
||||
and cache.nbytes == num_blocks * physical_page_size
|
||||
)
|
||||
|
||||
|
||||
class NixlBaseConnectorWorker:
|
||||
"""Base implementation of Worker side methods shared by pull and push."""
|
||||
|
||||
@@ -985,8 +944,10 @@ class NixlBaseConnectorWorker:
|
||||
)
|
||||
|
||||
def _sync_block_size_with_kernel(self) -> None:
|
||||
backends = get_current_attn_backends(self.vllm_config)
|
||||
kernel_block_size = select_common_block_size(self.block_size, backends)
|
||||
backends, specs = get_current_attn_backends_and_specs(
|
||||
self.vllm_config, self.kv_cache_config, self.attn_backends
|
||||
)
|
||||
kernel_block_size = select_common_block_size(self.block_size, backends, specs)
|
||||
# Number of blocks not accounting for kernel block mismatches
|
||||
self._logical_num_blocks = self.num_blocks
|
||||
if self.block_size != kernel_block_size:
|
||||
@@ -1425,7 +1386,7 @@ class NixlBaseConnectorWorker:
|
||||
if self._supports_pp_hma and self._has_packed_cache:
|
||||
# PP can reorder the same backends across stages of a packed model.
|
||||
backend_name = ",".join(
|
||||
sorted(backend.get_name() for backend in self.attn_backends)
|
||||
sorted({backend.get_name() for backend in self.attn_backends})
|
||||
)
|
||||
self.compat_hash = compute_nixl_compatibility_hash(
|
||||
self.vllm_config,
|
||||
@@ -1478,31 +1439,6 @@ class NixlBaseConnectorWorker:
|
||||
):
|
||||
raise NotImplementedError("PP push with packed KV caches requires MLA")
|
||||
|
||||
layer_specs: dict[str, KVCacheSpec] = {}
|
||||
compressed_region_owners: dict[int, torch.Tensor] = {}
|
||||
for layer_name, cache in xfer_buffers.items():
|
||||
layer_spec = self._layer_specs.get(layer_name)
|
||||
if isinstance(layer_spec, UniformTypeKVCacheSpecs):
|
||||
layer_spec = layer_spec.kv_cache_specs[layer_name]
|
||||
if layer_spec is None:
|
||||
continue
|
||||
layer_specs[layer_name] = layer_spec
|
||||
physical_page_size = (
|
||||
layer_spec.page_size_bytes
|
||||
if isinstance(layer_spec, MambaSpec)
|
||||
else layer_spec.page_size_bytes
|
||||
// self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
num_blocks = (
|
||||
self._logical_num_blocks
|
||||
if isinstance(layer_spec, MambaSpec)
|
||||
else self.num_blocks
|
||||
)
|
||||
if _uses_dense_virtual_transfer_pages(
|
||||
layer_spec, cache, physical_page_size, num_blocks
|
||||
):
|
||||
compressed_region_owners.setdefault(cache.data_ptr(), cache)
|
||||
|
||||
track_region_layers = self._tracks_region_layers()
|
||||
region_layers: list[list[str]] = []
|
||||
packed_member_layouts: dict[str, tuple[int, int]] = {}
|
||||
@@ -1527,7 +1463,7 @@ class NixlBaseConnectorWorker:
|
||||
# kernel requires a specific block size. This leads to SSM and FA layers
|
||||
# having different num_blocks.
|
||||
# `_physical_blocks_per_logical_kv_block` ratio is used to adjust for this.
|
||||
layer_spec = layer_specs.get(layer_name)
|
||||
layer_spec = self._layer_specs.get(layer_name)
|
||||
if layer_spec is None:
|
||||
logger.debug(
|
||||
"Skipping layer %s as no KVCache spec is present. "
|
||||
@@ -1587,32 +1523,6 @@ class NixlBaseConnectorWorker:
|
||||
region_device_id,
|
||||
)
|
||||
|
||||
if isinstance(layer_spec, KpoolTailSpec):
|
||||
compressed_owner = compressed_region_owners.get(cache.data_ptr())
|
||||
if compressed_owner is not None:
|
||||
owner_storage = compressed_owner.untyped_storage()
|
||||
owner_end = compressed_owner.data_ptr() + compressed_owner.nbytes
|
||||
tail_is_covered = (
|
||||
compressed_owner.is_contiguous()
|
||||
and owner_storage.data_ptr() == storage_addr
|
||||
and _tensor_byte_span_end(cache) <= owner_end
|
||||
)
|
||||
if not tail_is_covered:
|
||||
raise AssertionError(
|
||||
"Kpool tail cache is not fully covered by its compressed "
|
||||
f"indexer region: layer={layer_name}, "
|
||||
f"tail_shape={tuple(cache.shape)}, "
|
||||
f"tail_stride={tuple(cache.stride())}, "
|
||||
f"owner_shape={tuple(compressed_owner.shape)}, "
|
||||
f"owner_stride={tuple(compressed_owner.stride())}"
|
||||
)
|
||||
logger.debug(
|
||||
"Skipping layer %s because its compressed indexer region "
|
||||
"covers the same storage",
|
||||
layer_name,
|
||||
)
|
||||
continue
|
||||
|
||||
if isinstance(layer_spec, MambaSpec):
|
||||
physical_ratio = self._physical_blocks_per_logical_kv_block
|
||||
block_len = physical_page_size // physical_ratio
|
||||
@@ -1647,9 +1557,6 @@ class NixlBaseConnectorWorker:
|
||||
and cache[0].is_contiguous()
|
||||
and cache[0].nbytes == physical_page_size
|
||||
)
|
||||
virtual_transfer_pages = _uses_dense_virtual_transfer_pages(
|
||||
layer_spec, cache, physical_page_size, num_blocks
|
||||
)
|
||||
# Push workers address the pages of packed MLA rows by layer.
|
||||
packed_mla_push = (
|
||||
track_region_layers
|
||||
@@ -1658,12 +1565,7 @@ class NixlBaseConnectorWorker:
|
||||
and storage_is_block_major
|
||||
and is_mla_region
|
||||
)
|
||||
if virtual_transfer_pages:
|
||||
# A compressed kernel row can contain multiple NIXL transfer pages.
|
||||
region_specs = [
|
||||
(cache.data_ptr(), physical_page_size, physical_page_size)
|
||||
]
|
||||
elif packed_mla_push and use_layer_name_routing:
|
||||
if packed_mla_push and use_layer_name_routing:
|
||||
# PP>1 prefill stage: register only this layer's page of each row.
|
||||
region_specs = [
|
||||
(cache.data_ptr(), physical_page_size, block_stride)
|
||||
@@ -1671,7 +1573,7 @@ class NixlBaseConnectorWorker:
|
||||
elif packed_mla_push:
|
||||
# PP=1 decode or prefill: register whole rows, and advertise where
|
||||
# this layer's page sits so a PP>1 prefill can write into it.
|
||||
storage_block_len = registration_len // num_blocks
|
||||
storage_block_len = block_stride
|
||||
region_specs = [
|
||||
(registration_base, storage_block_len, storage_block_len)
|
||||
]
|
||||
@@ -3388,11 +3290,9 @@ class NixlBaseConnectorWorker:
|
||||
physical_block_ids.append(group)
|
||||
else:
|
||||
physical_block_ids.append(
|
||||
BlockTable.map_to_kernel_blocks(
|
||||
np.array(group),
|
||||
ratio,
|
||||
block_arange,
|
||||
).tolist()
|
||||
(np.array(group)[:, None] * ratio + block_arange)
|
||||
.reshape(-1)
|
||||
.tolist()
|
||||
)
|
||||
return physical_block_ids
|
||||
|
||||
|
||||
Reference in New Issue
Block a user