[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Dakai An
2026-10-09 15:48:37 -04:00
committed by GitHub
co-authored by Yifan Qiao OpenAI Codex OpenAI Codex Lucas Wilkinson Claude Opus 5.5 Lucas Wilkinson
parent 3766a67a32
commit 02e55d2af3
83 changed files with 1280 additions and 1986 deletions
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.platforms import current_platform
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backend import AttentionBackend
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheConfig
from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
if TYPE_CHECKING:
@@ -351,6 +352,35 @@ def get_current_attn_backends(
]
def get_current_attn_backends_and_specs(
vllm_config: VllmConfig,
kv_cache_config: KVCacheConfig,
fallback: list[type[AttentionBackend]],
) -> tuple[list[type[AttentionBackend]], list[AttentionSpec | None]]:
"""Distinct (backend, spec) pairs of the transfer layers, else ``fallback``.
Compressed caches are skipped: their kernel pages never split transferred
blocks.
"""
pairs: dict[tuple[type[AttentionBackend], AttentionSpec | None], None] = {}
layer_type = cast(type[Any], AttentionLayerBase)
for group in kv_cache_config.transfer_groups:
specs = getattr(group.kv_cache_spec, "kv_cache_specs", {})
layers = get_layers_from_vllm_config(vllm_config, layer_type, group.layer_names)
for name, layer in layers.items():
spec = specs.get(name, group.kv_cache_spec)
if isinstance(spec, AttentionSpec) and spec.tokens_per_state > 1:
layout = vllm_config.cache_config.get_resolved_kv_cache_layout()
assert layout.is_block_outermost, "Compressed caches must be packed"
continue
if not isinstance(spec, AttentionSpec):
spec = None
pairs[layer.get_attn_backend(), spec] = None
if not pairs:
return fallback, [None] * len(fallback)
return [backend for backend, _ in pairs], [spec for _, spec in pairs]
def get_current_attn_backend(
vllm_config: VllmConfig, layer_names: list[str] | None = None
) -> type[AttentionBackend]:
@@ -26,6 +26,7 @@ from vllm.distributed.kv_transfer.kv_connector.utils import (
EngineId,
TransferTopology,
get_current_attn_backends,
get_current_attn_backends_and_specs,
)
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
@@ -59,8 +60,8 @@ from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import (
AttentionSpec,
CircularBufferSpec,
FullAttentionSpec,
KpoolTailSpec,
KVCacheSpec,
MambaSpec,
MLAAttentionSpec,
@@ -69,7 +70,6 @@ from vllm.v1.kv_cache_interface import (
)
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.request import RequestStatus
from vllm.v1.worker.block_table import BlockTable
from vllm.v1.worker.utils import select_common_block_size
logger = init_logger(__name__)
@@ -1350,7 +1350,10 @@ class MooncakeConnectorWorker:
# physical block sizes. Pick the common (smallest) block size so that
# KV-cache registration and transfer work correctly for both models.
backends = get_current_attn_backends(self.vllm_config)
kernel_block_size = select_common_block_size(self.block_size, backends)
backends, specs = get_current_attn_backends_and_specs(
self.vllm_config, self.kv_cache_config, backends
)
kernel_block_size = select_common_block_size(self.block_size, backends, specs)
if self.block_size != kernel_block_size:
logger.info_once(
"User-specified logical block size (%s) does not match"
@@ -1727,11 +1730,12 @@ class MooncakeConnectorWorker:
)
group_specs = self.kv_cache_config.transfer_groups
return [
BlockTable.map_to_kernel_blocks(
np.array(group),
self._physical_blocks_per_logical_kv_block,
block_arange,
).tolist()
(
np.array(group)[:, None] * self._physical_blocks_per_logical_kv_block
+ block_arange
)
.reshape(-1)
.tolist()
if not isinstance(group_specs[i].kv_cache_spec, MambaSpec)
else group
for i, group in enumerate(block_ids)
@@ -2054,7 +2058,7 @@ class MooncakeConnectorWorker:
# hetero PP can align by name (#56033). A non-contiguous row cannot
# name a sub-span, so it is one region anchored at the allocation.
use_packed_row = (
not isinstance(layer_spec, (MambaSpec, KpoolTailSpec))
not isinstance(layer_spec, (MambaSpec, CircularBufferSpec))
and storage_is_block_major
and block_stride > 0
and (is_mla_region or not hnc_contiguous)
@@ -2121,9 +2125,7 @@ class MooncakeConnectorWorker:
block_len = region_cache.stride(0) * region_cache.element_size()
region_base_addresses.append(base_addr)
if isinstance(layer_spec, KpoolTailSpec):
kv_block_len = layer_spec.unpadded_page_size_bytes // 2
elif isinstance(layer_spec, AttentionSpec) and block_is_contiguous:
if isinstance(layer_spec, AttentionSpec) and block_is_contiguous:
assert (
layer_spec.page_size_bytes
% self._physical_blocks_per_logical_kv_block
@@ -2133,6 +2135,8 @@ class MooncakeConnectorWorker:
layer_spec.page_size_bytes
// self._physical_blocks_per_logical_kv_block
)
elif isinstance(layer_spec, MambaSpec) and block_is_contiguous:
kv_block_len = layer_spec.page_size_bytes
else:
kv_block_len = block_len
if kv_block_len > block_len:
@@ -28,6 +28,7 @@ from vllm.distributed.kv_transfer.kv_connector.utils import (
EngineTransferInfo,
TransferTopology,
get_current_attn_backends,
get_current_attn_backends_and_specs,
kv_postprocess_blksize_and_layout_on_receive,
kv_postprocess_blksize_on_receive,
)
@@ -79,7 +80,6 @@ from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
KpoolTailSpec,
KVCacheLayout,
KVCacheSpec,
MambaSpec,
@@ -88,7 +88,6 @@ from vllm.v1.kv_cache_interface import (
UniformTypeKVCacheSpecs,
iter_layer_specs,
)
from vllm.v1.worker.block_table import BlockTable
from vllm.v1.worker.utils import select_common_block_size
if TYPE_CHECKING:
@@ -120,46 +119,6 @@ def _share_storage_and_block_stride(caches: list[torch.Tensor]) -> bool:
return len(block_strides) == len(storage_ptrs) == 1
def _tensor_byte_span_end(cache: torch.Tensor) -> int:
"""Return the exclusive end address touched by a nonnegative-stride view."""
if cache.numel() == 0:
return cache.data_ptr()
if any(stride < 0 for stride in cache.stride()):
raise ValueError("NIXL cache views must have nonnegative strides")
max_element_offset = sum(
(size - 1) * stride for size, stride in zip(cache.shape, cache.stride())
)
return cache.data_ptr() + (max_element_offset + 1) * cache.element_size()
def _uses_dense_virtual_transfer_pages(
layer_spec: KVCacheSpec,
cache: torch.Tensor,
physical_page_size: int,
num_blocks: int,
) -> bool:
"""Return whether a compressed kernel view can be split into NIXL pages."""
if not (
isinstance(layer_spec, MLAAttentionSpec)
and layer_spec.tokens_per_state > 1
and cache.ndim == 4
and cache.shape[1] == 1
and cache.is_contiguous()
and physical_page_size > 0
and layer_spec.state_content_size_bytes > 0
):
return False
block_stride = cache.stride(0) * cache.element_size()
return (
block_stride > physical_page_size
and block_stride % physical_page_size == 0
and physical_page_size % layer_spec.state_content_size_bytes == 0
and cache.shape[0] * (block_stride // physical_page_size) == num_blocks
and cache.nbytes == num_blocks * physical_page_size
)
class NixlBaseConnectorWorker:
"""Base implementation of Worker side methods shared by pull and push."""
@@ -985,8 +944,10 @@ class NixlBaseConnectorWorker:
)
def _sync_block_size_with_kernel(self) -> None:
backends = get_current_attn_backends(self.vllm_config)
kernel_block_size = select_common_block_size(self.block_size, backends)
backends, specs = get_current_attn_backends_and_specs(
self.vllm_config, self.kv_cache_config, self.attn_backends
)
kernel_block_size = select_common_block_size(self.block_size, backends, specs)
# Number of blocks not accounting for kernel block mismatches
self._logical_num_blocks = self.num_blocks
if self.block_size != kernel_block_size:
@@ -1425,7 +1386,7 @@ class NixlBaseConnectorWorker:
if self._supports_pp_hma and self._has_packed_cache:
# PP can reorder the same backends across stages of a packed model.
backend_name = ",".join(
sorted(backend.get_name() for backend in self.attn_backends)
sorted({backend.get_name() for backend in self.attn_backends})
)
self.compat_hash = compute_nixl_compatibility_hash(
self.vllm_config,
@@ -1478,31 +1439,6 @@ class NixlBaseConnectorWorker:
):
raise NotImplementedError("PP push with packed KV caches requires MLA")
layer_specs: dict[str, KVCacheSpec] = {}
compressed_region_owners: dict[int, torch.Tensor] = {}
for layer_name, cache in xfer_buffers.items():
layer_spec = self._layer_specs.get(layer_name)
if isinstance(layer_spec, UniformTypeKVCacheSpecs):
layer_spec = layer_spec.kv_cache_specs[layer_name]
if layer_spec is None:
continue
layer_specs[layer_name] = layer_spec
physical_page_size = (
layer_spec.page_size_bytes
if isinstance(layer_spec, MambaSpec)
else layer_spec.page_size_bytes
// self._physical_blocks_per_logical_kv_block
)
num_blocks = (
self._logical_num_blocks
if isinstance(layer_spec, MambaSpec)
else self.num_blocks
)
if _uses_dense_virtual_transfer_pages(
layer_spec, cache, physical_page_size, num_blocks
):
compressed_region_owners.setdefault(cache.data_ptr(), cache)
track_region_layers = self._tracks_region_layers()
region_layers: list[list[str]] = []
packed_member_layouts: dict[str, tuple[int, int]] = {}
@@ -1527,7 +1463,7 @@ class NixlBaseConnectorWorker:
# kernel requires a specific block size. This leads to SSM and FA layers
# having different num_blocks.
# `_physical_blocks_per_logical_kv_block` ratio is used to adjust for this.
layer_spec = layer_specs.get(layer_name)
layer_spec = self._layer_specs.get(layer_name)
if layer_spec is None:
logger.debug(
"Skipping layer %s as no KVCache spec is present. "
@@ -1587,32 +1523,6 @@ class NixlBaseConnectorWorker:
region_device_id,
)
if isinstance(layer_spec, KpoolTailSpec):
compressed_owner = compressed_region_owners.get(cache.data_ptr())
if compressed_owner is not None:
owner_storage = compressed_owner.untyped_storage()
owner_end = compressed_owner.data_ptr() + compressed_owner.nbytes
tail_is_covered = (
compressed_owner.is_contiguous()
and owner_storage.data_ptr() == storage_addr
and _tensor_byte_span_end(cache) <= owner_end
)
if not tail_is_covered:
raise AssertionError(
"Kpool tail cache is not fully covered by its compressed "
f"indexer region: layer={layer_name}, "
f"tail_shape={tuple(cache.shape)}, "
f"tail_stride={tuple(cache.stride())}, "
f"owner_shape={tuple(compressed_owner.shape)}, "
f"owner_stride={tuple(compressed_owner.stride())}"
)
logger.debug(
"Skipping layer %s because its compressed indexer region "
"covers the same storage",
layer_name,
)
continue
if isinstance(layer_spec, MambaSpec):
physical_ratio = self._physical_blocks_per_logical_kv_block
block_len = physical_page_size // physical_ratio
@@ -1647,9 +1557,6 @@ class NixlBaseConnectorWorker:
and cache[0].is_contiguous()
and cache[0].nbytes == physical_page_size
)
virtual_transfer_pages = _uses_dense_virtual_transfer_pages(
layer_spec, cache, physical_page_size, num_blocks
)
# Push workers address the pages of packed MLA rows by layer.
packed_mla_push = (
track_region_layers
@@ -1658,12 +1565,7 @@ class NixlBaseConnectorWorker:
and storage_is_block_major
and is_mla_region
)
if virtual_transfer_pages:
# A compressed kernel row can contain multiple NIXL transfer pages.
region_specs = [
(cache.data_ptr(), physical_page_size, physical_page_size)
]
elif packed_mla_push and use_layer_name_routing:
if packed_mla_push and use_layer_name_routing:
# PP>1 prefill stage: register only this layer's page of each row.
region_specs = [
(cache.data_ptr(), physical_page_size, block_stride)
@@ -1671,7 +1573,7 @@ class NixlBaseConnectorWorker:
elif packed_mla_push:
# PP=1 decode or prefill: register whole rows, and advertise where
# this layer's page sits so a PP>1 prefill can write into it.
storage_block_len = registration_len // num_blocks
storage_block_len = block_stride
region_specs = [
(registration_base, storage_block_len, storage_block_len)
]
@@ -3388,11 +3290,9 @@ class NixlBaseConnectorWorker:
physical_block_ids.append(group)
else:
physical_block_ids.append(
BlockTable.map_to_kernel_blocks(
np.array(group),
ratio,
block_arange,
).tolist()
(np.array(group)[:, None] * ratio + block_arange)
.reshape(-1)
.tolist()
)
return physical_block_ids
@@ -1376,14 +1376,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
is_index_group_leader=self.indexer is not None,
non_causal_multi_token_decode=self.non_causal_multi_token_decode,
)
# SM100 FlashMLA paged kernels also express TMA coordinates in token rows.
uses_tma_rows = (
self.attn_backend.get_name() == "FLASHMLA_SPARSE"
and self.kv_cache_dtype in ("fp8_ds_mla", "nvfp4_ds_mla")
and current_platform.is_device_capability_family(100)
)
if self._uses_flat_kv_cache() or uses_tma_rows:
spec = replace(spec, block_stride_alignment=spec.state_content_size_bytes)
return spec
def _v_up_proj(self, x: torch.Tensor, out: torch.Tensor):
@@ -669,7 +669,7 @@ class DeepseekV41ModelState(DefaultModelState):
copy.create_metadata_builders(
self.vllm_config,
self.device,
group.metadata_builders[0].kernel_block_size,
group.kernel_block_size,
)
copies.append(copy)
own_groups.append(copies)
+13 -49
View File
@@ -33,9 +33,9 @@ from vllm.model_executor.utils import maybe_disable_graph_partition
from vllm.models.glm5next.nvidia.ops.kpool_compress import fwht128_quant_fp8
from vllm.models.glm5next.sparse_indexer import SparseAttnIndexerKpool
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES
from vllm.utils.math_utils import cdiv, next_power_of_2
from vllm.v1.kv_cache_interface import KpoolTailSpec, MLAAttentionSpec
from vllm.v1.attention.backends.mla.indexer import Glm5NextIndexerBackend
from vllm.v1.kv_cache_interface import CircularBufferSpec, MLAAttentionSpec
logger = init_logger(__name__)
@@ -89,15 +89,9 @@ class Glm5NextIndexerCache(DeepseekV32IndexerCache):
The indexer shares one block with the co-located MLA (a single
``MLAAttentionSpec`` / block_table), so ``block_size`` is the model-wide
``cache_config.block_size``. DeepGEMM's paged-MQA kernel
(``csrc/apis/attention.hpp``) requires ``block_kv`` to be exactly 32 or
64, so the storage block is virtually split into pool pages of the
largest such size that tiles it (``storage_kernel_block_size``); this
needs ``block_size`` to be a multiple of ``index_kpool * 32`` (512 for
``index_kpool = 16``). A smaller block (e.g. the default 64) silently
collapses ``storage_block_size`` (64 // 16 = 4) and only fails later at
the opaque C++ assert; ``get_kv_cache_spec`` guards this up front
instead.
``cache_config.block_size``. DeepGEMM's paged-MQA kernel takes 32- or
64-state pages (only 64 on SM120), which ``Glm5NextIndexerBackend``
declares as its kernel block sizes.
"""
def __init__(
@@ -113,12 +107,6 @@ class Glm5NextIndexerCache(DeepseekV32IndexerCache):
head_dim=head_dim, dtype=dtype, prefix=prefix, cache_config=cache_config
)
assert index_kpool > 1, "Glm5NextIndexerCache expects index_kpool > 1"
# Keep chunked-prefill boundaries aligned to complete pools.
assert cache_config.block_size % index_kpool == 0, (
"Glm5NextIndexerCache: cache_config.block_size "
f"({cache_config.block_size}) must be a multiple of index_kpool "
f"({index_kpool}) so chunked-prefill boundaries stay pool-aligned."
)
self._index_kpool = index_kpool
def get_kv_cache_spec(self, vllm_config: VllmConfig):
@@ -128,33 +116,10 @@ class Glm5NextIndexerCache(DeepseekV32IndexerCache):
# ``tokens_per_state`` is the KV-spec representation of kpool
# compression in the current cache-layout API.
assert isinstance(spec, MLAAttentionSpec)
spec = replace(spec, tokens_per_state=self._index_kpool)
return replace(spec, tokens_per_state=self._index_kpool)
# DeepGEMM paged-MQA takes block_kv in {32, 64}; the storage block
# (= block_size // index_kpool) is virtually split into pool pages of
# the largest such size that tiles it, so it must be a multiple of 32.
storage_block_size = spec.block_size // self._index_kpool
assert (
spec.block_size % self._index_kpool == 0 and storage_block_size % 32 == 0
), (
"Glm5NextIndexerCache: kpool indexer requires cache block_size to "
f"be a multiple of index_kpool * 32 ({self._index_kpool * 32}) so "
"that DeepGEMM paged-MQA pool pages (32 or 64 entries) tile the "
f"storage block, got block_size={spec.block_size} -> "
f"storage_block_size={storage_block_size}."
)
max_page_size = max(PAGED_MQA_PAGE_SIZES)
min_page_size = min(PAGED_MQA_PAGE_SIZES)
if storage_block_size <= max_page_size:
page_size = storage_block_size
elif storage_block_size % max_page_size == 0:
page_size = max_page_size
else:
page_size = min_page_size
return replace(
spec,
storage_block_size=page_size * self._index_kpool,
)
def get_attn_backend(self):
return Glm5NextIndexerBackend
class Glm5NextTailCache(DeepseekV32IndexerCache):
@@ -164,9 +129,9 @@ class Glm5NextTailCache(DeepseekV32IndexerCache):
``ring`` slots per request, overwritten in place by ``pos % ring``
as decode/spec-decode advances. Prefill seeds it (instead of discarding the
tail raw K+gate); the connector transfers it across PD; decode reads it to
compress the boundary pool correctly. ``KpoolTailSpec`` /
``KpoolTailManager`` provide the no-prune, 1-block/req allocation that lets
the in-progress pool survive across steps and across transfer.
compress the boundary pool correctly. ``CircularBufferSpec`` provides the
no-prune, 1-block/req allocation that lets the in-progress pool survive
across steps and across transfer.
Stores raw bf16 K (``head_dim``) as the "K" half of each block and the
bf16 gate score (``head_dim``) as the "V" half -- not the fp8-compressed
@@ -202,13 +167,12 @@ class Glm5NextTailCache(DeepseekV32IndexerCache):
f"({self.cache_config.block_size}) must be a multiple of the "
f"tail ring ({ring})"
)
return KpoolTailSpec(
return CircularBufferSpec(
block_size=ring,
num_kv_heads=2,
head_size=self.head_dim,
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=ring,
# Tiny per-request ring; every DCP rank keeps the full tail so the
# rank owning a pool can compress it.
dcp_sharded=False,
@@ -303,7 +267,7 @@ class Indexer(nn.Module):
# Paged tail cache (in-progress pool's raw K + gate score). Written by
# prefill (seeds the boundary pool) and decode (per-step stash); read by
# the decode kernel to compress the boundary pool. Transferred across PD
# so the decode side sees the prefill tail. See KpoolTailSpec/Manager.
# so the decode side sees the prefill tail. See CircularBufferManager.
self.tail_cache = Glm5NextTailCache(
head_dim=self.head_dim,
dtype=torch.bfloat16,
@@ -396,11 +396,6 @@ def _kpool_tail_seed_kernel(
ahead belongs to a different tail block (or is past the batch / padding,
slot < 0). ``tslot = block * RING + pos % RING``; the destination is
``tail[block, {0:K, 1:score}, pos % RING, :]``.
The tail cache aliases the indexer cache with the indexer's (padded) block
stride, so blocks are addressed through ``TAIL_BLOCK_ELEMS`` /
``KPOOL_HEAD`` (``tail.stride(0)`` / ``tail.stride(1)``), never as a dense
``[num_blocks, 2, KPOOL, HEAD_DIM]`` array.
"""
i = tl.program_id(0)
t = tl.load(tslot_ptr + i).to(tl.int64)
-20
View File
@@ -414,26 +414,6 @@ class CudaPlatformBase(Platform):
return valid_backends_priorities, invalid_reasons
@classmethod
def _get_indexer_block_alignment(cls, vllm_config: VllmConfig) -> int | None:
index_kpool = getattr(
vllm_config.model_config.hf_text_config, "index_kpool", None
)
if not index_kpool or index_kpool <= 1:
return None
from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES
# kpool paged-MQA indexer: the storage block (block_size /
# index_kpool) is virtually split into pool pages, so block_size
# must be a multiple of index_kpool times a legal pool page.
page = min(PAGED_MQA_PAGE_SIZES)
if cls.is_device_capability_family(120):
# On sm120 the DeepGEMM paged-MQA kernel only accepts block_kv
# 64 for the fp8 indexer cache, so align to the largest pool
# page here to make the page split land on 64 not the min 32.
page = max(PAGED_MQA_PAGE_SIZES)
return index_kpool * page
@classmethod
def get_attn_backend_cls(
cls,
+18 -21
View File
@@ -737,14 +737,14 @@ class Platform:
for candidate in candidates:
if all(b.supports_block_size(candidate) for b in backend_classes):
return candidate
raise ValueError(
"The attention backends share no supported KV cache block size ("
+ "; ".join(
f"{b.get_name()}: {b.get_supported_kernel_block_sizes()}"
for b in backend_classes
raise ValueError(
"The attention backends share no supported KV cache block size ("
+ "; ".join(
f"{b.get_name()}: {b.get_supported_kernel_block_sizes()}"
for b in backend_classes
)
+ ")."
)
+ ")."
)
@classmethod
def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None:
@@ -935,17 +935,6 @@ class Platform:
if cache_config.mamba_page_size_padded is not None:
cache_config.mamba_page_size_padded = shared_page
@classmethod
def _get_indexer_block_alignment(cls, vllm_config: "VllmConfig") -> int | None:
"""Extra ``block_size`` multiple a sparse indexer needs, else ``None``.
The CUDA kpool paged-MQA indexer virtually splits each storage block
into pool pages, so ``block_size`` must be a multiple of
``index_kpool * min(PAGED_MQA_PAGE_SIZES)`` — implemented in the CUDA
platform override. Other platforms impose no extra constraint.
"""
return None
@classmethod
def _align_hybrid_block_size(
cls,
@@ -1077,6 +1066,14 @@ class Platform:
),
cache_config.block_size,
)
# Also a multiple of every backend's smallest kernel block.
backend_block_alignment = lcm(
*(
min(s.base if isinstance(s, MultipleOf) else s for s in sizes)
for b in cls._find_non_ssm_backends(vllm_config)
if (sizes := b.get_supported_kernel_block_sizes())
)
)
if model_config.use_mla:
# TRTLLM/FlashInfer MLA decode kernels require the physical
# number of kernel blocks to be aligned to 128 / kernel_block_size.
@@ -1090,9 +1087,9 @@ class Platform:
mamba_page_size,
kernel_block_alignment_size * attn_page_size_1_token,
)
indexer_align = cls._get_indexer_block_alignment(vllm_config)
if indexer_align:
attn_block_size = indexer_align * cdiv(attn_block_size, indexer_align)
attn_block_size = backend_block_alignment * cdiv(
attn_block_size, backend_block_alignment
)
if cache_config.block_size < attn_block_size:
cache_config.block_size = attn_block_size
+5 -4
View File
@@ -29,10 +29,11 @@ _DEEPGEMM_BLACKWELL_EXCLUDED_MODEL_TYPES: set[str] = {
"qwen3_5_moe_text",
}
# KV page sizes (in cache entries) supported by the paged-MQA logits kernels
# (fp8_fp4_paged_mqa_logits / get_paged_mqa_logits_metadata). Larger storage
# blocks must be virtually split into one of these page sizes.
PAGED_MQA_PAGE_SIZES = (32, 64)
def get_paged_mqa_page_sizes() -> tuple[int, ...]:
"""KV page sizes (in cache entries) the paged-MQA logits kernels take (only
64 on SM120); larger storage blocks are split into one of these."""
return (64,) if current_platform.is_device_capability_family(120) else (32, 64)
def should_auto_disable_deep_gemm(model_type: str | None) -> bool:
+1 -7
View File
@@ -48,12 +48,10 @@ class AttentionType(str, Enum):
"""Attention between dec. Q and enc. K/V for encoder-decoder."""
@dataclass(frozen=True)
class MultipleOf:
base: int
def __init__(self, base: int):
self.base = base
def __repr__(self) -> str:
return f"MultipleOf({self.base})"
@@ -622,10 +620,6 @@ class AttentionMetadataBuilder(ABC, Generic[M]):
self.layer_names = layer_names
self.vllm_config = vllm_config
self.device = device
self.kernel_block_size: int | None = None
def set_kernel_block_size(self, kernel_block_size: int) -> None:
self.kernel_block_size = kernel_block_size
@classmethod
def get_cudagraph_support(
-5
View File
@@ -297,11 +297,6 @@ def create_composite_attention_backend(
)
self.routing = routing_policy(layer_names, vllm_config)
def set_kernel_block_size(self, kernel_block_size):
super().set_kernel_block_size(kernel_block_size)
for builder in self._builders:
builder.set_kernel_block_size(kernel_block_size)
@classmethod
def get_cudagraph_support(cls, vllm_config, kv_cache_spec):
return min(
@@ -95,7 +95,9 @@ class CompressedSlotMappingKernel(
block_table_ptr + batch_idx * block_table_stride + block_ids,
mask=mask & is_valid,
)
slot_ids = block_numbers * block_size + pos_after_compress % block_size
slot_ids = block_numbers.to(tl.int64) * block_size + (
pos_after_compress % block_size
)
# NOTE
slot_ids = tl.where(is_valid, slot_ids, PAD_ID)
@@ -24,6 +24,7 @@ from vllm.v1.attention.backend import (
from vllm.v1.attention.backends.fa_utils import flash_attn_supports_mla
from vllm.v1.attention.backends.mla.index_group import HiSparseMLAIndexGroup
from vllm.v1.attention.backends.mla.sparse_utils import (
align_blocks_to_rows,
flat_kv_row_view,
triton_convert_req_index_to_global_index,
)
@@ -42,7 +43,11 @@ class FlashAttnMLASparseBackend(AttentionBackend):
@staticmethod
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
return [64]
return [MultipleOf(64)]
@classmethod
def customize_spec(cls, spec: AttentionSpec) -> AttentionSpec:
return align_blocks_to_rows(spec)
@staticmethod
def get_name() -> str:
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""FlashInfer sparse MLA attention backend."""
import math
from dataclasses import dataclass
from typing import TYPE_CHECKING, ClassVar
@@ -30,6 +31,7 @@ from vllm.v1.attention.backend import (
)
from vllm.v1.attention.backends.mla.index_group import HiSparseMLAIndexGroup
from vllm.v1.attention.backends.mla.sparse_utils import (
align_blocks_to_rows,
flat_kv_row_view,
prepare_sparse_mla_safe_lengths,
triton_convert_req_index_to_global_index,
@@ -69,6 +71,13 @@ class _FlashInferMLASparseBackendBase(AttentionBackend):
def is_sparse(cls) -> bool:
return True
@classmethod
def customize_spec(cls, spec: AttentionSpec) -> AttentionSpec:
# The kernels read the rows as pages of their kernel block size.
(page,) = cls.get_supported_kernel_block_sizes(spec)
assert isinstance(page, MultipleOf)
return align_blocks_to_rows(spec, page.base)
class FlashInferMLASparseTRTLLMBackend(_FlashInferMLASparseBackendBase):
"""FlashInfer sparse MLA backend using the TRTLLM-gen launcher."""
@@ -84,7 +93,7 @@ class FlashInferMLASparseTRTLLMBackend(_FlashInferMLASparseBackendBase):
@staticmethod
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
return [32, 64]
return [MultipleOf(32)]
@staticmethod
def get_impl_cls() -> type[MLAAttentionImpl]:
@@ -173,7 +182,7 @@ class FlashInferMLASparseSM120Backend(_FlashInferMLASparseBackendBase):
@staticmethod
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
return [64, 256]
return [MultipleOf(64)]
@staticmethod
def get_impl_cls() -> type[MLAAttentionImpl]:
@@ -575,7 +584,7 @@ class FlashInferMLASparseImpl(SparseMLACommonImpl[FlashInferMLASparseMetadata]):
assert prefill_lse is not None
return output, torch.cat((decode_lse, prefill_lse))
_, block_stride_rows = flat_kv_row_view(
kv_rows, block_stride_rows = flat_kv_row_view(
kv_c_and_k_pe_cache, attn_metadata.block_size
)
@@ -602,7 +611,12 @@ class FlashInferMLASparseImpl(SparseMLACommonImpl[FlashInferMLASparseMetadata]):
return self._run_mqa_kernel(
q,
kv_c_and_k_pe_cache,
# Block sizes and strides have been aligned to 32 rows.
kv_rows.view(
-1,
math.gcd(block_stride_rows, attn_metadata.block_size, 64),
kv_rows.shape[-1],
),
topk_indices_physical,
seq_lens,
)
@@ -16,6 +16,7 @@ from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
)
from vllm.v1.attention.backends.mla.index_group import HiSparseMLAIndexGroup
from vllm.v1.attention.backends.mla.sparse_utils import (
flat_kv_row_view,
triton_convert_req_index_to_global_index,
)
@@ -184,6 +185,9 @@ class FlashInferMLASparseSM120Impl(SparseMLACommonImpl[FlashInferMLASparseMetada
output = torch.cat(outputs) if len(outputs) > 1 else outputs[0]
return output, None
kv_rows, block_stride_rows = flat_kv_row_view(
kv_c_and_k_pe_cache, attn_metadata.block_size
)
topk_indices_physical = cast(
torch.Tensor,
triton_convert_req_index_to_global_index(
@@ -191,13 +195,15 @@ class FlashInferMLASparseSM120Impl(SparseMLACommonImpl[FlashInferMLASparseMetada
attn_metadata.block_table,
topk_indices,
BLOCK_SIZE=attn_metadata.block_size,
BLOCK_STRIDE_ROWS=block_stride_rows,
NUM_TOPK_TOKENS=topk_indices.shape[1],
),
)
return (
self._run_mqa_kernel(
q,
kv_c_and_k_pe_cache,
# The SM120 kernels are built for 64-row pages only.
kv_rows.view(-1, 64, kv_rows.shape[-1]),
topk_indices_physical,
),
None,
@@ -26,7 +26,7 @@ needs no D2H sync. Per-step content (top-k slots) is written into the
reserved buffers by kernels inside the captured forward, and captured runs
read the refreshed plan buffers on replay.
KV cache format: plain contiguous E4M3 ``[num_blocks, block_size, 512]``
KV cache format: E4M3 ``[num_blocks, block_size, 512]`` (any block stride)
(uint8 storage) with a per-tensor ``k_scale``; BF16 caches also work. The
per-token x 128-channel-group ``ckv_scale_arr`` layout is supported by the
kernel but not wired yet (it needs a group-quantizing cache-write op).
@@ -57,9 +57,11 @@ from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
FlashInferMLASparseMetadataBuilder,
)
from vllm.v1.attention.backends.mla.sparse_utils import (
align_blocks_to_rows,
flat_kv_row_view,
triton_convert_req_index_to_global_index,
)
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout
from vllm.v1.kv_cache_interface import AttentionSpec
_FP8_KV_DTYPES = ("fp8", "fp8_e4m3")
_WORKSPACE_BYTES = 128 * 1024 * 1024
@@ -99,7 +101,12 @@ def _pack_topk_indices(
end = tl.load(indptr + row + 1)
mask = (cols < end - start) & (cols < WIDTH)
values = tl.load(slots + row * WIDTH + cols, mask=mask, other=0)
tl.store(indices + start + cols, tl.maximum(values, 0), mask=mask)
first_slot = tl.load(slots + row * WIDTH)
tl.store(
indices + start + cols,
tl.where(values >= 0, values, tl.maximum(first_slot, 0)),
mask=mask,
)
@triton.jit
@@ -149,6 +156,10 @@ class FlashInferMLASparseSM90Backend(AttentionBackend):
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
return [MultipleOf(64)]
@classmethod
def customize_spec(cls, spec: AttentionSpec) -> AttentionSpec:
return align_blocks_to_rows(spec)
@staticmethod
def get_name() -> str:
return "FLASHINFER_MLA_SPARSE_SM90"
@@ -222,10 +233,6 @@ class FlashInferMLASparseSM90Backend(AttentionBackend):
) -> tuple[int, ...]:
return (num_blocks, block_size, head_size)
@classmethod
def supported_kv_cache_layouts(cls) -> tuple[KVCacheLayout, ...]:
return (KVCacheLayout.LBHNC,)
class _SM90State:
"""Builder-owned wrapper, capture-stable buffers, and plan parameters.
@@ -602,6 +609,9 @@ class FlashInferMLASparseSM90Impl(SparseMLACommonImpl[FlashInferMLASparseSM90Met
assert self.topk_indices_buffer is not None
topk_indices = self.topk_indices_buffer[:num_tokens]
kv_c_and_k_pe_cache, block_stride_rows = flat_kv_row_view(
kv_c_and_k_pe_cache, attn_metadata.block_size
)
# return_valid_counts=True keeps the compacted-prefix layout: valid
# entries at [0, valid_count), -1 past it — exactly the prefix the
# planned per-row lengths address.
@@ -610,6 +620,7 @@ class FlashInferMLASparseSM90Impl(SparseMLACommonImpl[FlashInferMLASparseSM90Met
attn_metadata.block_table,
topk_indices,
BLOCK_SIZE=attn_metadata.block_size,
BLOCK_STRIDE_ROWS=block_stride_rows,
NUM_TOPK_TOKENS=topk_indices.shape[1],
return_valid_counts=True,
)
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from dataclasses import dataclass, replace
from typing import TYPE_CHECKING, ClassVar
import numpy as np
@@ -36,6 +36,7 @@ from vllm.v1.attention.backend import (
)
from vllm.v1.attention.backends.mla.index_group import HiSparseMLAIndexGroup
from vllm.v1.attention.backends.mla.sparse_utils import (
align_blocks_to_rows,
flat_kv_row_view,
request_row_bounds,
triton_convert_req_index_to_global_index,
@@ -53,7 +54,7 @@ from vllm.v1.attention.ops.flashmla import (
flash_mla_with_kvcache,
get_mla_metadata,
)
from vllm.v1.kv_cache_interface import AttentionSpec
from vllm.v1.kv_cache_interface import AttentionSpec, KVQuantMode
from vllm.v1.worker.workspace import current_workspace_manager
if TYPE_CHECKING:
@@ -132,6 +133,17 @@ QUANTIZED_DS_MLA_CACHE_FORMATS: frozenset[str] = frozenset(
FP8_DS_MLA_ROPE_DIM = 64
def _fp8_kv_pages(
kv_cache: torch.Tensor, block_size: int
) -> tuple[torch.Tensor, int | None]:
"""The cache as the fp8 kernel's 64-token pages, and the block stride in
rows when a block spans several pages."""
if block_size == 64:
return kv_cache, None
rows, block_stride_rows = flat_kv_row_view(kv_cache, block_size)
return rows.view(-1, 64, rows.shape[-1]), block_stride_rows
class FlashMLASparseBackend(AttentionBackend):
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
@@ -144,7 +156,23 @@ class FlashMLASparseBackend(AttentionBackend):
@staticmethod
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
return [64]
if (
isinstance(kv_cache_spec, AttentionSpec)
and kv_cache_spec.kv_quant_mode != KVQuantMode.NONE
and current_platform.is_device_capability_family(100)
):
return [64]
return [MultipleOf(64)]
@classmethod
def customize_spec(cls, spec: AttentionSpec) -> AttentionSpec:
# Off SM100 the quantized kernel reads 64-token pages, only needed
# when blocks span several.
if spec.kv_quant_mode != KVQuantMode.NONE and (
not current_platform.is_device_capability_family(100)
):
return replace(spec, block_stride_alignment=MultipleOf(64))
return align_blocks_to_rows(spec)
@staticmethod
def get_name() -> str:
@@ -998,13 +1026,16 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
# and the prefill rows per chunk, against the gathered workspace.
lse: torch.Tensor | None = None
topk_length: torch.Tensor | None = None
kv_pages, block_stride_rows = _fp8_kv_pages(
kv_c_and_k_pe_cache, attn_metadata.block_size
)
if self.pcp_dcp_kv_gather:
pass
elif num_prefill_tokens == 0 and not uses_host_cache:
topk_indices, topk_length = self._convert_logical_to_physical_topk(
topk_indices,
attn_metadata,
block_stride_rows=None,
block_stride_rows=block_stride_rows,
return_valid_counts=True,
)
elif num_prefill_tokens > 0:
@@ -1013,6 +1044,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
attn_metadata.block_table,
topk_indices,
BLOCK_SIZE=attn_metadata.block_size,
BLOCK_STRIDE_ROWS=block_stride_rows,
NUM_TOPK_TOKENS=topk_indices.shape[1],
HAS_PREFILL_WORKSPACE=has_prefill_workspace,
prefill_workspace_request_ids=prefill_request_ids,
@@ -1056,7 +1088,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
topk_indices = topk_indices.view(num_decodes, seq_len, -1)
attn_out, _ = self._fp8_flash_mla_kernel(
q=q,
kv_c_and_k_pe_cache=kv_c_and_k_pe_cache,
kv_c_and_k_pe_cache=kv_pages,
topk_indices=topk_indices,
kernel_metadata=fp8_metadata.decode.kernel_metadata,
)
@@ -1173,6 +1205,9 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
# req_id_per_token covers the whole batch; slice it to the MQA tokens
# (q may exclude prefill tokens routed to dense MHA).
req_id_per_token = attn_metadata.req_id_per_token[: topk_indices.shape[0]]
kv_pages, block_stride_rows = _fp8_kv_pages(
kv_c_and_k_pe_cache, attn_metadata.block_size
)
if self.dcp_world_size > 1:
# The indexer emits global token ids; keep this rank's shard and
# convert to local slots. compact_valid_to_front=False keeps the
@@ -1187,6 +1222,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
dcp_rank=self.dcp_rank,
cp_kv_cache_interleave_size=attn_metadata.cp_kv_cache_interleave_size,
BLOCK_SIZE=attn_metadata.block_size,
BLOCK_STRIDE_ROWS=block_stride_rows,
NUM_TOPK_TOKENS=topk_indices.shape[1],
compact_valid_to_front=False,
)
@@ -1202,7 +1238,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
topk_indices = self._convert_logical_to_physical_topk(
topk_indices,
attn_metadata,
block_stride_rows=None,
block_stride_rows=block_stride_rows,
return_valid_counts=False,
)
else:
@@ -1211,12 +1247,13 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
block_table,
topk_indices,
BLOCK_SIZE=attn_metadata.block_size,
BLOCK_STRIDE_ROWS=block_stride_rows,
NUM_TOPK_TOKENS=topk_indices.shape[1],
)
_attn_out, _lse = self._fp8_flash_mla_kernel(
q=q.unsqueeze(0), # unsqueeze to add batch_dim: (T, H, D) -> (1, T, H, D)
kv_c_and_k_pe_cache=kv_c_and_k_pe_cache,
kv_c_and_k_pe_cache=kv_pages,
topk_indices=topk_indices.unsqueeze(0), # (T, topk) -> (1, T, topk)
kernel_metadata=fp8_metadata,
)
+19 -47
View File
@@ -7,7 +7,7 @@ import numpy as np
import torch
import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.distributed import get_dcp_group, get_pcp_group
from vllm.logger import init_logger
from vllm.model_executor.warmup.jit_warmup import kernel_launcher, zip_inputs
@@ -22,6 +22,7 @@ from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.deep_gemm import (
get_paged_mqa_logits_metadata,
get_paged_mqa_page_sizes,
has_deep_gemm,
native_next_n_supported,
)
@@ -234,12 +235,25 @@ class DeepseekV32IndexerBackend(AttentionBackend):
return DeepseekV32IndexerMetadataBuilder
class Glm5NextIndexerBackend(DeepseekV32IndexerBackend):
@staticmethod
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
if kv_cache_spec is not None:
# Worker get_kv_cache_spec: runs without the current config.
index_kpool = int(kv_cache_spec.tokens_per_state)
else:
# Platform block-size selection: no spec yet, under the current config.
model_config = get_current_vllm_config().model_config
index_kpool = model_config.hf_text_config.index_kpool
return [index_kpool * n for n in get_paged_mqa_page_sizes()]
class KpoolTailBackend(DeepseekV32IndexerBackend):
"""Storage-only backend for the GLM-5.3-Flash kpool tail cache."""
@classmethod
def supported_kv_cache_layouts(cls) -> tuple[KVCacheLayout, ...]:
return (KVCacheLayout.LBHNC,)
return (KVCacheLayout.BLHNC,)
@staticmethod
def get_name() -> str:
@@ -1065,8 +1079,6 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
dtype=torch.int32,
device=self.device,
)
self.indexer_decode_block_table_buffer: torch.Tensor | None = None
self._max_num_batched_tokens = scheduler_config.max_num_batched_tokens
# DCP not supported yet
self.supports_draft_decode_metadata_update = self.dcp_world_size == 1
@@ -1353,16 +1365,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
dcp_local_seq_lens = common_attn_metadata.dcp_local_seq_lens
compressed_slot_mapping = slot_mapping
indexer_block_table = block_table
if self.compress_ratio > 1:
kernel_block_size = self.kernel_block_size
if (
kernel_block_size is not None
and self.kv_cache_spec.block_size != kernel_block_size
and self.kv_cache_spec.block_size % kernel_block_size == 0
):
factor = self.kv_cache_spec.block_size // kernel_block_size
indexer_block_table = (block_table[:, ::factor] // factor).contiguous()
padded_num_tokens = num_tokens
local_slot_mapping = slot_mapping
if self.use_pcp:
@@ -1378,7 +1381,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
local_slot_mapping,
query_start_loc,
seq_lens,
indexer_block_table,
block_table,
self.kv_cache_spec.num_states,
self.compress_ratio,
out=self.compressed_slot_mapping_buffer,
@@ -1497,7 +1500,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
seq_lens,
compressed_seq_lens,
compressed_seq_lens_cpu,
indexer_block_table,
block_table,
self.compress_ratio,
query_slice=query_slice,
skip_kv_gather=query_slice.start > 0,
@@ -1632,27 +1635,6 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
)
)
if self.compress_ratio > 1:
kernel_block_size = self.kernel_block_size
if (
kernel_block_size is not None
and self.kv_cache_spec.block_size != kernel_block_size
and self.kv_cache_spec.block_size % kernel_block_size == 0
):
factor = self.kv_cache_spec.block_size // kernel_block_size
compressed = block_table[:, ::factor] // factor
rows, cols = compressed.shape
if self.indexer_decode_block_table_buffer is None:
self.indexer_decode_block_table_buffer = torch.zeros(
(self._max_num_batched_tokens, cols),
dtype=torch.int32,
device=self.device,
)
self.indexer_decode_block_table_buffer[:rows, :cols].copy_(
compressed
)
block_table = self.indexer_decode_block_table_buffer[:rows, :cols]
# Flattening always returns a buffer view, including single-token
# batches. Keep its address stable across varlen graph replays.
seq_lens_is_buffer_view = not use_native or next_n > 1
@@ -1741,17 +1723,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
# One query token per request, so the per-token decode block table is
# the request block table (at the indexer kernel's page size). build()
# may copy it into a builder buffer, so re-derive it for this batch.
block_table = metadata.block_table[: metadata.num_decode_tokens]
kernel_block_size = self.kernel_block_size
if (
self.compress_ratio > 1
and kernel_block_size is not None
and self.kv_cache_spec.block_size != kernel_block_size
and self.kv_cache_spec.block_size % kernel_block_size == 0
):
factor = self.kv_cache_spec.block_size // kernel_block_size
block_table = block_table[:, ::factor] // factor
decode.block_table.copy_(block_table)
decode.block_table.copy_(metadata.block_table[: metadata.num_decode_tokens])
if decode.indices is not None:
decode.indices.copy_(self.arange_buffer[: metadata.num_decode_tokens])
if decode.per_req_decode_lens is not None:
@@ -35,6 +35,10 @@ from vllm.v1.attention.backend import (
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
AiterMLAHelper,
)
from vllm.v1.attention.backends.mla.sparse_utils import (
align_blocks_to_rows,
flat_kv_row_view,
)
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
rocm_sparse_attn_decode_bf16,
@@ -160,6 +164,7 @@ def _convert_req_index_to_global_index_kernel(
# shapes (compile-time where possible)
max_num_blocks_per_req: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
BLOCK_STRIDE_ROWS: tl.constexpr,
BLOCK_N: tl.constexpr, # tile width along columns
# strides (in elements)
bt_stride0,
@@ -201,9 +206,8 @@ def _convert_req_index_to_global_index_kernel(
bt_ptr = block_table_ptr + req * bt_stride0 + block_id * bt_stride1
base = tl.load(bt_ptr, mask=valid_block, other=0)
# # If token == -1 OR block_id OOB, output 0; else base * BLOCK_SIZE + offset
out_val = tl.where(
is_invalid_tok | (~valid_block), 0, base * BLOCK_SIZE + inblock_off
is_invalid_tok | (~valid_block), 0, base * BLOCK_STRIDE_ROWS + inblock_off
)
out_ptr_ij = out_ptr + seq_start + indice_id
out_ptr_ij_mask = (seq_start + indice_id) < seq_end
@@ -218,13 +222,14 @@ def triton_convert_req_index_to_global_index(
token_indices: torch.Tensor, # int32 [num_tokens, NUM_TOPK_TOKENS]
cu_seqlens: torch.Tensor, # int32 [num_tokens + 1]
paged_kv_indices: torch.Tensor, # int32 [num_tokens * topk] out_buffer
BLOCK_STRIDE_ROWS: int,
BLOCK_SIZE: int = 64,
NUM_TOPK_TOKENS: int = 2048,
BLOCK_N: int = 128, # tile width along columns
):
"""out[token_id, indice_id] =
block_table[req_id[token_id],
token_indices[token_id, indice_id] // BLOCK_SIZE] * BLOCK_SIZE
token_indices[token_id, indice_id] // BLOCK_SIZE] * BLOCK_STRIDE_ROWS
+ token_indices[token_id, indice_id] % BLOCK_SIZE
Only when token_indices[token_id, indice_id] == -1 do we output -1.
@@ -264,6 +269,7 @@ def triton_convert_req_index_to_global_index(
# shapes / constexprs
max_num_blocks_per_req,
BLOCK_SIZE,
BLOCK_STRIDE_ROWS,
BLOCK_N,
# strides
bt_stride0,
@@ -345,6 +351,10 @@ class ROCMAiterMLASparseBackend(AttentionBackend):
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
return [1, MultipleOf(16)]
@classmethod
def customize_spec(cls, spec: AttentionSpec) -> AttentionSpec:
return align_blocks_to_rows(spec)
@staticmethod
def get_name() -> str:
return "ROCM_AITER_MLA_SPARSE"
@@ -371,8 +381,7 @@ class ROCMAiterMLASparseBackend(AttentionBackend):
@classmethod
def supported_kv_cache_layouts(cls) -> tuple[KVCacheLayout, ...]:
# Global index conversion assumes contiguous pages within each layer.
return (KVCacheLayout.LBNHC, KVCacheLayout.LBHNC)
return (KVCacheLayout.LBNHC, KVCacheLayout.LBHNC, KVCacheLayout.BLHNC)
@classmethod
def supports_sink(cls) -> bool:
@@ -1155,6 +1164,9 @@ class ROCMAiterMLASparseImpl(
self.topk_indices_buffer[:num_actual_toks], attn_metadata.topk_tokens
)
kv_c_and_k_pe_cache, block_stride_rows = flat_kv_row_view(
kv_c_and_k_pe_cache, attn_metadata.block_size
)
triton_convert_req_index_to_global_index(
attn_metadata.req_id_per_token,
attn_metadata.block_table,
@@ -1162,6 +1174,7 @@ class ROCMAiterMLASparseImpl(
attn_metadata.paged_kv_indptr,
attn_metadata.paged_kv_indices,
BLOCK_SIZE=attn_metadata.block_size,
BLOCK_STRIDE_ROWS=block_stride_rows,
NUM_TOPK_TOKENS=attn_metadata.topk_tokens,
)
@@ -1179,5 +1192,5 @@ class ROCMAiterMLASparseImpl(
return output, None
mla_padded_q = AiterMLAHelper.get_mla_padded_q(self.num_heads, q)
return self._forward_mla(
layer, mla_padded_q, kv_c_and_k_pe_cache, attn_metadata
layer, mla_padded_q, kv_c_and_k_pe_cache.unsqueeze(1), attn_metadata
)
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Utility functions for sparse MLA backends."""
from dataclasses import dataclass
from dataclasses import dataclass, replace
from typing import Any
import numpy as np
@@ -22,6 +22,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
)
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import cdiv
from vllm.v1.kv_cache_interface import AttentionSpec
from vllm.v1.worker.block_table import get_block_table_width
@@ -40,6 +41,12 @@ def request_row_bounds(req_idx: np.ndarray) -> np.ndarray:
return bounds
def align_blocks_to_rows(spec: AttentionSpec, rows: int = 1) -> AttentionSpec:
"""Sparse kernels read the cache as flat rows, viewed as ``rows``-row pages,
so blocks must be whole pages apart."""
return replace(spec, block_stride_alignment=rows * spec.state_content_size_bytes)
def flat_kv_row_view(
kv_cache: torch.Tensor, # [num_blocks, block_size, head_dim]
block_size: int,
@@ -27,6 +27,7 @@ from vllm.v1.attention.backend import (
MLAAttentionImpl,
)
from vllm.v1.attention.backends.mla.sparse_utils import (
align_blocks_to_rows,
flat_kv_row_view,
triton_convert_req_index_to_global_index,
)
@@ -50,6 +51,10 @@ class XPUMLASparseBackend(AttentionBackend):
def get_name() -> str:
return "XPU_MLA_SPARSE"
@classmethod
def customize_spec(cls, spec: AttentionSpec) -> AttentionSpec:
return align_blocks_to_rows(spec)
@staticmethod
def get_metadata_cls() -> type["XPUMLASparseMetadata"]:
return XPUMLASparseMetadata
+4 -4
View File
@@ -661,8 +661,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
self.hash_block_size = hash_block_size
self.dcp_world_size = dcp_world_size
# Only groups that participate in prefix caching must satisfy the
# divisibility constraint; groups that opt out (e.g. GLM-5.3-Flash kpool
# tail, block_size=kpool) are scratch buffers and excluded.
# divisibility constraint; groups that opt out (e.g. circular buffers
# such as the GLM-5.3-Flash indexer tail) are scratch and excluded.
cacheable_block_sizes = [
manager.block_size
for manager, group in zip(
@@ -724,8 +724,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
"""
self.attention_groups: list[SpecGroup] = []
for i, g in enumerate(self.kv_cache_config.kv_cache_groups):
# Skip groups that opt out of prefix caching (e.g. GLM-5.3-Flash
# kpool tail): their blocks are per-request scratch, never
# Skip groups that opt out of prefix caching (e.g. circular
# buffers): their blocks are per-request scratch, never
# shareable, so they must not participate in hit lookup (their
# manager-level hooks already no-op). Their slot in the per-group
# hit tuple stays empty.
+41 -313
View File
@@ -10,7 +10,7 @@ from collections import defaultdict
from collections.abc import Callable, Iterable, Iterator, Sequence
from dataclasses import dataclass, field, replace
from functools import partial, reduce
from typing import TYPE_CHECKING, Any, NamedTuple, NewType, TypeAlias, cast, overload
from typing import TYPE_CHECKING, Any, NamedTuple, NewType, TypeAlias, overload
from vllm import envs
from vllm.config import VllmConfig
@@ -32,7 +32,6 @@ from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
HiddenStateCacheSpec,
HiSparseHotSpec,
KpoolTailSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheLayout,
@@ -1126,7 +1125,7 @@ def check_enough_kv_cache_memory(
# of the specs since grouping may unify them in-place.
groups = get_kv_cache_groups(vllm_config, dict(kv_cache_spec))
check_memory = (
available_memory - _pool_bytes_per_block(groups)
available_memory - _pool_bytes_per_block(vllm_config, groups)
if groups
else available_memory
)
@@ -1237,14 +1236,17 @@ def may_override_num_blocks(vllm_config: VllmConfig, num_blocks: int) -> int:
return num_blocks
def _pool_bytes_per_block(kv_cache_groups: list[KVCacheGroupSpec]) -> int:
def _pool_bytes_per_block(
vllm_config: VllmConfig, kv_cache_groups: list[KVCacheGroupSpec]
) -> int:
"""Bytes consumed by one block in the worker's shared KV cache pool, mirroring
the divisor used by `get_kv_cache_config_from_groups` to convert
`available_memory` into `num_blocks`. Used to compute the effective KV cache
capacity once `num_gpu_blocks_override` is applied.
"""
return _get_kv_cache_bytes_per_block(
[group for group in kv_cache_groups if not group.host_resident]
[group for group in kv_cache_groups if not group.host_resident],
vllm_config.cache_config.get_resolved_kv_cache_layout(),
)
@@ -1287,205 +1289,6 @@ def _get_kv_cache_groups_uniform_type(
return [KVCacheGroupSpec(list(spec.kv_cache_specs.keys()), spec)]
def _pp_balanced_mamba_group_count(
vllm_config: VllmConfig,
mamba_layer_names: list[str],
mla_layer_names: list[str],
) -> int | None:
"""Return a Mamba group count whose PP projections fit the MLA slots."""
num_groups = cdiv(len(mamba_layer_names), len(mla_layer_names))
pp_size = vllm_config.parallel_config.pipeline_parallel_size
if pp_size == 1:
return num_groups
from vllm.distributed.utils import get_pp_indices
from vllm.model_executor.models.utils import extract_layer_index
total_layers = vllm_config.model_config.get_total_num_hidden_layers()
mamba_indices = [extract_layer_index(name) for name in mamba_layer_names]
mla_indices = [extract_layer_index(name) for name in mla_layer_names]
for rank in range(pp_size):
start, end = get_pp_indices(total_layers, rank, pp_size)
num_mamba = sum(start <= index < end for index in mamba_indices)
num_mla = sum(start <= index < end for index in mla_indices)
if not num_mamba:
continue
if not num_mla:
return None
num_groups = max(num_groups, cdiv(num_mamba, num_mla))
return num_groups
def _get_kv_cache_groups_glm5_next(
vllm_config: VllmConfig,
kv_cache_spec: dict[str, KVCacheSpec],
) -> list[KVCacheGroupSpec] | None:
"""Build GLM-5.3-Flash groups with Mamba/MLA and tail/indexer aliasing."""
mamba_specs = {
name: spec
for name, spec in kv_cache_spec.items()
if isinstance(spec, MambaSpec)
}
tail_specs = {
name: spec
for name, spec in kv_cache_spec.items()
if isinstance(spec, KpoolTailSpec)
}
attn_specs = {
name: spec
for name, spec in kv_cache_spec.items()
if not isinstance(spec, (MambaSpec, KpoolTailSpec))
}
if not mamba_specs or not all(
type(spec) is MLAAttentionSpec for spec in attn_specs.values()
):
return None
mla_specs = cast(dict[str, MLAAttentionSpec], attn_specs)
idx_pages = {
spec.page_size_bytes for spec in mla_specs.values() if spec.tokens_per_state > 1
}
if not idx_pages:
return None
assert all(spec.page_size_padded is None for spec in mla_specs.values())
assert len(idx_pages) == 1
mla_names = [name for name, spec in mla_specs.items() if spec.tokens_per_state == 1]
mla_pages = {mla_specs[name].page_size_bytes for name in mla_names}
assert len(mla_pages) == 1
mla_page = mla_pages.pop()
uniform_spec = UniformTypeKVCacheSpecs.from_specs(attn_specs)
assert uniform_spec is not None
tail_group: KVCacheGroupSpec | None = None
if tail_specs:
idx_page = next(iter(idx_pages))
padded_tail_specs: dict[str, KVCacheSpec] = {
name: replace(spec, page_size_padded=idx_page)
for name, spec in tail_specs.items()
}
tail_uniform = UniformTypeKVCacheSpecs.from_specs(padded_tail_specs)
assert tail_uniform is not None
tail_group = KVCacheGroupSpec(list(padded_tail_specs), tail_uniform)
any_mamba = next(iter(mamba_specs.values()))
assert all(spec == any_mamba for spec in mamba_specs.values())
if any_mamba.real_page_size_bytes > mla_page:
raise ValueError(
f"the mamba state page ({any_mamba.real_page_size_bytes} bytes) "
f"does not fit the MLA page ({mla_page} bytes); increase tensor "
"parallelism or use a wider KV cache dtype"
)
padded_specs: dict[str, KVCacheSpec] = {
name: replace(any_mamba, page_size_padded=mla_page) for name in mamba_specs
}
num_groups = _pp_balanced_mamba_group_count(
vllm_config, list(mamba_specs), mla_names
)
if num_groups is None:
raise ValueError(
"a pipeline stage has mamba layers but no MLA layer to share "
"slots with; realign the stage boundaries (VLLM_PP_LAYER_PARTITION)"
)
mamba_grouped_names: list[list[str]] = [[] for _ in range(num_groups)]
for index, name in enumerate(mamba_specs):
mamba_grouped_names[index % num_groups].append(name)
return (
[KVCacheGroupSpec(list(attn_specs), uniform_spec)]
+ ([tail_group] if tail_group is not None else [])
+ create_kv_cache_group_specs(padded_specs, mamba_grouped_names)
)
def _glm5_next_tensor_layout(
kv_cache_groups: list[KVCacheGroupSpec],
) -> (
tuple[
KVCacheGroupSpec,
list[KVCacheGroupSpec],
list[str],
list[str],
int,
int,
list[str],
int,
]
| None
):
"""Recognize the GLM-5.3-Flash grouping after optional PP projection."""
uniform_groups = [
group
for group in kv_cache_groups
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
]
mamba_groups = [
group for group in kv_cache_groups if isinstance(group.kv_cache_spec, MambaSpec)
]
attn_group: KVCacheGroupSpec | None = None
tail_group: KVCacheGroupSpec | None = None
for group in uniform_groups:
inner = cast(UniformTypeKVCacheSpecs, group.kv_cache_spec).kv_cache_specs
if all(type(spec) is MLAAttentionSpec for spec in inner.values()):
attn_group = group
elif all(isinstance(spec, KpoolTailSpec) for spec in inner.values()):
tail_group = group
if attn_group is None or not mamba_groups:
return None
if len(uniform_groups) + len(mamba_groups) != len(kv_cache_groups):
return None
attn_uniform = cast(UniformTypeKVCacheSpecs, attn_group.kv_cache_spec)
mla_inner = cast(dict[str, MLAAttentionSpec], attn_uniform.kv_cache_specs)
if not all(
type(spec) is MLAAttentionSpec and spec.page_size_padded is None
for spec in mla_inner.values()
):
return None
mla_names = [
name for name in attn_group.layer_names if mla_inner[name].tokens_per_state == 1
]
idx_names = [
name for name in attn_group.layer_names if mla_inner[name].tokens_per_state > 1
]
mla_pages = {mla_inner[name].page_size_bytes for name in mla_names}
idx_pages = {mla_inner[name].page_size_bytes for name in idx_names}
if len(mla_pages) != 1 or len(idx_pages) != 1:
return None
mla_page = mla_pages.pop()
idx_page = idx_pages.pop()
if any(group.kv_cache_spec.page_size_bytes != mla_page for group in mamba_groups):
return None
tail_names: list[str] = []
tail_page = 0
if tail_group is not None:
tail_names = list(tail_group.layer_names)
tail_inner = cast(
UniformTypeKVCacheSpecs, tail_group.kv_cache_spec
).kv_cache_specs
tail_pages = {
cast(KpoolTailSpec, spec).unpadded_page_size_bytes
for spec in tail_inner.values()
}
if len(tail_pages) != 1 or len(tail_names) != len(idx_names):
return None
tail_page = tail_pages.pop()
if tail_page > idx_page:
return None
return (
attn_group,
mamba_groups,
mla_names,
idx_names,
mla_page,
idx_page,
tail_names,
tail_page,
)
def unify_kv_cache_spec_page_size(
kv_cache_spec: dict[str, KVCacheSpec],
) -> dict[str, KVCacheSpec]:
@@ -1709,14 +1512,24 @@ def _get_per_layer_spec(
return spec
def _get_layer_block_stride(spec: KVCacheSpec) -> int:
"""A layer-outer block stride: the page padded to its alignment."""
return round_up(spec.page_size_bytes, spec.get_block_stride_alignment())
def _get_kv_cache_bytes_per_block(
kv_cache_groups: list[KVCacheGroupSpec],
kv_cache_groups: list[KVCacheGroupSpec], layout: KVCacheLayout
) -> int:
"""Return the largest cache group's bytes per block."""
if (glm5_layout := _glm5_next_tensor_layout(kv_cache_groups)) is not None:
_, _, mla_names, idx_names, mla_page, idx_page, _, _ = glm5_layout
return len(mla_names) * mla_page + len(idx_names) * idx_page
if not layout.is_block_outermost:
# Each layer's block stride is its own padded page.
return max(
sum(
_get_layer_block_stride(_get_per_layer_spec(group, layer_name))
for layer_name in group.layer_names
)
for group in kv_cache_groups
)
bytes_per_block = max(
sum(
_get_per_layer_spec(group, layer_name).page_size_bytes
@@ -1731,7 +1544,7 @@ def _get_kv_cache_bytes_per_block(
if isinstance(group.kv_cache_spec, HiSparseHotSpec)
]
alignments.extend(
_get_per_layer_spec(group, layer_name).block_stride_alignment or 1
_get_per_layer_spec(group, layer_name).get_block_stride_alignment()
for group in kv_cache_groups
for layer_name in group.layer_names
)
@@ -1806,72 +1619,9 @@ def get_kv_cache_config_from_groups(
vllm_config, kv_cache_groups, available_memory, host_budget
)
if (glm5_layout := _glm5_next_tensor_layout(kv_cache_groups)) is not None:
(
attn_group,
mamba_groups,
mla_names,
idx_names,
mla_page,
idx_page,
tail_names,
_,
) = glm5_layout
bytes_per_block = len(mla_names) * mla_page + len(idx_names) * idx_page
num_blocks = may_override_num_blocks(
vllm_config, available_memory // bytes_per_block
)
size = bytes_per_block * num_blocks
attn_specs = cast(
UniformTypeKVCacheSpecs, attn_group.kv_cache_spec
).kv_cache_specs
kv_cache_tensors: list[KVCacheTensor] = []
def add_tensor(layer_name: str, spec: KVCacheSpec, offset: int) -> None:
kv_cache_tensors.append(
KVCacheTensor(
size=size,
layers=[layer_name],
layer_stride=spec.page_size_bytes * num_blocks,
block_stride=spec.page_size_bytes,
offset=offset,
)
)
for index, mla_name in enumerate(mla_names):
offset = index * mla_page * num_blocks
add_tensor(mla_name, attn_specs[mla_name], offset)
for group in mamba_groups:
if index < len(group.layer_names):
add_tensor(group.layer_names[index], group.kv_cache_spec, offset)
idx_base = len(mla_names) * mla_page * num_blocks
for index, idx_name in enumerate(idx_names):
offset = idx_base + index * idx_page * num_blocks
add_tensor(idx_name, attn_specs[idx_name], offset)
if tail_names:
tail_name = tail_names[index]
tail_group = next(
group for group in kv_cache_groups if tail_name in group.layer_names
)
tail_specs = cast(
UniformTypeKVCacheSpecs, tail_group.kv_cache_spec
).kv_cache_specs
add_tensor(tail_name, tail_specs[tail_name], offset)
return KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=kv_cache_tensors,
kv_cache_groups=kv_cache_groups,
prefix_cache_retention_interval=(
vllm_config.cache_config.prefix_cache_retention_interval
),
)
layout = vllm_config.cache_config.get_resolved_kv_cache_layout()
validate_kv_cache_layout(layout, kv_cache_groups)
bytes_per_block = _get_kv_cache_bytes_per_block(kv_cache_groups)
bytes_per_block = _get_kv_cache_bytes_per_block(kv_cache_groups, layout)
interleaved_block_stride = bytes_per_block if layout.is_block_outermost else None
num_blocks = available_memory // bytes_per_block
@@ -1901,17 +1651,17 @@ def get_kv_cache_config_from_groups(
byte_offset = 0
for spec, layer_names in layers_by_spec.items():
block_stride = interleaved_block_stride
padded_page = _get_layer_block_stride(spec)
if block_stride is None and padded_page != spec.page_size_bytes:
assert layout.is_block_compact, f"{layout.name} cannot pad pages"
block_stride = padded_page
layer_stride, block_stride, _, _, _ = compute_layout_strides(
spec,
num_blocks,
len(layer_names),
layout,
fixed_strides=(None, interleaved_block_stride, None, None, None),
)
offset = (
byte_offset
* max(layer_stride, spec.page_size_bytes)
// spec.page_size_bytes
fixed_strides=(None, block_stride, None, None, None),
)
kv_cache_tensors.append(
KVCacheTensor(
@@ -1919,10 +1669,13 @@ def get_kv_cache_config_from_groups(
layers=layer_names,
layer_stride=layer_stride,
block_stride=block_stride,
offset=offset,
offset=byte_offset,
)
)
byte_offset += len(layer_names) * spec.page_size_bytes
# Block-outer layers share each block; layer-outer ones own regions.
byte_offset += len(layer_names) * (
spec.page_size_bytes if layout.is_block_outermost else layer_stride
)
return KVCacheConfig(
num_blocks=num_blocks,
@@ -2438,9 +2191,6 @@ def get_kv_cache_groups(
# full attention, or all layers are sliding window attention with the
# same window size). Put all layers into one group.
return _get_kv_cache_groups_uniform_type(uniform_spec)
elif glm5_groups := _get_kv_cache_groups_glm5_next(vllm_config, kv_cache_spec):
return glm5_groups
# Hidden-state layers use their own block table and must not be absorbed
# into a compatible attention bucket.
hidden_specs = {
@@ -2594,31 +2344,7 @@ def _max_memory_usage_bytes_from_groups(
if not kv_cache_groups:
return 0
if (glm5_layout := _glm5_next_tensor_layout(kv_cache_groups)) is not None:
(
attn_group,
mamba_groups,
mla_names,
idx_names,
mla_page,
idx_page,
tail_names,
_,
) = glm5_layout
uniform_spec = cast(UniformTypeKVCacheSpecs, attn_group.kv_cache_spec)
total_blocks = uniform_spec.max_memory_usage_pages(vllm_config)
total_blocks += sum(
cdiv(
group.kv_cache_spec.max_memory_usage_bytes(vllm_config),
group.kv_cache_spec.page_size_bytes,
)
for group in mamba_groups
)
if tail_names:
total_blocks += 1
return total_blocks * (len(mla_names) * mla_page + len(idx_names) * idx_page)
bytes_per_block = _pool_bytes_per_block(kv_cache_groups)
bytes_per_block = _pool_bytes_per_block(vllm_config, kv_cache_groups)
total_blocks = 0
for group in kv_cache_groups:
spec = group.kv_cache_spec
@@ -2874,7 +2600,7 @@ def get_kv_cache_configs(
if not groups:
adjusted_memory.append(avail_mem)
continue
bytes_per_block = _pool_bytes_per_block(groups)
bytes_per_block = _pool_bytes_per_block(vllm_config, groups)
logger.info(
"Overriding num_gpu_blocks=%d with num_gpu_blocks_override=%d",
avail_mem // bytes_per_block,
@@ -2887,7 +2613,7 @@ def get_kv_cache_configs(
# the capacity check both plan against usable blocks. Allocation below
# still uses the full memory.
check_memory = [
avail_mem - _pool_bytes_per_block(groups) if groups else avail_mem
avail_mem - _pool_bytes_per_block(vllm_config, groups) if groups else avail_mem
for groups, avail_mem in zip(projected_groups_per_worker, available_memory)
]
@@ -2931,7 +2657,9 @@ def get_kv_cache_configs(
# strides and offsets stay consistent with the shrunken allocation.
groups = kv_cache_config.kv_cache_groups
kv_cache_configs[i] = get_kv_cache_config_from_groups(
vllm_config, groups, min_num_blocks * _pool_bytes_per_block(groups)
vllm_config,
groups,
min_num_blocks * _pool_bytes_per_block(vllm_config, groups),
)
for kv_cache_config in kv_cache_configs:
@@ -7,7 +7,6 @@ from collections.abc import Sequence
from typing import TYPE_CHECKING, ClassVar
from vllm.distributed.kv_events import MEDIUM_CPU
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_utils import (
@@ -27,7 +26,6 @@ from vllm.v1.kv_cache_interface import (
HiddenStateCacheSpec,
HiSparseHotSpec,
HiSparseResidentSpec,
KpoolTailSpec,
KVCacheGroupRole,
KVCacheSpec,
MambaSpec,
@@ -45,8 +43,6 @@ from vllm.v1.request import Request
if TYPE_CHECKING:
from vllm.v1.hisparse.coordinator import HiSparseCoordinator
logger = init_logger(__name__)
class SingleTypeKVCacheManager(ABC):
"""An abstract base class for a manager that handle the kv cache management
@@ -1299,10 +1295,6 @@ class CircularBufferManager(FullAttentionManager):
return 0
class KpoolTailManager(CircularBufferManager):
"""One-block circular scratch manager for ``KpoolTailSpec``."""
class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
def __init__(self, kv_cache_spec: ChunkedLocalAttentionSpec, **kwargs) -> None:
super().__init__(kv_cache_spec, **kwargs)
@@ -2687,11 +2679,6 @@ def register_all_kvcache_specs(vllm_config):
SlidingWindowManager,
uniform_type_base_spec=SlidingWindowMLASpec,
)
KVCacheSpecRegistry.register(
KpoolTailSpec,
KpoolTailManager,
uniform_type_base_spec=KpoolTailSpec,
)
KVCacheSpecRegistry.register(
MambaSpec, MambaManager, uniform_type_base_spec=MambaSpec
+2 -2
View File
@@ -335,8 +335,8 @@ class EngineCore:
vllm_config.cache_config.num_gpu_blocks = scheduler_kv_cache_config.num_blocks
kv_cache_groups = scheduler_kv_cache_config.kv_cache_groups
if kv_cache_groups:
# Exclude groups that opt out of prefix caching (e.g. GLM-5.3-Flash
# kpool tail, a 1-block/req scratch buffer with block_size=kpool):
# Exclude groups that opt out of prefix caching (e.g. a circular
# buffer, a 1-block/req scratch buffer with a small ring block):
# their small block_size would otherwise drag the global block_size
# below the real allocator block size and desync it from mamba.
participating = [
+3 -5
View File
@@ -55,8 +55,9 @@ def resolve_hisparse_specs(
if len(block_sizes) != 1:
raise ValueError("HiSparse requires one scheduler block size.")
backends = [attn_layers[name].get_attn_backend() for name in mla_specs]
specs = list(mla_specs.values())
try:
block_size = select_common_block_size(block_sizes.pop(), backends)
block_size = select_common_block_size(block_sizes.pop(), backends, specs)
except ValueError as error:
raise ValueError(
"HiSparse requires a GPU block size supported by every sparse "
@@ -104,16 +105,13 @@ def allocate_hisparse_kv_caches(
if isinstance(host_spec, UniformTypeKVCacheSpecs)
else host_spec
)
kernel_block_size = kernel_block_sizes[host_group_id]
if isinstance(spec, MLAAttentionSpec) and spec.storage_block_size is not None:
kernel_block_size = spec.storage_block_size
views = create_kv_cache_views(
backing,
spec,
kv_cache_config.num_blocks_of(tensor),
layout,
tensor,
kernel_block_size=kernel_block_size,
kernel_block_size=kernel_block_sizes[host_group_id],
)
kv_caches.update(zip(tensor.layers, views))
return kv_caches
+2 -2
View File
@@ -333,7 +333,7 @@ def get_hisparse_kv_cache_config(
device_groups = hisparse_layout.device_groups
layout = vllm_config.cache_config.get_resolved_kv_cache_layout()
validate_kv_cache_layout(layout, device_groups)
bytes_per_block = _get_kv_cache_bytes_per_block(device_groups)
bytes_per_block = _get_kv_cache_bytes_per_block(device_groups, layout)
num_blocks = may_override_num_blocks(
vllm_config, available_memory // bytes_per_block
)
@@ -345,7 +345,7 @@ def get_hisparse_kv_cache_config(
host_groups = [hisparse_layout.source_group]
host_layout = KVCacheLayout.LBNHC
validate_kv_cache_layout(host_layout, host_groups)
host_bytes_per_block = _get_kv_cache_bytes_per_block(host_groups)
host_bytes_per_block = _get_kv_cache_bytes_per_block(host_groups, host_layout)
host_size = host_bytes_per_block * hisparse_layout.host_num_blocks
kv_cache_tensors[:0] = _build_hisparse_kv_cache_tensors(
host_groups,
+21 -35
View File
@@ -4,6 +4,7 @@
from __future__ import annotations
import copy
import math
from collections import Counter
from collections.abc import Collection, Sequence
from dataclasses import dataclass, field, fields, replace
@@ -24,6 +25,7 @@ from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.attention.backend import MultipleOf
logger = init_logger(__name__)
@@ -168,13 +170,28 @@ class KVCacheSpec:
dcp_sharded: bool = field(default=False, kw_only=True)
"""Whether DCP shards this cache's token positions across ranks."""
block_stride_alignment: int | None = field(default=None, kw_only=True)
"""Required byte alignment between physical blocks, including packed layers."""
block_stride_alignment: int | MultipleOf | None = field(default=None, kw_only=True)
"""Required byte alignment between physical blocks, including packed layers.
``MultipleOf(n)``: whole kernel blocks, when blocks are split into kernel
blocks of up to ``n`` tokens."""
def __post_init__(self):
if self.block_stride_alignment is not None and self.block_stride_alignment <= 0:
alignment = self.block_stride_alignment
if isinstance(alignment, int) and alignment <= 0:
raise ValueError("block_stride_alignment must be positive")
def get_block_stride_alignment(self) -> int:
"""Bytes the stride between physical blocks must be a multiple of."""
from vllm.v1.attention.backend import MultipleOf
alignment = self.block_stride_alignment
if not isinstance(alignment, MultipleOf):
return alignment or 1
kernel_block_size = math.gcd(alignment.base, self.block_size)
if kernel_block_size == self.block_size:
return 1
return self.copy_with_new_block_size(kernel_block_size).page_size_bytes
@property
def prefix_cacheable(self) -> bool:
"""Whether this spec's group participates in prefix caching."""
@@ -674,8 +691,6 @@ class MLAAttentionSpec(FullAttentionSpec):
model_version: str | None = None
cache_role: SparseCacheRole = SparseCacheRole.SPARSE
is_index_group_leader: bool = False
storage_block_size: int | None = None
"""Token width used to view storage when it differs from the kernel block."""
# Group capability enabled when any member flattens a non-causal query block
# into decode rows. Runtime metadata still selects causal vs. non-causal mode.
non_causal_multi_token_decode: bool = False
@@ -696,7 +711,6 @@ class MLAAttentionSpec(FullAttentionSpec):
model_version_set = set(spec.model_version for spec in specs)
cache_role_set = {spec.cache_role for spec in specs}
index_group_leader_set = {spec.is_index_group_leader for spec in specs}
storage_block_size_set = set(spec.storage_block_size for spec in specs)
block_stride_alignment_set = {spec.block_stride_alignment for spec in specs}
assert (
len(cache_dtype_str_set) == 1
@@ -704,12 +718,11 @@ class MLAAttentionSpec(FullAttentionSpec):
and len(model_version_set) == 1
and len(cache_role_set) == 1
and len(index_group_leader_set) == 1
and len(storage_block_size_set) == 1
and len(block_stride_alignment_set) == 1
), (
"All attention layers in the same KV cache group must use the same "
"quantization method, tokens per state, model version, cache role, "
"index-sharing role, storage block size and block stride alignment."
"index-sharing role, and block stride alignment."
)
merged_spec = cls(
block_size=specs[0].block_size,
@@ -727,7 +740,6 @@ class MLAAttentionSpec(FullAttentionSpec):
model_version=model_version_set.pop(),
cache_role=cache_role_set.pop(),
is_index_group_leader=index_group_leader_set.pop(),
storage_block_size=storage_block_size_set.pop(),
block_stride_alignment=block_stride_alignment_set.pop(),
non_causal_multi_token_decode=any(
spec.non_causal_multi_token_decode for spec in specs
@@ -1019,32 +1031,6 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
)
@dataclass(frozen=True, kw_only=True)
class KpoolTailSpec(SlidingWindowSpec):
"""One-block circular scratch cache for a kpool indexer's raw tail."""
def max_admission_blocks_per_request(
self, max_in_flight_tokens: int, max_model_len: int
) -> int:
return 1
def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int:
return 1
def is_uniform_with_collection(
self, kv_cache_specs: dict[str, KVCacheSpec]
) -> bool:
return all(isinstance(spec, KpoolTailSpec) for spec in kv_cache_specs.values())
@property
def prefix_cacheable(self) -> bool:
return False
@property
def uses_slot_mapping(self) -> bool:
return False
@dataclass(frozen=True)
class MambaSpec(KVCacheSpec):
shapes: tuple[tuple[int, ...], ...]
+2 -6
View File
@@ -95,7 +95,7 @@ class Gemma4Proposer(SpecDecodeBaseProposer):
cm.block_table_tensor = self._per_group_block_tables[gid][:batch_size]
else:
cm = common_attn_metadata
attn_metadata = attn_group.get_metadata_builder().build_for_drafting(
attn_metadata = attn_group.build_metadata_for_drafting(
common_attn_metadata=cm, draft_index=draft_index
)
per_group_attn_metadata.append(attn_metadata)
@@ -291,11 +291,7 @@ class Gemma4Proposer(SpecDecodeBaseProposer):
self.draft_attn_groups = list(attention_groups.values())
if self.draft_attn_groups:
self.kv_cache_gid = self.draft_attn_groups[0].kv_cache_group_id
self.block_size = (
self.draft_attn_groups[0]
.get_metadata_builder()
.kv_cache_spec.block_size
)
self.block_size = self.draft_attn_groups[0].kv_cache_spec.block_size
else:
self.kv_cache_gid = 0
self.block_size = kv_cache_config.kv_cache_groups[
+2 -15
View File
@@ -991,7 +991,7 @@ class SpecDecodeBaseProposer:
per_group_attn_metadata: list[object] = []
per_layer_attn_metadata: dict[str, object] = {}
for attn_group in self.draft_attn_groups:
attn_metadata = attn_group.get_metadata_builder().build_for_drafting(
attn_metadata = attn_group.build_metadata_for_drafting(
common_attn_metadata=common_attn_metadata, draft_index=draft_index
)
per_group_attn_metadata.append(attn_metadata)
@@ -1765,20 +1765,7 @@ class SpecDecodeBaseProposer:
attention_groups[backend_key].layer_names.append(layer_name)
self.draft_attn_groups = list(attention_groups.values())
if kernel_block_sizes is not None and 0 <= self.kv_cache_gid < len(
kernel_block_sizes
):
# Slot mappings are computed against the block table, which is
# stored at kernel-block granularity. Use the kernel block size
# rather than the KV cache manager's block size; the two differ
# when manager blocks are split for the attention kernel.
self.block_size = kernel_block_sizes[self.kv_cache_gid]
else:
self.block_size = (
self.draft_attn_groups[0]
.get_metadata_builder()
.kv_cache_spec.block_size
)
self.block_size = self.draft_attn_groups[0].kv_cache_spec.block_size
logger.debug("Using block size %d for drafting layers", self.block_size)
def _determine_batch_execution_and_padding(
+2 -6
View File
@@ -144,7 +144,7 @@ class Step3p5MTPProposer(EagleProposer):
cm.slot_mapping = sm
else:
cm = common_attn_metadata
attn_metadata = attn_group.get_metadata_builder().build_for_drafting(
attn_metadata = attn_group.build_metadata_for_drafting(
common_attn_metadata=cm,
draft_index=draft_index,
)
@@ -244,11 +244,7 @@ class Step3p5MTPProposer(EagleProposer):
self.draft_attn_groups = list(attention_groups.values())
if self.draft_attn_groups:
self.kv_cache_gid = self.draft_attn_groups[0].kv_cache_group_id
self.block_size = (
self.draft_attn_groups[0]
.get_metadata_builder()
.kv_cache_spec.block_size
)
self.block_size = self.draft_attn_groups[0].kv_cache_spec.block_size
else:
self.kv_cache_gid = 0
self.block_size = kv_cache_config.kv_cache_groups[
+10 -119
View File
@@ -29,24 +29,16 @@ logger = init_logger(__name__)
def get_block_table_width(
max_num_blocks: int,
block_size: int,
kernel_block_size: int | None = None,
*,
token_alignment: int | None = 128,
) -> int:
"""Return the width after optional alignment and virtual block splitting."""
if kernel_block_size is None:
kernel_block_size = block_size
if block_size % kernel_block_size != 0:
raise ValueError(
f"kernel_block_size {kernel_block_size} must divide "
f"block_size {block_size} evenly"
)
"""Return the width after optional alignment."""
if token_alignment is not None:
if token_alignment <= 0:
raise ValueError("token_alignment must be positive")
block_alignment = token_alignment // math.gcd(token_alignment, block_size)
max_num_blocks = cdiv(max_num_blocks, block_alignment) * block_alignment
return max_num_blocks * block_size // kernel_block_size
return max_num_blocks
class SlotMappingMode(Enum):
@@ -63,7 +55,6 @@ class BlockTable:
max_num_batched_tokens: int,
pin_memory: bool,
device: torch.device,
kernel_block_size: int,
cp_kv_cache_interleave_size: int,
slot_mapping_mode: SlotMappingMode = SlotMappingMode.TOKEN_TO_KV_SLOT,
):
@@ -74,9 +65,6 @@ class BlockTable:
max_num_batched_tokens: Maximum number of tokens in a batch.
pin_memory: Whether to pin memory for faster GPU transfers.
device: Target device for the block table.
kernel_block_size: The block_size of underlying attention kernel.
Will be the same as `block_size` if `block_size` is supported
by the attention kernel.
slot_mapping_mode: How this cache group maps scheduled tokens to
cache slots. Mamba-like state caches do not use token slot
mappings and should use SlotMappingMode.NONE.
@@ -86,30 +74,8 @@ class BlockTable:
self.max_num_batched_tokens = max_num_batched_tokens
self.pin_memory = pin_memory
self.device = device
self.kv_cache_block_size = block_size
if kernel_block_size == block_size:
# Standard case: allocation and computation use same block size
# No block splitting needed, direct mapping
self.block_size = block_size
self.blocks_per_kv_block = 1
self.use_hybrid_blocks = False
else:
# Hybrid case: allocation block size differs from kernel block size
# Memory blocks are subdivided to match kernel requirements
# Example: 32-token memory blocks with 16-token kernel blocks
# → Each memory block corresponds to 2 kernel blocks
if block_size % kernel_block_size != 0:
raise ValueError(
f"kernel_block_size {kernel_block_size} must divide "
f"kv_manager_block_size size {block_size} evenly"
)
self.block_size = kernel_block_size
self.blocks_per_kv_block = block_size // kernel_block_size
self.use_hybrid_blocks = True
self.max_num_blocks_per_req = max_num_blocks_per_req * self.blocks_per_kv_block
self.block_size = block_size
self.max_num_blocks_per_req = max_num_blocks_per_req
self.block_table = self._make_buffer(
self.max_num_reqs, self.max_num_blocks_per_req, dtype=torch.int32
@@ -120,13 +86,6 @@ class BlockTable:
self.max_num_batched_tokens, dtype=torch.int64
)
if self.use_hybrid_blocks:
self._kernel_block_arange = np.arange(0, self.blocks_per_kv_block).reshape(
1, -1
)
else:
self._kernel_block_arange = None
try:
self.pcp_world_size = get_pcp_group().world_size
self.pcp_rank = get_pcp_group().rank_in_group
@@ -145,13 +104,11 @@ class BlockTable:
self.slot_mapping_mode = slot_mapping_mode
if self.slot_mapping_mode == SlotMappingMode.TOKEN_TO_KV_SLOT:
_COMPUTE_SLOT_MAPPING_KERNEL.register_warmup(
kv_cache_block_size=self.kv_cache_block_size,
blocks_per_kv_block=self.blocks_per_kv_block,
kv_cache_block_size=self.block_size,
total_cp_world_size=self.dcp_world_size,
total_cp_rank=self.dcp_rank,
cp_kv_cache_interleave_size=self.cp_kv_cache_interleave_size,
block_table_stride=self.block_table.gpu.stride(0),
block_size=self.block_size,
)
def append_row(
@@ -162,11 +119,6 @@ class BlockTable:
if not block_ids:
return
if self.use_hybrid_blocks:
block_ids = self.map_to_kernel_blocks(
np.array(block_ids), self.blocks_per_kv_block, self._kernel_block_arange
)
num_blocks = len(block_ids)
start = self.num_blocks_per_row[row_idx]
self.num_blocks_per_row[row_idx] += num_blocks
@@ -219,10 +171,8 @@ class BlockTable:
positions,
self.block_table.gpu,
self.block_table.gpu.stride(0),
self.block_size,
self.slot_mapping.gpu,
self.kv_cache_block_size,
self.blocks_per_kv_block,
self.block_size,
self.dcp_world_size,
self.dcp_rank,
self.cp_kv_cache_interleave_size,
@@ -235,37 +185,6 @@ class BlockTable:
self.block_table.gpu.fill_(0)
self.block_table.cpu.fill_(0)
@staticmethod
def map_to_kernel_blocks(
kv_manager_block_ids: np.ndarray,
blocks_per_kv_block: int,
kernel_block_arange: np.ndarray,
) -> np.ndarray:
"""Convert kv_manager_block_id IDs to kernel block IDs.
Example:
# kv_manager_block_ids: 32 tokens,
# Kernel block size: 16 tokens
# blocks_per_kv_block = 2
>>> kv_manager_block_ids = np.array([0, 1, 2])
>>> Result: [0, 1, 2, 3, 4, 5]
# Each kv_manager_block_id maps to 2 kernel block id:
# kv_manager_block_id 0 → kernel block id [0, 1]
# kv_manager_block_id 1 → kernel block id [2, 3]
# kv_manager_block_id 2 → kernel block id [4, 5]
"""
if blocks_per_kv_block == 1:
return kv_manager_block_ids
kernel_block_ids = (
kv_manager_block_ids.reshape(-1, 1) * blocks_per_kv_block
+ kernel_block_arange
)
return kernel_block_ids.reshape(-1)
def get_device_tensor(self, num_reqs: int) -> torch.Tensor:
"""Returns the device tensor of the block table."""
return self.block_table.gpu[:num_reqs]
@@ -296,16 +215,10 @@ class MultiGroupBlockTable:
pin_memory: bool,
device: torch.device,
block_sizes: list[int],
kernel_block_sizes: list[int],
max_num_blocks: list[int],
cp_kv_cache_interleave_size: int = 1,
slot_mapping_modes: list[SlotMappingMode] | None = None,
) -> None:
if len(kernel_block_sizes) != len(block_sizes):
raise ValueError(
f"kernel_block_sizes length ({len(kernel_block_sizes)}) "
f"must match block_sizes length ({len(block_sizes)})"
)
if slot_mapping_modes is None:
slot_mapping_modes = [SlotMappingMode.TOKEN_TO_KV_SLOT] * len(block_sizes)
if len(slot_mapping_modes) != len(block_sizes):
@@ -339,17 +252,11 @@ class MultiGroupBlockTable:
max_num_batched_tokens,
pin_memory,
device,
kernel_block_size,
cp_kv_cache_interleave_size,
slot_mapping_mode=slot_mapping_mode,
)
for (
block_size,
kernel_block_size,
max_num_blocks_per_req,
slot_mapping_mode,
) in zip(
block_sizes, kernel_block_sizes, max_num_blocks, slot_mapping_modes
for block_size, max_num_blocks_per_req, slot_mapping_mode in zip(
block_sizes, max_num_blocks, slot_mapping_modes
)
]
@@ -403,12 +310,10 @@ class ComputeSlotMappingKernel(
@dataclass(frozen=True)
class CompileKey:
kv_cache_block_size: int
blocks_per_kv_block: int
total_cp_world_size: int
total_cp_rank: int
cp_kv_cache_interleave_size: int
block_table_stride: int
block_size: int
@staticmethod
@triton.jit(do_not_specialize=["num_tokens", "max_num_tokens"])
@@ -419,10 +324,8 @@ class ComputeSlotMappingKernel(
positions_ptr, # [num_tokens], int64
block_table_ptr, # [max_num_reqs, max_num_blocks_per_req], int32 (flat)
block_table_stride, # max_num_blocks_per_req
block_size,
slot_mapping_ptr, # [max_num_tokens], int64
KV_CACHE_BLOCK_SIZE: tl.constexpr,
BLOCKS_PER_KV_BLOCK: tl.constexpr,
TOTAL_CP_WORLD_SIZE: tl.constexpr,
TOTAL_CP_RANK: tl.constexpr,
CP_KV_CACHE_INTERLEAVE_SIZE: tl.constexpr,
@@ -463,17 +366,12 @@ class ComputeSlotMappingKernel(
virtual_block_offsets % CP_KV_CACHE_INTERLEAVE_SIZE
)
block_indices = (
virtual_block_indices * BLOCKS_PER_KV_BLOCK
+ local_block_offsets // block_size
)
block_numbers = tl.load(
block_table_ptr + row_offset + block_indices,
block_table_ptr + row_offset + virtual_block_indices,
mask=mask & is_local,
other=0,
).to(tl.int64)
slot_offsets = local_block_offsets % block_size
slot_ids = block_numbers * block_size + slot_offsets
slot_ids = block_numbers * KV_CACHE_BLOCK_SIZE + local_block_offsets
slot_ids = tl.where(is_local, slot_ids, PAD_ID)
tl.store(slot_mapping_ptr + offsets, slot_ids, mask=mask)
@@ -481,13 +379,11 @@ class ComputeSlotMappingKernel(
self,
*,
block_table_stride: int,
block_size: int,
**compile_key_fields: int,
) -> CompileKey:
return self.CompileKey(
**compile_key_fields,
block_table_stride=triton_scalar_specialization_rep(block_table_stride),
block_size=triton_scalar_specialization_rep(block_size),
)
def get_warmup_keys(self, **dispatch_kwargs: int) -> list[CompileKey]:
@@ -507,10 +403,8 @@ class ComputeSlotMappingKernel(
shape=(1, compile_key.block_table_stride),
),
block_table_stride=compile_key.block_table_stride,
block_size=compile_key.block_size,
slot_mapping=int64_ptr,
kv_cache_block_size=compile_key.kv_cache_block_size,
blocks_per_kv_block=compile_key.blocks_per_kv_block,
total_cp_world_size=compile_key.total_cp_world_size,
total_cp_rank=compile_key.total_cp_rank,
cp_kv_cache_interleave_size=compile_key.cp_kv_cache_interleave_size,
@@ -526,17 +420,14 @@ class ComputeSlotMappingKernel(
positions: torch.Tensor,
block_table: torch.Tensor,
block_table_stride: int,
block_size: int,
slot_mapping: torch.Tensor,
kv_cache_block_size: int,
blocks_per_kv_block: int,
total_cp_world_size: int,
total_cp_rank: int,
cp_kv_cache_interleave_size: int,
) -> LaunchSpec:
return (num_reqs + 1,), dict(
KV_CACHE_BLOCK_SIZE=kv_cache_block_size,
BLOCKS_PER_KV_BLOCK=blocks_per_kv_block,
TOTAL_CP_WORLD_SIZE=total_cp_world_size,
TOTAL_CP_RANK=total_cp_rank,
CP_KV_CACHE_INTERLEAVE_SIZE=cp_kv_cache_interleave_size,
+2 -4
View File
@@ -95,11 +95,9 @@ def prepare_dcp_dummy_context_metadata(
max_valid_block_id = kv_cache_config.num_blocks - 1
assert max_valid_block_id > 0
for blk_table in input_batch.block_table.block_tables:
max_row_blocks = (
blk_table.max_num_blocks_per_req // blk_table.blocks_per_kv_block
)
block_ids = [
(block_idx % max_valid_block_id) + 1 for block_idx in range(max_row_blocks)
(block_idx % max_valid_block_id) + 1
for block_idx in range(blk_table.max_num_blocks_per_req)
]
for req_idx in range(num_reqs):
blk_table.add_row(block_ids, req_idx)
+13 -11
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Mapping, Sequence
from collections.abc import Iterable, Mapping, Sequence
from contextlib import AbstractContextManager, nullcontext
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, cast
@@ -40,6 +40,8 @@ from vllm.v1.worker.utils import (
add_kv_sharing_layers_to_kv_cache_groups,
allocate_kv_cache,
bind_kv_cache_to_layers,
customize_attention_spec,
map_kv_caches_to_kernel_blocks,
prepare_kernel_block_sizes,
)
@@ -141,7 +143,7 @@ def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]:
# Skip modules that don't need KV cache (eg encoder-only attention)
if spec := attn_module.get_kv_cache_spec(vllm_config):
if isinstance(spec, AttentionSpec):
spec = attn_module.get_attn_backend().customize_spec(spec)
spec = customize_attention_spec(attn_module.get_attn_backend(), spec)
kv_cache_spec[layer_name] = spec
if vllm_config.attention_config.hisparse_config is not None:
kv_cache_spec = resolve_hisparse_specs(vllm_config, kv_cache_spec, attn_layers)
@@ -382,6 +384,7 @@ def init_kv_cache(
kv_cache_allocation_context: AbstractContextManager | None = None,
*,
block_tables: "BlockTables | None" = None,
attn_groups: Iterable[AttentionGroup] = (),
) -> dict[str, Any]:
allocation_context = kv_cache_allocation_context or nullcontext()
with allocation_context:
@@ -409,9 +412,10 @@ def init_kv_cache(
num_attn_module = (
2 if vllm_config.model_config.hf_config.model_type == "longcat_flash" else 1
)
bindable_caches = {
name: cache for name, cache in kv_caches.items() if name in forward_context
}
bindable_caches = map_kv_caches_to_kernel_blocks(
{name: cache for name, cache in kv_caches.items() if name in forward_context},
attn_groups,
)
bind_kv_cache_to_layers(
bindable_caches,
forward_context,
@@ -536,8 +540,8 @@ def build_attn_metadata(
cached_metadata[reuse_key], block_table, slot_mapping
)
elif for_cudagraph_capture:
metadata = attn_metadata_builder.build_for_cudagraph_capture(
common_attn_metadata
metadata = attn_group.build_metadata_for_cudagraph_capture(
common_attn_metadata, ubatch_idx
)
else:
attn_metadata_extra_kwargs = (
@@ -548,10 +552,8 @@ def build_attn_metadata(
if model_specific_attn_metadata is not None
else {}
)
metadata = attn_metadata_builder.build(
common_prefix_len=0,
common_attn_metadata=common_attn_metadata,
**attn_metadata_extra_kwargs,
metadata = attn_group.build_metadata(
common_attn_metadata, ubatch_idx, **attn_metadata_extra_kwargs
)
if reuse_key is not None and reuse_key not in cached_metadata:
cached_metadata[reuse_key] = metadata
+4 -22
View File
@@ -25,7 +25,6 @@ class BlockTables:
max_num_batched_tokens: int,
max_num_blocks_per_group: list[int],
device: torch.device,
kernel_block_sizes: list[int],
cp_size: int = 1,
cp_rank: int = 0,
cp_interleave: int = 1,
@@ -33,7 +32,6 @@ class BlockTables:
dcp_sharded: list[bool] | None = None,
):
self.block_sizes = block_sizes
self.kernel_block_sizes = kernel_block_sizes
self.max_num_reqs = max_num_reqs
self.max_num_batched_tokens = max_num_batched_tokens
self.device = device
@@ -53,14 +51,9 @@ class BlockTables:
assert len(dcp_sharded) == self.num_kv_cache_groups
self.dcp_sharded = torch.tensor(dcp_sharded, dtype=torch.bool, device=device)
self.blocks_per_kv_block = [
bs // kbs for bs, kbs in zip(block_sizes, kernel_block_sizes)
]
# num_kv_cache_groups x [max_num_reqs, max_num_blocks]
self.block_tables: list[StagedWriteTensor] = []
for i in range(self.num_kv_cache_groups):
max_num_blocks = max_num_blocks_per_group[i] * self.blocks_per_kv_block[i]
for max_num_blocks in max_num_blocks_per_group:
block_table = StagedWriteTensor(
(self.max_num_reqs, max_num_blocks), dtype=torch.int32, device=device
)
@@ -109,9 +102,6 @@ class BlockTables:
self.block_sizes_tensor = torch.tensor(
self.block_sizes, dtype=torch.int32, device=self.device
)
self.kernel_block_sizes_tensor = torch.tensor(
self.kernel_block_sizes, dtype=torch.int32, device=self.device
)
self.slot_mapping_enabled = torch.tensor(
self._slot_mapping_enabled, dtype=torch.bool, device=self.device
)
@@ -126,9 +116,6 @@ class BlockTables:
for i in range(self.num_kv_cache_groups):
start = self.num_blocks.np[i, req_index] if not overwrite else 0
block_ids = new_block_ids[i]
bpk = self.blocks_per_kv_block[i]
if bpk > 1:
block_ids = [b * bpk + k for b in block_ids for k in range(bpk)]
if self.redirect_writes_to_null_block:
block_ids = [0] * len(block_ids)
end = start + len(block_ids)
@@ -218,7 +205,6 @@ class BlockTables:
self.block_table_ptrs,
self.block_table_strides,
self.block_sizes_tensor,
self.kernel_block_sizes_tensor,
self.slot_mapping_enabled,
self.dcp_sharded,
slot_mappings,
@@ -292,7 +278,6 @@ def _compute_slot_mappings_kernel(
block_table_ptrs, # [num_kv_cache_groups]
block_table_strides, # [num_kv_cache_groups]
block_sizes, # [num_kv_cache_groups]
kernel_block_sizes, # [num_kv_cache_groups]
slot_mapping_enabled, # [num_kv_cache_groups]
dcp_sharded, # [num_kv_cache_groups]
slot_mappings_ptr, # [num_kv_cache_groups, max_num_tokens]
@@ -322,7 +307,6 @@ def _compute_slot_mappings_kernel(
block_table_ptr = _load_ptr(block_table_ptrs + group_id, tl.int32)
block_table_stride = tl.load(block_table_strides + group_id)
kv_block_size = tl.load(block_sizes + group_id)
kernel_block_size = tl.load(kernel_block_sizes + group_id)
mapping_enabled = tl.load(slot_mapping_enabled + group_id)
if CP_SIZE != 1:
sharded = tl.load(dcp_sharded + group_id)
@@ -354,16 +338,14 @@ def _compute_slot_mappings_kernel(
local_positions = tl.where(sharded, local_positions, positions)
is_local = ~sharded | is_local
block_indices = tl.where(
mapping_enabled, local_positions // kernel_block_size, 0
)
block_offsets = local_positions % kernel_block_size
block_indices = tl.where(mapping_enabled, local_positions // kv_block_size, 0)
block_offsets = local_positions % kv_block_size
block_numbers = tl.load(
block_table_ptr + req_state_idx * block_table_stride + block_indices,
mask=is_local & is_real_req,
other=0,
)
slot_ids = block_numbers * kernel_block_size + block_offsets
slot_ids = block_numbers * kv_block_size + block_offsets
if CP_SIZE != 1:
slot_ids = tl.where(is_local, slot_ids, PAD_ID)
+9 -8
View File
@@ -252,17 +252,18 @@ def set_dummy_context(
input_batch.positions.copy_(torch.from_numpy(local_pos + context_len))
seq_len = context_len + query_len
for block_table, block_size, bpk in zip(
input_block_tables,
block_tables.kernel_block_sizes,
block_tables.blocks_per_kv_block,
):
for block_table, block_size in zip(input_block_tables, block_tables.block_sizes):
num_blocks = min(cdiv(seq_len, block_size), block_table.shape[1])
# Spans are disjoint until the pool runs out, then they wrap and share
# blocks: profiling only needs the reads to be realistic, not distinct.
block_ids = torch.arange(
num_reqs * num_blocks, dtype=block_table.dtype, device=block_table.device
) % (num_kv_blocks * bpk)
block_ids = (
torch.arange(
num_reqs * num_blocks,
dtype=block_table.dtype,
device=block_table.device,
)
% num_kv_blocks
)
block_table[:num_reqs, :num_blocks] = block_ids.view(num_reqs, num_blocks)
+7 -1
View File
@@ -688,7 +688,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
max_num_batched_tokens=self.max_num_tokens,
max_num_blocks_per_group=max_num_blocks_per_group,
device=self.device,
kernel_block_sizes=self.kernel_block_sizes,
slot_mapping_enabled=slot_mapping_enabled,
dcp_sharded=dcp_sharded,
cp_size=self.dcp_size,
@@ -773,6 +772,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.vllm_config,
kv_cache_allocation_context=kv_cache_allocation_context,
block_tables=self.block_tables,
# Draft groups build their own metadata, so they map the views too.
attn_groups=(
g
for groups in self.attn_groups
+ getattr(self.speculator, "attn_groups", [])
for g in groups
),
)
self.kv_caches = [
cache for cache in kv_caches_dict.values() if cache.device == self.device
@@ -432,7 +432,7 @@ class DFlashSpeculator(DraftModelSpeculator):
temperature,
seeds,
self.block_tables.input_block_tables[gid],
self.block_tables.kernel_block_sizes[gid],
self.block_tables.block_sizes[gid],
self.block_tables.cp_rank,
self.dcp_size,
self.block_tables.cp_interleave,
@@ -577,6 +577,11 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
idx_mapping = self.idx_mapping[:num_reqs]
if batch_desc.cg_mode == CUDAGraphMode.FULL:
for groups in self.attn_groups:
for group in groups:
group.map_to_kernel_block_table(
self.block_tables.input_block_tables[group.kv_cache_group_id]
)
assert self.decode_cudagraph_manager is not None
self.decode_cudagraph_manager.run_fullgraph(batch_desc)
return
-2
View File
@@ -96,7 +96,6 @@ class InputBatch:
device: torch.device,
vocab_size: int,
block_sizes: list[int], # The block_size of each kv cache group
kernel_block_sizes: list[int],
max_num_blocks_per_req: list[int],
logitsprocs: LogitsProcessors | None = None,
logitsprocs_need_output_token_ids: bool = False,
@@ -187,7 +186,6 @@ class InputBatch:
pin_memory=PIN_MEMORY,
device=device,
block_sizes=block_sizes,
kernel_block_sizes=kernel_block_sizes,
max_num_blocks=max_num_blocks_per_req,
cp_kv_cache_interleave_size=cp_kv_cache_interleave_size,
slot_mapping_modes=slot_mapping_modes,
+21 -20
View File
@@ -233,6 +233,8 @@ from .utils import (
allocate_kv_cache,
bind_kv_cache,
copy_kv_cache_blocks_inplace,
customize_attention_spec,
map_kv_caches_to_kernel_blocks,
prepare_kernel_block_sizes,
sanity_check_mm_encoder_outputs,
)
@@ -718,7 +720,6 @@ class GPUModelRunner(
max(self.max_model_len, self.max_encoder_len), placeholder_block_size
)
self._init_block_sizes = [placeholder_block_size]
self._init_kernel_block_sizes = [placeholder_block_size]
self._init_max_num_blocks = [placeholder_max_num_blocks]
self._init_slot_mapping_modes = [SlotMappingMode.TOKEN_TO_KV_SLOT]
self.cp_kv_cache_interleave_size = (
@@ -734,7 +735,6 @@ class GPUModelRunner(
device=self.device,
vocab_size=self.model_config.get_vocab_size(),
block_sizes=[placeholder_block_size],
kernel_block_sizes=[placeholder_block_size],
max_num_blocks_per_req=[placeholder_max_num_blocks],
num_spec_tokens=self.num_spec_tokens,
logitsprocs=build_logitsprocs(
@@ -2508,8 +2508,8 @@ class GPUModelRunner(
)
if for_cudagraph_capture:
attn_metadata_i = builder.build_for_cudagraph_capture(
common_attn_metadata
attn_metadata_i = attn_group.build_metadata_for_cudagraph_capture(
common_attn_metadata, ubid or 0
)
elif (
cache_key in cached_attn_metadata
@@ -2517,13 +2517,16 @@ class GPUModelRunner(
):
attn_metadata_i = builder.update_block_table(
cached_attn_metadata[cache_key],
common_attn_metadata.block_table_tensor,
attn_group.map_to_kernel_block_table(
common_attn_metadata.block_table_tensor, ubid or 0
),
common_attn_metadata.slot_mapping,
)
else:
attn_metadata_i = builder.build(
attn_metadata_i = attn_group.build_metadata(
common_attn_metadata,
ubid or 0,
common_prefix_len=cascade_attn_prefix_len,
common_attn_metadata=common_attn_metadata,
**extra_attn_metadata_args,
)
if builder.supports_update_block_table:
@@ -7143,9 +7146,7 @@ class GPUModelRunner(
return
self.reorder_batch_threshold = reduce(min_none_high, reorder_batch_thresholds) # type: ignore[assignment]
def may_reinitialize_input_batch(
self, kv_cache_config: KVCacheConfig, kernel_block_sizes: list[int]
) -> None:
def may_reinitialize_input_batch(self, kv_cache_config: KVCacheConfig) -> None:
"""Re-initialize the input batch if the block sizes are different from
what it was originally created with. This happens when the final
block size (determined after model loading) differs from the
@@ -7154,7 +7155,6 @@ class GPUModelRunner(
Args:
kv_cache_config: The KV cache configuration.
kernel_block_sizes: The kernel block sizes for each KV cache group.
"""
block_sizes = []
@@ -7179,14 +7179,12 @@ class GPUModelRunner(
if (
block_sizes != self._init_block_sizes
or kernel_block_sizes != self._init_kernel_block_sizes
or max_num_blocks != self._init_max_num_blocks
or slot_mapping_modes != self._init_slot_mapping_modes
or self.cp_kv_cache_interleave_size
!= self.parallel_config.cp_kv_cache_interleave_size
):
self._init_block_sizes = block_sizes
self._init_kernel_block_sizes = kernel_block_sizes
self._init_max_num_blocks = max_num_blocks
self._init_slot_mapping_modes = slot_mapping_modes
self.cp_kv_cache_interleave_size = (
@@ -7201,7 +7199,6 @@ class GPUModelRunner(
device=self.device,
vocab_size=self.model_config.get_vocab_size(),
block_sizes=block_sizes,
kernel_block_sizes=kernel_block_sizes,
max_num_blocks_per_req=max_num_blocks,
num_spec_tokens=self.num_spec_tokens,
logitsprocs=self.input_batch.logitsprocs,
@@ -7217,10 +7214,6 @@ class GPUModelRunner(
f"InputBatch block_sizes {self._init_block_sizes} != "
f"kv_cache block_sizes {block_sizes}"
)
assert self._init_kernel_block_sizes == kernel_block_sizes, (
f"InputBatch kernel_block_sizes {self._init_kernel_block_sizes} "
f"!= kv_cache kernel_block_sizes {kernel_block_sizes}"
)
def _attn_group_iterator(self) -> Iterator[AttentionGroup]:
return itertools.chain.from_iterable(self.attn_groups)
@@ -7267,12 +7260,16 @@ class GPUModelRunner(
num_attn_module = (
2 if self.model_config.hf_config.model_type == "longcat_flash" else 1
)
attn_groups = [g for groups in self.attn_groups for g in groups]
if self.speculative_config:
attn_groups += getattr(self.drafter, "draft_attn_groups", [])
bind_kv_cache(
kv_caches,
self.compilation_config.static_forward_context,
self.kv_caches,
num_attn_module,
kv_cache_groups=kv_cache_config.kv_cache_groups,
layer_kv_caches=map_kv_caches_to_kernel_blocks(kv_caches, attn_groups),
)
return kv_caches
@@ -7336,6 +7333,8 @@ class GPUModelRunner(
# backends for that group only supports block_size 64, we will return
# kernel_block_size 64 and split the 256-token-block to 4 blocks with 64
# tokens each.
# Block-outermost packed groups are not split; their attention groups
# map kernel blocks themselves.
kernel_block_sizes = prepare_kernel_block_sizes(
kv_cache_config, self.attn_groups
)
@@ -7345,7 +7344,7 @@ class GPUModelRunner(
self.initialize_metadata_builders(kv_cache_config, kernel_block_sizes)
# Reinitialize need to after initialize_attn_backend
self.may_reinitialize_input_batch(kv_cache_config, kernel_block_sizes)
self.may_reinitialize_input_batch(kv_cache_config)
# Capture warmup providers that depend on allocated KV-cache strides.
with self.jit_warmup_registry.activate(): # type: ignore[attr-defined]
kv_caches = self.initialize_kv_cache_tensors(
@@ -7422,7 +7421,9 @@ class GPUModelRunner(
# Skip modules that don't need KV cache (eg encoder-only attention)
if spec := attn_module.get_kv_cache_spec(self.vllm_config):
if isinstance(spec, AttentionSpec):
spec = attn_module.get_attn_backend().customize_spec(spec)
spec = customize_attention_spec(
attn_module.get_attn_backend(), spec
)
kv_cache_spec[layer_name] = spec
return kv_cache_spec
+207 -45
View File
@@ -3,7 +3,7 @@
import math
from collections import defaultdict
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from itertools import product as iprod
from typing import Any
@@ -24,6 +24,7 @@ from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionMetadataBuilder,
CommonAttentionMetadata,
MultipleOf,
)
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
@@ -35,7 +36,6 @@ from vllm.v1.kv_cache_interface import (
KVCacheLayout,
KVCacheSpec,
MambaSpec,
MLAAttentionSpec,
UniformTypeKVCacheSpecs,
create_kv_cache_views,
)
@@ -166,6 +166,9 @@ class KVBlockZeroer:
kv = static_forward_context[layer_name].kv_cache
if not isinstance(kv, torch.Tensor):
continue
if group.kernel_block_stride:
# Bound views address kernel blocks; zero by manager block.
kv = group.map_kv_cache_to_manager_blocks(kv, num_blocks)
if kv.device.type != self.device.type:
continue
dp = kv.data_ptr()
@@ -255,12 +258,22 @@ class KVBlockZeroer:
self.zero_block_ids([0])
@dataclass
@dataclass(eq=False)
class AttentionGroup:
backend: type[AttentionBackend]
layer_names: list[str]
kv_cache_spec: KVCacheSpec
kv_cache_group_id: int
# Set when the backend needs blocks smaller than the KV cache manager's: its
# block tables and KV cache views then address kernel blocks, kernel block j
# of block b being b * kernel_block_stride + j. The stride is set when a
# packed KV cache is bound (other layers' pages sit between its blocks);
# otherwise it stays 0 and the kernel blocks per block are used.
kernel_block_size: int | None = None
kernel_block_stride: int = 0
# Persistent per ubatch: CUDA graphs capture the mapped block table.
kernel_block_table: torch.Tensor | None = None
kernel_block_offsets: torch.Tensor | None = None
# When ubatching is enabled we will have a metadata builder for each ubatch
# so that if they use internal persistent buffers for cudagraphs, and they
# won't have to worry about conflicting with the other ubatches.
@@ -275,27 +288,44 @@ class AttentionGroup:
kernel_block_size: int | None = None,
num_metadata_builders: int = 1,
):
spec = self.kv_cache_spec
if kernel_block_size is not None and isinstance(spec, AttentionSpec):
# A backend in a packed KV cache group may need smaller blocks than
# the group is split into.
kernel_block_size = select_common_block_size(
kernel_block_size, [self.backend], [spec]
)
if kernel_block_size != spec.block_size:
self.kernel_block_size = kernel_block_size
if kernel_block_size is None:
kv_cache_spec_builder = self.kv_cache_spec
elif (
isinstance(self.kv_cache_spec, MLAAttentionSpec)
and self.kv_cache_spec.storage_block_size is not None
):
kv_cache_spec_builder = self.kv_cache_spec.copy_with_new_block_size(
self.kv_cache_spec.storage_block_size
)
else:
kv_cache_spec_builder = self.kv_cache_spec.copy_with_new_block_size(
kernel_block_size
)
builder_cls = self.backend.get_builder_cls()
builder_kwargs = {}
if builder_cls.requires_block_table_width:
if builder_cls.requires_block_table_width or self.kernel_block_size is not None:
max_num_blocks = self.kv_cache_spec.max_num_blocks_per_req(
vllm_config, vllm_config.model_config.max_model_len
)
builder_kwargs["block_table_width"] = get_block_table_width(
max_num_blocks, self.kv_cache_spec.block_size, kernel_block_size
width = get_block_table_width(max_num_blocks, self.kv_cache_spec.block_size)
if kernel_block_size is not None:
width *= self.kv_cache_spec.block_size // kernel_block_size
if builder_cls.requires_block_table_width:
builder_kwargs["block_table_width"] = width
if self.kernel_block_size is not None:
self.kernel_block_table = torch.zeros(
num_metadata_builders,
vllm_config.scheduler_config.max_num_seqs,
width,
dtype=torch.int32,
device=device,
)
self.kernel_block_offsets = torch.arange(
spec.block_size // self.kernel_block_size,
dtype=torch.int32,
device=device,
)
self.metadata_builders = [
builder_cls(
@@ -307,9 +337,109 @@ class AttentionGroup:
)
for _ in range(num_metadata_builders)
]
if kernel_block_size is not None:
for builder in self.metadata_builders:
builder.set_kernel_block_size(kernel_block_size)
def build_metadata(
self,
common_attn_metadata: CommonAttentionMetadata,
ubatch_idx: int = 0,
common_prefix_len: int = 0,
**kwargs,
) -> Any:
return self.get_metadata_builder(ubatch_idx).build(
common_prefix_len,
self._with_kernel_block_table(common_attn_metadata, ubatch_idx),
**kwargs,
)
def build_metadata_for_cudagraph_capture(
self, common_attn_metadata: CommonAttentionMetadata, ubatch_idx: int = 0
) -> Any:
return self.get_metadata_builder(ubatch_idx).build_for_cudagraph_capture(
self._with_kernel_block_table(common_attn_metadata, ubatch_idx)
)
def build_metadata_for_drafting(
self, common_attn_metadata: CommonAttentionMetadata, draft_index: int
) -> Any:
return self.get_metadata_builder().build_for_drafting(
self._with_kernel_block_table(common_attn_metadata), draft_index
)
def _with_kernel_block_table(
self, common_attn_metadata: CommonAttentionMetadata, ubatch_idx: int = 0
) -> CommonAttentionMetadata:
if self.kernel_block_size is None:
return common_attn_metadata
return replace(
common_attn_metadata,
block_table_tensor=self.map_to_kernel_block_table(
common_attn_metadata.block_table_tensor, ubatch_idx
),
)
def map_to_kernel_block_table(
self, block_table: torch.Tensor, ubatch_idx: int = 0
) -> torch.Tensor:
if self.kernel_block_size is None:
return block_table
assert self.kernel_block_table is not None
blocks_per_kv_block = self.kv_cache_spec.block_size // self.kernel_block_size
rows, cols = block_table.shape
out = self.kernel_block_table[ubatch_idx, :rows, : cols * blocks_per_kv_block]
torch.add(
self.kernel_block_offsets,
block_table.unsqueeze(-1),
alpha=self.kernel_block_stride or blocks_per_kv_block,
out=out.unflatten(1, (cols, -1)),
)
return out
def map_kv_cache_to_kernel_blocks(self, kv_cache: torch.Tensor) -> torch.Tensor:
"""Views a packed cache in manager blocks as kernel blocks."""
assert self.kernel_block_size is not None
spec = self.kv_cache_spec
kernel_rows = spec.get_num_kernel_states(self.kernel_block_size)
if kv_cache.shape[-2] == kernel_rows:
return kv_cache
# Token and pooled caches are written through manager-block slot
# mappings, so only compressed caches (written from their block table)
# can be mapped.
if (
spec.tokens_per_state <= 1
or kv_cache.shape[1] != 1
or kv_cache.stride(0) % (kernel_rows * kv_cache.stride(-2))
):
raise ValueError(
f"{self.backend.get_name()} needs {self.kernel_block_size}-token "
f"kernel blocks, which packed {spec.block_size}-token KV cache "
f"blocks cannot be split into. Reduce --block-size to "
f"{self.kernel_block_size} or set VLLM_KV_CACHE_LAYOUT to a "
"layer-compact layout (e.g. LBNHC)."
)
# Kernel block j of block b is b * kernel_block_stride + j. The view
# spans the other layers' pages between blocks (never addressing them),
# which ``view`` cannot express.
page_stride = kernel_rows * kv_cache.stride(-2)
self.kernel_block_stride = kv_cache.stride(0) // page_stride
num_pages = (kv_cache.shape[0] - 1) * self.kernel_block_stride + (
kv_cache.shape[-2] // kernel_rows
)
return kv_cache.as_strided(
(num_pages, *kv_cache.shape[1:-2], kernel_rows, kv_cache.shape[-1]),
(page_stride, *kv_cache.stride()[1:]),
)
def map_kv_cache_to_manager_blocks(
self, kv_cache: torch.Tensor, num_blocks: int
) -> torch.Tensor:
"""Views a packed cache in kernel blocks as manager blocks."""
assert self.kernel_block_stride
spec = self.kv_cache_spec
rows = spec.get_num_kernel_states(spec.block_size)
return kv_cache.as_strided(
(num_blocks, *kv_cache.shape[1:-2], rows, kv_cache.shape[-1]),
(kv_cache.stride(0) * self.kernel_block_stride, *kv_cache.stride()[1:]),
)
def get_metadata_builder(self, ubatch_id: int = 0) -> AttentionMetadataBuilder:
assert len(self.metadata_builders) > ubatch_id
@@ -327,17 +457,48 @@ class AttentionGroup:
self.get_metadata_builder().update_draft_decode_metadata(metadata)
def map_kv_caches_to_kernel_blocks(
kv_caches: dict[str, torch.Tensor], attn_groups: Iterable[AttentionGroup]
) -> dict[str, torch.Tensor]:
"""The views to bind to layers. The manager views in ``kv_caches`` stay for
KV connectors."""
mapped = dict(kv_caches)
for group in attn_groups:
if group.kernel_block_size is not None:
mapped.update(
(name, group.map_kv_cache_to_kernel_blocks(kv_caches[name]))
for name in group.layer_names
if name in kv_caches
)
return mapped
def customize_attention_spec(
backend: type[AttentionBackend], spec: AttentionSpec
) -> AttentionSpec:
"""The backend's spec. Blocks larger than every kernel block size a backend
takes are split into kernel blocks, which packed blocks must keep whole."""
spec = backend.customize_spec(spec)
sizes = backend.get_supported_kernel_block_sizes(spec)
fixed_sizes = [s for s in sizes if isinstance(s, int)]
if spec.block_stride_alignment is None and sizes and fixed_sizes == sizes:
spec = replace(spec, block_stride_alignment=MultipleOf(max(fixed_sizes)))
return spec
def _block_size_is_supported(
backends: list[type[AttentionBackend]], block_size: int
backends: list[type[AttentionBackend]],
block_size: int,
specs: Sequence[KVCacheSpec | None],
) -> bool:
"""Check if the block size is supported by all backends.
An exact ``int`` declaration must match exactly; a ``MultipleOf``
declaration accepts any multiple of its base.
"""
for backend in backends:
for backend, spec in zip(backends, specs, strict=True):
is_supported = False
for supported_size in backend.get_supported_kernel_block_sizes():
for supported_size in backend.get_supported_kernel_block_sizes(spec):
if isinstance(supported_size, int):
if block_size == supported_size:
is_supported = True
@@ -354,6 +515,7 @@ def _block_size_is_supported(
def select_common_block_size(
kv_manager_block_size: int,
backends: list[type[AttentionBackend]],
kv_cache_specs: Sequence[KVCacheSpec | None] | None = None,
) -> int:
"""Select a block size that is supported by all backends and is a factor of
kv_manager_block_size.
@@ -364,6 +526,7 @@ def select_common_block_size(
Args:
kv_manager_block_size: Block size of KV cache.
backends: List of attention backend classes.
kv_cache_specs: Per-backend cache specs for spec-specific constraints.
Returns:
The selected block size.
@@ -372,25 +535,27 @@ def select_common_block_size(
ValueError: If no valid block size found.
"""
if _block_size_is_supported(backends, kv_manager_block_size):
specs = kv_cache_specs or [None] * len(backends)
if _block_size_is_supported(backends, kv_manager_block_size, specs):
return kv_manager_block_size
# MultipleOf constraints also accept the manager size if they accept a divisor.
# Any remaining candidate must therefore be an explicit size from a backend.
candidates = {
size
for backend in backends
for size in backend.get_supported_kernel_block_sizes()
for backend, spec in zip(backends, specs, strict=True)
for size in backend.get_supported_kernel_block_sizes(spec)
if isinstance(size, int) and kv_manager_block_size % size == 0
}
for size in sorted(candidates, reverse=True):
if _block_size_is_supported(backends, size):
if _block_size_is_supported(backends, size, specs):
return size
raise ValueError(
f"No common block size for {kv_manager_block_size} ("
+ "; ".join(
f"{b.get_name()}: {b.get_supported_kernel_block_sizes()}" for b in backends
f"{b.get_name()}: {b.get_supported_kernel_block_sizes(s)}"
for b, s in zip(backends, specs, strict=True)
)
+ ")."
)
@@ -450,9 +615,6 @@ def allocate_kv_cache(
kernel_block_size = None
if kernel_block_sizes is not None and group_id < len(kernel_block_sizes):
kernel_block_size = kernel_block_sizes[group_id]
if isinstance(spec, MLAAttentionSpec) and spec.storage_block_size is not None:
kernel_block_size = spec.storage_block_size
views = create_kv_cache_views(
buf,
spec,
@@ -497,24 +659,20 @@ def prepare_kernel_block_sizes(
# This is an attention backend that supports virtual block splitting.
kv_manager_block_size = kv_cache_group.kv_cache_spec.block_size
group_backends = [g.backend for g in attn_groups[kv_cache_gid]]
storage_block_size = (
kv_cache_spec.storage_block_size
if isinstance(kv_cache_spec, MLAAttentionSpec)
else None
)
if storage_block_size is not None and _block_size_is_supported(
group_backends, storage_block_size
):
# Storage-block specs (e.g. the GLM-5.3-Flash kpool indexer
# cache) address the cache in pool pages, and every other
# consumer (cache views, metadata builders, hisparse) already
# uses storage_block_size as the kernel block. Fall back to the
# backend vote when the group's backends do not accept it.
selected_kernel_size = storage_block_size
else:
selected_kernel_size = select_common_block_size(
kv_manager_block_size, group_backends
group_specs = [g.kv_cache_spec for g in attn_groups[kv_cache_gid]]
# Block-outermost packing puts other layers' pages between the
# blocks, so they are never split group-wide.
selected_kernel_size = (
kv_manager_block_size
if any(
t.block_stride > t.layer_stride
for t in kv_cache_config.kv_cache_tensors
if not set(t.layers).isdisjoint(kv_cache_group.layer_names)
)
else select_common_block_size(
kv_manager_block_size, group_backends, group_specs
)
)
kernel_block_sizes.append(selected_kernel_size)
elif isinstance(kv_cache_spec, MambaSpec):
# This is likely Mamba or other non-attention cache, no splitting.
@@ -656,6 +814,7 @@ def bind_kv_cache(
runner_kv_caches: list[torch.Tensor],
num_attn_module: int = 1,
kv_cache_groups: Sequence[KVCacheGroupSpec] | None = None,
layer_kv_caches: dict[str, torch.Tensor] | None = None,
) -> None:
"""Bind the allocated KV cache to both ModelRunner and forward context so
that the KV cache can be used in the forward pass.
@@ -674,6 +833,9 @@ def bind_kv_cache(
runner_kv_caches: The kv_cache declared by ModelRunner.
kv_cache_groups: The KV cache groups of the model, used to resolve
layers that share a KV cache.
layer_kv_caches: The views to bind to layers when they differ from
kv_caches (e.g. kernel-block views). The runner keeps kv_caches,
which are addressed by scheduler block IDs.
"""
# Bind kv_caches to ModelRunner
@@ -699,7 +861,7 @@ def bind_kv_cache(
runner_kv_caches.append(kv_caches[layer_name])
bind_kv_cache_to_layers(
kv_caches, forward_context, num_attn_module, kv_cache_groups
layer_kv_caches or kv_caches, forward_context, num_attn_module, kv_cache_groups
)