[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
OpenAI Codex
OpenAI Codex
Lucas Wilkinson
Claude Opus 5.5
Lucas Wilkinson
parent
3766a67a32
commit
02e55d2af3
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheConfig
|
||||
from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -351,6 +352,35 @@ def get_current_attn_backends(
|
||||
]
|
||||
|
||||
|
||||
def get_current_attn_backends_and_specs(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
fallback: list[type[AttentionBackend]],
|
||||
) -> tuple[list[type[AttentionBackend]], list[AttentionSpec | None]]:
|
||||
"""Distinct (backend, spec) pairs of the transfer layers, else ``fallback``.
|
||||
|
||||
Compressed caches are skipped: their kernel pages never split transferred
|
||||
blocks.
|
||||
"""
|
||||
pairs: dict[tuple[type[AttentionBackend], AttentionSpec | None], None] = {}
|
||||
layer_type = cast(type[Any], AttentionLayerBase)
|
||||
for group in kv_cache_config.transfer_groups:
|
||||
specs = getattr(group.kv_cache_spec, "kv_cache_specs", {})
|
||||
layers = get_layers_from_vllm_config(vllm_config, layer_type, group.layer_names)
|
||||
for name, layer in layers.items():
|
||||
spec = specs.get(name, group.kv_cache_spec)
|
||||
if isinstance(spec, AttentionSpec) and spec.tokens_per_state > 1:
|
||||
layout = vllm_config.cache_config.get_resolved_kv_cache_layout()
|
||||
assert layout.is_block_outermost, "Compressed caches must be packed"
|
||||
continue
|
||||
if not isinstance(spec, AttentionSpec):
|
||||
spec = None
|
||||
pairs[layer.get_attn_backend(), spec] = None
|
||||
if not pairs:
|
||||
return fallback, [None] * len(fallback)
|
||||
return [backend for backend, _ in pairs], [spec for _, spec in pairs]
|
||||
|
||||
|
||||
def get_current_attn_backend(
|
||||
vllm_config: VllmConfig, layer_names: list[str] | None = None
|
||||
) -> type[AttentionBackend]:
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
EngineId,
|
||||
TransferTopology,
|
||||
get_current_attn_backends,
|
||||
get_current_attn_backends_and_specs,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
@@ -59,8 +60,8 @@ from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
@@ -69,7 +70,6 @@ from vllm.v1.kv_cache_interface import (
|
||||
)
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.request import RequestStatus
|
||||
from vllm.v1.worker.block_table import BlockTable
|
||||
from vllm.v1.worker.utils import select_common_block_size
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -1350,7 +1350,10 @@ class MooncakeConnectorWorker:
|
||||
# physical block sizes. Pick the common (smallest) block size so that
|
||||
# KV-cache registration and transfer work correctly for both models.
|
||||
backends = get_current_attn_backends(self.vllm_config)
|
||||
kernel_block_size = select_common_block_size(self.block_size, backends)
|
||||
backends, specs = get_current_attn_backends_and_specs(
|
||||
self.vllm_config, self.kv_cache_config, backends
|
||||
)
|
||||
kernel_block_size = select_common_block_size(self.block_size, backends, specs)
|
||||
if self.block_size != kernel_block_size:
|
||||
logger.info_once(
|
||||
"User-specified logical block size (%s) does not match"
|
||||
@@ -1727,11 +1730,12 @@ class MooncakeConnectorWorker:
|
||||
)
|
||||
group_specs = self.kv_cache_config.transfer_groups
|
||||
return [
|
||||
BlockTable.map_to_kernel_blocks(
|
||||
np.array(group),
|
||||
self._physical_blocks_per_logical_kv_block,
|
||||
block_arange,
|
||||
).tolist()
|
||||
(
|
||||
np.array(group)[:, None] * self._physical_blocks_per_logical_kv_block
|
||||
+ block_arange
|
||||
)
|
||||
.reshape(-1)
|
||||
.tolist()
|
||||
if not isinstance(group_specs[i].kv_cache_spec, MambaSpec)
|
||||
else group
|
||||
for i, group in enumerate(block_ids)
|
||||
@@ -2054,7 +2058,7 @@ class MooncakeConnectorWorker:
|
||||
# hetero PP can align by name (#56033). A non-contiguous row cannot
|
||||
# name a sub-span, so it is one region anchored at the allocation.
|
||||
use_packed_row = (
|
||||
not isinstance(layer_spec, (MambaSpec, KpoolTailSpec))
|
||||
not isinstance(layer_spec, (MambaSpec, CircularBufferSpec))
|
||||
and storage_is_block_major
|
||||
and block_stride > 0
|
||||
and (is_mla_region or not hnc_contiguous)
|
||||
@@ -2121,9 +2125,7 @@ class MooncakeConnectorWorker:
|
||||
block_len = region_cache.stride(0) * region_cache.element_size()
|
||||
region_base_addresses.append(base_addr)
|
||||
|
||||
if isinstance(layer_spec, KpoolTailSpec):
|
||||
kv_block_len = layer_spec.unpadded_page_size_bytes // 2
|
||||
elif isinstance(layer_spec, AttentionSpec) and block_is_contiguous:
|
||||
if isinstance(layer_spec, AttentionSpec) and block_is_contiguous:
|
||||
assert (
|
||||
layer_spec.page_size_bytes
|
||||
% self._physical_blocks_per_logical_kv_block
|
||||
@@ -2133,6 +2135,8 @@ class MooncakeConnectorWorker:
|
||||
layer_spec.page_size_bytes
|
||||
// self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
elif isinstance(layer_spec, MambaSpec) and block_is_contiguous:
|
||||
kv_block_len = layer_spec.page_size_bytes
|
||||
else:
|
||||
kv_block_len = block_len
|
||||
if kv_block_len > block_len:
|
||||
|
||||
@@ -28,6 +28,7 @@ from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
EngineTransferInfo,
|
||||
TransferTopology,
|
||||
get_current_attn_backends,
|
||||
get_current_attn_backends_and_specs,
|
||||
kv_postprocess_blksize_and_layout_on_receive,
|
||||
kv_postprocess_blksize_on_receive,
|
||||
)
|
||||
@@ -79,7 +80,6 @@ from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheLayout,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
@@ -88,7 +88,6 @@ from vllm.v1.kv_cache_interface import (
|
||||
UniformTypeKVCacheSpecs,
|
||||
iter_layer_specs,
|
||||
)
|
||||
from vllm.v1.worker.block_table import BlockTable
|
||||
from vllm.v1.worker.utils import select_common_block_size
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -120,46 +119,6 @@ def _share_storage_and_block_stride(caches: list[torch.Tensor]) -> bool:
|
||||
return len(block_strides) == len(storage_ptrs) == 1
|
||||
|
||||
|
||||
def _tensor_byte_span_end(cache: torch.Tensor) -> int:
|
||||
"""Return the exclusive end address touched by a nonnegative-stride view."""
|
||||
if cache.numel() == 0:
|
||||
return cache.data_ptr()
|
||||
if any(stride < 0 for stride in cache.stride()):
|
||||
raise ValueError("NIXL cache views must have nonnegative strides")
|
||||
max_element_offset = sum(
|
||||
(size - 1) * stride for size, stride in zip(cache.shape, cache.stride())
|
||||
)
|
||||
return cache.data_ptr() + (max_element_offset + 1) * cache.element_size()
|
||||
|
||||
|
||||
def _uses_dense_virtual_transfer_pages(
|
||||
layer_spec: KVCacheSpec,
|
||||
cache: torch.Tensor,
|
||||
physical_page_size: int,
|
||||
num_blocks: int,
|
||||
) -> bool:
|
||||
"""Return whether a compressed kernel view can be split into NIXL pages."""
|
||||
if not (
|
||||
isinstance(layer_spec, MLAAttentionSpec)
|
||||
and layer_spec.tokens_per_state > 1
|
||||
and cache.ndim == 4
|
||||
and cache.shape[1] == 1
|
||||
and cache.is_contiguous()
|
||||
and physical_page_size > 0
|
||||
and layer_spec.state_content_size_bytes > 0
|
||||
):
|
||||
return False
|
||||
|
||||
block_stride = cache.stride(0) * cache.element_size()
|
||||
return (
|
||||
block_stride > physical_page_size
|
||||
and block_stride % physical_page_size == 0
|
||||
and physical_page_size % layer_spec.state_content_size_bytes == 0
|
||||
and cache.shape[0] * (block_stride // physical_page_size) == num_blocks
|
||||
and cache.nbytes == num_blocks * physical_page_size
|
||||
)
|
||||
|
||||
|
||||
class NixlBaseConnectorWorker:
|
||||
"""Base implementation of Worker side methods shared by pull and push."""
|
||||
|
||||
@@ -985,8 +944,10 @@ class NixlBaseConnectorWorker:
|
||||
)
|
||||
|
||||
def _sync_block_size_with_kernel(self) -> None:
|
||||
backends = get_current_attn_backends(self.vllm_config)
|
||||
kernel_block_size = select_common_block_size(self.block_size, backends)
|
||||
backends, specs = get_current_attn_backends_and_specs(
|
||||
self.vllm_config, self.kv_cache_config, self.attn_backends
|
||||
)
|
||||
kernel_block_size = select_common_block_size(self.block_size, backends, specs)
|
||||
# Number of blocks not accounting for kernel block mismatches
|
||||
self._logical_num_blocks = self.num_blocks
|
||||
if self.block_size != kernel_block_size:
|
||||
@@ -1425,7 +1386,7 @@ class NixlBaseConnectorWorker:
|
||||
if self._supports_pp_hma and self._has_packed_cache:
|
||||
# PP can reorder the same backends across stages of a packed model.
|
||||
backend_name = ",".join(
|
||||
sorted(backend.get_name() for backend in self.attn_backends)
|
||||
sorted({backend.get_name() for backend in self.attn_backends})
|
||||
)
|
||||
self.compat_hash = compute_nixl_compatibility_hash(
|
||||
self.vllm_config,
|
||||
@@ -1478,31 +1439,6 @@ class NixlBaseConnectorWorker:
|
||||
):
|
||||
raise NotImplementedError("PP push with packed KV caches requires MLA")
|
||||
|
||||
layer_specs: dict[str, KVCacheSpec] = {}
|
||||
compressed_region_owners: dict[int, torch.Tensor] = {}
|
||||
for layer_name, cache in xfer_buffers.items():
|
||||
layer_spec = self._layer_specs.get(layer_name)
|
||||
if isinstance(layer_spec, UniformTypeKVCacheSpecs):
|
||||
layer_spec = layer_spec.kv_cache_specs[layer_name]
|
||||
if layer_spec is None:
|
||||
continue
|
||||
layer_specs[layer_name] = layer_spec
|
||||
physical_page_size = (
|
||||
layer_spec.page_size_bytes
|
||||
if isinstance(layer_spec, MambaSpec)
|
||||
else layer_spec.page_size_bytes
|
||||
// self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
num_blocks = (
|
||||
self._logical_num_blocks
|
||||
if isinstance(layer_spec, MambaSpec)
|
||||
else self.num_blocks
|
||||
)
|
||||
if _uses_dense_virtual_transfer_pages(
|
||||
layer_spec, cache, physical_page_size, num_blocks
|
||||
):
|
||||
compressed_region_owners.setdefault(cache.data_ptr(), cache)
|
||||
|
||||
track_region_layers = self._tracks_region_layers()
|
||||
region_layers: list[list[str]] = []
|
||||
packed_member_layouts: dict[str, tuple[int, int]] = {}
|
||||
@@ -1527,7 +1463,7 @@ class NixlBaseConnectorWorker:
|
||||
# kernel requires a specific block size. This leads to SSM and FA layers
|
||||
# having different num_blocks.
|
||||
# `_physical_blocks_per_logical_kv_block` ratio is used to adjust for this.
|
||||
layer_spec = layer_specs.get(layer_name)
|
||||
layer_spec = self._layer_specs.get(layer_name)
|
||||
if layer_spec is None:
|
||||
logger.debug(
|
||||
"Skipping layer %s as no KVCache spec is present. "
|
||||
@@ -1587,32 +1523,6 @@ class NixlBaseConnectorWorker:
|
||||
region_device_id,
|
||||
)
|
||||
|
||||
if isinstance(layer_spec, KpoolTailSpec):
|
||||
compressed_owner = compressed_region_owners.get(cache.data_ptr())
|
||||
if compressed_owner is not None:
|
||||
owner_storage = compressed_owner.untyped_storage()
|
||||
owner_end = compressed_owner.data_ptr() + compressed_owner.nbytes
|
||||
tail_is_covered = (
|
||||
compressed_owner.is_contiguous()
|
||||
and owner_storage.data_ptr() == storage_addr
|
||||
and _tensor_byte_span_end(cache) <= owner_end
|
||||
)
|
||||
if not tail_is_covered:
|
||||
raise AssertionError(
|
||||
"Kpool tail cache is not fully covered by its compressed "
|
||||
f"indexer region: layer={layer_name}, "
|
||||
f"tail_shape={tuple(cache.shape)}, "
|
||||
f"tail_stride={tuple(cache.stride())}, "
|
||||
f"owner_shape={tuple(compressed_owner.shape)}, "
|
||||
f"owner_stride={tuple(compressed_owner.stride())}"
|
||||
)
|
||||
logger.debug(
|
||||
"Skipping layer %s because its compressed indexer region "
|
||||
"covers the same storage",
|
||||
layer_name,
|
||||
)
|
||||
continue
|
||||
|
||||
if isinstance(layer_spec, MambaSpec):
|
||||
physical_ratio = self._physical_blocks_per_logical_kv_block
|
||||
block_len = physical_page_size // physical_ratio
|
||||
@@ -1647,9 +1557,6 @@ class NixlBaseConnectorWorker:
|
||||
and cache[0].is_contiguous()
|
||||
and cache[0].nbytes == physical_page_size
|
||||
)
|
||||
virtual_transfer_pages = _uses_dense_virtual_transfer_pages(
|
||||
layer_spec, cache, physical_page_size, num_blocks
|
||||
)
|
||||
# Push workers address the pages of packed MLA rows by layer.
|
||||
packed_mla_push = (
|
||||
track_region_layers
|
||||
@@ -1658,12 +1565,7 @@ class NixlBaseConnectorWorker:
|
||||
and storage_is_block_major
|
||||
and is_mla_region
|
||||
)
|
||||
if virtual_transfer_pages:
|
||||
# A compressed kernel row can contain multiple NIXL transfer pages.
|
||||
region_specs = [
|
||||
(cache.data_ptr(), physical_page_size, physical_page_size)
|
||||
]
|
||||
elif packed_mla_push and use_layer_name_routing:
|
||||
if packed_mla_push and use_layer_name_routing:
|
||||
# PP>1 prefill stage: register only this layer's page of each row.
|
||||
region_specs = [
|
||||
(cache.data_ptr(), physical_page_size, block_stride)
|
||||
@@ -1671,7 +1573,7 @@ class NixlBaseConnectorWorker:
|
||||
elif packed_mla_push:
|
||||
# PP=1 decode or prefill: register whole rows, and advertise where
|
||||
# this layer's page sits so a PP>1 prefill can write into it.
|
||||
storage_block_len = registration_len // num_blocks
|
||||
storage_block_len = block_stride
|
||||
region_specs = [
|
||||
(registration_base, storage_block_len, storage_block_len)
|
||||
]
|
||||
@@ -3388,11 +3290,9 @@ class NixlBaseConnectorWorker:
|
||||
physical_block_ids.append(group)
|
||||
else:
|
||||
physical_block_ids.append(
|
||||
BlockTable.map_to_kernel_blocks(
|
||||
np.array(group),
|
||||
ratio,
|
||||
block_arange,
|
||||
).tolist()
|
||||
(np.array(group)[:, None] * ratio + block_arange)
|
||||
.reshape(-1)
|
||||
.tolist()
|
||||
)
|
||||
return physical_block_ids
|
||||
|
||||
|
||||
@@ -1376,14 +1376,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
is_index_group_leader=self.indexer is not None,
|
||||
non_causal_multi_token_decode=self.non_causal_multi_token_decode,
|
||||
)
|
||||
# SM100 FlashMLA paged kernels also express TMA coordinates in token rows.
|
||||
uses_tma_rows = (
|
||||
self.attn_backend.get_name() == "FLASHMLA_SPARSE"
|
||||
and self.kv_cache_dtype in ("fp8_ds_mla", "nvfp4_ds_mla")
|
||||
and current_platform.is_device_capability_family(100)
|
||||
)
|
||||
if self._uses_flat_kv_cache() or uses_tma_rows:
|
||||
spec = replace(spec, block_stride_alignment=spec.state_content_size_bytes)
|
||||
return spec
|
||||
|
||||
def _v_up_proj(self, x: torch.Tensor, out: torch.Tensor):
|
||||
|
||||
@@ -669,7 +669,7 @@ class DeepseekV41ModelState(DefaultModelState):
|
||||
copy.create_metadata_builders(
|
||||
self.vllm_config,
|
||||
self.device,
|
||||
group.metadata_builders[0].kernel_block_size,
|
||||
group.kernel_block_size,
|
||||
)
|
||||
copies.append(copy)
|
||||
own_groups.append(copies)
|
||||
|
||||
@@ -33,9 +33,9 @@ from vllm.model_executor.utils import maybe_disable_graph_partition
|
||||
from vllm.models.glm5next.nvidia.ops.kpool_compress import fwht128_quant_fp8
|
||||
from vllm.models.glm5next.sparse_indexer import SparseAttnIndexerKpool
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES
|
||||
from vllm.utils.math_utils import cdiv, next_power_of_2
|
||||
from vllm.v1.kv_cache_interface import KpoolTailSpec, MLAAttentionSpec
|
||||
from vllm.v1.attention.backends.mla.indexer import Glm5NextIndexerBackend
|
||||
from vllm.v1.kv_cache_interface import CircularBufferSpec, MLAAttentionSpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -89,15 +89,9 @@ class Glm5NextIndexerCache(DeepseekV32IndexerCache):
|
||||
|
||||
The indexer shares one block with the co-located MLA (a single
|
||||
``MLAAttentionSpec`` / block_table), so ``block_size`` is the model-wide
|
||||
``cache_config.block_size``. DeepGEMM's paged-MQA kernel
|
||||
(``csrc/apis/attention.hpp``) requires ``block_kv`` to be exactly 32 or
|
||||
64, so the storage block is virtually split into pool pages of the
|
||||
largest such size that tiles it (``storage_kernel_block_size``); this
|
||||
needs ``block_size`` to be a multiple of ``index_kpool * 32`` (512 for
|
||||
``index_kpool = 16``). A smaller block (e.g. the default 64) silently
|
||||
collapses ``storage_block_size`` (64 // 16 = 4) and only fails later at
|
||||
the opaque C++ assert; ``get_kv_cache_spec`` guards this up front
|
||||
instead.
|
||||
``cache_config.block_size``. DeepGEMM's paged-MQA kernel takes 32- or
|
||||
64-state pages (only 64 on SM120), which ``Glm5NextIndexerBackend``
|
||||
declares as its kernel block sizes.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -113,12 +107,6 @@ class Glm5NextIndexerCache(DeepseekV32IndexerCache):
|
||||
head_dim=head_dim, dtype=dtype, prefix=prefix, cache_config=cache_config
|
||||
)
|
||||
assert index_kpool > 1, "Glm5NextIndexerCache expects index_kpool > 1"
|
||||
# Keep chunked-prefill boundaries aligned to complete pools.
|
||||
assert cache_config.block_size % index_kpool == 0, (
|
||||
"Glm5NextIndexerCache: cache_config.block_size "
|
||||
f"({cache_config.block_size}) must be a multiple of index_kpool "
|
||||
f"({index_kpool}) so chunked-prefill boundaries stay pool-aligned."
|
||||
)
|
||||
self._index_kpool = index_kpool
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig):
|
||||
@@ -128,33 +116,10 @@ class Glm5NextIndexerCache(DeepseekV32IndexerCache):
|
||||
# ``tokens_per_state`` is the KV-spec representation of kpool
|
||||
# compression in the current cache-layout API.
|
||||
assert isinstance(spec, MLAAttentionSpec)
|
||||
spec = replace(spec, tokens_per_state=self._index_kpool)
|
||||
return replace(spec, tokens_per_state=self._index_kpool)
|
||||
|
||||
# DeepGEMM paged-MQA takes block_kv in {32, 64}; the storage block
|
||||
# (= block_size // index_kpool) is virtually split into pool pages of
|
||||
# the largest such size that tiles it, so it must be a multiple of 32.
|
||||
storage_block_size = spec.block_size // self._index_kpool
|
||||
assert (
|
||||
spec.block_size % self._index_kpool == 0 and storage_block_size % 32 == 0
|
||||
), (
|
||||
"Glm5NextIndexerCache: kpool indexer requires cache block_size to "
|
||||
f"be a multiple of index_kpool * 32 ({self._index_kpool * 32}) so "
|
||||
"that DeepGEMM paged-MQA pool pages (32 or 64 entries) tile the "
|
||||
f"storage block, got block_size={spec.block_size} -> "
|
||||
f"storage_block_size={storage_block_size}."
|
||||
)
|
||||
max_page_size = max(PAGED_MQA_PAGE_SIZES)
|
||||
min_page_size = min(PAGED_MQA_PAGE_SIZES)
|
||||
if storage_block_size <= max_page_size:
|
||||
page_size = storage_block_size
|
||||
elif storage_block_size % max_page_size == 0:
|
||||
page_size = max_page_size
|
||||
else:
|
||||
page_size = min_page_size
|
||||
return replace(
|
||||
spec,
|
||||
storage_block_size=page_size * self._index_kpool,
|
||||
)
|
||||
def get_attn_backend(self):
|
||||
return Glm5NextIndexerBackend
|
||||
|
||||
|
||||
class Glm5NextTailCache(DeepseekV32IndexerCache):
|
||||
@@ -164,9 +129,9 @@ class Glm5NextTailCache(DeepseekV32IndexerCache):
|
||||
``ring`` slots per request, overwritten in place by ``pos % ring``
|
||||
as decode/spec-decode advances. Prefill seeds it (instead of discarding the
|
||||
tail raw K+gate); the connector transfers it across PD; decode reads it to
|
||||
compress the boundary pool correctly. ``KpoolTailSpec`` /
|
||||
``KpoolTailManager`` provide the no-prune, 1-block/req allocation that lets
|
||||
the in-progress pool survive across steps and across transfer.
|
||||
compress the boundary pool correctly. ``CircularBufferSpec`` provides the
|
||||
no-prune, 1-block/req allocation that lets the in-progress pool survive
|
||||
across steps and across transfer.
|
||||
|
||||
Stores raw bf16 K (``head_dim``) as the "K" half of each block and the
|
||||
bf16 gate score (``head_dim``) as the "V" half -- not the fp8-compressed
|
||||
@@ -202,13 +167,12 @@ class Glm5NextTailCache(DeepseekV32IndexerCache):
|
||||
f"({self.cache_config.block_size}) must be a multiple of the "
|
||||
f"tail ring ({ring})"
|
||||
)
|
||||
return KpoolTailSpec(
|
||||
return CircularBufferSpec(
|
||||
block_size=ring,
|
||||
num_kv_heads=2,
|
||||
head_size=self.head_dim,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=ring,
|
||||
# Tiny per-request ring; every DCP rank keeps the full tail so the
|
||||
# rank owning a pool can compress it.
|
||||
dcp_sharded=False,
|
||||
@@ -303,7 +267,7 @@ class Indexer(nn.Module):
|
||||
# Paged tail cache (in-progress pool's raw K + gate score). Written by
|
||||
# prefill (seeds the boundary pool) and decode (per-step stash); read by
|
||||
# the decode kernel to compress the boundary pool. Transferred across PD
|
||||
# so the decode side sees the prefill tail. See KpoolTailSpec/Manager.
|
||||
# so the decode side sees the prefill tail. See CircularBufferManager.
|
||||
self.tail_cache = Glm5NextTailCache(
|
||||
head_dim=self.head_dim,
|
||||
dtype=torch.bfloat16,
|
||||
|
||||
@@ -396,11 +396,6 @@ def _kpool_tail_seed_kernel(
|
||||
ahead belongs to a different tail block (or is past the batch / padding,
|
||||
slot < 0). ``tslot = block * RING + pos % RING``; the destination is
|
||||
``tail[block, {0:K, 1:score}, pos % RING, :]``.
|
||||
|
||||
The tail cache aliases the indexer cache with the indexer's (padded) block
|
||||
stride, so blocks are addressed through ``TAIL_BLOCK_ELEMS`` /
|
||||
``KPOOL_HEAD`` (``tail.stride(0)`` / ``tail.stride(1)``), never as a dense
|
||||
``[num_blocks, 2, KPOOL, HEAD_DIM]`` array.
|
||||
"""
|
||||
i = tl.program_id(0)
|
||||
t = tl.load(tslot_ptr + i).to(tl.int64)
|
||||
|
||||
@@ -414,26 +414,6 @@ class CudaPlatformBase(Platform):
|
||||
|
||||
return valid_backends_priorities, invalid_reasons
|
||||
|
||||
@classmethod
|
||||
def _get_indexer_block_alignment(cls, vllm_config: VllmConfig) -> int | None:
|
||||
index_kpool = getattr(
|
||||
vllm_config.model_config.hf_text_config, "index_kpool", None
|
||||
)
|
||||
if not index_kpool or index_kpool <= 1:
|
||||
return None
|
||||
from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES
|
||||
|
||||
# kpool paged-MQA indexer: the storage block (block_size /
|
||||
# index_kpool) is virtually split into pool pages, so block_size
|
||||
# must be a multiple of index_kpool times a legal pool page.
|
||||
page = min(PAGED_MQA_PAGE_SIZES)
|
||||
if cls.is_device_capability_family(120):
|
||||
# On sm120 the DeepGEMM paged-MQA kernel only accepts block_kv
|
||||
# 64 for the fp8 indexer cache, so align to the largest pool
|
||||
# page here to make the page split land on 64 not the min 32.
|
||||
page = max(PAGED_MQA_PAGE_SIZES)
|
||||
return index_kpool * page
|
||||
|
||||
@classmethod
|
||||
def get_attn_backend_cls(
|
||||
cls,
|
||||
|
||||
+18
-21
@@ -737,14 +737,14 @@ class Platform:
|
||||
for candidate in candidates:
|
||||
if all(b.supports_block_size(candidate) for b in backend_classes):
|
||||
return candidate
|
||||
raise ValueError(
|
||||
"The attention backends share no supported KV cache block size ("
|
||||
+ "; ".join(
|
||||
f"{b.get_name()}: {b.get_supported_kernel_block_sizes()}"
|
||||
for b in backend_classes
|
||||
raise ValueError(
|
||||
"The attention backends share no supported KV cache block size ("
|
||||
+ "; ".join(
|
||||
f"{b.get_name()}: {b.get_supported_kernel_block_sizes()}"
|
||||
for b in backend_classes
|
||||
)
|
||||
+ ")."
|
||||
)
|
||||
+ ")."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None:
|
||||
@@ -935,17 +935,6 @@ class Platform:
|
||||
if cache_config.mamba_page_size_padded is not None:
|
||||
cache_config.mamba_page_size_padded = shared_page
|
||||
|
||||
@classmethod
|
||||
def _get_indexer_block_alignment(cls, vllm_config: "VllmConfig") -> int | None:
|
||||
"""Extra ``block_size`` multiple a sparse indexer needs, else ``None``.
|
||||
|
||||
The CUDA kpool paged-MQA indexer virtually splits each storage block
|
||||
into pool pages, so ``block_size`` must be a multiple of
|
||||
``index_kpool * min(PAGED_MQA_PAGE_SIZES)`` — implemented in the CUDA
|
||||
platform override. Other platforms impose no extra constraint.
|
||||
"""
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _align_hybrid_block_size(
|
||||
cls,
|
||||
@@ -1077,6 +1066,14 @@ class Platform:
|
||||
),
|
||||
cache_config.block_size,
|
||||
)
|
||||
# Also a multiple of every backend's smallest kernel block.
|
||||
backend_block_alignment = lcm(
|
||||
*(
|
||||
min(s.base if isinstance(s, MultipleOf) else s for s in sizes)
|
||||
for b in cls._find_non_ssm_backends(vllm_config)
|
||||
if (sizes := b.get_supported_kernel_block_sizes())
|
||||
)
|
||||
)
|
||||
if model_config.use_mla:
|
||||
# TRTLLM/FlashInfer MLA decode kernels require the physical
|
||||
# number of kernel blocks to be aligned to 128 / kernel_block_size.
|
||||
@@ -1090,9 +1087,9 @@ class Platform:
|
||||
mamba_page_size,
|
||||
kernel_block_alignment_size * attn_page_size_1_token,
|
||||
)
|
||||
indexer_align = cls._get_indexer_block_alignment(vllm_config)
|
||||
if indexer_align:
|
||||
attn_block_size = indexer_align * cdiv(attn_block_size, indexer_align)
|
||||
attn_block_size = backend_block_alignment * cdiv(
|
||||
attn_block_size, backend_block_alignment
|
||||
)
|
||||
|
||||
if cache_config.block_size < attn_block_size:
|
||||
cache_config.block_size = attn_block_size
|
||||
|
||||
@@ -29,10 +29,11 @@ _DEEPGEMM_BLACKWELL_EXCLUDED_MODEL_TYPES: set[str] = {
|
||||
"qwen3_5_moe_text",
|
||||
}
|
||||
|
||||
# KV page sizes (in cache entries) supported by the paged-MQA logits kernels
|
||||
# (fp8_fp4_paged_mqa_logits / get_paged_mqa_logits_metadata). Larger storage
|
||||
# blocks must be virtually split into one of these page sizes.
|
||||
PAGED_MQA_PAGE_SIZES = (32, 64)
|
||||
|
||||
def get_paged_mqa_page_sizes() -> tuple[int, ...]:
|
||||
"""KV page sizes (in cache entries) the paged-MQA logits kernels take (only
|
||||
64 on SM120); larger storage blocks are split into one of these."""
|
||||
return (64,) if current_platform.is_device_capability_family(120) else (32, 64)
|
||||
|
||||
|
||||
def should_auto_disable_deep_gemm(model_type: str | None) -> bool:
|
||||
|
||||
@@ -48,12 +48,10 @@ class AttentionType(str, Enum):
|
||||
"""Attention between dec. Q and enc. K/V for encoder-decoder."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MultipleOf:
|
||||
base: int
|
||||
|
||||
def __init__(self, base: int):
|
||||
self.base = base
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"MultipleOf({self.base})"
|
||||
|
||||
@@ -622,10 +620,6 @@ class AttentionMetadataBuilder(ABC, Generic[M]):
|
||||
self.layer_names = layer_names
|
||||
self.vllm_config = vllm_config
|
||||
self.device = device
|
||||
self.kernel_block_size: int | None = None
|
||||
|
||||
def set_kernel_block_size(self, kernel_block_size: int) -> None:
|
||||
self.kernel_block_size = kernel_block_size
|
||||
|
||||
@classmethod
|
||||
def get_cudagraph_support(
|
||||
|
||||
@@ -297,11 +297,6 @@ def create_composite_attention_backend(
|
||||
)
|
||||
self.routing = routing_policy(layer_names, vllm_config)
|
||||
|
||||
def set_kernel_block_size(self, kernel_block_size):
|
||||
super().set_kernel_block_size(kernel_block_size)
|
||||
for builder in self._builders:
|
||||
builder.set_kernel_block_size(kernel_block_size)
|
||||
|
||||
@classmethod
|
||||
def get_cudagraph_support(cls, vllm_config, kv_cache_spec):
|
||||
return min(
|
||||
|
||||
@@ -95,7 +95,9 @@ class CompressedSlotMappingKernel(
|
||||
block_table_ptr + batch_idx * block_table_stride + block_ids,
|
||||
mask=mask & is_valid,
|
||||
)
|
||||
slot_ids = block_numbers * block_size + pos_after_compress % block_size
|
||||
slot_ids = block_numbers.to(tl.int64) * block_size + (
|
||||
pos_after_compress % block_size
|
||||
)
|
||||
|
||||
# NOTE
|
||||
slot_ids = tl.where(is_valid, slot_ids, PAD_ID)
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm.v1.attention.backend import (
|
||||
from vllm.v1.attention.backends.fa_utils import flash_attn_supports_mla
|
||||
from vllm.v1.attention.backends.mla.index_group import HiSparseMLAIndexGroup
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
align_blocks_to_rows,
|
||||
flat_kv_row_view,
|
||||
triton_convert_req_index_to_global_index,
|
||||
)
|
||||
@@ -42,7 +43,11 @@ class FlashAttnMLASparseBackend(AttentionBackend):
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
|
||||
return [64]
|
||||
return [MultipleOf(64)]
|
||||
|
||||
@classmethod
|
||||
def customize_spec(cls, spec: AttentionSpec) -> AttentionSpec:
|
||||
return align_blocks_to_rows(spec)
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""FlashInfer sparse MLA attention backend."""
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
@@ -30,6 +31,7 @@ from vllm.v1.attention.backend import (
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.index_group import HiSparseMLAIndexGroup
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
align_blocks_to_rows,
|
||||
flat_kv_row_view,
|
||||
prepare_sparse_mla_safe_lengths,
|
||||
triton_convert_req_index_to_global_index,
|
||||
@@ -69,6 +71,13 @@ class _FlashInferMLASparseBackendBase(AttentionBackend):
|
||||
def is_sparse(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def customize_spec(cls, spec: AttentionSpec) -> AttentionSpec:
|
||||
# The kernels read the rows as pages of their kernel block size.
|
||||
(page,) = cls.get_supported_kernel_block_sizes(spec)
|
||||
assert isinstance(page, MultipleOf)
|
||||
return align_blocks_to_rows(spec, page.base)
|
||||
|
||||
|
||||
class FlashInferMLASparseTRTLLMBackend(_FlashInferMLASparseBackendBase):
|
||||
"""FlashInfer sparse MLA backend using the TRTLLM-gen launcher."""
|
||||
@@ -84,7 +93,7 @@ class FlashInferMLASparseTRTLLMBackend(_FlashInferMLASparseBackendBase):
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
|
||||
return [32, 64]
|
||||
return [MultipleOf(32)]
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type[MLAAttentionImpl]:
|
||||
@@ -173,7 +182,7 @@ class FlashInferMLASparseSM120Backend(_FlashInferMLASparseBackendBase):
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
|
||||
return [64, 256]
|
||||
return [MultipleOf(64)]
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type[MLAAttentionImpl]:
|
||||
@@ -575,7 +584,7 @@ class FlashInferMLASparseImpl(SparseMLACommonImpl[FlashInferMLASparseMetadata]):
|
||||
assert prefill_lse is not None
|
||||
return output, torch.cat((decode_lse, prefill_lse))
|
||||
|
||||
_, block_stride_rows = flat_kv_row_view(
|
||||
kv_rows, block_stride_rows = flat_kv_row_view(
|
||||
kv_c_and_k_pe_cache, attn_metadata.block_size
|
||||
)
|
||||
|
||||
@@ -602,7 +611,12 @@ class FlashInferMLASparseImpl(SparseMLACommonImpl[FlashInferMLASparseMetadata]):
|
||||
|
||||
return self._run_mqa_kernel(
|
||||
q,
|
||||
kv_c_and_k_pe_cache,
|
||||
# Block sizes and strides have been aligned to 32 rows.
|
||||
kv_rows.view(
|
||||
-1,
|
||||
math.gcd(block_stride_rows, attn_metadata.block_size, 64),
|
||||
kv_rows.shape[-1],
|
||||
),
|
||||
topk_indices_physical,
|
||||
seq_lens,
|
||||
)
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.index_group import HiSparseMLAIndexGroup
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
flat_kv_row_view,
|
||||
triton_convert_req_index_to_global_index,
|
||||
)
|
||||
|
||||
@@ -184,6 +185,9 @@ class FlashInferMLASparseSM120Impl(SparseMLACommonImpl[FlashInferMLASparseMetada
|
||||
output = torch.cat(outputs) if len(outputs) > 1 else outputs[0]
|
||||
return output, None
|
||||
|
||||
kv_rows, block_stride_rows = flat_kv_row_view(
|
||||
kv_c_and_k_pe_cache, attn_metadata.block_size
|
||||
)
|
||||
topk_indices_physical = cast(
|
||||
torch.Tensor,
|
||||
triton_convert_req_index_to_global_index(
|
||||
@@ -191,13 +195,15 @@ class FlashInferMLASparseSM120Impl(SparseMLACommonImpl[FlashInferMLASparseMetada
|
||||
attn_metadata.block_table,
|
||||
topk_indices,
|
||||
BLOCK_SIZE=attn_metadata.block_size,
|
||||
BLOCK_STRIDE_ROWS=block_stride_rows,
|
||||
NUM_TOPK_TOKENS=topk_indices.shape[1],
|
||||
),
|
||||
)
|
||||
return (
|
||||
self._run_mqa_kernel(
|
||||
q,
|
||||
kv_c_and_k_pe_cache,
|
||||
# The SM120 kernels are built for 64-row pages only.
|
||||
kv_rows.view(-1, 64, kv_rows.shape[-1]),
|
||||
topk_indices_physical,
|
||||
),
|
||||
None,
|
||||
|
||||
@@ -26,7 +26,7 @@ needs no D2H sync. Per-step content (top-k slots) is written into the
|
||||
reserved buffers by kernels inside the captured forward, and captured runs
|
||||
read the refreshed plan buffers on replay.
|
||||
|
||||
KV cache format: plain contiguous E4M3 ``[num_blocks, block_size, 512]``
|
||||
KV cache format: E4M3 ``[num_blocks, block_size, 512]`` (any block stride)
|
||||
(uint8 storage) with a per-tensor ``k_scale``; BF16 caches also work. The
|
||||
per-token x 128-channel-group ``ckv_scale_arr`` layout is supported by the
|
||||
kernel but not wired yet (it needs a group-quantizing cache-write op).
|
||||
@@ -57,9 +57,11 @@ from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||
FlashInferMLASparseMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
align_blocks_to_rows,
|
||||
flat_kv_row_view,
|
||||
triton_convert_req_index_to_global_index,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec
|
||||
|
||||
_FP8_KV_DTYPES = ("fp8", "fp8_e4m3")
|
||||
_WORKSPACE_BYTES = 128 * 1024 * 1024
|
||||
@@ -99,7 +101,12 @@ def _pack_topk_indices(
|
||||
end = tl.load(indptr + row + 1)
|
||||
mask = (cols < end - start) & (cols < WIDTH)
|
||||
values = tl.load(slots + row * WIDTH + cols, mask=mask, other=0)
|
||||
tl.store(indices + start + cols, tl.maximum(values, 0), mask=mask)
|
||||
first_slot = tl.load(slots + row * WIDTH)
|
||||
tl.store(
|
||||
indices + start + cols,
|
||||
tl.where(values >= 0, values, tl.maximum(first_slot, 0)),
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -149,6 +156,10 @@ class FlashInferMLASparseSM90Backend(AttentionBackend):
|
||||
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
|
||||
return [MultipleOf(64)]
|
||||
|
||||
@classmethod
|
||||
def customize_spec(cls, spec: AttentionSpec) -> AttentionSpec:
|
||||
return align_blocks_to_rows(spec)
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "FLASHINFER_MLA_SPARSE_SM90"
|
||||
@@ -222,10 +233,6 @@ class FlashInferMLASparseSM90Backend(AttentionBackend):
|
||||
) -> tuple[int, ...]:
|
||||
return (num_blocks, block_size, head_size)
|
||||
|
||||
@classmethod
|
||||
def supported_kv_cache_layouts(cls) -> tuple[KVCacheLayout, ...]:
|
||||
return (KVCacheLayout.LBHNC,)
|
||||
|
||||
|
||||
class _SM90State:
|
||||
"""Builder-owned wrapper, capture-stable buffers, and plan parameters.
|
||||
@@ -602,6 +609,9 @@ class FlashInferMLASparseSM90Impl(SparseMLACommonImpl[FlashInferMLASparseSM90Met
|
||||
|
||||
assert self.topk_indices_buffer is not None
|
||||
topk_indices = self.topk_indices_buffer[:num_tokens]
|
||||
kv_c_and_k_pe_cache, block_stride_rows = flat_kv_row_view(
|
||||
kv_c_and_k_pe_cache, attn_metadata.block_size
|
||||
)
|
||||
# return_valid_counts=True keeps the compacted-prefix layout: valid
|
||||
# entries at [0, valid_count), -1 past it — exactly the prefix the
|
||||
# planned per-row lengths address.
|
||||
@@ -610,6 +620,7 @@ class FlashInferMLASparseSM90Impl(SparseMLACommonImpl[FlashInferMLASparseSM90Met
|
||||
attn_metadata.block_table,
|
||||
topk_indices,
|
||||
BLOCK_SIZE=attn_metadata.block_size,
|
||||
BLOCK_STRIDE_ROWS=block_stride_rows,
|
||||
NUM_TOPK_TOKENS=topk_indices.shape[1],
|
||||
return_valid_counts=True,
|
||||
)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
import numpy as np
|
||||
@@ -36,6 +36,7 @@ from vllm.v1.attention.backend import (
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.index_group import HiSparseMLAIndexGroup
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
align_blocks_to_rows,
|
||||
flat_kv_row_view,
|
||||
request_row_bounds,
|
||||
triton_convert_req_index_to_global_index,
|
||||
@@ -53,7 +54,7 @@ from vllm.v1.attention.ops.flashmla import (
|
||||
flash_mla_with_kvcache,
|
||||
get_mla_metadata,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, KVQuantMode
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -132,6 +133,17 @@ QUANTIZED_DS_MLA_CACHE_FORMATS: frozenset[str] = frozenset(
|
||||
FP8_DS_MLA_ROPE_DIM = 64
|
||||
|
||||
|
||||
def _fp8_kv_pages(
|
||||
kv_cache: torch.Tensor, block_size: int
|
||||
) -> tuple[torch.Tensor, int | None]:
|
||||
"""The cache as the fp8 kernel's 64-token pages, and the block stride in
|
||||
rows when a block spans several pages."""
|
||||
if block_size == 64:
|
||||
return kv_cache, None
|
||||
rows, block_stride_rows = flat_kv_row_view(kv_cache, block_size)
|
||||
return rows.view(-1, 64, rows.shape[-1]), block_stride_rows
|
||||
|
||||
|
||||
class FlashMLASparseBackend(AttentionBackend):
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
||||
@@ -144,7 +156,23 @@ class FlashMLASparseBackend(AttentionBackend):
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
|
||||
return [64]
|
||||
if (
|
||||
isinstance(kv_cache_spec, AttentionSpec)
|
||||
and kv_cache_spec.kv_quant_mode != KVQuantMode.NONE
|
||||
and current_platform.is_device_capability_family(100)
|
||||
):
|
||||
return [64]
|
||||
return [MultipleOf(64)]
|
||||
|
||||
@classmethod
|
||||
def customize_spec(cls, spec: AttentionSpec) -> AttentionSpec:
|
||||
# Off SM100 the quantized kernel reads 64-token pages, only needed
|
||||
# when blocks span several.
|
||||
if spec.kv_quant_mode != KVQuantMode.NONE and (
|
||||
not current_platform.is_device_capability_family(100)
|
||||
):
|
||||
return replace(spec, block_stride_alignment=MultipleOf(64))
|
||||
return align_blocks_to_rows(spec)
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
@@ -998,13 +1026,16 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
# and the prefill rows per chunk, against the gathered workspace.
|
||||
lse: torch.Tensor | None = None
|
||||
topk_length: torch.Tensor | None = None
|
||||
kv_pages, block_stride_rows = _fp8_kv_pages(
|
||||
kv_c_and_k_pe_cache, attn_metadata.block_size
|
||||
)
|
||||
if self.pcp_dcp_kv_gather:
|
||||
pass
|
||||
elif num_prefill_tokens == 0 and not uses_host_cache:
|
||||
topk_indices, topk_length = self._convert_logical_to_physical_topk(
|
||||
topk_indices,
|
||||
attn_metadata,
|
||||
block_stride_rows=None,
|
||||
block_stride_rows=block_stride_rows,
|
||||
return_valid_counts=True,
|
||||
)
|
||||
elif num_prefill_tokens > 0:
|
||||
@@ -1013,6 +1044,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
attn_metadata.block_table,
|
||||
topk_indices,
|
||||
BLOCK_SIZE=attn_metadata.block_size,
|
||||
BLOCK_STRIDE_ROWS=block_stride_rows,
|
||||
NUM_TOPK_TOKENS=topk_indices.shape[1],
|
||||
HAS_PREFILL_WORKSPACE=has_prefill_workspace,
|
||||
prefill_workspace_request_ids=prefill_request_ids,
|
||||
@@ -1056,7 +1088,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
topk_indices = topk_indices.view(num_decodes, seq_len, -1)
|
||||
attn_out, _ = self._fp8_flash_mla_kernel(
|
||||
q=q,
|
||||
kv_c_and_k_pe_cache=kv_c_and_k_pe_cache,
|
||||
kv_c_and_k_pe_cache=kv_pages,
|
||||
topk_indices=topk_indices,
|
||||
kernel_metadata=fp8_metadata.decode.kernel_metadata,
|
||||
)
|
||||
@@ -1173,6 +1205,9 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
# req_id_per_token covers the whole batch; slice it to the MQA tokens
|
||||
# (q may exclude prefill tokens routed to dense MHA).
|
||||
req_id_per_token = attn_metadata.req_id_per_token[: topk_indices.shape[0]]
|
||||
kv_pages, block_stride_rows = _fp8_kv_pages(
|
||||
kv_c_and_k_pe_cache, attn_metadata.block_size
|
||||
)
|
||||
if self.dcp_world_size > 1:
|
||||
# The indexer emits global token ids; keep this rank's shard and
|
||||
# convert to local slots. compact_valid_to_front=False keeps the
|
||||
@@ -1187,6 +1222,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
dcp_rank=self.dcp_rank,
|
||||
cp_kv_cache_interleave_size=attn_metadata.cp_kv_cache_interleave_size,
|
||||
BLOCK_SIZE=attn_metadata.block_size,
|
||||
BLOCK_STRIDE_ROWS=block_stride_rows,
|
||||
NUM_TOPK_TOKENS=topk_indices.shape[1],
|
||||
compact_valid_to_front=False,
|
||||
)
|
||||
@@ -1202,7 +1238,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
topk_indices = self._convert_logical_to_physical_topk(
|
||||
topk_indices,
|
||||
attn_metadata,
|
||||
block_stride_rows=None,
|
||||
block_stride_rows=block_stride_rows,
|
||||
return_valid_counts=False,
|
||||
)
|
||||
else:
|
||||
@@ -1211,12 +1247,13 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
block_table,
|
||||
topk_indices,
|
||||
BLOCK_SIZE=attn_metadata.block_size,
|
||||
BLOCK_STRIDE_ROWS=block_stride_rows,
|
||||
NUM_TOPK_TOKENS=topk_indices.shape[1],
|
||||
)
|
||||
|
||||
_attn_out, _lse = self._fp8_flash_mla_kernel(
|
||||
q=q.unsqueeze(0), # unsqueeze to add batch_dim: (T, H, D) -> (1, T, H, D)
|
||||
kv_c_and_k_pe_cache=kv_c_and_k_pe_cache,
|
||||
kv_c_and_k_pe_cache=kv_pages,
|
||||
topk_indices=topk_indices.unsqueeze(0), # (T, topk) -> (1, T, topk)
|
||||
kernel_metadata=fp8_metadata,
|
||||
)
|
||||
|
||||
@@ -7,7 +7,7 @@ import numpy as np
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.distributed import get_dcp_group, get_pcp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.warmup.jit_warmup import kernel_launcher, zip_inputs
|
||||
@@ -22,6 +22,7 @@ from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.deep_gemm import (
|
||||
get_paged_mqa_logits_metadata,
|
||||
get_paged_mqa_page_sizes,
|
||||
has_deep_gemm,
|
||||
native_next_n_supported,
|
||||
)
|
||||
@@ -234,12 +235,25 @@ class DeepseekV32IndexerBackend(AttentionBackend):
|
||||
return DeepseekV32IndexerMetadataBuilder
|
||||
|
||||
|
||||
class Glm5NextIndexerBackend(DeepseekV32IndexerBackend):
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
|
||||
if kv_cache_spec is not None:
|
||||
# Worker get_kv_cache_spec: runs without the current config.
|
||||
index_kpool = int(kv_cache_spec.tokens_per_state)
|
||||
else:
|
||||
# Platform block-size selection: no spec yet, under the current config.
|
||||
model_config = get_current_vllm_config().model_config
|
||||
index_kpool = model_config.hf_text_config.index_kpool
|
||||
return [index_kpool * n for n in get_paged_mqa_page_sizes()]
|
||||
|
||||
|
||||
class KpoolTailBackend(DeepseekV32IndexerBackend):
|
||||
"""Storage-only backend for the GLM-5.3-Flash kpool tail cache."""
|
||||
|
||||
@classmethod
|
||||
def supported_kv_cache_layouts(cls) -> tuple[KVCacheLayout, ...]:
|
||||
return (KVCacheLayout.LBHNC,)
|
||||
return (KVCacheLayout.BLHNC,)
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
@@ -1065,8 +1079,6 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
self.indexer_decode_block_table_buffer: torch.Tensor | None = None
|
||||
self._max_num_batched_tokens = scheduler_config.max_num_batched_tokens
|
||||
# DCP not supported yet
|
||||
self.supports_draft_decode_metadata_update = self.dcp_world_size == 1
|
||||
|
||||
@@ -1353,16 +1365,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
dcp_local_seq_lens = common_attn_metadata.dcp_local_seq_lens
|
||||
|
||||
compressed_slot_mapping = slot_mapping
|
||||
indexer_block_table = block_table
|
||||
if self.compress_ratio > 1:
|
||||
kernel_block_size = self.kernel_block_size
|
||||
if (
|
||||
kernel_block_size is not None
|
||||
and self.kv_cache_spec.block_size != kernel_block_size
|
||||
and self.kv_cache_spec.block_size % kernel_block_size == 0
|
||||
):
|
||||
factor = self.kv_cache_spec.block_size // kernel_block_size
|
||||
indexer_block_table = (block_table[:, ::factor] // factor).contiguous()
|
||||
padded_num_tokens = num_tokens
|
||||
local_slot_mapping = slot_mapping
|
||||
if self.use_pcp:
|
||||
@@ -1378,7 +1381,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
local_slot_mapping,
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
indexer_block_table,
|
||||
block_table,
|
||||
self.kv_cache_spec.num_states,
|
||||
self.compress_ratio,
|
||||
out=self.compressed_slot_mapping_buffer,
|
||||
@@ -1497,7 +1500,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
seq_lens,
|
||||
compressed_seq_lens,
|
||||
compressed_seq_lens_cpu,
|
||||
indexer_block_table,
|
||||
block_table,
|
||||
self.compress_ratio,
|
||||
query_slice=query_slice,
|
||||
skip_kv_gather=query_slice.start > 0,
|
||||
@@ -1632,27 +1635,6 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
)
|
||||
)
|
||||
|
||||
if self.compress_ratio > 1:
|
||||
kernel_block_size = self.kernel_block_size
|
||||
if (
|
||||
kernel_block_size is not None
|
||||
and self.kv_cache_spec.block_size != kernel_block_size
|
||||
and self.kv_cache_spec.block_size % kernel_block_size == 0
|
||||
):
|
||||
factor = self.kv_cache_spec.block_size // kernel_block_size
|
||||
compressed = block_table[:, ::factor] // factor
|
||||
rows, cols = compressed.shape
|
||||
if self.indexer_decode_block_table_buffer is None:
|
||||
self.indexer_decode_block_table_buffer = torch.zeros(
|
||||
(self._max_num_batched_tokens, cols),
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
self.indexer_decode_block_table_buffer[:rows, :cols].copy_(
|
||||
compressed
|
||||
)
|
||||
block_table = self.indexer_decode_block_table_buffer[:rows, :cols]
|
||||
|
||||
# Flattening always returns a buffer view, including single-token
|
||||
# batches. Keep its address stable across varlen graph replays.
|
||||
seq_lens_is_buffer_view = not use_native or next_n > 1
|
||||
@@ -1741,17 +1723,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
# One query token per request, so the per-token decode block table is
|
||||
# the request block table (at the indexer kernel's page size). build()
|
||||
# may copy it into a builder buffer, so re-derive it for this batch.
|
||||
block_table = metadata.block_table[: metadata.num_decode_tokens]
|
||||
kernel_block_size = self.kernel_block_size
|
||||
if (
|
||||
self.compress_ratio > 1
|
||||
and kernel_block_size is not None
|
||||
and self.kv_cache_spec.block_size != kernel_block_size
|
||||
and self.kv_cache_spec.block_size % kernel_block_size == 0
|
||||
):
|
||||
factor = self.kv_cache_spec.block_size // kernel_block_size
|
||||
block_table = block_table[:, ::factor] // factor
|
||||
decode.block_table.copy_(block_table)
|
||||
decode.block_table.copy_(metadata.block_table[: metadata.num_decode_tokens])
|
||||
if decode.indices is not None:
|
||||
decode.indices.copy_(self.arange_buffer[: metadata.num_decode_tokens])
|
||||
if decode.per_req_decode_lens is not None:
|
||||
|
||||
@@ -35,6 +35,10 @@ from vllm.v1.attention.backend import (
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
|
||||
AiterMLAHelper,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
align_blocks_to_rows,
|
||||
flat_kv_row_view,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
rocm_sparse_attn_decode_bf16,
|
||||
@@ -160,6 +164,7 @@ def _convert_req_index_to_global_index_kernel(
|
||||
# shapes (compile-time where possible)
|
||||
max_num_blocks_per_req: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
BLOCK_STRIDE_ROWS: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr, # tile width along columns
|
||||
# strides (in elements)
|
||||
bt_stride0,
|
||||
@@ -201,9 +206,8 @@ def _convert_req_index_to_global_index_kernel(
|
||||
bt_ptr = block_table_ptr + req * bt_stride0 + block_id * bt_stride1
|
||||
base = tl.load(bt_ptr, mask=valid_block, other=0)
|
||||
|
||||
# # If token == -1 OR block_id OOB, output 0; else base * BLOCK_SIZE + offset
|
||||
out_val = tl.where(
|
||||
is_invalid_tok | (~valid_block), 0, base * BLOCK_SIZE + inblock_off
|
||||
is_invalid_tok | (~valid_block), 0, base * BLOCK_STRIDE_ROWS + inblock_off
|
||||
)
|
||||
out_ptr_ij = out_ptr + seq_start + indice_id
|
||||
out_ptr_ij_mask = (seq_start + indice_id) < seq_end
|
||||
@@ -218,13 +222,14 @@ def triton_convert_req_index_to_global_index(
|
||||
token_indices: torch.Tensor, # int32 [num_tokens, NUM_TOPK_TOKENS]
|
||||
cu_seqlens: torch.Tensor, # int32 [num_tokens + 1]
|
||||
paged_kv_indices: torch.Tensor, # int32 [num_tokens * topk] out_buffer
|
||||
BLOCK_STRIDE_ROWS: int,
|
||||
BLOCK_SIZE: int = 64,
|
||||
NUM_TOPK_TOKENS: int = 2048,
|
||||
BLOCK_N: int = 128, # tile width along columns
|
||||
):
|
||||
"""out[token_id, indice_id] =
|
||||
block_table[req_id[token_id],
|
||||
token_indices[token_id, indice_id] // BLOCK_SIZE] * BLOCK_SIZE
|
||||
token_indices[token_id, indice_id] // BLOCK_SIZE] * BLOCK_STRIDE_ROWS
|
||||
+ token_indices[token_id, indice_id] % BLOCK_SIZE
|
||||
|
||||
Only when token_indices[token_id, indice_id] == -1 do we output -1.
|
||||
@@ -264,6 +269,7 @@ def triton_convert_req_index_to_global_index(
|
||||
# shapes / constexprs
|
||||
max_num_blocks_per_req,
|
||||
BLOCK_SIZE,
|
||||
BLOCK_STRIDE_ROWS,
|
||||
BLOCK_N,
|
||||
# strides
|
||||
bt_stride0,
|
||||
@@ -345,6 +351,10 @@ class ROCMAiterMLASparseBackend(AttentionBackend):
|
||||
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
|
||||
return [1, MultipleOf(16)]
|
||||
|
||||
@classmethod
|
||||
def customize_spec(cls, spec: AttentionSpec) -> AttentionSpec:
|
||||
return align_blocks_to_rows(spec)
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "ROCM_AITER_MLA_SPARSE"
|
||||
@@ -371,8 +381,7 @@ class ROCMAiterMLASparseBackend(AttentionBackend):
|
||||
|
||||
@classmethod
|
||||
def supported_kv_cache_layouts(cls) -> tuple[KVCacheLayout, ...]:
|
||||
# Global index conversion assumes contiguous pages within each layer.
|
||||
return (KVCacheLayout.LBNHC, KVCacheLayout.LBHNC)
|
||||
return (KVCacheLayout.LBNHC, KVCacheLayout.LBHNC, KVCacheLayout.BLHNC)
|
||||
|
||||
@classmethod
|
||||
def supports_sink(cls) -> bool:
|
||||
@@ -1155,6 +1164,9 @@ class ROCMAiterMLASparseImpl(
|
||||
self.topk_indices_buffer[:num_actual_toks], attn_metadata.topk_tokens
|
||||
)
|
||||
|
||||
kv_c_and_k_pe_cache, block_stride_rows = flat_kv_row_view(
|
||||
kv_c_and_k_pe_cache, attn_metadata.block_size
|
||||
)
|
||||
triton_convert_req_index_to_global_index(
|
||||
attn_metadata.req_id_per_token,
|
||||
attn_metadata.block_table,
|
||||
@@ -1162,6 +1174,7 @@ class ROCMAiterMLASparseImpl(
|
||||
attn_metadata.paged_kv_indptr,
|
||||
attn_metadata.paged_kv_indices,
|
||||
BLOCK_SIZE=attn_metadata.block_size,
|
||||
BLOCK_STRIDE_ROWS=block_stride_rows,
|
||||
NUM_TOPK_TOKENS=attn_metadata.topk_tokens,
|
||||
)
|
||||
|
||||
@@ -1179,5 +1192,5 @@ class ROCMAiterMLASparseImpl(
|
||||
return output, None
|
||||
mla_padded_q = AiterMLAHelper.get_mla_padded_q(self.num_heads, q)
|
||||
return self._forward_mla(
|
||||
layer, mla_padded_q, kv_c_and_k_pe_cache, attn_metadata
|
||||
layer, mla_padded_q, kv_c_and_k_pe_cache.unsqueeze(1), attn_metadata
|
||||
)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Utility functions for sparse MLA backends."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
@@ -22,6 +22,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec
|
||||
from vllm.v1.worker.block_table import get_block_table_width
|
||||
|
||||
|
||||
@@ -40,6 +41,12 @@ def request_row_bounds(req_idx: np.ndarray) -> np.ndarray:
|
||||
return bounds
|
||||
|
||||
|
||||
def align_blocks_to_rows(spec: AttentionSpec, rows: int = 1) -> AttentionSpec:
|
||||
"""Sparse kernels read the cache as flat rows, viewed as ``rows``-row pages,
|
||||
so blocks must be whole pages apart."""
|
||||
return replace(spec, block_stride_alignment=rows * spec.state_content_size_bytes)
|
||||
|
||||
|
||||
def flat_kv_row_view(
|
||||
kv_cache: torch.Tensor, # [num_blocks, block_size, head_dim]
|
||||
block_size: int,
|
||||
|
||||
@@ -27,6 +27,7 @@ from vllm.v1.attention.backend import (
|
||||
MLAAttentionImpl,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
align_blocks_to_rows,
|
||||
flat_kv_row_view,
|
||||
triton_convert_req_index_to_global_index,
|
||||
)
|
||||
@@ -50,6 +51,10 @@ class XPUMLASparseBackend(AttentionBackend):
|
||||
def get_name() -> str:
|
||||
return "XPU_MLA_SPARSE"
|
||||
|
||||
@classmethod
|
||||
def customize_spec(cls, spec: AttentionSpec) -> AttentionSpec:
|
||||
return align_blocks_to_rows(spec)
|
||||
|
||||
@staticmethod
|
||||
def get_metadata_cls() -> type["XPUMLASparseMetadata"]:
|
||||
return XPUMLASparseMetadata
|
||||
|
||||
@@ -661,8 +661,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
self.hash_block_size = hash_block_size
|
||||
self.dcp_world_size = dcp_world_size
|
||||
# Only groups that participate in prefix caching must satisfy the
|
||||
# divisibility constraint; groups that opt out (e.g. GLM-5.3-Flash kpool
|
||||
# tail, block_size=kpool) are scratch buffers and excluded.
|
||||
# divisibility constraint; groups that opt out (e.g. circular buffers
|
||||
# such as the GLM-5.3-Flash indexer tail) are scratch and excluded.
|
||||
cacheable_block_sizes = [
|
||||
manager.block_size
|
||||
for manager, group in zip(
|
||||
@@ -724,8 +724,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
"""
|
||||
self.attention_groups: list[SpecGroup] = []
|
||||
for i, g in enumerate(self.kv_cache_config.kv_cache_groups):
|
||||
# Skip groups that opt out of prefix caching (e.g. GLM-5.3-Flash
|
||||
# kpool tail): their blocks are per-request scratch, never
|
||||
# Skip groups that opt out of prefix caching (e.g. circular
|
||||
# buffers): their blocks are per-request scratch, never
|
||||
# shareable, so they must not participate in hit lookup (their
|
||||
# manager-level hooks already no-op). Their slot in the per-group
|
||||
# hit tuple stays empty.
|
||||
|
||||
+41
-313
@@ -10,7 +10,7 @@ from collections import defaultdict
|
||||
from collections.abc import Callable, Iterable, Iterator, Sequence
|
||||
from dataclasses import dataclass, field, replace
|
||||
from functools import partial, reduce
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, NewType, TypeAlias, cast, overload
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, NewType, TypeAlias, overload
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import VllmConfig
|
||||
@@ -32,7 +32,6 @@ from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
HiddenStateCacheSpec,
|
||||
HiSparseHotSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheLayout,
|
||||
@@ -1126,7 +1125,7 @@ def check_enough_kv_cache_memory(
|
||||
# of the specs since grouping may unify them in-place.
|
||||
groups = get_kv_cache_groups(vllm_config, dict(kv_cache_spec))
|
||||
check_memory = (
|
||||
available_memory - _pool_bytes_per_block(groups)
|
||||
available_memory - _pool_bytes_per_block(vllm_config, groups)
|
||||
if groups
|
||||
else available_memory
|
||||
)
|
||||
@@ -1237,14 +1236,17 @@ def may_override_num_blocks(vllm_config: VllmConfig, num_blocks: int) -> int:
|
||||
return num_blocks
|
||||
|
||||
|
||||
def _pool_bytes_per_block(kv_cache_groups: list[KVCacheGroupSpec]) -> int:
|
||||
def _pool_bytes_per_block(
|
||||
vllm_config: VllmConfig, kv_cache_groups: list[KVCacheGroupSpec]
|
||||
) -> int:
|
||||
"""Bytes consumed by one block in the worker's shared KV cache pool, mirroring
|
||||
the divisor used by `get_kv_cache_config_from_groups` to convert
|
||||
`available_memory` into `num_blocks`. Used to compute the effective KV cache
|
||||
capacity once `num_gpu_blocks_override` is applied.
|
||||
"""
|
||||
return _get_kv_cache_bytes_per_block(
|
||||
[group for group in kv_cache_groups if not group.host_resident]
|
||||
[group for group in kv_cache_groups if not group.host_resident],
|
||||
vllm_config.cache_config.get_resolved_kv_cache_layout(),
|
||||
)
|
||||
|
||||
|
||||
@@ -1287,205 +1289,6 @@ def _get_kv_cache_groups_uniform_type(
|
||||
return [KVCacheGroupSpec(list(spec.kv_cache_specs.keys()), spec)]
|
||||
|
||||
|
||||
def _pp_balanced_mamba_group_count(
|
||||
vllm_config: VllmConfig,
|
||||
mamba_layer_names: list[str],
|
||||
mla_layer_names: list[str],
|
||||
) -> int | None:
|
||||
"""Return a Mamba group count whose PP projections fit the MLA slots."""
|
||||
num_groups = cdiv(len(mamba_layer_names), len(mla_layer_names))
|
||||
pp_size = vllm_config.parallel_config.pipeline_parallel_size
|
||||
if pp_size == 1:
|
||||
return num_groups
|
||||
|
||||
from vllm.distributed.utils import get_pp_indices
|
||||
from vllm.model_executor.models.utils import extract_layer_index
|
||||
|
||||
total_layers = vllm_config.model_config.get_total_num_hidden_layers()
|
||||
mamba_indices = [extract_layer_index(name) for name in mamba_layer_names]
|
||||
mla_indices = [extract_layer_index(name) for name in mla_layer_names]
|
||||
for rank in range(pp_size):
|
||||
start, end = get_pp_indices(total_layers, rank, pp_size)
|
||||
num_mamba = sum(start <= index < end for index in mamba_indices)
|
||||
num_mla = sum(start <= index < end for index in mla_indices)
|
||||
if not num_mamba:
|
||||
continue
|
||||
if not num_mla:
|
||||
return None
|
||||
num_groups = max(num_groups, cdiv(num_mamba, num_mla))
|
||||
return num_groups
|
||||
|
||||
|
||||
def _get_kv_cache_groups_glm5_next(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
) -> list[KVCacheGroupSpec] | None:
|
||||
"""Build GLM-5.3-Flash groups with Mamba/MLA and tail/indexer aliasing."""
|
||||
mamba_specs = {
|
||||
name: spec
|
||||
for name, spec in kv_cache_spec.items()
|
||||
if isinstance(spec, MambaSpec)
|
||||
}
|
||||
tail_specs = {
|
||||
name: spec
|
||||
for name, spec in kv_cache_spec.items()
|
||||
if isinstance(spec, KpoolTailSpec)
|
||||
}
|
||||
attn_specs = {
|
||||
name: spec
|
||||
for name, spec in kv_cache_spec.items()
|
||||
if not isinstance(spec, (MambaSpec, KpoolTailSpec))
|
||||
}
|
||||
if not mamba_specs or not all(
|
||||
type(spec) is MLAAttentionSpec for spec in attn_specs.values()
|
||||
):
|
||||
return None
|
||||
|
||||
mla_specs = cast(dict[str, MLAAttentionSpec], attn_specs)
|
||||
idx_pages = {
|
||||
spec.page_size_bytes for spec in mla_specs.values() if spec.tokens_per_state > 1
|
||||
}
|
||||
if not idx_pages:
|
||||
return None
|
||||
|
||||
assert all(spec.page_size_padded is None for spec in mla_specs.values())
|
||||
assert len(idx_pages) == 1
|
||||
mla_names = [name for name, spec in mla_specs.items() if spec.tokens_per_state == 1]
|
||||
mla_pages = {mla_specs[name].page_size_bytes for name in mla_names}
|
||||
assert len(mla_pages) == 1
|
||||
mla_page = mla_pages.pop()
|
||||
uniform_spec = UniformTypeKVCacheSpecs.from_specs(attn_specs)
|
||||
assert uniform_spec is not None
|
||||
|
||||
tail_group: KVCacheGroupSpec | None = None
|
||||
if tail_specs:
|
||||
idx_page = next(iter(idx_pages))
|
||||
padded_tail_specs: dict[str, KVCacheSpec] = {
|
||||
name: replace(spec, page_size_padded=idx_page)
|
||||
for name, spec in tail_specs.items()
|
||||
}
|
||||
tail_uniform = UniformTypeKVCacheSpecs.from_specs(padded_tail_specs)
|
||||
assert tail_uniform is not None
|
||||
tail_group = KVCacheGroupSpec(list(padded_tail_specs), tail_uniform)
|
||||
|
||||
any_mamba = next(iter(mamba_specs.values()))
|
||||
assert all(spec == any_mamba for spec in mamba_specs.values())
|
||||
if any_mamba.real_page_size_bytes > mla_page:
|
||||
raise ValueError(
|
||||
f"the mamba state page ({any_mamba.real_page_size_bytes} bytes) "
|
||||
f"does not fit the MLA page ({mla_page} bytes); increase tensor "
|
||||
"parallelism or use a wider KV cache dtype"
|
||||
)
|
||||
padded_specs: dict[str, KVCacheSpec] = {
|
||||
name: replace(any_mamba, page_size_padded=mla_page) for name in mamba_specs
|
||||
}
|
||||
num_groups = _pp_balanced_mamba_group_count(
|
||||
vllm_config, list(mamba_specs), mla_names
|
||||
)
|
||||
if num_groups is None:
|
||||
raise ValueError(
|
||||
"a pipeline stage has mamba layers but no MLA layer to share "
|
||||
"slots with; realign the stage boundaries (VLLM_PP_LAYER_PARTITION)"
|
||||
)
|
||||
mamba_grouped_names: list[list[str]] = [[] for _ in range(num_groups)]
|
||||
for index, name in enumerate(mamba_specs):
|
||||
mamba_grouped_names[index % num_groups].append(name)
|
||||
|
||||
return (
|
||||
[KVCacheGroupSpec(list(attn_specs), uniform_spec)]
|
||||
+ ([tail_group] if tail_group is not None else [])
|
||||
+ create_kv_cache_group_specs(padded_specs, mamba_grouped_names)
|
||||
)
|
||||
|
||||
|
||||
def _glm5_next_tensor_layout(
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
) -> (
|
||||
tuple[
|
||||
KVCacheGroupSpec,
|
||||
list[KVCacheGroupSpec],
|
||||
list[str],
|
||||
list[str],
|
||||
int,
|
||||
int,
|
||||
list[str],
|
||||
int,
|
||||
]
|
||||
| None
|
||||
):
|
||||
"""Recognize the GLM-5.3-Flash grouping after optional PP projection."""
|
||||
uniform_groups = [
|
||||
group
|
||||
for group in kv_cache_groups
|
||||
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
]
|
||||
mamba_groups = [
|
||||
group for group in kv_cache_groups if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
]
|
||||
attn_group: KVCacheGroupSpec | None = None
|
||||
tail_group: KVCacheGroupSpec | None = None
|
||||
for group in uniform_groups:
|
||||
inner = cast(UniformTypeKVCacheSpecs, group.kv_cache_spec).kv_cache_specs
|
||||
if all(type(spec) is MLAAttentionSpec for spec in inner.values()):
|
||||
attn_group = group
|
||||
elif all(isinstance(spec, KpoolTailSpec) for spec in inner.values()):
|
||||
tail_group = group
|
||||
if attn_group is None or not mamba_groups:
|
||||
return None
|
||||
if len(uniform_groups) + len(mamba_groups) != len(kv_cache_groups):
|
||||
return None
|
||||
|
||||
attn_uniform = cast(UniformTypeKVCacheSpecs, attn_group.kv_cache_spec)
|
||||
mla_inner = cast(dict[str, MLAAttentionSpec], attn_uniform.kv_cache_specs)
|
||||
if not all(
|
||||
type(spec) is MLAAttentionSpec and spec.page_size_padded is None
|
||||
for spec in mla_inner.values()
|
||||
):
|
||||
return None
|
||||
mla_names = [
|
||||
name for name in attn_group.layer_names if mla_inner[name].tokens_per_state == 1
|
||||
]
|
||||
idx_names = [
|
||||
name for name in attn_group.layer_names if mla_inner[name].tokens_per_state > 1
|
||||
]
|
||||
mla_pages = {mla_inner[name].page_size_bytes for name in mla_names}
|
||||
idx_pages = {mla_inner[name].page_size_bytes for name in idx_names}
|
||||
if len(mla_pages) != 1 or len(idx_pages) != 1:
|
||||
return None
|
||||
mla_page = mla_pages.pop()
|
||||
idx_page = idx_pages.pop()
|
||||
if any(group.kv_cache_spec.page_size_bytes != mla_page for group in mamba_groups):
|
||||
return None
|
||||
|
||||
tail_names: list[str] = []
|
||||
tail_page = 0
|
||||
if tail_group is not None:
|
||||
tail_names = list(tail_group.layer_names)
|
||||
tail_inner = cast(
|
||||
UniformTypeKVCacheSpecs, tail_group.kv_cache_spec
|
||||
).kv_cache_specs
|
||||
tail_pages = {
|
||||
cast(KpoolTailSpec, spec).unpadded_page_size_bytes
|
||||
for spec in tail_inner.values()
|
||||
}
|
||||
if len(tail_pages) != 1 or len(tail_names) != len(idx_names):
|
||||
return None
|
||||
tail_page = tail_pages.pop()
|
||||
if tail_page > idx_page:
|
||||
return None
|
||||
|
||||
return (
|
||||
attn_group,
|
||||
mamba_groups,
|
||||
mla_names,
|
||||
idx_names,
|
||||
mla_page,
|
||||
idx_page,
|
||||
tail_names,
|
||||
tail_page,
|
||||
)
|
||||
|
||||
|
||||
def unify_kv_cache_spec_page_size(
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
) -> dict[str, KVCacheSpec]:
|
||||
@@ -1709,14 +1512,24 @@ def _get_per_layer_spec(
|
||||
return spec
|
||||
|
||||
|
||||
def _get_layer_block_stride(spec: KVCacheSpec) -> int:
|
||||
"""A layer-outer block stride: the page padded to its alignment."""
|
||||
return round_up(spec.page_size_bytes, spec.get_block_stride_alignment())
|
||||
|
||||
|
||||
def _get_kv_cache_bytes_per_block(
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
kv_cache_groups: list[KVCacheGroupSpec], layout: KVCacheLayout
|
||||
) -> int:
|
||||
"""Return the largest cache group's bytes per block."""
|
||||
if (glm5_layout := _glm5_next_tensor_layout(kv_cache_groups)) is not None:
|
||||
_, _, mla_names, idx_names, mla_page, idx_page, _, _ = glm5_layout
|
||||
return len(mla_names) * mla_page + len(idx_names) * idx_page
|
||||
|
||||
if not layout.is_block_outermost:
|
||||
# Each layer's block stride is its own padded page.
|
||||
return max(
|
||||
sum(
|
||||
_get_layer_block_stride(_get_per_layer_spec(group, layer_name))
|
||||
for layer_name in group.layer_names
|
||||
)
|
||||
for group in kv_cache_groups
|
||||
)
|
||||
bytes_per_block = max(
|
||||
sum(
|
||||
_get_per_layer_spec(group, layer_name).page_size_bytes
|
||||
@@ -1731,7 +1544,7 @@ def _get_kv_cache_bytes_per_block(
|
||||
if isinstance(group.kv_cache_spec, HiSparseHotSpec)
|
||||
]
|
||||
alignments.extend(
|
||||
_get_per_layer_spec(group, layer_name).block_stride_alignment or 1
|
||||
_get_per_layer_spec(group, layer_name).get_block_stride_alignment()
|
||||
for group in kv_cache_groups
|
||||
for layer_name in group.layer_names
|
||||
)
|
||||
@@ -1806,72 +1619,9 @@ def get_kv_cache_config_from_groups(
|
||||
vllm_config, kv_cache_groups, available_memory, host_budget
|
||||
)
|
||||
|
||||
if (glm5_layout := _glm5_next_tensor_layout(kv_cache_groups)) is not None:
|
||||
(
|
||||
attn_group,
|
||||
mamba_groups,
|
||||
mla_names,
|
||||
idx_names,
|
||||
mla_page,
|
||||
idx_page,
|
||||
tail_names,
|
||||
_,
|
||||
) = glm5_layout
|
||||
bytes_per_block = len(mla_names) * mla_page + len(idx_names) * idx_page
|
||||
num_blocks = may_override_num_blocks(
|
||||
vllm_config, available_memory // bytes_per_block
|
||||
)
|
||||
size = bytes_per_block * num_blocks
|
||||
attn_specs = cast(
|
||||
UniformTypeKVCacheSpecs, attn_group.kv_cache_spec
|
||||
).kv_cache_specs
|
||||
|
||||
kv_cache_tensors: list[KVCacheTensor] = []
|
||||
|
||||
def add_tensor(layer_name: str, spec: KVCacheSpec, offset: int) -> None:
|
||||
kv_cache_tensors.append(
|
||||
KVCacheTensor(
|
||||
size=size,
|
||||
layers=[layer_name],
|
||||
layer_stride=spec.page_size_bytes * num_blocks,
|
||||
block_stride=spec.page_size_bytes,
|
||||
offset=offset,
|
||||
)
|
||||
)
|
||||
|
||||
for index, mla_name in enumerate(mla_names):
|
||||
offset = index * mla_page * num_blocks
|
||||
add_tensor(mla_name, attn_specs[mla_name], offset)
|
||||
for group in mamba_groups:
|
||||
if index < len(group.layer_names):
|
||||
add_tensor(group.layer_names[index], group.kv_cache_spec, offset)
|
||||
|
||||
idx_base = len(mla_names) * mla_page * num_blocks
|
||||
for index, idx_name in enumerate(idx_names):
|
||||
offset = idx_base + index * idx_page * num_blocks
|
||||
add_tensor(idx_name, attn_specs[idx_name], offset)
|
||||
if tail_names:
|
||||
tail_name = tail_names[index]
|
||||
tail_group = next(
|
||||
group for group in kv_cache_groups if tail_name in group.layer_names
|
||||
)
|
||||
tail_specs = cast(
|
||||
UniformTypeKVCacheSpecs, tail_group.kv_cache_spec
|
||||
).kv_cache_specs
|
||||
add_tensor(tail_name, tail_specs[tail_name], offset)
|
||||
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=kv_cache_tensors,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
prefix_cache_retention_interval=(
|
||||
vllm_config.cache_config.prefix_cache_retention_interval
|
||||
),
|
||||
)
|
||||
|
||||
layout = vllm_config.cache_config.get_resolved_kv_cache_layout()
|
||||
validate_kv_cache_layout(layout, kv_cache_groups)
|
||||
bytes_per_block = _get_kv_cache_bytes_per_block(kv_cache_groups)
|
||||
bytes_per_block = _get_kv_cache_bytes_per_block(kv_cache_groups, layout)
|
||||
interleaved_block_stride = bytes_per_block if layout.is_block_outermost else None
|
||||
|
||||
num_blocks = available_memory // bytes_per_block
|
||||
@@ -1901,17 +1651,17 @@ def get_kv_cache_config_from_groups(
|
||||
|
||||
byte_offset = 0
|
||||
for spec, layer_names in layers_by_spec.items():
|
||||
block_stride = interleaved_block_stride
|
||||
padded_page = _get_layer_block_stride(spec)
|
||||
if block_stride is None and padded_page != spec.page_size_bytes:
|
||||
assert layout.is_block_compact, f"{layout.name} cannot pad pages"
|
||||
block_stride = padded_page
|
||||
layer_stride, block_stride, _, _, _ = compute_layout_strides(
|
||||
spec,
|
||||
num_blocks,
|
||||
len(layer_names),
|
||||
layout,
|
||||
fixed_strides=(None, interleaved_block_stride, None, None, None),
|
||||
)
|
||||
offset = (
|
||||
byte_offset
|
||||
* max(layer_stride, spec.page_size_bytes)
|
||||
// spec.page_size_bytes
|
||||
fixed_strides=(None, block_stride, None, None, None),
|
||||
)
|
||||
kv_cache_tensors.append(
|
||||
KVCacheTensor(
|
||||
@@ -1919,10 +1669,13 @@ def get_kv_cache_config_from_groups(
|
||||
layers=layer_names,
|
||||
layer_stride=layer_stride,
|
||||
block_stride=block_stride,
|
||||
offset=offset,
|
||||
offset=byte_offset,
|
||||
)
|
||||
)
|
||||
byte_offset += len(layer_names) * spec.page_size_bytes
|
||||
# Block-outer layers share each block; layer-outer ones own regions.
|
||||
byte_offset += len(layer_names) * (
|
||||
spec.page_size_bytes if layout.is_block_outermost else layer_stride
|
||||
)
|
||||
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
@@ -2438,9 +2191,6 @@ def get_kv_cache_groups(
|
||||
# full attention, or all layers are sliding window attention with the
|
||||
# same window size). Put all layers into one group.
|
||||
return _get_kv_cache_groups_uniform_type(uniform_spec)
|
||||
elif glm5_groups := _get_kv_cache_groups_glm5_next(vllm_config, kv_cache_spec):
|
||||
return glm5_groups
|
||||
|
||||
# Hidden-state layers use their own block table and must not be absorbed
|
||||
# into a compatible attention bucket.
|
||||
hidden_specs = {
|
||||
@@ -2594,31 +2344,7 @@ def _max_memory_usage_bytes_from_groups(
|
||||
if not kv_cache_groups:
|
||||
return 0
|
||||
|
||||
if (glm5_layout := _glm5_next_tensor_layout(kv_cache_groups)) is not None:
|
||||
(
|
||||
attn_group,
|
||||
mamba_groups,
|
||||
mla_names,
|
||||
idx_names,
|
||||
mla_page,
|
||||
idx_page,
|
||||
tail_names,
|
||||
_,
|
||||
) = glm5_layout
|
||||
uniform_spec = cast(UniformTypeKVCacheSpecs, attn_group.kv_cache_spec)
|
||||
total_blocks = uniform_spec.max_memory_usage_pages(vllm_config)
|
||||
total_blocks += sum(
|
||||
cdiv(
|
||||
group.kv_cache_spec.max_memory_usage_bytes(vllm_config),
|
||||
group.kv_cache_spec.page_size_bytes,
|
||||
)
|
||||
for group in mamba_groups
|
||||
)
|
||||
if tail_names:
|
||||
total_blocks += 1
|
||||
return total_blocks * (len(mla_names) * mla_page + len(idx_names) * idx_page)
|
||||
|
||||
bytes_per_block = _pool_bytes_per_block(kv_cache_groups)
|
||||
bytes_per_block = _pool_bytes_per_block(vllm_config, kv_cache_groups)
|
||||
total_blocks = 0
|
||||
for group in kv_cache_groups:
|
||||
spec = group.kv_cache_spec
|
||||
@@ -2874,7 +2600,7 @@ def get_kv_cache_configs(
|
||||
if not groups:
|
||||
adjusted_memory.append(avail_mem)
|
||||
continue
|
||||
bytes_per_block = _pool_bytes_per_block(groups)
|
||||
bytes_per_block = _pool_bytes_per_block(vllm_config, groups)
|
||||
logger.info(
|
||||
"Overriding num_gpu_blocks=%d with num_gpu_blocks_override=%d",
|
||||
avail_mem // bytes_per_block,
|
||||
@@ -2887,7 +2613,7 @@ def get_kv_cache_configs(
|
||||
# the capacity check both plan against usable blocks. Allocation below
|
||||
# still uses the full memory.
|
||||
check_memory = [
|
||||
avail_mem - _pool_bytes_per_block(groups) if groups else avail_mem
|
||||
avail_mem - _pool_bytes_per_block(vllm_config, groups) if groups else avail_mem
|
||||
for groups, avail_mem in zip(projected_groups_per_worker, available_memory)
|
||||
]
|
||||
|
||||
@@ -2931,7 +2657,9 @@ def get_kv_cache_configs(
|
||||
# strides and offsets stay consistent with the shrunken allocation.
|
||||
groups = kv_cache_config.kv_cache_groups
|
||||
kv_cache_configs[i] = get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, min_num_blocks * _pool_bytes_per_block(groups)
|
||||
vllm_config,
|
||||
groups,
|
||||
min_num_blocks * _pool_bytes_per_block(vllm_config, groups),
|
||||
)
|
||||
|
||||
for kv_cache_config in kv_cache_configs:
|
||||
|
||||
@@ -7,7 +7,6 @@ from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from vllm.distributed.kv_events import MEDIUM_CPU
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
@@ -27,7 +26,6 @@ from vllm.v1.kv_cache_interface import (
|
||||
HiddenStateCacheSpec,
|
||||
HiSparseHotSpec,
|
||||
HiSparseResidentSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheGroupRole,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
@@ -45,8 +43,6 @@ from vllm.v1.request import Request
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.hisparse.coordinator import HiSparseCoordinator
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class SingleTypeKVCacheManager(ABC):
|
||||
"""An abstract base class for a manager that handle the kv cache management
|
||||
@@ -1299,10 +1295,6 @@ class CircularBufferManager(FullAttentionManager):
|
||||
return 0
|
||||
|
||||
|
||||
class KpoolTailManager(CircularBufferManager):
|
||||
"""One-block circular scratch manager for ``KpoolTailSpec``."""
|
||||
|
||||
|
||||
class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
|
||||
def __init__(self, kv_cache_spec: ChunkedLocalAttentionSpec, **kwargs) -> None:
|
||||
super().__init__(kv_cache_spec, **kwargs)
|
||||
@@ -2687,11 +2679,6 @@ def register_all_kvcache_specs(vllm_config):
|
||||
SlidingWindowManager,
|
||||
uniform_type_base_spec=SlidingWindowMLASpec,
|
||||
)
|
||||
KVCacheSpecRegistry.register(
|
||||
KpoolTailSpec,
|
||||
KpoolTailManager,
|
||||
uniform_type_base_spec=KpoolTailSpec,
|
||||
)
|
||||
|
||||
KVCacheSpecRegistry.register(
|
||||
MambaSpec, MambaManager, uniform_type_base_spec=MambaSpec
|
||||
|
||||
@@ -335,8 +335,8 @@ class EngineCore:
|
||||
vllm_config.cache_config.num_gpu_blocks = scheduler_kv_cache_config.num_blocks
|
||||
kv_cache_groups = scheduler_kv_cache_config.kv_cache_groups
|
||||
if kv_cache_groups:
|
||||
# Exclude groups that opt out of prefix caching (e.g. GLM-5.3-Flash
|
||||
# kpool tail, a 1-block/req scratch buffer with block_size=kpool):
|
||||
# Exclude groups that opt out of prefix caching (e.g. a circular
|
||||
# buffer, a 1-block/req scratch buffer with a small ring block):
|
||||
# their small block_size would otherwise drag the global block_size
|
||||
# below the real allocator block size and desync it from mamba.
|
||||
participating = [
|
||||
|
||||
@@ -55,8 +55,9 @@ def resolve_hisparse_specs(
|
||||
if len(block_sizes) != 1:
|
||||
raise ValueError("HiSparse requires one scheduler block size.")
|
||||
backends = [attn_layers[name].get_attn_backend() for name in mla_specs]
|
||||
specs = list(mla_specs.values())
|
||||
try:
|
||||
block_size = select_common_block_size(block_sizes.pop(), backends)
|
||||
block_size = select_common_block_size(block_sizes.pop(), backends, specs)
|
||||
except ValueError as error:
|
||||
raise ValueError(
|
||||
"HiSparse requires a GPU block size supported by every sparse "
|
||||
@@ -104,16 +105,13 @@ def allocate_hisparse_kv_caches(
|
||||
if isinstance(host_spec, UniformTypeKVCacheSpecs)
|
||||
else host_spec
|
||||
)
|
||||
kernel_block_size = kernel_block_sizes[host_group_id]
|
||||
if isinstance(spec, MLAAttentionSpec) and spec.storage_block_size is not None:
|
||||
kernel_block_size = spec.storage_block_size
|
||||
views = create_kv_cache_views(
|
||||
backing,
|
||||
spec,
|
||||
kv_cache_config.num_blocks_of(tensor),
|
||||
layout,
|
||||
tensor,
|
||||
kernel_block_size=kernel_block_size,
|
||||
kernel_block_size=kernel_block_sizes[host_group_id],
|
||||
)
|
||||
kv_caches.update(zip(tensor.layers, views))
|
||||
return kv_caches
|
||||
|
||||
@@ -333,7 +333,7 @@ def get_hisparse_kv_cache_config(
|
||||
device_groups = hisparse_layout.device_groups
|
||||
layout = vllm_config.cache_config.get_resolved_kv_cache_layout()
|
||||
validate_kv_cache_layout(layout, device_groups)
|
||||
bytes_per_block = _get_kv_cache_bytes_per_block(device_groups)
|
||||
bytes_per_block = _get_kv_cache_bytes_per_block(device_groups, layout)
|
||||
num_blocks = may_override_num_blocks(
|
||||
vllm_config, available_memory // bytes_per_block
|
||||
)
|
||||
@@ -345,7 +345,7 @@ def get_hisparse_kv_cache_config(
|
||||
host_groups = [hisparse_layout.source_group]
|
||||
host_layout = KVCacheLayout.LBNHC
|
||||
validate_kv_cache_layout(host_layout, host_groups)
|
||||
host_bytes_per_block = _get_kv_cache_bytes_per_block(host_groups)
|
||||
host_bytes_per_block = _get_kv_cache_bytes_per_block(host_groups, host_layout)
|
||||
host_size = host_bytes_per_block * hisparse_layout.host_num_blocks
|
||||
kv_cache_tensors[:0] = _build_hisparse_kv_cache_tensors(
|
||||
host_groups,
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import math
|
||||
from collections import Counter
|
||||
from collections.abc import Collection, Sequence
|
||||
from dataclasses import dataclass, field, fields, replace
|
||||
@@ -24,6 +25,7 @@ from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.attention.backend import MultipleOf
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -168,13 +170,28 @@ class KVCacheSpec:
|
||||
dcp_sharded: bool = field(default=False, kw_only=True)
|
||||
"""Whether DCP shards this cache's token positions across ranks."""
|
||||
|
||||
block_stride_alignment: int | None = field(default=None, kw_only=True)
|
||||
"""Required byte alignment between physical blocks, including packed layers."""
|
||||
block_stride_alignment: int | MultipleOf | None = field(default=None, kw_only=True)
|
||||
"""Required byte alignment between physical blocks, including packed layers.
|
||||
``MultipleOf(n)``: whole kernel blocks, when blocks are split into kernel
|
||||
blocks of up to ``n`` tokens."""
|
||||
|
||||
def __post_init__(self):
|
||||
if self.block_stride_alignment is not None and self.block_stride_alignment <= 0:
|
||||
alignment = self.block_stride_alignment
|
||||
if isinstance(alignment, int) and alignment <= 0:
|
||||
raise ValueError("block_stride_alignment must be positive")
|
||||
|
||||
def get_block_stride_alignment(self) -> int:
|
||||
"""Bytes the stride between physical blocks must be a multiple of."""
|
||||
from vllm.v1.attention.backend import MultipleOf
|
||||
|
||||
alignment = self.block_stride_alignment
|
||||
if not isinstance(alignment, MultipleOf):
|
||||
return alignment or 1
|
||||
kernel_block_size = math.gcd(alignment.base, self.block_size)
|
||||
if kernel_block_size == self.block_size:
|
||||
return 1
|
||||
return self.copy_with_new_block_size(kernel_block_size).page_size_bytes
|
||||
|
||||
@property
|
||||
def prefix_cacheable(self) -> bool:
|
||||
"""Whether this spec's group participates in prefix caching."""
|
||||
@@ -674,8 +691,6 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
model_version: str | None = None
|
||||
cache_role: SparseCacheRole = SparseCacheRole.SPARSE
|
||||
is_index_group_leader: bool = False
|
||||
storage_block_size: int | None = None
|
||||
"""Token width used to view storage when it differs from the kernel block."""
|
||||
# Group capability enabled when any member flattens a non-causal query block
|
||||
# into decode rows. Runtime metadata still selects causal vs. non-causal mode.
|
||||
non_causal_multi_token_decode: bool = False
|
||||
@@ -696,7 +711,6 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
model_version_set = set(spec.model_version for spec in specs)
|
||||
cache_role_set = {spec.cache_role for spec in specs}
|
||||
index_group_leader_set = {spec.is_index_group_leader for spec in specs}
|
||||
storage_block_size_set = set(spec.storage_block_size for spec in specs)
|
||||
block_stride_alignment_set = {spec.block_stride_alignment for spec in specs}
|
||||
assert (
|
||||
len(cache_dtype_str_set) == 1
|
||||
@@ -704,12 +718,11 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
and len(model_version_set) == 1
|
||||
and len(cache_role_set) == 1
|
||||
and len(index_group_leader_set) == 1
|
||||
and len(storage_block_size_set) == 1
|
||||
and len(block_stride_alignment_set) == 1
|
||||
), (
|
||||
"All attention layers in the same KV cache group must use the same "
|
||||
"quantization method, tokens per state, model version, cache role, "
|
||||
"index-sharing role, storage block size and block stride alignment."
|
||||
"index-sharing role, and block stride alignment."
|
||||
)
|
||||
merged_spec = cls(
|
||||
block_size=specs[0].block_size,
|
||||
@@ -727,7 +740,6 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
model_version=model_version_set.pop(),
|
||||
cache_role=cache_role_set.pop(),
|
||||
is_index_group_leader=index_group_leader_set.pop(),
|
||||
storage_block_size=storage_block_size_set.pop(),
|
||||
block_stride_alignment=block_stride_alignment_set.pop(),
|
||||
non_causal_multi_token_decode=any(
|
||||
spec.non_causal_multi_token_decode for spec in specs
|
||||
@@ -1019,32 +1031,6 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class KpoolTailSpec(SlidingWindowSpec):
|
||||
"""One-block circular scratch cache for a kpool indexer's raw tail."""
|
||||
|
||||
def max_admission_blocks_per_request(
|
||||
self, max_in_flight_tokens: int, max_model_len: int
|
||||
) -> int:
|
||||
return 1
|
||||
|
||||
def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int:
|
||||
return 1
|
||||
|
||||
def is_uniform_with_collection(
|
||||
self, kv_cache_specs: dict[str, KVCacheSpec]
|
||||
) -> bool:
|
||||
return all(isinstance(spec, KpoolTailSpec) for spec in kv_cache_specs.values())
|
||||
|
||||
@property
|
||||
def prefix_cacheable(self) -> bool:
|
||||
return False
|
||||
|
||||
@property
|
||||
def uses_slot_mapping(self) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MambaSpec(KVCacheSpec):
|
||||
shapes: tuple[tuple[int, ...], ...]
|
||||
|
||||
@@ -95,7 +95,7 @@ class Gemma4Proposer(SpecDecodeBaseProposer):
|
||||
cm.block_table_tensor = self._per_group_block_tables[gid][:batch_size]
|
||||
else:
|
||||
cm = common_attn_metadata
|
||||
attn_metadata = attn_group.get_metadata_builder().build_for_drafting(
|
||||
attn_metadata = attn_group.build_metadata_for_drafting(
|
||||
common_attn_metadata=cm, draft_index=draft_index
|
||||
)
|
||||
per_group_attn_metadata.append(attn_metadata)
|
||||
@@ -291,11 +291,7 @@ class Gemma4Proposer(SpecDecodeBaseProposer):
|
||||
self.draft_attn_groups = list(attention_groups.values())
|
||||
if self.draft_attn_groups:
|
||||
self.kv_cache_gid = self.draft_attn_groups[0].kv_cache_group_id
|
||||
self.block_size = (
|
||||
self.draft_attn_groups[0]
|
||||
.get_metadata_builder()
|
||||
.kv_cache_spec.block_size
|
||||
)
|
||||
self.block_size = self.draft_attn_groups[0].kv_cache_spec.block_size
|
||||
else:
|
||||
self.kv_cache_gid = 0
|
||||
self.block_size = kv_cache_config.kv_cache_groups[
|
||||
|
||||
@@ -991,7 +991,7 @@ class SpecDecodeBaseProposer:
|
||||
per_group_attn_metadata: list[object] = []
|
||||
per_layer_attn_metadata: dict[str, object] = {}
|
||||
for attn_group in self.draft_attn_groups:
|
||||
attn_metadata = attn_group.get_metadata_builder().build_for_drafting(
|
||||
attn_metadata = attn_group.build_metadata_for_drafting(
|
||||
common_attn_metadata=common_attn_metadata, draft_index=draft_index
|
||||
)
|
||||
per_group_attn_metadata.append(attn_metadata)
|
||||
@@ -1765,20 +1765,7 @@ class SpecDecodeBaseProposer:
|
||||
attention_groups[backend_key].layer_names.append(layer_name)
|
||||
|
||||
self.draft_attn_groups = list(attention_groups.values())
|
||||
if kernel_block_sizes is not None and 0 <= self.kv_cache_gid < len(
|
||||
kernel_block_sizes
|
||||
):
|
||||
# Slot mappings are computed against the block table, which is
|
||||
# stored at kernel-block granularity. Use the kernel block size
|
||||
# rather than the KV cache manager's block size; the two differ
|
||||
# when manager blocks are split for the attention kernel.
|
||||
self.block_size = kernel_block_sizes[self.kv_cache_gid]
|
||||
else:
|
||||
self.block_size = (
|
||||
self.draft_attn_groups[0]
|
||||
.get_metadata_builder()
|
||||
.kv_cache_spec.block_size
|
||||
)
|
||||
self.block_size = self.draft_attn_groups[0].kv_cache_spec.block_size
|
||||
logger.debug("Using block size %d for drafting layers", self.block_size)
|
||||
|
||||
def _determine_batch_execution_and_padding(
|
||||
|
||||
@@ -144,7 +144,7 @@ class Step3p5MTPProposer(EagleProposer):
|
||||
cm.slot_mapping = sm
|
||||
else:
|
||||
cm = common_attn_metadata
|
||||
attn_metadata = attn_group.get_metadata_builder().build_for_drafting(
|
||||
attn_metadata = attn_group.build_metadata_for_drafting(
|
||||
common_attn_metadata=cm,
|
||||
draft_index=draft_index,
|
||||
)
|
||||
@@ -244,11 +244,7 @@ class Step3p5MTPProposer(EagleProposer):
|
||||
self.draft_attn_groups = list(attention_groups.values())
|
||||
if self.draft_attn_groups:
|
||||
self.kv_cache_gid = self.draft_attn_groups[0].kv_cache_group_id
|
||||
self.block_size = (
|
||||
self.draft_attn_groups[0]
|
||||
.get_metadata_builder()
|
||||
.kv_cache_spec.block_size
|
||||
)
|
||||
self.block_size = self.draft_attn_groups[0].kv_cache_spec.block_size
|
||||
else:
|
||||
self.kv_cache_gid = 0
|
||||
self.block_size = kv_cache_config.kv_cache_groups[
|
||||
|
||||
+10
-119
@@ -29,24 +29,16 @@ logger = init_logger(__name__)
|
||||
def get_block_table_width(
|
||||
max_num_blocks: int,
|
||||
block_size: int,
|
||||
kernel_block_size: int | None = None,
|
||||
*,
|
||||
token_alignment: int | None = 128,
|
||||
) -> int:
|
||||
"""Return the width after optional alignment and virtual block splitting."""
|
||||
if kernel_block_size is None:
|
||||
kernel_block_size = block_size
|
||||
if block_size % kernel_block_size != 0:
|
||||
raise ValueError(
|
||||
f"kernel_block_size {kernel_block_size} must divide "
|
||||
f"block_size {block_size} evenly"
|
||||
)
|
||||
"""Return the width after optional alignment."""
|
||||
if token_alignment is not None:
|
||||
if token_alignment <= 0:
|
||||
raise ValueError("token_alignment must be positive")
|
||||
block_alignment = token_alignment // math.gcd(token_alignment, block_size)
|
||||
max_num_blocks = cdiv(max_num_blocks, block_alignment) * block_alignment
|
||||
return max_num_blocks * block_size // kernel_block_size
|
||||
return max_num_blocks
|
||||
|
||||
|
||||
class SlotMappingMode(Enum):
|
||||
@@ -63,7 +55,6 @@ class BlockTable:
|
||||
max_num_batched_tokens: int,
|
||||
pin_memory: bool,
|
||||
device: torch.device,
|
||||
kernel_block_size: int,
|
||||
cp_kv_cache_interleave_size: int,
|
||||
slot_mapping_mode: SlotMappingMode = SlotMappingMode.TOKEN_TO_KV_SLOT,
|
||||
):
|
||||
@@ -74,9 +65,6 @@ class BlockTable:
|
||||
max_num_batched_tokens: Maximum number of tokens in a batch.
|
||||
pin_memory: Whether to pin memory for faster GPU transfers.
|
||||
device: Target device for the block table.
|
||||
kernel_block_size: The block_size of underlying attention kernel.
|
||||
Will be the same as `block_size` if `block_size` is supported
|
||||
by the attention kernel.
|
||||
slot_mapping_mode: How this cache group maps scheduled tokens to
|
||||
cache slots. Mamba-like state caches do not use token slot
|
||||
mappings and should use SlotMappingMode.NONE.
|
||||
@@ -86,30 +74,8 @@ class BlockTable:
|
||||
self.max_num_batched_tokens = max_num_batched_tokens
|
||||
self.pin_memory = pin_memory
|
||||
self.device = device
|
||||
self.kv_cache_block_size = block_size
|
||||
|
||||
if kernel_block_size == block_size:
|
||||
# Standard case: allocation and computation use same block size
|
||||
# No block splitting needed, direct mapping
|
||||
self.block_size = block_size
|
||||
self.blocks_per_kv_block = 1
|
||||
self.use_hybrid_blocks = False
|
||||
else:
|
||||
# Hybrid case: allocation block size differs from kernel block size
|
||||
# Memory blocks are subdivided to match kernel requirements
|
||||
# Example: 32-token memory blocks with 16-token kernel blocks
|
||||
# → Each memory block corresponds to 2 kernel blocks
|
||||
if block_size % kernel_block_size != 0:
|
||||
raise ValueError(
|
||||
f"kernel_block_size {kernel_block_size} must divide "
|
||||
f"kv_manager_block_size size {block_size} evenly"
|
||||
)
|
||||
|
||||
self.block_size = kernel_block_size
|
||||
self.blocks_per_kv_block = block_size // kernel_block_size
|
||||
self.use_hybrid_blocks = True
|
||||
|
||||
self.max_num_blocks_per_req = max_num_blocks_per_req * self.blocks_per_kv_block
|
||||
self.block_size = block_size
|
||||
self.max_num_blocks_per_req = max_num_blocks_per_req
|
||||
|
||||
self.block_table = self._make_buffer(
|
||||
self.max_num_reqs, self.max_num_blocks_per_req, dtype=torch.int32
|
||||
@@ -120,13 +86,6 @@ class BlockTable:
|
||||
self.max_num_batched_tokens, dtype=torch.int64
|
||||
)
|
||||
|
||||
if self.use_hybrid_blocks:
|
||||
self._kernel_block_arange = np.arange(0, self.blocks_per_kv_block).reshape(
|
||||
1, -1
|
||||
)
|
||||
else:
|
||||
self._kernel_block_arange = None
|
||||
|
||||
try:
|
||||
self.pcp_world_size = get_pcp_group().world_size
|
||||
self.pcp_rank = get_pcp_group().rank_in_group
|
||||
@@ -145,13 +104,11 @@ class BlockTable:
|
||||
self.slot_mapping_mode = slot_mapping_mode
|
||||
if self.slot_mapping_mode == SlotMappingMode.TOKEN_TO_KV_SLOT:
|
||||
_COMPUTE_SLOT_MAPPING_KERNEL.register_warmup(
|
||||
kv_cache_block_size=self.kv_cache_block_size,
|
||||
blocks_per_kv_block=self.blocks_per_kv_block,
|
||||
kv_cache_block_size=self.block_size,
|
||||
total_cp_world_size=self.dcp_world_size,
|
||||
total_cp_rank=self.dcp_rank,
|
||||
cp_kv_cache_interleave_size=self.cp_kv_cache_interleave_size,
|
||||
block_table_stride=self.block_table.gpu.stride(0),
|
||||
block_size=self.block_size,
|
||||
)
|
||||
|
||||
def append_row(
|
||||
@@ -162,11 +119,6 @@ class BlockTable:
|
||||
if not block_ids:
|
||||
return
|
||||
|
||||
if self.use_hybrid_blocks:
|
||||
block_ids = self.map_to_kernel_blocks(
|
||||
np.array(block_ids), self.blocks_per_kv_block, self._kernel_block_arange
|
||||
)
|
||||
|
||||
num_blocks = len(block_ids)
|
||||
start = self.num_blocks_per_row[row_idx]
|
||||
self.num_blocks_per_row[row_idx] += num_blocks
|
||||
@@ -219,10 +171,8 @@ class BlockTable:
|
||||
positions,
|
||||
self.block_table.gpu,
|
||||
self.block_table.gpu.stride(0),
|
||||
self.block_size,
|
||||
self.slot_mapping.gpu,
|
||||
self.kv_cache_block_size,
|
||||
self.blocks_per_kv_block,
|
||||
self.block_size,
|
||||
self.dcp_world_size,
|
||||
self.dcp_rank,
|
||||
self.cp_kv_cache_interleave_size,
|
||||
@@ -235,37 +185,6 @@ class BlockTable:
|
||||
self.block_table.gpu.fill_(0)
|
||||
self.block_table.cpu.fill_(0)
|
||||
|
||||
@staticmethod
|
||||
def map_to_kernel_blocks(
|
||||
kv_manager_block_ids: np.ndarray,
|
||||
blocks_per_kv_block: int,
|
||||
kernel_block_arange: np.ndarray,
|
||||
) -> np.ndarray:
|
||||
"""Convert kv_manager_block_id IDs to kernel block IDs.
|
||||
|
||||
Example:
|
||||
# kv_manager_block_ids: 32 tokens,
|
||||
# Kernel block size: 16 tokens
|
||||
# blocks_per_kv_block = 2
|
||||
>>> kv_manager_block_ids = np.array([0, 1, 2])
|
||||
>>> Result: [0, 1, 2, 3, 4, 5]
|
||||
|
||||
# Each kv_manager_block_id maps to 2 kernel block id:
|
||||
# kv_manager_block_id 0 → kernel block id [0, 1]
|
||||
# kv_manager_block_id 1 → kernel block id [2, 3]
|
||||
# kv_manager_block_id 2 → kernel block id [4, 5]
|
||||
|
||||
"""
|
||||
if blocks_per_kv_block == 1:
|
||||
return kv_manager_block_ids
|
||||
|
||||
kernel_block_ids = (
|
||||
kv_manager_block_ids.reshape(-1, 1) * blocks_per_kv_block
|
||||
+ kernel_block_arange
|
||||
)
|
||||
|
||||
return kernel_block_ids.reshape(-1)
|
||||
|
||||
def get_device_tensor(self, num_reqs: int) -> torch.Tensor:
|
||||
"""Returns the device tensor of the block table."""
|
||||
return self.block_table.gpu[:num_reqs]
|
||||
@@ -296,16 +215,10 @@ class MultiGroupBlockTable:
|
||||
pin_memory: bool,
|
||||
device: torch.device,
|
||||
block_sizes: list[int],
|
||||
kernel_block_sizes: list[int],
|
||||
max_num_blocks: list[int],
|
||||
cp_kv_cache_interleave_size: int = 1,
|
||||
slot_mapping_modes: list[SlotMappingMode] | None = None,
|
||||
) -> None:
|
||||
if len(kernel_block_sizes) != len(block_sizes):
|
||||
raise ValueError(
|
||||
f"kernel_block_sizes length ({len(kernel_block_sizes)}) "
|
||||
f"must match block_sizes length ({len(block_sizes)})"
|
||||
)
|
||||
if slot_mapping_modes is None:
|
||||
slot_mapping_modes = [SlotMappingMode.TOKEN_TO_KV_SLOT] * len(block_sizes)
|
||||
if len(slot_mapping_modes) != len(block_sizes):
|
||||
@@ -339,17 +252,11 @@ class MultiGroupBlockTable:
|
||||
max_num_batched_tokens,
|
||||
pin_memory,
|
||||
device,
|
||||
kernel_block_size,
|
||||
cp_kv_cache_interleave_size,
|
||||
slot_mapping_mode=slot_mapping_mode,
|
||||
)
|
||||
for (
|
||||
block_size,
|
||||
kernel_block_size,
|
||||
max_num_blocks_per_req,
|
||||
slot_mapping_mode,
|
||||
) in zip(
|
||||
block_sizes, kernel_block_sizes, max_num_blocks, slot_mapping_modes
|
||||
for block_size, max_num_blocks_per_req, slot_mapping_mode in zip(
|
||||
block_sizes, max_num_blocks, slot_mapping_modes
|
||||
)
|
||||
]
|
||||
|
||||
@@ -403,12 +310,10 @@ class ComputeSlotMappingKernel(
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
kv_cache_block_size: int
|
||||
blocks_per_kv_block: int
|
||||
total_cp_world_size: int
|
||||
total_cp_rank: int
|
||||
cp_kv_cache_interleave_size: int
|
||||
block_table_stride: int
|
||||
block_size: int
|
||||
|
||||
@staticmethod
|
||||
@triton.jit(do_not_specialize=["num_tokens", "max_num_tokens"])
|
||||
@@ -419,10 +324,8 @@ class ComputeSlotMappingKernel(
|
||||
positions_ptr, # [num_tokens], int64
|
||||
block_table_ptr, # [max_num_reqs, max_num_blocks_per_req], int32 (flat)
|
||||
block_table_stride, # max_num_blocks_per_req
|
||||
block_size,
|
||||
slot_mapping_ptr, # [max_num_tokens], int64
|
||||
KV_CACHE_BLOCK_SIZE: tl.constexpr,
|
||||
BLOCKS_PER_KV_BLOCK: tl.constexpr,
|
||||
TOTAL_CP_WORLD_SIZE: tl.constexpr,
|
||||
TOTAL_CP_RANK: tl.constexpr,
|
||||
CP_KV_CACHE_INTERLEAVE_SIZE: tl.constexpr,
|
||||
@@ -463,17 +366,12 @@ class ComputeSlotMappingKernel(
|
||||
virtual_block_offsets % CP_KV_CACHE_INTERLEAVE_SIZE
|
||||
)
|
||||
|
||||
block_indices = (
|
||||
virtual_block_indices * BLOCKS_PER_KV_BLOCK
|
||||
+ local_block_offsets // block_size
|
||||
)
|
||||
block_numbers = tl.load(
|
||||
block_table_ptr + row_offset + block_indices,
|
||||
block_table_ptr + row_offset + virtual_block_indices,
|
||||
mask=mask & is_local,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
slot_offsets = local_block_offsets % block_size
|
||||
slot_ids = block_numbers * block_size + slot_offsets
|
||||
slot_ids = block_numbers * KV_CACHE_BLOCK_SIZE + local_block_offsets
|
||||
slot_ids = tl.where(is_local, slot_ids, PAD_ID)
|
||||
tl.store(slot_mapping_ptr + offsets, slot_ids, mask=mask)
|
||||
|
||||
@@ -481,13 +379,11 @@ class ComputeSlotMappingKernel(
|
||||
self,
|
||||
*,
|
||||
block_table_stride: int,
|
||||
block_size: int,
|
||||
**compile_key_fields: int,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(
|
||||
**compile_key_fields,
|
||||
block_table_stride=triton_scalar_specialization_rep(block_table_stride),
|
||||
block_size=triton_scalar_specialization_rep(block_size),
|
||||
)
|
||||
|
||||
def get_warmup_keys(self, **dispatch_kwargs: int) -> list[CompileKey]:
|
||||
@@ -507,10 +403,8 @@ class ComputeSlotMappingKernel(
|
||||
shape=(1, compile_key.block_table_stride),
|
||||
),
|
||||
block_table_stride=compile_key.block_table_stride,
|
||||
block_size=compile_key.block_size,
|
||||
slot_mapping=int64_ptr,
|
||||
kv_cache_block_size=compile_key.kv_cache_block_size,
|
||||
blocks_per_kv_block=compile_key.blocks_per_kv_block,
|
||||
total_cp_world_size=compile_key.total_cp_world_size,
|
||||
total_cp_rank=compile_key.total_cp_rank,
|
||||
cp_kv_cache_interleave_size=compile_key.cp_kv_cache_interleave_size,
|
||||
@@ -526,17 +420,14 @@ class ComputeSlotMappingKernel(
|
||||
positions: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
block_table_stride: int,
|
||||
block_size: int,
|
||||
slot_mapping: torch.Tensor,
|
||||
kv_cache_block_size: int,
|
||||
blocks_per_kv_block: int,
|
||||
total_cp_world_size: int,
|
||||
total_cp_rank: int,
|
||||
cp_kv_cache_interleave_size: int,
|
||||
) -> LaunchSpec:
|
||||
return (num_reqs + 1,), dict(
|
||||
KV_CACHE_BLOCK_SIZE=kv_cache_block_size,
|
||||
BLOCKS_PER_KV_BLOCK=blocks_per_kv_block,
|
||||
TOTAL_CP_WORLD_SIZE=total_cp_world_size,
|
||||
TOTAL_CP_RANK=total_cp_rank,
|
||||
CP_KV_CACHE_INTERLEAVE_SIZE=cp_kv_cache_interleave_size,
|
||||
|
||||
@@ -95,11 +95,9 @@ def prepare_dcp_dummy_context_metadata(
|
||||
max_valid_block_id = kv_cache_config.num_blocks - 1
|
||||
assert max_valid_block_id > 0
|
||||
for blk_table in input_batch.block_table.block_tables:
|
||||
max_row_blocks = (
|
||||
blk_table.max_num_blocks_per_req // blk_table.blocks_per_kv_block
|
||||
)
|
||||
block_ids = [
|
||||
(block_idx % max_valid_block_id) + 1 for block_idx in range(max_row_blocks)
|
||||
(block_idx % max_valid_block_id) + 1
|
||||
for block_idx in range(blk_table.max_num_blocks_per_req)
|
||||
]
|
||||
for req_idx in range(num_reqs):
|
||||
blk_table.add_row(block_ids, req_idx)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
@@ -40,6 +40,8 @@ from vllm.v1.worker.utils import (
|
||||
add_kv_sharing_layers_to_kv_cache_groups,
|
||||
allocate_kv_cache,
|
||||
bind_kv_cache_to_layers,
|
||||
customize_attention_spec,
|
||||
map_kv_caches_to_kernel_blocks,
|
||||
prepare_kernel_block_sizes,
|
||||
)
|
||||
|
||||
@@ -141,7 +143,7 @@ def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]:
|
||||
# Skip modules that don't need KV cache (eg encoder-only attention)
|
||||
if spec := attn_module.get_kv_cache_spec(vllm_config):
|
||||
if isinstance(spec, AttentionSpec):
|
||||
spec = attn_module.get_attn_backend().customize_spec(spec)
|
||||
spec = customize_attention_spec(attn_module.get_attn_backend(), spec)
|
||||
kv_cache_spec[layer_name] = spec
|
||||
if vllm_config.attention_config.hisparse_config is not None:
|
||||
kv_cache_spec = resolve_hisparse_specs(vllm_config, kv_cache_spec, attn_layers)
|
||||
@@ -382,6 +384,7 @@ def init_kv_cache(
|
||||
kv_cache_allocation_context: AbstractContextManager | None = None,
|
||||
*,
|
||||
block_tables: "BlockTables | None" = None,
|
||||
attn_groups: Iterable[AttentionGroup] = (),
|
||||
) -> dict[str, Any]:
|
||||
allocation_context = kv_cache_allocation_context or nullcontext()
|
||||
with allocation_context:
|
||||
@@ -409,9 +412,10 @@ def init_kv_cache(
|
||||
num_attn_module = (
|
||||
2 if vllm_config.model_config.hf_config.model_type == "longcat_flash" else 1
|
||||
)
|
||||
bindable_caches = {
|
||||
name: cache for name, cache in kv_caches.items() if name in forward_context
|
||||
}
|
||||
bindable_caches = map_kv_caches_to_kernel_blocks(
|
||||
{name: cache for name, cache in kv_caches.items() if name in forward_context},
|
||||
attn_groups,
|
||||
)
|
||||
bind_kv_cache_to_layers(
|
||||
bindable_caches,
|
||||
forward_context,
|
||||
@@ -536,8 +540,8 @@ def build_attn_metadata(
|
||||
cached_metadata[reuse_key], block_table, slot_mapping
|
||||
)
|
||||
elif for_cudagraph_capture:
|
||||
metadata = attn_metadata_builder.build_for_cudagraph_capture(
|
||||
common_attn_metadata
|
||||
metadata = attn_group.build_metadata_for_cudagraph_capture(
|
||||
common_attn_metadata, ubatch_idx
|
||||
)
|
||||
else:
|
||||
attn_metadata_extra_kwargs = (
|
||||
@@ -548,10 +552,8 @@ def build_attn_metadata(
|
||||
if model_specific_attn_metadata is not None
|
||||
else {}
|
||||
)
|
||||
metadata = attn_metadata_builder.build(
|
||||
common_prefix_len=0,
|
||||
common_attn_metadata=common_attn_metadata,
|
||||
**attn_metadata_extra_kwargs,
|
||||
metadata = attn_group.build_metadata(
|
||||
common_attn_metadata, ubatch_idx, **attn_metadata_extra_kwargs
|
||||
)
|
||||
if reuse_key is not None and reuse_key not in cached_metadata:
|
||||
cached_metadata[reuse_key] = metadata
|
||||
|
||||
@@ -25,7 +25,6 @@ class BlockTables:
|
||||
max_num_batched_tokens: int,
|
||||
max_num_blocks_per_group: list[int],
|
||||
device: torch.device,
|
||||
kernel_block_sizes: list[int],
|
||||
cp_size: int = 1,
|
||||
cp_rank: int = 0,
|
||||
cp_interleave: int = 1,
|
||||
@@ -33,7 +32,6 @@ class BlockTables:
|
||||
dcp_sharded: list[bool] | None = None,
|
||||
):
|
||||
self.block_sizes = block_sizes
|
||||
self.kernel_block_sizes = kernel_block_sizes
|
||||
self.max_num_reqs = max_num_reqs
|
||||
self.max_num_batched_tokens = max_num_batched_tokens
|
||||
self.device = device
|
||||
@@ -53,14 +51,9 @@ class BlockTables:
|
||||
assert len(dcp_sharded) == self.num_kv_cache_groups
|
||||
self.dcp_sharded = torch.tensor(dcp_sharded, dtype=torch.bool, device=device)
|
||||
|
||||
self.blocks_per_kv_block = [
|
||||
bs // kbs for bs, kbs in zip(block_sizes, kernel_block_sizes)
|
||||
]
|
||||
|
||||
# num_kv_cache_groups x [max_num_reqs, max_num_blocks]
|
||||
self.block_tables: list[StagedWriteTensor] = []
|
||||
for i in range(self.num_kv_cache_groups):
|
||||
max_num_blocks = max_num_blocks_per_group[i] * self.blocks_per_kv_block[i]
|
||||
for max_num_blocks in max_num_blocks_per_group:
|
||||
block_table = StagedWriteTensor(
|
||||
(self.max_num_reqs, max_num_blocks), dtype=torch.int32, device=device
|
||||
)
|
||||
@@ -109,9 +102,6 @@ class BlockTables:
|
||||
self.block_sizes_tensor = torch.tensor(
|
||||
self.block_sizes, dtype=torch.int32, device=self.device
|
||||
)
|
||||
self.kernel_block_sizes_tensor = torch.tensor(
|
||||
self.kernel_block_sizes, dtype=torch.int32, device=self.device
|
||||
)
|
||||
self.slot_mapping_enabled = torch.tensor(
|
||||
self._slot_mapping_enabled, dtype=torch.bool, device=self.device
|
||||
)
|
||||
@@ -126,9 +116,6 @@ class BlockTables:
|
||||
for i in range(self.num_kv_cache_groups):
|
||||
start = self.num_blocks.np[i, req_index] if not overwrite else 0
|
||||
block_ids = new_block_ids[i]
|
||||
bpk = self.blocks_per_kv_block[i]
|
||||
if bpk > 1:
|
||||
block_ids = [b * bpk + k for b in block_ids for k in range(bpk)]
|
||||
if self.redirect_writes_to_null_block:
|
||||
block_ids = [0] * len(block_ids)
|
||||
end = start + len(block_ids)
|
||||
@@ -218,7 +205,6 @@ class BlockTables:
|
||||
self.block_table_ptrs,
|
||||
self.block_table_strides,
|
||||
self.block_sizes_tensor,
|
||||
self.kernel_block_sizes_tensor,
|
||||
self.slot_mapping_enabled,
|
||||
self.dcp_sharded,
|
||||
slot_mappings,
|
||||
@@ -292,7 +278,6 @@ def _compute_slot_mappings_kernel(
|
||||
block_table_ptrs, # [num_kv_cache_groups]
|
||||
block_table_strides, # [num_kv_cache_groups]
|
||||
block_sizes, # [num_kv_cache_groups]
|
||||
kernel_block_sizes, # [num_kv_cache_groups]
|
||||
slot_mapping_enabled, # [num_kv_cache_groups]
|
||||
dcp_sharded, # [num_kv_cache_groups]
|
||||
slot_mappings_ptr, # [num_kv_cache_groups, max_num_tokens]
|
||||
@@ -322,7 +307,6 @@ def _compute_slot_mappings_kernel(
|
||||
block_table_ptr = _load_ptr(block_table_ptrs + group_id, tl.int32)
|
||||
block_table_stride = tl.load(block_table_strides + group_id)
|
||||
kv_block_size = tl.load(block_sizes + group_id)
|
||||
kernel_block_size = tl.load(kernel_block_sizes + group_id)
|
||||
mapping_enabled = tl.load(slot_mapping_enabled + group_id)
|
||||
if CP_SIZE != 1:
|
||||
sharded = tl.load(dcp_sharded + group_id)
|
||||
@@ -354,16 +338,14 @@ def _compute_slot_mappings_kernel(
|
||||
local_positions = tl.where(sharded, local_positions, positions)
|
||||
is_local = ~sharded | is_local
|
||||
|
||||
block_indices = tl.where(
|
||||
mapping_enabled, local_positions // kernel_block_size, 0
|
||||
)
|
||||
block_offsets = local_positions % kernel_block_size
|
||||
block_indices = tl.where(mapping_enabled, local_positions // kv_block_size, 0)
|
||||
block_offsets = local_positions % kv_block_size
|
||||
block_numbers = tl.load(
|
||||
block_table_ptr + req_state_idx * block_table_stride + block_indices,
|
||||
mask=is_local & is_real_req,
|
||||
other=0,
|
||||
)
|
||||
slot_ids = block_numbers * kernel_block_size + block_offsets
|
||||
slot_ids = block_numbers * kv_block_size + block_offsets
|
||||
if CP_SIZE != 1:
|
||||
slot_ids = tl.where(is_local, slot_ids, PAD_ID)
|
||||
|
||||
|
||||
@@ -252,17 +252,18 @@ def set_dummy_context(
|
||||
input_batch.positions.copy_(torch.from_numpy(local_pos + context_len))
|
||||
|
||||
seq_len = context_len + query_len
|
||||
for block_table, block_size, bpk in zip(
|
||||
input_block_tables,
|
||||
block_tables.kernel_block_sizes,
|
||||
block_tables.blocks_per_kv_block,
|
||||
):
|
||||
for block_table, block_size in zip(input_block_tables, block_tables.block_sizes):
|
||||
num_blocks = min(cdiv(seq_len, block_size), block_table.shape[1])
|
||||
# Spans are disjoint until the pool runs out, then they wrap and share
|
||||
# blocks: profiling only needs the reads to be realistic, not distinct.
|
||||
block_ids = torch.arange(
|
||||
num_reqs * num_blocks, dtype=block_table.dtype, device=block_table.device
|
||||
) % (num_kv_blocks * bpk)
|
||||
block_ids = (
|
||||
torch.arange(
|
||||
num_reqs * num_blocks,
|
||||
dtype=block_table.dtype,
|
||||
device=block_table.device,
|
||||
)
|
||||
% num_kv_blocks
|
||||
)
|
||||
block_table[:num_reqs, :num_blocks] = block_ids.view(num_reqs, num_blocks)
|
||||
|
||||
|
||||
|
||||
@@ -688,7 +688,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
max_num_batched_tokens=self.max_num_tokens,
|
||||
max_num_blocks_per_group=max_num_blocks_per_group,
|
||||
device=self.device,
|
||||
kernel_block_sizes=self.kernel_block_sizes,
|
||||
slot_mapping_enabled=slot_mapping_enabled,
|
||||
dcp_sharded=dcp_sharded,
|
||||
cp_size=self.dcp_size,
|
||||
@@ -773,6 +772,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.vllm_config,
|
||||
kv_cache_allocation_context=kv_cache_allocation_context,
|
||||
block_tables=self.block_tables,
|
||||
# Draft groups build their own metadata, so they map the views too.
|
||||
attn_groups=(
|
||||
g
|
||||
for groups in self.attn_groups
|
||||
+ getattr(self.speculator, "attn_groups", [])
|
||||
for g in groups
|
||||
),
|
||||
)
|
||||
self.kv_caches = [
|
||||
cache for cache in kv_caches_dict.values() if cache.device == self.device
|
||||
|
||||
@@ -432,7 +432,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
temperature,
|
||||
seeds,
|
||||
self.block_tables.input_block_tables[gid],
|
||||
self.block_tables.kernel_block_sizes[gid],
|
||||
self.block_tables.block_sizes[gid],
|
||||
self.block_tables.cp_rank,
|
||||
self.dcp_size,
|
||||
self.block_tables.cp_interleave,
|
||||
|
||||
@@ -577,6 +577,11 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
|
||||
idx_mapping = self.idx_mapping[:num_reqs]
|
||||
|
||||
if batch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
for groups in self.attn_groups:
|
||||
for group in groups:
|
||||
group.map_to_kernel_block_table(
|
||||
self.block_tables.input_block_tables[group.kv_cache_group_id]
|
||||
)
|
||||
assert self.decode_cudagraph_manager is not None
|
||||
self.decode_cudagraph_manager.run_fullgraph(batch_desc)
|
||||
return
|
||||
|
||||
@@ -96,7 +96,6 @@ class InputBatch:
|
||||
device: torch.device,
|
||||
vocab_size: int,
|
||||
block_sizes: list[int], # The block_size of each kv cache group
|
||||
kernel_block_sizes: list[int],
|
||||
max_num_blocks_per_req: list[int],
|
||||
logitsprocs: LogitsProcessors | None = None,
|
||||
logitsprocs_need_output_token_ids: bool = False,
|
||||
@@ -187,7 +186,6 @@ class InputBatch:
|
||||
pin_memory=PIN_MEMORY,
|
||||
device=device,
|
||||
block_sizes=block_sizes,
|
||||
kernel_block_sizes=kernel_block_sizes,
|
||||
max_num_blocks=max_num_blocks_per_req,
|
||||
cp_kv_cache_interleave_size=cp_kv_cache_interleave_size,
|
||||
slot_mapping_modes=slot_mapping_modes,
|
||||
|
||||
@@ -233,6 +233,8 @@ from .utils import (
|
||||
allocate_kv_cache,
|
||||
bind_kv_cache,
|
||||
copy_kv_cache_blocks_inplace,
|
||||
customize_attention_spec,
|
||||
map_kv_caches_to_kernel_blocks,
|
||||
prepare_kernel_block_sizes,
|
||||
sanity_check_mm_encoder_outputs,
|
||||
)
|
||||
@@ -718,7 +720,6 @@ class GPUModelRunner(
|
||||
max(self.max_model_len, self.max_encoder_len), placeholder_block_size
|
||||
)
|
||||
self._init_block_sizes = [placeholder_block_size]
|
||||
self._init_kernel_block_sizes = [placeholder_block_size]
|
||||
self._init_max_num_blocks = [placeholder_max_num_blocks]
|
||||
self._init_slot_mapping_modes = [SlotMappingMode.TOKEN_TO_KV_SLOT]
|
||||
self.cp_kv_cache_interleave_size = (
|
||||
@@ -734,7 +735,6 @@ class GPUModelRunner(
|
||||
device=self.device,
|
||||
vocab_size=self.model_config.get_vocab_size(),
|
||||
block_sizes=[placeholder_block_size],
|
||||
kernel_block_sizes=[placeholder_block_size],
|
||||
max_num_blocks_per_req=[placeholder_max_num_blocks],
|
||||
num_spec_tokens=self.num_spec_tokens,
|
||||
logitsprocs=build_logitsprocs(
|
||||
@@ -2508,8 +2508,8 @@ class GPUModelRunner(
|
||||
)
|
||||
|
||||
if for_cudagraph_capture:
|
||||
attn_metadata_i = builder.build_for_cudagraph_capture(
|
||||
common_attn_metadata
|
||||
attn_metadata_i = attn_group.build_metadata_for_cudagraph_capture(
|
||||
common_attn_metadata, ubid or 0
|
||||
)
|
||||
elif (
|
||||
cache_key in cached_attn_metadata
|
||||
@@ -2517,13 +2517,16 @@ class GPUModelRunner(
|
||||
):
|
||||
attn_metadata_i = builder.update_block_table(
|
||||
cached_attn_metadata[cache_key],
|
||||
common_attn_metadata.block_table_tensor,
|
||||
attn_group.map_to_kernel_block_table(
|
||||
common_attn_metadata.block_table_tensor, ubid or 0
|
||||
),
|
||||
common_attn_metadata.slot_mapping,
|
||||
)
|
||||
else:
|
||||
attn_metadata_i = builder.build(
|
||||
attn_metadata_i = attn_group.build_metadata(
|
||||
common_attn_metadata,
|
||||
ubid or 0,
|
||||
common_prefix_len=cascade_attn_prefix_len,
|
||||
common_attn_metadata=common_attn_metadata,
|
||||
**extra_attn_metadata_args,
|
||||
)
|
||||
if builder.supports_update_block_table:
|
||||
@@ -7143,9 +7146,7 @@ class GPUModelRunner(
|
||||
return
|
||||
self.reorder_batch_threshold = reduce(min_none_high, reorder_batch_thresholds) # type: ignore[assignment]
|
||||
|
||||
def may_reinitialize_input_batch(
|
||||
self, kv_cache_config: KVCacheConfig, kernel_block_sizes: list[int]
|
||||
) -> None:
|
||||
def may_reinitialize_input_batch(self, kv_cache_config: KVCacheConfig) -> None:
|
||||
"""Re-initialize the input batch if the block sizes are different from
|
||||
what it was originally created with. This happens when the final
|
||||
block size (determined after model loading) differs from the
|
||||
@@ -7154,7 +7155,6 @@ class GPUModelRunner(
|
||||
|
||||
Args:
|
||||
kv_cache_config: The KV cache configuration.
|
||||
kernel_block_sizes: The kernel block sizes for each KV cache group.
|
||||
|
||||
"""
|
||||
block_sizes = []
|
||||
@@ -7179,14 +7179,12 @@ class GPUModelRunner(
|
||||
|
||||
if (
|
||||
block_sizes != self._init_block_sizes
|
||||
or kernel_block_sizes != self._init_kernel_block_sizes
|
||||
or max_num_blocks != self._init_max_num_blocks
|
||||
or slot_mapping_modes != self._init_slot_mapping_modes
|
||||
or self.cp_kv_cache_interleave_size
|
||||
!= self.parallel_config.cp_kv_cache_interleave_size
|
||||
):
|
||||
self._init_block_sizes = block_sizes
|
||||
self._init_kernel_block_sizes = kernel_block_sizes
|
||||
self._init_max_num_blocks = max_num_blocks
|
||||
self._init_slot_mapping_modes = slot_mapping_modes
|
||||
self.cp_kv_cache_interleave_size = (
|
||||
@@ -7201,7 +7199,6 @@ class GPUModelRunner(
|
||||
device=self.device,
|
||||
vocab_size=self.model_config.get_vocab_size(),
|
||||
block_sizes=block_sizes,
|
||||
kernel_block_sizes=kernel_block_sizes,
|
||||
max_num_blocks_per_req=max_num_blocks,
|
||||
num_spec_tokens=self.num_spec_tokens,
|
||||
logitsprocs=self.input_batch.logitsprocs,
|
||||
@@ -7217,10 +7214,6 @@ class GPUModelRunner(
|
||||
f"InputBatch block_sizes {self._init_block_sizes} != "
|
||||
f"kv_cache block_sizes {block_sizes}"
|
||||
)
|
||||
assert self._init_kernel_block_sizes == kernel_block_sizes, (
|
||||
f"InputBatch kernel_block_sizes {self._init_kernel_block_sizes} "
|
||||
f"!= kv_cache kernel_block_sizes {kernel_block_sizes}"
|
||||
)
|
||||
|
||||
def _attn_group_iterator(self) -> Iterator[AttentionGroup]:
|
||||
return itertools.chain.from_iterable(self.attn_groups)
|
||||
@@ -7267,12 +7260,16 @@ class GPUModelRunner(
|
||||
num_attn_module = (
|
||||
2 if self.model_config.hf_config.model_type == "longcat_flash" else 1
|
||||
)
|
||||
attn_groups = [g for groups in self.attn_groups for g in groups]
|
||||
if self.speculative_config:
|
||||
attn_groups += getattr(self.drafter, "draft_attn_groups", [])
|
||||
bind_kv_cache(
|
||||
kv_caches,
|
||||
self.compilation_config.static_forward_context,
|
||||
self.kv_caches,
|
||||
num_attn_module,
|
||||
kv_cache_groups=kv_cache_config.kv_cache_groups,
|
||||
layer_kv_caches=map_kv_caches_to_kernel_blocks(kv_caches, attn_groups),
|
||||
)
|
||||
return kv_caches
|
||||
|
||||
@@ -7336,6 +7333,8 @@ class GPUModelRunner(
|
||||
# backends for that group only supports block_size 64, we will return
|
||||
# kernel_block_size 64 and split the 256-token-block to 4 blocks with 64
|
||||
# tokens each.
|
||||
# Block-outermost packed groups are not split; their attention groups
|
||||
# map kernel blocks themselves.
|
||||
kernel_block_sizes = prepare_kernel_block_sizes(
|
||||
kv_cache_config, self.attn_groups
|
||||
)
|
||||
@@ -7345,7 +7344,7 @@ class GPUModelRunner(
|
||||
self.initialize_metadata_builders(kv_cache_config, kernel_block_sizes)
|
||||
|
||||
# Reinitialize need to after initialize_attn_backend
|
||||
self.may_reinitialize_input_batch(kv_cache_config, kernel_block_sizes)
|
||||
self.may_reinitialize_input_batch(kv_cache_config)
|
||||
# Capture warmup providers that depend on allocated KV-cache strides.
|
||||
with self.jit_warmup_registry.activate(): # type: ignore[attr-defined]
|
||||
kv_caches = self.initialize_kv_cache_tensors(
|
||||
@@ -7422,7 +7421,9 @@ class GPUModelRunner(
|
||||
# Skip modules that don't need KV cache (eg encoder-only attention)
|
||||
if spec := attn_module.get_kv_cache_spec(self.vllm_config):
|
||||
if isinstance(spec, AttentionSpec):
|
||||
spec = attn_module.get_attn_backend().customize_spec(spec)
|
||||
spec = customize_attention_spec(
|
||||
attn_module.get_attn_backend(), spec
|
||||
)
|
||||
kv_cache_spec[layer_name] = spec
|
||||
|
||||
return kv_cache_spec
|
||||
|
||||
+207
-45
@@ -3,7 +3,7 @@
|
||||
import math
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass, field, replace
|
||||
from itertools import product as iprod
|
||||
from typing import Any
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionMetadataBuilder,
|
||||
CommonAttentionMetadata,
|
||||
MultipleOf,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
@@ -35,7 +36,6 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheLayout,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
create_kv_cache_views,
|
||||
)
|
||||
@@ -166,6 +166,9 @@ class KVBlockZeroer:
|
||||
kv = static_forward_context[layer_name].kv_cache
|
||||
if not isinstance(kv, torch.Tensor):
|
||||
continue
|
||||
if group.kernel_block_stride:
|
||||
# Bound views address kernel blocks; zero by manager block.
|
||||
kv = group.map_kv_cache_to_manager_blocks(kv, num_blocks)
|
||||
if kv.device.type != self.device.type:
|
||||
continue
|
||||
dp = kv.data_ptr()
|
||||
@@ -255,12 +258,22 @@ class KVBlockZeroer:
|
||||
self.zero_block_ids([0])
|
||||
|
||||
|
||||
@dataclass
|
||||
@dataclass(eq=False)
|
||||
class AttentionGroup:
|
||||
backend: type[AttentionBackend]
|
||||
layer_names: list[str]
|
||||
kv_cache_spec: KVCacheSpec
|
||||
kv_cache_group_id: int
|
||||
# Set when the backend needs blocks smaller than the KV cache manager's: its
|
||||
# block tables and KV cache views then address kernel blocks, kernel block j
|
||||
# of block b being b * kernel_block_stride + j. The stride is set when a
|
||||
# packed KV cache is bound (other layers' pages sit between its blocks);
|
||||
# otherwise it stays 0 and the kernel blocks per block are used.
|
||||
kernel_block_size: int | None = None
|
||||
kernel_block_stride: int = 0
|
||||
# Persistent per ubatch: CUDA graphs capture the mapped block table.
|
||||
kernel_block_table: torch.Tensor | None = None
|
||||
kernel_block_offsets: torch.Tensor | None = None
|
||||
# When ubatching is enabled we will have a metadata builder for each ubatch
|
||||
# so that if they use internal persistent buffers for cudagraphs, and they
|
||||
# won't have to worry about conflicting with the other ubatches.
|
||||
@@ -275,27 +288,44 @@ class AttentionGroup:
|
||||
kernel_block_size: int | None = None,
|
||||
num_metadata_builders: int = 1,
|
||||
):
|
||||
spec = self.kv_cache_spec
|
||||
if kernel_block_size is not None and isinstance(spec, AttentionSpec):
|
||||
# A backend in a packed KV cache group may need smaller blocks than
|
||||
# the group is split into.
|
||||
kernel_block_size = select_common_block_size(
|
||||
kernel_block_size, [self.backend], [spec]
|
||||
)
|
||||
if kernel_block_size != spec.block_size:
|
||||
self.kernel_block_size = kernel_block_size
|
||||
if kernel_block_size is None:
|
||||
kv_cache_spec_builder = self.kv_cache_spec
|
||||
elif (
|
||||
isinstance(self.kv_cache_spec, MLAAttentionSpec)
|
||||
and self.kv_cache_spec.storage_block_size is not None
|
||||
):
|
||||
kv_cache_spec_builder = self.kv_cache_spec.copy_with_new_block_size(
|
||||
self.kv_cache_spec.storage_block_size
|
||||
)
|
||||
else:
|
||||
kv_cache_spec_builder = self.kv_cache_spec.copy_with_new_block_size(
|
||||
kernel_block_size
|
||||
)
|
||||
builder_cls = self.backend.get_builder_cls()
|
||||
builder_kwargs = {}
|
||||
if builder_cls.requires_block_table_width:
|
||||
if builder_cls.requires_block_table_width or self.kernel_block_size is not None:
|
||||
max_num_blocks = self.kv_cache_spec.max_num_blocks_per_req(
|
||||
vllm_config, vllm_config.model_config.max_model_len
|
||||
)
|
||||
builder_kwargs["block_table_width"] = get_block_table_width(
|
||||
max_num_blocks, self.kv_cache_spec.block_size, kernel_block_size
|
||||
width = get_block_table_width(max_num_blocks, self.kv_cache_spec.block_size)
|
||||
if kernel_block_size is not None:
|
||||
width *= self.kv_cache_spec.block_size // kernel_block_size
|
||||
if builder_cls.requires_block_table_width:
|
||||
builder_kwargs["block_table_width"] = width
|
||||
if self.kernel_block_size is not None:
|
||||
self.kernel_block_table = torch.zeros(
|
||||
num_metadata_builders,
|
||||
vllm_config.scheduler_config.max_num_seqs,
|
||||
width,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
self.kernel_block_offsets = torch.arange(
|
||||
spec.block_size // self.kernel_block_size,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
self.metadata_builders = [
|
||||
builder_cls(
|
||||
@@ -307,9 +337,109 @@ class AttentionGroup:
|
||||
)
|
||||
for _ in range(num_metadata_builders)
|
||||
]
|
||||
if kernel_block_size is not None:
|
||||
for builder in self.metadata_builders:
|
||||
builder.set_kernel_block_size(kernel_block_size)
|
||||
|
||||
def build_metadata(
|
||||
self,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
ubatch_idx: int = 0,
|
||||
common_prefix_len: int = 0,
|
||||
**kwargs,
|
||||
) -> Any:
|
||||
return self.get_metadata_builder(ubatch_idx).build(
|
||||
common_prefix_len,
|
||||
self._with_kernel_block_table(common_attn_metadata, ubatch_idx),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def build_metadata_for_cudagraph_capture(
|
||||
self, common_attn_metadata: CommonAttentionMetadata, ubatch_idx: int = 0
|
||||
) -> Any:
|
||||
return self.get_metadata_builder(ubatch_idx).build_for_cudagraph_capture(
|
||||
self._with_kernel_block_table(common_attn_metadata, ubatch_idx)
|
||||
)
|
||||
|
||||
def build_metadata_for_drafting(
|
||||
self, common_attn_metadata: CommonAttentionMetadata, draft_index: int
|
||||
) -> Any:
|
||||
return self.get_metadata_builder().build_for_drafting(
|
||||
self._with_kernel_block_table(common_attn_metadata), draft_index
|
||||
)
|
||||
|
||||
def _with_kernel_block_table(
|
||||
self, common_attn_metadata: CommonAttentionMetadata, ubatch_idx: int = 0
|
||||
) -> CommonAttentionMetadata:
|
||||
if self.kernel_block_size is None:
|
||||
return common_attn_metadata
|
||||
return replace(
|
||||
common_attn_metadata,
|
||||
block_table_tensor=self.map_to_kernel_block_table(
|
||||
common_attn_metadata.block_table_tensor, ubatch_idx
|
||||
),
|
||||
)
|
||||
|
||||
def map_to_kernel_block_table(
|
||||
self, block_table: torch.Tensor, ubatch_idx: int = 0
|
||||
) -> torch.Tensor:
|
||||
if self.kernel_block_size is None:
|
||||
return block_table
|
||||
assert self.kernel_block_table is not None
|
||||
blocks_per_kv_block = self.kv_cache_spec.block_size // self.kernel_block_size
|
||||
rows, cols = block_table.shape
|
||||
out = self.kernel_block_table[ubatch_idx, :rows, : cols * blocks_per_kv_block]
|
||||
torch.add(
|
||||
self.kernel_block_offsets,
|
||||
block_table.unsqueeze(-1),
|
||||
alpha=self.kernel_block_stride or blocks_per_kv_block,
|
||||
out=out.unflatten(1, (cols, -1)),
|
||||
)
|
||||
return out
|
||||
|
||||
def map_kv_cache_to_kernel_blocks(self, kv_cache: torch.Tensor) -> torch.Tensor:
|
||||
"""Views a packed cache in manager blocks as kernel blocks."""
|
||||
assert self.kernel_block_size is not None
|
||||
spec = self.kv_cache_spec
|
||||
kernel_rows = spec.get_num_kernel_states(self.kernel_block_size)
|
||||
if kv_cache.shape[-2] == kernel_rows:
|
||||
return kv_cache
|
||||
# Token and pooled caches are written through manager-block slot
|
||||
# mappings, so only compressed caches (written from their block table)
|
||||
# can be mapped.
|
||||
if (
|
||||
spec.tokens_per_state <= 1
|
||||
or kv_cache.shape[1] != 1
|
||||
or kv_cache.stride(0) % (kernel_rows * kv_cache.stride(-2))
|
||||
):
|
||||
raise ValueError(
|
||||
f"{self.backend.get_name()} needs {self.kernel_block_size}-token "
|
||||
f"kernel blocks, which packed {spec.block_size}-token KV cache "
|
||||
f"blocks cannot be split into. Reduce --block-size to "
|
||||
f"{self.kernel_block_size} or set VLLM_KV_CACHE_LAYOUT to a "
|
||||
"layer-compact layout (e.g. LBNHC)."
|
||||
)
|
||||
# Kernel block j of block b is b * kernel_block_stride + j. The view
|
||||
# spans the other layers' pages between blocks (never addressing them),
|
||||
# which ``view`` cannot express.
|
||||
page_stride = kernel_rows * kv_cache.stride(-2)
|
||||
self.kernel_block_stride = kv_cache.stride(0) // page_stride
|
||||
num_pages = (kv_cache.shape[0] - 1) * self.kernel_block_stride + (
|
||||
kv_cache.shape[-2] // kernel_rows
|
||||
)
|
||||
return kv_cache.as_strided(
|
||||
(num_pages, *kv_cache.shape[1:-2], kernel_rows, kv_cache.shape[-1]),
|
||||
(page_stride, *kv_cache.stride()[1:]),
|
||||
)
|
||||
|
||||
def map_kv_cache_to_manager_blocks(
|
||||
self, kv_cache: torch.Tensor, num_blocks: int
|
||||
) -> torch.Tensor:
|
||||
"""Views a packed cache in kernel blocks as manager blocks."""
|
||||
assert self.kernel_block_stride
|
||||
spec = self.kv_cache_spec
|
||||
rows = spec.get_num_kernel_states(spec.block_size)
|
||||
return kv_cache.as_strided(
|
||||
(num_blocks, *kv_cache.shape[1:-2], rows, kv_cache.shape[-1]),
|
||||
(kv_cache.stride(0) * self.kernel_block_stride, *kv_cache.stride()[1:]),
|
||||
)
|
||||
|
||||
def get_metadata_builder(self, ubatch_id: int = 0) -> AttentionMetadataBuilder:
|
||||
assert len(self.metadata_builders) > ubatch_id
|
||||
@@ -327,17 +457,48 @@ class AttentionGroup:
|
||||
self.get_metadata_builder().update_draft_decode_metadata(metadata)
|
||||
|
||||
|
||||
def map_kv_caches_to_kernel_blocks(
|
||||
kv_caches: dict[str, torch.Tensor], attn_groups: Iterable[AttentionGroup]
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""The views to bind to layers. The manager views in ``kv_caches`` stay for
|
||||
KV connectors."""
|
||||
mapped = dict(kv_caches)
|
||||
for group in attn_groups:
|
||||
if group.kernel_block_size is not None:
|
||||
mapped.update(
|
||||
(name, group.map_kv_cache_to_kernel_blocks(kv_caches[name]))
|
||||
for name in group.layer_names
|
||||
if name in kv_caches
|
||||
)
|
||||
return mapped
|
||||
|
||||
|
||||
def customize_attention_spec(
|
||||
backend: type[AttentionBackend], spec: AttentionSpec
|
||||
) -> AttentionSpec:
|
||||
"""The backend's spec. Blocks larger than every kernel block size a backend
|
||||
takes are split into kernel blocks, which packed blocks must keep whole."""
|
||||
spec = backend.customize_spec(spec)
|
||||
sizes = backend.get_supported_kernel_block_sizes(spec)
|
||||
fixed_sizes = [s for s in sizes if isinstance(s, int)]
|
||||
if spec.block_stride_alignment is None and sizes and fixed_sizes == sizes:
|
||||
spec = replace(spec, block_stride_alignment=MultipleOf(max(fixed_sizes)))
|
||||
return spec
|
||||
|
||||
|
||||
def _block_size_is_supported(
|
||||
backends: list[type[AttentionBackend]], block_size: int
|
||||
backends: list[type[AttentionBackend]],
|
||||
block_size: int,
|
||||
specs: Sequence[KVCacheSpec | None],
|
||||
) -> bool:
|
||||
"""Check if the block size is supported by all backends.
|
||||
|
||||
An exact ``int`` declaration must match exactly; a ``MultipleOf``
|
||||
declaration accepts any multiple of its base.
|
||||
"""
|
||||
for backend in backends:
|
||||
for backend, spec in zip(backends, specs, strict=True):
|
||||
is_supported = False
|
||||
for supported_size in backend.get_supported_kernel_block_sizes():
|
||||
for supported_size in backend.get_supported_kernel_block_sizes(spec):
|
||||
if isinstance(supported_size, int):
|
||||
if block_size == supported_size:
|
||||
is_supported = True
|
||||
@@ -354,6 +515,7 @@ def _block_size_is_supported(
|
||||
def select_common_block_size(
|
||||
kv_manager_block_size: int,
|
||||
backends: list[type[AttentionBackend]],
|
||||
kv_cache_specs: Sequence[KVCacheSpec | None] | None = None,
|
||||
) -> int:
|
||||
"""Select a block size that is supported by all backends and is a factor of
|
||||
kv_manager_block_size.
|
||||
@@ -364,6 +526,7 @@ def select_common_block_size(
|
||||
Args:
|
||||
kv_manager_block_size: Block size of KV cache.
|
||||
backends: List of attention backend classes.
|
||||
kv_cache_specs: Per-backend cache specs for spec-specific constraints.
|
||||
|
||||
Returns:
|
||||
The selected block size.
|
||||
@@ -372,25 +535,27 @@ def select_common_block_size(
|
||||
ValueError: If no valid block size found.
|
||||
|
||||
"""
|
||||
if _block_size_is_supported(backends, kv_manager_block_size):
|
||||
specs = kv_cache_specs or [None] * len(backends)
|
||||
if _block_size_is_supported(backends, kv_manager_block_size, specs):
|
||||
return kv_manager_block_size
|
||||
|
||||
# MultipleOf constraints also accept the manager size if they accept a divisor.
|
||||
# Any remaining candidate must therefore be an explicit size from a backend.
|
||||
candidates = {
|
||||
size
|
||||
for backend in backends
|
||||
for size in backend.get_supported_kernel_block_sizes()
|
||||
for backend, spec in zip(backends, specs, strict=True)
|
||||
for size in backend.get_supported_kernel_block_sizes(spec)
|
||||
if isinstance(size, int) and kv_manager_block_size % size == 0
|
||||
}
|
||||
|
||||
for size in sorted(candidates, reverse=True):
|
||||
if _block_size_is_supported(backends, size):
|
||||
if _block_size_is_supported(backends, size, specs):
|
||||
return size
|
||||
raise ValueError(
|
||||
f"No common block size for {kv_manager_block_size} ("
|
||||
+ "; ".join(
|
||||
f"{b.get_name()}: {b.get_supported_kernel_block_sizes()}" for b in backends
|
||||
f"{b.get_name()}: {b.get_supported_kernel_block_sizes(s)}"
|
||||
for b, s in zip(backends, specs, strict=True)
|
||||
)
|
||||
+ ")."
|
||||
)
|
||||
@@ -450,9 +615,6 @@ def allocate_kv_cache(
|
||||
kernel_block_size = None
|
||||
if kernel_block_sizes is not None and group_id < len(kernel_block_sizes):
|
||||
kernel_block_size = kernel_block_sizes[group_id]
|
||||
if isinstance(spec, MLAAttentionSpec) and spec.storage_block_size is not None:
|
||||
kernel_block_size = spec.storage_block_size
|
||||
|
||||
views = create_kv_cache_views(
|
||||
buf,
|
||||
spec,
|
||||
@@ -497,24 +659,20 @@ def prepare_kernel_block_sizes(
|
||||
# This is an attention backend that supports virtual block splitting.
|
||||
kv_manager_block_size = kv_cache_group.kv_cache_spec.block_size
|
||||
group_backends = [g.backend for g in attn_groups[kv_cache_gid]]
|
||||
storage_block_size = (
|
||||
kv_cache_spec.storage_block_size
|
||||
if isinstance(kv_cache_spec, MLAAttentionSpec)
|
||||
else None
|
||||
)
|
||||
if storage_block_size is not None and _block_size_is_supported(
|
||||
group_backends, storage_block_size
|
||||
):
|
||||
# Storage-block specs (e.g. the GLM-5.3-Flash kpool indexer
|
||||
# cache) address the cache in pool pages, and every other
|
||||
# consumer (cache views, metadata builders, hisparse) already
|
||||
# uses storage_block_size as the kernel block. Fall back to the
|
||||
# backend vote when the group's backends do not accept it.
|
||||
selected_kernel_size = storage_block_size
|
||||
else:
|
||||
selected_kernel_size = select_common_block_size(
|
||||
kv_manager_block_size, group_backends
|
||||
group_specs = [g.kv_cache_spec for g in attn_groups[kv_cache_gid]]
|
||||
# Block-outermost packing puts other layers' pages between the
|
||||
# blocks, so they are never split group-wide.
|
||||
selected_kernel_size = (
|
||||
kv_manager_block_size
|
||||
if any(
|
||||
t.block_stride > t.layer_stride
|
||||
for t in kv_cache_config.kv_cache_tensors
|
||||
if not set(t.layers).isdisjoint(kv_cache_group.layer_names)
|
||||
)
|
||||
else select_common_block_size(
|
||||
kv_manager_block_size, group_backends, group_specs
|
||||
)
|
||||
)
|
||||
kernel_block_sizes.append(selected_kernel_size)
|
||||
elif isinstance(kv_cache_spec, MambaSpec):
|
||||
# This is likely Mamba or other non-attention cache, no splitting.
|
||||
@@ -656,6 +814,7 @@ def bind_kv_cache(
|
||||
runner_kv_caches: list[torch.Tensor],
|
||||
num_attn_module: int = 1,
|
||||
kv_cache_groups: Sequence[KVCacheGroupSpec] | None = None,
|
||||
layer_kv_caches: dict[str, torch.Tensor] | None = None,
|
||||
) -> None:
|
||||
"""Bind the allocated KV cache to both ModelRunner and forward context so
|
||||
that the KV cache can be used in the forward pass.
|
||||
@@ -674,6 +833,9 @@ def bind_kv_cache(
|
||||
runner_kv_caches: The kv_cache declared by ModelRunner.
|
||||
kv_cache_groups: The KV cache groups of the model, used to resolve
|
||||
layers that share a KV cache.
|
||||
layer_kv_caches: The views to bind to layers when they differ from
|
||||
kv_caches (e.g. kernel-block views). The runner keeps kv_caches,
|
||||
which are addressed by scheduler block IDs.
|
||||
|
||||
"""
|
||||
# Bind kv_caches to ModelRunner
|
||||
@@ -699,7 +861,7 @@ def bind_kv_cache(
|
||||
runner_kv_caches.append(kv_caches[layer_name])
|
||||
|
||||
bind_kv_cache_to_layers(
|
||||
kv_caches, forward_context, num_attn_module, kv_cache_groups
|
||||
layer_kv_caches or kv_caches, forward_context, num_attn_module, kv_cache_groups
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user