[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Dakai An
2026-10-09 15:48:37 -04:00
committed by GitHub
co-authored by Yifan Qiao OpenAI Codex OpenAI Codex Lucas Wilkinson Claude Opus 5.5 Lucas Wilkinson
parent 3766a67a32
commit 02e55d2af3
83 changed files with 1280 additions and 1986 deletions
+10 -119
View File
@@ -29,24 +29,16 @@ logger = init_logger(__name__)
def get_block_table_width(
max_num_blocks: int,
block_size: int,
kernel_block_size: int | None = None,
*,
token_alignment: int | None = 128,
) -> int:
"""Return the width after optional alignment and virtual block splitting."""
if kernel_block_size is None:
kernel_block_size = block_size
if block_size % kernel_block_size != 0:
raise ValueError(
f"kernel_block_size {kernel_block_size} must divide "
f"block_size {block_size} evenly"
)
"""Return the width after optional alignment."""
if token_alignment is not None:
if token_alignment <= 0:
raise ValueError("token_alignment must be positive")
block_alignment = token_alignment // math.gcd(token_alignment, block_size)
max_num_blocks = cdiv(max_num_blocks, block_alignment) * block_alignment
return max_num_blocks * block_size // kernel_block_size
return max_num_blocks
class SlotMappingMode(Enum):
@@ -63,7 +55,6 @@ class BlockTable:
max_num_batched_tokens: int,
pin_memory: bool,
device: torch.device,
kernel_block_size: int,
cp_kv_cache_interleave_size: int,
slot_mapping_mode: SlotMappingMode = SlotMappingMode.TOKEN_TO_KV_SLOT,
):
@@ -74,9 +65,6 @@ class BlockTable:
max_num_batched_tokens: Maximum number of tokens in a batch.
pin_memory: Whether to pin memory for faster GPU transfers.
device: Target device for the block table.
kernel_block_size: The block_size of underlying attention kernel.
Will be the same as `block_size` if `block_size` is supported
by the attention kernel.
slot_mapping_mode: How this cache group maps scheduled tokens to
cache slots. Mamba-like state caches do not use token slot
mappings and should use SlotMappingMode.NONE.
@@ -86,30 +74,8 @@ class BlockTable:
self.max_num_batched_tokens = max_num_batched_tokens
self.pin_memory = pin_memory
self.device = device
self.kv_cache_block_size = block_size
if kernel_block_size == block_size:
# Standard case: allocation and computation use same block size
# No block splitting needed, direct mapping
self.block_size = block_size
self.blocks_per_kv_block = 1
self.use_hybrid_blocks = False
else:
# Hybrid case: allocation block size differs from kernel block size
# Memory blocks are subdivided to match kernel requirements
# Example: 32-token memory blocks with 16-token kernel blocks
# → Each memory block corresponds to 2 kernel blocks
if block_size % kernel_block_size != 0:
raise ValueError(
f"kernel_block_size {kernel_block_size} must divide "
f"kv_manager_block_size size {block_size} evenly"
)
self.block_size = kernel_block_size
self.blocks_per_kv_block = block_size // kernel_block_size
self.use_hybrid_blocks = True
self.max_num_blocks_per_req = max_num_blocks_per_req * self.blocks_per_kv_block
self.block_size = block_size
self.max_num_blocks_per_req = max_num_blocks_per_req
self.block_table = self._make_buffer(
self.max_num_reqs, self.max_num_blocks_per_req, dtype=torch.int32
@@ -120,13 +86,6 @@ class BlockTable:
self.max_num_batched_tokens, dtype=torch.int64
)
if self.use_hybrid_blocks:
self._kernel_block_arange = np.arange(0, self.blocks_per_kv_block).reshape(
1, -1
)
else:
self._kernel_block_arange = None
try:
self.pcp_world_size = get_pcp_group().world_size
self.pcp_rank = get_pcp_group().rank_in_group
@@ -145,13 +104,11 @@ class BlockTable:
self.slot_mapping_mode = slot_mapping_mode
if self.slot_mapping_mode == SlotMappingMode.TOKEN_TO_KV_SLOT:
_COMPUTE_SLOT_MAPPING_KERNEL.register_warmup(
kv_cache_block_size=self.kv_cache_block_size,
blocks_per_kv_block=self.blocks_per_kv_block,
kv_cache_block_size=self.block_size,
total_cp_world_size=self.dcp_world_size,
total_cp_rank=self.dcp_rank,
cp_kv_cache_interleave_size=self.cp_kv_cache_interleave_size,
block_table_stride=self.block_table.gpu.stride(0),
block_size=self.block_size,
)
def append_row(
@@ -162,11 +119,6 @@ class BlockTable:
if not block_ids:
return
if self.use_hybrid_blocks:
block_ids = self.map_to_kernel_blocks(
np.array(block_ids), self.blocks_per_kv_block, self._kernel_block_arange
)
num_blocks = len(block_ids)
start = self.num_blocks_per_row[row_idx]
self.num_blocks_per_row[row_idx] += num_blocks
@@ -219,10 +171,8 @@ class BlockTable:
positions,
self.block_table.gpu,
self.block_table.gpu.stride(0),
self.block_size,
self.slot_mapping.gpu,
self.kv_cache_block_size,
self.blocks_per_kv_block,
self.block_size,
self.dcp_world_size,
self.dcp_rank,
self.cp_kv_cache_interleave_size,
@@ -235,37 +185,6 @@ class BlockTable:
self.block_table.gpu.fill_(0)
self.block_table.cpu.fill_(0)
@staticmethod
def map_to_kernel_blocks(
kv_manager_block_ids: np.ndarray,
blocks_per_kv_block: int,
kernel_block_arange: np.ndarray,
) -> np.ndarray:
"""Convert kv_manager_block_id IDs to kernel block IDs.
Example:
# kv_manager_block_ids: 32 tokens,
# Kernel block size: 16 tokens
# blocks_per_kv_block = 2
>>> kv_manager_block_ids = np.array([0, 1, 2])
>>> Result: [0, 1, 2, 3, 4, 5]
# Each kv_manager_block_id maps to 2 kernel block id:
# kv_manager_block_id 0 → kernel block id [0, 1]
# kv_manager_block_id 1 → kernel block id [2, 3]
# kv_manager_block_id 2 → kernel block id [4, 5]
"""
if blocks_per_kv_block == 1:
return kv_manager_block_ids
kernel_block_ids = (
kv_manager_block_ids.reshape(-1, 1) * blocks_per_kv_block
+ kernel_block_arange
)
return kernel_block_ids.reshape(-1)
def get_device_tensor(self, num_reqs: int) -> torch.Tensor:
"""Returns the device tensor of the block table."""
return self.block_table.gpu[:num_reqs]
@@ -296,16 +215,10 @@ class MultiGroupBlockTable:
pin_memory: bool,
device: torch.device,
block_sizes: list[int],
kernel_block_sizes: list[int],
max_num_blocks: list[int],
cp_kv_cache_interleave_size: int = 1,
slot_mapping_modes: list[SlotMappingMode] | None = None,
) -> None:
if len(kernel_block_sizes) != len(block_sizes):
raise ValueError(
f"kernel_block_sizes length ({len(kernel_block_sizes)}) "
f"must match block_sizes length ({len(block_sizes)})"
)
if slot_mapping_modes is None:
slot_mapping_modes = [SlotMappingMode.TOKEN_TO_KV_SLOT] * len(block_sizes)
if len(slot_mapping_modes) != len(block_sizes):
@@ -339,17 +252,11 @@ class MultiGroupBlockTable:
max_num_batched_tokens,
pin_memory,
device,
kernel_block_size,
cp_kv_cache_interleave_size,
slot_mapping_mode=slot_mapping_mode,
)
for (
block_size,
kernel_block_size,
max_num_blocks_per_req,
slot_mapping_mode,
) in zip(
block_sizes, kernel_block_sizes, max_num_blocks, slot_mapping_modes
for block_size, max_num_blocks_per_req, slot_mapping_mode in zip(
block_sizes, max_num_blocks, slot_mapping_modes
)
]
@@ -403,12 +310,10 @@ class ComputeSlotMappingKernel(
@dataclass(frozen=True)
class CompileKey:
kv_cache_block_size: int
blocks_per_kv_block: int
total_cp_world_size: int
total_cp_rank: int
cp_kv_cache_interleave_size: int
block_table_stride: int
block_size: int
@staticmethod
@triton.jit(do_not_specialize=["num_tokens", "max_num_tokens"])
@@ -419,10 +324,8 @@ class ComputeSlotMappingKernel(
positions_ptr, # [num_tokens], int64
block_table_ptr, # [max_num_reqs, max_num_blocks_per_req], int32 (flat)
block_table_stride, # max_num_blocks_per_req
block_size,
slot_mapping_ptr, # [max_num_tokens], int64
KV_CACHE_BLOCK_SIZE: tl.constexpr,
BLOCKS_PER_KV_BLOCK: tl.constexpr,
TOTAL_CP_WORLD_SIZE: tl.constexpr,
TOTAL_CP_RANK: tl.constexpr,
CP_KV_CACHE_INTERLEAVE_SIZE: tl.constexpr,
@@ -463,17 +366,12 @@ class ComputeSlotMappingKernel(
virtual_block_offsets % CP_KV_CACHE_INTERLEAVE_SIZE
)
block_indices = (
virtual_block_indices * BLOCKS_PER_KV_BLOCK
+ local_block_offsets // block_size
)
block_numbers = tl.load(
block_table_ptr + row_offset + block_indices,
block_table_ptr + row_offset + virtual_block_indices,
mask=mask & is_local,
other=0,
).to(tl.int64)
slot_offsets = local_block_offsets % block_size
slot_ids = block_numbers * block_size + slot_offsets
slot_ids = block_numbers * KV_CACHE_BLOCK_SIZE + local_block_offsets
slot_ids = tl.where(is_local, slot_ids, PAD_ID)
tl.store(slot_mapping_ptr + offsets, slot_ids, mask=mask)
@@ -481,13 +379,11 @@ class ComputeSlotMappingKernel(
self,
*,
block_table_stride: int,
block_size: int,
**compile_key_fields: int,
) -> CompileKey:
return self.CompileKey(
**compile_key_fields,
block_table_stride=triton_scalar_specialization_rep(block_table_stride),
block_size=triton_scalar_specialization_rep(block_size),
)
def get_warmup_keys(self, **dispatch_kwargs: int) -> list[CompileKey]:
@@ -507,10 +403,8 @@ class ComputeSlotMappingKernel(
shape=(1, compile_key.block_table_stride),
),
block_table_stride=compile_key.block_table_stride,
block_size=compile_key.block_size,
slot_mapping=int64_ptr,
kv_cache_block_size=compile_key.kv_cache_block_size,
blocks_per_kv_block=compile_key.blocks_per_kv_block,
total_cp_world_size=compile_key.total_cp_world_size,
total_cp_rank=compile_key.total_cp_rank,
cp_kv_cache_interleave_size=compile_key.cp_kv_cache_interleave_size,
@@ -526,17 +420,14 @@ class ComputeSlotMappingKernel(
positions: torch.Tensor,
block_table: torch.Tensor,
block_table_stride: int,
block_size: int,
slot_mapping: torch.Tensor,
kv_cache_block_size: int,
blocks_per_kv_block: int,
total_cp_world_size: int,
total_cp_rank: int,
cp_kv_cache_interleave_size: int,
) -> LaunchSpec:
return (num_reqs + 1,), dict(
KV_CACHE_BLOCK_SIZE=kv_cache_block_size,
BLOCKS_PER_KV_BLOCK=blocks_per_kv_block,
TOTAL_CP_WORLD_SIZE=total_cp_world_size,
TOTAL_CP_RANK=total_cp_rank,
CP_KV_CACHE_INTERLEAVE_SIZE=cp_kv_cache_interleave_size,
+2 -4
View File
@@ -95,11 +95,9 @@ def prepare_dcp_dummy_context_metadata(
max_valid_block_id = kv_cache_config.num_blocks - 1
assert max_valid_block_id > 0
for blk_table in input_batch.block_table.block_tables:
max_row_blocks = (
blk_table.max_num_blocks_per_req // blk_table.blocks_per_kv_block
)
block_ids = [
(block_idx % max_valid_block_id) + 1 for block_idx in range(max_row_blocks)
(block_idx % max_valid_block_id) + 1
for block_idx in range(blk_table.max_num_blocks_per_req)
]
for req_idx in range(num_reqs):
blk_table.add_row(block_ids, req_idx)
+13 -11
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Mapping, Sequence
from collections.abc import Iterable, Mapping, Sequence
from contextlib import AbstractContextManager, nullcontext
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, cast
@@ -40,6 +40,8 @@ from vllm.v1.worker.utils import (
add_kv_sharing_layers_to_kv_cache_groups,
allocate_kv_cache,
bind_kv_cache_to_layers,
customize_attention_spec,
map_kv_caches_to_kernel_blocks,
prepare_kernel_block_sizes,
)
@@ -141,7 +143,7 @@ def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]:
# Skip modules that don't need KV cache (eg encoder-only attention)
if spec := attn_module.get_kv_cache_spec(vllm_config):
if isinstance(spec, AttentionSpec):
spec = attn_module.get_attn_backend().customize_spec(spec)
spec = customize_attention_spec(attn_module.get_attn_backend(), spec)
kv_cache_spec[layer_name] = spec
if vllm_config.attention_config.hisparse_config is not None:
kv_cache_spec = resolve_hisparse_specs(vllm_config, kv_cache_spec, attn_layers)
@@ -382,6 +384,7 @@ def init_kv_cache(
kv_cache_allocation_context: AbstractContextManager | None = None,
*,
block_tables: "BlockTables | None" = None,
attn_groups: Iterable[AttentionGroup] = (),
) -> dict[str, Any]:
allocation_context = kv_cache_allocation_context or nullcontext()
with allocation_context:
@@ -409,9 +412,10 @@ def init_kv_cache(
num_attn_module = (
2 if vllm_config.model_config.hf_config.model_type == "longcat_flash" else 1
)
bindable_caches = {
name: cache for name, cache in kv_caches.items() if name in forward_context
}
bindable_caches = map_kv_caches_to_kernel_blocks(
{name: cache for name, cache in kv_caches.items() if name in forward_context},
attn_groups,
)
bind_kv_cache_to_layers(
bindable_caches,
forward_context,
@@ -536,8 +540,8 @@ def build_attn_metadata(
cached_metadata[reuse_key], block_table, slot_mapping
)
elif for_cudagraph_capture:
metadata = attn_metadata_builder.build_for_cudagraph_capture(
common_attn_metadata
metadata = attn_group.build_metadata_for_cudagraph_capture(
common_attn_metadata, ubatch_idx
)
else:
attn_metadata_extra_kwargs = (
@@ -548,10 +552,8 @@ def build_attn_metadata(
if model_specific_attn_metadata is not None
else {}
)
metadata = attn_metadata_builder.build(
common_prefix_len=0,
common_attn_metadata=common_attn_metadata,
**attn_metadata_extra_kwargs,
metadata = attn_group.build_metadata(
common_attn_metadata, ubatch_idx, **attn_metadata_extra_kwargs
)
if reuse_key is not None and reuse_key not in cached_metadata:
cached_metadata[reuse_key] = metadata
+4 -22
View File
@@ -25,7 +25,6 @@ class BlockTables:
max_num_batched_tokens: int,
max_num_blocks_per_group: list[int],
device: torch.device,
kernel_block_sizes: list[int],
cp_size: int = 1,
cp_rank: int = 0,
cp_interleave: int = 1,
@@ -33,7 +32,6 @@ class BlockTables:
dcp_sharded: list[bool] | None = None,
):
self.block_sizes = block_sizes
self.kernel_block_sizes = kernel_block_sizes
self.max_num_reqs = max_num_reqs
self.max_num_batched_tokens = max_num_batched_tokens
self.device = device
@@ -53,14 +51,9 @@ class BlockTables:
assert len(dcp_sharded) == self.num_kv_cache_groups
self.dcp_sharded = torch.tensor(dcp_sharded, dtype=torch.bool, device=device)
self.blocks_per_kv_block = [
bs // kbs for bs, kbs in zip(block_sizes, kernel_block_sizes)
]
# num_kv_cache_groups x [max_num_reqs, max_num_blocks]
self.block_tables: list[StagedWriteTensor] = []
for i in range(self.num_kv_cache_groups):
max_num_blocks = max_num_blocks_per_group[i] * self.blocks_per_kv_block[i]
for max_num_blocks in max_num_blocks_per_group:
block_table = StagedWriteTensor(
(self.max_num_reqs, max_num_blocks), dtype=torch.int32, device=device
)
@@ -109,9 +102,6 @@ class BlockTables:
self.block_sizes_tensor = torch.tensor(
self.block_sizes, dtype=torch.int32, device=self.device
)
self.kernel_block_sizes_tensor = torch.tensor(
self.kernel_block_sizes, dtype=torch.int32, device=self.device
)
self.slot_mapping_enabled = torch.tensor(
self._slot_mapping_enabled, dtype=torch.bool, device=self.device
)
@@ -126,9 +116,6 @@ class BlockTables:
for i in range(self.num_kv_cache_groups):
start = self.num_blocks.np[i, req_index] if not overwrite else 0
block_ids = new_block_ids[i]
bpk = self.blocks_per_kv_block[i]
if bpk > 1:
block_ids = [b * bpk + k for b in block_ids for k in range(bpk)]
if self.redirect_writes_to_null_block:
block_ids = [0] * len(block_ids)
end = start + len(block_ids)
@@ -218,7 +205,6 @@ class BlockTables:
self.block_table_ptrs,
self.block_table_strides,
self.block_sizes_tensor,
self.kernel_block_sizes_tensor,
self.slot_mapping_enabled,
self.dcp_sharded,
slot_mappings,
@@ -292,7 +278,6 @@ def _compute_slot_mappings_kernel(
block_table_ptrs, # [num_kv_cache_groups]
block_table_strides, # [num_kv_cache_groups]
block_sizes, # [num_kv_cache_groups]
kernel_block_sizes, # [num_kv_cache_groups]
slot_mapping_enabled, # [num_kv_cache_groups]
dcp_sharded, # [num_kv_cache_groups]
slot_mappings_ptr, # [num_kv_cache_groups, max_num_tokens]
@@ -322,7 +307,6 @@ def _compute_slot_mappings_kernel(
block_table_ptr = _load_ptr(block_table_ptrs + group_id, tl.int32)
block_table_stride = tl.load(block_table_strides + group_id)
kv_block_size = tl.load(block_sizes + group_id)
kernel_block_size = tl.load(kernel_block_sizes + group_id)
mapping_enabled = tl.load(slot_mapping_enabled + group_id)
if CP_SIZE != 1:
sharded = tl.load(dcp_sharded + group_id)
@@ -354,16 +338,14 @@ def _compute_slot_mappings_kernel(
local_positions = tl.where(sharded, local_positions, positions)
is_local = ~sharded | is_local
block_indices = tl.where(
mapping_enabled, local_positions // kernel_block_size, 0
)
block_offsets = local_positions % kernel_block_size
block_indices = tl.where(mapping_enabled, local_positions // kv_block_size, 0)
block_offsets = local_positions % kv_block_size
block_numbers = tl.load(
block_table_ptr + req_state_idx * block_table_stride + block_indices,
mask=is_local & is_real_req,
other=0,
)
slot_ids = block_numbers * kernel_block_size + block_offsets
slot_ids = block_numbers * kv_block_size + block_offsets
if CP_SIZE != 1:
slot_ids = tl.where(is_local, slot_ids, PAD_ID)
+9 -8
View File
@@ -252,17 +252,18 @@ def set_dummy_context(
input_batch.positions.copy_(torch.from_numpy(local_pos + context_len))
seq_len = context_len + query_len
for block_table, block_size, bpk in zip(
input_block_tables,
block_tables.kernel_block_sizes,
block_tables.blocks_per_kv_block,
):
for block_table, block_size in zip(input_block_tables, block_tables.block_sizes):
num_blocks = min(cdiv(seq_len, block_size), block_table.shape[1])
# Spans are disjoint until the pool runs out, then they wrap and share
# blocks: profiling only needs the reads to be realistic, not distinct.
block_ids = torch.arange(
num_reqs * num_blocks, dtype=block_table.dtype, device=block_table.device
) % (num_kv_blocks * bpk)
block_ids = (
torch.arange(
num_reqs * num_blocks,
dtype=block_table.dtype,
device=block_table.device,
)
% num_kv_blocks
)
block_table[:num_reqs, :num_blocks] = block_ids.view(num_reqs, num_blocks)
+7 -1
View File
@@ -688,7 +688,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
max_num_batched_tokens=self.max_num_tokens,
max_num_blocks_per_group=max_num_blocks_per_group,
device=self.device,
kernel_block_sizes=self.kernel_block_sizes,
slot_mapping_enabled=slot_mapping_enabled,
dcp_sharded=dcp_sharded,
cp_size=self.dcp_size,
@@ -773,6 +772,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.vllm_config,
kv_cache_allocation_context=kv_cache_allocation_context,
block_tables=self.block_tables,
# Draft groups build their own metadata, so they map the views too.
attn_groups=(
g
for groups in self.attn_groups
+ getattr(self.speculator, "attn_groups", [])
for g in groups
),
)
self.kv_caches = [
cache for cache in kv_caches_dict.values() if cache.device == self.device
@@ -432,7 +432,7 @@ class DFlashSpeculator(DraftModelSpeculator):
temperature,
seeds,
self.block_tables.input_block_tables[gid],
self.block_tables.kernel_block_sizes[gid],
self.block_tables.block_sizes[gid],
self.block_tables.cp_rank,
self.dcp_size,
self.block_tables.cp_interleave,
@@ -577,6 +577,11 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
idx_mapping = self.idx_mapping[:num_reqs]
if batch_desc.cg_mode == CUDAGraphMode.FULL:
for groups in self.attn_groups:
for group in groups:
group.map_to_kernel_block_table(
self.block_tables.input_block_tables[group.kv_cache_group_id]
)
assert self.decode_cudagraph_manager is not None
self.decode_cudagraph_manager.run_fullgraph(batch_desc)
return
-2
View File
@@ -96,7 +96,6 @@ class InputBatch:
device: torch.device,
vocab_size: int,
block_sizes: list[int], # The block_size of each kv cache group
kernel_block_sizes: list[int],
max_num_blocks_per_req: list[int],
logitsprocs: LogitsProcessors | None = None,
logitsprocs_need_output_token_ids: bool = False,
@@ -187,7 +186,6 @@ class InputBatch:
pin_memory=PIN_MEMORY,
device=device,
block_sizes=block_sizes,
kernel_block_sizes=kernel_block_sizes,
max_num_blocks=max_num_blocks_per_req,
cp_kv_cache_interleave_size=cp_kv_cache_interleave_size,
slot_mapping_modes=slot_mapping_modes,
+21 -20
View File
@@ -233,6 +233,8 @@ from .utils import (
allocate_kv_cache,
bind_kv_cache,
copy_kv_cache_blocks_inplace,
customize_attention_spec,
map_kv_caches_to_kernel_blocks,
prepare_kernel_block_sizes,
sanity_check_mm_encoder_outputs,
)
@@ -718,7 +720,6 @@ class GPUModelRunner(
max(self.max_model_len, self.max_encoder_len), placeholder_block_size
)
self._init_block_sizes = [placeholder_block_size]
self._init_kernel_block_sizes = [placeholder_block_size]
self._init_max_num_blocks = [placeholder_max_num_blocks]
self._init_slot_mapping_modes = [SlotMappingMode.TOKEN_TO_KV_SLOT]
self.cp_kv_cache_interleave_size = (
@@ -734,7 +735,6 @@ class GPUModelRunner(
device=self.device,
vocab_size=self.model_config.get_vocab_size(),
block_sizes=[placeholder_block_size],
kernel_block_sizes=[placeholder_block_size],
max_num_blocks_per_req=[placeholder_max_num_blocks],
num_spec_tokens=self.num_spec_tokens,
logitsprocs=build_logitsprocs(
@@ -2508,8 +2508,8 @@ class GPUModelRunner(
)
if for_cudagraph_capture:
attn_metadata_i = builder.build_for_cudagraph_capture(
common_attn_metadata
attn_metadata_i = attn_group.build_metadata_for_cudagraph_capture(
common_attn_metadata, ubid or 0
)
elif (
cache_key in cached_attn_metadata
@@ -2517,13 +2517,16 @@ class GPUModelRunner(
):
attn_metadata_i = builder.update_block_table(
cached_attn_metadata[cache_key],
common_attn_metadata.block_table_tensor,
attn_group.map_to_kernel_block_table(
common_attn_metadata.block_table_tensor, ubid or 0
),
common_attn_metadata.slot_mapping,
)
else:
attn_metadata_i = builder.build(
attn_metadata_i = attn_group.build_metadata(
common_attn_metadata,
ubid or 0,
common_prefix_len=cascade_attn_prefix_len,
common_attn_metadata=common_attn_metadata,
**extra_attn_metadata_args,
)
if builder.supports_update_block_table:
@@ -7143,9 +7146,7 @@ class GPUModelRunner(
return
self.reorder_batch_threshold = reduce(min_none_high, reorder_batch_thresholds) # type: ignore[assignment]
def may_reinitialize_input_batch(
self, kv_cache_config: KVCacheConfig, kernel_block_sizes: list[int]
) -> None:
def may_reinitialize_input_batch(self, kv_cache_config: KVCacheConfig) -> None:
"""Re-initialize the input batch if the block sizes are different from
what it was originally created with. This happens when the final
block size (determined after model loading) differs from the
@@ -7154,7 +7155,6 @@ class GPUModelRunner(
Args:
kv_cache_config: The KV cache configuration.
kernel_block_sizes: The kernel block sizes for each KV cache group.
"""
block_sizes = []
@@ -7179,14 +7179,12 @@ class GPUModelRunner(
if (
block_sizes != self._init_block_sizes
or kernel_block_sizes != self._init_kernel_block_sizes
or max_num_blocks != self._init_max_num_blocks
or slot_mapping_modes != self._init_slot_mapping_modes
or self.cp_kv_cache_interleave_size
!= self.parallel_config.cp_kv_cache_interleave_size
):
self._init_block_sizes = block_sizes
self._init_kernel_block_sizes = kernel_block_sizes
self._init_max_num_blocks = max_num_blocks
self._init_slot_mapping_modes = slot_mapping_modes
self.cp_kv_cache_interleave_size = (
@@ -7201,7 +7199,6 @@ class GPUModelRunner(
device=self.device,
vocab_size=self.model_config.get_vocab_size(),
block_sizes=block_sizes,
kernel_block_sizes=kernel_block_sizes,
max_num_blocks_per_req=max_num_blocks,
num_spec_tokens=self.num_spec_tokens,
logitsprocs=self.input_batch.logitsprocs,
@@ -7217,10 +7214,6 @@ class GPUModelRunner(
f"InputBatch block_sizes {self._init_block_sizes} != "
f"kv_cache block_sizes {block_sizes}"
)
assert self._init_kernel_block_sizes == kernel_block_sizes, (
f"InputBatch kernel_block_sizes {self._init_kernel_block_sizes} "
f"!= kv_cache kernel_block_sizes {kernel_block_sizes}"
)
def _attn_group_iterator(self) -> Iterator[AttentionGroup]:
return itertools.chain.from_iterable(self.attn_groups)
@@ -7267,12 +7260,16 @@ class GPUModelRunner(
num_attn_module = (
2 if self.model_config.hf_config.model_type == "longcat_flash" else 1
)
attn_groups = [g for groups in self.attn_groups for g in groups]
if self.speculative_config:
attn_groups += getattr(self.drafter, "draft_attn_groups", [])
bind_kv_cache(
kv_caches,
self.compilation_config.static_forward_context,
self.kv_caches,
num_attn_module,
kv_cache_groups=kv_cache_config.kv_cache_groups,
layer_kv_caches=map_kv_caches_to_kernel_blocks(kv_caches, attn_groups),
)
return kv_caches
@@ -7336,6 +7333,8 @@ class GPUModelRunner(
# backends for that group only supports block_size 64, we will return
# kernel_block_size 64 and split the 256-token-block to 4 blocks with 64
# tokens each.
# Block-outermost packed groups are not split; their attention groups
# map kernel blocks themselves.
kernel_block_sizes = prepare_kernel_block_sizes(
kv_cache_config, self.attn_groups
)
@@ -7345,7 +7344,7 @@ class GPUModelRunner(
self.initialize_metadata_builders(kv_cache_config, kernel_block_sizes)
# Reinitialize need to after initialize_attn_backend
self.may_reinitialize_input_batch(kv_cache_config, kernel_block_sizes)
self.may_reinitialize_input_batch(kv_cache_config)
# Capture warmup providers that depend on allocated KV-cache strides.
with self.jit_warmup_registry.activate(): # type: ignore[attr-defined]
kv_caches = self.initialize_kv_cache_tensors(
@@ -7422,7 +7421,9 @@ class GPUModelRunner(
# Skip modules that don't need KV cache (eg encoder-only attention)
if spec := attn_module.get_kv_cache_spec(self.vllm_config):
if isinstance(spec, AttentionSpec):
spec = attn_module.get_attn_backend().customize_spec(spec)
spec = customize_attention_spec(
attn_module.get_attn_backend(), spec
)
kv_cache_spec[layer_name] = spec
return kv_cache_spec
+207 -45
View File
@@ -3,7 +3,7 @@
import math
from collections import defaultdict
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from itertools import product as iprod
from typing import Any
@@ -24,6 +24,7 @@ from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionMetadataBuilder,
CommonAttentionMetadata,
MultipleOf,
)
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
@@ -35,7 +36,6 @@ from vllm.v1.kv_cache_interface import (
KVCacheLayout,
KVCacheSpec,
MambaSpec,
MLAAttentionSpec,
UniformTypeKVCacheSpecs,
create_kv_cache_views,
)
@@ -166,6 +166,9 @@ class KVBlockZeroer:
kv = static_forward_context[layer_name].kv_cache
if not isinstance(kv, torch.Tensor):
continue
if group.kernel_block_stride:
# Bound views address kernel blocks; zero by manager block.
kv = group.map_kv_cache_to_manager_blocks(kv, num_blocks)
if kv.device.type != self.device.type:
continue
dp = kv.data_ptr()
@@ -255,12 +258,22 @@ class KVBlockZeroer:
self.zero_block_ids([0])
@dataclass
@dataclass(eq=False)
class AttentionGroup:
backend: type[AttentionBackend]
layer_names: list[str]
kv_cache_spec: KVCacheSpec
kv_cache_group_id: int
# Set when the backend needs blocks smaller than the KV cache manager's: its
# block tables and KV cache views then address kernel blocks, kernel block j
# of block b being b * kernel_block_stride + j. The stride is set when a
# packed KV cache is bound (other layers' pages sit between its blocks);
# otherwise it stays 0 and the kernel blocks per block are used.
kernel_block_size: int | None = None
kernel_block_stride: int = 0
# Persistent per ubatch: CUDA graphs capture the mapped block table.
kernel_block_table: torch.Tensor | None = None
kernel_block_offsets: torch.Tensor | None = None
# When ubatching is enabled we will have a metadata builder for each ubatch
# so that if they use internal persistent buffers for cudagraphs, and they
# won't have to worry about conflicting with the other ubatches.
@@ -275,27 +288,44 @@ class AttentionGroup:
kernel_block_size: int | None = None,
num_metadata_builders: int = 1,
):
spec = self.kv_cache_spec
if kernel_block_size is not None and isinstance(spec, AttentionSpec):
# A backend in a packed KV cache group may need smaller blocks than
# the group is split into.
kernel_block_size = select_common_block_size(
kernel_block_size, [self.backend], [spec]
)
if kernel_block_size != spec.block_size:
self.kernel_block_size = kernel_block_size
if kernel_block_size is None:
kv_cache_spec_builder = self.kv_cache_spec
elif (
isinstance(self.kv_cache_spec, MLAAttentionSpec)
and self.kv_cache_spec.storage_block_size is not None
):
kv_cache_spec_builder = self.kv_cache_spec.copy_with_new_block_size(
self.kv_cache_spec.storage_block_size
)
else:
kv_cache_spec_builder = self.kv_cache_spec.copy_with_new_block_size(
kernel_block_size
)
builder_cls = self.backend.get_builder_cls()
builder_kwargs = {}
if builder_cls.requires_block_table_width:
if builder_cls.requires_block_table_width or self.kernel_block_size is not None:
max_num_blocks = self.kv_cache_spec.max_num_blocks_per_req(
vllm_config, vllm_config.model_config.max_model_len
)
builder_kwargs["block_table_width"] = get_block_table_width(
max_num_blocks, self.kv_cache_spec.block_size, kernel_block_size
width = get_block_table_width(max_num_blocks, self.kv_cache_spec.block_size)
if kernel_block_size is not None:
width *= self.kv_cache_spec.block_size // kernel_block_size
if builder_cls.requires_block_table_width:
builder_kwargs["block_table_width"] = width
if self.kernel_block_size is not None:
self.kernel_block_table = torch.zeros(
num_metadata_builders,
vllm_config.scheduler_config.max_num_seqs,
width,
dtype=torch.int32,
device=device,
)
self.kernel_block_offsets = torch.arange(
spec.block_size // self.kernel_block_size,
dtype=torch.int32,
device=device,
)
self.metadata_builders = [
builder_cls(
@@ -307,9 +337,109 @@ class AttentionGroup:
)
for _ in range(num_metadata_builders)
]
if kernel_block_size is not None:
for builder in self.metadata_builders:
builder.set_kernel_block_size(kernel_block_size)
def build_metadata(
self,
common_attn_metadata: CommonAttentionMetadata,
ubatch_idx: int = 0,
common_prefix_len: int = 0,
**kwargs,
) -> Any:
return self.get_metadata_builder(ubatch_idx).build(
common_prefix_len,
self._with_kernel_block_table(common_attn_metadata, ubatch_idx),
**kwargs,
)
def build_metadata_for_cudagraph_capture(
self, common_attn_metadata: CommonAttentionMetadata, ubatch_idx: int = 0
) -> Any:
return self.get_metadata_builder(ubatch_idx).build_for_cudagraph_capture(
self._with_kernel_block_table(common_attn_metadata, ubatch_idx)
)
def build_metadata_for_drafting(
self, common_attn_metadata: CommonAttentionMetadata, draft_index: int
) -> Any:
return self.get_metadata_builder().build_for_drafting(
self._with_kernel_block_table(common_attn_metadata), draft_index
)
def _with_kernel_block_table(
self, common_attn_metadata: CommonAttentionMetadata, ubatch_idx: int = 0
) -> CommonAttentionMetadata:
if self.kernel_block_size is None:
return common_attn_metadata
return replace(
common_attn_metadata,
block_table_tensor=self.map_to_kernel_block_table(
common_attn_metadata.block_table_tensor, ubatch_idx
),
)
def map_to_kernel_block_table(
self, block_table: torch.Tensor, ubatch_idx: int = 0
) -> torch.Tensor:
if self.kernel_block_size is None:
return block_table
assert self.kernel_block_table is not None
blocks_per_kv_block = self.kv_cache_spec.block_size // self.kernel_block_size
rows, cols = block_table.shape
out = self.kernel_block_table[ubatch_idx, :rows, : cols * blocks_per_kv_block]
torch.add(
self.kernel_block_offsets,
block_table.unsqueeze(-1),
alpha=self.kernel_block_stride or blocks_per_kv_block,
out=out.unflatten(1, (cols, -1)),
)
return out
def map_kv_cache_to_kernel_blocks(self, kv_cache: torch.Tensor) -> torch.Tensor:
"""Views a packed cache in manager blocks as kernel blocks."""
assert self.kernel_block_size is not None
spec = self.kv_cache_spec
kernel_rows = spec.get_num_kernel_states(self.kernel_block_size)
if kv_cache.shape[-2] == kernel_rows:
return kv_cache
# Token and pooled caches are written through manager-block slot
# mappings, so only compressed caches (written from their block table)
# can be mapped.
if (
spec.tokens_per_state <= 1
or kv_cache.shape[1] != 1
or kv_cache.stride(0) % (kernel_rows * kv_cache.stride(-2))
):
raise ValueError(
f"{self.backend.get_name()} needs {self.kernel_block_size}-token "
f"kernel blocks, which packed {spec.block_size}-token KV cache "
f"blocks cannot be split into. Reduce --block-size to "
f"{self.kernel_block_size} or set VLLM_KV_CACHE_LAYOUT to a "
"layer-compact layout (e.g. LBNHC)."
)
# Kernel block j of block b is b * kernel_block_stride + j. The view
# spans the other layers' pages between blocks (never addressing them),
# which ``view`` cannot express.
page_stride = kernel_rows * kv_cache.stride(-2)
self.kernel_block_stride = kv_cache.stride(0) // page_stride
num_pages = (kv_cache.shape[0] - 1) * self.kernel_block_stride + (
kv_cache.shape[-2] // kernel_rows
)
return kv_cache.as_strided(
(num_pages, *kv_cache.shape[1:-2], kernel_rows, kv_cache.shape[-1]),
(page_stride, *kv_cache.stride()[1:]),
)
def map_kv_cache_to_manager_blocks(
self, kv_cache: torch.Tensor, num_blocks: int
) -> torch.Tensor:
"""Views a packed cache in kernel blocks as manager blocks."""
assert self.kernel_block_stride
spec = self.kv_cache_spec
rows = spec.get_num_kernel_states(spec.block_size)
return kv_cache.as_strided(
(num_blocks, *kv_cache.shape[1:-2], rows, kv_cache.shape[-1]),
(kv_cache.stride(0) * self.kernel_block_stride, *kv_cache.stride()[1:]),
)
def get_metadata_builder(self, ubatch_id: int = 0) -> AttentionMetadataBuilder:
assert len(self.metadata_builders) > ubatch_id
@@ -327,17 +457,48 @@ class AttentionGroup:
self.get_metadata_builder().update_draft_decode_metadata(metadata)
def map_kv_caches_to_kernel_blocks(
kv_caches: dict[str, torch.Tensor], attn_groups: Iterable[AttentionGroup]
) -> dict[str, torch.Tensor]:
"""The views to bind to layers. The manager views in ``kv_caches`` stay for
KV connectors."""
mapped = dict(kv_caches)
for group in attn_groups:
if group.kernel_block_size is not None:
mapped.update(
(name, group.map_kv_cache_to_kernel_blocks(kv_caches[name]))
for name in group.layer_names
if name in kv_caches
)
return mapped
def customize_attention_spec(
backend: type[AttentionBackend], spec: AttentionSpec
) -> AttentionSpec:
"""The backend's spec. Blocks larger than every kernel block size a backend
takes are split into kernel blocks, which packed blocks must keep whole."""
spec = backend.customize_spec(spec)
sizes = backend.get_supported_kernel_block_sizes(spec)
fixed_sizes = [s for s in sizes if isinstance(s, int)]
if spec.block_stride_alignment is None and sizes and fixed_sizes == sizes:
spec = replace(spec, block_stride_alignment=MultipleOf(max(fixed_sizes)))
return spec
def _block_size_is_supported(
backends: list[type[AttentionBackend]], block_size: int
backends: list[type[AttentionBackend]],
block_size: int,
specs: Sequence[KVCacheSpec | None],
) -> bool:
"""Check if the block size is supported by all backends.
An exact ``int`` declaration must match exactly; a ``MultipleOf``
declaration accepts any multiple of its base.
"""
for backend in backends:
for backend, spec in zip(backends, specs, strict=True):
is_supported = False
for supported_size in backend.get_supported_kernel_block_sizes():
for supported_size in backend.get_supported_kernel_block_sizes(spec):
if isinstance(supported_size, int):
if block_size == supported_size:
is_supported = True
@@ -354,6 +515,7 @@ def _block_size_is_supported(
def select_common_block_size(
kv_manager_block_size: int,
backends: list[type[AttentionBackend]],
kv_cache_specs: Sequence[KVCacheSpec | None] | None = None,
) -> int:
"""Select a block size that is supported by all backends and is a factor of
kv_manager_block_size.
@@ -364,6 +526,7 @@ def select_common_block_size(
Args:
kv_manager_block_size: Block size of KV cache.
backends: List of attention backend classes.
kv_cache_specs: Per-backend cache specs for spec-specific constraints.
Returns:
The selected block size.
@@ -372,25 +535,27 @@ def select_common_block_size(
ValueError: If no valid block size found.
"""
if _block_size_is_supported(backends, kv_manager_block_size):
specs = kv_cache_specs or [None] * len(backends)
if _block_size_is_supported(backends, kv_manager_block_size, specs):
return kv_manager_block_size
# MultipleOf constraints also accept the manager size if they accept a divisor.
# Any remaining candidate must therefore be an explicit size from a backend.
candidates = {
size
for backend in backends
for size in backend.get_supported_kernel_block_sizes()
for backend, spec in zip(backends, specs, strict=True)
for size in backend.get_supported_kernel_block_sizes(spec)
if isinstance(size, int) and kv_manager_block_size % size == 0
}
for size in sorted(candidates, reverse=True):
if _block_size_is_supported(backends, size):
if _block_size_is_supported(backends, size, specs):
return size
raise ValueError(
f"No common block size for {kv_manager_block_size} ("
+ "; ".join(
f"{b.get_name()}: {b.get_supported_kernel_block_sizes()}" for b in backends
f"{b.get_name()}: {b.get_supported_kernel_block_sizes(s)}"
for b, s in zip(backends, specs, strict=True)
)
+ ")."
)
@@ -450,9 +615,6 @@ def allocate_kv_cache(
kernel_block_size = None
if kernel_block_sizes is not None and group_id < len(kernel_block_sizes):
kernel_block_size = kernel_block_sizes[group_id]
if isinstance(spec, MLAAttentionSpec) and spec.storage_block_size is not None:
kernel_block_size = spec.storage_block_size
views = create_kv_cache_views(
buf,
spec,
@@ -497,24 +659,20 @@ def prepare_kernel_block_sizes(
# This is an attention backend that supports virtual block splitting.
kv_manager_block_size = kv_cache_group.kv_cache_spec.block_size
group_backends = [g.backend for g in attn_groups[kv_cache_gid]]
storage_block_size = (
kv_cache_spec.storage_block_size
if isinstance(kv_cache_spec, MLAAttentionSpec)
else None
)
if storage_block_size is not None and _block_size_is_supported(
group_backends, storage_block_size
):
# Storage-block specs (e.g. the GLM-5.3-Flash kpool indexer
# cache) address the cache in pool pages, and every other
# consumer (cache views, metadata builders, hisparse) already
# uses storage_block_size as the kernel block. Fall back to the
# backend vote when the group's backends do not accept it.
selected_kernel_size = storage_block_size
else:
selected_kernel_size = select_common_block_size(
kv_manager_block_size, group_backends
group_specs = [g.kv_cache_spec for g in attn_groups[kv_cache_gid]]
# Block-outermost packing puts other layers' pages between the
# blocks, so they are never split group-wide.
selected_kernel_size = (
kv_manager_block_size
if any(
t.block_stride > t.layer_stride
for t in kv_cache_config.kv_cache_tensors
if not set(t.layers).isdisjoint(kv_cache_group.layer_names)
)
else select_common_block_size(
kv_manager_block_size, group_backends, group_specs
)
)
kernel_block_sizes.append(selected_kernel_size)
elif isinstance(kv_cache_spec, MambaSpec):
# This is likely Mamba or other non-attention cache, no splitting.
@@ -656,6 +814,7 @@ def bind_kv_cache(
runner_kv_caches: list[torch.Tensor],
num_attn_module: int = 1,
kv_cache_groups: Sequence[KVCacheGroupSpec] | None = None,
layer_kv_caches: dict[str, torch.Tensor] | None = None,
) -> None:
"""Bind the allocated KV cache to both ModelRunner and forward context so
that the KV cache can be used in the forward pass.
@@ -674,6 +833,9 @@ def bind_kv_cache(
runner_kv_caches: The kv_cache declared by ModelRunner.
kv_cache_groups: The KV cache groups of the model, used to resolve
layers that share a KV cache.
layer_kv_caches: The views to bind to layers when they differ from
kv_caches (e.g. kernel-block views). The runner keeps kv_caches,
which are addressed by scheduler block IDs.
"""
# Bind kv_caches to ModelRunner
@@ -699,7 +861,7 @@ def bind_kv_cache(
runner_kv_caches.append(kv_caches[layer_name])
bind_kv_cache_to_layers(
kv_caches, forward_context, num_attn_module, kv_cache_groups
layer_kv_caches or kv_caches, forward_context, num_attn_module, kv_cache_groups
)