[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Dakai An
2026-10-09 15:48:37 -04:00
committed by GitHub
co-authored by Yifan Qiao OpenAI Codex OpenAI Codex Lucas Wilkinson Claude Opus 5.5 Lucas Wilkinson
parent 3766a67a32
commit 02e55d2af3
83 changed files with 1280 additions and 1986 deletions
+13 -11
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Mapping, Sequence
from collections.abc import Iterable, Mapping, Sequence
from contextlib import AbstractContextManager, nullcontext
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, cast
@@ -40,6 +40,8 @@ from vllm.v1.worker.utils import (
add_kv_sharing_layers_to_kv_cache_groups,
allocate_kv_cache,
bind_kv_cache_to_layers,
customize_attention_spec,
map_kv_caches_to_kernel_blocks,
prepare_kernel_block_sizes,
)
@@ -141,7 +143,7 @@ def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]:
# Skip modules that don't need KV cache (eg encoder-only attention)
if spec := attn_module.get_kv_cache_spec(vllm_config):
if isinstance(spec, AttentionSpec):
spec = attn_module.get_attn_backend().customize_spec(spec)
spec = customize_attention_spec(attn_module.get_attn_backend(), spec)
kv_cache_spec[layer_name] = spec
if vllm_config.attention_config.hisparse_config is not None:
kv_cache_spec = resolve_hisparse_specs(vllm_config, kv_cache_spec, attn_layers)
@@ -382,6 +384,7 @@ def init_kv_cache(
kv_cache_allocation_context: AbstractContextManager | None = None,
*,
block_tables: "BlockTables | None" = None,
attn_groups: Iterable[AttentionGroup] = (),
) -> dict[str, Any]:
allocation_context = kv_cache_allocation_context or nullcontext()
with allocation_context:
@@ -409,9 +412,10 @@ def init_kv_cache(
num_attn_module = (
2 if vllm_config.model_config.hf_config.model_type == "longcat_flash" else 1
)
bindable_caches = {
name: cache for name, cache in kv_caches.items() if name in forward_context
}
bindable_caches = map_kv_caches_to_kernel_blocks(
{name: cache for name, cache in kv_caches.items() if name in forward_context},
attn_groups,
)
bind_kv_cache_to_layers(
bindable_caches,
forward_context,
@@ -536,8 +540,8 @@ def build_attn_metadata(
cached_metadata[reuse_key], block_table, slot_mapping
)
elif for_cudagraph_capture:
metadata = attn_metadata_builder.build_for_cudagraph_capture(
common_attn_metadata
metadata = attn_group.build_metadata_for_cudagraph_capture(
common_attn_metadata, ubatch_idx
)
else:
attn_metadata_extra_kwargs = (
@@ -548,10 +552,8 @@ def build_attn_metadata(
if model_specific_attn_metadata is not None
else {}
)
metadata = attn_metadata_builder.build(
common_prefix_len=0,
common_attn_metadata=common_attn_metadata,
**attn_metadata_extra_kwargs,
metadata = attn_group.build_metadata(
common_attn_metadata, ubatch_idx, **attn_metadata_extra_kwargs
)
if reuse_key is not None and reuse_key not in cached_metadata:
cached_metadata[reuse_key] = metadata
+4 -22
View File
@@ -25,7 +25,6 @@ class BlockTables:
max_num_batched_tokens: int,
max_num_blocks_per_group: list[int],
device: torch.device,
kernel_block_sizes: list[int],
cp_size: int = 1,
cp_rank: int = 0,
cp_interleave: int = 1,
@@ -33,7 +32,6 @@ class BlockTables:
dcp_sharded: list[bool] | None = None,
):
self.block_sizes = block_sizes
self.kernel_block_sizes = kernel_block_sizes
self.max_num_reqs = max_num_reqs
self.max_num_batched_tokens = max_num_batched_tokens
self.device = device
@@ -53,14 +51,9 @@ class BlockTables:
assert len(dcp_sharded) == self.num_kv_cache_groups
self.dcp_sharded = torch.tensor(dcp_sharded, dtype=torch.bool, device=device)
self.blocks_per_kv_block = [
bs // kbs for bs, kbs in zip(block_sizes, kernel_block_sizes)
]
# num_kv_cache_groups x [max_num_reqs, max_num_blocks]
self.block_tables: list[StagedWriteTensor] = []
for i in range(self.num_kv_cache_groups):
max_num_blocks = max_num_blocks_per_group[i] * self.blocks_per_kv_block[i]
for max_num_blocks in max_num_blocks_per_group:
block_table = StagedWriteTensor(
(self.max_num_reqs, max_num_blocks), dtype=torch.int32, device=device
)
@@ -109,9 +102,6 @@ class BlockTables:
self.block_sizes_tensor = torch.tensor(
self.block_sizes, dtype=torch.int32, device=self.device
)
self.kernel_block_sizes_tensor = torch.tensor(
self.kernel_block_sizes, dtype=torch.int32, device=self.device
)
self.slot_mapping_enabled = torch.tensor(
self._slot_mapping_enabled, dtype=torch.bool, device=self.device
)
@@ -126,9 +116,6 @@ class BlockTables:
for i in range(self.num_kv_cache_groups):
start = self.num_blocks.np[i, req_index] if not overwrite else 0
block_ids = new_block_ids[i]
bpk = self.blocks_per_kv_block[i]
if bpk > 1:
block_ids = [b * bpk + k for b in block_ids for k in range(bpk)]
if self.redirect_writes_to_null_block:
block_ids = [0] * len(block_ids)
end = start + len(block_ids)
@@ -218,7 +205,6 @@ class BlockTables:
self.block_table_ptrs,
self.block_table_strides,
self.block_sizes_tensor,
self.kernel_block_sizes_tensor,
self.slot_mapping_enabled,
self.dcp_sharded,
slot_mappings,
@@ -292,7 +278,6 @@ def _compute_slot_mappings_kernel(
block_table_ptrs, # [num_kv_cache_groups]
block_table_strides, # [num_kv_cache_groups]
block_sizes, # [num_kv_cache_groups]
kernel_block_sizes, # [num_kv_cache_groups]
slot_mapping_enabled, # [num_kv_cache_groups]
dcp_sharded, # [num_kv_cache_groups]
slot_mappings_ptr, # [num_kv_cache_groups, max_num_tokens]
@@ -322,7 +307,6 @@ def _compute_slot_mappings_kernel(
block_table_ptr = _load_ptr(block_table_ptrs + group_id, tl.int32)
block_table_stride = tl.load(block_table_strides + group_id)
kv_block_size = tl.load(block_sizes + group_id)
kernel_block_size = tl.load(kernel_block_sizes + group_id)
mapping_enabled = tl.load(slot_mapping_enabled + group_id)
if CP_SIZE != 1:
sharded = tl.load(dcp_sharded + group_id)
@@ -354,16 +338,14 @@ def _compute_slot_mappings_kernel(
local_positions = tl.where(sharded, local_positions, positions)
is_local = ~sharded | is_local
block_indices = tl.where(
mapping_enabled, local_positions // kernel_block_size, 0
)
block_offsets = local_positions % kernel_block_size
block_indices = tl.where(mapping_enabled, local_positions // kv_block_size, 0)
block_offsets = local_positions % kv_block_size
block_numbers = tl.load(
block_table_ptr + req_state_idx * block_table_stride + block_indices,
mask=is_local & is_real_req,
other=0,
)
slot_ids = block_numbers * kernel_block_size + block_offsets
slot_ids = block_numbers * kv_block_size + block_offsets
if CP_SIZE != 1:
slot_ids = tl.where(is_local, slot_ids, PAD_ID)
+9 -8
View File
@@ -252,17 +252,18 @@ def set_dummy_context(
input_batch.positions.copy_(torch.from_numpy(local_pos + context_len))
seq_len = context_len + query_len
for block_table, block_size, bpk in zip(
input_block_tables,
block_tables.kernel_block_sizes,
block_tables.blocks_per_kv_block,
):
for block_table, block_size in zip(input_block_tables, block_tables.block_sizes):
num_blocks = min(cdiv(seq_len, block_size), block_table.shape[1])
# Spans are disjoint until the pool runs out, then they wrap and share
# blocks: profiling only needs the reads to be realistic, not distinct.
block_ids = torch.arange(
num_reqs * num_blocks, dtype=block_table.dtype, device=block_table.device
) % (num_kv_blocks * bpk)
block_ids = (
torch.arange(
num_reqs * num_blocks,
dtype=block_table.dtype,
device=block_table.device,
)
% num_kv_blocks
)
block_table[:num_reqs, :num_blocks] = block_ids.view(num_reqs, num_blocks)
+7 -1
View File
@@ -688,7 +688,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
max_num_batched_tokens=self.max_num_tokens,
max_num_blocks_per_group=max_num_blocks_per_group,
device=self.device,
kernel_block_sizes=self.kernel_block_sizes,
slot_mapping_enabled=slot_mapping_enabled,
dcp_sharded=dcp_sharded,
cp_size=self.dcp_size,
@@ -773,6 +772,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.vllm_config,
kv_cache_allocation_context=kv_cache_allocation_context,
block_tables=self.block_tables,
# Draft groups build their own metadata, so they map the views too.
attn_groups=(
g
for groups in self.attn_groups
+ getattr(self.speculator, "attn_groups", [])
for g in groups
),
)
self.kv_caches = [
cache for cache in kv_caches_dict.values() if cache.device == self.device
@@ -432,7 +432,7 @@ class DFlashSpeculator(DraftModelSpeculator):
temperature,
seeds,
self.block_tables.input_block_tables[gid],
self.block_tables.kernel_block_sizes[gid],
self.block_tables.block_sizes[gid],
self.block_tables.cp_rank,
self.dcp_size,
self.block_tables.cp_interleave,
@@ -577,6 +577,11 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
idx_mapping = self.idx_mapping[:num_reqs]
if batch_desc.cg_mode == CUDAGraphMode.FULL:
for groups in self.attn_groups:
for group in groups:
group.map_to_kernel_block_table(
self.block_tables.input_block_tables[group.kv_cache_group_id]
)
assert self.decode_cudagraph_manager is not None
self.decode_cudagraph_manager.run_fullgraph(batch_desc)
return