[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
OpenAI Codex
OpenAI Codex
Lucas Wilkinson
Claude Opus 5.5
Lucas Wilkinson
parent
3766a67a32
commit
02e55d2af3
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
@@ -40,6 +40,8 @@ from vllm.v1.worker.utils import (
|
||||
add_kv_sharing_layers_to_kv_cache_groups,
|
||||
allocate_kv_cache,
|
||||
bind_kv_cache_to_layers,
|
||||
customize_attention_spec,
|
||||
map_kv_caches_to_kernel_blocks,
|
||||
prepare_kernel_block_sizes,
|
||||
)
|
||||
|
||||
@@ -141,7 +143,7 @@ def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]:
|
||||
# Skip modules that don't need KV cache (eg encoder-only attention)
|
||||
if spec := attn_module.get_kv_cache_spec(vllm_config):
|
||||
if isinstance(spec, AttentionSpec):
|
||||
spec = attn_module.get_attn_backend().customize_spec(spec)
|
||||
spec = customize_attention_spec(attn_module.get_attn_backend(), spec)
|
||||
kv_cache_spec[layer_name] = spec
|
||||
if vllm_config.attention_config.hisparse_config is not None:
|
||||
kv_cache_spec = resolve_hisparse_specs(vllm_config, kv_cache_spec, attn_layers)
|
||||
@@ -382,6 +384,7 @@ def init_kv_cache(
|
||||
kv_cache_allocation_context: AbstractContextManager | None = None,
|
||||
*,
|
||||
block_tables: "BlockTables | None" = None,
|
||||
attn_groups: Iterable[AttentionGroup] = (),
|
||||
) -> dict[str, Any]:
|
||||
allocation_context = kv_cache_allocation_context or nullcontext()
|
||||
with allocation_context:
|
||||
@@ -409,9 +412,10 @@ def init_kv_cache(
|
||||
num_attn_module = (
|
||||
2 if vllm_config.model_config.hf_config.model_type == "longcat_flash" else 1
|
||||
)
|
||||
bindable_caches = {
|
||||
name: cache for name, cache in kv_caches.items() if name in forward_context
|
||||
}
|
||||
bindable_caches = map_kv_caches_to_kernel_blocks(
|
||||
{name: cache for name, cache in kv_caches.items() if name in forward_context},
|
||||
attn_groups,
|
||||
)
|
||||
bind_kv_cache_to_layers(
|
||||
bindable_caches,
|
||||
forward_context,
|
||||
@@ -536,8 +540,8 @@ def build_attn_metadata(
|
||||
cached_metadata[reuse_key], block_table, slot_mapping
|
||||
)
|
||||
elif for_cudagraph_capture:
|
||||
metadata = attn_metadata_builder.build_for_cudagraph_capture(
|
||||
common_attn_metadata
|
||||
metadata = attn_group.build_metadata_for_cudagraph_capture(
|
||||
common_attn_metadata, ubatch_idx
|
||||
)
|
||||
else:
|
||||
attn_metadata_extra_kwargs = (
|
||||
@@ -548,10 +552,8 @@ def build_attn_metadata(
|
||||
if model_specific_attn_metadata is not None
|
||||
else {}
|
||||
)
|
||||
metadata = attn_metadata_builder.build(
|
||||
common_prefix_len=0,
|
||||
common_attn_metadata=common_attn_metadata,
|
||||
**attn_metadata_extra_kwargs,
|
||||
metadata = attn_group.build_metadata(
|
||||
common_attn_metadata, ubatch_idx, **attn_metadata_extra_kwargs
|
||||
)
|
||||
if reuse_key is not None and reuse_key not in cached_metadata:
|
||||
cached_metadata[reuse_key] = metadata
|
||||
|
||||
@@ -25,7 +25,6 @@ class BlockTables:
|
||||
max_num_batched_tokens: int,
|
||||
max_num_blocks_per_group: list[int],
|
||||
device: torch.device,
|
||||
kernel_block_sizes: list[int],
|
||||
cp_size: int = 1,
|
||||
cp_rank: int = 0,
|
||||
cp_interleave: int = 1,
|
||||
@@ -33,7 +32,6 @@ class BlockTables:
|
||||
dcp_sharded: list[bool] | None = None,
|
||||
):
|
||||
self.block_sizes = block_sizes
|
||||
self.kernel_block_sizes = kernel_block_sizes
|
||||
self.max_num_reqs = max_num_reqs
|
||||
self.max_num_batched_tokens = max_num_batched_tokens
|
||||
self.device = device
|
||||
@@ -53,14 +51,9 @@ class BlockTables:
|
||||
assert len(dcp_sharded) == self.num_kv_cache_groups
|
||||
self.dcp_sharded = torch.tensor(dcp_sharded, dtype=torch.bool, device=device)
|
||||
|
||||
self.blocks_per_kv_block = [
|
||||
bs // kbs for bs, kbs in zip(block_sizes, kernel_block_sizes)
|
||||
]
|
||||
|
||||
# num_kv_cache_groups x [max_num_reqs, max_num_blocks]
|
||||
self.block_tables: list[StagedWriteTensor] = []
|
||||
for i in range(self.num_kv_cache_groups):
|
||||
max_num_blocks = max_num_blocks_per_group[i] * self.blocks_per_kv_block[i]
|
||||
for max_num_blocks in max_num_blocks_per_group:
|
||||
block_table = StagedWriteTensor(
|
||||
(self.max_num_reqs, max_num_blocks), dtype=torch.int32, device=device
|
||||
)
|
||||
@@ -109,9 +102,6 @@ class BlockTables:
|
||||
self.block_sizes_tensor = torch.tensor(
|
||||
self.block_sizes, dtype=torch.int32, device=self.device
|
||||
)
|
||||
self.kernel_block_sizes_tensor = torch.tensor(
|
||||
self.kernel_block_sizes, dtype=torch.int32, device=self.device
|
||||
)
|
||||
self.slot_mapping_enabled = torch.tensor(
|
||||
self._slot_mapping_enabled, dtype=torch.bool, device=self.device
|
||||
)
|
||||
@@ -126,9 +116,6 @@ class BlockTables:
|
||||
for i in range(self.num_kv_cache_groups):
|
||||
start = self.num_blocks.np[i, req_index] if not overwrite else 0
|
||||
block_ids = new_block_ids[i]
|
||||
bpk = self.blocks_per_kv_block[i]
|
||||
if bpk > 1:
|
||||
block_ids = [b * bpk + k for b in block_ids for k in range(bpk)]
|
||||
if self.redirect_writes_to_null_block:
|
||||
block_ids = [0] * len(block_ids)
|
||||
end = start + len(block_ids)
|
||||
@@ -218,7 +205,6 @@ class BlockTables:
|
||||
self.block_table_ptrs,
|
||||
self.block_table_strides,
|
||||
self.block_sizes_tensor,
|
||||
self.kernel_block_sizes_tensor,
|
||||
self.slot_mapping_enabled,
|
||||
self.dcp_sharded,
|
||||
slot_mappings,
|
||||
@@ -292,7 +278,6 @@ def _compute_slot_mappings_kernel(
|
||||
block_table_ptrs, # [num_kv_cache_groups]
|
||||
block_table_strides, # [num_kv_cache_groups]
|
||||
block_sizes, # [num_kv_cache_groups]
|
||||
kernel_block_sizes, # [num_kv_cache_groups]
|
||||
slot_mapping_enabled, # [num_kv_cache_groups]
|
||||
dcp_sharded, # [num_kv_cache_groups]
|
||||
slot_mappings_ptr, # [num_kv_cache_groups, max_num_tokens]
|
||||
@@ -322,7 +307,6 @@ def _compute_slot_mappings_kernel(
|
||||
block_table_ptr = _load_ptr(block_table_ptrs + group_id, tl.int32)
|
||||
block_table_stride = tl.load(block_table_strides + group_id)
|
||||
kv_block_size = tl.load(block_sizes + group_id)
|
||||
kernel_block_size = tl.load(kernel_block_sizes + group_id)
|
||||
mapping_enabled = tl.load(slot_mapping_enabled + group_id)
|
||||
if CP_SIZE != 1:
|
||||
sharded = tl.load(dcp_sharded + group_id)
|
||||
@@ -354,16 +338,14 @@ def _compute_slot_mappings_kernel(
|
||||
local_positions = tl.where(sharded, local_positions, positions)
|
||||
is_local = ~sharded | is_local
|
||||
|
||||
block_indices = tl.where(
|
||||
mapping_enabled, local_positions // kernel_block_size, 0
|
||||
)
|
||||
block_offsets = local_positions % kernel_block_size
|
||||
block_indices = tl.where(mapping_enabled, local_positions // kv_block_size, 0)
|
||||
block_offsets = local_positions % kv_block_size
|
||||
block_numbers = tl.load(
|
||||
block_table_ptr + req_state_idx * block_table_stride + block_indices,
|
||||
mask=is_local & is_real_req,
|
||||
other=0,
|
||||
)
|
||||
slot_ids = block_numbers * kernel_block_size + block_offsets
|
||||
slot_ids = block_numbers * kv_block_size + block_offsets
|
||||
if CP_SIZE != 1:
|
||||
slot_ids = tl.where(is_local, slot_ids, PAD_ID)
|
||||
|
||||
|
||||
@@ -252,17 +252,18 @@ def set_dummy_context(
|
||||
input_batch.positions.copy_(torch.from_numpy(local_pos + context_len))
|
||||
|
||||
seq_len = context_len + query_len
|
||||
for block_table, block_size, bpk in zip(
|
||||
input_block_tables,
|
||||
block_tables.kernel_block_sizes,
|
||||
block_tables.blocks_per_kv_block,
|
||||
):
|
||||
for block_table, block_size in zip(input_block_tables, block_tables.block_sizes):
|
||||
num_blocks = min(cdiv(seq_len, block_size), block_table.shape[1])
|
||||
# Spans are disjoint until the pool runs out, then they wrap and share
|
||||
# blocks: profiling only needs the reads to be realistic, not distinct.
|
||||
block_ids = torch.arange(
|
||||
num_reqs * num_blocks, dtype=block_table.dtype, device=block_table.device
|
||||
) % (num_kv_blocks * bpk)
|
||||
block_ids = (
|
||||
torch.arange(
|
||||
num_reqs * num_blocks,
|
||||
dtype=block_table.dtype,
|
||||
device=block_table.device,
|
||||
)
|
||||
% num_kv_blocks
|
||||
)
|
||||
block_table[:num_reqs, :num_blocks] = block_ids.view(num_reqs, num_blocks)
|
||||
|
||||
|
||||
|
||||
@@ -688,7 +688,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
max_num_batched_tokens=self.max_num_tokens,
|
||||
max_num_blocks_per_group=max_num_blocks_per_group,
|
||||
device=self.device,
|
||||
kernel_block_sizes=self.kernel_block_sizes,
|
||||
slot_mapping_enabled=slot_mapping_enabled,
|
||||
dcp_sharded=dcp_sharded,
|
||||
cp_size=self.dcp_size,
|
||||
@@ -773,6 +772,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.vllm_config,
|
||||
kv_cache_allocation_context=kv_cache_allocation_context,
|
||||
block_tables=self.block_tables,
|
||||
# Draft groups build their own metadata, so they map the views too.
|
||||
attn_groups=(
|
||||
g
|
||||
for groups in self.attn_groups
|
||||
+ getattr(self.speculator, "attn_groups", [])
|
||||
for g in groups
|
||||
),
|
||||
)
|
||||
self.kv_caches = [
|
||||
cache for cache in kv_caches_dict.values() if cache.device == self.device
|
||||
|
||||
@@ -432,7 +432,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
temperature,
|
||||
seeds,
|
||||
self.block_tables.input_block_tables[gid],
|
||||
self.block_tables.kernel_block_sizes[gid],
|
||||
self.block_tables.block_sizes[gid],
|
||||
self.block_tables.cp_rank,
|
||||
self.dcp_size,
|
||||
self.block_tables.cp_interleave,
|
||||
|
||||
@@ -577,6 +577,11 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
|
||||
idx_mapping = self.idx_mapping[:num_reqs]
|
||||
|
||||
if batch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
for groups in self.attn_groups:
|
||||
for group in groups:
|
||||
group.map_to_kernel_block_table(
|
||||
self.block_tables.input_block_tables[group.kv_cache_group_id]
|
||||
)
|
||||
assert self.decode_cudagraph_manager is not None
|
||||
self.decode_cudagraph_manager.run_fullgraph(batch_desc)
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user