[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Dakai An
2026-10-09 15:48:37 -04:00
committed by GitHub
co-authored by Yifan Qiao OpenAI Codex OpenAI Codex Lucas Wilkinson Claude Opus 5.5 Lucas Wilkinson
parent 3766a67a32
commit 02e55d2af3
83 changed files with 1280 additions and 1986 deletions
+21 -20
View File
@@ -233,6 +233,8 @@ from .utils import (
allocate_kv_cache,
bind_kv_cache,
copy_kv_cache_blocks_inplace,
customize_attention_spec,
map_kv_caches_to_kernel_blocks,
prepare_kernel_block_sizes,
sanity_check_mm_encoder_outputs,
)
@@ -718,7 +720,6 @@ class GPUModelRunner(
max(self.max_model_len, self.max_encoder_len), placeholder_block_size
)
self._init_block_sizes = [placeholder_block_size]
self._init_kernel_block_sizes = [placeholder_block_size]
self._init_max_num_blocks = [placeholder_max_num_blocks]
self._init_slot_mapping_modes = [SlotMappingMode.TOKEN_TO_KV_SLOT]
self.cp_kv_cache_interleave_size = (
@@ -734,7 +735,6 @@ class GPUModelRunner(
device=self.device,
vocab_size=self.model_config.get_vocab_size(),
block_sizes=[placeholder_block_size],
kernel_block_sizes=[placeholder_block_size],
max_num_blocks_per_req=[placeholder_max_num_blocks],
num_spec_tokens=self.num_spec_tokens,
logitsprocs=build_logitsprocs(
@@ -2508,8 +2508,8 @@ class GPUModelRunner(
)
if for_cudagraph_capture:
attn_metadata_i = builder.build_for_cudagraph_capture(
common_attn_metadata
attn_metadata_i = attn_group.build_metadata_for_cudagraph_capture(
common_attn_metadata, ubid or 0
)
elif (
cache_key in cached_attn_metadata
@@ -2517,13 +2517,16 @@ class GPUModelRunner(
):
attn_metadata_i = builder.update_block_table(
cached_attn_metadata[cache_key],
common_attn_metadata.block_table_tensor,
attn_group.map_to_kernel_block_table(
common_attn_metadata.block_table_tensor, ubid or 0
),
common_attn_metadata.slot_mapping,
)
else:
attn_metadata_i = builder.build(
attn_metadata_i = attn_group.build_metadata(
common_attn_metadata,
ubid or 0,
common_prefix_len=cascade_attn_prefix_len,
common_attn_metadata=common_attn_metadata,
**extra_attn_metadata_args,
)
if builder.supports_update_block_table:
@@ -7143,9 +7146,7 @@ class GPUModelRunner(
return
self.reorder_batch_threshold = reduce(min_none_high, reorder_batch_thresholds) # type: ignore[assignment]
def may_reinitialize_input_batch(
self, kv_cache_config: KVCacheConfig, kernel_block_sizes: list[int]
) -> None:
def may_reinitialize_input_batch(self, kv_cache_config: KVCacheConfig) -> None:
"""Re-initialize the input batch if the block sizes are different from
what it was originally created with. This happens when the final
block size (determined after model loading) differs from the
@@ -7154,7 +7155,6 @@ class GPUModelRunner(
Args:
kv_cache_config: The KV cache configuration.
kernel_block_sizes: The kernel block sizes for each KV cache group.
"""
block_sizes = []
@@ -7179,14 +7179,12 @@ class GPUModelRunner(
if (
block_sizes != self._init_block_sizes
or kernel_block_sizes != self._init_kernel_block_sizes
or max_num_blocks != self._init_max_num_blocks
or slot_mapping_modes != self._init_slot_mapping_modes
or self.cp_kv_cache_interleave_size
!= self.parallel_config.cp_kv_cache_interleave_size
):
self._init_block_sizes = block_sizes
self._init_kernel_block_sizes = kernel_block_sizes
self._init_max_num_blocks = max_num_blocks
self._init_slot_mapping_modes = slot_mapping_modes
self.cp_kv_cache_interleave_size = (
@@ -7201,7 +7199,6 @@ class GPUModelRunner(
device=self.device,
vocab_size=self.model_config.get_vocab_size(),
block_sizes=block_sizes,
kernel_block_sizes=kernel_block_sizes,
max_num_blocks_per_req=max_num_blocks,
num_spec_tokens=self.num_spec_tokens,
logitsprocs=self.input_batch.logitsprocs,
@@ -7217,10 +7214,6 @@ class GPUModelRunner(
f"InputBatch block_sizes {self._init_block_sizes} != "
f"kv_cache block_sizes {block_sizes}"
)
assert self._init_kernel_block_sizes == kernel_block_sizes, (
f"InputBatch kernel_block_sizes {self._init_kernel_block_sizes} "
f"!= kv_cache kernel_block_sizes {kernel_block_sizes}"
)
def _attn_group_iterator(self) -> Iterator[AttentionGroup]:
return itertools.chain.from_iterable(self.attn_groups)
@@ -7267,12 +7260,16 @@ class GPUModelRunner(
num_attn_module = (
2 if self.model_config.hf_config.model_type == "longcat_flash" else 1
)
attn_groups = [g for groups in self.attn_groups for g in groups]
if self.speculative_config:
attn_groups += getattr(self.drafter, "draft_attn_groups", [])
bind_kv_cache(
kv_caches,
self.compilation_config.static_forward_context,
self.kv_caches,
num_attn_module,
kv_cache_groups=kv_cache_config.kv_cache_groups,
layer_kv_caches=map_kv_caches_to_kernel_blocks(kv_caches, attn_groups),
)
return kv_caches
@@ -7336,6 +7333,8 @@ class GPUModelRunner(
# backends for that group only supports block_size 64, we will return
# kernel_block_size 64 and split the 256-token-block to 4 blocks with 64
# tokens each.
# Block-outermost packed groups are not split; their attention groups
# map kernel blocks themselves.
kernel_block_sizes = prepare_kernel_block_sizes(
kv_cache_config, self.attn_groups
)
@@ -7345,7 +7344,7 @@ class GPUModelRunner(
self.initialize_metadata_builders(kv_cache_config, kernel_block_sizes)
# Reinitialize need to after initialize_attn_backend
self.may_reinitialize_input_batch(kv_cache_config, kernel_block_sizes)
self.may_reinitialize_input_batch(kv_cache_config)
# Capture warmup providers that depend on allocated KV-cache strides.
with self.jit_warmup_registry.activate(): # type: ignore[attr-defined]
kv_caches = self.initialize_kv_cache_tensors(
@@ -7422,7 +7421,9 @@ class GPUModelRunner(
# Skip modules that don't need KV cache (eg encoder-only attention)
if spec := attn_module.get_kv_cache_spec(self.vllm_config):
if isinstance(spec, AttentionSpec):
spec = attn_module.get_attn_backend().customize_spec(spec)
spec = customize_attention_spec(
attn_module.get_attn_backend(), spec
)
kv_cache_spec[layer_name] = spec
return kv_cache_spec