[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
OpenAI Codex
OpenAI Codex
Lucas Wilkinson
Claude Opus 5.5
Lucas Wilkinson
parent
3766a67a32
commit
02e55d2af3
@@ -233,6 +233,8 @@ from .utils import (
|
||||
allocate_kv_cache,
|
||||
bind_kv_cache,
|
||||
copy_kv_cache_blocks_inplace,
|
||||
customize_attention_spec,
|
||||
map_kv_caches_to_kernel_blocks,
|
||||
prepare_kernel_block_sizes,
|
||||
sanity_check_mm_encoder_outputs,
|
||||
)
|
||||
@@ -718,7 +720,6 @@ class GPUModelRunner(
|
||||
max(self.max_model_len, self.max_encoder_len), placeholder_block_size
|
||||
)
|
||||
self._init_block_sizes = [placeholder_block_size]
|
||||
self._init_kernel_block_sizes = [placeholder_block_size]
|
||||
self._init_max_num_blocks = [placeholder_max_num_blocks]
|
||||
self._init_slot_mapping_modes = [SlotMappingMode.TOKEN_TO_KV_SLOT]
|
||||
self.cp_kv_cache_interleave_size = (
|
||||
@@ -734,7 +735,6 @@ class GPUModelRunner(
|
||||
device=self.device,
|
||||
vocab_size=self.model_config.get_vocab_size(),
|
||||
block_sizes=[placeholder_block_size],
|
||||
kernel_block_sizes=[placeholder_block_size],
|
||||
max_num_blocks_per_req=[placeholder_max_num_blocks],
|
||||
num_spec_tokens=self.num_spec_tokens,
|
||||
logitsprocs=build_logitsprocs(
|
||||
@@ -2508,8 +2508,8 @@ class GPUModelRunner(
|
||||
)
|
||||
|
||||
if for_cudagraph_capture:
|
||||
attn_metadata_i = builder.build_for_cudagraph_capture(
|
||||
common_attn_metadata
|
||||
attn_metadata_i = attn_group.build_metadata_for_cudagraph_capture(
|
||||
common_attn_metadata, ubid or 0
|
||||
)
|
||||
elif (
|
||||
cache_key in cached_attn_metadata
|
||||
@@ -2517,13 +2517,16 @@ class GPUModelRunner(
|
||||
):
|
||||
attn_metadata_i = builder.update_block_table(
|
||||
cached_attn_metadata[cache_key],
|
||||
common_attn_metadata.block_table_tensor,
|
||||
attn_group.map_to_kernel_block_table(
|
||||
common_attn_metadata.block_table_tensor, ubid or 0
|
||||
),
|
||||
common_attn_metadata.slot_mapping,
|
||||
)
|
||||
else:
|
||||
attn_metadata_i = builder.build(
|
||||
attn_metadata_i = attn_group.build_metadata(
|
||||
common_attn_metadata,
|
||||
ubid or 0,
|
||||
common_prefix_len=cascade_attn_prefix_len,
|
||||
common_attn_metadata=common_attn_metadata,
|
||||
**extra_attn_metadata_args,
|
||||
)
|
||||
if builder.supports_update_block_table:
|
||||
@@ -7143,9 +7146,7 @@ class GPUModelRunner(
|
||||
return
|
||||
self.reorder_batch_threshold = reduce(min_none_high, reorder_batch_thresholds) # type: ignore[assignment]
|
||||
|
||||
def may_reinitialize_input_batch(
|
||||
self, kv_cache_config: KVCacheConfig, kernel_block_sizes: list[int]
|
||||
) -> None:
|
||||
def may_reinitialize_input_batch(self, kv_cache_config: KVCacheConfig) -> None:
|
||||
"""Re-initialize the input batch if the block sizes are different from
|
||||
what it was originally created with. This happens when the final
|
||||
block size (determined after model loading) differs from the
|
||||
@@ -7154,7 +7155,6 @@ class GPUModelRunner(
|
||||
|
||||
Args:
|
||||
kv_cache_config: The KV cache configuration.
|
||||
kernel_block_sizes: The kernel block sizes for each KV cache group.
|
||||
|
||||
"""
|
||||
block_sizes = []
|
||||
@@ -7179,14 +7179,12 @@ class GPUModelRunner(
|
||||
|
||||
if (
|
||||
block_sizes != self._init_block_sizes
|
||||
or kernel_block_sizes != self._init_kernel_block_sizes
|
||||
or max_num_blocks != self._init_max_num_blocks
|
||||
or slot_mapping_modes != self._init_slot_mapping_modes
|
||||
or self.cp_kv_cache_interleave_size
|
||||
!= self.parallel_config.cp_kv_cache_interleave_size
|
||||
):
|
||||
self._init_block_sizes = block_sizes
|
||||
self._init_kernel_block_sizes = kernel_block_sizes
|
||||
self._init_max_num_blocks = max_num_blocks
|
||||
self._init_slot_mapping_modes = slot_mapping_modes
|
||||
self.cp_kv_cache_interleave_size = (
|
||||
@@ -7201,7 +7199,6 @@ class GPUModelRunner(
|
||||
device=self.device,
|
||||
vocab_size=self.model_config.get_vocab_size(),
|
||||
block_sizes=block_sizes,
|
||||
kernel_block_sizes=kernel_block_sizes,
|
||||
max_num_blocks_per_req=max_num_blocks,
|
||||
num_spec_tokens=self.num_spec_tokens,
|
||||
logitsprocs=self.input_batch.logitsprocs,
|
||||
@@ -7217,10 +7214,6 @@ class GPUModelRunner(
|
||||
f"InputBatch block_sizes {self._init_block_sizes} != "
|
||||
f"kv_cache block_sizes {block_sizes}"
|
||||
)
|
||||
assert self._init_kernel_block_sizes == kernel_block_sizes, (
|
||||
f"InputBatch kernel_block_sizes {self._init_kernel_block_sizes} "
|
||||
f"!= kv_cache kernel_block_sizes {kernel_block_sizes}"
|
||||
)
|
||||
|
||||
def _attn_group_iterator(self) -> Iterator[AttentionGroup]:
|
||||
return itertools.chain.from_iterable(self.attn_groups)
|
||||
@@ -7267,12 +7260,16 @@ class GPUModelRunner(
|
||||
num_attn_module = (
|
||||
2 if self.model_config.hf_config.model_type == "longcat_flash" else 1
|
||||
)
|
||||
attn_groups = [g for groups in self.attn_groups for g in groups]
|
||||
if self.speculative_config:
|
||||
attn_groups += getattr(self.drafter, "draft_attn_groups", [])
|
||||
bind_kv_cache(
|
||||
kv_caches,
|
||||
self.compilation_config.static_forward_context,
|
||||
self.kv_caches,
|
||||
num_attn_module,
|
||||
kv_cache_groups=kv_cache_config.kv_cache_groups,
|
||||
layer_kv_caches=map_kv_caches_to_kernel_blocks(kv_caches, attn_groups),
|
||||
)
|
||||
return kv_caches
|
||||
|
||||
@@ -7336,6 +7333,8 @@ class GPUModelRunner(
|
||||
# backends for that group only supports block_size 64, we will return
|
||||
# kernel_block_size 64 and split the 256-token-block to 4 blocks with 64
|
||||
# tokens each.
|
||||
# Block-outermost packed groups are not split; their attention groups
|
||||
# map kernel blocks themselves.
|
||||
kernel_block_sizes = prepare_kernel_block_sizes(
|
||||
kv_cache_config, self.attn_groups
|
||||
)
|
||||
@@ -7345,7 +7344,7 @@ class GPUModelRunner(
|
||||
self.initialize_metadata_builders(kv_cache_config, kernel_block_sizes)
|
||||
|
||||
# Reinitialize need to after initialize_attn_backend
|
||||
self.may_reinitialize_input_batch(kv_cache_config, kernel_block_sizes)
|
||||
self.may_reinitialize_input_batch(kv_cache_config)
|
||||
# Capture warmup providers that depend on allocated KV-cache strides.
|
||||
with self.jit_warmup_registry.activate(): # type: ignore[attr-defined]
|
||||
kv_caches = self.initialize_kv_cache_tensors(
|
||||
@@ -7422,7 +7421,9 @@ class GPUModelRunner(
|
||||
# Skip modules that don't need KV cache (eg encoder-only attention)
|
||||
if spec := attn_module.get_kv_cache_spec(self.vllm_config):
|
||||
if isinstance(spec, AttentionSpec):
|
||||
spec = attn_module.get_attn_backend().customize_spec(spec)
|
||||
spec = customize_attention_spec(
|
||||
attn_module.get_attn_backend(), spec
|
||||
)
|
||||
kv_cache_spec[layer_name] = spec
|
||||
|
||||
return kv_cache_spec
|
||||
|
||||
Reference in New Issue
Block a user