[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
OpenAI Codex
OpenAI Codex
Lucas Wilkinson
Claude Opus 5.5
Lucas Wilkinson
parent
3766a67a32
commit
02e55d2af3
@@ -95,7 +95,7 @@ class Gemma4Proposer(SpecDecodeBaseProposer):
|
||||
cm.block_table_tensor = self._per_group_block_tables[gid][:batch_size]
|
||||
else:
|
||||
cm = common_attn_metadata
|
||||
attn_metadata = attn_group.get_metadata_builder().build_for_drafting(
|
||||
attn_metadata = attn_group.build_metadata_for_drafting(
|
||||
common_attn_metadata=cm, draft_index=draft_index
|
||||
)
|
||||
per_group_attn_metadata.append(attn_metadata)
|
||||
@@ -291,11 +291,7 @@ class Gemma4Proposer(SpecDecodeBaseProposer):
|
||||
self.draft_attn_groups = list(attention_groups.values())
|
||||
if self.draft_attn_groups:
|
||||
self.kv_cache_gid = self.draft_attn_groups[0].kv_cache_group_id
|
||||
self.block_size = (
|
||||
self.draft_attn_groups[0]
|
||||
.get_metadata_builder()
|
||||
.kv_cache_spec.block_size
|
||||
)
|
||||
self.block_size = self.draft_attn_groups[0].kv_cache_spec.block_size
|
||||
else:
|
||||
self.kv_cache_gid = 0
|
||||
self.block_size = kv_cache_config.kv_cache_groups[
|
||||
|
||||
@@ -991,7 +991,7 @@ class SpecDecodeBaseProposer:
|
||||
per_group_attn_metadata: list[object] = []
|
||||
per_layer_attn_metadata: dict[str, object] = {}
|
||||
for attn_group in self.draft_attn_groups:
|
||||
attn_metadata = attn_group.get_metadata_builder().build_for_drafting(
|
||||
attn_metadata = attn_group.build_metadata_for_drafting(
|
||||
common_attn_metadata=common_attn_metadata, draft_index=draft_index
|
||||
)
|
||||
per_group_attn_metadata.append(attn_metadata)
|
||||
@@ -1765,20 +1765,7 @@ class SpecDecodeBaseProposer:
|
||||
attention_groups[backend_key].layer_names.append(layer_name)
|
||||
|
||||
self.draft_attn_groups = list(attention_groups.values())
|
||||
if kernel_block_sizes is not None and 0 <= self.kv_cache_gid < len(
|
||||
kernel_block_sizes
|
||||
):
|
||||
# Slot mappings are computed against the block table, which is
|
||||
# stored at kernel-block granularity. Use the kernel block size
|
||||
# rather than the KV cache manager's block size; the two differ
|
||||
# when manager blocks are split for the attention kernel.
|
||||
self.block_size = kernel_block_sizes[self.kv_cache_gid]
|
||||
else:
|
||||
self.block_size = (
|
||||
self.draft_attn_groups[0]
|
||||
.get_metadata_builder()
|
||||
.kv_cache_spec.block_size
|
||||
)
|
||||
self.block_size = self.draft_attn_groups[0].kv_cache_spec.block_size
|
||||
logger.debug("Using block size %d for drafting layers", self.block_size)
|
||||
|
||||
def _determine_batch_execution_and_padding(
|
||||
|
||||
@@ -144,7 +144,7 @@ class Step3p5MTPProposer(EagleProposer):
|
||||
cm.slot_mapping = sm
|
||||
else:
|
||||
cm = common_attn_metadata
|
||||
attn_metadata = attn_group.get_metadata_builder().build_for_drafting(
|
||||
attn_metadata = attn_group.build_metadata_for_drafting(
|
||||
common_attn_metadata=cm,
|
||||
draft_index=draft_index,
|
||||
)
|
||||
@@ -244,11 +244,7 @@ class Step3p5MTPProposer(EagleProposer):
|
||||
self.draft_attn_groups = list(attention_groups.values())
|
||||
if self.draft_attn_groups:
|
||||
self.kv_cache_gid = self.draft_attn_groups[0].kv_cache_group_id
|
||||
self.block_size = (
|
||||
self.draft_attn_groups[0]
|
||||
.get_metadata_builder()
|
||||
.kv_cache_spec.block_size
|
||||
)
|
||||
self.block_size = self.draft_attn_groups[0].kv_cache_spec.block_size
|
||||
else:
|
||||
self.kv_cache_gid = 0
|
||||
self.block_size = kv_cache_config.kv_cache_groups[
|
||||
|
||||
Reference in New Issue
Block a user