[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Dakai An
2026-10-09 15:48:37 -04:00
committed by GitHub
co-authored by Yifan Qiao OpenAI Codex OpenAI Codex Lucas Wilkinson Claude Opus 5.5 Lucas Wilkinson
parent 3766a67a32
commit 02e55d2af3
83 changed files with 1280 additions and 1986 deletions
+9 -8
View File
@@ -252,17 +252,18 @@ def set_dummy_context(
input_batch.positions.copy_(torch.from_numpy(local_pos + context_len))
seq_len = context_len + query_len
for block_table, block_size, bpk in zip(
input_block_tables,
block_tables.kernel_block_sizes,
block_tables.blocks_per_kv_block,
):
for block_table, block_size in zip(input_block_tables, block_tables.block_sizes):
num_blocks = min(cdiv(seq_len, block_size), block_table.shape[1])
# Spans are disjoint until the pool runs out, then they wrap and share
# blocks: profiling only needs the reads to be realistic, not distinct.
block_ids = torch.arange(
num_reqs * num_blocks, dtype=block_table.dtype, device=block_table.device
) % (num_kv_blocks * bpk)
block_ids = (
torch.arange(
num_reqs * num_blocks,
dtype=block_table.dtype,
device=block_table.device,
)
% num_kv_blocks
)
block_table[:num_reqs, :num_blocks] = block_ids.view(num_reqs, num_blocks)