[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
OpenAI Codex
OpenAI Codex
Lucas Wilkinson
Claude Opus 5.5
Lucas Wilkinson
parent
3766a67a32
commit
02e55d2af3
@@ -252,17 +252,18 @@ def set_dummy_context(
|
||||
input_batch.positions.copy_(torch.from_numpy(local_pos + context_len))
|
||||
|
||||
seq_len = context_len + query_len
|
||||
for block_table, block_size, bpk in zip(
|
||||
input_block_tables,
|
||||
block_tables.kernel_block_sizes,
|
||||
block_tables.blocks_per_kv_block,
|
||||
):
|
||||
for block_table, block_size in zip(input_block_tables, block_tables.block_sizes):
|
||||
num_blocks = min(cdiv(seq_len, block_size), block_table.shape[1])
|
||||
# Spans are disjoint until the pool runs out, then they wrap and share
|
||||
# blocks: profiling only needs the reads to be realistic, not distinct.
|
||||
block_ids = torch.arange(
|
||||
num_reqs * num_blocks, dtype=block_table.dtype, device=block_table.device
|
||||
) % (num_kv_blocks * bpk)
|
||||
block_ids = (
|
||||
torch.arange(
|
||||
num_reqs * num_blocks,
|
||||
dtype=block_table.dtype,
|
||||
device=block_table.device,
|
||||
)
|
||||
% num_kv_blocks
|
||||
)
|
||||
block_table[:num_reqs, :num_blocks] = block_ids.view(num_reqs, num_blocks)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user