[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Dakai An
2026-10-09 15:48:37 -04:00
committed by GitHub
co-authored by Yifan Qiao OpenAI Codex OpenAI Codex Lucas Wilkinson Claude Opus 5.5 Lucas Wilkinson
parent 3766a67a32
commit 02e55d2af3
83 changed files with 1280 additions and 1986 deletions
+4 -22
View File
@@ -25,7 +25,6 @@ class BlockTables:
max_num_batched_tokens: int,
max_num_blocks_per_group: list[int],
device: torch.device,
kernel_block_sizes: list[int],
cp_size: int = 1,
cp_rank: int = 0,
cp_interleave: int = 1,
@@ -33,7 +32,6 @@ class BlockTables:
dcp_sharded: list[bool] | None = None,
):
self.block_sizes = block_sizes
self.kernel_block_sizes = kernel_block_sizes
self.max_num_reqs = max_num_reqs
self.max_num_batched_tokens = max_num_batched_tokens
self.device = device
@@ -53,14 +51,9 @@ class BlockTables:
assert len(dcp_sharded) == self.num_kv_cache_groups
self.dcp_sharded = torch.tensor(dcp_sharded, dtype=torch.bool, device=device)
self.blocks_per_kv_block = [
bs // kbs for bs, kbs in zip(block_sizes, kernel_block_sizes)
]
# num_kv_cache_groups x [max_num_reqs, max_num_blocks]
self.block_tables: list[StagedWriteTensor] = []
for i in range(self.num_kv_cache_groups):
max_num_blocks = max_num_blocks_per_group[i] * self.blocks_per_kv_block[i]
for max_num_blocks in max_num_blocks_per_group:
block_table = StagedWriteTensor(
(self.max_num_reqs, max_num_blocks), dtype=torch.int32, device=device
)
@@ -109,9 +102,6 @@ class BlockTables:
self.block_sizes_tensor = torch.tensor(
self.block_sizes, dtype=torch.int32, device=self.device
)
self.kernel_block_sizes_tensor = torch.tensor(
self.kernel_block_sizes, dtype=torch.int32, device=self.device
)
self.slot_mapping_enabled = torch.tensor(
self._slot_mapping_enabled, dtype=torch.bool, device=self.device
)
@@ -126,9 +116,6 @@ class BlockTables:
for i in range(self.num_kv_cache_groups):
start = self.num_blocks.np[i, req_index] if not overwrite else 0
block_ids = new_block_ids[i]
bpk = self.blocks_per_kv_block[i]
if bpk > 1:
block_ids = [b * bpk + k for b in block_ids for k in range(bpk)]
if self.redirect_writes_to_null_block:
block_ids = [0] * len(block_ids)
end = start + len(block_ids)
@@ -218,7 +205,6 @@ class BlockTables:
self.block_table_ptrs,
self.block_table_strides,
self.block_sizes_tensor,
self.kernel_block_sizes_tensor,
self.slot_mapping_enabled,
self.dcp_sharded,
slot_mappings,
@@ -292,7 +278,6 @@ def _compute_slot_mappings_kernel(
block_table_ptrs, # [num_kv_cache_groups]
block_table_strides, # [num_kv_cache_groups]
block_sizes, # [num_kv_cache_groups]
kernel_block_sizes, # [num_kv_cache_groups]
slot_mapping_enabled, # [num_kv_cache_groups]
dcp_sharded, # [num_kv_cache_groups]
slot_mappings_ptr, # [num_kv_cache_groups, max_num_tokens]
@@ -322,7 +307,6 @@ def _compute_slot_mappings_kernel(
block_table_ptr = _load_ptr(block_table_ptrs + group_id, tl.int32)
block_table_stride = tl.load(block_table_strides + group_id)
kv_block_size = tl.load(block_sizes + group_id)
kernel_block_size = tl.load(kernel_block_sizes + group_id)
mapping_enabled = tl.load(slot_mapping_enabled + group_id)
if CP_SIZE != 1:
sharded = tl.load(dcp_sharded + group_id)
@@ -354,16 +338,14 @@ def _compute_slot_mappings_kernel(
local_positions = tl.where(sharded, local_positions, positions)
is_local = ~sharded | is_local
block_indices = tl.where(
mapping_enabled, local_positions // kernel_block_size, 0
)
block_offsets = local_positions % kernel_block_size
block_indices = tl.where(mapping_enabled, local_positions // kv_block_size, 0)
block_offsets = local_positions % kv_block_size
block_numbers = tl.load(
block_table_ptr + req_state_idx * block_table_stride + block_indices,
mask=is_local & is_real_req,
other=0,
)
slot_ids = block_numbers * kernel_block_size + block_offsets
slot_ids = block_numbers * kv_block_size + block_offsets
if CP_SIZE != 1:
slot_ids = tl.where(is_local, slot_ids, PAD_ID)