[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Dakai An
2026-10-09 15:48:37 -04:00
committed by GitHub
co-authored by Yifan Qiao OpenAI Codex OpenAI Codex Lucas Wilkinson Claude Opus 5.5 Lucas Wilkinson
parent 3766a67a32
commit 02e55d2af3
83 changed files with 1280 additions and 1986 deletions
+11 -20
View File
@@ -22,7 +22,6 @@ def test_block_tables_apply_staged_writes_fuses_kv_groups(monkeypatch):
max_num_batched_tokens=64,
max_num_blocks_per_group=[8, 8, 8],
device=device,
kernel_block_sizes=[16, 16, 8],
)
def fail_if_apply_write_called():
@@ -48,33 +47,31 @@ def test_block_tables_apply_staged_writes_fuses_kv_groups(monkeypatch):
block_tables.block_tables[0].gpu[0, :2],
torch.tensor([1, 2], dtype=torch.int32, device=device),
)
# Group 1 has blocks_per_kv_block == 2, so each KV block expands to two
# kernel block IDs.
assert torch.equal(
block_tables.block_tables[1].gpu[0, :4],
torch.tensor([20, 21, 22, 23], dtype=torch.int32, device=device),
block_tables.block_tables[1].gpu[0, :2],
torch.tensor([10, 11], dtype=torch.int32, device=device),
)
assert torch.equal(
block_tables.block_tables[0].gpu[1, :1],
torch.tensor([3], dtype=torch.int32, device=device),
)
assert torch.equal(
block_tables.block_tables[1].gpu[1, :2],
torch.tensor([24, 25], dtype=torch.int32, device=device),
block_tables.block_tables[1].gpu[1, :1],
torch.tensor([12], dtype=torch.int32, device=device),
)
assert torch.equal(
block_tables.block_tables[2].gpu[1, :2],
torch.tensor([5, 6], dtype=torch.int32, device=device),
)
assert block_tables.num_blocks.np[0, 0] == 2
assert block_tables.num_blocks.np[1, 0] == 4
assert block_tables.num_blocks.np[1, 0] == 2
assert block_tables.num_blocks.np[2, 0] == 0
assert block_tables.num_blocks.np[0, 1] == 1
assert block_tables.num_blocks.np[1, 1] == 2
assert block_tables.num_blocks.np[1, 1] == 1
assert block_tables.num_blocks.np[2, 1] == 2
assert torch.equal(
block_tables.num_blocks.gpu[:, :2],
torch.tensor([[2, 1], [4, 2], [0, 2]], dtype=torch.int32, device=device),
torch.tensor([[2, 1], [2, 1], [0, 2]], dtype=torch.int32, device=device),
)
for block_table in block_tables.block_tables:
@@ -96,15 +93,15 @@ def test_block_tables_apply_staged_writes_fuses_kv_groups(monkeypatch):
torch.tensor([1, 2, 7], dtype=torch.int32, device=device),
)
assert torch.equal(
block_tables.block_tables[1].gpu[0, :6],
torch.tensor([20, 21, 22, 23, 26, 27], dtype=torch.int32, device=device),
block_tables.block_tables[1].gpu[0, :3],
torch.tensor([10, 11, 13], dtype=torch.int32, device=device),
)
assert torch.equal(
block_tables.block_tables[2].gpu[0, :1],
torch.tensor([8], dtype=torch.int32, device=device),
)
assert block_tables.num_blocks.np[0, 0] == 3
assert block_tables.num_blocks.np[1, 0] == 6
assert block_tables.num_blocks.np[1, 0] == 3
assert block_tables.num_blocks.np[2, 0] == 1
@@ -116,7 +113,6 @@ def test_block_tables_apply_staged_writes_single_group():
max_num_batched_tokens=16,
max_num_blocks_per_group=[4],
device=device,
kernel_block_sizes=[16],
)
block_tables.append_block_ids(
@@ -141,7 +137,6 @@ def test_block_tables_skip_custom_slot_mapping_groups():
max_num_batched_tokens=4,
max_num_blocks_per_group=[1, 1],
device=device,
kernel_block_sizes=[8, 262144],
slot_mapping_enabled=[False, True],
)
block_tables.append_block_ids(
@@ -170,7 +165,7 @@ def test_block_tables_skip_custom_slot_mapping_groups():
@pytest.mark.parametrize("cp_rank", range(4))
def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
def test_dcp_slot_mapping_only_shards_sharded_groups(cp_rank: int):
"""Only sharded groups use DCP interleave in logical-block coordinates."""
device = torch.device("cuda")
block_tables = BlockTables(
@@ -179,7 +174,6 @@ def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
max_num_batched_tokens=1024,
max_num_blocks_per_group=[2, 8],
device=device,
kernel_block_sizes=[64, 64],
dcp_sharded=[True, False],
cp_size=4,
cp_rank=cp_rank,
@@ -230,7 +224,6 @@ def test_v1_block_table_move_row_clears_vacated_row():
max_num_batched_tokens=64,
pin_memory=False,
device=torch.device("cuda"),
kernel_block_size=16,
cp_kv_cache_interleave_size=1,
)
block_table.add_row([7, 8, 9], row_idx=0)
@@ -260,7 +253,6 @@ def test_get_dummy_block_tables_returns_zeroed_rows():
max_num_batched_tokens=64,
max_num_blocks_per_group=[8],
device=device,
kernel_block_sizes=[16],
)
# Simulate a real step: stage a request's blocks and gather them into
# the persistent input block tables.
@@ -292,7 +284,6 @@ def test_dummy_request_slot_mapping_is_pad():
max_num_batched_tokens=16,
max_num_blocks_per_group=[4],
device=device,
kernel_block_sizes=[4],
)
block_tables.append_block_ids(req_index=0, new_block_ids=([7, 8],), overwrite=True)
block_tables.apply_staged_writes()