[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Dakai An
2026-10-09 15:48:37 -04:00
committed by GitHub
co-authored by Yifan Qiao OpenAI Codex OpenAI Codex Lucas Wilkinson Claude Opus 5.5 Lucas Wilkinson
parent 3766a67a32
commit 02e55d2af3
83 changed files with 1280 additions and 1986 deletions
@@ -313,3 +313,49 @@ def test_deepgemm_fp8_fp4_paged_mqa_logits(batch_size: int, next_n: int):
ref_logits = ref_logits.masked_fill(~mask, 0)
diff = calc_diff(logits, ref_logits)
assert diff < 1e-3, f"{diff=}"
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@pytest.mark.skipif(not has_deep_gemm(), reason="DeepGEMM not available")
@pytest.mark.skipif(
not current_platform.has_device_capability(90), reason="SM90 and SM100 only"
)
def test_deepgemm_paged_mqa_packed_manager_block_stride():
torch.manual_seed(0)
num_blocks, block_size, packed_rows = 4, 256, 320
num_heads, head_dim, row_bytes = 64, 128, 132
page_size, pages_per_block = 64, block_size // 64
dense_stride, packed_stride = pages_per_block, packed_rows // page_size
compact_pages = kv_cache_cast_to_fp8(
torch.randn(
num_blocks, block_size, 1, head_dim, device="cuda", dtype=torch.bfloat16
).view(num_blocks * pages_per_block, page_size, 1, head_dim)
)
packed_backing = torch.zeros(
num_blocks,
packed_rows,
row_bytes,
device="cuda",
dtype=torch.uint8,
)
packed_manager = packed_backing[:, :block_size]
packed_manager.copy_(compact_pages.squeeze(2).view(num_blocks, block_size, -1))
packed_pages = packed_backing.view(-1, page_size, row_bytes).unsqueeze(2)
offsets = torch.arange(pages_per_block, device="cuda", dtype=torch.int32)
tables = [
(stride + offsets).unsqueeze(0) for stride in (dense_stride, packed_stride)
]
q = torch.randn(1, 1, num_heads, head_dim, device="cuda", dtype=torch.bfloat16).to(
torch.float8_e4m3fn
)
weights = torch.randn(1, num_heads, device="cuda", dtype=torch.float32)
context_lens = torch.full((1, 1), block_size, device="cuda", dtype=torch.int32)
schedule = get_paged_mqa_logits_metadata(context_lens, page_size, get_num_sms())
logits = [
fp8_fp4_paged_mqa_logits(
(q, None), cache, weights, context_lens, table, schedule, block_size, False
)
for cache, table in zip((compact_pages, packed_pages), tables)
]
torch.testing.assert_close(*logits, rtol=0, atol=0)
@@ -463,7 +463,7 @@ def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype):
set_random_seed(412)
num_tokens, num_heads, block_size = 2, 8, 16
q = torch.randn(num_tokens, num_heads, Q_HEAD_DIM, device="cuda").to(dtype)
kv = torch.randn(2 * block_size, 1, Q_HEAD_DIM, device="cuda").to(dtype)
kv = torch.randn(2, 2, block_size, Q_HEAD_DIM, device="cuda").to(dtype)[:, 1]
selected = torch.tensor([[1, 7, 18], [0, 3, 20]], dtype=torch.int32, device="cuda")
sinks = torch.linspace(-3.0, 5.0, num_heads, device="cuda")
impl = object.__new__(ROCMAiterMLASparseImpl)
@@ -504,7 +504,7 @@ def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype):
SimpleNamespace(_q_scale=None, _k_scale=None),
)
references = [
_sink_reference(q[i], kv[:, 0][selected[i].long()], sinks, SM_SCALE)
_sink_reference(q[i], kv.flatten(0, 1)[selected[i].long()], sinks, SM_SCALE)
for i in range(num_tokens)
]
expected, _, value_scale, score_error = (