[Bugfix][GLM-5.3-Flash] kpool corruption with speculative decoding (#58454)

Signed-off-by: Matt Mastracci <matthew@mastracci.com>
Signed-off-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Matt Mastracci
2026-09-25 12:51:34 -04:00
committed by GitHub
co-authored by Yifan Qiao Claude Opus 5.5 Robert Shaw mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent f83ea61cfe
commit 2617fe9383
5 changed files with 208 additions and 49 deletions
@@ -293,8 +293,9 @@ def _run_kernel(kv, tail, tail_slot, key, score, ape, slot_map, pos):
@pytest.mark.parametrize("pool_size", [4, 16])
def test_decode_writer_matches_prefill_writer(pool_size):
"""Compare production decode and prefill writers for pool sizes 4 and 16."""
@pytest.mark.parametrize("ring_pools", [1, 2])
def test_decode_writer_matches_prefill_writer(pool_size, ring_pools):
ring = ring_pools * pool_size
n_pools, page, nblk = 8, 64, 4
n_tok = n_pools * pool_size
dev = "cuda"
@@ -317,14 +318,14 @@ def test_decode_writer_matches_prefill_writer(pool_size):
# One request owning tail block 0, fed one token per decode step.
kv_decode = torch.zeros_like(kv_prefill)
tail = torch.zeros(nblk, 2, pool_size, HEAD_DIM, dtype=torch.bfloat16, device=dev)
tail = torch.zeros(nblk, 2, ring, HEAD_DIM, dtype=torch.bfloat16, device=dev)
for t in range(n_tok):
completes = t % pool_size == pool_size - 1
kpool_decode_update_and_maybe_write_cache_batched(
kv_decode,
tail,
# token-granular: every token has a valid tail slot
torch.tensor([[t % pool_size]], dtype=torch.int32, device=dev),
torch.tensor([[t % ring]], dtype=torch.int32, device=dev),
k[t].view(1, 1, HEAD_DIM),
score[t].view(1, 1, HEAD_DIM),
ape,
@@ -351,6 +352,91 @@ def test_decode_writer_matches_prefill_writer(pool_size):
)
@pytest.mark.parametrize("ring_pools", [1, 2])
def test_rejected_draft_redo_needs_ring_slots(ring_pools):
"""With a one-pool ring, the drafts behind a rejected pool-completing draft
overwrote the pool's earlier keys, so its redo compressed wrong keys."""
pool, spec, page, nblk = 4, 3, 64, 2
ring = ring_pools * pool
dev = "cuda"
torch.manual_seed(1)
n_tok = 3 * pool
k = torch.randn(n_tok, HEAD_DIM, dtype=torch.bfloat16, device=dev)
score = torch.randn(n_tok, HEAD_DIM, dtype=torch.bfloat16, device=dev)
ape = torch.randn(pool, HEAD_DIM, dtype=torch.float32, device=dev)
kv_ref = torch.zeros(nblk, page, HEAD_DIM + 4, dtype=torch.uint8, device=dev)
kpool_compress_and_write_cache(
kv_ref,
k.view(3, pool, HEAD_DIM),
score.view(3, pool, HEAD_DIM),
ape,
torch.arange(3, dtype=torch.int64, device=dev),
pool_size=pool,
head_dim=HEAD_DIM,
round_scale=ROUND_SCALE,
)
kv = torch.zeros_like(kv_ref)
tail = torch.zeros(nblk, 2, ring, HEAD_DIM, dtype=torch.bfloat16, device=dev)
def pool_bytes(cache, p):
# Page layout: [page * HEAD_DIM bytes of K rows | page * 4 bytes of scales].
flat = cache[0].reshape(-1)
k_bytes = flat[p * HEAD_DIM : (p + 1) * HEAD_DIM]
s_bytes = flat[page * HEAD_DIM + 4 * p : page * HEAD_DIM + 4 * (p + 1)]
return torch.cat([k_bytes, s_bytes])
def step(positions, keys, scores):
pos = torch.tensor([positions], dtype=torch.int32, device=dev)
slots = [(p // pool) if p % pool == pool - 1 else -1 for p in positions]
kpool_decode_update_and_maybe_write_cache_batched(
kv,
tail,
pos % ring,
keys.view(1, -1, HEAD_DIM),
scores.view(1, -1, HEAD_DIM),
ape,
torch.tensor([slots], dtype=torch.int32, device=dev),
pos,
pool,
HEAD_DIM,
round_scale=ROUND_SCALE,
)
for t in range(7):
step([t], k[t], score[t])
# Control: verified token 7 completes pool 1 before drafts 8..10 are stashed.
drafts = torch.randn(spec, HEAD_DIM, dtype=torch.bfloat16, device=dev)
draft_scores = torch.randn(spec, HEAD_DIM, dtype=torch.bfloat16, device=dev)
step(
[7, 8, 9, 10],
torch.cat([k[7:8], drafts]),
torch.cat([score[7:8], draft_scores]),
)
step([8, 9, 10, 11], k[8:12], score[8:12]) # all drafts rejected
for p in (1, 2):
assert torch.equal(pool_bytes(kv, p), pool_bytes(kv_ref, p)), p
# Draft 7 completes pool 1 and is rejected. With a one-pool ring, drafts
# 8 and 9 overwrite the slots of positions 4 and 5, which are read by the
# redo of 7.
kv.zero_()
tail.zero_()
for t in range(6):
step([t], k[t], score[t])
step(
[6, 7, 8, 9],
torch.cat([k[6:7], drafts]),
torch.cat([score[6:7], draft_scores]),
)
step([7, 8, 9, 10], k[7:11], score[7:11])
pool1_ok = torch.equal(pool_bytes(kv, 1), pool_bytes(kv_ref, 1))
if ring_pools == 1:
assert not pool1_ok, "expected a one-pool ring to corrupt pool 1"
else:
assert pool1_ok
def test_leading_invalid_tail_slot():
"""A request whose FIRST token carries an invalid (-1) tail slot while a
later token is a real pool completion.