[Bugfix][GLM-5.3-Flash] kpool corruption with speculative decoding (#58454)
Signed-off-by: Matt Mastracci <matthew@mastracci.com> Signed-off-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
Claude Opus 5.5
Robert Shaw
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
f83ea61cfe
commit
2617fe9383
@@ -293,8 +293,9 @@ def _run_kernel(kv, tail, tail_slot, key, score, ape, slot_map, pos):
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pool_size", [4, 16])
|
||||
def test_decode_writer_matches_prefill_writer(pool_size):
|
||||
"""Compare production decode and prefill writers for pool sizes 4 and 16."""
|
||||
@pytest.mark.parametrize("ring_pools", [1, 2])
|
||||
def test_decode_writer_matches_prefill_writer(pool_size, ring_pools):
|
||||
ring = ring_pools * pool_size
|
||||
n_pools, page, nblk = 8, 64, 4
|
||||
n_tok = n_pools * pool_size
|
||||
dev = "cuda"
|
||||
@@ -317,14 +318,14 @@ def test_decode_writer_matches_prefill_writer(pool_size):
|
||||
|
||||
# One request owning tail block 0, fed one token per decode step.
|
||||
kv_decode = torch.zeros_like(kv_prefill)
|
||||
tail = torch.zeros(nblk, 2, pool_size, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
tail = torch.zeros(nblk, 2, ring, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
for t in range(n_tok):
|
||||
completes = t % pool_size == pool_size - 1
|
||||
kpool_decode_update_and_maybe_write_cache_batched(
|
||||
kv_decode,
|
||||
tail,
|
||||
# token-granular: every token has a valid tail slot
|
||||
torch.tensor([[t % pool_size]], dtype=torch.int32, device=dev),
|
||||
torch.tensor([[t % ring]], dtype=torch.int32, device=dev),
|
||||
k[t].view(1, 1, HEAD_DIM),
|
||||
score[t].view(1, 1, HEAD_DIM),
|
||||
ape,
|
||||
@@ -351,6 +352,91 @@ def test_decode_writer_matches_prefill_writer(pool_size):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ring_pools", [1, 2])
|
||||
def test_rejected_draft_redo_needs_ring_slots(ring_pools):
|
||||
"""With a one-pool ring, the drafts behind a rejected pool-completing draft
|
||||
overwrote the pool's earlier keys, so its redo compressed wrong keys."""
|
||||
pool, spec, page, nblk = 4, 3, 64, 2
|
||||
ring = ring_pools * pool
|
||||
dev = "cuda"
|
||||
torch.manual_seed(1)
|
||||
n_tok = 3 * pool
|
||||
k = torch.randn(n_tok, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
score = torch.randn(n_tok, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
ape = torch.randn(pool, HEAD_DIM, dtype=torch.float32, device=dev)
|
||||
kv_ref = torch.zeros(nblk, page, HEAD_DIM + 4, dtype=torch.uint8, device=dev)
|
||||
kpool_compress_and_write_cache(
|
||||
kv_ref,
|
||||
k.view(3, pool, HEAD_DIM),
|
||||
score.view(3, pool, HEAD_DIM),
|
||||
ape,
|
||||
torch.arange(3, dtype=torch.int64, device=dev),
|
||||
pool_size=pool,
|
||||
head_dim=HEAD_DIM,
|
||||
round_scale=ROUND_SCALE,
|
||||
)
|
||||
|
||||
kv = torch.zeros_like(kv_ref)
|
||||
tail = torch.zeros(nblk, 2, ring, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
|
||||
def pool_bytes(cache, p):
|
||||
# Page layout: [page * HEAD_DIM bytes of K rows | page * 4 bytes of scales].
|
||||
flat = cache[0].reshape(-1)
|
||||
k_bytes = flat[p * HEAD_DIM : (p + 1) * HEAD_DIM]
|
||||
s_bytes = flat[page * HEAD_DIM + 4 * p : page * HEAD_DIM + 4 * (p + 1)]
|
||||
return torch.cat([k_bytes, s_bytes])
|
||||
|
||||
def step(positions, keys, scores):
|
||||
pos = torch.tensor([positions], dtype=torch.int32, device=dev)
|
||||
slots = [(p // pool) if p % pool == pool - 1 else -1 for p in positions]
|
||||
kpool_decode_update_and_maybe_write_cache_batched(
|
||||
kv,
|
||||
tail,
|
||||
pos % ring,
|
||||
keys.view(1, -1, HEAD_DIM),
|
||||
scores.view(1, -1, HEAD_DIM),
|
||||
ape,
|
||||
torch.tensor([slots], dtype=torch.int32, device=dev),
|
||||
pos,
|
||||
pool,
|
||||
HEAD_DIM,
|
||||
round_scale=ROUND_SCALE,
|
||||
)
|
||||
|
||||
for t in range(7):
|
||||
step([t], k[t], score[t])
|
||||
# Control: verified token 7 completes pool 1 before drafts 8..10 are stashed.
|
||||
drafts = torch.randn(spec, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
draft_scores = torch.randn(spec, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
step(
|
||||
[7, 8, 9, 10],
|
||||
torch.cat([k[7:8], drafts]),
|
||||
torch.cat([score[7:8], draft_scores]),
|
||||
)
|
||||
step([8, 9, 10, 11], k[8:12], score[8:12]) # all drafts rejected
|
||||
for p in (1, 2):
|
||||
assert torch.equal(pool_bytes(kv, p), pool_bytes(kv_ref, p)), p
|
||||
|
||||
# Draft 7 completes pool 1 and is rejected. With a one-pool ring, drafts
|
||||
# 8 and 9 overwrite the slots of positions 4 and 5, which are read by the
|
||||
# redo of 7.
|
||||
kv.zero_()
|
||||
tail.zero_()
|
||||
for t in range(6):
|
||||
step([t], k[t], score[t])
|
||||
step(
|
||||
[6, 7, 8, 9],
|
||||
torch.cat([k[6:7], drafts]),
|
||||
torch.cat([score[6:7], draft_scores]),
|
||||
)
|
||||
step([7, 8, 9, 10], k[7:11], score[7:11])
|
||||
pool1_ok = torch.equal(pool_bytes(kv, 1), pool_bytes(kv_ref, 1))
|
||||
if ring_pools == 1:
|
||||
assert not pool1_ok, "expected a one-pool ring to corrupt pool 1"
|
||||
else:
|
||||
assert pool1_ok
|
||||
|
||||
|
||||
def test_leading_invalid_tail_slot():
|
||||
"""A request whose FIRST token carries an invalid (-1) tail slot while a
|
||||
later token is a real pool completion.
|
||||
|
||||
@@ -24,6 +24,8 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.models.glm5next.common.attention import Glm5NextTailCache
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
KpoolTailBackend,
|
||||
@@ -56,6 +58,30 @@ def test_tail_backend_layout_matches_kernel_pointer_arithmetic():
|
||||
assert content_stride == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_speculative_tokens, ring", [(0, 4), (1, 8), (4, 8), (7, 16), (13, 32)]
|
||||
)
|
||||
def test_tail_ring_divides_the_attention_block(num_speculative_tokens, ring):
|
||||
"""With 7 draft tokens the ring was 12 slots. 12 does not divide the
|
||||
640-token KDA attention block, so the scheduler block grew to their lcm
|
||||
and prefix-cache hits were cut down to multiples of 1920."""
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
cache = Glm5NextTailCache(
|
||||
head_dim=128,
|
||||
dtype=torch.bfloat16,
|
||||
prefix="tail",
|
||||
cache_config=SimpleNamespace(block_size=640),
|
||||
index_kpool=KPOOL,
|
||||
)
|
||||
spec = cache.get_kv_cache_spec(
|
||||
SimpleNamespace(num_speculative_tokens=num_speculative_tokens)
|
||||
)
|
||||
|
||||
assert spec.block_size == spec.sliding_window == ring
|
||||
assert ring >= KPOOL + num_speculative_tokens
|
||||
assert 640 % ring == 0
|
||||
|
||||
|
||||
def test_tail_spec_opts_out_of_generic_slot_mapping():
|
||||
"""The tail row is one block wide (padded to the block-table alignment), so
|
||||
the generic kernel's ``pos // kpool`` column index runs off the end of the
|
||||
@@ -296,31 +322,30 @@ def test_builder_reuses_slot_mapping_storage():
|
||||
|
||||
|
||||
class TailRingMirror:
|
||||
"""Mirror of _kpool_tail_seed_kernel / _kpool_decode_update_batched_kernel
|
||||
addressing: block = tail_slot // kpool, ring offset = pos % kpool; a pool
|
||||
completing at pos reads ring slots (pool_start + s) % kpool and uses the
|
||||
current token's own K/score for the last member."""
|
||||
"""Mirror of the tail-ring addressing in _kpool_tail_seed_kernel and
|
||||
_kpool_decode_update_batched_kernel."""
|
||||
|
||||
def __init__(self, num_blocks, kpool=KPOOL):
|
||||
def __init__(self, num_blocks, kpool=KPOOL, ring=None):
|
||||
self.kpool = kpool
|
||||
self.k = torch.full((num_blocks, kpool, 3), float("nan"))
|
||||
self.s = torch.full((num_blocks, kpool, 3), float("nan"))
|
||||
self.ring = ring or kpool
|
||||
self.k = torch.full((num_blocks, self.ring, 3), float("nan"))
|
||||
self.s = torch.full((num_blocks, self.ring, 3), float("nan"))
|
||||
|
||||
def stash(self, tail_slot, pos, k, s):
|
||||
blk, off = tail_slot // self.kpool, pos % self.kpool
|
||||
blk, off = tail_slot // self.ring, pos % self.ring
|
||||
self.k[blk, off] = k
|
||||
self.s[blk, off] = s
|
||||
|
||||
seed = stash # the seed kernel writes with the same addressing
|
||||
|
||||
def complete(self, tail_slot, pos, k, s):
|
||||
blk = tail_slot // self.kpool
|
||||
blk = tail_slot // self.ring
|
||||
start = pos - (self.kpool - 1)
|
||||
kk = torch.stack(
|
||||
[self.k[blk, (start + i) % self.kpool] for i in range(self.kpool)]
|
||||
[self.k[blk, (start + i) % self.ring] for i in range(self.kpool)]
|
||||
)
|
||||
ss = torch.stack(
|
||||
[self.s[blk, (start + i) % self.kpool] for i in range(self.kpool)]
|
||||
[self.s[blk, (start + i) % self.ring] for i in range(self.kpool)]
|
||||
)
|
||||
kk[-1], ss[-1] = k, s # is_current for the completing token
|
||||
w = torch.softmax(ss, dim=0)
|
||||
@@ -454,3 +479,35 @@ def test_triton_mapping_reads_strided_block_table():
|
||||
for req, blk in enumerate(own_blocks):
|
||||
start, end = int(qsl[req]), int(qsl[req + 1])
|
||||
assert (got[start:end] // KPOOL == blk).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ring_pools", [1, 2])
|
||||
def test_rejected_completing_draft_needs_ring_slots(ring_pools):
|
||||
"""With a one-pool ring, the drafts behind a rejected pool-completing draft
|
||||
overwrote the pool's earlier keys, so its redo compressed wrong keys."""
|
||||
ring_size = ring_pools * KPOOL
|
||||
truth = TailRingMirror(num_blocks=2, ring=ring_size)
|
||||
ring = TailRingMirror(num_blocks=2, ring=ring_size)
|
||||
block = 1
|
||||
|
||||
def slot(pos):
|
||||
return block * ring_size + pos % ring_size
|
||||
|
||||
for pos in range(4, 7):
|
||||
truth.stash(slot(pos), pos, *token_kv(0, pos))
|
||||
ring.stash(slot(pos), pos, *token_kv(0, pos))
|
||||
expected = truth.complete(slot(7), 7, *token_kv(0, 7))
|
||||
|
||||
# Draft 7 completes the pool and is rejected. With a one-pool ring, drafts
|
||||
# 8..10 overwrite the slots of positions 4..6, which are read by the redo
|
||||
# of 7.
|
||||
for pos in range(7, 11):
|
||||
k, s = token_kv(9, pos) # draft values
|
||||
if pos % KPOOL == KPOOL - 1:
|
||||
ring.complete(slot(pos), pos, k, s)
|
||||
ring.stash(slot(pos), pos, k, s)
|
||||
redo = ring.complete(slot(7), 7, *token_kv(0, 7))
|
||||
if ring_pools == 1:
|
||||
assert not torch.allclose(redo, expected)
|
||||
else:
|
||||
torch.testing.assert_close(redo, expected)
|
||||
|
||||
Reference in New Issue
Block a user