[Bugfix][GLM-5.3-Flash] kpool corruption with speculative decoding (#58454)

Signed-off-by: Matt Mastracci <matthew@mastracci.com>
Signed-off-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Matt Mastracci
2026-09-25 12:51:34 -04:00
committed by GitHub
co-authored by Yifan Qiao Claude Opus 5.5 Robert Shaw mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent f83ea61cfe
commit 2617fe9383
5 changed files with 208 additions and 49 deletions
@@ -293,8 +293,9 @@ def _run_kernel(kv, tail, tail_slot, key, score, ape, slot_map, pos):
@pytest.mark.parametrize("pool_size", [4, 16])
def test_decode_writer_matches_prefill_writer(pool_size):
"""Compare production decode and prefill writers for pool sizes 4 and 16."""
@pytest.mark.parametrize("ring_pools", [1, 2])
def test_decode_writer_matches_prefill_writer(pool_size, ring_pools):
ring = ring_pools * pool_size
n_pools, page, nblk = 8, 64, 4
n_tok = n_pools * pool_size
dev = "cuda"
@@ -317,14 +318,14 @@ def test_decode_writer_matches_prefill_writer(pool_size):
# One request owning tail block 0, fed one token per decode step.
kv_decode = torch.zeros_like(kv_prefill)
tail = torch.zeros(nblk, 2, pool_size, HEAD_DIM, dtype=torch.bfloat16, device=dev)
tail = torch.zeros(nblk, 2, ring, HEAD_DIM, dtype=torch.bfloat16, device=dev)
for t in range(n_tok):
completes = t % pool_size == pool_size - 1
kpool_decode_update_and_maybe_write_cache_batched(
kv_decode,
tail,
# token-granular: every token has a valid tail slot
torch.tensor([[t % pool_size]], dtype=torch.int32, device=dev),
torch.tensor([[t % ring]], dtype=torch.int32, device=dev),
k[t].view(1, 1, HEAD_DIM),
score[t].view(1, 1, HEAD_DIM),
ape,
@@ -351,6 +352,91 @@ def test_decode_writer_matches_prefill_writer(pool_size):
)
@pytest.mark.parametrize("ring_pools", [1, 2])
def test_rejected_draft_redo_needs_ring_slots(ring_pools):
"""With a one-pool ring, the drafts behind a rejected pool-completing draft
overwrote the pool's earlier keys, so its redo compressed wrong keys."""
pool, spec, page, nblk = 4, 3, 64, 2
ring = ring_pools * pool
dev = "cuda"
torch.manual_seed(1)
n_tok = 3 * pool
k = torch.randn(n_tok, HEAD_DIM, dtype=torch.bfloat16, device=dev)
score = torch.randn(n_tok, HEAD_DIM, dtype=torch.bfloat16, device=dev)
ape = torch.randn(pool, HEAD_DIM, dtype=torch.float32, device=dev)
kv_ref = torch.zeros(nblk, page, HEAD_DIM + 4, dtype=torch.uint8, device=dev)
kpool_compress_and_write_cache(
kv_ref,
k.view(3, pool, HEAD_DIM),
score.view(3, pool, HEAD_DIM),
ape,
torch.arange(3, dtype=torch.int64, device=dev),
pool_size=pool,
head_dim=HEAD_DIM,
round_scale=ROUND_SCALE,
)
kv = torch.zeros_like(kv_ref)
tail = torch.zeros(nblk, 2, ring, HEAD_DIM, dtype=torch.bfloat16, device=dev)
def pool_bytes(cache, p):
# Page layout: [page * HEAD_DIM bytes of K rows | page * 4 bytes of scales].
flat = cache[0].reshape(-1)
k_bytes = flat[p * HEAD_DIM : (p + 1) * HEAD_DIM]
s_bytes = flat[page * HEAD_DIM + 4 * p : page * HEAD_DIM + 4 * (p + 1)]
return torch.cat([k_bytes, s_bytes])
def step(positions, keys, scores):
pos = torch.tensor([positions], dtype=torch.int32, device=dev)
slots = [(p // pool) if p % pool == pool - 1 else -1 for p in positions]
kpool_decode_update_and_maybe_write_cache_batched(
kv,
tail,
pos % ring,
keys.view(1, -1, HEAD_DIM),
scores.view(1, -1, HEAD_DIM),
ape,
torch.tensor([slots], dtype=torch.int32, device=dev),
pos,
pool,
HEAD_DIM,
round_scale=ROUND_SCALE,
)
for t in range(7):
step([t], k[t], score[t])
# Control: verified token 7 completes pool 1 before drafts 8..10 are stashed.
drafts = torch.randn(spec, HEAD_DIM, dtype=torch.bfloat16, device=dev)
draft_scores = torch.randn(spec, HEAD_DIM, dtype=torch.bfloat16, device=dev)
step(
[7, 8, 9, 10],
torch.cat([k[7:8], drafts]),
torch.cat([score[7:8], draft_scores]),
)
step([8, 9, 10, 11], k[8:12], score[8:12]) # all drafts rejected
for p in (1, 2):
assert torch.equal(pool_bytes(kv, p), pool_bytes(kv_ref, p)), p
# Draft 7 completes pool 1 and is rejected. With a one-pool ring, drafts
# 8 and 9 overwrite the slots of positions 4 and 5, which are read by the
# redo of 7.
kv.zero_()
tail.zero_()
for t in range(6):
step([t], k[t], score[t])
step(
[6, 7, 8, 9],
torch.cat([k[6:7], drafts]),
torch.cat([score[6:7], draft_scores]),
)
step([7, 8, 9, 10], k[7:11], score[7:11])
pool1_ok = torch.equal(pool_bytes(kv, 1), pool_bytes(kv_ref, 1))
if ring_pools == 1:
assert not pool1_ok, "expected a one-pool ring to corrupt pool 1"
else:
assert pool1_ok
def test_leading_invalid_tail_slot():
"""A request whose FIRST token carries an invalid (-1) tail slot while a
later token is a real pool completion.
@@ -24,6 +24,8 @@ from types import SimpleNamespace
import pytest
import torch
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.models.glm5next.common.attention import Glm5NextTailCache
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.mla.indexer import (
KpoolTailBackend,
@@ -56,6 +58,30 @@ def test_tail_backend_layout_matches_kernel_pointer_arithmetic():
assert content_stride == 1
@pytest.mark.parametrize(
"num_speculative_tokens, ring", [(0, 4), (1, 8), (4, 8), (7, 16), (13, 32)]
)
def test_tail_ring_divides_the_attention_block(num_speculative_tokens, ring):
"""With 7 draft tokens the ring was 12 slots. 12 does not divide the
640-token KDA attention block, so the scheduler block grew to their lcm
and prefix-cache hits were cut down to multiples of 1920."""
with set_current_vllm_config(VllmConfig()):
cache = Glm5NextTailCache(
head_dim=128,
dtype=torch.bfloat16,
prefix="tail",
cache_config=SimpleNamespace(block_size=640),
index_kpool=KPOOL,
)
spec = cache.get_kv_cache_spec(
SimpleNamespace(num_speculative_tokens=num_speculative_tokens)
)
assert spec.block_size == spec.sliding_window == ring
assert ring >= KPOOL + num_speculative_tokens
assert 640 % ring == 0
def test_tail_spec_opts_out_of_generic_slot_mapping():
"""The tail row is one block wide (padded to the block-table alignment), so
the generic kernel's ``pos // kpool`` column index runs off the end of the
@@ -296,31 +322,30 @@ def test_builder_reuses_slot_mapping_storage():
class TailRingMirror:
"""Mirror of _kpool_tail_seed_kernel / _kpool_decode_update_batched_kernel
addressing: block = tail_slot // kpool, ring offset = pos % kpool; a pool
completing at pos reads ring slots (pool_start + s) % kpool and uses the
current token's own K/score for the last member."""
"""Mirror of the tail-ring addressing in _kpool_tail_seed_kernel and
_kpool_decode_update_batched_kernel."""
def __init__(self, num_blocks, kpool=KPOOL):
def __init__(self, num_blocks, kpool=KPOOL, ring=None):
self.kpool = kpool
self.k = torch.full((num_blocks, kpool, 3), float("nan"))
self.s = torch.full((num_blocks, kpool, 3), float("nan"))
self.ring = ring or kpool
self.k = torch.full((num_blocks, self.ring, 3), float("nan"))
self.s = torch.full((num_blocks, self.ring, 3), float("nan"))
def stash(self, tail_slot, pos, k, s):
blk, off = tail_slot // self.kpool, pos % self.kpool
blk, off = tail_slot // self.ring, pos % self.ring
self.k[blk, off] = k
self.s[blk, off] = s
seed = stash # the seed kernel writes with the same addressing
def complete(self, tail_slot, pos, k, s):
blk = tail_slot // self.kpool
blk = tail_slot // self.ring
start = pos - (self.kpool - 1)
kk = torch.stack(
[self.k[blk, (start + i) % self.kpool] for i in range(self.kpool)]
[self.k[blk, (start + i) % self.ring] for i in range(self.kpool)]
)
ss = torch.stack(
[self.s[blk, (start + i) % self.kpool] for i in range(self.kpool)]
[self.s[blk, (start + i) % self.ring] for i in range(self.kpool)]
)
kk[-1], ss[-1] = k, s # is_current for the completing token
w = torch.softmax(ss, dim=0)
@@ -454,3 +479,35 @@ def test_triton_mapping_reads_strided_block_table():
for req, blk in enumerate(own_blocks):
start, end = int(qsl[req]), int(qsl[req + 1])
assert (got[start:end] // KPOOL == blk).all()
@pytest.mark.parametrize("ring_pools", [1, 2])
def test_rejected_completing_draft_needs_ring_slots(ring_pools):
"""With a one-pool ring, the drafts behind a rejected pool-completing draft
overwrote the pool's earlier keys, so its redo compressed wrong keys."""
ring_size = ring_pools * KPOOL
truth = TailRingMirror(num_blocks=2, ring=ring_size)
ring = TailRingMirror(num_blocks=2, ring=ring_size)
block = 1
def slot(pos):
return block * ring_size + pos % ring_size
for pos in range(4, 7):
truth.stash(slot(pos), pos, *token_kv(0, pos))
ring.stash(slot(pos), pos, *token_kv(0, pos))
expected = truth.complete(slot(7), 7, *token_kv(0, 7))
# Draft 7 completes the pool and is rejected. With a one-pool ring, drafts
# 8..10 overwrite the slots of positions 4..6, which are read by the redo
# of 7.
for pos in range(7, 11):
k, s = token_kv(9, pos) # draft values
if pos % KPOOL == KPOOL - 1:
ring.complete(slot(pos), pos, k, s)
ring.stash(slot(pos), pos, k, s)
redo = ring.complete(slot(7), 7, *token_kv(0, 7))
if ring_pools == 1:
assert not torch.allclose(redo, expected)
else:
torch.testing.assert_close(redo, expected)
+18 -16
View File
@@ -327,37 +327,38 @@ def _kpool_tail_seed_kernel(
KPOOL_HEAD: tl.constexpr,
HEAD_DIM: tl.constexpr,
KPOOL: tl.constexpr,
RING: tl.constexpr,
BLOCK_D: tl.constexpr,
):
"""Copy token ``i``'s raw K + gate into its request's tail block.
"""Copy token ``i``'s raw K + gate into its request's tail ring.
Token ``i`` is among its request's last KPOOL tokens iff the token KPOOL
ahead belongs to a different tail block (or is past the batch / padding,
slot < 0). ``tslot = block * KPOOL + pos % KPOOL``; the destination is
``tail[block, {0:K, 1:score}, pos % KPOOL, :]``.
slot < 0). ``tslot = block * RING + pos % RING``; the destination is
``tail[block, {0:K, 1:score}, pos % RING, :]``.
"""
i = tl.program_id(0)
t = tl.load(tslot_ptr + i).to(tl.int64)
if t < 0:
return
blk = t // KPOOL # t >= 0 here, so trunc == floor
blk = t // RING # t >= 0 here, so trunc == floor
ahead = tl.load(tslot_ptr + i + KPOOL, mask=i + KPOOL < n_tokens, other=-1).to(
tl.int64
)
# Match the torch semantics exactly: a negative ahead slot floors to a
# block id that differs from every real block -> token is in the tail.
# Only divide non-negative slots (Triton int div truncates, torch floors).
if ahead >= 0 and ahead // KPOOL == blk:
if ahead >= 0 and ahead // RING == blk:
return
offs = tl.arange(0, BLOCK_D)
m = offs < HEAD_DIM
block_base = blk * TAIL_BLOCK_ELEMS
base = block_base + (t % KPOOL) * HEAD_DIM
base = block_base + (t % RING) * HEAD_DIM
k = tl.load(key_ptr + i * HEAD_DIM + offs, mask=m)
s = tl.load(score_ptr + i * HEAD_DIM + offs, mask=m)
tl.store(tail_ptr + base + offs, k, mask=m)
tl.store(
tail_ptr + block_base + KPOOL_HEAD + (t % KPOOL) * HEAD_DIM + offs, s, mask=m
tail_ptr + block_base + KPOOL_HEAD + (t % RING) * HEAD_DIM + offs, s, mask=m
)
@@ -385,6 +386,7 @@ def kpool_seed_tail_cache(
KPOOL_HEAD=tail_kv_cache.stride(1),
HEAD_DIM=head_dim,
KPOOL=kpool,
RING=tail_kv_cache.shape[2],
BLOCK_D=triton.next_power_of_2(head_dim),
)
@@ -412,6 +414,7 @@ def _kpool_decode_update_batched_kernel(
PAGE_SIZE: tl.constexpr,
BUF_NUMEL_PER_PAGE: tl.constexpr,
POOL_SIZE: tl.constexpr,
RING: tl.constexpr,
TAIL_BLOCK_ELEMS: tl.constexpr,
KPOOL_HEAD: tl.constexpr,
HEAD_DIM: tl.constexpr,
@@ -428,10 +431,7 @@ def _kpool_decode_update_batched_kernel(
the tail-ring slots that tokens t < t* (same request) just stashed in this
same invocation. ``tl.range`` iterates sequentially within the program, so
those stashes are visible to the later completion read. Cross-request
programs are independent (distinct tail blocks). With NEXT_N < POOL_SIZE
(the spec-verify case: NEXT_N ~= num_spec+1, POOL_SIZE=16) at most one
completion can occur per request per call, but the ordered loop is correct
for any NEXT_N.
programs are independent (distinct tail blocks). RING >= POOL_SIZE.
"""
req = tl.program_id(0)
offs = tl.arange(0, BLOCK_D)
@@ -445,7 +445,7 @@ def _kpool_decode_update_batched_kernel(
pos_valid = (cache_loc >= 0) & (pos >= 0)
slot = safe_pos % POOL_SIZE
phys_slot = safe_pos % POOL_SIZE
phys_slot = safe_pos % RING
# Derive the tail block from THIS token's tail_slot (the request's block
# is constant across a pool, but a padded / invalid entry carries a
@@ -453,7 +453,7 @@ def _kpool_decode_update_batched_kernel(
# token's base address). Clamp so an invalid entry can never form an
# out-of-bounds base; the accesses below are gated on pos_valid anyway.
tail_slot = tl.load(tail_slot_mapping_ptr + idx)
block = tl.maximum(tail_slot, 0).to(tl.int64) // POOL_SIZE
block = tl.maximum(tail_slot, 0).to(tl.int64) // RING
block_base = block * TAIL_BLOCK_ELEMS
# The tail-ring stash must run for EVERY real token, so it is gated on
@@ -481,7 +481,7 @@ def _kpool_decode_update_batched_kernel(
max_score = tl.full((BLOCK_D,), -float("inf"), tl.float32)
for pool_slot in tl.static_range(0, POOL_SIZE):
is_current = pool_slot == slot
phys = (pool_logical_start + pool_slot) % POOL_SIZE
phys = (pool_logical_start + pool_slot) % RING
score_buf = tl.load(
tail_kv_ptr + block_base + KPOOL_HEAD + phys * HEAD_DIM + offs,
mask=dim_mask,
@@ -499,7 +499,7 @@ def _kpool_decode_update_batched_kernel(
denom = tl.full((BLOCK_D,), 0.0, tl.float32)
for pool_slot in tl.static_range(0, POOL_SIZE):
is_current = pool_slot == slot
phys = (pool_logical_start + pool_slot) % POOL_SIZE
phys = (pool_logical_start + pool_slot) % RING
score_buf = tl.load(
tail_kv_ptr + block_base + KPOOL_HEAD + phys * HEAD_DIM + offs,
mask=dim_mask,
@@ -609,7 +609,8 @@ def kpool_decode_update_and_maybe_write_cache_batched(
return
assert tail_kv_cache.ndim == 4
assert tail_kv_cache.shape[1] == 2
assert tail_kv_cache.shape[2] == pool_size
ring = tail_kv_cache.shape[2]
assert ring >= pool_size and ring % pool_size == 0, (ring, pool_size)
assert tail_kv_cache.shape[3] == head_dim
assert tail_kv_cache.dtype == torch.bfloat16
assert key.ndim == 3 and key.shape[2] == head_dim
@@ -658,6 +659,7 @@ def kpool_decode_update_and_maybe_write_cache_batched(
PAGE_SIZE=page_size,
BUF_NUMEL_PER_PAGE=buf.stride(0),
POOL_SIZE=pool_size,
RING=ring,
TAIL_BLOCK_ELEMS=tail_kv_cache.stride(0),
KPOOL_HEAD=tail_kv_cache.stride(1),
HEAD_DIM=head_dim,
+15 -3
View File
@@ -34,6 +34,7 @@ from vllm.models.glm5next.sparse_indexer import SparseAttnIndexerKpool
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig
from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES
from vllm.utils.math_utils import cdiv, next_power_of_2
from vllm.v1.kv_cache_interface import KpoolTailSpec, MLAAttentionSpec
logger = init_logger(__name__)
@@ -160,7 +161,7 @@ class Glm5NextTailCache(DeepseekV32IndexerCache):
"""Paged circular buffer for the kpool indexer's in-progress (tail) pool.
Holds the trailing incomplete pool's raw K + gate score: one block of
``index_kpool`` slots per request, overwritten in place by ``pos % kpool``
``ring`` slots per request, overwritten in place by ``pos % ring``
as decode/spec-decode advances. Prefill seeds it (instead of discarding the
tail raw K+gate); the connector transfers it across PD; decode reads it to
compress the boundary pool correctly. ``KpoolTailSpec`` /
@@ -190,13 +191,24 @@ class Glm5NextTailCache(DeepseekV32IndexerCache):
def get_kv_cache_spec(self, vllm_config: VllmConfig):
# The two head slots form [K, gate score] in the generic
# [block, head, state, content] cache view.
# Drafts are stashed before acceptance. With a one-pool ring, the
# drafts behind a rejected pool-completing draft overwrite the keys
# read by its redo.
span = self._index_kpool + vllm_config.num_speculative_tokens
ring = self._index_kpool * next_power_of_2(cdiv(span, self._index_kpool))
# ring must divide the attention block size (a multiple of 128).
assert self.cache_config.block_size % ring == 0, (
f"Glm5NextTailCache: cache_config.block_size "
f"({self.cache_config.block_size}) must be a multiple of the "
f"tail ring ({ring})"
)
return KpoolTailSpec(
block_size=self._index_kpool,
block_size=ring,
num_kv_heads=2,
head_size=self.head_dim,
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=self._index_kpool,
sliding_window=ring,
)
def get_attn_backend(self):
@@ -387,14 +387,15 @@ def _kpool_tail_seed_kernel(
KPOOL_HEAD: tl.constexpr,
HEAD_DIM: tl.constexpr,
KPOOL: tl.constexpr,
RING: tl.constexpr,
BLOCK_D: tl.constexpr,
):
"""Copy token ``i``'s raw K + gate into its request's tail block.
"""Copy token ``i``'s raw K + gate into its request's tail ring.
Token ``i`` is among its request's last KPOOL tokens iff the token KPOOL
ahead belongs to a different tail block (or is past the batch / padding,
slot < 0). ``tslot = block * KPOOL + pos % KPOOL``; the destination is
``tail[block, {0:K, 1:score}, pos % KPOOL, :]``.
slot < 0). ``tslot = block * RING + pos % RING``; the destination is
``tail[block, {0:K, 1:score}, pos % RING, :]``.
The tail cache aliases the indexer cache with the indexer's (padded) block
stride, so blocks are addressed through ``TAIL_BLOCK_ELEMS`` /
@@ -405,18 +406,18 @@ def _kpool_tail_seed_kernel(
t = tl.load(tslot_ptr + i).to(tl.int64)
if t < 0:
return
blk = t // KPOOL # t >= 0 here, so trunc == floor
blk = t // RING # t >= 0 here, so trunc == floor
ahead = tl.load(tslot_ptr + i + KPOOL, mask=i + KPOOL < n_tokens, other=-1).to(
tl.int64
)
# Match the torch semantics exactly: a negative ahead slot floors to a
# block id that differs from every real block -> token is in the tail.
# Only divide non-negative slots (Triton int div truncates, torch floors).
if ahead >= 0 and ahead // KPOOL == blk:
if ahead >= 0 and ahead // RING == blk:
return
offs = tl.arange(0, BLOCK_D)
m = offs < HEAD_DIM
base = blk * TAIL_BLOCK_ELEMS + (t % KPOOL) * HEAD_DIM
base = blk * TAIL_BLOCK_ELEMS + (t % RING) * HEAD_DIM
k = tl.load(key_ptr + i * HEAD_DIM + offs, mask=m)
s = tl.load(score_ptr + i * HEAD_DIM + offs, mask=m)
tl.store(tail_ptr + base + offs, k, mask=m)
@@ -449,6 +450,7 @@ def kpool_seed_tail_cache(
KPOOL_HEAD=tail_kv_cache.stride(1),
HEAD_DIM=head_dim,
KPOOL=kpool,
RING=tail_kv_cache.shape[2],
BLOCK_D=triton.next_power_of_2(head_dim),
)
@@ -476,6 +478,7 @@ def _kpool_decode_update_batched_kernel(
PAGE_SIZE: tl.constexpr,
BUF_NUMEL_PER_PAGE: tl.constexpr,
POOL_SIZE: tl.constexpr,
RING: tl.constexpr,
TAIL_BLOCK_ELEMS: tl.constexpr,
KPOOL_HEAD: tl.constexpr,
HEAD_DIM: tl.constexpr,
@@ -490,10 +493,7 @@ def _kpool_decode_update_batched_kernel(
the tail-ring slots that tokens t < t* (same request) just stashed in this
same invocation. ``tl.range`` iterates sequentially within the program, so
those stashes are visible to the later completion read. Cross-request
programs are independent (distinct tail blocks). With NEXT_N < POOL_SIZE
(the spec-verify case: NEXT_N ~= num_spec+1, POOL_SIZE=16) at most one
completion can occur per request per call, but the ordered loop is correct
for any NEXT_N.
programs are independent (distinct tail blocks). RING >= POOL_SIZE.
"""
req = tl.program_id(0)
offs = tl.arange(0, BLOCK_D)
@@ -507,7 +507,7 @@ def _kpool_decode_update_batched_kernel(
pos_valid = (cache_loc >= 0) & (pos >= 0)
slot = safe_pos % POOL_SIZE
phys_slot = safe_pos % POOL_SIZE
phys_slot = safe_pos % RING
# Derive the tail block from THIS token's tail_slot (the request's block
# is constant across a pool, but a padded / invalid entry carries a
@@ -515,7 +515,7 @@ def _kpool_decode_update_batched_kernel(
# token's base address). Clamp so an invalid entry can never form an
# out-of-bounds base; the accesses below are gated on pos_valid anyway.
tail_slot = tl.load(tail_slot_mapping_ptr + idx)
block = tl.maximum(tail_slot, 0).to(tl.int64) // POOL_SIZE
block = tl.maximum(tail_slot, 0).to(tl.int64) // RING
block_base = block * TAIL_BLOCK_ELEMS
# The tail-ring stash must run for EVERY real token, so it is gated on
@@ -543,7 +543,7 @@ def _kpool_decode_update_batched_kernel(
max_score = tl.full((BLOCK_D,), -float("inf"), tl.float32)
for pool_slot in tl.static_range(0, POOL_SIZE):
is_current = pool_slot == slot
phys = (pool_logical_start + pool_slot) % POOL_SIZE
phys = (pool_logical_start + pool_slot) % RING
score_buf = tl.load(
tail_kv_ptr + block_base + KPOOL_HEAD + phys * HEAD_DIM + offs,
mask=dim_mask,
@@ -561,7 +561,7 @@ def _kpool_decode_update_batched_kernel(
denom = tl.full((BLOCK_D,), 0.0, tl.float32)
for pool_slot in tl.static_range(0, POOL_SIZE):
is_current = pool_slot == slot
phys = (pool_logical_start + pool_slot) % POOL_SIZE
phys = (pool_logical_start + pool_slot) % RING
score_buf = tl.load(
tail_kv_ptr + block_base + KPOOL_HEAD + phys * HEAD_DIM + offs,
mask=dim_mask,
@@ -671,7 +671,8 @@ def kpool_decode_update_and_maybe_write_cache_batched(
return
assert tail_kv_cache.ndim == 4
assert tail_kv_cache.shape[1] == 2
assert tail_kv_cache.shape[2] == pool_size
ring = tail_kv_cache.shape[2]
assert ring >= pool_size and ring % pool_size == 0, (ring, pool_size)
assert tail_kv_cache.shape[3] == head_dim
assert tail_kv_cache.dtype == torch.bfloat16
assert key.ndim == 3 and key.shape[2] == head_dim
@@ -717,6 +718,7 @@ def kpool_decode_update_and_maybe_write_cache_batched(
PAGE_SIZE=page_size,
BUF_NUMEL_PER_PAGE=buf.stride(0),
POOL_SIZE=pool_size,
RING=ring,
TAIL_BLOCK_ELEMS=tail_kv_cache.stride(0),
KPOOL_HEAD=tail_kv_cache.stride(1),
HEAD_DIM=head_dim,