diff --git a/tests/kernels/test_kpool_decode_update_batched.py b/tests/kernels/test_kpool_decode_update_batched.py index 37ba4b58c8..b0fbcf7faf 100644 --- a/tests/kernels/test_kpool_decode_update_batched.py +++ b/tests/kernels/test_kpool_decode_update_batched.py @@ -293,8 +293,9 @@ def _run_kernel(kv, tail, tail_slot, key, score, ape, slot_map, pos): @pytest.mark.parametrize("pool_size", [4, 16]) -def test_decode_writer_matches_prefill_writer(pool_size): - """Compare production decode and prefill writers for pool sizes 4 and 16.""" +@pytest.mark.parametrize("ring_pools", [1, 2]) +def test_decode_writer_matches_prefill_writer(pool_size, ring_pools): + ring = ring_pools * pool_size n_pools, page, nblk = 8, 64, 4 n_tok = n_pools * pool_size dev = "cuda" @@ -317,14 +318,14 @@ def test_decode_writer_matches_prefill_writer(pool_size): # One request owning tail block 0, fed one token per decode step. kv_decode = torch.zeros_like(kv_prefill) - tail = torch.zeros(nblk, 2, pool_size, HEAD_DIM, dtype=torch.bfloat16, device=dev) + tail = torch.zeros(nblk, 2, ring, HEAD_DIM, dtype=torch.bfloat16, device=dev) for t in range(n_tok): completes = t % pool_size == pool_size - 1 kpool_decode_update_and_maybe_write_cache_batched( kv_decode, tail, # token-granular: every token has a valid tail slot - torch.tensor([[t % pool_size]], dtype=torch.int32, device=dev), + torch.tensor([[t % ring]], dtype=torch.int32, device=dev), k[t].view(1, 1, HEAD_DIM), score[t].view(1, 1, HEAD_DIM), ape, @@ -351,6 +352,91 @@ def test_decode_writer_matches_prefill_writer(pool_size): ) +@pytest.mark.parametrize("ring_pools", [1, 2]) +def test_rejected_draft_redo_needs_ring_slots(ring_pools): + """With a one-pool ring, the drafts behind a rejected pool-completing draft + overwrote the pool's earlier keys, so its redo compressed wrong keys.""" + pool, spec, page, nblk = 4, 3, 64, 2 + ring = ring_pools * pool + dev = "cuda" + torch.manual_seed(1) + n_tok = 3 * pool + k = torch.randn(n_tok, HEAD_DIM, dtype=torch.bfloat16, device=dev) + score = torch.randn(n_tok, HEAD_DIM, dtype=torch.bfloat16, device=dev) + ape = torch.randn(pool, HEAD_DIM, dtype=torch.float32, device=dev) + kv_ref = torch.zeros(nblk, page, HEAD_DIM + 4, dtype=torch.uint8, device=dev) + kpool_compress_and_write_cache( + kv_ref, + k.view(3, pool, HEAD_DIM), + score.view(3, pool, HEAD_DIM), + ape, + torch.arange(3, dtype=torch.int64, device=dev), + pool_size=pool, + head_dim=HEAD_DIM, + round_scale=ROUND_SCALE, + ) + + kv = torch.zeros_like(kv_ref) + tail = torch.zeros(nblk, 2, ring, HEAD_DIM, dtype=torch.bfloat16, device=dev) + + def pool_bytes(cache, p): + # Page layout: [page * HEAD_DIM bytes of K rows | page * 4 bytes of scales]. + flat = cache[0].reshape(-1) + k_bytes = flat[p * HEAD_DIM : (p + 1) * HEAD_DIM] + s_bytes = flat[page * HEAD_DIM + 4 * p : page * HEAD_DIM + 4 * (p + 1)] + return torch.cat([k_bytes, s_bytes]) + + def step(positions, keys, scores): + pos = torch.tensor([positions], dtype=torch.int32, device=dev) + slots = [(p // pool) if p % pool == pool - 1 else -1 for p in positions] + kpool_decode_update_and_maybe_write_cache_batched( + kv, + tail, + pos % ring, + keys.view(1, -1, HEAD_DIM), + scores.view(1, -1, HEAD_DIM), + ape, + torch.tensor([slots], dtype=torch.int32, device=dev), + pos, + pool, + HEAD_DIM, + round_scale=ROUND_SCALE, + ) + + for t in range(7): + step([t], k[t], score[t]) + # Control: verified token 7 completes pool 1 before drafts 8..10 are stashed. + drafts = torch.randn(spec, HEAD_DIM, dtype=torch.bfloat16, device=dev) + draft_scores = torch.randn(spec, HEAD_DIM, dtype=torch.bfloat16, device=dev) + step( + [7, 8, 9, 10], + torch.cat([k[7:8], drafts]), + torch.cat([score[7:8], draft_scores]), + ) + step([8, 9, 10, 11], k[8:12], score[8:12]) # all drafts rejected + for p in (1, 2): + assert torch.equal(pool_bytes(kv, p), pool_bytes(kv_ref, p)), p + + # Draft 7 completes pool 1 and is rejected. With a one-pool ring, drafts + # 8 and 9 overwrite the slots of positions 4 and 5, which are read by the + # redo of 7. + kv.zero_() + tail.zero_() + for t in range(6): + step([t], k[t], score[t]) + step( + [6, 7, 8, 9], + torch.cat([k[6:7], drafts]), + torch.cat([score[6:7], draft_scores]), + ) + step([7, 8, 9, 10], k[7:11], score[7:11]) + pool1_ok = torch.equal(pool_bytes(kv, 1), pool_bytes(kv_ref, 1)) + if ring_pools == 1: + assert not pool1_ok, "expected a one-pool ring to corrupt pool 1" + else: + assert pool1_ok + + def test_leading_invalid_tail_slot(): """A request whose FIRST token carries an invalid (-1) tail slot while a later token is a real pool completion. diff --git a/tests/v1/attention/test_kpool_tail_slot_mapping.py b/tests/v1/attention/test_kpool_tail_slot_mapping.py index ed0e250b18..157366e49c 100644 --- a/tests/v1/attention/test_kpool_tail_slot_mapping.py +++ b/tests/v1/attention/test_kpool_tail_slot_mapping.py @@ -24,6 +24,8 @@ from types import SimpleNamespace import pytest import torch +from vllm.config import VllmConfig, set_current_vllm_config +from vllm.models.glm5next.common.attention import Glm5NextTailCache from vllm.v1.attention.backend import CommonAttentionMetadata from vllm.v1.attention.backends.mla.indexer import ( KpoolTailBackend, @@ -56,6 +58,30 @@ def test_tail_backend_layout_matches_kernel_pointer_arithmetic(): assert content_stride == 1 +@pytest.mark.parametrize( + "num_speculative_tokens, ring", [(0, 4), (1, 8), (4, 8), (7, 16), (13, 32)] +) +def test_tail_ring_divides_the_attention_block(num_speculative_tokens, ring): + """With 7 draft tokens the ring was 12 slots. 12 does not divide the + 640-token KDA attention block, so the scheduler block grew to their lcm + and prefix-cache hits were cut down to multiples of 1920.""" + with set_current_vllm_config(VllmConfig()): + cache = Glm5NextTailCache( + head_dim=128, + dtype=torch.bfloat16, + prefix="tail", + cache_config=SimpleNamespace(block_size=640), + index_kpool=KPOOL, + ) + spec = cache.get_kv_cache_spec( + SimpleNamespace(num_speculative_tokens=num_speculative_tokens) + ) + + assert spec.block_size == spec.sliding_window == ring + assert ring >= KPOOL + num_speculative_tokens + assert 640 % ring == 0 + + def test_tail_spec_opts_out_of_generic_slot_mapping(): """The tail row is one block wide (padded to the block-table alignment), so the generic kernel's ``pos // kpool`` column index runs off the end of the @@ -296,31 +322,30 @@ def test_builder_reuses_slot_mapping_storage(): class TailRingMirror: - """Mirror of _kpool_tail_seed_kernel / _kpool_decode_update_batched_kernel - addressing: block = tail_slot // kpool, ring offset = pos % kpool; a pool - completing at pos reads ring slots (pool_start + s) % kpool and uses the - current token's own K/score for the last member.""" + """Mirror of the tail-ring addressing in _kpool_tail_seed_kernel and + _kpool_decode_update_batched_kernel.""" - def __init__(self, num_blocks, kpool=KPOOL): + def __init__(self, num_blocks, kpool=KPOOL, ring=None): self.kpool = kpool - self.k = torch.full((num_blocks, kpool, 3), float("nan")) - self.s = torch.full((num_blocks, kpool, 3), float("nan")) + self.ring = ring or kpool + self.k = torch.full((num_blocks, self.ring, 3), float("nan")) + self.s = torch.full((num_blocks, self.ring, 3), float("nan")) def stash(self, tail_slot, pos, k, s): - blk, off = tail_slot // self.kpool, pos % self.kpool + blk, off = tail_slot // self.ring, pos % self.ring self.k[blk, off] = k self.s[blk, off] = s seed = stash # the seed kernel writes with the same addressing def complete(self, tail_slot, pos, k, s): - blk = tail_slot // self.kpool + blk = tail_slot // self.ring start = pos - (self.kpool - 1) kk = torch.stack( - [self.k[blk, (start + i) % self.kpool] for i in range(self.kpool)] + [self.k[blk, (start + i) % self.ring] for i in range(self.kpool)] ) ss = torch.stack( - [self.s[blk, (start + i) % self.kpool] for i in range(self.kpool)] + [self.s[blk, (start + i) % self.ring] for i in range(self.kpool)] ) kk[-1], ss[-1] = k, s # is_current for the completing token w = torch.softmax(ss, dim=0) @@ -454,3 +479,35 @@ def test_triton_mapping_reads_strided_block_table(): for req, blk in enumerate(own_blocks): start, end = int(qsl[req]), int(qsl[req + 1]) assert (got[start:end] // KPOOL == blk).all() + + +@pytest.mark.parametrize("ring_pools", [1, 2]) +def test_rejected_completing_draft_needs_ring_slots(ring_pools): + """With a one-pool ring, the drafts behind a rejected pool-completing draft + overwrote the pool's earlier keys, so its redo compressed wrong keys.""" + ring_size = ring_pools * KPOOL + truth = TailRingMirror(num_blocks=2, ring=ring_size) + ring = TailRingMirror(num_blocks=2, ring=ring_size) + block = 1 + + def slot(pos): + return block * ring_size + pos % ring_size + + for pos in range(4, 7): + truth.stash(slot(pos), pos, *token_kv(0, pos)) + ring.stash(slot(pos), pos, *token_kv(0, pos)) + expected = truth.complete(slot(7), 7, *token_kv(0, 7)) + + # Draft 7 completes the pool and is rejected. With a one-pool ring, drafts + # 8..10 overwrite the slots of positions 4..6, which are read by the redo + # of 7. + for pos in range(7, 11): + k, s = token_kv(9, pos) # draft values + if pos % KPOOL == KPOOL - 1: + ring.complete(slot(pos), pos, k, s) + ring.stash(slot(pos), pos, k, s) + redo = ring.complete(slot(7), 7, *token_kv(0, 7)) + if ring_pools == 1: + assert not torch.allclose(redo, expected) + else: + torch.testing.assert_close(redo, expected) diff --git a/vllm/models/glm5next/amd/ops/kpool_compress.py b/vllm/models/glm5next/amd/ops/kpool_compress.py index 97d8314c09..766950f332 100644 --- a/vllm/models/glm5next/amd/ops/kpool_compress.py +++ b/vllm/models/glm5next/amd/ops/kpool_compress.py @@ -327,37 +327,38 @@ def _kpool_tail_seed_kernel( KPOOL_HEAD: tl.constexpr, HEAD_DIM: tl.constexpr, KPOOL: tl.constexpr, + RING: tl.constexpr, BLOCK_D: tl.constexpr, ): - """Copy token ``i``'s raw K + gate into its request's tail block. + """Copy token ``i``'s raw K + gate into its request's tail ring. Token ``i`` is among its request's last KPOOL tokens iff the token KPOOL ahead belongs to a different tail block (or is past the batch / padding, - slot < 0). ``tslot = block * KPOOL + pos % KPOOL``; the destination is - ``tail[block, {0:K, 1:score}, pos % KPOOL, :]``. + slot < 0). ``tslot = block * RING + pos % RING``; the destination is + ``tail[block, {0:K, 1:score}, pos % RING, :]``. """ i = tl.program_id(0) t = tl.load(tslot_ptr + i).to(tl.int64) if t < 0: return - blk = t // KPOOL # t >= 0 here, so trunc == floor + blk = t // RING # t >= 0 here, so trunc == floor ahead = tl.load(tslot_ptr + i + KPOOL, mask=i + KPOOL < n_tokens, other=-1).to( tl.int64 ) # Match the torch semantics exactly: a negative ahead slot floors to a # block id that differs from every real block -> token is in the tail. # Only divide non-negative slots (Triton int div truncates, torch floors). - if ahead >= 0 and ahead // KPOOL == blk: + if ahead >= 0 and ahead // RING == blk: return offs = tl.arange(0, BLOCK_D) m = offs < HEAD_DIM block_base = blk * TAIL_BLOCK_ELEMS - base = block_base + (t % KPOOL) * HEAD_DIM + base = block_base + (t % RING) * HEAD_DIM k = tl.load(key_ptr + i * HEAD_DIM + offs, mask=m) s = tl.load(score_ptr + i * HEAD_DIM + offs, mask=m) tl.store(tail_ptr + base + offs, k, mask=m) tl.store( - tail_ptr + block_base + KPOOL_HEAD + (t % KPOOL) * HEAD_DIM + offs, s, mask=m + tail_ptr + block_base + KPOOL_HEAD + (t % RING) * HEAD_DIM + offs, s, mask=m ) @@ -385,6 +386,7 @@ def kpool_seed_tail_cache( KPOOL_HEAD=tail_kv_cache.stride(1), HEAD_DIM=head_dim, KPOOL=kpool, + RING=tail_kv_cache.shape[2], BLOCK_D=triton.next_power_of_2(head_dim), ) @@ -412,6 +414,7 @@ def _kpool_decode_update_batched_kernel( PAGE_SIZE: tl.constexpr, BUF_NUMEL_PER_PAGE: tl.constexpr, POOL_SIZE: tl.constexpr, + RING: tl.constexpr, TAIL_BLOCK_ELEMS: tl.constexpr, KPOOL_HEAD: tl.constexpr, HEAD_DIM: tl.constexpr, @@ -428,10 +431,7 @@ def _kpool_decode_update_batched_kernel( the tail-ring slots that tokens t < t* (same request) just stashed in this same invocation. ``tl.range`` iterates sequentially within the program, so those stashes are visible to the later completion read. Cross-request - programs are independent (distinct tail blocks). With NEXT_N < POOL_SIZE - (the spec-verify case: NEXT_N ~= num_spec+1, POOL_SIZE=16) at most one - completion can occur per request per call, but the ordered loop is correct - for any NEXT_N. + programs are independent (distinct tail blocks). RING >= POOL_SIZE. """ req = tl.program_id(0) offs = tl.arange(0, BLOCK_D) @@ -445,7 +445,7 @@ def _kpool_decode_update_batched_kernel( pos_valid = (cache_loc >= 0) & (pos >= 0) slot = safe_pos % POOL_SIZE - phys_slot = safe_pos % POOL_SIZE + phys_slot = safe_pos % RING # Derive the tail block from THIS token's tail_slot (the request's block # is constant across a pool, but a padded / invalid entry carries a @@ -453,7 +453,7 @@ def _kpool_decode_update_batched_kernel( # token's base address). Clamp so an invalid entry can never form an # out-of-bounds base; the accesses below are gated on pos_valid anyway. tail_slot = tl.load(tail_slot_mapping_ptr + idx) - block = tl.maximum(tail_slot, 0).to(tl.int64) // POOL_SIZE + block = tl.maximum(tail_slot, 0).to(tl.int64) // RING block_base = block * TAIL_BLOCK_ELEMS # The tail-ring stash must run for EVERY real token, so it is gated on @@ -481,7 +481,7 @@ def _kpool_decode_update_batched_kernel( max_score = tl.full((BLOCK_D,), -float("inf"), tl.float32) for pool_slot in tl.static_range(0, POOL_SIZE): is_current = pool_slot == slot - phys = (pool_logical_start + pool_slot) % POOL_SIZE + phys = (pool_logical_start + pool_slot) % RING score_buf = tl.load( tail_kv_ptr + block_base + KPOOL_HEAD + phys * HEAD_DIM + offs, mask=dim_mask, @@ -499,7 +499,7 @@ def _kpool_decode_update_batched_kernel( denom = tl.full((BLOCK_D,), 0.0, tl.float32) for pool_slot in tl.static_range(0, POOL_SIZE): is_current = pool_slot == slot - phys = (pool_logical_start + pool_slot) % POOL_SIZE + phys = (pool_logical_start + pool_slot) % RING score_buf = tl.load( tail_kv_ptr + block_base + KPOOL_HEAD + phys * HEAD_DIM + offs, mask=dim_mask, @@ -609,7 +609,8 @@ def kpool_decode_update_and_maybe_write_cache_batched( return assert tail_kv_cache.ndim == 4 assert tail_kv_cache.shape[1] == 2 - assert tail_kv_cache.shape[2] == pool_size + ring = tail_kv_cache.shape[2] + assert ring >= pool_size and ring % pool_size == 0, (ring, pool_size) assert tail_kv_cache.shape[3] == head_dim assert tail_kv_cache.dtype == torch.bfloat16 assert key.ndim == 3 and key.shape[2] == head_dim @@ -658,6 +659,7 @@ def kpool_decode_update_and_maybe_write_cache_batched( PAGE_SIZE=page_size, BUF_NUMEL_PER_PAGE=buf.stride(0), POOL_SIZE=pool_size, + RING=ring, TAIL_BLOCK_ELEMS=tail_kv_cache.stride(0), KPOOL_HEAD=tail_kv_cache.stride(1), HEAD_DIM=head_dim, diff --git a/vllm/models/glm5next/common/attention.py b/vllm/models/glm5next/common/attention.py index 4f776e8a8e..25f51117a8 100644 --- a/vllm/models/glm5next/common/attention.py +++ b/vllm/models/glm5next/common/attention.py @@ -34,6 +34,7 @@ from vllm.models.glm5next.sparse_indexer import SparseAttnIndexerKpool from vllm.platforms import current_platform from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES +from vllm.utils.math_utils import cdiv, next_power_of_2 from vllm.v1.kv_cache_interface import KpoolTailSpec, MLAAttentionSpec logger = init_logger(__name__) @@ -160,7 +161,7 @@ class Glm5NextTailCache(DeepseekV32IndexerCache): """Paged circular buffer for the kpool indexer's in-progress (tail) pool. Holds the trailing incomplete pool's raw K + gate score: one block of - ``index_kpool`` slots per request, overwritten in place by ``pos % kpool`` + ``ring`` slots per request, overwritten in place by ``pos % ring`` as decode/spec-decode advances. Prefill seeds it (instead of discarding the tail raw K+gate); the connector transfers it across PD; decode reads it to compress the boundary pool correctly. ``KpoolTailSpec`` / @@ -190,13 +191,24 @@ class Glm5NextTailCache(DeepseekV32IndexerCache): def get_kv_cache_spec(self, vllm_config: VllmConfig): # The two head slots form [K, gate score] in the generic # [block, head, state, content] cache view. + # Drafts are stashed before acceptance. With a one-pool ring, the + # drafts behind a rejected pool-completing draft overwrite the keys + # read by its redo. + span = self._index_kpool + vllm_config.num_speculative_tokens + ring = self._index_kpool * next_power_of_2(cdiv(span, self._index_kpool)) + # ring must divide the attention block size (a multiple of 128). + assert self.cache_config.block_size % ring == 0, ( + f"Glm5NextTailCache: cache_config.block_size " + f"({self.cache_config.block_size}) must be a multiple of the " + f"tail ring ({ring})" + ) return KpoolTailSpec( - block_size=self._index_kpool, + block_size=ring, num_kv_heads=2, head_size=self.head_dim, head_size_v=0, dtype=torch.bfloat16, - sliding_window=self._index_kpool, + sliding_window=ring, ) def get_attn_backend(self): diff --git a/vllm/models/glm5next/nvidia/ops/kpool_compress.py b/vllm/models/glm5next/nvidia/ops/kpool_compress.py index 416d3b5c48..db496d300b 100644 --- a/vllm/models/glm5next/nvidia/ops/kpool_compress.py +++ b/vllm/models/glm5next/nvidia/ops/kpool_compress.py @@ -387,14 +387,15 @@ def _kpool_tail_seed_kernel( KPOOL_HEAD: tl.constexpr, HEAD_DIM: tl.constexpr, KPOOL: tl.constexpr, + RING: tl.constexpr, BLOCK_D: tl.constexpr, ): - """Copy token ``i``'s raw K + gate into its request's tail block. + """Copy token ``i``'s raw K + gate into its request's tail ring. Token ``i`` is among its request's last KPOOL tokens iff the token KPOOL ahead belongs to a different tail block (or is past the batch / padding, - slot < 0). ``tslot = block * KPOOL + pos % KPOOL``; the destination is - ``tail[block, {0:K, 1:score}, pos % KPOOL, :]``. + slot < 0). ``tslot = block * RING + pos % RING``; the destination is + ``tail[block, {0:K, 1:score}, pos % RING, :]``. The tail cache aliases the indexer cache with the indexer's (padded) block stride, so blocks are addressed through ``TAIL_BLOCK_ELEMS`` / @@ -405,18 +406,18 @@ def _kpool_tail_seed_kernel( t = tl.load(tslot_ptr + i).to(tl.int64) if t < 0: return - blk = t // KPOOL # t >= 0 here, so trunc == floor + blk = t // RING # t >= 0 here, so trunc == floor ahead = tl.load(tslot_ptr + i + KPOOL, mask=i + KPOOL < n_tokens, other=-1).to( tl.int64 ) # Match the torch semantics exactly: a negative ahead slot floors to a # block id that differs from every real block -> token is in the tail. # Only divide non-negative slots (Triton int div truncates, torch floors). - if ahead >= 0 and ahead // KPOOL == blk: + if ahead >= 0 and ahead // RING == blk: return offs = tl.arange(0, BLOCK_D) m = offs < HEAD_DIM - base = blk * TAIL_BLOCK_ELEMS + (t % KPOOL) * HEAD_DIM + base = blk * TAIL_BLOCK_ELEMS + (t % RING) * HEAD_DIM k = tl.load(key_ptr + i * HEAD_DIM + offs, mask=m) s = tl.load(score_ptr + i * HEAD_DIM + offs, mask=m) tl.store(tail_ptr + base + offs, k, mask=m) @@ -449,6 +450,7 @@ def kpool_seed_tail_cache( KPOOL_HEAD=tail_kv_cache.stride(1), HEAD_DIM=head_dim, KPOOL=kpool, + RING=tail_kv_cache.shape[2], BLOCK_D=triton.next_power_of_2(head_dim), ) @@ -476,6 +478,7 @@ def _kpool_decode_update_batched_kernel( PAGE_SIZE: tl.constexpr, BUF_NUMEL_PER_PAGE: tl.constexpr, POOL_SIZE: tl.constexpr, + RING: tl.constexpr, TAIL_BLOCK_ELEMS: tl.constexpr, KPOOL_HEAD: tl.constexpr, HEAD_DIM: tl.constexpr, @@ -490,10 +493,7 @@ def _kpool_decode_update_batched_kernel( the tail-ring slots that tokens t < t* (same request) just stashed in this same invocation. ``tl.range`` iterates sequentially within the program, so those stashes are visible to the later completion read. Cross-request - programs are independent (distinct tail blocks). With NEXT_N < POOL_SIZE - (the spec-verify case: NEXT_N ~= num_spec+1, POOL_SIZE=16) at most one - completion can occur per request per call, but the ordered loop is correct - for any NEXT_N. + programs are independent (distinct tail blocks). RING >= POOL_SIZE. """ req = tl.program_id(0) offs = tl.arange(0, BLOCK_D) @@ -507,7 +507,7 @@ def _kpool_decode_update_batched_kernel( pos_valid = (cache_loc >= 0) & (pos >= 0) slot = safe_pos % POOL_SIZE - phys_slot = safe_pos % POOL_SIZE + phys_slot = safe_pos % RING # Derive the tail block from THIS token's tail_slot (the request's block # is constant across a pool, but a padded / invalid entry carries a @@ -515,7 +515,7 @@ def _kpool_decode_update_batched_kernel( # token's base address). Clamp so an invalid entry can never form an # out-of-bounds base; the accesses below are gated on pos_valid anyway. tail_slot = tl.load(tail_slot_mapping_ptr + idx) - block = tl.maximum(tail_slot, 0).to(tl.int64) // POOL_SIZE + block = tl.maximum(tail_slot, 0).to(tl.int64) // RING block_base = block * TAIL_BLOCK_ELEMS # The tail-ring stash must run for EVERY real token, so it is gated on @@ -543,7 +543,7 @@ def _kpool_decode_update_batched_kernel( max_score = tl.full((BLOCK_D,), -float("inf"), tl.float32) for pool_slot in tl.static_range(0, POOL_SIZE): is_current = pool_slot == slot - phys = (pool_logical_start + pool_slot) % POOL_SIZE + phys = (pool_logical_start + pool_slot) % RING score_buf = tl.load( tail_kv_ptr + block_base + KPOOL_HEAD + phys * HEAD_DIM + offs, mask=dim_mask, @@ -561,7 +561,7 @@ def _kpool_decode_update_batched_kernel( denom = tl.full((BLOCK_D,), 0.0, tl.float32) for pool_slot in tl.static_range(0, POOL_SIZE): is_current = pool_slot == slot - phys = (pool_logical_start + pool_slot) % POOL_SIZE + phys = (pool_logical_start + pool_slot) % RING score_buf = tl.load( tail_kv_ptr + block_base + KPOOL_HEAD + phys * HEAD_DIM + offs, mask=dim_mask, @@ -671,7 +671,8 @@ def kpool_decode_update_and_maybe_write_cache_batched( return assert tail_kv_cache.ndim == 4 assert tail_kv_cache.shape[1] == 2 - assert tail_kv_cache.shape[2] == pool_size + ring = tail_kv_cache.shape[2] + assert ring >= pool_size and ring % pool_size == 0, (ring, pool_size) assert tail_kv_cache.shape[3] == head_dim assert tail_kv_cache.dtype == torch.bfloat16 assert key.ndim == 3 and key.shape[2] == head_dim @@ -717,6 +718,7 @@ def kpool_decode_update_and_maybe_write_cache_batched( PAGE_SIZE=page_size, BUF_NUMEL_PER_PAGE=buf.stride(0), POOL_SIZE=pool_size, + RING=ring, TAIL_BLOCK_ELEMS=tail_kv_cache.stride(0), KPOOL_HEAD=tail_kv_cache.stride(1), HEAD_DIM=head_dim,