[Bugfix][GLM-5.3-Flash] kpool corruption with speculative decoding (#58454)
Signed-off-by: Matt Mastracci <matthew@mastracci.com> Signed-off-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
Claude Opus 5.5
Robert Shaw
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
f83ea61cfe
commit
2617fe9383
@@ -293,8 +293,9 @@ def _run_kernel(kv, tail, tail_slot, key, score, ape, slot_map, pos):
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pool_size", [4, 16])
|
||||
def test_decode_writer_matches_prefill_writer(pool_size):
|
||||
"""Compare production decode and prefill writers for pool sizes 4 and 16."""
|
||||
@pytest.mark.parametrize("ring_pools", [1, 2])
|
||||
def test_decode_writer_matches_prefill_writer(pool_size, ring_pools):
|
||||
ring = ring_pools * pool_size
|
||||
n_pools, page, nblk = 8, 64, 4
|
||||
n_tok = n_pools * pool_size
|
||||
dev = "cuda"
|
||||
@@ -317,14 +318,14 @@ def test_decode_writer_matches_prefill_writer(pool_size):
|
||||
|
||||
# One request owning tail block 0, fed one token per decode step.
|
||||
kv_decode = torch.zeros_like(kv_prefill)
|
||||
tail = torch.zeros(nblk, 2, pool_size, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
tail = torch.zeros(nblk, 2, ring, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
for t in range(n_tok):
|
||||
completes = t % pool_size == pool_size - 1
|
||||
kpool_decode_update_and_maybe_write_cache_batched(
|
||||
kv_decode,
|
||||
tail,
|
||||
# token-granular: every token has a valid tail slot
|
||||
torch.tensor([[t % pool_size]], dtype=torch.int32, device=dev),
|
||||
torch.tensor([[t % ring]], dtype=torch.int32, device=dev),
|
||||
k[t].view(1, 1, HEAD_DIM),
|
||||
score[t].view(1, 1, HEAD_DIM),
|
||||
ape,
|
||||
@@ -351,6 +352,91 @@ def test_decode_writer_matches_prefill_writer(pool_size):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ring_pools", [1, 2])
|
||||
def test_rejected_draft_redo_needs_ring_slots(ring_pools):
|
||||
"""With a one-pool ring, the drafts behind a rejected pool-completing draft
|
||||
overwrote the pool's earlier keys, so its redo compressed wrong keys."""
|
||||
pool, spec, page, nblk = 4, 3, 64, 2
|
||||
ring = ring_pools * pool
|
||||
dev = "cuda"
|
||||
torch.manual_seed(1)
|
||||
n_tok = 3 * pool
|
||||
k = torch.randn(n_tok, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
score = torch.randn(n_tok, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
ape = torch.randn(pool, HEAD_DIM, dtype=torch.float32, device=dev)
|
||||
kv_ref = torch.zeros(nblk, page, HEAD_DIM + 4, dtype=torch.uint8, device=dev)
|
||||
kpool_compress_and_write_cache(
|
||||
kv_ref,
|
||||
k.view(3, pool, HEAD_DIM),
|
||||
score.view(3, pool, HEAD_DIM),
|
||||
ape,
|
||||
torch.arange(3, dtype=torch.int64, device=dev),
|
||||
pool_size=pool,
|
||||
head_dim=HEAD_DIM,
|
||||
round_scale=ROUND_SCALE,
|
||||
)
|
||||
|
||||
kv = torch.zeros_like(kv_ref)
|
||||
tail = torch.zeros(nblk, 2, ring, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
|
||||
def pool_bytes(cache, p):
|
||||
# Page layout: [page * HEAD_DIM bytes of K rows | page * 4 bytes of scales].
|
||||
flat = cache[0].reshape(-1)
|
||||
k_bytes = flat[p * HEAD_DIM : (p + 1) * HEAD_DIM]
|
||||
s_bytes = flat[page * HEAD_DIM + 4 * p : page * HEAD_DIM + 4 * (p + 1)]
|
||||
return torch.cat([k_bytes, s_bytes])
|
||||
|
||||
def step(positions, keys, scores):
|
||||
pos = torch.tensor([positions], dtype=torch.int32, device=dev)
|
||||
slots = [(p // pool) if p % pool == pool - 1 else -1 for p in positions]
|
||||
kpool_decode_update_and_maybe_write_cache_batched(
|
||||
kv,
|
||||
tail,
|
||||
pos % ring,
|
||||
keys.view(1, -1, HEAD_DIM),
|
||||
scores.view(1, -1, HEAD_DIM),
|
||||
ape,
|
||||
torch.tensor([slots], dtype=torch.int32, device=dev),
|
||||
pos,
|
||||
pool,
|
||||
HEAD_DIM,
|
||||
round_scale=ROUND_SCALE,
|
||||
)
|
||||
|
||||
for t in range(7):
|
||||
step([t], k[t], score[t])
|
||||
# Control: verified token 7 completes pool 1 before drafts 8..10 are stashed.
|
||||
drafts = torch.randn(spec, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
draft_scores = torch.randn(spec, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
step(
|
||||
[7, 8, 9, 10],
|
||||
torch.cat([k[7:8], drafts]),
|
||||
torch.cat([score[7:8], draft_scores]),
|
||||
)
|
||||
step([8, 9, 10, 11], k[8:12], score[8:12]) # all drafts rejected
|
||||
for p in (1, 2):
|
||||
assert torch.equal(pool_bytes(kv, p), pool_bytes(kv_ref, p)), p
|
||||
|
||||
# Draft 7 completes pool 1 and is rejected. With a one-pool ring, drafts
|
||||
# 8 and 9 overwrite the slots of positions 4 and 5, which are read by the
|
||||
# redo of 7.
|
||||
kv.zero_()
|
||||
tail.zero_()
|
||||
for t in range(6):
|
||||
step([t], k[t], score[t])
|
||||
step(
|
||||
[6, 7, 8, 9],
|
||||
torch.cat([k[6:7], drafts]),
|
||||
torch.cat([score[6:7], draft_scores]),
|
||||
)
|
||||
step([7, 8, 9, 10], k[7:11], score[7:11])
|
||||
pool1_ok = torch.equal(pool_bytes(kv, 1), pool_bytes(kv_ref, 1))
|
||||
if ring_pools == 1:
|
||||
assert not pool1_ok, "expected a one-pool ring to corrupt pool 1"
|
||||
else:
|
||||
assert pool1_ok
|
||||
|
||||
|
||||
def test_leading_invalid_tail_slot():
|
||||
"""A request whose FIRST token carries an invalid (-1) tail slot while a
|
||||
later token is a real pool completion.
|
||||
|
||||
@@ -24,6 +24,8 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.models.glm5next.common.attention import Glm5NextTailCache
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
KpoolTailBackend,
|
||||
@@ -56,6 +58,30 @@ def test_tail_backend_layout_matches_kernel_pointer_arithmetic():
|
||||
assert content_stride == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_speculative_tokens, ring", [(0, 4), (1, 8), (4, 8), (7, 16), (13, 32)]
|
||||
)
|
||||
def test_tail_ring_divides_the_attention_block(num_speculative_tokens, ring):
|
||||
"""With 7 draft tokens the ring was 12 slots. 12 does not divide the
|
||||
640-token KDA attention block, so the scheduler block grew to their lcm
|
||||
and prefix-cache hits were cut down to multiples of 1920."""
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
cache = Glm5NextTailCache(
|
||||
head_dim=128,
|
||||
dtype=torch.bfloat16,
|
||||
prefix="tail",
|
||||
cache_config=SimpleNamespace(block_size=640),
|
||||
index_kpool=KPOOL,
|
||||
)
|
||||
spec = cache.get_kv_cache_spec(
|
||||
SimpleNamespace(num_speculative_tokens=num_speculative_tokens)
|
||||
)
|
||||
|
||||
assert spec.block_size == spec.sliding_window == ring
|
||||
assert ring >= KPOOL + num_speculative_tokens
|
||||
assert 640 % ring == 0
|
||||
|
||||
|
||||
def test_tail_spec_opts_out_of_generic_slot_mapping():
|
||||
"""The tail row is one block wide (padded to the block-table alignment), so
|
||||
the generic kernel's ``pos // kpool`` column index runs off the end of the
|
||||
@@ -296,31 +322,30 @@ def test_builder_reuses_slot_mapping_storage():
|
||||
|
||||
|
||||
class TailRingMirror:
|
||||
"""Mirror of _kpool_tail_seed_kernel / _kpool_decode_update_batched_kernel
|
||||
addressing: block = tail_slot // kpool, ring offset = pos % kpool; a pool
|
||||
completing at pos reads ring slots (pool_start + s) % kpool and uses the
|
||||
current token's own K/score for the last member."""
|
||||
"""Mirror of the tail-ring addressing in _kpool_tail_seed_kernel and
|
||||
_kpool_decode_update_batched_kernel."""
|
||||
|
||||
def __init__(self, num_blocks, kpool=KPOOL):
|
||||
def __init__(self, num_blocks, kpool=KPOOL, ring=None):
|
||||
self.kpool = kpool
|
||||
self.k = torch.full((num_blocks, kpool, 3), float("nan"))
|
||||
self.s = torch.full((num_blocks, kpool, 3), float("nan"))
|
||||
self.ring = ring or kpool
|
||||
self.k = torch.full((num_blocks, self.ring, 3), float("nan"))
|
||||
self.s = torch.full((num_blocks, self.ring, 3), float("nan"))
|
||||
|
||||
def stash(self, tail_slot, pos, k, s):
|
||||
blk, off = tail_slot // self.kpool, pos % self.kpool
|
||||
blk, off = tail_slot // self.ring, pos % self.ring
|
||||
self.k[blk, off] = k
|
||||
self.s[blk, off] = s
|
||||
|
||||
seed = stash # the seed kernel writes with the same addressing
|
||||
|
||||
def complete(self, tail_slot, pos, k, s):
|
||||
blk = tail_slot // self.kpool
|
||||
blk = tail_slot // self.ring
|
||||
start = pos - (self.kpool - 1)
|
||||
kk = torch.stack(
|
||||
[self.k[blk, (start + i) % self.kpool] for i in range(self.kpool)]
|
||||
[self.k[blk, (start + i) % self.ring] for i in range(self.kpool)]
|
||||
)
|
||||
ss = torch.stack(
|
||||
[self.s[blk, (start + i) % self.kpool] for i in range(self.kpool)]
|
||||
[self.s[blk, (start + i) % self.ring] for i in range(self.kpool)]
|
||||
)
|
||||
kk[-1], ss[-1] = k, s # is_current for the completing token
|
||||
w = torch.softmax(ss, dim=0)
|
||||
@@ -454,3 +479,35 @@ def test_triton_mapping_reads_strided_block_table():
|
||||
for req, blk in enumerate(own_blocks):
|
||||
start, end = int(qsl[req]), int(qsl[req + 1])
|
||||
assert (got[start:end] // KPOOL == blk).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ring_pools", [1, 2])
|
||||
def test_rejected_completing_draft_needs_ring_slots(ring_pools):
|
||||
"""With a one-pool ring, the drafts behind a rejected pool-completing draft
|
||||
overwrote the pool's earlier keys, so its redo compressed wrong keys."""
|
||||
ring_size = ring_pools * KPOOL
|
||||
truth = TailRingMirror(num_blocks=2, ring=ring_size)
|
||||
ring = TailRingMirror(num_blocks=2, ring=ring_size)
|
||||
block = 1
|
||||
|
||||
def slot(pos):
|
||||
return block * ring_size + pos % ring_size
|
||||
|
||||
for pos in range(4, 7):
|
||||
truth.stash(slot(pos), pos, *token_kv(0, pos))
|
||||
ring.stash(slot(pos), pos, *token_kv(0, pos))
|
||||
expected = truth.complete(slot(7), 7, *token_kv(0, 7))
|
||||
|
||||
# Draft 7 completes the pool and is rejected. With a one-pool ring, drafts
|
||||
# 8..10 overwrite the slots of positions 4..6, which are read by the redo
|
||||
# of 7.
|
||||
for pos in range(7, 11):
|
||||
k, s = token_kv(9, pos) # draft values
|
||||
if pos % KPOOL == KPOOL - 1:
|
||||
ring.complete(slot(pos), pos, k, s)
|
||||
ring.stash(slot(pos), pos, k, s)
|
||||
redo = ring.complete(slot(7), 7, *token_kv(0, 7))
|
||||
if ring_pools == 1:
|
||||
assert not torch.allclose(redo, expected)
|
||||
else:
|
||||
torch.testing.assert_close(redo, expected)
|
||||
|
||||
@@ -327,37 +327,38 @@ def _kpool_tail_seed_kernel(
|
||||
KPOOL_HEAD: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
KPOOL: tl.constexpr,
|
||||
RING: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
"""Copy token ``i``'s raw K + gate into its request's tail block.
|
||||
"""Copy token ``i``'s raw K + gate into its request's tail ring.
|
||||
|
||||
Token ``i`` is among its request's last KPOOL tokens iff the token KPOOL
|
||||
ahead belongs to a different tail block (or is past the batch / padding,
|
||||
slot < 0). ``tslot = block * KPOOL + pos % KPOOL``; the destination is
|
||||
``tail[block, {0:K, 1:score}, pos % KPOOL, :]``.
|
||||
slot < 0). ``tslot = block * RING + pos % RING``; the destination is
|
||||
``tail[block, {0:K, 1:score}, pos % RING, :]``.
|
||||
"""
|
||||
i = tl.program_id(0)
|
||||
t = tl.load(tslot_ptr + i).to(tl.int64)
|
||||
if t < 0:
|
||||
return
|
||||
blk = t // KPOOL # t >= 0 here, so trunc == floor
|
||||
blk = t // RING # t >= 0 here, so trunc == floor
|
||||
ahead = tl.load(tslot_ptr + i + KPOOL, mask=i + KPOOL < n_tokens, other=-1).to(
|
||||
tl.int64
|
||||
)
|
||||
# Match the torch semantics exactly: a negative ahead slot floors to a
|
||||
# block id that differs from every real block -> token is in the tail.
|
||||
# Only divide non-negative slots (Triton int div truncates, torch floors).
|
||||
if ahead >= 0 and ahead // KPOOL == blk:
|
||||
if ahead >= 0 and ahead // RING == blk:
|
||||
return
|
||||
offs = tl.arange(0, BLOCK_D)
|
||||
m = offs < HEAD_DIM
|
||||
block_base = blk * TAIL_BLOCK_ELEMS
|
||||
base = block_base + (t % KPOOL) * HEAD_DIM
|
||||
base = block_base + (t % RING) * HEAD_DIM
|
||||
k = tl.load(key_ptr + i * HEAD_DIM + offs, mask=m)
|
||||
s = tl.load(score_ptr + i * HEAD_DIM + offs, mask=m)
|
||||
tl.store(tail_ptr + base + offs, k, mask=m)
|
||||
tl.store(
|
||||
tail_ptr + block_base + KPOOL_HEAD + (t % KPOOL) * HEAD_DIM + offs, s, mask=m
|
||||
tail_ptr + block_base + KPOOL_HEAD + (t % RING) * HEAD_DIM + offs, s, mask=m
|
||||
)
|
||||
|
||||
|
||||
@@ -385,6 +386,7 @@ def kpool_seed_tail_cache(
|
||||
KPOOL_HEAD=tail_kv_cache.stride(1),
|
||||
HEAD_DIM=head_dim,
|
||||
KPOOL=kpool,
|
||||
RING=tail_kv_cache.shape[2],
|
||||
BLOCK_D=triton.next_power_of_2(head_dim),
|
||||
)
|
||||
|
||||
@@ -412,6 +414,7 @@ def _kpool_decode_update_batched_kernel(
|
||||
PAGE_SIZE: tl.constexpr,
|
||||
BUF_NUMEL_PER_PAGE: tl.constexpr,
|
||||
POOL_SIZE: tl.constexpr,
|
||||
RING: tl.constexpr,
|
||||
TAIL_BLOCK_ELEMS: tl.constexpr,
|
||||
KPOOL_HEAD: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
@@ -428,10 +431,7 @@ def _kpool_decode_update_batched_kernel(
|
||||
the tail-ring slots that tokens t < t* (same request) just stashed in this
|
||||
same invocation. ``tl.range`` iterates sequentially within the program, so
|
||||
those stashes are visible to the later completion read. Cross-request
|
||||
programs are independent (distinct tail blocks). With NEXT_N < POOL_SIZE
|
||||
(the spec-verify case: NEXT_N ~= num_spec+1, POOL_SIZE=16) at most one
|
||||
completion can occur per request per call, but the ordered loop is correct
|
||||
for any NEXT_N.
|
||||
programs are independent (distinct tail blocks). RING >= POOL_SIZE.
|
||||
"""
|
||||
req = tl.program_id(0)
|
||||
offs = tl.arange(0, BLOCK_D)
|
||||
@@ -445,7 +445,7 @@ def _kpool_decode_update_batched_kernel(
|
||||
pos_valid = (cache_loc >= 0) & (pos >= 0)
|
||||
|
||||
slot = safe_pos % POOL_SIZE
|
||||
phys_slot = safe_pos % POOL_SIZE
|
||||
phys_slot = safe_pos % RING
|
||||
|
||||
# Derive the tail block from THIS token's tail_slot (the request's block
|
||||
# is constant across a pool, but a padded / invalid entry carries a
|
||||
@@ -453,7 +453,7 @@ def _kpool_decode_update_batched_kernel(
|
||||
# token's base address). Clamp so an invalid entry can never form an
|
||||
# out-of-bounds base; the accesses below are gated on pos_valid anyway.
|
||||
tail_slot = tl.load(tail_slot_mapping_ptr + idx)
|
||||
block = tl.maximum(tail_slot, 0).to(tl.int64) // POOL_SIZE
|
||||
block = tl.maximum(tail_slot, 0).to(tl.int64) // RING
|
||||
block_base = block * TAIL_BLOCK_ELEMS
|
||||
|
||||
# The tail-ring stash must run for EVERY real token, so it is gated on
|
||||
@@ -481,7 +481,7 @@ def _kpool_decode_update_batched_kernel(
|
||||
max_score = tl.full((BLOCK_D,), -float("inf"), tl.float32)
|
||||
for pool_slot in tl.static_range(0, POOL_SIZE):
|
||||
is_current = pool_slot == slot
|
||||
phys = (pool_logical_start + pool_slot) % POOL_SIZE
|
||||
phys = (pool_logical_start + pool_slot) % RING
|
||||
score_buf = tl.load(
|
||||
tail_kv_ptr + block_base + KPOOL_HEAD + phys * HEAD_DIM + offs,
|
||||
mask=dim_mask,
|
||||
@@ -499,7 +499,7 @@ def _kpool_decode_update_batched_kernel(
|
||||
denom = tl.full((BLOCK_D,), 0.0, tl.float32)
|
||||
for pool_slot in tl.static_range(0, POOL_SIZE):
|
||||
is_current = pool_slot == slot
|
||||
phys = (pool_logical_start + pool_slot) % POOL_SIZE
|
||||
phys = (pool_logical_start + pool_slot) % RING
|
||||
score_buf = tl.load(
|
||||
tail_kv_ptr + block_base + KPOOL_HEAD + phys * HEAD_DIM + offs,
|
||||
mask=dim_mask,
|
||||
@@ -609,7 +609,8 @@ def kpool_decode_update_and_maybe_write_cache_batched(
|
||||
return
|
||||
assert tail_kv_cache.ndim == 4
|
||||
assert tail_kv_cache.shape[1] == 2
|
||||
assert tail_kv_cache.shape[2] == pool_size
|
||||
ring = tail_kv_cache.shape[2]
|
||||
assert ring >= pool_size and ring % pool_size == 0, (ring, pool_size)
|
||||
assert tail_kv_cache.shape[3] == head_dim
|
||||
assert tail_kv_cache.dtype == torch.bfloat16
|
||||
assert key.ndim == 3 and key.shape[2] == head_dim
|
||||
@@ -658,6 +659,7 @@ def kpool_decode_update_and_maybe_write_cache_batched(
|
||||
PAGE_SIZE=page_size,
|
||||
BUF_NUMEL_PER_PAGE=buf.stride(0),
|
||||
POOL_SIZE=pool_size,
|
||||
RING=ring,
|
||||
TAIL_BLOCK_ELEMS=tail_kv_cache.stride(0),
|
||||
KPOOL_HEAD=tail_kv_cache.stride(1),
|
||||
HEAD_DIM=head_dim,
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.models.glm5next.sparse_indexer import SparseAttnIndexerKpool
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig
|
||||
from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES
|
||||
from vllm.utils.math_utils import cdiv, next_power_of_2
|
||||
from vllm.v1.kv_cache_interface import KpoolTailSpec, MLAAttentionSpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -160,7 +161,7 @@ class Glm5NextTailCache(DeepseekV32IndexerCache):
|
||||
"""Paged circular buffer for the kpool indexer's in-progress (tail) pool.
|
||||
|
||||
Holds the trailing incomplete pool's raw K + gate score: one block of
|
||||
``index_kpool`` slots per request, overwritten in place by ``pos % kpool``
|
||||
``ring`` slots per request, overwritten in place by ``pos % ring``
|
||||
as decode/spec-decode advances. Prefill seeds it (instead of discarding the
|
||||
tail raw K+gate); the connector transfers it across PD; decode reads it to
|
||||
compress the boundary pool correctly. ``KpoolTailSpec`` /
|
||||
@@ -190,13 +191,24 @@ class Glm5NextTailCache(DeepseekV32IndexerCache):
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig):
|
||||
# The two head slots form [K, gate score] in the generic
|
||||
# [block, head, state, content] cache view.
|
||||
# Drafts are stashed before acceptance. With a one-pool ring, the
|
||||
# drafts behind a rejected pool-completing draft overwrite the keys
|
||||
# read by its redo.
|
||||
span = self._index_kpool + vllm_config.num_speculative_tokens
|
||||
ring = self._index_kpool * next_power_of_2(cdiv(span, self._index_kpool))
|
||||
# ring must divide the attention block size (a multiple of 128).
|
||||
assert self.cache_config.block_size % ring == 0, (
|
||||
f"Glm5NextTailCache: cache_config.block_size "
|
||||
f"({self.cache_config.block_size}) must be a multiple of the "
|
||||
f"tail ring ({ring})"
|
||||
)
|
||||
return KpoolTailSpec(
|
||||
block_size=self._index_kpool,
|
||||
block_size=ring,
|
||||
num_kv_heads=2,
|
||||
head_size=self.head_dim,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=self._index_kpool,
|
||||
sliding_window=ring,
|
||||
)
|
||||
|
||||
def get_attn_backend(self):
|
||||
|
||||
@@ -387,14 +387,15 @@ def _kpool_tail_seed_kernel(
|
||||
KPOOL_HEAD: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
KPOOL: tl.constexpr,
|
||||
RING: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
"""Copy token ``i``'s raw K + gate into its request's tail block.
|
||||
"""Copy token ``i``'s raw K + gate into its request's tail ring.
|
||||
|
||||
Token ``i`` is among its request's last KPOOL tokens iff the token KPOOL
|
||||
ahead belongs to a different tail block (or is past the batch / padding,
|
||||
slot < 0). ``tslot = block * KPOOL + pos % KPOOL``; the destination is
|
||||
``tail[block, {0:K, 1:score}, pos % KPOOL, :]``.
|
||||
slot < 0). ``tslot = block * RING + pos % RING``; the destination is
|
||||
``tail[block, {0:K, 1:score}, pos % RING, :]``.
|
||||
|
||||
The tail cache aliases the indexer cache with the indexer's (padded) block
|
||||
stride, so blocks are addressed through ``TAIL_BLOCK_ELEMS`` /
|
||||
@@ -405,18 +406,18 @@ def _kpool_tail_seed_kernel(
|
||||
t = tl.load(tslot_ptr + i).to(tl.int64)
|
||||
if t < 0:
|
||||
return
|
||||
blk = t // KPOOL # t >= 0 here, so trunc == floor
|
||||
blk = t // RING # t >= 0 here, so trunc == floor
|
||||
ahead = tl.load(tslot_ptr + i + KPOOL, mask=i + KPOOL < n_tokens, other=-1).to(
|
||||
tl.int64
|
||||
)
|
||||
# Match the torch semantics exactly: a negative ahead slot floors to a
|
||||
# block id that differs from every real block -> token is in the tail.
|
||||
# Only divide non-negative slots (Triton int div truncates, torch floors).
|
||||
if ahead >= 0 and ahead // KPOOL == blk:
|
||||
if ahead >= 0 and ahead // RING == blk:
|
||||
return
|
||||
offs = tl.arange(0, BLOCK_D)
|
||||
m = offs < HEAD_DIM
|
||||
base = blk * TAIL_BLOCK_ELEMS + (t % KPOOL) * HEAD_DIM
|
||||
base = blk * TAIL_BLOCK_ELEMS + (t % RING) * HEAD_DIM
|
||||
k = tl.load(key_ptr + i * HEAD_DIM + offs, mask=m)
|
||||
s = tl.load(score_ptr + i * HEAD_DIM + offs, mask=m)
|
||||
tl.store(tail_ptr + base + offs, k, mask=m)
|
||||
@@ -449,6 +450,7 @@ def kpool_seed_tail_cache(
|
||||
KPOOL_HEAD=tail_kv_cache.stride(1),
|
||||
HEAD_DIM=head_dim,
|
||||
KPOOL=kpool,
|
||||
RING=tail_kv_cache.shape[2],
|
||||
BLOCK_D=triton.next_power_of_2(head_dim),
|
||||
)
|
||||
|
||||
@@ -476,6 +478,7 @@ def _kpool_decode_update_batched_kernel(
|
||||
PAGE_SIZE: tl.constexpr,
|
||||
BUF_NUMEL_PER_PAGE: tl.constexpr,
|
||||
POOL_SIZE: tl.constexpr,
|
||||
RING: tl.constexpr,
|
||||
TAIL_BLOCK_ELEMS: tl.constexpr,
|
||||
KPOOL_HEAD: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
@@ -490,10 +493,7 @@ def _kpool_decode_update_batched_kernel(
|
||||
the tail-ring slots that tokens t < t* (same request) just stashed in this
|
||||
same invocation. ``tl.range`` iterates sequentially within the program, so
|
||||
those stashes are visible to the later completion read. Cross-request
|
||||
programs are independent (distinct tail blocks). With NEXT_N < POOL_SIZE
|
||||
(the spec-verify case: NEXT_N ~= num_spec+1, POOL_SIZE=16) at most one
|
||||
completion can occur per request per call, but the ordered loop is correct
|
||||
for any NEXT_N.
|
||||
programs are independent (distinct tail blocks). RING >= POOL_SIZE.
|
||||
"""
|
||||
req = tl.program_id(0)
|
||||
offs = tl.arange(0, BLOCK_D)
|
||||
@@ -507,7 +507,7 @@ def _kpool_decode_update_batched_kernel(
|
||||
pos_valid = (cache_loc >= 0) & (pos >= 0)
|
||||
|
||||
slot = safe_pos % POOL_SIZE
|
||||
phys_slot = safe_pos % POOL_SIZE
|
||||
phys_slot = safe_pos % RING
|
||||
|
||||
# Derive the tail block from THIS token's tail_slot (the request's block
|
||||
# is constant across a pool, but a padded / invalid entry carries a
|
||||
@@ -515,7 +515,7 @@ def _kpool_decode_update_batched_kernel(
|
||||
# token's base address). Clamp so an invalid entry can never form an
|
||||
# out-of-bounds base; the accesses below are gated on pos_valid anyway.
|
||||
tail_slot = tl.load(tail_slot_mapping_ptr + idx)
|
||||
block = tl.maximum(tail_slot, 0).to(tl.int64) // POOL_SIZE
|
||||
block = tl.maximum(tail_slot, 0).to(tl.int64) // RING
|
||||
block_base = block * TAIL_BLOCK_ELEMS
|
||||
|
||||
# The tail-ring stash must run for EVERY real token, so it is gated on
|
||||
@@ -543,7 +543,7 @@ def _kpool_decode_update_batched_kernel(
|
||||
max_score = tl.full((BLOCK_D,), -float("inf"), tl.float32)
|
||||
for pool_slot in tl.static_range(0, POOL_SIZE):
|
||||
is_current = pool_slot == slot
|
||||
phys = (pool_logical_start + pool_slot) % POOL_SIZE
|
||||
phys = (pool_logical_start + pool_slot) % RING
|
||||
score_buf = tl.load(
|
||||
tail_kv_ptr + block_base + KPOOL_HEAD + phys * HEAD_DIM + offs,
|
||||
mask=dim_mask,
|
||||
@@ -561,7 +561,7 @@ def _kpool_decode_update_batched_kernel(
|
||||
denom = tl.full((BLOCK_D,), 0.0, tl.float32)
|
||||
for pool_slot in tl.static_range(0, POOL_SIZE):
|
||||
is_current = pool_slot == slot
|
||||
phys = (pool_logical_start + pool_slot) % POOL_SIZE
|
||||
phys = (pool_logical_start + pool_slot) % RING
|
||||
score_buf = tl.load(
|
||||
tail_kv_ptr + block_base + KPOOL_HEAD + phys * HEAD_DIM + offs,
|
||||
mask=dim_mask,
|
||||
@@ -671,7 +671,8 @@ def kpool_decode_update_and_maybe_write_cache_batched(
|
||||
return
|
||||
assert tail_kv_cache.ndim == 4
|
||||
assert tail_kv_cache.shape[1] == 2
|
||||
assert tail_kv_cache.shape[2] == pool_size
|
||||
ring = tail_kv_cache.shape[2]
|
||||
assert ring >= pool_size and ring % pool_size == 0, (ring, pool_size)
|
||||
assert tail_kv_cache.shape[3] == head_dim
|
||||
assert tail_kv_cache.dtype == torch.bfloat16
|
||||
assert key.ndim == 3 and key.shape[2] == head_dim
|
||||
@@ -717,6 +718,7 @@ def kpool_decode_update_and_maybe_write_cache_batched(
|
||||
PAGE_SIZE=page_size,
|
||||
BUF_NUMEL_PER_PAGE=buf.stride(0),
|
||||
POOL_SIZE=pool_size,
|
||||
RING=ring,
|
||||
TAIL_BLOCK_ELEMS=tail_kv_cache.stride(0),
|
||||
KPOOL_HEAD=tail_kv_cache.stride(1),
|
||||
HEAD_DIM=head_dim,
|
||||
|
||||
Reference in New Issue
Block a user