[Bugfix] Support non-uniform page sizes in KVBlockZeroer (#49704)

Signed-off-by: Elvir Crncevic <elvircrn@gmail.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Elvir Crnčević
2026-07-24 13:38:59 -07:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 2279575cd9
commit 972848f276
3 changed files with 92 additions and 31 deletions
+50 -3
View File
@@ -20,9 +20,10 @@ def test_block_ids_are_not_overwritten_while_copy_is_in_flight():
zeroer.device = device
zeroer._meta = (
torch.tensor([storage.data_ptr()], dtype=torch.uint64, device=device),
page_size_el,
page_size_el,
1,
torch.tensor([page_size_el], dtype=torch.int64, device=device),
page_size_el // page_size_el, # max_chunks = 1
page_size_el, # blk_size
1, # n_segs
)
stream = torch.cuda.Stream()
@@ -39,3 +40,49 @@ def test_block_ids_are_not_overwritten_while_copy_is_in_flight():
assert torch.all(storage[1] == 0)
assert torch.all(storage[2] == 0)
assert torch.all(storage[3] == 1)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
def test_non_uniform_page_sizes():
"""Two segments with different page sizes (e.g. MLA + DSA indexer)."""
device = torch.device("cuda")
num_blocks = 4
page_size_a = 10496 # int32 elements
page_size_b = 2112
storage_a = torch.ones((num_blocks, page_size_a), dtype=torch.int32, device=device)
storage_b = torch.ones((num_blocks, page_size_b), dtype=torch.int32, device=device)
zeroer = KVBlockZeroer.__new__(KVBlockZeroer)
zeroer.device = device
seg_page_sizes = [page_size_a, page_size_b]
max_ps = max(seg_page_sizes)
def largest_power_of_2_divisor(n):
return n & -n
blk_size = min(min(largest_power_of_2_divisor(ps) for ps in seg_page_sizes), 1024)
zeroer._meta = (
torch.tensor(
[storage_a.data_ptr(), storage_b.data_ptr()],
dtype=torch.uint64,
device=device,
),
torch.tensor(seg_page_sizes, dtype=torch.int64, device=device),
max_ps // blk_size,
blk_size,
2,
)
stream = torch.cuda.Stream()
with torch.cuda.stream(stream):
zeroer.zero_block_ids([1, 2])
stream.synchronize()
for storage in (storage_a, storage_b):
assert torch.all(storage[0] == 1)
assert torch.all(storage[1] == 0)
assert torch.all(storage[2] == 0)
assert torch.all(storage[3] == 1)
@@ -36,7 +36,8 @@ _FLA_POST_CONV_WARMUP_LENGTHS = (1, 2, 16)
@dataclass(frozen=True)
class _ZeroKvWarmupConfig:
page_size_el: int
seg_page_sizes: torch.Tensor
max_chunks: int
block_size: int
n_segs: int
@@ -160,9 +161,10 @@ def _zero_kv_warmup_config(runner: object) -> _ZeroKvWarmupConfig | None:
if meta is None:
return None
_, page_size_el, block_size, n_segs = meta
_, seg_page_sizes, max_chunks, block_size, n_segs = meta
return _ZeroKvWarmupConfig(
page_size_el=int(page_size_el),
seg_page_sizes=seg_page_sizes,
max_chunks=int(max_chunks),
block_size=int(block_size),
n_segs=int(n_segs),
)
@@ -185,8 +187,9 @@ def _warm_zero_kv_blocks_kernel(
from vllm.v1.worker.utils import _zero_kv_blocks_kernel
max_n_blocks = max(_ZERO_KV_N_BLOCKS)
max_page_size = int(config.seg_page_sizes.max().item())
scratch = torch.empty(
max_n_blocks * config.page_size_el,
max_n_blocks * max_page_size,
dtype=torch.int32,
device=device,
)
@@ -198,13 +201,14 @@ def _warm_zero_kv_blocks_kernel(
for n_blocks in _ZERO_KV_N_BLOCKS:
block_ids = torch.arange(n_blocks, dtype=torch.int64, device=device)
grid = (n_blocks * config.n_segs * (config.page_size_el // config.block_size),)
grid = (n_blocks * config.n_segs * config.max_chunks,)
_zero_kv_blocks_kernel[grid](
seg_addrs,
config.seg_page_sizes,
block_ids,
n_blocks,
N_SEGS=config.n_segs,
PAGE_SIZE_EL=config.page_size_el,
MAX_CHUNKS=config.max_chunks,
BLOCK_SIZE=config.block_size,
)
+32 -22
View File
@@ -43,10 +43,11 @@ logger = init_logger(__name__)
@triton.jit
def _zero_kv_blocks_kernel(
seg_addrs_ptr,
seg_page_sizes_ptr,
block_ids_ptr,
n_blocks,
N_SEGS: tl.constexpr,
PAGE_SIZE_EL: tl.constexpr,
MAX_CHUNKS: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
"""Zero KV cache blocks across all segments in a single launch.
@@ -56,25 +57,33 @@ def _zero_kv_blocks_kernel(
buffer. For backends where K/V is outermost (block_dim=1) there are
two segments per buffer (one for K, one for V).
Segments may have different page sizes (e.g. models with multiple KV
cache groups like MLA + DSA indexer). Each segment's page size is
read from seg_page_sizes_ptr; programs whose chunk_index falls beyond
their segment's page size early-exit.
seg_addrs_ptr holds absolute byte addresses (int64) for each segment,
allowing segments to live in different CUDA allocations.
Programs are mapped as (block_index, seg_index, chunk_index).
"""
pid = tl.program_id(0)
chunks = PAGE_SIZE_EL // BLOCK_SIZE
work_per_block = N_SEGS * chunks
work_per_block = N_SEGS * MAX_CHUNKS
block_index = pid // work_per_block
if block_index >= n_blocks:
return
remainder = pid % work_per_block
seg_index = remainder // chunks
chunk_index = remainder % chunks
seg_index = remainder // MAX_CHUNKS
chunk_index = remainder % MAX_CHUNKS
page_size_el = tl.load(seg_page_sizes_ptr + seg_index)
if chunk_index >= page_size_el // BLOCK_SIZE:
return
block_id = tl.load(block_ids_ptr + block_index)
seg_addr = tl.load(seg_addrs_ptr + seg_index)
ptr = tl.cast(seg_addr, tl.pointer_type(tl.int32))
offset = (
block_id.to(tl.int64) * PAGE_SIZE_EL + chunk_index.to(tl.int64) * BLOCK_SIZE
block_id.to(tl.int64) * page_size_el.to(tl.int64)
+ chunk_index.to(tl.int64) * BLOCK_SIZE
)
cols = tl.arange(0, BLOCK_SIZE).to(tl.int64)
tl.store(ptr + offset + cols, tl.zeros([BLOCK_SIZE], dtype=tl.int32))
@@ -104,19 +113,19 @@ class KVBlockZeroer:
Block IDs from the scheduler reference logical blocks whose size
may differ from the kernel block size (virtual block splitting).
PAGE_SIZE_EL accounts for this ratio so that
``block_id * PAGE_SIZE_EL`` lands at the correct offset.
Each segment's page_size_el accounts for this ratio so that
``block_id * page_size_el`` lands at the correct offset.
Only AttentionSpec layers are processed; Mamba layers are skipped.
"""
self.device = device
self._meta: tuple[torch.Tensor, int, int, int] | None = None
self._meta: tuple[torch.Tensor, torch.Tensor, int, int, int] | None = None
if runner_only_attn_layers is None:
runner_only_attn_layers = set()
seen_ptrs: set[int] = set()
seg_addrs: list[int] = []
page_size_el: int | None = None
seg_page_sizes: list[int] = []
for group in attn_groups_iter:
spec = group.kv_cache_spec
@@ -149,12 +158,6 @@ class KVBlockZeroer:
assert cur_bytes % 4 == 0
kernel_block_el = cur_bytes // 4
cur_page_el = kernel_block_el * ratio
if page_size_el is None:
page_size_el = cur_page_el
else:
assert page_size_el == cur_page_el, (
f"Non-uniform page sizes: {page_size_el} vs {cur_page_el}"
)
block_stride_bytes = cur_bytes
outer_dims = [
@@ -166,15 +169,21 @@ class KVBlockZeroer:
for outer in iprod(*(range(kv.shape[d]) for d in outer_dims)):
off_bytes = sum(i * s for i, s in zip(outer, outer_strides))
seg_addrs.append(dp + off_bytes)
seg_page_sizes.append(cur_page_el)
if not seg_addrs or page_size_el is None:
if not seg_addrs:
self._meta = None
return
blk_size = min(largest_power_of_2_divisor(page_size_el), 1024)
max_page_size_el = max(seg_page_sizes)
blk_size = min(
min(largest_power_of_2_divisor(ps) for ps in seg_page_sizes),
1024,
)
self._meta = (
torch.tensor(seg_addrs, dtype=torch.uint64, device=self.device),
page_size_el,
torch.tensor(seg_page_sizes, dtype=torch.int64, device=self.device),
max_page_size_el // blk_size,
blk_size,
len(seg_addrs),
)
@@ -183,16 +192,17 @@ class KVBlockZeroer:
"""Zero the KV cache memory for the given block IDs."""
if not block_ids or self._meta is None:
return
seg_addrs, page_size_el, blk_size, n_segs = self._meta
seg_addrs, seg_page_sizes, max_chunks, blk_size, n_segs = self._meta
n_blocks = len(block_ids)
idx = async_tensor_h2d(block_ids, device=self.device, dtype=torch.int64)
grid = (n_blocks * n_segs * (page_size_el // blk_size),)
grid = (n_blocks * n_segs * max_chunks,)
_zero_kv_blocks_kernel[grid](
seg_addrs,
seg_page_sizes,
idx,
n_blocks,
N_SEGS=n_segs,
PAGE_SIZE_EL=page_size_el,
MAX_CHUNKS=max_chunks,
BLOCK_SIZE=blk_size,
)