[Bugfix] Bound KV block zeroing launch geometry (#52058)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Lucas Wilkinson
2026-08-13 03:42:26 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent f9538af28c
commit 79f3183f86
2 changed files with 86 additions and 37 deletions
+17 -29
View File
@@ -17,7 +17,6 @@ from vllm.model_executor.models.interfaces import MultiModalEmbeddings
from vllm.model_executor.models.utils import extract_layer_index
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import largest_power_of_2_divisor
from vllm.utils.mem_utils import MemorySnapshot, format_gib
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backend import (
@@ -52,15 +51,12 @@ def raise_if_nan_logits(num_nans_in_logits: Mapping[str, int]) -> None:
raise RuntimeError(f"NaNs detected in logits: {corrupted_requests}")
@triton.jit(do_not_specialize=["n_blocks"])
@triton.jit
def _zero_kv_blocks_kernel(
seg_addrs_ptr,
seg_block_strides_ptr,
seg_page_sizes_ptr,
block_ids_ptr,
n_blocks,
N_SEGS: tl.constexpr,
MAX_CHUNKS: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
"""Zero KV cache blocks across all segments in a single launch.
@@ -78,29 +74,27 @@ def _zero_kv_blocks_kernel(
seg_addrs_ptr holds absolute byte addresses (int64) for each segment,
allowing segments to live in different CUDA allocations.
Programs are mapped as (block_index, seg_index, chunk_index).
Programs are mapped directly onto a 3-D grid as
(block_index, seg_index, chunk_index).
"""
pid = tl.program_id(0)
work_per_block = N_SEGS * MAX_CHUNKS
block_index = pid // work_per_block
if block_index >= n_blocks:
return
remainder = pid % work_per_block
seg_index = remainder // MAX_CHUNKS
chunk_index = remainder % MAX_CHUNKS
block_index = tl.program_id(0)
seg_index = tl.program_id(1)
chunk_index = tl.program_id(2)
block_stride_el = tl.load(seg_block_strides_ptr + seg_index)
page_size_el = tl.load(seg_page_sizes_ptr + seg_index)
if chunk_index >= page_size_el // BLOCK_SIZE:
chunk_offset = chunk_index.to(tl.int64) * BLOCK_SIZE
if chunk_offset >= page_size_el:
return
block_id = tl.load(block_ids_ptr + block_index)
seg_addr = tl.load(seg_addrs_ptr + seg_index)
ptr = tl.cast(seg_addr, tl.pointer_type(tl.int32))
offset = (
block_id.to(tl.int64) * block_stride_el.to(tl.int64)
+ chunk_index.to(tl.int64) * BLOCK_SIZE
block_offset = block_id.to(tl.int64) * block_stride_el.to(tl.int64)
cols = chunk_offset + tl.arange(0, BLOCK_SIZE).to(tl.int64)
tl.store(
ptr + block_offset + cols,
tl.zeros([BLOCK_SIZE], dtype=tl.int32),
mask=cols < page_size_el,
)
cols = tl.arange(0, BLOCK_SIZE).to(tl.int64)
tl.store(ptr + offset + cols, tl.zeros([BLOCK_SIZE], dtype=tl.int32))
class KVBlockZeroer:
@@ -204,15 +198,12 @@ class KVBlockZeroer:
return
max_page_size_el = max(seg_page_sizes)
blk_size = min(
min(largest_power_of_2_divisor(ps) for ps in seg_page_sizes),
1024,
)
blk_size = min(1 << (max_page_size_el - 1).bit_length(), 1024)
self._meta = (
torch.tensor(seg_addrs, dtype=torch.uint64, device=self.device),
torch.tensor(seg_block_strides, dtype=torch.int64, device=self.device),
torch.tensor(seg_page_sizes, dtype=torch.int64, device=self.device),
max_page_size_el // blk_size,
(max_page_size_el + blk_size - 1) // blk_size,
blk_size,
len(seg_addrs),
)
@@ -231,15 +222,12 @@ class KVBlockZeroer:
) = self._meta
n_blocks = len(block_ids)
idx = async_tensor_h2d(block_ids, device=self.device, dtype=torch.int64)
grid = (n_blocks * n_segs * max_chunks,)
grid = (n_blocks, n_segs, max_chunks)
_zero_kv_blocks_kernel[grid](
seg_addrs,
seg_block_strides,
seg_page_sizes,
idx,
n_blocks,
N_SEGS=n_segs,
MAX_CHUNKS=max_chunks,
BLOCK_SIZE=blk_size,
)