[Core] Simplify KVBlockZeroer index tensor handling (#48399)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Nick Hill
2026-07-23 18:12:46 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent a76df87db8
commit 521aa80f71
4 changed files with 6 additions and 55 deletions
+4 -7
View File
@@ -14,14 +14,10 @@ def test_block_ids_are_not_overwritten_while_copy_is_in_flight():
page_size_el = 4
storage = torch.ones((num_blocks, page_size_el), dtype=torch.int32, device=device)
# Build the minimal zeroer state directly so the test can focus on ID-buffer
# lifetime without constructing model attention groups.
# Build the minimal zeroer state directly so the test can focus on the
# in-flight copy behavior without constructing model attention groups.
zeroer = KVBlockZeroer.__new__(KVBlockZeroer)
zeroer.device = device
zeroer.pin_memory = True
zeroer.max_concurrency = 2
zeroer._id_cap = 8
zeroer._allocate_id_buffers()
zeroer._meta = (
torch.tensor([storage.data_ptr()], dtype=torch.uint64, device=device),
page_size_el,
@@ -32,7 +28,8 @@ def test_block_ids_are_not_overwritten_while_copy_is_in_flight():
stream = torch.cuda.Stream()
with torch.cuda.stream(stream):
# Keep the first nonblocking H2D copy pending while the host submits the
# second call. A single shared pinned source would be overwritten here.
# second call. Each call must stage from its own pinned source so the
# first copy is not corrupted before it runs.
torch.cuda._sleep(10_000_000)
zeroer.zero_block_ids([1])
zeroer.zero_block_ids([2])
+1 -3
View File
@@ -51,7 +51,7 @@ from vllm.sequence import IntermediateTensors
from vllm.tasks import SupportedTask
from vllm.utils.math_utils import cdiv
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
from vllm.utils.torch_utils import PIN_MEMORY, STR_DTYPE_TO_TORCH_DTYPE
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
from vllm.v1.outputs import DraftTokenIds, ModelRunnerOutput
@@ -521,12 +521,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
"""Build KV-block zeroing metadata; invoked from gpu_worker."""
self.kv_block_zeroer = KVBlockZeroer(
self.device,
pin_memory=PIN_MEMORY,
attn_groups_iter=(g for groups in self.attn_groups for g in groups),
kernel_block_sizes=self.kernel_block_sizes,
cache_dtype=self.cache_config.cache_dtype,
static_forward_context=self.compilation_config.static_forward_context,
max_concurrency=self.vllm_config.max_concurrent_batches,
)
@torch.inference_mode()
-2
View File
@@ -1134,13 +1134,11 @@ class GPUModelRunner(
"""
self._kv_block_zeroer = KVBlockZeroer(
self.device,
pin_memory=PIN_MEMORY,
attn_groups_iter=self._kv_cache_spec_attn_group_iterator(),
kernel_block_sizes=self._kernel_block_sizes,
cache_dtype=self.cache_config.cache_dtype,
runner_only_attn_layers=self.runner_only_attn_layers,
static_forward_context=self.compilation_config.static_forward_context,
max_concurrency=self.vllm_config.max_concurrent_batches,
)
def _zero_block_ids(self, block_ids: list[int]) -> None:
+1 -43
View File
@@ -91,13 +91,11 @@ class KVBlockZeroer:
def __init__(
self,
device: torch.device,
pin_memory: bool,
attn_groups_iter: Iterable["AttentionGroup"],
kernel_block_sizes: list[int],
cache_dtype: str,
static_forward_context: dict[str, Any],
runner_only_attn_layers: set[str] | None = None,
max_concurrency: int = 1,
) -> None:
"""Precompute the absolute-address table for the Triton zeroing kernel.
@@ -112,15 +110,7 @@ class KVBlockZeroer:
Only AttentionSpec layers are processed; Mamba layers are skipped.
"""
self.device = device
self.pin_memory = pin_memory
if max_concurrency < 1:
raise ValueError("max_concurrency must be at least 1")
self.max_concurrency = max_concurrency
self._meta: tuple[torch.Tensor, int, int, int] | None = None
self._id_cap: int = 0
self._ids_pinned: list[torch.Tensor] = []
self._ids_gpu: list[torch.Tensor] = []
self._id_buffer_index = 0
if runner_only_attn_layers is None:
runner_only_attn_layers = set()
@@ -182,8 +172,6 @@ class KVBlockZeroer:
return
blk_size = min(largest_power_of_2_divisor(page_size_el), 1024)
self._id_cap = 8192
self._allocate_id_buffers()
self._meta = (
torch.tensor(seg_addrs, dtype=torch.uint64, device=self.device),
page_size_el,
@@ -191,43 +179,13 @@ class KVBlockZeroer:
len(seg_addrs),
)
def _allocate_id_buffers(self) -> None:
self._ids_pinned = [
torch.empty(
self._id_cap,
dtype=torch.int64,
pin_memory=self.pin_memory,
)
for _ in range(self.max_concurrency)
]
self._ids_gpu = [
torch.empty(self._id_cap, dtype=torch.int64, device=self.device)
for _ in range(self.max_concurrency)
]
self._id_buffer_index = 0
def zero_block_ids(self, block_ids: list[int]) -> None:
"""Zero the KV cache memory for the given block IDs."""
if not block_ids or self._meta is None:
return
seg_addrs, page_size_el, blk_size, n_segs = self._meta
n_blocks = len(block_ids)
if n_blocks > self._id_cap:
# The old pinned buffers may still be the source of an in-flight
# nonblocking copy. Growing is rare, so we don't mind the sync overhead
torch.accelerator.synchronize()
self._id_cap = n_blocks * 2
self._allocate_id_buffers()
# The H2D copy is nonblocking, so its pinned source must not be mutated
# while this batch is in flight. Rotate through as many buffers as concurrent
# in-flight batches, to avoid collisions.
buffer_index = self._id_buffer_index
self._id_buffer_index = (buffer_index + 1) % self.max_concurrency
ids_pinned = self._ids_pinned[buffer_index]
ids_pinned[:n_blocks].numpy()[:] = block_ids
idx = self._ids_gpu[buffer_index][:n_blocks]
idx.copy_(ids_pinned[:n_blocks], non_blocking=True)
idx = async_tensor_h2d(block_ids, device=self.device, dtype=torch.int64)
grid = (n_blocks * n_segs * (page_size_el // blk_size),)
_zero_kv_blocks_kernel[grid](
seg_addrs,