From 521aa80f719bb11bf973d2d51873ca966afe373d Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Thu, 23 Jul 2026 11:12:46 +0100 Subject: [PATCH] [Core] Simplify KVBlockZeroer index tensor handling (#48399) Signed-off-by: Nick Hill Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- tests/v1/worker/test_kv_block_zeroer.py | 11 +++---- vllm/v1/worker/gpu/model_runner.py | 4 +-- vllm/v1/worker/gpu_model_runner.py | 2 -- vllm/v1/worker/utils.py | 44 +------------------------ 4 files changed, 6 insertions(+), 55 deletions(-) diff --git a/tests/v1/worker/test_kv_block_zeroer.py b/tests/v1/worker/test_kv_block_zeroer.py index 8f15229912..365e4adace 100644 --- a/tests/v1/worker/test_kv_block_zeroer.py +++ b/tests/v1/worker/test_kv_block_zeroer.py @@ -14,14 +14,10 @@ def test_block_ids_are_not_overwritten_while_copy_is_in_flight(): page_size_el = 4 storage = torch.ones((num_blocks, page_size_el), dtype=torch.int32, device=device) - # Build the minimal zeroer state directly so the test can focus on ID-buffer - # lifetime without constructing model attention groups. + # Build the minimal zeroer state directly so the test can focus on the + # in-flight copy behavior without constructing model attention groups. zeroer = KVBlockZeroer.__new__(KVBlockZeroer) zeroer.device = device - zeroer.pin_memory = True - zeroer.max_concurrency = 2 - zeroer._id_cap = 8 - zeroer._allocate_id_buffers() zeroer._meta = ( torch.tensor([storage.data_ptr()], dtype=torch.uint64, device=device), page_size_el, @@ -32,7 +28,8 @@ def test_block_ids_are_not_overwritten_while_copy_is_in_flight(): stream = torch.cuda.Stream() with torch.cuda.stream(stream): # Keep the first nonblocking H2D copy pending while the host submits the - # second call. A single shared pinned source would be overwritten here. + # second call. Each call must stage from its own pinned source so the + # first copy is not corrupted before it runs. torch.cuda._sleep(10_000_000) zeroer.zero_block_ids([1]) zeroer.zero_block_ids([2]) diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 6392d150d4..3f86b5595c 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -51,7 +51,7 @@ from vllm.sequence import IntermediateTensors from vllm.tasks import SupportedTask from vllm.utils.math_utils import cdiv from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib -from vllm.utils.torch_utils import PIN_MEMORY, STR_DTYPE_TO_TORCH_DTYPE +from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec from vllm.v1.outputs import DraftTokenIds, ModelRunnerOutput @@ -521,12 +521,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): """Build KV-block zeroing metadata; invoked from gpu_worker.""" self.kv_block_zeroer = KVBlockZeroer( self.device, - pin_memory=PIN_MEMORY, attn_groups_iter=(g for groups in self.attn_groups for g in groups), kernel_block_sizes=self.kernel_block_sizes, cache_dtype=self.cache_config.cache_dtype, static_forward_context=self.compilation_config.static_forward_context, - max_concurrency=self.vllm_config.max_concurrent_batches, ) @torch.inference_mode() diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 56a27f3b2a..e8804721f8 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -1134,13 +1134,11 @@ class GPUModelRunner( """ self._kv_block_zeroer = KVBlockZeroer( self.device, - pin_memory=PIN_MEMORY, attn_groups_iter=self._kv_cache_spec_attn_group_iterator(), kernel_block_sizes=self._kernel_block_sizes, cache_dtype=self.cache_config.cache_dtype, runner_only_attn_layers=self.runner_only_attn_layers, static_forward_context=self.compilation_config.static_forward_context, - max_concurrency=self.vllm_config.max_concurrent_batches, ) def _zero_block_ids(self, block_ids: list[int]) -> None: diff --git a/vllm/v1/worker/utils.py b/vllm/v1/worker/utils.py index fda37dcdab..d83242b160 100644 --- a/vllm/v1/worker/utils.py +++ b/vllm/v1/worker/utils.py @@ -91,13 +91,11 @@ class KVBlockZeroer: def __init__( self, device: torch.device, - pin_memory: bool, attn_groups_iter: Iterable["AttentionGroup"], kernel_block_sizes: list[int], cache_dtype: str, static_forward_context: dict[str, Any], runner_only_attn_layers: set[str] | None = None, - max_concurrency: int = 1, ) -> None: """Precompute the absolute-address table for the Triton zeroing kernel. @@ -112,15 +110,7 @@ class KVBlockZeroer: Only AttentionSpec layers are processed; Mamba layers are skipped. """ self.device = device - self.pin_memory = pin_memory - if max_concurrency < 1: - raise ValueError("max_concurrency must be at least 1") - self.max_concurrency = max_concurrency self._meta: tuple[torch.Tensor, int, int, int] | None = None - self._id_cap: int = 0 - self._ids_pinned: list[torch.Tensor] = [] - self._ids_gpu: list[torch.Tensor] = [] - self._id_buffer_index = 0 if runner_only_attn_layers is None: runner_only_attn_layers = set() @@ -182,8 +172,6 @@ class KVBlockZeroer: return blk_size = min(largest_power_of_2_divisor(page_size_el), 1024) - self._id_cap = 8192 - self._allocate_id_buffers() self._meta = ( torch.tensor(seg_addrs, dtype=torch.uint64, device=self.device), page_size_el, @@ -191,43 +179,13 @@ class KVBlockZeroer: len(seg_addrs), ) - def _allocate_id_buffers(self) -> None: - self._ids_pinned = [ - torch.empty( - self._id_cap, - dtype=torch.int64, - pin_memory=self.pin_memory, - ) - for _ in range(self.max_concurrency) - ] - self._ids_gpu = [ - torch.empty(self._id_cap, dtype=torch.int64, device=self.device) - for _ in range(self.max_concurrency) - ] - self._id_buffer_index = 0 - def zero_block_ids(self, block_ids: list[int]) -> None: """Zero the KV cache memory for the given block IDs.""" if not block_ids or self._meta is None: return seg_addrs, page_size_el, blk_size, n_segs = self._meta n_blocks = len(block_ids) - if n_blocks > self._id_cap: - # The old pinned buffers may still be the source of an in-flight - # nonblocking copy. Growing is rare, so we don't mind the sync overhead - torch.accelerator.synchronize() - self._id_cap = n_blocks * 2 - self._allocate_id_buffers() - - # The H2D copy is nonblocking, so its pinned source must not be mutated - # while this batch is in flight. Rotate through as many buffers as concurrent - # in-flight batches, to avoid collisions. - buffer_index = self._id_buffer_index - self._id_buffer_index = (buffer_index + 1) % self.max_concurrency - ids_pinned = self._ids_pinned[buffer_index] - ids_pinned[:n_blocks].numpy()[:] = block_ids - idx = self._ids_gpu[buffer_index][:n_blocks] - idx.copy_(ids_pinned[:n_blocks], non_blocking=True) + idx = async_tensor_h2d(block_ids, device=self.device, dtype=torch.int64) grid = (n_blocks * n_segs * (page_size_el // blk_size),) _zero_kv_blocks_kernel[grid]( seg_addrs,