[Core] Simplify KVBlockZeroer index tensor handling (#48399)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
a76df87db8
commit
521aa80f71
@@ -14,14 +14,10 @@ def test_block_ids_are_not_overwritten_while_copy_is_in_flight():
|
||||
page_size_el = 4
|
||||
storage = torch.ones((num_blocks, page_size_el), dtype=torch.int32, device=device)
|
||||
|
||||
# Build the minimal zeroer state directly so the test can focus on ID-buffer
|
||||
# lifetime without constructing model attention groups.
|
||||
# Build the minimal zeroer state directly so the test can focus on the
|
||||
# in-flight copy behavior without constructing model attention groups.
|
||||
zeroer = KVBlockZeroer.__new__(KVBlockZeroer)
|
||||
zeroer.device = device
|
||||
zeroer.pin_memory = True
|
||||
zeroer.max_concurrency = 2
|
||||
zeroer._id_cap = 8
|
||||
zeroer._allocate_id_buffers()
|
||||
zeroer._meta = (
|
||||
torch.tensor([storage.data_ptr()], dtype=torch.uint64, device=device),
|
||||
page_size_el,
|
||||
@@ -32,7 +28,8 @@ def test_block_ids_are_not_overwritten_while_copy_is_in_flight():
|
||||
stream = torch.cuda.Stream()
|
||||
with torch.cuda.stream(stream):
|
||||
# Keep the first nonblocking H2D copy pending while the host submits the
|
||||
# second call. A single shared pinned source would be overwritten here.
|
||||
# second call. Each call must stage from its own pinned source so the
|
||||
# first copy is not corrupted before it runs.
|
||||
torch.cuda._sleep(10_000_000)
|
||||
zeroer.zero_block_ids([1])
|
||||
zeroer.zero_block_ids([2])
|
||||
|
||||
@@ -51,7 +51,7 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
|
||||
from vllm.utils.torch_utils import PIN_MEMORY, STR_DTYPE_TO_TORCH_DTYPE
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
|
||||
from vllm.v1.outputs import DraftTokenIds, ModelRunnerOutput
|
||||
@@ -521,12 +521,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
"""Build KV-block zeroing metadata; invoked from gpu_worker."""
|
||||
self.kv_block_zeroer = KVBlockZeroer(
|
||||
self.device,
|
||||
pin_memory=PIN_MEMORY,
|
||||
attn_groups_iter=(g for groups in self.attn_groups for g in groups),
|
||||
kernel_block_sizes=self.kernel_block_sizes,
|
||||
cache_dtype=self.cache_config.cache_dtype,
|
||||
static_forward_context=self.compilation_config.static_forward_context,
|
||||
max_concurrency=self.vllm_config.max_concurrent_batches,
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
|
||||
@@ -1134,13 +1134,11 @@ class GPUModelRunner(
|
||||
"""
|
||||
self._kv_block_zeroer = KVBlockZeroer(
|
||||
self.device,
|
||||
pin_memory=PIN_MEMORY,
|
||||
attn_groups_iter=self._kv_cache_spec_attn_group_iterator(),
|
||||
kernel_block_sizes=self._kernel_block_sizes,
|
||||
cache_dtype=self.cache_config.cache_dtype,
|
||||
runner_only_attn_layers=self.runner_only_attn_layers,
|
||||
static_forward_context=self.compilation_config.static_forward_context,
|
||||
max_concurrency=self.vllm_config.max_concurrent_batches,
|
||||
)
|
||||
|
||||
def _zero_block_ids(self, block_ids: list[int]) -> None:
|
||||
|
||||
+1
-43
@@ -91,13 +91,11 @@ class KVBlockZeroer:
|
||||
def __init__(
|
||||
self,
|
||||
device: torch.device,
|
||||
pin_memory: bool,
|
||||
attn_groups_iter: Iterable["AttentionGroup"],
|
||||
kernel_block_sizes: list[int],
|
||||
cache_dtype: str,
|
||||
static_forward_context: dict[str, Any],
|
||||
runner_only_attn_layers: set[str] | None = None,
|
||||
max_concurrency: int = 1,
|
||||
) -> None:
|
||||
"""Precompute the absolute-address table for the Triton zeroing kernel.
|
||||
|
||||
@@ -112,15 +110,7 @@ class KVBlockZeroer:
|
||||
Only AttentionSpec layers are processed; Mamba layers are skipped.
|
||||
"""
|
||||
self.device = device
|
||||
self.pin_memory = pin_memory
|
||||
if max_concurrency < 1:
|
||||
raise ValueError("max_concurrency must be at least 1")
|
||||
self.max_concurrency = max_concurrency
|
||||
self._meta: tuple[torch.Tensor, int, int, int] | None = None
|
||||
self._id_cap: int = 0
|
||||
self._ids_pinned: list[torch.Tensor] = []
|
||||
self._ids_gpu: list[torch.Tensor] = []
|
||||
self._id_buffer_index = 0
|
||||
|
||||
if runner_only_attn_layers is None:
|
||||
runner_only_attn_layers = set()
|
||||
@@ -182,8 +172,6 @@ class KVBlockZeroer:
|
||||
return
|
||||
|
||||
blk_size = min(largest_power_of_2_divisor(page_size_el), 1024)
|
||||
self._id_cap = 8192
|
||||
self._allocate_id_buffers()
|
||||
self._meta = (
|
||||
torch.tensor(seg_addrs, dtype=torch.uint64, device=self.device),
|
||||
page_size_el,
|
||||
@@ -191,43 +179,13 @@ class KVBlockZeroer:
|
||||
len(seg_addrs),
|
||||
)
|
||||
|
||||
def _allocate_id_buffers(self) -> None:
|
||||
self._ids_pinned = [
|
||||
torch.empty(
|
||||
self._id_cap,
|
||||
dtype=torch.int64,
|
||||
pin_memory=self.pin_memory,
|
||||
)
|
||||
for _ in range(self.max_concurrency)
|
||||
]
|
||||
self._ids_gpu = [
|
||||
torch.empty(self._id_cap, dtype=torch.int64, device=self.device)
|
||||
for _ in range(self.max_concurrency)
|
||||
]
|
||||
self._id_buffer_index = 0
|
||||
|
||||
def zero_block_ids(self, block_ids: list[int]) -> None:
|
||||
"""Zero the KV cache memory for the given block IDs."""
|
||||
if not block_ids or self._meta is None:
|
||||
return
|
||||
seg_addrs, page_size_el, blk_size, n_segs = self._meta
|
||||
n_blocks = len(block_ids)
|
||||
if n_blocks > self._id_cap:
|
||||
# The old pinned buffers may still be the source of an in-flight
|
||||
# nonblocking copy. Growing is rare, so we don't mind the sync overhead
|
||||
torch.accelerator.synchronize()
|
||||
self._id_cap = n_blocks * 2
|
||||
self._allocate_id_buffers()
|
||||
|
||||
# The H2D copy is nonblocking, so its pinned source must not be mutated
|
||||
# while this batch is in flight. Rotate through as many buffers as concurrent
|
||||
# in-flight batches, to avoid collisions.
|
||||
buffer_index = self._id_buffer_index
|
||||
self._id_buffer_index = (buffer_index + 1) % self.max_concurrency
|
||||
ids_pinned = self._ids_pinned[buffer_index]
|
||||
ids_pinned[:n_blocks].numpy()[:] = block_ids
|
||||
idx = self._ids_gpu[buffer_index][:n_blocks]
|
||||
idx.copy_(ids_pinned[:n_blocks], non_blocking=True)
|
||||
idx = async_tensor_h2d(block_ids, device=self.device, dtype=torch.int64)
|
||||
grid = (n_blocks * n_segs * (page_size_el // blk_size),)
|
||||
_zero_kv_blocks_kernel[grid](
|
||||
seg_addrs,
|
||||
|
||||
Reference in New Issue
Block a user