[HiSparse] Own GPU residency instead of editing core block-table rows (#60647)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
9bd074a996
commit
d5f356dae0
@@ -7,10 +7,11 @@ from __future__ import annotations
|
||||
import math
|
||||
import mmap
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, TypeAlias
|
||||
|
||||
import numpy as np
|
||||
import psutil
|
||||
import torch
|
||||
|
||||
@@ -21,8 +22,9 @@ from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv, round_up
|
||||
from vllm.utils.mem_utils import get_max_shared_memory_bytes
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
from vllm.utils.torch_utils import async_tensor_h2d, current_stream
|
||||
from vllm.v1.attention.backend import max_decode_query_len
|
||||
from vllm.v1.hisparse.types import SparseKVResidencyUpdate
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
from vllm.v1.simple_kv_offload.cuda_mem_ops import pin_tensor
|
||||
|
||||
@@ -499,16 +501,19 @@ class HiSparsePrefillStagingPlan:
|
||||
|
||||
def ensure_gpu_sources(
|
||||
self,
|
||||
resident_block_table: torch.Tensor,
|
||||
resident_state_rows: torch.Tensor,
|
||||
state_indices: torch.Tensor,
|
||||
resident_block_size: int,
|
||||
) -> None:
|
||||
"""Resolve which staged rows can be served from the resident cache.
|
||||
|
||||
Computed once per plan (the resident block table is shared by every
|
||||
layer in the group); non-null resident pages become miss_mask=0 rows
|
||||
gathered device-to-device by ``gather_prefill_cache``.
|
||||
``resident_state_rows`` is a resident group's persistent table by
|
||||
request state row and ``state_indices`` holds each staged request's
|
||||
state row. Computed once per plan and resident group (every layer in
|
||||
the group shares the table); non-null resident pages become
|
||||
miss_mask=0 rows gathered device-to-device by ``gather_prefill_cache``.
|
||||
"""
|
||||
source_key = (resident_block_table.data_ptr(), resident_block_size)
|
||||
source_key = (resident_state_rows.data_ptr(), resident_block_size)
|
||||
if self.gpu_source_key == source_key:
|
||||
return
|
||||
block_size = self.block_size
|
||||
@@ -519,8 +524,11 @@ class HiSparsePrefillStagingPlan:
|
||||
host_ids = self.row_ids[0].view(num_unique, block_size)[:, 0] // block_size
|
||||
new_bt = self.block_table.to(torch.int64)
|
||||
num_rows, num_cols = new_bt.shape
|
||||
if num_rows == 0 or resident_block_table.shape[0] < num_rows:
|
||||
if num_rows == 0 or state_indices.shape[0] < num_rows:
|
||||
return
|
||||
resident_block_table = resident_state_rows.index_select(
|
||||
0, state_indices[:num_rows].clamp(min=0)
|
||||
)
|
||||
# One representative (row, col) per unique host block: any request
|
||||
# referencing the block holds an equivalent (refcounted) resident view.
|
||||
flat_pos = torch.arange(num_rows * num_cols, device=device)
|
||||
@@ -1107,11 +1115,50 @@ class HiSparseRuntime:
|
||||
return physical_topk_indices
|
||||
|
||||
|
||||
def update_hisparse_residency(
|
||||
residency: torch.Tensor,
|
||||
updates: Mapping[str, SparseKVResidencyUpdate],
|
||||
request_ids: Sequence[str],
|
||||
request_state_indices: torch.Tensor,
|
||||
) -> None:
|
||||
"""Apply scheduled requests' residency changes.
|
||||
|
||||
``residency`` is ``[state rows, resident groups, pages]``: the GPU block of
|
||||
each page, or block 0 when the page is read from the host.
|
||||
"""
|
||||
if not updates:
|
||||
return
|
||||
batch_rows = {request_id: row for row, request_id in enumerate(request_ids)}
|
||||
num_groups = residency.shape[1]
|
||||
rows: list[int] = []
|
||||
pages: list[int] = []
|
||||
block_ids: list[list[int]] = [[] for _ in range(num_groups)]
|
||||
for request_id, update in updates.items():
|
||||
rows.append(batch_rows[request_id])
|
||||
pages.extend(update.pages)
|
||||
for group_block_ids, update_block_ids in zip(
|
||||
block_ids, update.block_ids, strict=True
|
||||
):
|
||||
group_block_ids.extend(update_block_ids)
|
||||
host_values = np.empty((len(pages), 2 + num_groups), dtype=np.int32)
|
||||
host_values[:, 0] = np.repeat(
|
||||
rows, [len(update.pages) for update in updates.values()]
|
||||
)
|
||||
host_values[:, 1] = pages
|
||||
host_values[:, 2:] = np.asarray(block_ids, dtype=np.int32).T
|
||||
values = async_tensor_h2d(
|
||||
host_values, device=request_state_indices.device, dtype=torch.int32
|
||||
)
|
||||
state_rows = request_state_indices[values[:, 0]]
|
||||
residency[state_rows, :, values[:, 1]] = values[:, 2:]
|
||||
|
||||
|
||||
class HiSparseCacheHandle:
|
||||
"""Attention-facing handle for resident KV and sparse offload state."""
|
||||
|
||||
def __init__(self, runtime: HiSparseRuntime) -> None:
|
||||
self.view: PagedCacheView | None = None
|
||||
self.residency: torch.Tensor | None = None
|
||||
self.block_table: torch.Tensor | None = None
|
||||
self.source_block_table: torch.Tensor | None = None
|
||||
self.slot_mapping: torch.Tensor | None = None
|
||||
@@ -1128,6 +1175,7 @@ class HiSparseCacheHandle:
|
||||
# Speculator layers write their rows after the target forward.
|
||||
self.draft_layer = False
|
||||
self.index_group_caches: list[HiSparseCacheHandle] = [self]
|
||||
self._batch_block_table: torch.Tensor | None = None
|
||||
|
||||
def prepare_group_for_batch(self, attn_metadata: Any | None) -> None:
|
||||
assert self.runtime.is_group_leader
|
||||
@@ -1136,6 +1184,7 @@ class HiSparseCacheHandle:
|
||||
|
||||
def _prepare_for_batch(self, attn_metadata: Any | None) -> None:
|
||||
self.dummy_batch = attn_metadata is None
|
||||
self._batch_block_table = None
|
||||
self.runtime.begin_forward()
|
||||
self.num_actual_tokens = (
|
||||
attn_metadata.num_actual_tokens if attn_metadata is not None else 0
|
||||
@@ -1151,6 +1200,17 @@ class HiSparseCacheHandle:
|
||||
not self.decode_batch or self.runtime.eager_host_mirror
|
||||
)
|
||||
|
||||
def batch_block_table(self) -> torch.Tensor:
|
||||
"""Resident rows by this step's batch row, for the prefill paths."""
|
||||
if self._batch_block_table is None:
|
||||
assert self.block_table is not None
|
||||
indices = self.runtime.request_state_indices
|
||||
assert indices is not None
|
||||
self._batch_block_table = self.block_table.index_select(
|
||||
0, indices.clamp(min=0)
|
||||
)
|
||||
return self._batch_block_table
|
||||
|
||||
def write_target(
|
||||
self, num_input_rows: int, num_slot_rows: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor, int]:
|
||||
|
||||
Reference in New Issue
Block a user