[HiSparse] Own GPU residency instead of editing core block-table rows (#60647)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Matthew Bonanni
2026-10-09 21:25:44 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 9bd074a996
commit d5f356dae0
15 changed files with 429 additions and 128 deletions
+68 -8
View File
@@ -7,10 +7,11 @@ from __future__ import annotations
import math
import mmap
import time
from collections.abc import Callable
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, TypeAlias
import numpy as np
import psutil
import torch
@@ -21,8 +22,9 @@ from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv, round_up
from vllm.utils.mem_utils import get_max_shared_memory_bytes
from vllm.utils.torch_utils import current_stream
from vllm.utils.torch_utils import async_tensor_h2d, current_stream
from vllm.v1.attention.backend import max_decode_query_len
from vllm.v1.hisparse.types import SparseKVResidencyUpdate
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
from vllm.v1.simple_kv_offload.cuda_mem_ops import pin_tensor
@@ -499,16 +501,19 @@ class HiSparsePrefillStagingPlan:
def ensure_gpu_sources(
self,
resident_block_table: torch.Tensor,
resident_state_rows: torch.Tensor,
state_indices: torch.Tensor,
resident_block_size: int,
) -> None:
"""Resolve which staged rows can be served from the resident cache.
Computed once per plan (the resident block table is shared by every
layer in the group); non-null resident pages become miss_mask=0 rows
gathered device-to-device by ``gather_prefill_cache``.
``resident_state_rows`` is a resident group's persistent table by
request state row and ``state_indices`` holds each staged request's
state row. Computed once per plan and resident group (every layer in
the group shares the table); non-null resident pages become
miss_mask=0 rows gathered device-to-device by ``gather_prefill_cache``.
"""
source_key = (resident_block_table.data_ptr(), resident_block_size)
source_key = (resident_state_rows.data_ptr(), resident_block_size)
if self.gpu_source_key == source_key:
return
block_size = self.block_size
@@ -519,8 +524,11 @@ class HiSparsePrefillStagingPlan:
host_ids = self.row_ids[0].view(num_unique, block_size)[:, 0] // block_size
new_bt = self.block_table.to(torch.int64)
num_rows, num_cols = new_bt.shape
if num_rows == 0 or resident_block_table.shape[0] < num_rows:
if num_rows == 0 or state_indices.shape[0] < num_rows:
return
resident_block_table = resident_state_rows.index_select(
0, state_indices[:num_rows].clamp(min=0)
)
# One representative (row, col) per unique host block: any request
# referencing the block holds an equivalent (refcounted) resident view.
flat_pos = torch.arange(num_rows * num_cols, device=device)
@@ -1107,11 +1115,50 @@ class HiSparseRuntime:
return physical_topk_indices
def update_hisparse_residency(
residency: torch.Tensor,
updates: Mapping[str, SparseKVResidencyUpdate],
request_ids: Sequence[str],
request_state_indices: torch.Tensor,
) -> None:
"""Apply scheduled requests' residency changes.
``residency`` is ``[state rows, resident groups, pages]``: the GPU block of
each page, or block 0 when the page is read from the host.
"""
if not updates:
return
batch_rows = {request_id: row for row, request_id in enumerate(request_ids)}
num_groups = residency.shape[1]
rows: list[int] = []
pages: list[int] = []
block_ids: list[list[int]] = [[] for _ in range(num_groups)]
for request_id, update in updates.items():
rows.append(batch_rows[request_id])
pages.extend(update.pages)
for group_block_ids, update_block_ids in zip(
block_ids, update.block_ids, strict=True
):
group_block_ids.extend(update_block_ids)
host_values = np.empty((len(pages), 2 + num_groups), dtype=np.int32)
host_values[:, 0] = np.repeat(
rows, [len(update.pages) for update in updates.values()]
)
host_values[:, 1] = pages
host_values[:, 2:] = np.asarray(block_ids, dtype=np.int32).T
values = async_tensor_h2d(
host_values, device=request_state_indices.device, dtype=torch.int32
)
state_rows = request_state_indices[values[:, 0]]
residency[state_rows, :, values[:, 1]] = values[:, 2:]
class HiSparseCacheHandle:
"""Attention-facing handle for resident KV and sparse offload state."""
def __init__(self, runtime: HiSparseRuntime) -> None:
self.view: PagedCacheView | None = None
self.residency: torch.Tensor | None = None
self.block_table: torch.Tensor | None = None
self.source_block_table: torch.Tensor | None = None
self.slot_mapping: torch.Tensor | None = None
@@ -1128,6 +1175,7 @@ class HiSparseCacheHandle:
# Speculator layers write their rows after the target forward.
self.draft_layer = False
self.index_group_caches: list[HiSparseCacheHandle] = [self]
self._batch_block_table: torch.Tensor | None = None
def prepare_group_for_batch(self, attn_metadata: Any | None) -> None:
assert self.runtime.is_group_leader
@@ -1136,6 +1184,7 @@ class HiSparseCacheHandle:
def _prepare_for_batch(self, attn_metadata: Any | None) -> None:
self.dummy_batch = attn_metadata is None
self._batch_block_table = None
self.runtime.begin_forward()
self.num_actual_tokens = (
attn_metadata.num_actual_tokens if attn_metadata is not None else 0
@@ -1151,6 +1200,17 @@ class HiSparseCacheHandle:
not self.decode_batch or self.runtime.eager_host_mirror
)
def batch_block_table(self) -> torch.Tensor:
"""Resident rows by this step's batch row, for the prefill paths."""
if self._batch_block_table is None:
assert self.block_table is not None
indices = self.runtime.request_state_indices
assert indices is not None
self._batch_block_table = self.block_table.index_select(
0, indices.clamp(min=0)
)
return self._batch_block_table
def write_target(
self, num_input_rows: int, num_slot_rows: int
) -> tuple[torch.Tensor, torch.Tensor, int]: