[Bugfix][HiSparse] Drop the unaccounted, write-only prefill mirror staging buffer (#60429)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Matthew Bonanni
2026-10-08 14:00:49 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 194da61de1
commit 4ed4c4433b
6 changed files with 5 additions and 81 deletions
+3 -52
View File
@@ -1120,10 +1120,7 @@ class HiSparseCacheHandle:
self.num_decode_tokens = 0
self.req_id_per_token: torch.Tensor | None = None
self.host_mirror_required = False
self.mirror_from_resident = False
self.mirror_slot_mapping: torch.Tensor | None = None
self.mirror_staging_cache: torch.Tensor | None = None
self.mirror_staging_slots: torch.Tensor | None = None
self.submit_layer_mirror: Callable[[], None] | None = None
# Speculator layers write their rows after the target forward.
self.draft_layer = False
@@ -1160,26 +1157,6 @@ class HiSparseCacheHandle:
num_rows = min(num_rows, self.runtime.max_num_reqs)
return self.view.cache, self.slot_mapping[:num_rows], num_rows
def mirror_write_target(
self, num_rows: int
) -> tuple[torch.Tensor, torch.Tensor] | None:
if (
not self.host_mirror_required
or self.decode_batch
or self.mirror_from_resident
):
return None
cache = self.mirror_staging_cache
slots = self.mirror_staging_slots
if cache is None or slots is None:
raise RuntimeError("HiSparse prefill mirror staging is not bound.")
if num_rows > slots.numel():
raise RuntimeError(
"HiSparse prefill mirror exceeds staging capacity: "
f"{num_rows} > {slots.numel()}."
)
return cache, slots[:num_rows]
def finish_kv_update(self) -> None:
if self.dummy_batch:
return
@@ -1242,9 +1219,8 @@ def initialize_hisparse_runtime_buffers(
cache_handles: list[HiSparseCacheHandle],
*,
max_num_reqs: int,
max_num_batched_tokens: int,
) -> None:
"""Allocate shared request state and per-layer prefill staging buffers."""
"""Allocate the request state shared by every HiSparse runtime."""
assert cache_handles
resident = cache_handles[0].view
assert resident is not None
@@ -1252,26 +1228,8 @@ def initialize_hisparse_runtime_buffers(
request_state_indices = torch.full(
(max_num_reqs,), -1, dtype=torch.int32, device=device
)
staging_blocks = (
max_num_batched_tokens + resident.block_size - 1
) // resident.block_size
mirror_staging_caches = torch.empty(
(
len(cache_handles),
staging_blocks,
resident.block_size,
resident.cache.shape[-1],
),
dtype=resident.cache.dtype,
device=device,
)
mirror_staging_slots = torch.arange(
max_num_batched_tokens, dtype=torch.int64, device=device
)
for layer_index, cache_handle in enumerate(cache_handles):
for cache_handle in cache_handles:
cache_handle.runtime.request_state_indices = request_state_indices
cache_handle.mirror_staging_cache = mirror_staging_caches[layer_index]
cache_handle.mirror_staging_slots = mirror_staging_slots
def create_hisparse_cache_handle(
@@ -1325,11 +1283,4 @@ def create_hisparse_cache_handle(
config.device_buffer_size,
max_num_reqs,
)
handle = HiSparseCacheHandle(runtime)
speculative_config = vllm_config.speculative_config
handle.mirror_from_resident = bool(
vllm_config.scheduler_config.async_scheduling
and speculative_config is not None
and speculative_config.uses_draft_kv_cache()
)
return handle
return HiSparseCacheHandle(runtime)