[Perf][HiSparse] Avoid repeated prefix scans and residency updates (#57930)

Signed-off-by: S1ro1 <matej.sirovatka@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Matej Sirovatka
2026-10-01 21:48:18 +00:00
committed by GitHub
co-authored by Lucas Wilkinson Claude Fable 5.1 Matthew Bonanni
parent 7566d83bd3
commit ca65eb67d9
5 changed files with 236 additions and 83 deletions
+26 -4
View File
@@ -99,9 +99,12 @@ class HiSparseCoordinator:
kv_cache_config: KVCacheConfig,
managers: tuple[SingleTypeKVCacheManager, ...],
max_model_len: int,
*,
num_reprefillable_tokens: int,
) -> None:
self.managers = managers
self.max_model_len = max_model_len
self.num_reprefillable_tokens = num_reprefillable_tokens
groups = kv_cache_config.kv_cache_groups
resident_managers: list[HiSparseResidentManager] = []
@@ -264,6 +267,7 @@ class HiSparseCoordinator:
if manager.adopt_resident_page(request_id, host_idx, block):
manager.block_pool.touch([block])
state.pinned_clean.add(host_idx)
self.block_table_updates.add(request_id)
def _record_copies(self, request_id: str, num_computed_tokens: int) -> None:
"""Index the GPU copies of just-published host blocks for later hits."""
@@ -431,6 +435,22 @@ class HiSparseCoordinator:
# Host publication and spills
# ------------------------------------------------------------------
def advance_scheduled(self, requests: Iterable[tuple[str, int]]) -> None:
"""Run the per-step residency work for each scheduled request.
``requests`` pairs a request id with the number of tokens computed
once the step completes. As in ``KVCacheCoordinator.cache_blocks``, the
last ``num_reprefillable_tokens`` are not final yet and are not written
back.
"""
if not self.resident_managers:
return
for request_id, num_tokens in requests:
self.plan_prefix_materialization(
request_id, max(0, num_tokens - self.num_reprefillable_tokens)
)
self.update_residency(request_id)
def plan_prefix_materialization(
self, request_id: str, num_computed_tokens: int
) -> None:
@@ -445,9 +465,8 @@ class HiSparseCoordinator:
)
state = self._get_request_state(request_id)
budget = max(self.max_spill_pages - len(self.spills_to_send), 0)
for page_idx in range(num_pages):
if page_idx < importing_pages:
continue
first_page = max(state.ready_prefix_pages, importing_pages)
for page_idx in range(first_page, num_pages):
if budget == 0:
break
if page_idx in state.pending_pages:
@@ -852,7 +871,10 @@ def get_hisparse_coordinator(
assert isinstance(coordinator, HiSparseCoordinator)
return coordinator
coordinator = HiSparseCoordinator(
kv_cache_manager.kv_cache_config, managers, kv_cache_manager.max_model_len
kv_cache_manager.kv_cache_config,
managers,
kv_cache_manager.max_model_len,
num_reprefillable_tokens=kv_cache_manager.coordinator.num_reprefillable_tokens,
)
if coordinator.host_manager is None:
raise ValueError("No HiSparse cache group is configured.")