[Perf][HiSparse] Avoid repeated prefix scans and residency updates (#57930)
Signed-off-by: S1ro1 <matej.sirovatka@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Lucas Wilkinson
Claude Fable 5.1
Matthew Bonanni
parent
7566d83bd3
commit
ca65eb67d9
@@ -99,9 +99,12 @@ class HiSparseCoordinator:
|
||||
kv_cache_config: KVCacheConfig,
|
||||
managers: tuple[SingleTypeKVCacheManager, ...],
|
||||
max_model_len: int,
|
||||
*,
|
||||
num_reprefillable_tokens: int,
|
||||
) -> None:
|
||||
self.managers = managers
|
||||
self.max_model_len = max_model_len
|
||||
self.num_reprefillable_tokens = num_reprefillable_tokens
|
||||
groups = kv_cache_config.kv_cache_groups
|
||||
|
||||
resident_managers: list[HiSparseResidentManager] = []
|
||||
@@ -264,6 +267,7 @@ class HiSparseCoordinator:
|
||||
if manager.adopt_resident_page(request_id, host_idx, block):
|
||||
manager.block_pool.touch([block])
|
||||
state.pinned_clean.add(host_idx)
|
||||
self.block_table_updates.add(request_id)
|
||||
|
||||
def _record_copies(self, request_id: str, num_computed_tokens: int) -> None:
|
||||
"""Index the GPU copies of just-published host blocks for later hits."""
|
||||
@@ -431,6 +435,22 @@ class HiSparseCoordinator:
|
||||
# Host publication and spills
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def advance_scheduled(self, requests: Iterable[tuple[str, int]]) -> None:
|
||||
"""Run the per-step residency work for each scheduled request.
|
||||
|
||||
``requests`` pairs a request id with the number of tokens computed
|
||||
once the step completes. As in ``KVCacheCoordinator.cache_blocks``, the
|
||||
last ``num_reprefillable_tokens`` are not final yet and are not written
|
||||
back.
|
||||
"""
|
||||
if not self.resident_managers:
|
||||
return
|
||||
for request_id, num_tokens in requests:
|
||||
self.plan_prefix_materialization(
|
||||
request_id, max(0, num_tokens - self.num_reprefillable_tokens)
|
||||
)
|
||||
self.update_residency(request_id)
|
||||
|
||||
def plan_prefix_materialization(
|
||||
self, request_id: str, num_computed_tokens: int
|
||||
) -> None:
|
||||
@@ -445,9 +465,8 @@ class HiSparseCoordinator:
|
||||
)
|
||||
state = self._get_request_state(request_id)
|
||||
budget = max(self.max_spill_pages - len(self.spills_to_send), 0)
|
||||
for page_idx in range(num_pages):
|
||||
if page_idx < importing_pages:
|
||||
continue
|
||||
first_page = max(state.ready_prefix_pages, importing_pages)
|
||||
for page_idx in range(first_page, num_pages):
|
||||
if budget == 0:
|
||||
break
|
||||
if page_idx in state.pending_pages:
|
||||
@@ -852,7 +871,10 @@ def get_hisparse_coordinator(
|
||||
assert isinstance(coordinator, HiSparseCoordinator)
|
||||
return coordinator
|
||||
coordinator = HiSparseCoordinator(
|
||||
kv_cache_manager.kv_cache_config, managers, kv_cache_manager.max_model_len
|
||||
kv_cache_manager.kv_cache_config,
|
||||
managers,
|
||||
kv_cache_manager.max_model_len,
|
||||
num_reprefillable_tokens=kv_cache_manager.coordinator.num_reprefillable_tokens,
|
||||
)
|
||||
if coordinator.host_manager is None:
|
||||
raise ValueError("No HiSparse cache group is configured.")
|
||||
|
||||
Reference in New Issue
Block a user