[Perf][HiSparse] Avoid repeated prefix scans and residency updates (#57930)

Signed-off-by: S1ro1 <matej.sirovatka@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Matej Sirovatka
2026-10-01 21:48:18 +00:00
committed by GitHub
co-authored by Lucas Wilkinson Claude Fable 5.1 Matthew Bonanni
parent 7566d83bd3
commit ca65eb67d9
5 changed files with 236 additions and 83 deletions
@@ -106,21 +106,6 @@ class HiSparseConnectorScheduler:
# HiSparse rebinds worker state every step, so the load must start
# before the forward rather than being deferred to post-forward.
scheduler_output.has_sync_kv_loads = True
scheduler_output.block_table_updates = (
self.coordinator.take_block_table_updates() or None
)
command = self.coordinator.build_offload_command()
host_block_copies = self.coordinator.take_host_block_copies()
source_group_id = self.coordinator.host_group_id
assert source_group_id is not None
source_block_ids = [
block_id
for request in scheduler_output.scheduled_new_reqs
for block_id in request.block_ids[source_group_id]
]
for new_block_ids in scheduler_output.scheduled_cached_reqs.new_block_ids:
if new_block_ids is not None:
source_block_ids.extend(new_block_ids[source_group_id])
num_computed_tokens = {
request.req_id: request.num_computed_tokens
for request in scheduler_output.scheduled_new_reqs
@@ -141,6 +126,29 @@ class HiSparseConnectorScheduler:
scheduler_output.num_scheduled_tokens.items()
)
)
# Runs first: it plans this step's transfers and table updates.
self.coordinator.advance_scheduled(
(
request_id,
min(start + count, self.requests[request_id].num_tokens),
)
for request_id, start, count in scheduled_requests
)
scheduler_output.block_table_updates = (
self.coordinator.take_block_table_updates() or None
)
command = self.coordinator.build_offload_command()
host_block_copies = self.coordinator.take_host_block_copies()
source_group_id = self.coordinator.host_group_id
assert source_group_id is not None
source_block_ids = [
block_id
for request in scheduler_output.scheduled_new_reqs
for block_id in request.block_ids[source_group_id]
]
for new_block_ids in scheduler_output.scheduled_cached_reqs.new_block_ids:
if new_block_ids is not None:
source_block_ids.extend(new_block_ids[source_group_id])
row_mirrors = {}
for request_id, scheduled_start, scheduled_count in scheduled_requests:
mirror_start = scheduled_start