[Perf][HiSparse] Avoid repeated prefix scans and residency updates (#57930)
Signed-off-by: S1ro1 <matej.sirovatka@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Lucas Wilkinson
Claude Fable 5.1
Matthew Bonanni
parent
7566d83bd3
commit
ca65eb67d9
@@ -106,21 +106,6 @@ class HiSparseConnectorScheduler:
|
||||
# HiSparse rebinds worker state every step, so the load must start
|
||||
# before the forward rather than being deferred to post-forward.
|
||||
scheduler_output.has_sync_kv_loads = True
|
||||
scheduler_output.block_table_updates = (
|
||||
self.coordinator.take_block_table_updates() or None
|
||||
)
|
||||
command = self.coordinator.build_offload_command()
|
||||
host_block_copies = self.coordinator.take_host_block_copies()
|
||||
source_group_id = self.coordinator.host_group_id
|
||||
assert source_group_id is not None
|
||||
source_block_ids = [
|
||||
block_id
|
||||
for request in scheduler_output.scheduled_new_reqs
|
||||
for block_id in request.block_ids[source_group_id]
|
||||
]
|
||||
for new_block_ids in scheduler_output.scheduled_cached_reqs.new_block_ids:
|
||||
if new_block_ids is not None:
|
||||
source_block_ids.extend(new_block_ids[source_group_id])
|
||||
num_computed_tokens = {
|
||||
request.req_id: request.num_computed_tokens
|
||||
for request in scheduler_output.scheduled_new_reqs
|
||||
@@ -141,6 +126,29 @@ class HiSparseConnectorScheduler:
|
||||
scheduler_output.num_scheduled_tokens.items()
|
||||
)
|
||||
)
|
||||
# Runs first: it plans this step's transfers and table updates.
|
||||
self.coordinator.advance_scheduled(
|
||||
(
|
||||
request_id,
|
||||
min(start + count, self.requests[request_id].num_tokens),
|
||||
)
|
||||
for request_id, start, count in scheduled_requests
|
||||
)
|
||||
scheduler_output.block_table_updates = (
|
||||
self.coordinator.take_block_table_updates() or None
|
||||
)
|
||||
command = self.coordinator.build_offload_command()
|
||||
host_block_copies = self.coordinator.take_host_block_copies()
|
||||
source_group_id = self.coordinator.host_group_id
|
||||
assert source_group_id is not None
|
||||
source_block_ids = [
|
||||
block_id
|
||||
for request in scheduler_output.scheduled_new_reqs
|
||||
for block_id in request.block_ids[source_group_id]
|
||||
]
|
||||
for new_block_ids in scheduler_output.scheduled_cached_reqs.new_block_ids:
|
||||
if new_block_ids is not None:
|
||||
source_block_ids.extend(new_block_ids[source_group_id])
|
||||
row_mirrors = {}
|
||||
for request_id, scheduled_start, scheduled_count in scheduled_requests:
|
||||
mirror_start = scheduled_start
|
||||
|
||||
Reference in New Issue
Block a user