[Core][KV Connector] Start async KV loads after the forward launch when no sync loads are scheduled (#53333)

Signed-off-by: Summer Yang <girasoleyang@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Summer Yang
2026-08-28 21:58:00 +00:00
committed by GitHub
co-authored by Nick Hill
parent 96242aa50d
commit 2aac565cae
17 changed files with 293 additions and 118 deletions
+14
View File
@@ -54,6 +54,7 @@ class ActiveKVConnector(KVConnector):
self.kv_connector.register_kv_caches(kv_caches_dict)
self.kv_connector.set_host_xfer_buffer_ops(copy_kv_blocks)
self._pending_load_start = False
self._disabled = False
def pre_forward(self, scheduler_output: "SchedulerOutput") -> None:
@@ -65,6 +66,16 @@ class ActiveKVConnector(KVConnector):
self.kv_connector.handle_preemptions(kv_connector_metadata)
self.kv_connector.bind_connector_metadata(kv_connector_metadata)
if scheduler_output.has_sync_kv_loads:
# Sync loads need to run before this step's forward.
self._start_load_kv()
else:
# Start any async loads in post-forward instead, keeping
# their host-side submission cost off the critical path.
self._pending_load_start = True
def _start_load_kv(self) -> None:
self._pending_load_start = False
# TODO: sort out KV Connectors' use of forward_context
if is_forward_context_available():
self.kv_connector.start_load_kv(get_forward_context())
@@ -78,6 +89,9 @@ class ActiveKVConnector(KVConnector):
if self._disabled:
return None
if self._pending_load_start:
self._start_load_kv()
output = KVConnectorOutput()
if wait_for_save:
self.kv_connector.wait_for_save()