[Core][KV Connector] Start async KV loads after the forward launch when no sync loads are scheduled (#53333)
Signed-off-by: Summer Yang <girasoleyang@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -54,6 +54,7 @@ class ActiveKVConnector(KVConnector):
|
||||
self.kv_connector.register_kv_caches(kv_caches_dict)
|
||||
self.kv_connector.set_host_xfer_buffer_ops(copy_kv_blocks)
|
||||
|
||||
self._pending_load_start = False
|
||||
self._disabled = False
|
||||
|
||||
def pre_forward(self, scheduler_output: "SchedulerOutput") -> None:
|
||||
@@ -65,6 +66,16 @@ class ActiveKVConnector(KVConnector):
|
||||
self.kv_connector.handle_preemptions(kv_connector_metadata)
|
||||
self.kv_connector.bind_connector_metadata(kv_connector_metadata)
|
||||
|
||||
if scheduler_output.has_sync_kv_loads:
|
||||
# Sync loads need to run before this step's forward.
|
||||
self._start_load_kv()
|
||||
else:
|
||||
# Start any async loads in post-forward instead, keeping
|
||||
# their host-side submission cost off the critical path.
|
||||
self._pending_load_start = True
|
||||
|
||||
def _start_load_kv(self) -> None:
|
||||
self._pending_load_start = False
|
||||
# TODO: sort out KV Connectors' use of forward_context
|
||||
if is_forward_context_available():
|
||||
self.kv_connector.start_load_kv(get_forward_context())
|
||||
@@ -78,6 +89,9 @@ class ActiveKVConnector(KVConnector):
|
||||
if self._disabled:
|
||||
return None
|
||||
|
||||
if self._pending_load_start:
|
||||
self._start_load_kv()
|
||||
|
||||
output = KVConnectorOutput()
|
||||
if wait_for_save:
|
||||
self.kv_connector.wait_for_save()
|
||||
|
||||
Reference in New Issue
Block a user