[Core][KV Connector] Start async KV loads after the forward launch when no sync loads are scheduled (#53333)
Signed-off-by: Summer Yang <girasoleyang@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -289,8 +289,8 @@ class KVConnectorBase_V1(ABC):
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
|
||||
"""
|
||||
Start loading the KV cache from the connector to vLLM's paged
|
||||
KV buffer. This is called from the forward context before the
|
||||
forward pass to enable async loading during model execution.
|
||||
KV buffer. Loads required by the current forward start before it;
|
||||
independent asynchronous loads may start after it is submitted.
|
||||
|
||||
Args:
|
||||
forward_context (ForwardContext): the forward context.
|
||||
|
||||
@@ -133,6 +133,21 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
"please check and use the latest version"
|
||||
)
|
||||
|
||||
def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None:
|
||||
super().bind_connector_metadata(connector_metadata)
|
||||
engine = self._lmcache_engine
|
||||
if hasattr(engine, "bind_connector_metadata"):
|
||||
engine.bind_connector_metadata(connector_metadata)
|
||||
else:
|
||||
# Compat for engines predating the bind hook: start_load_kv may
|
||||
# run after the forward (SchedulerOutput.has_sync_kv_loads), so
|
||||
# the per-step layerwise state it used to reset must be reset
|
||||
# here. Remove once LMCache ships bind_connector_metadata.
|
||||
if hasattr(engine, "current_layer"):
|
||||
engine.current_layer = 0
|
||||
if hasattr(engine, "layerwise_retrievers"):
|
||||
engine.layerwise_retrievers = []
|
||||
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
|
||||
"""
|
||||
Start loading the KV cache from the connector to vLLM's paged
|
||||
|
||||
+11
-4
@@ -783,6 +783,17 @@ class LMCacheConnectorV1Impl:
|
||||
####################
|
||||
# Worker side APIs
|
||||
####################
|
||||
def bind_connector_metadata(self, metadata: "KVConnectorMetadata") -> None:
|
||||
"""Per-step init, called when the runner binds this step's metadata.
|
||||
|
||||
The layerwise hooks fire during every forward once metadata is
|
||||
bound, while start_load_kv may run after the forward launch on
|
||||
steps without sync loads (SchedulerOutput.has_sync_kv_loads), so
|
||||
the per-step state they consume must be reset here.
|
||||
"""
|
||||
self.current_layer = 0
|
||||
self.layerwise_retrievers = []
|
||||
|
||||
@_lmcache_nvtx_annotate
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
logger.info("Registering KV caches")
|
||||
@@ -806,8 +817,6 @@ class LMCacheConnectorV1Impl:
|
||||
The number of elements in kv_caches and layer_names should be
|
||||
the same.
|
||||
"""
|
||||
self.current_layer = 0
|
||||
|
||||
if len(self.kv_caches) == 0:
|
||||
self._init_kv_caches_from_forward_context(forward_context)
|
||||
|
||||
@@ -826,8 +835,6 @@ class LMCacheConnectorV1Impl:
|
||||
|
||||
self.lmcache_engine.post_init(kvcaches=kvcaches)
|
||||
|
||||
self.layerwise_retrievers = []
|
||||
|
||||
for idx, request in enumerate(metadata.requests):
|
||||
if request.load_spec is None:
|
||||
continue
|
||||
|
||||
@@ -283,8 +283,10 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
self.connector_worker.register_kv_caches(kv_caches)
|
||||
|
||||
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
|
||||
# No-op: loads are issued in get_finished() for compute overlap.
|
||||
pass
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
|
||||
self.connector_worker.start_load_kv(metadata)
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
# No layerwise support - no-op
|
||||
@@ -301,8 +303,10 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
return
|
||||
|
||||
def wait_for_save(self):
|
||||
# No-op: stores are issued in get_finished() for compute overlap.
|
||||
pass
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
|
||||
self.connector_worker.wait_for_save(metadata)
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
|
||||
@@ -1743,36 +1743,17 @@ class MooncakeStoreWorker:
|
||||
"Started %d Mooncake KV-load receive thread(s)", self.num_recv_threads
|
||||
)
|
||||
|
||||
def start_load_kv(
|
||||
self,
|
||||
metadata: MooncakeStoreConnectorMetadata,
|
||||
):
|
||||
"""No-op: loads are issued in get_finished() for overlap."""
|
||||
pass
|
||||
def start_load_kv(self, metadata: MooncakeStoreConnectorMetadata):
|
||||
"""Issue async loads.
|
||||
|
||||
def wait_for_save(
|
||||
self,
|
||||
metadata: MooncakeStoreConnectorMetadata,
|
||||
):
|
||||
"""No-op: stores are issued in get_finished() for overlap."""
|
||||
pass
|
||||
|
||||
def get_finished(
|
||||
self,
|
||||
finished_req_ids: set[str],
|
||||
meta: MooncakeStoreConnectorMetadata,
|
||||
) -> tuple[set[str], set[str]]:
|
||||
"""Issue all I/O and get completed send/recv request IDs.
|
||||
|
||||
All load and store I/O requests are issued here (after model
|
||||
compute is launched on the compute stream) for better
|
||||
compute-I/O overlap.
|
||||
Runs after the forward launch on steps without sync loads
|
||||
(SchedulerOutput.has_sync_kv_loads), keeping load submission off
|
||||
the critical path while preserving compute-I/O overlap.
|
||||
"""
|
||||
if self._capacity_only:
|
||||
return set(), set()
|
||||
return
|
||||
|
||||
# Issue async loads
|
||||
for request in meta.requests:
|
||||
for request in metadata.requests:
|
||||
load_spec = request.load_spec
|
||||
if load_spec is None or not load_spec.can_load:
|
||||
continue
|
||||
@@ -1781,21 +1762,40 @@ class MooncakeStoreWorker:
|
||||
self.recv_request_queue.put(request)
|
||||
|
||||
assert self.load_async, "load_async must be True for better performance."
|
||||
# Issue stores with CUDA event synchronization.
|
||||
if self.can_put:
|
||||
current_event = None
|
||||
for request in meta.requests:
|
||||
if request.can_save:
|
||||
current_event = torch.cuda.Event()
|
||||
current_event.record()
|
||||
break
|
||||
|
||||
for request in meta.requests:
|
||||
if not request.can_save:
|
||||
continue
|
||||
request.current_event = current_event
|
||||
assert self.kv_send_thread is not None
|
||||
self.kv_send_thread.add_request(request)
|
||||
def wait_for_save(self, metadata: MooncakeStoreConnectorMetadata):
|
||||
"""Issue async stores with CUDA event synchronization.
|
||||
|
||||
Runs after the forward launch for compute-I/O overlap.
|
||||
"""
|
||||
if self._capacity_only or not self.can_put:
|
||||
return
|
||||
|
||||
current_event = None
|
||||
for request in metadata.requests:
|
||||
if request.can_save:
|
||||
current_event = torch.cuda.Event()
|
||||
current_event.record()
|
||||
break
|
||||
|
||||
for request in metadata.requests:
|
||||
if not request.can_save:
|
||||
continue
|
||||
request.current_event = current_event
|
||||
assert self.kv_send_thread is not None
|
||||
self.kv_send_thread.add_request(request)
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str], meta: MooncakeStoreConnectorMetadata
|
||||
) -> tuple[set[str], set[str]]:
|
||||
"""Get completed send/recv request IDs.
|
||||
|
||||
Loads are issued in start_load_kv() and stores in wait_for_save().
|
||||
"""
|
||||
if self._capacity_only:
|
||||
return set(), set()
|
||||
|
||||
if self.can_put:
|
||||
self._close_ended_store_requests(finished_req_ids, meta)
|
||||
|
||||
# Blocks read by a store job are released by the scheduler when the job
|
||||
|
||||
@@ -196,10 +196,11 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
self.worker_handler.handle_preemptions(kv_connector_metadata)
|
||||
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
|
||||
pass # Launch loads ops in get_finished() after launching model execution
|
||||
if self.worker_handler is not None:
|
||||
self.worker_handler.start_load_kv()
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
pass # Always load asynchronously and deferred to get_finished()
|
||||
pass # Always load asynchronously, issued in start_load_kv()
|
||||
|
||||
def save_kv_layer(
|
||||
self,
|
||||
@@ -208,10 +209,11 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
attn_metadata: "AttentionMetadata",
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
pass # Always save asynchronously and deferred to get_finished()
|
||||
pass # Always save asynchronously, issued in wait_for_save()
|
||||
|
||||
def wait_for_save(self) -> None:
|
||||
pass # All stores are driven by get_finished() and no wait needed
|
||||
if self.worker_handler is not None:
|
||||
self.worker_handler.wait_for_save()
|
||||
|
||||
def get_finished(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user