[Core][KV Connector] Start async KV loads after the forward launch when no sync loads are scheduled (#53333)

Signed-off-by: Summer Yang <girasoleyang@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Summer Yang
2026-08-28 21:58:00 +00:00
committed by GitHub
co-authored by Nick Hill
parent 96242aa50d
commit 2aac565cae
17 changed files with 293 additions and 118 deletions
@@ -289,8 +289,8 @@ class KVConnectorBase_V1(ABC):
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
"""
Start loading the KV cache from the connector to vLLM's paged
KV buffer. This is called from the forward context before the
forward pass to enable async loading during model execution.
KV buffer. Loads required by the current forward start before it;
independent asynchronous loads may start after it is submitted.
Args:
forward_context (ForwardContext): the forward context.
@@ -133,6 +133,21 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
"please check and use the latest version"
)
def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None:
super().bind_connector_metadata(connector_metadata)
engine = self._lmcache_engine
if hasattr(engine, "bind_connector_metadata"):
engine.bind_connector_metadata(connector_metadata)
else:
# Compat for engines predating the bind hook: start_load_kv may
# run after the forward (SchedulerOutput.has_sync_kv_loads), so
# the per-step layerwise state it used to reset must be reset
# here. Remove once LMCache ships bind_connector_metadata.
if hasattr(engine, "current_layer"):
engine.current_layer = 0
if hasattr(engine, "layerwise_retrievers"):
engine.layerwise_retrievers = []
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
"""
Start loading the KV cache from the connector to vLLM's paged
@@ -783,6 +783,17 @@ class LMCacheConnectorV1Impl:
####################
# Worker side APIs
####################
def bind_connector_metadata(self, metadata: "KVConnectorMetadata") -> None:
"""Per-step init, called when the runner binds this step's metadata.
The layerwise hooks fire during every forward once metadata is
bound, while start_load_kv may run after the forward launch on
steps without sync loads (SchedulerOutput.has_sync_kv_loads), so
the per-step state they consume must be reset here.
"""
self.current_layer = 0
self.layerwise_retrievers = []
@_lmcache_nvtx_annotate
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
logger.info("Registering KV caches")
@@ -806,8 +817,6 @@ class LMCacheConnectorV1Impl:
The number of elements in kv_caches and layer_names should be
the same.
"""
self.current_layer = 0
if len(self.kv_caches) == 0:
self._init_kv_caches_from_forward_context(forward_context)
@@ -826,8 +835,6 @@ class LMCacheConnectorV1Impl:
self.lmcache_engine.post_init(kvcaches=kvcaches)
self.layerwise_retrievers = []
for idx, request in enumerate(metadata.requests):
if request.load_spec is None:
continue
@@ -283,8 +283,10 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
self.connector_worker.register_kv_caches(kv_caches)
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
# No-op: loads are issued in get_finished() for compute overlap.
pass
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
self.connector_worker.start_load_kv(metadata)
def wait_for_layer_load(self, layer_name: str) -> None:
# No layerwise support - no-op
@@ -301,8 +303,10 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
return
def wait_for_save(self):
# No-op: stores are issued in get_finished() for compute overlap.
pass
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
self.connector_worker.wait_for_save(metadata)
def get_finished(
self, finished_req_ids: set[str]
@@ -1743,36 +1743,17 @@ class MooncakeStoreWorker:
"Started %d Mooncake KV-load receive thread(s)", self.num_recv_threads
)
def start_load_kv(
self,
metadata: MooncakeStoreConnectorMetadata,
):
"""No-op: loads are issued in get_finished() for overlap."""
pass
def start_load_kv(self, metadata: MooncakeStoreConnectorMetadata):
"""Issue async loads.
def wait_for_save(
self,
metadata: MooncakeStoreConnectorMetadata,
):
"""No-op: stores are issued in get_finished() for overlap."""
pass
def get_finished(
self,
finished_req_ids: set[str],
meta: MooncakeStoreConnectorMetadata,
) -> tuple[set[str], set[str]]:
"""Issue all I/O and get completed send/recv request IDs.
All load and store I/O requests are issued here (after model
compute is launched on the compute stream) for better
compute-I/O overlap.
Runs after the forward launch on steps without sync loads
(SchedulerOutput.has_sync_kv_loads), keeping load submission off
the critical path while preserving compute-I/O overlap.
"""
if self._capacity_only:
return set(), set()
return
# Issue async loads
for request in meta.requests:
for request in metadata.requests:
load_spec = request.load_spec
if load_spec is None or not load_spec.can_load:
continue
@@ -1781,21 +1762,40 @@ class MooncakeStoreWorker:
self.recv_request_queue.put(request)
assert self.load_async, "load_async must be True for better performance."
# Issue stores with CUDA event synchronization.
if self.can_put:
current_event = None
for request in meta.requests:
if request.can_save:
current_event = torch.cuda.Event()
current_event.record()
break
for request in meta.requests:
if not request.can_save:
continue
request.current_event = current_event
assert self.kv_send_thread is not None
self.kv_send_thread.add_request(request)
def wait_for_save(self, metadata: MooncakeStoreConnectorMetadata):
"""Issue async stores with CUDA event synchronization.
Runs after the forward launch for compute-I/O overlap.
"""
if self._capacity_only or not self.can_put:
return
current_event = None
for request in metadata.requests:
if request.can_save:
current_event = torch.cuda.Event()
current_event.record()
break
for request in metadata.requests:
if not request.can_save:
continue
request.current_event = current_event
assert self.kv_send_thread is not None
self.kv_send_thread.add_request(request)
def get_finished(
self, finished_req_ids: set[str], meta: MooncakeStoreConnectorMetadata
) -> tuple[set[str], set[str]]:
"""Get completed send/recv request IDs.
Loads are issued in start_load_kv() and stores in wait_for_save().
"""
if self._capacity_only:
return set(), set()
if self.can_put:
self._close_ended_store_requests(finished_req_ids, meta)
# Blocks read by a store job are released by the scheduler when the job
@@ -196,10 +196,11 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
self.worker_handler.handle_preemptions(kv_connector_metadata)
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
pass # Launch loads ops in get_finished() after launching model execution
if self.worker_handler is not None:
self.worker_handler.start_load_kv()
def wait_for_layer_load(self, layer_name: str) -> None:
pass # Always load asynchronously and deferred to get_finished()
pass # Always load asynchronously, issued in start_load_kv()
def save_kv_layer(
self,
@@ -208,10 +209,11 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
attn_metadata: "AttentionMetadata",
**kwargs: Any,
) -> None:
pass # Always save asynchronously and deferred to get_finished()
pass # Always save asynchronously, issued in wait_for_save()
def wait_for_save(self) -> None:
pass # All stores are driven by get_finished() and no wait needed
if self.worker_handler is not None:
self.worker_handler.wait_for_save()
def get_finished(
self,