[Core][KV Connector] Start async KV loads after the forward launch when no sync loads are scheduled (#53333)

Signed-off-by: Summer Yang <girasoleyang@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Summer Yang
2026-08-28 21:58:00 +00:00
committed by GitHub
co-authored by Nick Hill
parent 96242aa50d
commit 2aac565cae
17 changed files with 293 additions and 118 deletions
@@ -289,8 +289,8 @@ class KVConnectorBase_V1(ABC):
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
"""
Start loading the KV cache from the connector to vLLM's paged
KV buffer. This is called from the forward context before the
forward pass to enable async loading during model execution.
KV buffer. Loads required by the current forward start before it;
independent asynchronous loads may start after it is submitted.
Args:
forward_context (ForwardContext): the forward context.
@@ -133,6 +133,21 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
"please check and use the latest version"
)
def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None:
super().bind_connector_metadata(connector_metadata)
engine = self._lmcache_engine
if hasattr(engine, "bind_connector_metadata"):
engine.bind_connector_metadata(connector_metadata)
else:
# Compat for engines predating the bind hook: start_load_kv may
# run after the forward (SchedulerOutput.has_sync_kv_loads), so
# the per-step layerwise state it used to reset must be reset
# here. Remove once LMCache ships bind_connector_metadata.
if hasattr(engine, "current_layer"):
engine.current_layer = 0
if hasattr(engine, "layerwise_retrievers"):
engine.layerwise_retrievers = []
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
"""
Start loading the KV cache from the connector to vLLM's paged
@@ -783,6 +783,17 @@ class LMCacheConnectorV1Impl:
####################
# Worker side APIs
####################
def bind_connector_metadata(self, metadata: "KVConnectorMetadata") -> None:
"""Per-step init, called when the runner binds this step's metadata.
The layerwise hooks fire during every forward once metadata is
bound, while start_load_kv may run after the forward launch on
steps without sync loads (SchedulerOutput.has_sync_kv_loads), so
the per-step state they consume must be reset here.
"""
self.current_layer = 0
self.layerwise_retrievers = []
@_lmcache_nvtx_annotate
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
logger.info("Registering KV caches")
@@ -806,8 +817,6 @@ class LMCacheConnectorV1Impl:
The number of elements in kv_caches and layer_names should be
the same.
"""
self.current_layer = 0
if len(self.kv_caches) == 0:
self._init_kv_caches_from_forward_context(forward_context)
@@ -826,8 +835,6 @@ class LMCacheConnectorV1Impl:
self.lmcache_engine.post_init(kvcaches=kvcaches)
self.layerwise_retrievers = []
for idx, request in enumerate(metadata.requests):
if request.load_spec is None:
continue
@@ -283,8 +283,10 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
self.connector_worker.register_kv_caches(kv_caches)
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
# No-op: loads are issued in get_finished() for compute overlap.
pass
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
self.connector_worker.start_load_kv(metadata)
def wait_for_layer_load(self, layer_name: str) -> None:
# No layerwise support - no-op
@@ -301,8 +303,10 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
return
def wait_for_save(self):
# No-op: stores are issued in get_finished() for compute overlap.
pass
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
self.connector_worker.wait_for_save(metadata)
def get_finished(
self, finished_req_ids: set[str]
@@ -1743,36 +1743,17 @@ class MooncakeStoreWorker:
"Started %d Mooncake KV-load receive thread(s)", self.num_recv_threads
)
def start_load_kv(
self,
metadata: MooncakeStoreConnectorMetadata,
):
"""No-op: loads are issued in get_finished() for overlap."""
pass
def start_load_kv(self, metadata: MooncakeStoreConnectorMetadata):
"""Issue async loads.
def wait_for_save(
self,
metadata: MooncakeStoreConnectorMetadata,
):
"""No-op: stores are issued in get_finished() for overlap."""
pass
def get_finished(
self,
finished_req_ids: set[str],
meta: MooncakeStoreConnectorMetadata,
) -> tuple[set[str], set[str]]:
"""Issue all I/O and get completed send/recv request IDs.
All load and store I/O requests are issued here (after model
compute is launched on the compute stream) for better
compute-I/O overlap.
Runs after the forward launch on steps without sync loads
(SchedulerOutput.has_sync_kv_loads), keeping load submission off
the critical path while preserving compute-I/O overlap.
"""
if self._capacity_only:
return set(), set()
return
# Issue async loads
for request in meta.requests:
for request in metadata.requests:
load_spec = request.load_spec
if load_spec is None or not load_spec.can_load:
continue
@@ -1781,21 +1762,40 @@ class MooncakeStoreWorker:
self.recv_request_queue.put(request)
assert self.load_async, "load_async must be True for better performance."
# Issue stores with CUDA event synchronization.
if self.can_put:
current_event = None
for request in meta.requests:
if request.can_save:
current_event = torch.cuda.Event()
current_event.record()
break
for request in meta.requests:
if not request.can_save:
continue
request.current_event = current_event
assert self.kv_send_thread is not None
self.kv_send_thread.add_request(request)
def wait_for_save(self, metadata: MooncakeStoreConnectorMetadata):
"""Issue async stores with CUDA event synchronization.
Runs after the forward launch for compute-I/O overlap.
"""
if self._capacity_only or not self.can_put:
return
current_event = None
for request in metadata.requests:
if request.can_save:
current_event = torch.cuda.Event()
current_event.record()
break
for request in metadata.requests:
if not request.can_save:
continue
request.current_event = current_event
assert self.kv_send_thread is not None
self.kv_send_thread.add_request(request)
def get_finished(
self, finished_req_ids: set[str], meta: MooncakeStoreConnectorMetadata
) -> tuple[set[str], set[str]]:
"""Get completed send/recv request IDs.
Loads are issued in start_load_kv() and stores in wait_for_save().
"""
if self._capacity_only:
return set(), set()
if self.can_put:
self._close_ended_store_requests(finished_req_ids, meta)
# Blocks read by a store job are released by the scheduler when the job
@@ -196,10 +196,11 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
self.worker_handler.handle_preemptions(kv_connector_metadata)
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
pass # Launch loads ops in get_finished() after launching model execution
if self.worker_handler is not None:
self.worker_handler.start_load_kv()
def wait_for_layer_load(self, layer_name: str) -> None:
pass # Always load asynchronously and deferred to get_finished()
pass # Always load asynchronously, issued in start_load_kv()
def save_kv_layer(
self,
@@ -208,10 +209,11 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
attn_metadata: "AttentionMetadata",
**kwargs: Any,
) -> None:
pass # Always save asynchronously and deferred to get_finished()
pass # Always save asynchronously, issued in wait_for_save()
def wait_for_save(self) -> None:
pass # All stores are driven by get_finished() and no wait needed
if self.worker_handler is not None:
self.worker_handler.wait_for_save()
def get_finished(
self,
+4
View File
@@ -262,6 +262,10 @@ class SchedulerOutput:
# KV Cache Connector metadata.
kv_connector_metadata: KVConnectorMetadata | None = None
# Whether any scheduled request consumes KV that the connector loads
# synchronously during this step (load_async=False).
has_sync_kv_loads: bool = False
# EC Cache Connector metadata
ec_connector_metadata: ECConnectorMetadata | None = None
# EC Cache Manager metadata
+6
View File
@@ -532,6 +532,8 @@ class Scheduler(SchedulerInterface):
scheduled_spec_decode_tokens: dict[str, list[int]] = {}
# Whether the running batch contains any prefill requests.
prefill_scheduled = False
# Whether any scheduled request has a synchronous connector KV load.
has_sync_kv_loads = False
# For logging.
scheduled_timestamp = time.monotonic()
@@ -1142,6 +1144,9 @@ class Scheduler(SchedulerInterface):
continue
self.running.append(request)
if num_external_computed_tokens > 0:
# load_kv_async is False here
has_sync_kv_loads = True
if self.log_stats:
request.record_event(
EngineCoreEventType.SCHEDULED, scheduled_timestamp
@@ -1318,6 +1323,7 @@ class Scheduler(SchedulerInterface):
finished_req_ids=self.finished_req_ids,
free_encoder_mm_hashes=self.encoder_cache_manager.get_freed_mm_hashes(),
new_block_ids_to_zero=self._get_new_block_ids_to_zero(),
has_sync_kv_loads=has_sync_kv_loads,
kv_cache_block_copies=pending_kv_cache_block_copies,
partial_tail_offloads=pending_partial_tail_offloads,
num_spec_tokens_to_schedule=num_spec_tokens_to_schedule,
+40 -39
View File
@@ -231,58 +231,59 @@ class SimpleCPUOffloadWorker:
self._connector_metadata = None
def start_load_kv(self) -> None:
# NOTE: we defer launching both load and store to get_finished(),
# which runs after model execution. This hides the CPU-side
# block copy op overhead (~5ms) behind GPU compute.
pass
# Invoked after the forward launch on steps without sync loads
# (SchedulerOutput.has_sync_kv_loads), so the CPU-side block copy
# op overhead (~5ms) stays hidden behind GPU compute. Stores are
# issued in wait_for_save().
metadata = self._connector_metadata
if metadata is not None and metadata.load_cpu_blocks:
backend = self._backend
assert backend is not None
backend.launch_copy(
metadata.load_cpu_blocks,
metadata.load_gpu_blocks,
is_store=False,
event_idx=metadata.load_event,
events_list=self._load_events,
)
def wait_for_save(self) -> None:
pass
def get_finished(
self,
finished_req_ids: set[str],
) -> tuple[set[str] | None, set[str] | None]:
"""Submit transfers and report completed events to the scheduler.
"""Submit async stores.
Stores (GPU->CPU) read the live KV cache, which the compute stream may
still be writing under v1 overlapped execution, so they are ordered
after a compute-done event recorded on the current stream. Loads
(CPU->GPU) read stable pinned host memory and launch immediately. See
after a compute-done event recorded on the current stream. See
#45704 for the bug and #39306 for the srcAccessOrder rationale.
"""
metadata = self._connector_metadata
if metadata is not None and metadata.store_gpu_blocks:
backend = self._backend
assert backend is not None
if self._store_compute_done is None:
self._store_compute_done = torch.Event()
self._store_compute_done.record(torch.cuda.current_stream())
backend.launch_copy(
metadata.store_gpu_blocks,
metadata.store_cpu_blocks,
is_store=True,
event_idx=metadata.store_event,
events_list=self._store_events,
wait_event=self._store_compute_done,
)
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
"""Report completed transfer events to the scheduler.
Loads are submitted in start_load_kv() and stores in wait_for_save().
Returns:
tuple of (finished_sending, finished_recving).
- finished_sending: always None (stores use worker metadata).
- finished_recving: req_ids whose loads have completed.
"""
# (1) Submit transfers
metadata = self._connector_metadata
if metadata is not None:
backend = self._backend
assert backend is not None
if metadata.load_cpu_blocks:
backend.launch_copy(
metadata.load_cpu_blocks,
metadata.load_gpu_blocks,
is_store=False,
event_idx=metadata.load_event,
events_list=self._load_events,
)
if metadata.store_gpu_blocks:
if self._store_compute_done is None:
self._store_compute_done = torch.Event()
self._store_compute_done.record(torch.cuda.current_stream())
backend.launch_copy(
metadata.store_gpu_blocks,
metadata.store_cpu_blocks,
is_store=True,
event_idx=metadata.store_event,
events_list=self._store_events,
wait_event=self._store_compute_done,
)
# (2) Track completed transfer events
finished_recving: set[str] = set()
if self._pending_load_event_indices:
+14
View File
@@ -54,6 +54,7 @@ class ActiveKVConnector(KVConnector):
self.kv_connector.register_kv_caches(kv_caches_dict)
self.kv_connector.set_host_xfer_buffer_ops(copy_kv_blocks)
self._pending_load_start = False
self._disabled = False
def pre_forward(self, scheduler_output: "SchedulerOutput") -> None:
@@ -65,6 +66,16 @@ class ActiveKVConnector(KVConnector):
self.kv_connector.handle_preemptions(kv_connector_metadata)
self.kv_connector.bind_connector_metadata(kv_connector_metadata)
if scheduler_output.has_sync_kv_loads:
# Sync loads need to run before this step's forward.
self._start_load_kv()
else:
# Start any async loads in post-forward instead, keeping
# their host-side submission cost off the critical path.
self._pending_load_start = True
def _start_load_kv(self) -> None:
self._pending_load_start = False
# TODO: sort out KV Connectors' use of forward_context
if is_forward_context_available():
self.kv_connector.start_load_kv(get_forward_context())
@@ -78,6 +89,9 @@ class ActiveKVConnector(KVConnector):
if self._disabled:
return None
if self._pending_load_start:
self._start_load_kv()
output = KVConnectorOutput()
if wait_for_save:
self.kv_connector.wait_for_save()
@@ -79,14 +79,18 @@ class KVConnectorModelRunnerMixin:
assert scheduler_output.kv_connector_metadata is not None
kv_connector.bind_connector_metadata(scheduler_output.kv_connector_metadata)
# Background KV cache transfers happen here.
# These transfers are designed to be async and the requests
# involved may be disjoint from the running requests.
# Do this here to save a collective_rpc.
kv_connector.start_load_kv(get_forward_context())
# Start this step's KV loads, ordered after any in-flight KV block
# zeroing. Sync loads feed this step's forward so must precede it;
# otherwise start (async) loads after the forward launch, keeping
# their host-side submission cost off the critical path.
start_after_forward = not scheduler_output.has_sync_kv_loads
if not start_after_forward:
kv_connector.start_load_kv(get_forward_context())
try:
yield output
finally:
if start_after_forward:
kv_connector.start_load_kv(get_forward_context())
if wait_for_save and not defer_finalize:
kv_connector.wait_for_save()