[Bugfix][KVConnector] Finalize saves on steps without a forward (#57775)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Kimi Code <noreply@moonshot.ai>
This commit is contained in:
Yifan Qiao
2026-09-25 00:06:36 +00:00
committed by GitHub
co-authored by OpenAI Codex Kimi Code
parent 7833ff25dd
commit b3a87b2d04
11 changed files with 56 additions and 37 deletions
@@ -31,7 +31,7 @@ The class provides the following primitives:
wait_for_layer_load() - blocks until layer i load is done
save_kv_layer() - starts saving KV for layer i (maybe async)
wait_for_save() - blocks until all saves are done
wait_for_save() - submits saves and performs required synchronization
get_transfer_results() - returns async send/receive completions and
receive failures in one snapshot.
@@ -356,11 +356,14 @@ class KVConnectorBase_V1(ABC):
@abstractmethod
def wait_for_save(self):
"""Block until all the save operations is done. This is called
as the forward context exits to ensure that the async saving
from save_kv_layer is complete before finishing the forward.
"""Submit saves and perform required synchronization.
This prevents overwrites of paged KV buffer before saving done.
Called once per step, including empty steps, after any target and draft
forwards. Metadata remains bound during this call. Do nothing if there
is no save work.
Async saves must retain their source blocks until completion is reported
to the scheduler. Otherwise, wait for saves to finish before returning.
"""
pass
@@ -142,6 +142,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
engine.current_layer = 0
if hasattr(engine, "layerwise_retrievers"):
engine.layerwise_retrievers = []
engine.layerwise_storers = []
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
"""Start loading the KV cache from the connector to vLLM's paged
@@ -606,6 +606,7 @@ class LMCacheConnectorV1Impl:
self.async_loading = config.enable_async_loading
self.layerwise_retrievers: list[Generator[torch.Tensor | None, None, None]] = []
self.layerwise_storers: list[Generator[Any, None, None]] = []
self._stats_monitor = LMCStatsMonitor.GetOrCreate()
if role == KVConnectorRole.SCHEDULER:
# Create lookup client using factory
@@ -798,6 +799,7 @@ class LMCacheConnectorV1Impl:
"""
self.current_layer = 0
self.layerwise_retrievers = []
self.layerwise_storers = []
@_lmcache_nvtx_annotate
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
@@ -131,19 +131,14 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
pass
def wait_for_save(self):
# Store deferral is handled in get_finished(), which always runs even
# when wait_for_save() is skipped (e.g. kv_connector_no_forward).
pass
assert self.connector_worker is not None
assert isinstance(self._connector_metadata, OffloadingConnectorMetadata)
# Defer store jobs to the next step's start_kv_transfers.
self.connector_worker.prepare_store_kv(self._connector_metadata)
def get_finished(self, finished_req_ids: set[str]) -> tuple[set[str], set[str]]:
assert self.connector_worker is not None
assert isinstance(self._connector_metadata, OffloadingConnectorMetadata)
# Defer store jobs to the next step's start_kv_transfers. Done here
# (rather than wait_for_save) so stores are queued even on steps where
# wait_for_save is skipped.
self.connector_worker.prepare_store_kv(self._connector_metadata)
return self.connector_worker.get_finished(finished_req_ids)
def build_connector_worker_meta(self) -> OffloadingWorkerMetadata | None: