[Bugfix][KVConnector] Finalize saves on steps without a forward (#57775)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Kimi Code <noreply@moonshot.ai>
This commit is contained in:
co-authored by
OpenAI Codex
Kimi Code
parent
7833ff25dd
commit
b3a87b2d04
@@ -31,7 +31,7 @@ The class provides the following primitives:
|
||||
wait_for_layer_load() - blocks until layer i load is done
|
||||
|
||||
save_kv_layer() - starts saving KV for layer i (maybe async)
|
||||
wait_for_save() - blocks until all saves are done
|
||||
wait_for_save() - submits saves and performs required synchronization
|
||||
|
||||
get_transfer_results() - returns async send/receive completions and
|
||||
receive failures in one snapshot.
|
||||
@@ -356,11 +356,14 @@ class KVConnectorBase_V1(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def wait_for_save(self):
|
||||
"""Block until all the save operations is done. This is called
|
||||
as the forward context exits to ensure that the async saving
|
||||
from save_kv_layer is complete before finishing the forward.
|
||||
"""Submit saves and perform required synchronization.
|
||||
|
||||
This prevents overwrites of paged KV buffer before saving done.
|
||||
Called once per step, including empty steps, after any target and draft
|
||||
forwards. Metadata remains bound during this call. Do nothing if there
|
||||
is no save work.
|
||||
|
||||
Async saves must retain their source blocks until completion is reported
|
||||
to the scheduler. Otherwise, wait for saves to finish before returning.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
@@ -142,6 +142,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
engine.current_layer = 0
|
||||
if hasattr(engine, "layerwise_retrievers"):
|
||||
engine.layerwise_retrievers = []
|
||||
engine.layerwise_storers = []
|
||||
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
|
||||
"""Start loading the KV cache from the connector to vLLM's paged
|
||||
|
||||
@@ -606,6 +606,7 @@ class LMCacheConnectorV1Impl:
|
||||
|
||||
self.async_loading = config.enable_async_loading
|
||||
self.layerwise_retrievers: list[Generator[torch.Tensor | None, None, None]] = []
|
||||
self.layerwise_storers: list[Generator[Any, None, None]] = []
|
||||
self._stats_monitor = LMCStatsMonitor.GetOrCreate()
|
||||
if role == KVConnectorRole.SCHEDULER:
|
||||
# Create lookup client using factory
|
||||
@@ -798,6 +799,7 @@ class LMCacheConnectorV1Impl:
|
||||
"""
|
||||
self.current_layer = 0
|
||||
self.layerwise_retrievers = []
|
||||
self.layerwise_storers = []
|
||||
|
||||
@_lmcache_nvtx_annotate
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
|
||||
@@ -131,19 +131,14 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
pass
|
||||
|
||||
def wait_for_save(self):
|
||||
# Store deferral is handled in get_finished(), which always runs even
|
||||
# when wait_for_save() is skipped (e.g. kv_connector_no_forward).
|
||||
pass
|
||||
assert self.connector_worker is not None
|
||||
assert isinstance(self._connector_metadata, OffloadingConnectorMetadata)
|
||||
# Defer store jobs to the next step's start_kv_transfers.
|
||||
self.connector_worker.prepare_store_kv(self._connector_metadata)
|
||||
|
||||
def get_finished(self, finished_req_ids: set[str]) -> tuple[set[str], set[str]]:
|
||||
assert self.connector_worker is not None
|
||||
assert isinstance(self._connector_metadata, OffloadingConnectorMetadata)
|
||||
|
||||
# Defer store jobs to the next step's start_kv_transfers. Done here
|
||||
# (rather than wait_for_save) so stores are queued even on steps where
|
||||
# wait_for_save is skipped.
|
||||
self.connector_worker.prepare_store_kv(self._connector_metadata)
|
||||
|
||||
return self.connector_worker.get_finished(finished_req_ids)
|
||||
|
||||
def build_connector_worker_meta(self) -> OffloadingWorkerMetadata | None:
|
||||
|
||||
Reference in New Issue
Block a user