[Bugfix][KVConnector] Finalize saves on steps without a forward (#57775)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Kimi Code <noreply@moonshot.ai>
This commit is contained in:
co-authored by
OpenAI Codex
Kimi Code
parent
7833ff25dd
commit
b3a87b2d04
@@ -517,6 +517,7 @@ class RequestRunner:
|
||||
if complete_transfers:
|
||||
self.offloading_spec.complete_transfers()
|
||||
|
||||
self.worker_connector.wait_for_save()
|
||||
finished_sending, finished_recving = self.worker_connector.get_finished(
|
||||
scheduler_output.finished_req_ids
|
||||
)
|
||||
|
||||
@@ -71,6 +71,7 @@ def test_kv_connector_mixin_clears_metadata():
|
||||
assert connector._connector_metadata is None
|
||||
# Test connector wrapper records method calls
|
||||
assert connector.call_record.get("bind_connector_metadata", 0) == 1
|
||||
assert connector.call_record.get("wait_for_save", 0) == 1
|
||||
assert connector.call_record.get("clear_connector_metadata", 0) == 1
|
||||
finally:
|
||||
# Ensure we clean up the global connector between tests
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterator
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_events import BlockStored
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.lmcache_connector import (
|
||||
LMCacheConnectorV1,
|
||||
LMCacheKVEvents,
|
||||
@@ -783,3 +785,32 @@ class TestIntegrationScenarios:
|
||||
assert aggregated_events[0].block_hashes == ["hash_common"]
|
||||
assert aggregated_events[0].parent_block_hash == "parent_common"
|
||||
assert aggregated_events[0].token_ids == [1, 2, 3]
|
||||
|
||||
|
||||
def test_legacy_layerwise_engine_does_not_reuse_storers_on_empty_steps():
|
||||
"""Empty steps must not advance generators exhausted by the last forward."""
|
||||
|
||||
class LegacyLayerwiseEngine:
|
||||
layerwise_storers: list[Iterator[None]]
|
||||
|
||||
def save_kv_layer(self, *args, **kwargs):
|
||||
self.layerwise_storers.append(iter([None]))
|
||||
|
||||
def wait_for_save(self):
|
||||
for storer in self.layerwise_storers:
|
||||
next(storer)
|
||||
|
||||
connector = LMCacheConnectorV1.__new__(LMCacheConnectorV1)
|
||||
connector._lmcache_engine = LegacyLayerwiseEngine()
|
||||
metadata = KVConnectorMetadata()
|
||||
connector.bind_connector_metadata(metadata)
|
||||
connector.save_kv_layer("layer", MagicMock(), MagicMock())
|
||||
connector.wait_for_save()
|
||||
connector.clear_connector_metadata()
|
||||
|
||||
for _ in range(2):
|
||||
connector.bind_connector_metadata(metadata)
|
||||
connector.wait_for_save()
|
||||
connector.clear_connector_metadata()
|
||||
|
||||
assert connector._lmcache_engine.layerwise_storers == []
|
||||
|
||||
@@ -96,4 +96,4 @@ def test_no_forward_starts_deferred_load_once(monkeypatch: pytest.MonkeyPatch):
|
||||
|
||||
connector.no_forward(_scheduler_output(False)) # type: ignore[arg-type]
|
||||
|
||||
assert events == ["handle", "bind", "start", "clear"]
|
||||
assert events == ["handle", "bind", "start", "wait", "clear"]
|
||||
|
||||
Reference in New Issue
Block a user