[Bugfix][KVConnector] Finalize saves on steps without a forward (#57775)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Kimi Code <noreply@moonshot.ai>
This commit is contained in:
Yifan Qiao
2026-09-25 00:06:36 +00:00
committed by GitHub
co-authored by OpenAI Codex Kimi Code
parent 7833ff25dd
commit b3a87b2d04
11 changed files with 56 additions and 37 deletions
@@ -517,6 +517,7 @@ class RequestRunner:
if complete_transfers:
self.offloading_spec.complete_transfers()
self.worker_connector.wait_for_save()
finished_sending, finished_recving = self.worker_connector.get_finished(
scheduler_output.finished_req_ids
)
@@ -71,6 +71,7 @@ def test_kv_connector_mixin_clears_metadata():
assert connector._connector_metadata is None
# Test connector wrapper records method calls
assert connector.call_record.get("bind_connector_metadata", 0) == 1
assert connector.call_record.get("wait_for_save", 0) == 1
assert connector.call_record.get("clear_connector_metadata", 0) == 1
finally:
# Ensure we clean up the global connector between tests
@@ -1,10 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterator
from unittest.mock import MagicMock
import pytest
from vllm.distributed.kv_events import BlockStored
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
from vllm.distributed.kv_transfer.kv_connector.v1.lmcache_connector import (
LMCacheConnectorV1,
LMCacheKVEvents,
@@ -783,3 +785,32 @@ class TestIntegrationScenarios:
assert aggregated_events[0].block_hashes == ["hash_common"]
assert aggregated_events[0].parent_block_hash == "parent_common"
assert aggregated_events[0].token_ids == [1, 2, 3]
def test_legacy_layerwise_engine_does_not_reuse_storers_on_empty_steps():
"""Empty steps must not advance generators exhausted by the last forward."""
class LegacyLayerwiseEngine:
layerwise_storers: list[Iterator[None]]
def save_kv_layer(self, *args, **kwargs):
self.layerwise_storers.append(iter([None]))
def wait_for_save(self):
for storer in self.layerwise_storers:
next(storer)
connector = LMCacheConnectorV1.__new__(LMCacheConnectorV1)
connector._lmcache_engine = LegacyLayerwiseEngine()
metadata = KVConnectorMetadata()
connector.bind_connector_metadata(metadata)
connector.save_kv_layer("layer", MagicMock(), MagicMock())
connector.wait_for_save()
connector.clear_connector_metadata()
for _ in range(2):
connector.bind_connector_metadata(metadata)
connector.wait_for_save()
connector.clear_connector_metadata()
assert connector._lmcache_engine.layerwise_storers == []
+1 -1
View File
@@ -96,4 +96,4 @@ def test_no_forward_starts_deferred_load_once(monkeypatch: pytest.MonkeyPatch):
connector.no_forward(_scheduler_output(False)) # type: ignore[arg-type]
assert events == ["handle", "bind", "start", "clear"]
assert events == ["handle", "bind", "start", "wait", "clear"]