From 410f6da5c4bb62010728502035bee1b5f0eab2ac Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Sun, 13 Sep 2026 01:56:14 -0400 Subject: [PATCH] [Bugfix][KV Offload] Submit CPU stores on no-forward steps (#56621) Signed-off-by: Lucas Wilkinson --- tests/v1/simple_kv_offload/test_worker.py | 103 +++++++++++++++++++++- vllm/v1/simple_kv_offload/worker.py | 11 ++- 2 files changed, 110 insertions(+), 4 deletions(-) diff --git a/tests/v1/simple_kv_offload/test_worker.py b/tests/v1/simple_kv_offload/test_worker.py index a62dacc310..1739290051 100644 --- a/tests/v1/simple_kv_offload/test_worker.py +++ b/tests/v1/simple_kv_offload/test_worker.py @@ -10,6 +10,7 @@ read partially written / stale blocks and silently corrupt the CPU cache. from __future__ import annotations import time +from contextlib import nullcontext from unittest.mock import MagicMock import pytest @@ -21,7 +22,13 @@ if not current_platform.is_cuda_alike(): pytest.skip("Requires CUDA or ROCm", allow_module_level=True) from tests.v1.attention.utils import dense_kv_cache_tensor, dense_kv_cache_views +from tests.v1.kv_connector.unit.test_kv_connector_lifecycle import ( + _make_empty_scheduler_output, +) from vllm.config import CacheConfig +from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import ( + SimpleCPUOffloadConnector, +) from vllm.v1.core.kv_cache_utils import ( get_kv_cache_config_from_groups, is_kv_cache_spec_uniform, @@ -41,11 +48,12 @@ from vllm.v1.simple_kv_offload.cuda_mem_ops import ( CU_MEMCPY_SRC_ACCESS_ORDER_ANY, CU_MEMCPY_SRC_ACCESS_ORDER_STREAM, build_params, - pin_tensor, ) from vllm.v1.simple_kv_offload.disk_backend import DiskBackend from vllm.v1.simple_kv_offload.metadata import SimpleCPUOffloadMetadata from vllm.v1.simple_kv_offload.worker import SimpleCPUOffloadWorker +from vllm.v1.worker.gpu.kv_connector import ActiveKVConnector +from vllm.v1.worker.kv_connector_model_runner_mixin import KVConnectorModelRunnerMixin from vllm.v1.worker.utils import allocate_kv_cache NUM_BLOCKS = 64 @@ -58,8 +66,11 @@ SLEEP_CYCLES = 50_000_000 def _make_backend() -> tuple[DmaCopyBackend, torch.Tensor, torch.Tensor]: gpu = {"k": torch.zeros((NUM_BLOCKS, BLOCK_BYTES), dtype=torch.int8, device="cuda")} - cpu = {"k": torch.zeros((NUM_BLOCKS, BLOCK_BYTES), dtype=torch.int8, device="cpu")} - pin_tensor(cpu["k"]) + cpu = { + "k": torch.zeros( + (NUM_BLOCKS, BLOCK_BYTES), dtype=torch.int8, device="cpu", pin_memory=True + ) + } low_pri, _ = torch.cuda.Stream.priority_range() backend = DmaCopyBackend() backend.init( @@ -72,6 +83,92 @@ def _make_backend() -> tuple[DmaCopyBackend, torch.Tensor, torch.Tensor]: return backend, gpu["k"], cpu["k"] +@pytest.mark.parametrize("use_v2", [False, True]) +def test_no_forward_step_completes_cpu_store(monkeypatch, use_v2): + """A finished request's final store must drain without another model step.""" + backend, gpu, cpu = _make_backend() + worker = SimpleCPUOffloadWorker(None, None, cpu_capacity_bytes=0) + worker._backend = backend + connector = SimpleCPUOffloadConnector.__new__(SimpleCPUOffloadConnector) + connector.worker_handler = worker + output = _make_empty_scheduler_output() + output.kv_connector_metadata = SimpleCPUOffloadMetadata( + store_event=0, store_gpu_blocks=[0], store_cpu_blocks=[0] + ) + module = "vllm.v1.worker.kv_connector_model_runner_mixin" + monkeypatch.setattr(f"{module}.get_kv_transfer_group", lambda: connector) + monkeypatch.setattr(f"{module}.set_forward_context", lambda *a: nullcontext()) + monkeypatch.setattr(f"{module}.get_forward_context", lambda: None) + try: + gpu.fill_(91) + if use_v2: + active = ActiveKVConnector.__new__(ActiveKVConnector) + active.kv_connector = connector + active._disabled = False + active._pending_load_start = False + module = "vllm.v1.worker.gpu.kv_connector" + monkeypatch.setattr(f"{module}.is_forward_context_available", lambda: True) + monkeypatch.setattr(f"{module}.get_forward_context", lambda: None) + result = active.no_forward(output) + else: + result = KVConnectorModelRunnerMixin.kv_connector_no_forward(output, None) + completion = ( + result.kv_connector_output.kv_connector_worker_meta + if result.kv_connector_output is not None + else None + ) + deadline = time.monotonic() + 5 + while completion is None and time.monotonic() < deadline: + connector.get_finished(set()) + completion = connector.build_connector_worker_meta() + time.sleep(0.001) + assert completion is not None, "CPU store never completed on the empty step" + assert completion.completed_store_events == {0: 1} + assert torch.equal(cpu[0], gpu[0].cpu()) + finally: + backend.shutdown() + + +def test_deferred_store_waits_for_draft_forward(monkeypatch): + """Target completion must not submit the store before draft finalization.""" + backend, gpu, cpu = _make_backend() + worker = SimpleCPUOffloadWorker(None, None, cpu_capacity_bytes=0) + worker._backend = backend + connector = SimpleCPUOffloadConnector.__new__(SimpleCPUOffloadConnector) + connector.worker_handler = worker + output = _make_empty_scheduler_output() + output.kv_connector_metadata = SimpleCPUOffloadMetadata( + store_event=0, store_gpu_blocks=[0], store_cpu_blocks=[0] + ) + module = "vllm.v1.worker.kv_connector_model_runner_mixin" + monkeypatch.setattr(f"{module}.get_kv_transfer_group", lambda: connector) + monkeypatch.setattr(f"{module}.has_kv_transfer_group", lambda: True) + monkeypatch.setattr(f"{module}.get_forward_context", lambda: None) + launch = MagicMock(wraps=backend.launch_copy) + monkeypatch.setattr(backend, "launch_copy", launch) + try: + with KVConnectorModelRunnerMixin._get_kv_connector_output( + output, defer_finalize=True + ): + gpu.fill_(17) + launch.assert_not_called() + gpu.fill_(91) + KVConnectorModelRunnerMixin.finalize_kv_connector() + assert launch.call_count == 1 + completion = None + deadline = time.monotonic() + 5 + while completion is None and time.monotonic() < deadline: + connector.get_finished(set()) + completion = connector.build_connector_worker_meta() + time.sleep(0.001) + assert completion is not None + assert completion.completed_store_events == {0: 1} + assert torch.equal(cpu[0], gpu[0].cpu()) + assert launch.call_count == 1 + finally: + backend.shutdown() + + def _drive_store( backend: DmaCopyBackend, gpu: torch.Tensor, diff --git a/vllm/v1/simple_kv_offload/worker.py b/vllm/v1/simple_kv_offload/worker.py index 9efc544f2f..fc950af40f 100644 --- a/vllm/v1/simple_kv_offload/worker.py +++ b/vllm/v1/simple_kv_offload/worker.py @@ -67,6 +67,7 @@ class SimpleCPUOffloadWorker: # Metadata for the current step self._connector_metadata: SimpleCPUOffloadMetadata | None = None + self._store_submitted = False # Compute-done event recorded before each store; reused across steps # (get_finished runs once per step, copy queue is FIFO). @@ -244,12 +245,15 @@ class SimpleCPUOffloadWorker: def bind_connector_metadata(self, metadata: SimpleCPUOffloadMetadata) -> None: self._connector_metadata = metadata + self._store_submitted = False if metadata.load_event >= 0: self._pending_load_event_indices.add(metadata.load_event) if metadata.store_event >= 0: self._pending_store_event_indices.add(metadata.store_event) def clear_connector_metadata(self) -> None: + # No-forward steps skip the normal wait_for_save hook. + self.wait_for_save() self._connector_metadata = None def start_load_kv(self) -> None: @@ -278,7 +282,11 @@ class SimpleCPUOffloadWorker: #45704 for the bug and #39306 for the srcAccessOrder rationale. """ metadata = self._connector_metadata - if metadata is not None and metadata.store_gpu_blocks: + if ( + metadata is not None + and metadata.store_gpu_blocks + and not self._store_submitted + ): backend = self._backend assert backend is not None if self._store_compute_done is None: @@ -292,6 +300,7 @@ class SimpleCPUOffloadWorker: events_list=self._store_events, wait_event=self._store_compute_done, ) + self._store_submitted = True def get_finished( self, finished_req_ids: set[str]