From 52c3c4a42fd13b62ba985b9ceb9b9969964ee83e Mon Sep 17 00:00:00 2001 From: MINJUN GIL Date: Tue, 28 Jul 2026 13:10:52 +0900 Subject: [PATCH] [Bugfix][KV Offload][OBJ] Preserve job completion during cleanup (#49947) Signed-off-by: MINJUN GIL Co-authored-by: OpenAI Codex --- tests/v1/kv_offload/tiering/test_obj_tier.py | 111 ++++++++++++++++++- vllm/v1/kv_offload/tiering/obj/manager.py | 29 ++++- 2 files changed, 133 insertions(+), 7 deletions(-) diff --git a/tests/v1/kv_offload/tiering/test_obj_tier.py b/tests/v1/kv_offload/tiering/test_obj_tier.py index 82ba183ea5..661438dce6 100644 --- a/tests/v1/kv_offload/tiering/test_obj_tier.py +++ b/tests/v1/kv_offload/tiering/test_obj_tier.py @@ -35,6 +35,10 @@ from vllm.v1.kv_offload.config import ( OffloadingParallelConfig, ) from vllm.v1.kv_offload.tiering.base import JobMetadata, JobResult +from vllm.v1.kv_offload.tiering.manager import ( + CPUPrimaryTierOffloadingManager, + TieringOffloadingManager, +) from vllm.v1.kv_offload.tiering.obj.config import ObjStoreConfig from vllm.v1.kv_offload.tiering.obj.manager import ObjectStoreSecondaryTierManager @@ -220,12 +224,14 @@ def _make_events_spec(enable_kv_cache_events: bool) -> SimpleNamespace: def _make_tier( num_blocks: int = 4, offloading_spec: SimpleNamespace = _OFFLOADING_SPEC, + primary_kv_view: memoryview | None = None, **tier_kwargs, ) -> tuple[ObjectStoreSecondaryTierManager, MockNixlAgent]: """Create a tier backed by a fresh MockNixlAgent.""" mock_agent = MockNixlAgent() - tensor = torch.zeros((num_blocks, _BLOCK_ELEMENTS), dtype=_DTYPE) - view = memoryview(tensor.numpy()) + if primary_kv_view is None: + tensor = torch.zeros((num_blocks, _BLOCK_ELEMENTS), dtype=_DTYPE) + primary_kv_view = memoryview(tensor.numpy()) with ( patch("vllm.v1.kv_offload.tiering.obj.manager.nixl_agent_config"), patch( @@ -235,7 +241,7 @@ def _make_tier( ): tier = ObjectStoreSecondaryTierManager( offloading_spec=offloading_spec, - primary_kv_view=view, + primary_kv_view=primary_kv_view, tier_type="obj", store_config=_STORE_CONFIG, prefix=_RUN_PREFIX, @@ -449,6 +455,105 @@ class TestMockObjTierFailures: assert not by_id[1].success assert by_id[2].success + def test_release_xfer_failure_retries_without_losing_result(self, monkeypatch): + tier, agent = _make_tier(num_blocks=4) + agent.check_xfer_state = MagicMock(side_effect=RuntimeError("poll failed")) + release_xfer = MagicMock( + side_effect=[RuntimeError("transfer is still active"), None] + ) + monkeypatch.setattr(agent, "release_xfer_handle", release_xfer) + + tier.submit_store(make_job(1, [key(1)], [0])) + + # The transfer handle could not be released safely, so the job must + # remain tracked and must not be finalized yet. + assert list(tier.get_finished_jobs()) == [] + assert 1 in tier._transfers + + # Cleanup is retried without polling again or changing the failure + # verdict. The completion is then returned exactly once. + results = list(tier.get_finished_jobs()) + assert len(results) == 1 + assert results[0].job_id == 1 + assert not results[0].success + assert agent.check_xfer_state.call_count == 2 + assert release_xfer.call_count == 2 + assert not tier._transfers + assert list(tier.get_finished_jobs()) == [] + + @pytest.mark.parametrize( + "cleanup_method", ["release_dlist_handle", "deregister_memory"] + ) + def test_post_transfer_cleanup_failure_does_not_lose_result( + self, monkeypatch, cleanup_method + ): + tier, agent = _make_tier(num_blocks=4) + monkeypatch.setattr( + agent, + cleanup_method, + MagicMock(side_effect=RuntimeError("cleanup failed")), + ) + + tier.submit_store(make_job(1, [key(1)], [0])) + results = list(tier.get_finished_jobs()) + + assert len(results) == 1 + assert results[0].job_id == 1 + assert results[0].success + assert not tier._transfers + assert list(tier.get_finished_jobs()) == [] + + def test_xfer_cleanup_retry_finalizes_parent_job_and_primary_pin(self, monkeypatch): + num_blocks = 4 + tensor = torch.zeros((num_blocks, _BLOCK_ELEMENTS), dtype=_DTYPE) + primary_kv_view = memoryview(tensor.numpy()) + mmap_region = MagicMock() + mmap_region.create_kv_memoryview.return_value = primary_kv_view + primary_tier = CPUPrimaryTierOffloadingManager( + num_blocks=num_blocks, mmap_region=mmap_region + ) + obj_tier, agent = _make_tier( + num_blocks=num_blocks, primary_kv_view=primary_kv_view + ) + manager = TieringOffloadingManager( + primary_tier=primary_tier, secondary_tiers=[obj_tier] + ) + + keys = [key(1)] + primary_result = primary_tier.prepare_store(keys, _CTX) + assert primary_result is not None + primary_tier.complete_store(keys, _CTX, success=True) + job = manager.create_store_job(keys, _CTX) + obj_tier.submit_store(job) + + block = primary_tier._policy.get(keys[0]) + assert block is not None + assert block.ref_cnt == 1 + assert len(manager._transfer_jobs) == 1 + + agent.check_xfer_state = MagicMock(side_effect=RuntimeError("poll failed")) + release_xfer = MagicMock( + side_effect=[RuntimeError("transfer is still active"), None] + ) + monkeypatch.setattr(agent, "release_xfer_handle", release_xfer) + schedule_context = ScheduleEndContext(new_req_ids=[], preempted_req_ids=()) + + manager.on_schedule_end(schedule_context) + + assert len(manager._transfer_jobs) == 1 + assert block.ref_cnt == 1 + assert len(obj_tier._transfers) == 1 + assert manager.has_pending_work() + + manager.on_schedule_end(schedule_context) + + assert manager._transfer_jobs == {} + assert block.ref_cnt == 0 + assert obj_tier._transfers == {} + assert not manager.has_pending_work() + assert agent.check_xfer_state.call_count == 2 + assert release_xfer.call_count == 2 + class TestMockObjTierShutdown: def test_shutdown_clears_in_flight_transfers(self): diff --git a/vllm/v1/kv_offload/tiering/obj/manager.py b/vllm/v1/kv_offload/tiering/obj/manager.py index c7e0d4c4be..2dfc2d30fa 100644 --- a/vllm/v1/kv_offload/tiering/obj/manager.py +++ b/vllm/v1/kv_offload/tiering/obj/manager.py @@ -307,15 +307,36 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager): else: if state == NIXL_PROC: continue - elif state == NIXL_DONE: + if state == NIXL_DONE: success = True else: success = False logger.warning("transfer failed job=%d state=%s", job_id, state) + + try: + self._agent.release_xfer_handle(entry.xfer_handle) + except Exception as exc: + # Keep the entry until NIXL confirms that the transfer handle + # can be released. The transfer may still access primary-tier + # memory, so publishing its result would allow unsafe reuse. + logger.warning("release_xfer_handle failed for job %d: %s", job_id, exc) + continue + + # Once the transfer handle is released, these remaining cleanup + # failures must not suppress the job completion. They can leak + # NIXL metadata, but cannot leave an active data transfer behind. + try: + self._agent.release_dlist_handle(entry.obj_handle) + except Exception as exc: + logger.warning( + "release_dlist_handle failed for job %d: %s", job_id, exc + ) + try: + self._agent.deregister_memory(entry.files_desc) + except Exception as exc: + logger.warning("deregister_memory failed for job %d: %s", job_id, exc) + del self._transfers[job_id] - self._agent.release_xfer_handle(entry.xfer_handle) - self._agent.release_dlist_handle(entry.obj_handle) - self._agent.deregister_memory(entry.files_desc) self._pending_results.append(JobResult(job_id=job_id, success=success)) def get_finished_jobs(self) -> Iterable[JobResult]: