[Bugfix][KV Offloading] Offload last block at request finish and prevent reuse race (#48596)

Signed-off-by: Alex <alex.tech.lab@outlook.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
AlexHuang
2026-07-17 16:50:49 +03:00
committed by GitHub
co-authored by Or Ozeri
parent d4b4562917
commit f38f3d11fb
3 changed files with 115 additions and 30 deletions
@@ -64,6 +64,47 @@ def test_scheduler_reports_allocation_failure(request_runner):
assert reduced[_ConnectorMetricName.ALLOCATION_FAILURE] == 1
@pytest.mark.parametrize("async_scheduling", [True, False])
@pytest.mark.parametrize("prompt_offset", [-1, -2])
def test_last_block_offloaded_at_request_finish(
request_runner, async_scheduling: bool, prompt_offset: int
):
"""EOS fills the last block at request finish — verify req_status is kept alive.
prompt = block_size + prompt_offset tokens → not a full block at schedule time,
so _build_store_jobs creates no store job. After EOS, request_finished
keeps req_status alive so _build_store_jobs can process it on the next step.
prompt_offset=-1: EOS fills the block → store job created on next step.
prompt_offset=-2: block remains partial → no store job, cleanup in
_build_store_jobs deletes req_status.
"""
block_size = 4
runner = request_runner(
block_size=block_size,
num_gpu_blocks=10,
async_scheduling=async_scheduling,
)
# prompt = block_size + prompt_offset tokens
runner.new_request(token_ids=[0] * (block_size + prompt_offset))
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(list(keys))
)
# Run with one step (EOS)
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
)
cs = runner.connector_scheduler
# Verify req_status is kept alive for _build_store_jobs to process
# regardless of whether there are storable blocks
assert "0" in cs._req_status, (
"req_status was deleted but should be kept alive "
"for _build_store_jobs to process finished_req_ids."
)
def test_scheduler_reports_lookup_sync_delay(request_runner):
runner = request_runner(
block_size=4,
@@ -560,11 +601,19 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo
# Group 1 (sliding window, window=2): only the last 2 blocks
# are within the window → loads blocks 1,2
expected_loaded=((0, 0), (0, 1), (0, 2), (1, 1), (1, 2)),
# The deferred store from the previous request's last block
# completes during this step, and its blocks are flushed because
# they were reallocated to the new request.
# Only block 1 (sliding window group) is stored — block 0's
# deferred store is flushed because it was reallocated.
expected_stored=((0, 1),),
expected_flushed=((0, 1),),
)
# one touch in get_num_new_matched_tokens x 2 groups
# 4 touch calls: 2 from get_num_new_matched_tokens (2 groups)
# + 2 from _get_reqs_to_store (2 groups)
touch_calls = runner.manager.touch.call_args_list
assert len(touch_calls) == 2
assert len(touch_calls) == 4
# full attention group touched all 3 blocks
assert len(touch_calls[0].args[0]) == 3
# sliding window group touched just the last 2 blocks
@@ -3,7 +3,7 @@
import time
from collections.abc import Iterable, Sequence
from dataclasses import dataclass, field
from itertools import islice
from itertools import chain, islice
from typing import Any, NamedTuple
from vllm.config import VllmConfig
@@ -444,6 +444,24 @@ class OffloadingConnectorScheduler:
if not pending:
del self._block_id_to_pending_jobs[bid]
def _calc_num_offloadable_tokens(
self, req_status: RequestOffloadState, num_computed_tokens: int
) -> int:
num = min(num_computed_tokens, req_status.req.num_tokens)
max_offload_tokens = req_status.max_offload_tokens
if max_offload_tokens is not None:
num = min(num, max_offload_tokens)
if self.config.offload_prompt_only:
num = min(num, req_status.req.num_prompt_tokens)
return num
def _maybe_cleanup_finished_req(
self, req_id: str, req_status: RequestOffloadState
) -> None:
"""Clean up req_status if finished and no in-flight jobs."""
if req_status.req.is_finished() and not req_status.transfer_jobs:
del self._req_status[req_id]
def _maximal_prefix_lookup(
self, keys: Iterable[OffloadKey], req_context: ReqContext
) -> int | None:
@@ -911,28 +929,24 @@ class OffloadingConnectorScheduler:
) -> dict[int, TransferJob]:
blocks_per_chunk = self.config.blocks_per_chunk
store_jobs: dict[int, TransferJob] = {}
for req_id in scheduler_output.num_scheduled_tokens:
for req_id in chain(
scheduler_output.num_scheduled_tokens,
scheduler_output.finished_req_ids or (),
):
req_status = self._req_status.get(req_id)
if req_status is None:
continue
req = req_status.req
num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id]
num_tokens_after_batch = req.num_computed_tokens + num_scheduled_tokens
# with async scheduling, some tokens may be missing
num_offloadable_tokens = min(num_tokens_after_batch, req.num_tokens)
max_offload_tokens = req_status.max_offload_tokens
if max_offload_tokens is not None:
num_offloadable_tokens = min(num_offloadable_tokens, max_offload_tokens)
if req.is_finished():
num_tokens_after_batch = req.num_tokens
else:
num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id]
num_tokens_after_batch = req.num_computed_tokens + num_scheduled_tokens
# Skip decode-phase chunks: clamp to the prompt length so only
# prefill chunks become eligible for store. next_stored_chunk_idx
# never advances past this boundary, so decode chunks are never
# queued in this or any later step.
if self.config.offload_prompt_only:
num_offloadable_tokens = min(
num_offloadable_tokens, req.num_prompt_tokens
)
num_offloadable_tokens = self._calc_num_offloadable_tokens(
req_status, num_tokens_after_batch
)
# Filter out chunks skipped due to sliding window attention / SSM
# or unreachable by the load path's alignment constraints.
@@ -983,6 +997,7 @@ class OffloadingConnectorScheduler:
if not new_offload_keys:
req_status.advance_stored_idx(num_offloadable_tokens)
self._maybe_cleanup_finished_req(req_id, req_status)
continue
store_output = self.manager.prepare_store(
@@ -993,10 +1008,12 @@ class OffloadingConnectorScheduler:
_ConnectorMetricName.ALLOCATION_FAILURE
)
logger.warning("Request %s: cannot store chunks", req_id)
self._maybe_cleanup_finished_req(req_id, req_status)
continue
if not store_output.keys_to_store:
req_status.advance_stored_idx(num_offloadable_tokens)
self._maybe_cleanup_finished_req(req_id, req_status)
continue
self._touch(req_status)
@@ -1093,6 +1110,13 @@ class OffloadingConnectorScheduler:
job_id,
)
if req.is_finished():
# Register non-sliding-window blocks for flush detection.
for bid in non_sliding_window_block_ids:
self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id)
if bid in self._current_batch_allocated_block_ids:
self._current_batch_jobs_to_flush.add(job_id)
return store_jobs
def build_connector_meta(
@@ -1255,8 +1279,6 @@ class OffloadingConnectorScheduler:
Optional KVTransferParams to be included in the request outputs
returned by the engine.
"""
# TODO(orozery): possibly kickoff offload for last block
# which may have been deferred due to async scheduling
req_status = self._req_status.get(request.request_id)
if req_status is None:
@@ -1269,20 +1291,20 @@ class OffloadingConnectorScheduler:
self.manager.on_request_finished(req_status.req_context)
self._maybe_observe_lookup_async_delay(req_status)
if not req_status.transfer_jobs:
# No in-flight jobs: no later complete_store()/complete_load() calls
# need this request's state.
del self._req_status[request.request_id]
return False, None
# In-flight jobs remain after the request stopped. Their completion may
# still call manager.complete_store()/complete_load(), so keep req_status.
# Pending stores outlive the request's block ownership; register them so
# future reuse of those blocks triggers a flush.
# Update offload keys with final block hash so _build_store_jobs can
# create store jobs for the last block(s) on the next schedule step.
req_status.update_offload_keys()
# Keep req_status alive: _build_store_jobs will process finished_req_ids
# on the next step and handle cleanup after creating store jobs.
# Register non_sliding_window_block_ids so future block reuse triggers
# a flush via _block_id_to_pending_jobs.
for job_id in req_status.transfer_jobs:
job_status = self._jobs[job_id]
for bid in job_status.non_sliding_window_block_ids or ():
self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id)
return False, None
def take_events(self) -> Iterable[KVCacheEvent]:
@@ -280,7 +280,21 @@ class OffloadingConnectorWorker:
def handle_preemptions(self, kv_connector_metadata: OffloadingConnectorMetadata):
assert self.worker is not None
# Pop jobs_to_flush from store_jobs into _unsubmitted_store_jobs
# so the existing submission loop below submits them before wait().
if kv_connector_metadata.jobs_to_flush:
for job_id in kv_connector_metadata.jobs_to_flush:
entry = kv_connector_metadata.store_jobs.pop(job_id, None)
if entry is not None:
assert isinstance(entry.src_spec, GPULoadStoreSpec)
self._unsubmitted_store_jobs.append(
(job_id, entry.src_spec, entry.dst_spec)
)
# Submit deferred stores from previous step (and jobs_to_flush above).
for job_id, src_spec, dst_spec in self._unsubmitted_store_jobs:
assert isinstance(src_spec, GPULoadStoreSpec)
success = self.worker.submit_store(job_id, src_spec, dst_spec)
assert success
self._unsubmitted_store_jobs.clear()