[Bugfix][KV Offloading] Offload last block at request finish and prevent reuse race (#48596)
Signed-off-by: Alex <alex.tech.lab@outlook.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -64,6 +64,47 @@ def test_scheduler_reports_allocation_failure(request_runner):
|
||||
assert reduced[_ConnectorMetricName.ALLOCATION_FAILURE] == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
@pytest.mark.parametrize("prompt_offset", [-1, -2])
|
||||
def test_last_block_offloaded_at_request_finish(
|
||||
request_runner, async_scheduling: bool, prompt_offset: int
|
||||
):
|
||||
"""EOS fills the last block at request finish — verify req_status is kept alive.
|
||||
|
||||
prompt = block_size + prompt_offset tokens → not a full block at schedule time,
|
||||
so _build_store_jobs creates no store job. After EOS, request_finished
|
||||
keeps req_status alive so _build_store_jobs can process it on the next step.
|
||||
|
||||
prompt_offset=-1: EOS fills the block → store job created on next step.
|
||||
prompt_offset=-2: block remains partial → no store job, cleanup in
|
||||
_build_store_jobs deletes req_status.
|
||||
"""
|
||||
block_size = 4
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=10,
|
||||
async_scheduling=async_scheduling,
|
||||
)
|
||||
# prompt = block_size + prompt_offset tokens
|
||||
runner.new_request(token_ids=[0] * (block_size + prompt_offset))
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(list(keys))
|
||||
)
|
||||
|
||||
# Run with one step (EOS)
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
)
|
||||
|
||||
cs = runner.connector_scheduler
|
||||
# Verify req_status is kept alive for _build_store_jobs to process
|
||||
# regardless of whether there are storable blocks
|
||||
assert "0" in cs._req_status, (
|
||||
"req_status was deleted but should be kept alive "
|
||||
"for _build_store_jobs to process finished_req_ids."
|
||||
)
|
||||
|
||||
|
||||
def test_scheduler_reports_lookup_sync_delay(request_runner):
|
||||
runner = request_runner(
|
||||
block_size=4,
|
||||
@@ -560,11 +601,19 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo
|
||||
# Group 1 (sliding window, window=2): only the last 2 blocks
|
||||
# are within the window → loads blocks 1,2
|
||||
expected_loaded=((0, 0), (0, 1), (0, 2), (1, 1), (1, 2)),
|
||||
# The deferred store from the previous request's last block
|
||||
# completes during this step, and its blocks are flushed because
|
||||
# they were reallocated to the new request.
|
||||
# Only block 1 (sliding window group) is stored — block 0's
|
||||
# deferred store is flushed because it was reallocated.
|
||||
expected_stored=((0, 1),),
|
||||
expected_flushed=((0, 1),),
|
||||
)
|
||||
|
||||
# one touch in get_num_new_matched_tokens x 2 groups
|
||||
# 4 touch calls: 2 from get_num_new_matched_tokens (2 groups)
|
||||
# + 2 from _get_reqs_to_store (2 groups)
|
||||
touch_calls = runner.manager.touch.call_args_list
|
||||
assert len(touch_calls) == 2
|
||||
assert len(touch_calls) == 4
|
||||
# full attention group touched all 3 blocks
|
||||
assert len(touch_calls[0].args[0]) == 3
|
||||
# sliding window group touched just the last 2 blocks
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import time
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from itertools import islice
|
||||
from itertools import chain, islice
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
@@ -444,6 +444,24 @@ class OffloadingConnectorScheduler:
|
||||
if not pending:
|
||||
del self._block_id_to_pending_jobs[bid]
|
||||
|
||||
def _calc_num_offloadable_tokens(
|
||||
self, req_status: RequestOffloadState, num_computed_tokens: int
|
||||
) -> int:
|
||||
num = min(num_computed_tokens, req_status.req.num_tokens)
|
||||
max_offload_tokens = req_status.max_offload_tokens
|
||||
if max_offload_tokens is not None:
|
||||
num = min(num, max_offload_tokens)
|
||||
if self.config.offload_prompt_only:
|
||||
num = min(num, req_status.req.num_prompt_tokens)
|
||||
return num
|
||||
|
||||
def _maybe_cleanup_finished_req(
|
||||
self, req_id: str, req_status: RequestOffloadState
|
||||
) -> None:
|
||||
"""Clean up req_status if finished and no in-flight jobs."""
|
||||
if req_status.req.is_finished() and not req_status.transfer_jobs:
|
||||
del self._req_status[req_id]
|
||||
|
||||
def _maximal_prefix_lookup(
|
||||
self, keys: Iterable[OffloadKey], req_context: ReqContext
|
||||
) -> int | None:
|
||||
@@ -911,28 +929,24 @@ class OffloadingConnectorScheduler:
|
||||
) -> dict[int, TransferJob]:
|
||||
blocks_per_chunk = self.config.blocks_per_chunk
|
||||
store_jobs: dict[int, TransferJob] = {}
|
||||
for req_id in scheduler_output.num_scheduled_tokens:
|
||||
for req_id in chain(
|
||||
scheduler_output.num_scheduled_tokens,
|
||||
scheduler_output.finished_req_ids or (),
|
||||
):
|
||||
req_status = self._req_status.get(req_id)
|
||||
if req_status is None:
|
||||
continue
|
||||
req = req_status.req
|
||||
|
||||
num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id]
|
||||
num_tokens_after_batch = req.num_computed_tokens + num_scheduled_tokens
|
||||
# with async scheduling, some tokens may be missing
|
||||
num_offloadable_tokens = min(num_tokens_after_batch, req.num_tokens)
|
||||
max_offload_tokens = req_status.max_offload_tokens
|
||||
if max_offload_tokens is not None:
|
||||
num_offloadable_tokens = min(num_offloadable_tokens, max_offload_tokens)
|
||||
if req.is_finished():
|
||||
num_tokens_after_batch = req.num_tokens
|
||||
else:
|
||||
num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id]
|
||||
num_tokens_after_batch = req.num_computed_tokens + num_scheduled_tokens
|
||||
|
||||
# Skip decode-phase chunks: clamp to the prompt length so only
|
||||
# prefill chunks become eligible for store. next_stored_chunk_idx
|
||||
# never advances past this boundary, so decode chunks are never
|
||||
# queued in this or any later step.
|
||||
if self.config.offload_prompt_only:
|
||||
num_offloadable_tokens = min(
|
||||
num_offloadable_tokens, req.num_prompt_tokens
|
||||
)
|
||||
num_offloadable_tokens = self._calc_num_offloadable_tokens(
|
||||
req_status, num_tokens_after_batch
|
||||
)
|
||||
|
||||
# Filter out chunks skipped due to sliding window attention / SSM
|
||||
# or unreachable by the load path's alignment constraints.
|
||||
@@ -983,6 +997,7 @@ class OffloadingConnectorScheduler:
|
||||
|
||||
if not new_offload_keys:
|
||||
req_status.advance_stored_idx(num_offloadable_tokens)
|
||||
self._maybe_cleanup_finished_req(req_id, req_status)
|
||||
continue
|
||||
|
||||
store_output = self.manager.prepare_store(
|
||||
@@ -993,10 +1008,12 @@ class OffloadingConnectorScheduler:
|
||||
_ConnectorMetricName.ALLOCATION_FAILURE
|
||||
)
|
||||
logger.warning("Request %s: cannot store chunks", req_id)
|
||||
self._maybe_cleanup_finished_req(req_id, req_status)
|
||||
continue
|
||||
|
||||
if not store_output.keys_to_store:
|
||||
req_status.advance_stored_idx(num_offloadable_tokens)
|
||||
self._maybe_cleanup_finished_req(req_id, req_status)
|
||||
continue
|
||||
|
||||
self._touch(req_status)
|
||||
@@ -1093,6 +1110,13 @@ class OffloadingConnectorScheduler:
|
||||
job_id,
|
||||
)
|
||||
|
||||
if req.is_finished():
|
||||
# Register non-sliding-window blocks for flush detection.
|
||||
for bid in non_sliding_window_block_ids:
|
||||
self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id)
|
||||
if bid in self._current_batch_allocated_block_ids:
|
||||
self._current_batch_jobs_to_flush.add(job_id)
|
||||
|
||||
return store_jobs
|
||||
|
||||
def build_connector_meta(
|
||||
@@ -1255,8 +1279,6 @@ class OffloadingConnectorScheduler:
|
||||
Optional KVTransferParams to be included in the request outputs
|
||||
returned by the engine.
|
||||
"""
|
||||
# TODO(orozery): possibly kickoff offload for last block
|
||||
# which may have been deferred due to async scheduling
|
||||
req_status = self._req_status.get(request.request_id)
|
||||
|
||||
if req_status is None:
|
||||
@@ -1269,20 +1291,20 @@ class OffloadingConnectorScheduler:
|
||||
|
||||
self.manager.on_request_finished(req_status.req_context)
|
||||
self._maybe_observe_lookup_async_delay(req_status)
|
||||
if not req_status.transfer_jobs:
|
||||
# No in-flight jobs: no later complete_store()/complete_load() calls
|
||||
# need this request's state.
|
||||
del self._req_status[request.request_id]
|
||||
return False, None
|
||||
|
||||
# In-flight jobs remain after the request stopped. Their completion may
|
||||
# still call manager.complete_store()/complete_load(), so keep req_status.
|
||||
# Pending stores outlive the request's block ownership; register them so
|
||||
# future reuse of those blocks triggers a flush.
|
||||
# Update offload keys with final block hash so _build_store_jobs can
|
||||
# create store jobs for the last block(s) on the next schedule step.
|
||||
req_status.update_offload_keys()
|
||||
|
||||
# Keep req_status alive: _build_store_jobs will process finished_req_ids
|
||||
# on the next step and handle cleanup after creating store jobs.
|
||||
# Register non_sliding_window_block_ids so future block reuse triggers
|
||||
# a flush via _block_id_to_pending_jobs.
|
||||
for job_id in req_status.transfer_jobs:
|
||||
job_status = self._jobs[job_id]
|
||||
for bid in job_status.non_sliding_window_block_ids or ():
|
||||
self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id)
|
||||
|
||||
return False, None
|
||||
|
||||
def take_events(self) -> Iterable[KVCacheEvent]:
|
||||
|
||||
@@ -280,7 +280,21 @@ class OffloadingConnectorWorker:
|
||||
|
||||
def handle_preemptions(self, kv_connector_metadata: OffloadingConnectorMetadata):
|
||||
assert self.worker is not None
|
||||
|
||||
# Pop jobs_to_flush from store_jobs into _unsubmitted_store_jobs
|
||||
# so the existing submission loop below submits them before wait().
|
||||
if kv_connector_metadata.jobs_to_flush:
|
||||
for job_id in kv_connector_metadata.jobs_to_flush:
|
||||
entry = kv_connector_metadata.store_jobs.pop(job_id, None)
|
||||
if entry is not None:
|
||||
assert isinstance(entry.src_spec, GPULoadStoreSpec)
|
||||
self._unsubmitted_store_jobs.append(
|
||||
(job_id, entry.src_spec, entry.dst_spec)
|
||||
)
|
||||
|
||||
# Submit deferred stores from previous step (and jobs_to_flush above).
|
||||
for job_id, src_spec, dst_spec in self._unsubmitted_store_jobs:
|
||||
assert isinstance(src_spec, GPULoadStoreSpec)
|
||||
success = self.worker.submit_store(job_id, src_spec, dst_spec)
|
||||
assert success
|
||||
self._unsubmitted_store_jobs.clear()
|
||||
|
||||
Reference in New Issue
Block a user