[Bugfix][KV Offloading] Fix eager SimpleCPUOffload cache registration and final flush (#53532)

Signed-off-by: maithilijoshi20 <maithilijoshi2003@gmail.com>
Signed-off-by: maithilijoshi20 <97733343+maithilijoshi20@users.noreply.github.com>
Co-authored-by: maithilijoshi20 <maithilijoshi2003@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
maithilijoshi20
2026-09-02 03:23:09 +00:00
committed by GitHub
co-authored by maithilijoshi20 OpenAI Codex Yifan Qiao
parent dbf1a044ea
commit 12b9573c98
3 changed files with 354 additions and 94 deletions
+248 -6
View File
@@ -18,6 +18,9 @@ from vllm.config import (
SchedulerConfig,
VllmConfig,
)
from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import (
SimpleCPUOffloadConnector,
)
from vllm.utils.hashing import sha256
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
@@ -39,12 +42,16 @@ from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
KVCacheTensor,
SlidingWindowSpec,
)
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request
from vllm.v1.simple_kv_offload.manager import SimpleCPUOffloadScheduler
from vllm.v1.simple_kv_offload.metadata import SimpleCPUOffloadWorkerMetadata
pytestmark = pytest.mark.skip_global_cleanup
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
@@ -77,15 +84,26 @@ def _make_kv_cache_config(
) # Ensure specs are registered for tests
for g in range(num_groups):
layer_names = [f"layer_{g}"]
spec = (
FullAttentionSpec(
block_size=BLOCK_SIZE,
num_kv_heads=NUM_KV_HEADS,
head_size=HEAD_SIZE,
dtype=DTYPE,
)
if g == 0
else SlidingWindowSpec(
block_size=BLOCK_SIZE,
num_kv_heads=NUM_KV_HEADS,
head_size=HEAD_SIZE,
dtype=DTYPE,
sliding_window=BLOCK_SIZE * 4,
)
)
groups.append(
KVCacheGroupSpec(
layer_names,
FullAttentionSpec(
block_size=BLOCK_SIZE,
num_kv_heads=NUM_KV_HEADS,
head_size=HEAD_SIZE,
dtype=DTYPE,
),
spec,
)
)
tensors.append(
@@ -412,6 +430,230 @@ def test_eager_store_and_load_roundtrip() -> None:
assert len(meta2.load_cpu_blocks) == len(meta2.load_gpu_blocks)
def test_eager_store_preserves_secondary_block_hashes() -> None:
"""CPU copies retain fine-grained hashes owned by the GPU block."""
fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False)
sched = fix.scheduler
req = make_request(num_blocks=1)
kv_blocks = _alloc_and_register(fix, req, num_blocks=1)
gpu_block = kv_blocks.blocks[0][0]
primary_hash = gpu_block.block_hash
primary_num_tokens = gpu_block.block_hash_num_tokens
assert primary_hash is not None
fine_grained_req = Request(
request_id="req-fine-grained-hash",
prompt_token_ids=req.prompt_token_ids,
sampling_params=req.sampling_params,
pooling_params=None,
mm_features=None,
block_hasher=get_request_block_hasher(BLOCK_SIZE // 2, sha256),
)
secondary_hash = make_block_hash_with_group_id(fine_grained_req.block_hashes[0], 0)
fix.gpu_block_pool._insert_block_hash(
secondary_hash, gpu_block, num_tokens=BLOCK_SIZE // 2
)
assert (
secondary_hash
in fix.gpu_block_pool.cached_block_hashes_by_block[gpu_block.block_id]
)
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
block_ids = kv_blocks.get_block_ids()
meta = sched.build_connector_meta(
make_scheduler_output(
{req.request_id: BLOCK_SIZE},
new_reqs={req.request_id: block_ids},
)
)
simulate_store_completion(sched, meta.store_event)
cpu_block = sched.cpu_block_pool.cached_block_hash_to_block.get_one_block(
secondary_hash
)
assert cpu_block is not None
assert cpu_block.block_hash == primary_hash
assert cpu_block.block_hash_num_tokens == primary_num_tokens
assert (
secondary_hash
in sched.cpu_block_pool.cached_block_hashes_by_block[cpu_block.block_id]
)
# ---------------------------------------------------------------------------
# Test 1b: Finished eager stores
# ---------------------------------------------------------------------------
def test_finished_eager_stores_are_batched() -> None:
"""Finished requests in one step retain, complete, and cache every store."""
fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False)
sched = fix.scheduler
requests = [make_request(num_blocks=2) for _ in range(2)]
block_ids = []
for request in requests:
kv_blocks = _alloc_and_register(fix, request, num_blocks=2)
sched.update_state_after_alloc(request, kv_blocks, num_external_tokens=0)
block_ids.append(kv_blocks.get_block_ids())
for request, request_block_ids in zip(requests, block_ids):
sched.request_finished_all_groups(request, request_block_ids)
meta = sched.build_connector_meta(make_scheduler_output({}))
assert len(meta.store_gpu_blocks) == 4
assert len(meta.store_cpu_blocks) == 4
simulate_store_completion(sched, meta.store_event)
for request in requests:
for block_hash in request.block_hashes[:2]:
cached = sched.cpu_block_pool.cached_block_hash_to_block.get_one_block(
make_block_hash_with_group_id(block_hash, 0)
)
assert cached is not None
request = requests[0]
cached_request = Request(
request_id="req-finished-eager-load",
prompt_token_ids=request.prompt_token_ids,
sampling_params=request.sampling_params,
pooling_params=None,
mm_features=None,
block_hasher=request._block_hasher,
)
hit_tokens, is_async = sched.get_num_new_matched_tokens(
cached_request, num_computed_tokens=0
)
assert hit_tokens == 2 * BLOCK_SIZE
assert is_async is True
cached_gpu_blocks = fix.gpu_block_pool.get_new_blocks(2)
sched.update_state_after_alloc(
cached_request,
KVCacheBlocks(blocks=(cached_gpu_blocks,)),
num_external_tokens=hit_tokens,
)
load_meta = sched.build_connector_meta(
make_scheduler_output(
{cached_request.request_id: 1},
new_reqs={
cached_request.request_id: (
[block.block_id for block in cached_gpu_blocks],
)
},
)
)
assert load_meta.load_event >= 0
assert len(load_meta.load_gpu_blocks) == 2
assert len(load_meta.load_cpu_blocks) == 2
def test_finished_eager_store_skips_existing_cpu_cache_entries() -> None:
"""A final flush must not consume CPU capacity for an existing hash."""
fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False)
sched = fix.scheduler
first = make_request(num_blocks=2)
first_blocks = _alloc_and_register(fix, first, num_blocks=2)
sched.update_state_after_alloc(first, first_blocks, num_external_tokens=0)
sched.request_finished_all_groups(first, first_blocks.get_block_ids())
first_meta = sched.build_connector_meta(make_scheduler_output({}))
simulate_store_completion(sched, first_meta.store_event)
free_blocks = get_cpu_free_blocks(sched)
duplicate = Request(
request_id="req-finished-eager-duplicate",
prompt_token_ids=first.prompt_token_ids,
sampling_params=first.sampling_params,
pooling_params=None,
mm_features=None,
block_hasher=first._block_hasher,
)
duplicate_blocks = _alloc_and_register(fix, duplicate, num_blocks=2)
sched.update_state_after_alloc(duplicate, duplicate_blocks, num_external_tokens=0)
sched.request_finished_all_groups(duplicate, duplicate_blocks.get_block_ids())
duplicate_meta = sched.build_connector_meta(make_scheduler_output({}))
assert duplicate_meta.store_event == -1
assert get_cpu_free_blocks(sched) == free_blocks
def test_finished_eager_store_is_reported_pending_before_metadata_build() -> None:
"""Queued finished stores must keep the connector pending until submitted."""
fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False)
sched = fix.scheduler
request = make_request(num_blocks=2)
kv_blocks = _alloc_and_register(fix, request, num_blocks=2)
connector = SimpleCPUOffloadConnector.__new__(SimpleCPUOffloadConnector)
connector.scheduler_manager = sched
sched.update_state_after_alloc(request, kv_blocks, num_external_tokens=0)
sched.request_finished_all_groups(request, kv_blocks.get_block_ids())
assert sched.has_pending_stores()
assert connector.has_pending_push_work()
meta = sched.build_connector_meta(make_scheduler_output({}))
assert meta.store_event >= 0
assert not sched._pending_finished_stores
assert connector.has_pending_push_work()
simulate_store_completion(sched, meta.store_event)
assert not sched.has_pending_stores()
assert not connector.has_pending_push_work()
def test_finished_eager_store_caches_all_groups() -> None:
"""The final flush retains completed blocks from every KV cache group."""
fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, num_groups=2, lazy=False)
sched = fix.scheduler
request = make_request(num_blocks=2)
group_blocks = tuple(
_allocate_gpu_blocks(fix.gpu_block_pool, request, 2, group_id=group_id)
for group_id in range(2)
)
request.num_computed_tokens = 2 * BLOCK_SIZE
kv_blocks = KVCacheBlocks(blocks=group_blocks)
sched.update_state_after_alloc(request, kv_blocks, num_external_tokens=0)
sched.request_finished_all_groups(request, kv_blocks.get_block_ids())
meta = sched.build_connector_meta(make_scheduler_output({}))
assert len(meta.store_gpu_blocks) == 4
simulate_store_completion(sched, meta.store_event)
for group_id in range(2):
for block_hash in request.block_hashes[:2]:
cached = sched.cpu_block_pool.cached_block_hash_to_block.get_one_block(
make_block_hash_with_group_id(block_hash, group_id)
)
assert cached is not None
def test_finished_eager_store_respects_cpu_capacity() -> None:
"""A finished request skips excess blocks when the CPU cache is full."""
fix = make_scheduler(num_cpu_blocks=3, num_gpu_blocks=16, lazy=False)
sched = fix.scheduler
request = make_request(num_blocks=3)
kv_blocks = _alloc_and_register(fix, request, num_blocks=3)
sched.update_state_after_alloc(request, kv_blocks, num_external_tokens=0)
available = get_cpu_free_blocks(sched)
sched.request_finished_all_groups(request, kv_blocks.get_block_ids())
meta = sched.build_connector_meta(make_scheduler_output({}))
assert len(meta.store_gpu_blocks) == available
assert len(meta.store_cpu_blocks) == available
def test_reset_releases_unsubmitted_finished_stores() -> None:
"""Reset releases finished-store refs that were not sent to a worker."""
fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False)
sched = fix.scheduler
request = make_request(num_blocks=2)
kv_blocks = _alloc_and_register(fix, request, num_blocks=2)
sched.update_state_after_alloc(request, kv_blocks, num_external_tokens=0)
sched.request_finished_all_groups(request, kv_blocks.get_block_ids())
assert sched.reset()
# ---------------------------------------------------------------------------
# Test 1b: Boundary — max_hit_len cap drops the last full block when the
# prompt is an exact multiple of BLOCK_SIZE.
@@ -291,8 +291,7 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
)
return False, None
# NOTE: New API only for SimpleCPUOffloadConnector.
def has_pending_transfers(self) -> bool:
def has_pending_push_work(self) -> bool:
if self.scheduler_manager is not None:
return self.scheduler_manager.has_pending_stores()
return False
+105 -86
View File
@@ -174,6 +174,7 @@ class SimpleCPUOffloadScheduler:
self._reqs_to_store: dict[str, StoreRequestState] = {}
self._store_event_to_reqs: dict[int, list[str]] = {}
self._in_flight_store_gpu_blocks: set[int] = set()
self._pending_finished_stores: list[TransferMeta] = []
self._abandoned_reqs_to_load: dict[str, LoadRequestState] = {}
# Event counters
@@ -414,6 +415,11 @@ class SimpleCPUOffloadScheduler:
# --- Stores ---
store_event = -1
store_gpu, store_cpu, store_req_ids = self.prepare_store_specs(scheduler_output)
for transfer in self._pending_finished_stores:
store_gpu.extend(transfer.gpu_block_ids)
store_cpu.extend(transfer.cpu_block_ids)
self._pending_finished_stores.clear()
if store_gpu:
store_event = self._store_event_counter
self._store_event_counter += 1
@@ -492,7 +498,6 @@ class SimpleCPUOffloadScheduler:
self._cursor = None
gpu_ids: list[int] = []
block_hashes: list[bytes] = []
last_visited = self._cursor
for covered, node in enumerate(free_queue.iter_blocks_after(self._cursor)):
@@ -508,16 +513,13 @@ class SimpleCPUOffloadScheduler:
and cpu_pool.cached_block_hash_to_block.get_one_block(bhash) is None
):
gpu_ids.append(node.block_id)
block_hashes.append(bhash)
self._cursor = last_visited
# Batch-allocate CPU blocks and stamp hashes.
# Batch-allocate CPU blocks.
if gpu_ids:
cpu_blocks = cpu_pool.get_new_blocks(len(gpu_ids))
cpu_ids = [blk.block_id for blk in cpu_blocks]
for cpu_blk, bhash in zip(cpu_blocks, block_hashes): # type: ignore[assignment]
cpu_blk._block_hash = bhash # type: ignore[assignment]
# Touch GPU blocks to prevent eviction during async copy.
gpu_pool.touch([gpu_pool.blocks[bid] for bid in gpu_ids])
else:
@@ -531,9 +533,8 @@ class SimpleCPUOffloadScheduler:
"""Identify newly computed blocks to offload from scheduler requests.
Only considers blocks whose KV data has been **confirmed computed** by
the GPU. This means blocks from the current step are NOT stored until the
next step. If a request finishes in the same step as its last full block,
that block may be missed. (TODO: flush on finish.)
the GPU. Blocks from the current step are stored on a later step, or by
the finish-time flush if the request completes first.
Returns:
(gpu_block_ids, cpu_block_ids, req_ids) for the store event.
@@ -547,7 +548,6 @@ class SimpleCPUOffloadScheduler:
if gpu_block_pool is None:
return [], [], []
cpu_block_pool = self.cpu_block_pool
num_free = cpu_block_pool.get_num_free_blocks()
kv_cache_groups = self.cpu_kv_cache_config.kv_cache_groups
num_groups = len(kv_cache_groups)
# Dedup against blocks already scheduled.
@@ -575,73 +575,15 @@ class SimpleCPUOffloadScheduler:
if not block_ids_by_group:
continue
# --- Phase 1: Scan blocks, classify as cached vs to-store ---
gpu_block_ids: list[int] = []
block_hashes_to_store: list[bytes] = []
advanced_per_group: list[int] = [0] * num_groups
out_of_space = False
# Confirmed tokens: KV data written and visible to all streams.
req = state.request
confirmed_tokens = req.num_computed_tokens - req.num_output_placeholders
# Cap to blocks with confirmed KV data.
aligned_tokens = confirmed_tokens // self.block_size * self.block_size
gpu_block_ids, advanced_per_group = self._select_eager_blocks_to_store(
state, block_ids_by_group
)
for g in range(num_groups):
# FIXME (yifan): handle CPU cache eviction, where
# num_stored_blocks can be stale and omit evicted blocks in
# the middle of the request.
already_stored_g = state.num_stored_blocks[g]
group_gpu_ids = block_ids_by_group[g]
g_block_size = (
kv_cache_groups[g].kv_cache_spec.block_size * self.cp_world_size
)
ready_blocks_g = aligned_tokens // g_block_size
scannable = group_gpu_ids[already_stored_g:ready_blocks_g]
for gpu_block_id in scannable:
gpu_block = gpu_block_pool.blocks[gpu_block_id]
if gpu_block.is_null:
advanced_per_group[g] += 1
continue
bhash_with_group = gpu_block.block_hash
if bhash_with_group is None:
# Masked-out SWA position the coordinator chose not to
# hash; it can never serve a prefix-cache hit, so skip.
advanced_per_group[g] += 1
continue
# Skip if already scheduled for store or already cached in CPU.
if (
gpu_block_id in in_flight
or cpu_block_pool.cached_block_hash_to_block.get_one_block(
bhash_with_group
)
is not None
):
advanced_per_group[g] += 1
continue
if num_free <= 0:
out_of_space = True
break
num_free -= 1
gpu_block_ids.append(gpu_block_id)
block_hashes_to_store.append(bhash_with_group)
advanced_per_group[g] += 1
if out_of_space:
break
# --- Phase 2: Batch allocate CPU blocks and stamp hashes ---
# --- Phase 2: Batch allocate CPU blocks ---
n_to_alloc = len(gpu_block_ids)
if n_to_alloc > 0:
cpu_blocks_alloc = cpu_block_pool.get_new_blocks(n_to_alloc)
cpu_block_ids = [blk.block_id for blk in cpu_blocks_alloc]
for cpu_blk, bhash in zip(cpu_blocks_alloc, block_hashes_to_store):
cpu_blk._block_hash = bhash # type: ignore[assignment]
else:
cpu_block_ids = []
@@ -669,6 +611,53 @@ class SimpleCPUOffloadScheduler:
return merged_gpu_block_ids, merged_cpu_block_ids, req_ids
def _select_eager_blocks_to_store(
self,
state: StoreRequestState,
block_ids_by_group: tuple[list[int], ...],
) -> tuple[list[int], list[int]]:
"""Return confirmed, uncached eager blocks and per-group cursor advances."""
assert self._gpu_block_pool is not None
gpu_block_ids: list[int] = []
advanced_per_group = [0] * len(self.cpu_kv_cache_config.kv_cache_groups)
confirmed_tokens = (
state.request.num_computed_tokens - state.request.num_output_placeholders
)
aligned_tokens = confirmed_tokens // self.block_size * self.block_size
num_free = self.cpu_block_pool.get_num_free_blocks()
for g, group_gpu_ids in enumerate(block_ids_by_group):
if len(gpu_block_ids) >= num_free:
break
# FIXME (yifan): handle CPU cache eviction, where
# num_stored_blocks can be stale and omit evicted blocks in
# the middle of the request.
group_size = (
self.cpu_kv_cache_config.kv_cache_groups[g].kv_cache_spec.block_size
* self.cp_world_size
)
ready = min(len(group_gpu_ids), aligned_tokens // group_size)
for gpu_block_id in group_gpu_ids[state.num_stored_blocks[g] : ready]:
gpu_block = self._gpu_block_pool.blocks[gpu_block_id]
if gpu_block.is_null or gpu_block.block_hash is None:
advanced_per_group[g] += 1
continue
if (
gpu_block_id in self._in_flight_store_gpu_blocks
or self.cpu_block_pool.cached_block_hash_to_block.get_one_block(
gpu_block.block_hash
)
is not None
):
advanced_per_group[g] += 1
continue
if len(gpu_block_ids) >= num_free:
break
gpu_block_ids.append(gpu_block_id)
advanced_per_group[g] += 1
return gpu_block_ids, advanced_per_group
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
"""Handle async transfer completions from worker.
@@ -726,24 +715,31 @@ class SimpleCPUOffloadScheduler:
def _process_store_completion(
self, gpu_block_ids: list[int], cpu_block_ids: list[int]
) -> None:
"""Cache CPU blocks per-group and release GPU refs.
Block hashes were stamped on CPU blocks at allocation time (in
``_prepare_*_store_specs``). Here we just register them in the
cache map so they become discoverable by the load path.
"""
"""Register copied blocks in the CPU prefix cache and release refs."""
assert len(cpu_block_ids) == len(gpu_block_ids)
cpu_blocks = [self.cpu_block_pool.blocks[bid] for bid in cpu_block_ids]
for cpu_block in cpu_blocks:
bhash = cpu_block.block_hash
assert bhash is not None
self.cpu_block_pool.cached_block_hash_to_block.insert(bhash, cpu_block)
assert self._gpu_block_pool is not None
for gpu_block_id, cpu_block in zip(gpu_block_ids, cpu_blocks):
gpu_block = self._gpu_block_pool.blocks[gpu_block_id]
primary_hash = gpu_block.block_hash
assert primary_hash is not None
self.cpu_block_pool._insert_block_hash(
primary_hash,
cpu_block,
num_tokens=gpu_block.block_hash_num_tokens,
)
secondary_hashes = self._gpu_block_pool.cached_block_hashes_by_block.get(
gpu_block_id, ()
)
for block_hash in secondary_hashes:
self.cpu_block_pool._insert_block_hash(
block_hash, cpu_block, num_tokens=None
)
# Free CPU and GPU blocks' ref counts to turn them into prefix cache
self.cpu_block_pool.free_blocks(cpu_blocks)
assert self._gpu_block_pool is not None
self._gpu_block_pool.free_blocks(
self._gpu_block_pool.blocks[bid] for bid in gpu_block_ids
)
@@ -751,8 +747,6 @@ class SimpleCPUOffloadScheduler:
def _release_transfer_refs(self, transfer: TransferMeta) -> None:
"""Release transfer refs without making copied data cacheable."""
cpu_blocks = [self.cpu_block_pool.blocks[bid] for bid in transfer.cpu_block_ids]
for cpu_block in cpu_blocks:
cpu_block.reset_hash()
self.cpu_block_pool.free_blocks(cpu_blocks)
assert self._gpu_block_pool is not None
self._gpu_block_pool.free_blocks(
@@ -760,9 +754,11 @@ class SimpleCPUOffloadScheduler:
)
def has_pending_stores(self) -> bool:
"""Return True if there are in-flight store transfers."""
"""Return True if a store transfer is queued or in flight."""
return bool(
self._store_event_to_blocks or self._abandoned_store_event_to_blocks
self._pending_finished_stores
or self._store_event_to_blocks
or self._abandoned_store_event_to_blocks
)
def request_finished(
@@ -804,8 +800,27 @@ class SimpleCPUOffloadScheduler:
request: "Request",
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
self._queue_finished_eager_store(request, block_ids)
return self.request_finished(request, block_ids=[])
def _queue_finished_eager_store(
self, request: "Request", block_ids: tuple[list[int], ...]
) -> None:
"""Queue confirmed eager blocks omitted after a request enters decode."""
state = self._reqs_to_store.get(request.request_id)
gpu_pool = self._gpu_block_pool
if state is None or gpu_pool is None:
return
gpu_ids, _ = self._select_eager_blocks_to_store(state, block_ids)
if not gpu_ids:
return
cpu_blocks = self.cpu_block_pool.get_new_blocks(len(gpu_ids))
self._pending_finished_stores.append(
TransferMeta(gpu_ids, [b.block_id for b in cpu_blocks])
)
self._in_flight_store_gpu_blocks.update(gpu_ids)
gpu_pool.touch([gpu_pool.blocks[bid] for bid in gpu_ids])
def _free_pending_cpu_hit(self, pending: tuple) -> None:
"""Release the temporary CPU block pin taken in get_num_new_matched_tokens()."""
cpu_hit_blocks, _ = pending
@@ -879,6 +894,10 @@ class SimpleCPUOffloadScheduler:
"""
self._abandoned_store_event_to_blocks.update(self._store_event_to_blocks)
for transfer in self._pending_finished_stores:
self._release_transfer_refs(transfer)
self._pending_finished_stores.clear()
self._store_event_to_blocks.clear()
self._in_flight_store_gpu_blocks.clear()