From 4ae6228284c4e9776f62d0867f543e621864176c Mon Sep 17 00:00:00 2001 From: mevince <26759093+mevince@users.noreply.github.com> Date: Thu, 3 Sep 2026 08:58:07 -0700 Subject: [PATCH] [Bugfix][KV Connector] Populate SimpleCPUOffload BlockStored metadata (#54325) --- .../unit/test_simple_cpu_offload_connector.py | 92 ++ tests/v1/simple_kv_offload/test_kv_events.py | 790 ++++++++++++++++++ .../v1/simple_cpu_offload_connector.py | 4 + vllm/v1/simple_kv_offload/manager.py | 274 +++++- 4 files changed, 1132 insertions(+), 28 deletions(-) create mode 100644 tests/v1/kv_connector/unit/test_simple_cpu_offload_connector.py create mode 100644 tests/v1/simple_kv_offload/test_kv_events.py diff --git a/tests/v1/kv_connector/unit/test_simple_cpu_offload_connector.py b/tests/v1/kv_connector/unit/test_simple_cpu_offload_connector.py new file mode 100644 index 0000000000..69f9c2da41 --- /dev/null +++ b/tests/v1/kv_connector/unit/test_simple_cpu_offload_connector.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for SimpleCPUOffloadConnector-level behavior.""" + +from __future__ import annotations + +import pytest + +from tests.v1.kv_connector.unit.utils import create_vllm_config +from tests.v1.simple_kv_offload.test_scheduler import ( + _BYTES_PER_BLOCK, + BLOCK_SIZE, + DTYPE, + HEAD_SIZE, + NUM_KV_HEADS, +) +from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorRole +from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import ( + SimpleCPUOffloadConnector, +) +from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheConfig, + KVCacheGroupSpec, + KVCacheTensor, +) + + +def _make_kv_cache_config(num_blocks: int = 16) -> KVCacheConfig: + fa_layers = ["layer0"] + spec = FullAttentionSpec( + block_size=BLOCK_SIZE, + num_kv_heads=NUM_KV_HEADS, + head_size=HEAD_SIZE, + dtype=DTYPE, + ) + groups = [KVCacheGroupSpec(fa_layers, spec)] + size = _BYTES_PER_BLOCK * num_blocks + tensors = [ + KVCacheTensor( + size=size, + layers=fa_layers, + layer_stride=size, + block_stride=_BYTES_PER_BLOCK, + ) + ] + return KVCacheConfig( + num_blocks=num_blocks, + kv_cache_tensors=tensors, + kv_cache_groups=groups, + ) + + +def _make_connector( + extra_config: dict | None = None, + num_gpu_blocks: int = 16, + cpu_bytes: int = _BYTES_PER_BLOCK * 8, +) -> SimpleCPUOffloadConnector: + kv_cache_config = _make_kv_cache_config(num_gpu_blocks) + config_dict = { + "cpu_bytes_to_use": cpu_bytes, + } + if extra_config: + config_dict.update(extra_config) + vllm_config = create_vllm_config( + kv_connector="SimpleCPUOffloadConnector", + kv_role="kv_both", + kv_connector_extra_config=config_dict, + ) + connector = SimpleCPUOffloadConnector( + vllm_config=vllm_config, + role=KVConnectorRole.SCHEDULER, + kv_cache_config=kv_cache_config, + ) + return connector + + +def test_disk_mode_rejects_non_positive_capacity() -> None: + """Disk mode with disk_capacity_bytes <= 0 raises ValueError. + + Without this guard, the worker operates in disk mode but the scheduler + classifies events as MEDIUM_CPU instead of MEDIUM_STORAGE, silently + mislabeling events. + """ + with pytest.raises(ValueError, match="disk_capacity_bytes > 0"): + _make_connector( + extra_config={ + "kv_offload_backend": "disk", + "disk_path": "/tmp/fake", + "disk_capacity_bytes": 0, + } + ) diff --git a/tests/v1/simple_kv_offload/test_kv_events.py b/tests/v1/simple_kv_offload/test_kv_events.py new file mode 100644 index 0000000000..3bc6c32e7c --- /dev/null +++ b/tests/v1/simple_kv_offload/test_kv_events.py @@ -0,0 +1,790 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""KV cache event emission tests for SimpleCPUOffloadScheduler. + +Verifies that BlockStored/BlockRemoved events carry the correct storage +medium (MEDIUM_CPU / MEDIUM_STORAGE) and per-group metadata when +``enable_kv_cache_events`` is set, and that no events are emitted when it +is unset. +""" + +from __future__ import annotations + +from tests.v1.simple_kv_offload.test_scheduler import ( + _BYTES_PER_BLOCK, + BLOCK_SIZE, + DTYPE, + HEAD_SIZE, + NUM_KV_HEADS, + SchedulerFixture, + _alloc_and_register, + _allocate_cp_gpu_blocks, + _make_cp_request, + _make_kv_cache_config, + _make_vllm_config, + make_request, + make_scheduler_output, + simulate_store_completion, +) +from vllm import SamplingParams +from vllm.config.kv_events import KVEventsConfig +from vllm.distributed.kv_events import ( + MEDIUM_CPU, + MEDIUM_STORAGE, + BlockRemoved, + BlockStored, +) +from vllm.lora.request import LoRARequest +from vllm.utils.hashing import sha256 +from vllm.v1.core.block_pool import BlockPool +from vllm.v1.core.kv_cache_manager import KVCacheBlocks +from vllm.v1.core.kv_cache_utils import ( + get_block_hash, + get_request_block_hasher, + make_block_hash_with_group_id, + maybe_convert_block_hash, +) +from vllm.v1.core.single_type_kv_cache_manager import register_all_kvcache_specs +from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheConfig, + KVCacheGroupSpec, + KVCacheTensor, + MambaSpec, + SlidingWindowSpec, +) +from vllm.v1.request import Request +from vllm.v1.simple_kv_offload.manager import ( + SimpleCPUOffloadScheduler, + StoreRequestState, +) + + +def _make_mixed_kv_cache_config( + num_blocks: int, + sliding_window: int | None = None, + with_mamba: bool = False, + mamba_block_size: int = BLOCK_SIZE, +) -> KVCacheConfig: + """Build a KVCacheConfig with a FullAttention group plus optional + SlidingWindow and Mamba groups for per-group metadata tests.""" + register_all_kvcache_specs(vllm_config=None) + groups = [] + tensors = [] + + # Group 0 is always FullAttention (the scheduler requires one FA group). + fa_layers = ["layer_fa"] + fa_spec = FullAttentionSpec( + block_size=BLOCK_SIZE, + num_kv_heads=NUM_KV_HEADS, + head_size=HEAD_SIZE, + dtype=DTYPE, + ) + groups.append(KVCacheGroupSpec(fa_layers, fa_spec)) + fa_bytes = _BYTES_PER_BLOCK * num_blocks + tensors.append( + KVCacheTensor( + size=fa_bytes, + layers=fa_layers, + layer_stride=fa_bytes, + block_stride=_BYTES_PER_BLOCK, + ) + ) + + if sliding_window is not None: + sw_layers = ["layer_sw"] + sw_spec = SlidingWindowSpec( + block_size=BLOCK_SIZE, + num_kv_heads=NUM_KV_HEADS, + head_size=HEAD_SIZE, + dtype=DTYPE, + sliding_window=sliding_window, + ) + groups.append(KVCacheGroupSpec(sw_layers, sw_spec)) + sw_bytes = _BYTES_PER_BLOCK * num_blocks + tensors.append( + KVCacheTensor( + size=sw_bytes, + layers=sw_layers, + layer_stride=sw_bytes, + block_stride=_BYTES_PER_BLOCK, + ) + ) + + if with_mamba: + m_layers = ["layer_m"] + m_spec = MambaSpec( + block_size=mamba_block_size, + shapes=((NUM_KV_HEADS, HEAD_SIZE, HEAD_SIZE),), + dtypes=(DTYPE,), + mamba_cache_mode="align", + ) + groups.append(KVCacheGroupSpec(m_layers, m_spec)) + m_bytes = _BYTES_PER_BLOCK * num_blocks + tensors.append( + KVCacheTensor( + size=m_bytes, + layers=m_layers, + layer_stride=m_bytes, + block_stride=_BYTES_PER_BLOCK, + ) + ) + + return KVCacheConfig( + num_blocks=num_blocks, + kv_cache_tensors=tensors, + kv_cache_groups=groups, + ) + + +def make_events_scheduler( + num_cpu_blocks: int = 8, + num_gpu_blocks: int = 16, + num_groups: int = 1, + lazy: bool = False, + disk_capacity_bytes: int = 0, + dcp_world_size: int = 1, + kv_cache_config: KVCacheConfig | None = None, + enable_events: bool = True, +) -> SchedulerFixture: + """Build a SimpleCPUOffloadScheduler with kv cache events enabled.""" + if kv_cache_config is None: + kv_cache_config = _make_kv_cache_config(num_gpu_blocks, num_groups) + vllm_config = _make_vllm_config() + if enable_events: + vllm_config.kv_events_config = KVEventsConfig(enable_kv_cache_events=True) + vllm_config.parallel_config.decode_context_parallel_size = dcp_world_size + + virtual_block_size = BLOCK_SIZE * dcp_world_size + cpu_capacity_bytes = _BYTES_PER_BLOCK * num_cpu_blocks * num_groups + + sched = SimpleCPUOffloadScheduler( + vllm_config=vllm_config, + kv_cache_config=kv_cache_config, + cpu_capacity_bytes=cpu_capacity_bytes, + scheduler_block_size=virtual_block_size, + hash_block_size=virtual_block_size, + lazy_offload=lazy, + disk_capacity_bytes=disk_capacity_bytes, + ) + + gpu_block_pool = BlockPool( + num_gpu_blocks=num_gpu_blocks, + enable_caching=True, + hash_block_size=virtual_block_size, + ) + sched.bind_gpu_block_pool(gpu_block_pool) + return SchedulerFixture( + scheduler=sched, + gpu_block_pool=gpu_block_pool, + vllm_config=vllm_config, + kv_cache_config=kv_cache_config, + num_groups=num_groups, + ) + + +def _do_eager_store( + fix: SchedulerFixture, + num_blocks: int = 2, + lora_request: LoRARequest | None = None, +) -> tuple[object, Request]: + """Run an eager store of ``num_blocks`` and complete it. + + Returns (meta, req) for further assertions. + """ + sched = fix.scheduler + if lora_request is not None: + # lora_request must be set in the constructor so block_hashes + # (computed in __init__) include the lora extra_keys. + req = Request( + request_id="req-lora", + prompt_token_ids=list(range(num_blocks * BLOCK_SIZE + 1)), + sampling_params=SamplingParams(max_tokens=1), + pooling_params=None, + mm_features=None, + lora_request=lora_request, + block_hasher=get_request_block_hasher(BLOCK_SIZE, sha256), + ) + else: + req = make_request(num_blocks=num_blocks) + kv_blocks = _alloc_and_register(fix, req, num_blocks) + sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) + block_ids = kv_blocks.get_block_ids() + sched_out = make_scheduler_output( + {req.request_id: num_blocks * BLOCK_SIZE}, + new_reqs={req.request_id: block_ids}, + ) + meta = sched.build_connector_meta(sched_out) + assert meta.store_event >= 0 + simulate_store_completion(sched, meta.store_event) + return meta, req + + +# --------------------------------------------------------------------------- +# events disabled -> no events +# --------------------------------------------------------------------------- +def test_events_disabled_no_events() -> None: + """When enable_kv_cache_events is False, take_events returns nothing.""" + fix = make_events_scheduler(enable_events=False) + sched = fix.scheduler + assert sched.enable_kv_cache_events is False + _do_eager_store(fix, num_blocks=2) + events = list(sched.take_events()) + assert events == [], f"expected no events, got {events}" + + +# --------------------------------------------------------------------------- +# eager store emits BlockStored with per-group metadata +# --------------------------------------------------------------------------- +def test_block_stored_per_group_metadata_full_attention() -> None: + """BlockStored carries medium, group_idx, kind, sliding_window, block_size, + and the correct block hash.""" + fix = make_events_scheduler() + sched = fix.scheduler + assert sched.kv_event_medium == MEDIUM_CPU + _, req = _do_eager_store(fix, num_blocks=2) + events = list(sched.take_events()) + stored = [e for e in events if isinstance(e, BlockStored)] + assert len(stored) == 2 + ev = stored[0] + assert ev.medium == MEDIUM_CPU + assert ev.group_idx == 0 + assert ev.kv_cache_spec_kind == "full_attention" + assert ev.kv_cache_spec_sliding_window is None + assert ev.block_size == BLOCK_SIZE + assert ev.locality == "LOCAL" + expected_hash_0 = maybe_convert_block_hash( + get_block_hash(make_block_hash_with_group_id(req.block_hashes[0], 0)) + ) + assert ev.block_hashes == [expected_hash_0] + assert ev.parent_block_hash is None + assert ev.token_ids == req.prompt_token_ids[0:BLOCK_SIZE] + + # Second block's parent_block_hash equals the first block's hash. + ev1 = stored[1] + expected_hash_1 = maybe_convert_block_hash( + get_block_hash(make_block_hash_with_group_id(req.block_hashes[1], 0)) + ) + assert ev1.block_hashes == [expected_hash_1] + assert ev1.parent_block_hash == expected_hash_0 + assert ev1.token_ids == req.prompt_token_ids[BLOCK_SIZE : 2 * BLOCK_SIZE] + + +def test_eager_store_lora_metadata() -> None: + """BlockStored carries lora_id and lora_name from req.lora_request.""" + fix = make_events_scheduler() + sched = fix.scheduler + + _do_eager_store( + fix, + num_blocks=1, + lora_request=LoRARequest( + lora_name="test-lora", + lora_int_id=42, + lora_path="/fake/lora/path", + ), + ) + + events = list(sched.take_events()) + stored = [e for e in events if isinstance(e, BlockStored)] + assert len(stored) == 1 + ev = stored[0] + assert ev.lora_id == 42, f"expected lora_id=42, got {ev.lora_id}" + assert ev.lora_name == "test-lora", ( + f"expected lora_name='test-lora', got {ev.lora_name!r}" + ) + assert ev.extra_keys is not None, "expected extra_keys for lora request" + assert any("test-lora" in keys for keys in ev.extra_keys), ( + f"expected 'test-lora' in extra_keys, got {ev.extra_keys}" + ) + assert len(ev.token_ids) == BLOCK_SIZE + + +# --------------------------------------------------------------------------- +# disk mode -> MEDIUM_STORAGE +# --------------------------------------------------------------------------- +def test_disk_mode_medium_storage_on_block_stored() -> None: + """disk_capacity_bytes>0 sets kv_event_medium=STORAGE on BlockStored.""" + fix = make_events_scheduler( + disk_capacity_bytes=_BYTES_PER_BLOCK * 8, + enable_events=True, + ) + sched = fix.scheduler + assert sched.kv_event_medium == MEDIUM_STORAGE + _do_eager_store(fix, num_blocks=1) + events = list(sched.take_events()) + stored = [e for e in events if isinstance(e, BlockStored)] + assert len(stored) == 1 + assert stored[0].medium == MEDIUM_STORAGE + assert stored[0].locality == "LOCAL" + + +def test_finished_eager_store_emits_all_storage_events() -> None: + """Finished stores preserve and announce every registered hash.""" + fix = make_events_scheduler( + disk_capacity_bytes=_BYTES_PER_BLOCK * 8, + enable_events=True, + ) + sched = fix.scheduler + req = make_request(num_blocks=2) + kv_blocks = _alloc_and_register(fix, req, num_blocks=2) + gpu_blocks = kv_blocks.blocks[0] + primary_hashes = [] + for gpu_block in gpu_blocks: + assert gpu_block.block_hash is not None + primary_hashes.append(gpu_block.block_hash) + + fine_grained_req = Request( + request_id="req-finished-fine-grained-hash", + prompt_token_ids=req.prompt_token_ids, + sampling_params=req.sampling_params, + pooling_params=None, + mm_features=None, + block_hasher=get_request_block_hasher(BLOCK_SIZE // 2, sha256), + ) + secondary_hash = make_block_hash_with_group_id(fine_grained_req.block_hashes[0], 0) + fix.gpu_block_pool._insert_block_hash( + secondary_hash, gpu_blocks[0], num_tokens=BLOCK_SIZE // 2 + ) + + sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) + sched.request_finished_all_groups(req, kv_blocks.get_block_ids()) + meta = sched.build_connector_meta(make_scheduler_output({})) + assert meta.store_event >= 0 + simulate_store_completion(sched, meta.store_event) + + primary_cpu_block = sched.cpu_block_pool.cached_block_hash_to_block.get_one_block( + primary_hashes[0] + ) + secondary_cpu_block = sched.cpu_block_pool.cached_block_hash_to_block.get_one_block( + secondary_hash + ) + assert primary_cpu_block is not None + assert secondary_cpu_block is primary_cpu_block + + stored = [e for e in sched.take_events() if isinstance(e, BlockStored)] + primary_event_hashes = [ + maybe_convert_block_hash(get_block_hash(block_hash)) + for block_hash in primary_hashes + ] + secondary_event_hash = maybe_convert_block_hash(get_block_hash(secondary_hash)) + by_hash = {event.block_hashes[0]: event for event in stored} + assert set(by_hash) == {*primary_event_hashes, secondary_event_hash} + assert ( + by_hash[primary_event_hashes[0]].token_ids == req.prompt_token_ids[:BLOCK_SIZE] + ) + assert ( + by_hash[primary_event_hashes[1]].token_ids + == req.prompt_token_ids[BLOCK_SIZE : 2 * BLOCK_SIZE] + ) + assert by_hash[primary_event_hashes[1]].parent_block_hash == primary_event_hashes[0] + secondary_event = by_hash[secondary_event_hash] + assert secondary_event.medium == MEDIUM_STORAGE + assert secondary_event.block_size == 0 + assert secondary_event.token_ids == [] + assert secondary_event.parent_block_hash is None + + sched.cpu_block_pool.evict_blocks({primary_cpu_block.block_id}) + removed = [ + event for event in sched.take_events() if isinstance(event, BlockRemoved) + ] + assert {event.block_hashes[0] for event in removed} == { + primary_event_hashes[0], + secondary_event_hash, + } + assert all(event.medium == MEDIUM_STORAGE for event in removed) + + +def test_secondary_hash_block_stored_metadata() -> None: + """Secondary hashes carry their own token range and block size.""" + fix = make_events_scheduler( + disk_capacity_bytes=_BYTES_PER_BLOCK * 8, + enable_events=True, + ) + sched = fix.scheduler + sched.hash_block_size = BLOCK_SIZE // 2 + req = Request( + request_id="req-secondary-hash", + prompt_token_ids=list(range(BLOCK_SIZE + 1)), + sampling_params=SamplingParams(max_tokens=1), + pooling_params=None, + mm_features=None, + block_hasher=get_request_block_hasher(BLOCK_SIZE // 2, sha256), + ) + req.num_computed_tokens = BLOCK_SIZE + gpu_block = fix.gpu_block_pool.get_new_blocks(1)[0] + primary_hash = make_block_hash_with_group_id(req.block_hashes[1], 0) + secondary_hash = make_block_hash_with_group_id(req.block_hashes[0], 0) + fix.gpu_block_pool._insert_block_hash( + primary_hash, gpu_block, num_tokens=BLOCK_SIZE + ) + fix.gpu_block_pool._insert_block_hash( + secondary_hash, gpu_block, num_tokens=BLOCK_SIZE // 2 + ) + state = StoreRequestState( + request=req, + block_ids=([gpu_block.block_id],), + num_stored_blocks=[0], + ) + + gpu_block_ids, advanced_per_group, block_meta = sched._select_eager_blocks_to_store( + state, + ([gpu_block.block_id],), + ) + assert gpu_block_ids == [gpu_block.block_id] + assert advanced_per_group == [1] + assert block_meta is not None + assert set(block_meta[0]) == {primary_hash, secondary_hash} + + cpu_block = sched.cpu_block_pool.get_new_blocks(1)[0] + sched._process_store_completion( + gpu_block_ids, + [cpu_block.block_id], + block_meta, + ) + + primary_event_hash = maybe_convert_block_hash(get_block_hash(primary_hash)) + secondary_event_hash = maybe_convert_block_hash(get_block_hash(secondary_hash)) + stored = [event for event in sched.take_events() if isinstance(event, BlockStored)] + by_hash = {event.block_hashes[0]: event for event in stored} + assert set(by_hash) == {primary_event_hash, secondary_event_hash} + assert by_hash[primary_event_hash].block_size == BLOCK_SIZE + assert by_hash[primary_event_hash].token_ids == req.prompt_token_ids[:BLOCK_SIZE] + assert by_hash[secondary_event_hash].block_size == BLOCK_SIZE // 2 + assert ( + by_hash[secondary_event_hash].token_ids + == req.prompt_token_ids[: BLOCK_SIZE // 2] + ) + + primary_cpu_block = sched.cpu_block_pool.cached_block_hash_to_block.get_one_block( + primary_hash + ) + secondary_cpu_block = sched.cpu_block_pool.cached_block_hash_to_block.get_one_block( + secondary_hash + ) + assert primary_cpu_block is not None + assert secondary_cpu_block is primary_cpu_block + + +# --------------------------------------------------------------------------- +# BlockRemoved relabeled to correct medium +# --------------------------------------------------------------------------- +def test_block_removed_relabeled_medium_cpu() -> None: + """Evicted BlockRemoved (emitted as GPU by BlockPool) is relabeled CPU.""" + # 5 total = 4 usable (null_block takes 1). Fill 4, then store 2 more to + # evict 2 LRU blocks, producing BlockRemoved events. + fix = make_events_scheduler(num_cpu_blocks=5, num_gpu_blocks=16) + sched = fix.scheduler + + # Fill CPU with 4 blocks (2 + 2). + _do_eager_store(fix, num_blocks=2) + # Drain store events so they don't clutter the eviction check. + list(sched.take_events()) + _do_eager_store(fix, num_blocks=2) + list(sched.take_events()) + + # Now store 2 more -> evicts 2 LRU blocks -> BlockRemoved emitted by pool + # with MEDIUM_GPU, then relabeled by scheduler.take_events(). + _do_eager_store(fix, num_blocks=2) + events = list(sched.take_events()) + + removed = [e for e in events if isinstance(e, BlockRemoved)] + assert len(removed) >= 2, ( + f"expected >=2 BlockRemoved from eviction, got {len(removed)}" + ) + for ev in removed: + assert ev.medium == MEDIUM_CPU, ( + f"BlockRemoved should be relabeled CPU, got {ev.medium}" + ) + assert ev.locality == "LOCAL", ( + f"BlockRemoved should be LOCAL, got {ev.locality}" + ) + stored = [e for e in events if isinstance(e, BlockStored)] + assert len(stored) == 2 + + +# --------------------------------------------------------------------------- +# lazy store emits BlockStored +# --------------------------------------------------------------------------- +def test_lazy_store_emits_block_stored() -> None: + """Lazy-mode store completion also emits BlockStored with MEDIUM_CPU.""" + fix = make_events_scheduler(num_cpu_blocks=8, num_gpu_blocks=8, lazy=True) + sched = fix.scheduler + gpu_pool = fix.gpu_block_pool + + # Allocate, hash, free -> hashed blocks in free queue. + req = make_request(num_blocks=2) + gpu_blocks = [] + for blk in gpu_pool.get_new_blocks(2): + gpu_blocks.append(blk) + gpu_pool.cache_full_blocks( + request=req, + blocks=gpu_blocks, + num_cached_blocks=0, + num_full_blocks=2, + block_size=BLOCK_SIZE, + kv_cache_group_id=0, + ) + gpu_pool.free_blocks(gpu_blocks) + + # Push hashed blocks to LRU head (8 total - 1 null = 7 usable; 2 freed + # hashed + 5 other free -> consume 5 fillers). + fillers = gpu_pool.get_new_blocks(5) + + meta = sched.build_connector_meta(make_scheduler_output({})) + assert meta.store_event >= 0 + simulate_store_completion(sched, meta.store_event) + gpu_pool.free_blocks(fillers) + + events = list(sched.take_events()) + stored = [e for e in events if isinstance(e, BlockStored)] + assert len(stored) == 2, f"expected 2 BlockStored, got {len(stored)}" + for ev in stored: + assert ev.medium == MEDIUM_CPU + assert ev.group_idx == 0 + assert ev.kv_cache_spec_kind == "full_attention" + assert ev.locality == "LOCAL" + assert ev.token_ids == [] + assert ev.parent_block_hash is None + + +# --------------------------------------------------------------------------- +# cp_world_size scales block_size for non-Mamba +# --------------------------------------------------------------------------- +def test_cp_world_size_scales_block_size() -> None: + """block_size on BlockStored = spec.block_size * cp_world_size (FA).""" + fix = make_events_scheduler(dcp_world_size=2) + sched = fix.scheduler + gpu_pool = fix.gpu_block_pool + vbs = BLOCK_SIZE * 2 + assert sched.cp_world_size == 2 + + req = _make_cp_request(num_blocks=1, virtual_block_size=vbs) + gpu_blocks = _allocate_cp_gpu_blocks(gpu_pool, req, 1, vbs) + kv_blocks = KVCacheBlocks(blocks=(gpu_blocks,)) + req.num_computed_tokens = vbs + sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) + sched_out = make_scheduler_output( + {req.request_id: vbs}, + new_reqs={req.request_id: kv_blocks.get_block_ids()}, + ) + meta = sched.build_connector_meta(sched_out) + assert meta.store_event >= 0 + simulate_store_completion(sched, meta.store_event) + + events = list(sched.take_events()) + stored = [e for e in events if isinstance(e, BlockStored)] + assert len(stored) == 1 + # spec.block_size=BLOCK_SIZE, cp_world_size=2 -> BLOCK_SIZE * 2 + assert stored[0].block_size == BLOCK_SIZE * 2 + assert stored[0].locality == "LOCAL" + + +# --------------------------------------------------------------------------- +# Mamba block_size NOT scaled by cp_world_size +# --------------------------------------------------------------------------- +def test_mamba_block_size_not_scaled() -> None: + """Mamba group: block_size = spec.block_size (no cp scaling). + + With cp_world_size=2, scheduler_block_size=hash_block_size=32. The + coordinator constraints force Mamba spec.block_size=32 (must divide + scheduler_block_size and be a multiple of hash_block_size). The event + block_size for Mamba must be 32 (spec.block_size * 1), NOT 64 + (spec.block_size * cp_world_size) -- that is the bug this guards. + """ + kv_cfg = _make_mixed_kv_cache_config( + num_blocks=16, with_mamba=True, mamba_block_size=32 + ) + fix = make_events_scheduler( + num_cpu_blocks=8, + num_gpu_blocks=16, + kv_cache_config=kv_cfg, + dcp_world_size=2, + num_groups=2, + ) + sched = fix.scheduler + + cpu_pool = sched.cpu_block_pool + gpu_pool = fix.gpu_block_pool + + cpu_blk = cpu_pool.get_new_blocks(1)[0] + gpu_blk = gpu_pool.get_new_blocks(1)[0] + + raw_hash = make_request(num_blocks=1).block_hashes[0] + mamba_hash = make_block_hash_with_group_id(raw_hash, group_id=1) + gpu_blk._block_hash = mamba_hash # type: ignore[attr-defined] + + sched._process_store_completion( + gpu_block_ids=[gpu_blk.block_id], + cpu_block_ids=[cpu_blk.block_id], + ) + + events = list(sched.take_events()) + stored = [e for e in events if isinstance(e, BlockStored)] + assert len(stored) == 1 + ev = stored[0] + assert ev.group_idx == 1 + assert ev.kv_cache_spec_kind == "mamba" + assert ev.kv_cache_spec_sliding_window is None + assert ev.locality == "LOCAL" + assert ev.block_size == 32, ( + f"Mamba block_size should be unscaled (32), got {ev.block_size}" + ) + + +# --------------------------------------------------------------------------- +# multi-group FA+SW stores carry correct group_idx and kind +# --------------------------------------------------------------------------- +def test_multi_group_fa_block_stored_group_idx() -> None: + """Two distinct groups (FA + SW): each BlockStored carries its group_idx + and kind. HybridKVCacheCoordinator requires >=2 distinct spec groups, so + we use FA (group 0) + SlidingWindow (group 1).""" + kv_cfg = _make_mixed_kv_cache_config(num_blocks=16, sliding_window=32) + fix = make_events_scheduler( + num_cpu_blocks=8, + num_gpu_blocks=16, + kv_cache_config=kv_cfg, + num_groups=2, + ) + sched = fix.scheduler + + cpu_pool = sched.cpu_block_pool + gpu_pool = fix.gpu_block_pool + raw_hash = make_request(num_blocks=1).block_hashes[0] + + # Store one block for each group via direct _process_store_completion. + for gidx in (0, 1): + cpu_blk = cpu_pool.get_new_blocks(1)[0] + gpu_blk = gpu_pool.get_new_blocks(1)[0] + gpu_blk._block_hash = make_block_hash_with_group_id( # type: ignore + raw_hash, group_id=gidx + ) + sched._process_store_completion( + gpu_block_ids=[gpu_blk.block_id], + cpu_block_ids=[cpu_blk.block_id], + ) + + events = list(sched.take_events()) + stored = [e for e in events if isinstance(e, BlockStored)] + assert len(stored) == 2 + by_group = {ev.group_idx: ev for ev in stored} + assert set(by_group) == {0, 1} + assert by_group[0].kv_cache_spec_kind == "full_attention" + assert by_group[0].kv_cache_spec_sliding_window is None + assert by_group[1].kv_cache_spec_kind == "sliding_window" + assert by_group[1].kv_cache_spec_sliding_window == 32 + for ev in stored: + assert ev.locality == "LOCAL" + + +def test_disk_mode_block_removed_relabeled_storage() -> None: + """Disk mode: evicted BlockRemoved relabeled to MEDIUM_STORAGE.""" + fix = make_events_scheduler( + num_cpu_blocks=5, + num_gpu_blocks=16, + disk_capacity_bytes=_BYTES_PER_BLOCK * 5, + ) + sched = fix.scheduler + assert sched.kv_event_medium == MEDIUM_STORAGE + + # Fill disk with 4 blocks (2 + 2). + _do_eager_store(fix, num_blocks=2) + list(sched.take_events()) + _do_eager_store(fix, num_blocks=2) + list(sched.take_events()) + + # Store 2 more -> evicts 2 LRU blocks -> BlockRemoved relabeled STORAGE. + _do_eager_store(fix, num_blocks=2) + events = list(sched.take_events()) + + removed = [e for e in events if isinstance(e, BlockRemoved)] + assert len(removed) >= 2, ( + f"expected >=2 BlockRemoved from eviction, got {len(removed)}" + ) + for ev in removed: + assert ev.medium == MEDIUM_STORAGE, ( + f"BlockRemoved should be relabeled STORAGE, got {ev.medium}" + ) + assert ev.locality == "LOCAL", ( + f"BlockRemoved should be LOCAL, got {ev.locality}" + ) + stored = [e for e in events if isinstance(e, BlockStored)] + assert len(stored) == 2 + for ev in stored: + assert ev.medium == MEDIUM_STORAGE + + +def test_mamba_dcp_metadata_guard() -> None: + """Mamba+DCP: token_ids guard emits [] when capture and emission sizes differ. + + The metadata capture at _prepare_eager_store_specs uses + g_block_size = spec.block_size * cp_world_size = 32 * 2 = 64 to slice + tokens. The event emission uses block_size = spec.block_size * 1 = 32 + for Mamba (SSM state is not CP-sharded). The guard in + _process_store_completion detects len(token_ids) != block_size and emits + token_ids=[] instead of misleading data. When #49962 fixes the capture + path, the guard stops triggering and correct token_ids flow through. + """ + vbs = BLOCK_SIZE * 2 + kv_cfg = _make_mixed_kv_cache_config( + num_blocks=16, with_mamba=True, mamba_block_size=32 + ) + fix = make_events_scheduler( + num_cpu_blocks=8, + num_gpu_blocks=16, + kv_cache_config=kv_cfg, + dcp_world_size=2, + num_groups=2, + ) + sched = fix.scheduler + gpu_pool = fix.gpu_block_pool + + # Mamba g_block_size = 32 * 2 = 64, so 2 Mamba blocks need 128 computed + # tokens to be "ready"; FA g_block_size = 16 * 2 = 32 needs only 64. + req = _make_cp_request(num_blocks=4, virtual_block_size=vbs) + fa_blocks = _allocate_cp_gpu_blocks(gpu_pool, req, 2, vbs, group_id=0) + mamba_blocks = _allocate_cp_gpu_blocks(gpu_pool, req, 2, vbs, group_id=1) + kv_blocks = KVCacheBlocks(blocks=(fa_blocks, mamba_blocks)) + req.num_computed_tokens = 4 * vbs + sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) + + block_ids = kv_blocks.get_block_ids() + sched_out = make_scheduler_output( + {req.request_id: 4 * vbs}, + new_reqs={req.request_id: block_ids}, + ) + meta = sched.build_connector_meta(sched_out) + assert meta.store_event >= 0 + simulate_store_completion(sched, meta.store_event) + + events = list(sched.take_events()) + stored = [e for e in events if isinstance(e, BlockStored)] + assert len(stored) == 4, ( + f"expected 4 BlockStored (2 FA + 2 Mamba), got {len(stored)}" + ) + + by_group: dict[int, list[BlockStored]] = {} + for ev in stored: + by_group.setdefault(ev.group_idx, []).append(ev) + + # FA group: capture size (64) == emission size (64) → token_ids present. + fa_events = by_group[0] + assert len(fa_events) == 2 + for ev in fa_events: + assert ev.block_size == vbs + assert len(ev.token_ids) == vbs, ( + f"FA token_ids should be {vbs}, got {len(ev.token_ids)}" + ) + + # Mamba group: capture size (64) != emission size (32) → guard emits []. + mamba_events = by_group[1] + assert len(mamba_events) == 2 + for ev in mamba_events: + assert ev.block_size == 32 + assert ev.token_ids == [], ( + f"Mamba token_ids should be [] (guard), got {ev.token_ids}" + ) + assert ev.parent_block_hash is None, ( + "Mamba parent_block_hash should be None (guard)" + ) + assert ev.extra_keys is None, "Mamba extra_keys should be None (guard)" diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py index 47f1e098e4..0589c83232 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py @@ -111,6 +111,10 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA): raise ValueError( 'kv_offload_backend="disk" requires disk_path to be set.' ) + if disk_capacity_bytes <= 0: + raise ValueError( + 'kv_offload_backend="disk" requires disk_capacity_bytes > 0.' + ) else: ignored = [k for k in _DISK_ONLY_KEYS if k in extra_config] if ignored: diff --git a/vllm/v1/simple_kv_offload/manager.py b/vllm/v1/simple_kv_offload/manager.py index 423aca7955..d350211199 100644 --- a/vllm/v1/simple_kv_offload/manager.py +++ b/vllm/v1/simple_kv_offload/manager.py @@ -8,7 +8,13 @@ from dataclasses import dataclass, field, replace from typing import TYPE_CHECKING, Any from vllm.config import VllmConfig -from vllm.distributed.kv_events import KVCacheEvent +from vllm.distributed.kv_events import ( + MEDIUM_CPU, + MEDIUM_STORAGE, + BlockRemoved, + BlockStored, + KVCacheEvent, +) from vllm.distributed.kv_transfer.kv_connector.utils import yield_req_data from vllm.logger import init_logger from vllm.utils.math_utils import cdiv @@ -17,11 +23,23 @@ from vllm.v1.core.kv_cache_coordinator import ( KVCacheCoordinator, get_kv_cache_coordinator, ) +from vllm.v1.core.kv_cache_utils import ( + BlockHashWithGroupId, + ExternalBlockHash, + generate_block_hash_extra_keys, + get_block_hash, + get_group_id, + make_block_hash_with_group_id, + maybe_convert_block_hash, + resolve_block_hashes, +) from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.kv_cache_interface import ( FullAttentionSpec, MambaSpec, SlidingWindowSpec, + get_kv_cache_spec_kind, + get_kv_cache_spec_sliding_window, ) from vllm.v1.outputs import KVConnectorOutput from vllm.v1.simple_kv_offload.metadata import ( @@ -38,10 +56,27 @@ if TYPE_CHECKING: logger = init_logger(__name__) +@dataclass +class BlockStoreMeta: + """Per-block metadata snapshot for BlockStored event emission. + + Captured at store-prep time (when the Request is available) and carried + through TransferMeta to async completion, where the Request is gone. + """ + + token_ids: list[int] + parent_block_hash: ExternalBlockHash | None + lora_id: int | None + lora_name: str | None + extra_keys: tuple[Any, ...] | None + block_size: int | None = None + + @dataclass class TransferMeta: gpu_block_ids: list[int] cpu_block_ids: list[int] + block_meta: list[dict[BlockHashWithGroupId, BlockStoreMeta]] | None = None @dataclass @@ -98,6 +133,7 @@ class SimpleCPUOffloadScheduler: kv_cache_config, offload_capacity ) self.num_cpu_blocks = self.cpu_kv_cache_config.num_blocks + self.kv_event_medium = MEDIUM_STORAGE if disk_capacity_bytes > 0 else MEDIUM_CPU # Find the full attention kv group for prefix cache matching. self.fa_gidx = -1 for g_idx, g in enumerate(self.cpu_kv_cache_config.kv_cache_groups): @@ -414,17 +450,22 @@ class SimpleCPUOffloadScheduler: ) -> SimpleCPUOffloadMetadata: # --- Stores --- store_event = -1 - store_gpu, store_cpu, store_req_ids = self.prepare_store_specs(scheduler_output) + store_gpu, store_cpu, store_req_ids, store_meta = self.prepare_store_specs( + scheduler_output + ) for transfer in self._pending_finished_stores: store_gpu.extend(transfer.gpu_block_ids) store_cpu.extend(transfer.cpu_block_ids) + if store_meta is not None: + assert transfer.block_meta is not None + store_meta.extend(transfer.block_meta) self._pending_finished_stores.clear() if store_gpu: store_event = self._store_event_counter self._store_event_counter += 1 self._store_event_to_blocks[store_event] = TransferMeta( - store_gpu, store_cpu + store_gpu, store_cpu, store_meta ) if store_req_ids: # For eager mode only, track req->blocks mapping self._store_event_to_reqs[store_event] = store_req_ids @@ -469,7 +510,12 @@ class SimpleCPUOffloadScheduler: def prepare_store_specs( self, scheduler_output: SchedulerOutput - ) -> tuple[list[int], list[int], list[str]]: + ) -> tuple[ + list[int], + list[int], + list[str], + list[dict[BlockHashWithGroupId, BlockStoreMeta]] | None, + ]: """Prepare store specs for the store event.""" if self._lazy_mode: return self._prepare_lazy_store_specs() @@ -478,7 +524,12 @@ class SimpleCPUOffloadScheduler: def _prepare_lazy_store_specs( self, - ) -> tuple[list[int], list[int], list[str]]: + ) -> tuple[ + list[int], + list[int], + list[str], + list[dict[BlockHashWithGroupId, BlockStoreMeta]] | None, + ]: """Single-pass cursor walk: offload cached GPU blocks near eviction. Walks the GPU free queue from the cursor, counting blocks that are @@ -487,7 +538,7 @@ class SimpleCPUOffloadScheduler: """ gpu_pool = self._gpu_block_pool if gpu_pool is None or self._target_free <= 0: - return [], [], [] + return [], [], [], None free_queue = gpu_pool.free_block_queue cpu_pool = self.cpu_block_pool @@ -525,11 +576,16 @@ class SimpleCPUOffloadScheduler: else: cpu_ids = [] - return gpu_ids, cpu_ids, [] + return gpu_ids, cpu_ids, [], None def _prepare_eager_store_specs( self, scheduler_output: SchedulerOutput - ) -> tuple[list[int], list[int], list[str]]: + ) -> tuple[ + list[int], + list[int], + list[str], + list[dict[BlockHashWithGroupId, BlockStoreMeta]] | None, + ]: """Identify newly computed blocks to offload from scheduler requests. Only considers blocks whose KV data has been **confirmed computed** by @@ -537,19 +593,22 @@ class SimpleCPUOffloadScheduler: the finish-time flush if the request completes first. Returns: - (gpu_block_ids, cpu_block_ids, req_ids) for the store event. + (gpu_block_ids, cpu_block_ids, req_ids, block_meta) for the store + event. ``block_meta`` is None when kv cache events are disabled. """ merged_gpu_block_ids: list[int] = [] merged_cpu_block_ids: list[int] = [] req_ids: list[str] = [] + merged_block_meta: list[dict[BlockHashWithGroupId, BlockStoreMeta]] | None = ( + [] if self.enable_kv_cache_events else None + ) gpu_block_pool = self._gpu_block_pool if gpu_block_pool is None: - return [], [], [] + return [], [], [], merged_block_meta cpu_block_pool = self.cpu_block_pool - kv_cache_groups = self.cpu_kv_cache_config.kv_cache_groups - num_groups = len(kv_cache_groups) + num_groups = len(self.cpu_kv_cache_config.kv_cache_groups) # Dedup against blocks already scheduled. in_flight = self._in_flight_store_gpu_blocks @@ -558,13 +617,13 @@ class SimpleCPUOffloadScheduler: if state is None or state.finished: continue - # Accumulate new block IDs. if preempted: state.block_ids = tuple([] for _ in range(num_groups)) state.num_stored_blocks = [0] * num_groups if new_block_id_groups: for g in range(min(num_groups, len(new_block_id_groups))): if new_block_id_groups[g] is not None: + # Accumulate new block IDs. state.block_ids[g].extend(new_block_id_groups[g]) num_new_tokens = scheduler_output.num_scheduled_tokens.get(req_id, 0) @@ -575,8 +634,8 @@ class SimpleCPUOffloadScheduler: if not block_ids_by_group: continue - gpu_block_ids, advanced_per_group = self._select_eager_blocks_to_store( - state, block_ids_by_group + gpu_block_ids, advanced_per_group, block_meta = ( + self._select_eager_blocks_to_store(state, block_ids_by_group) ) # --- Phase 2: Batch allocate CPU blocks --- @@ -592,6 +651,9 @@ class SimpleCPUOffloadScheduler: merged_gpu_block_ids.extend(gpu_block_ids) merged_cpu_block_ids.extend(cpu_block_ids) in_flight.update(gpu_block_ids) + if merged_block_meta is not None: + assert block_meta is not None + merged_block_meta.extend(block_meta) # Touch GPU blocks to prevent freeing during async copy gpu_block_pool.touch( @@ -609,20 +671,26 @@ class SimpleCPUOffloadScheduler: for g in range(num_groups): state.num_stored_blocks[g] += advanced_per_group[g] - return merged_gpu_block_ids, merged_cpu_block_ids, req_ids + return merged_gpu_block_ids, merged_cpu_block_ids, req_ids, merged_block_meta def _select_eager_blocks_to_store( self, state: StoreRequestState, block_ids_by_group: tuple[list[int], ...], - ) -> tuple[list[int], list[int]]: - """Return confirmed, uncached eager blocks and per-group cursor advances.""" + ) -> tuple[ + list[int], + list[int], + list[dict[BlockHashWithGroupId, BlockStoreMeta]] | None, + ]: + """Return confirmed eager blocks, cursor advances, and event metadata.""" assert self._gpu_block_pool is not None gpu_block_ids: list[int] = [] advanced_per_group = [0] * len(self.cpu_kv_cache_config.kv_cache_groups) - confirmed_tokens = ( - state.request.num_computed_tokens - state.request.num_output_placeholders + block_meta: list[dict[BlockHashWithGroupId, BlockStoreMeta]] | None = ( + [] if self.enable_kv_cache_events else None ) + request = state.request + confirmed_tokens = request.num_computed_tokens - request.num_output_placeholders aligned_tokens = confirmed_tokens // self.block_size * self.block_size num_free = self.cpu_block_pool.get_num_free_blocks() @@ -637,7 +705,17 @@ class SimpleCPUOffloadScheduler: * self.cp_world_size ) ready = min(len(group_gpu_ids), aligned_tokens // group_size) - for gpu_block_id in group_gpu_ids[state.num_stored_blocks[g] : ready]: + resolved_hashes = ( + resolve_block_hashes( + request.block_hashes, self.hash_block_size, group_size + ) + if block_meta is not None + else None + ) + curr_mm_idx = 0 + secondary_mm_idx = 0 + start = state.num_stored_blocks[g] + for i, gpu_block_id in enumerate(group_gpu_ids[start:ready], start=start): gpu_block = self._gpu_block_pool.blocks[gpu_block_id] if gpu_block.is_null or gpu_block.block_hash is None: advanced_per_group[g] += 1 @@ -655,8 +733,65 @@ class SimpleCPUOffloadScheduler: break gpu_block_ids.append(gpu_block_id) advanced_per_group[g] += 1 + if block_meta is not None: + token_start = i * group_size + token_end = token_start + group_size + assert resolved_hashes is not None + parent_hash = ( + None + if i == 0 + else maybe_convert_block_hash(resolved_hashes[i - 1]) + ) + lora_req = request.lora_request + extra_keys, curr_mm_idx = generate_block_hash_extra_keys( + request, token_start, token_end, curr_mm_idx + ) + meta_by_hash = { + gpu_block.block_hash: BlockStoreMeta( + token_ids=list( + request.all_token_ids[token_start:token_end] + ), + parent_block_hash=parent_hash, + lora_id=lora_req.adapter_id if lora_req else None, + lora_name=lora_req.name if lora_req else None, + extra_keys=extra_keys, + ) + } + first_hash_idx = token_start // self.hash_block_size + last_hash_idx = token_end // self.hash_block_size + for hash_idx in range(first_hash_idx, last_hash_idx): + block_hash = make_block_hash_with_group_id( + request.block_hashes[hash_idx], g + ) + if block_hash == gpu_block.block_hash: + continue + hash_start = hash_idx * self.hash_block_size + hash_end = hash_start + self.hash_block_size + secondary_extra_keys, secondary_mm_idx = ( + generate_block_hash_extra_keys( + request, + hash_start, + hash_end, + secondary_mm_idx, + ) + ) + meta_by_hash[block_hash] = BlockStoreMeta( + token_ids=list(request.all_token_ids[hash_start:hash_end]), + parent_block_hash=( + None + if hash_idx == 0 + else maybe_convert_block_hash( + request.block_hashes[hash_idx - 1] + ) + ), + lora_id=lora_req.adapter_id if lora_req else None, + lora_name=lora_req.name if lora_req else None, + extra_keys=secondary_extra_keys, + block_size=self.hash_block_size, + ) + block_meta.append(meta_by_hash) - return gpu_block_ids, advanced_per_group + return gpu_block_ids, advanced_per_group, block_meta def update_connector_output(self, connector_output: KVConnectorOutput) -> None: """Handle async transfer completions from worker. @@ -695,7 +830,11 @@ class SimpleCPUOffloadScheduler: if not self._lazy_mode: self._in_flight_store_gpu_blocks.difference_update(transfer.gpu_block_ids) - self._process_store_completion(transfer.gpu_block_ids, transfer.cpu_block_ids) + self._process_store_completion( + transfer.gpu_block_ids, + transfer.cpu_block_ids, + transfer.block_meta, + ) logger.debug( "Store event %d completed: cached %d blocks to CPU", event_idx, @@ -713,15 +852,19 @@ class SimpleCPUOffloadScheduler: self._cleanup_store_request(req_id) def _process_store_completion( - self, gpu_block_ids: list[int], cpu_block_ids: list[int] + self, + gpu_block_ids: list[int], + cpu_block_ids: list[int], + block_meta: list[dict[BlockHashWithGroupId, BlockStoreMeta]] | None = None, ) -> None: """Register copied blocks in the CPU prefix cache and release refs.""" assert len(cpu_block_ids) == len(gpu_block_ids) + assert block_meta is None or len(block_meta) == len(gpu_block_ids) cpu_blocks = [self.cpu_block_pool.blocks[bid] for bid in cpu_block_ids] assert self._gpu_block_pool is not None - for gpu_block_id, cpu_block in zip(gpu_block_ids, cpu_blocks): + for i, (gpu_block_id, cpu_block) in enumerate(zip(gpu_block_ids, cpu_blocks)): gpu_block = self._gpu_block_pool.blocks[gpu_block_id] primary_hash = gpu_block.block_hash assert primary_hash is not None @@ -737,6 +880,76 @@ class SimpleCPUOffloadScheduler: self.cpu_block_pool._insert_block_hash( block_hash, cpu_block, num_tokens=None ) + if self.enable_kv_cache_events: + meta_by_hash = block_meta[i] if block_meta is not None else {} + primary_group_idx = get_group_id(primary_hash) + primary_spec = self.cpu_kv_cache_config.kv_cache_groups[ + primary_group_idx + ].kv_cache_spec + primary_block_size = primary_spec.block_size * ( + self.cp_world_size if not isinstance(primary_spec, MambaSpec) else 1 + ) + events_to_emit: list[ + tuple[BlockHashWithGroupId, BlockStoreMeta | None, int] + ] = [ + ( + primary_hash, + meta_by_hash.get(primary_hash), + primary_block_size, + ) + ] + for secondary_hash in secondary_hashes: + secondary_meta = meta_by_hash.get(secondary_hash) + events_to_emit.append( + ( + secondary_hash, + secondary_meta, + secondary_meta.block_size + if secondary_meta is not None + and secondary_meta.block_size is not None + else 0, + ) + ) + + for block_hash, meta, event_block_size in events_to_emit: + group_idx = get_group_id(block_hash) + spec = self.cpu_kv_cache_config.kv_cache_groups[ + group_idx + ].kv_cache_spec + if meta is not None and len(meta.token_ids) != event_block_size: + # token_ids were sliced with a different g_block_size + # (e.g. Mamba+DCP where capture uses spec.block_size * + # cp_world_size but the event emits spec.block_size * 1). + # Emit empty metadata until #49962 fixes the sizing. + token_ids = [] + parent_block_hash = None + extra_keys = None + else: + token_ids = meta.token_ids if meta else [] + parent_block_hash = meta.parent_block_hash if meta else None + extra_keys = meta.extra_keys if meta else None + self.cpu_block_pool.kv_event_queue.append( + BlockStored( + block_hashes=[ + maybe_convert_block_hash(get_block_hash(block_hash)) + ], + parent_block_hash=parent_block_hash, + token_ids=token_ids, + block_size=event_block_size, + lora_id=meta.lora_id if meta else None, + medium=self.kv_event_medium, + lora_name=meta.lora_name if meta else None, + extra_keys=( + [extra_keys] if extra_keys is not None else None + ), + group_idx=group_idx, + kv_cache_spec_kind=get_kv_cache_spec_kind(spec).value, + kv_cache_spec_sliding_window=( + get_kv_cache_spec_sliding_window(spec) + ), + locality="LOCAL", + ) + ) # Free CPU and GPU blocks' ref counts to turn them into prefix cache self.cpu_block_pool.free_blocks(cpu_blocks) @@ -811,12 +1024,12 @@ class SimpleCPUOffloadScheduler: gpu_pool = self._gpu_block_pool if state is None or gpu_pool is None: return - gpu_ids, _ = self._select_eager_blocks_to_store(state, block_ids) + gpu_ids, _, block_meta = self._select_eager_blocks_to_store(state, block_ids) if not gpu_ids: return cpu_blocks = self.cpu_block_pool.get_new_blocks(len(gpu_ids)) self._pending_finished_stores.append( - TransferMeta(gpu_ids, [b.block_id for b in cpu_blocks]) + TransferMeta(gpu_ids, [b.block_id for b in cpu_blocks], block_meta) ) self._in_flight_store_gpu_blocks.update(gpu_ids) gpu_pool.touch([gpu_pool.blocks[bid] for bid in gpu_ids]) @@ -882,7 +1095,12 @@ class SimpleCPUOffloadScheduler: state.store_events.clear() def take_events(self) -> Iterable[KVCacheEvent]: - return self.cpu_block_pool.take_events() + events = self.cpu_block_pool.take_events() + for event in events: + if isinstance(event, BlockRemoved): + event.medium = self.kv_event_medium + event.locality = "LOCAL" + return events def reset(self) -> bool: """Abandon pending transfers and reset the CPU cache when safe.