diff --git a/tests/v1/simple_kv_offload/test_kv_events.py b/tests/v1/simple_kv_offload/test_kv_events.py index 3bc6c32e7c..f21dae7e0b 100644 --- a/tests/v1/simple_kv_offload/test_kv_events.py +++ b/tests/v1/simple_kv_offload/test_kv_events.py @@ -714,17 +714,8 @@ def test_disk_mode_block_removed_relabeled_storage() -> None: assert ev.medium == MEDIUM_STORAGE -def test_mamba_dcp_metadata_guard() -> None: - """Mamba+DCP: token_ids guard emits [] when capture and emission sizes differ. - - The metadata capture at _prepare_eager_store_specs uses - g_block_size = spec.block_size * cp_world_size = 32 * 2 = 64 to slice - tokens. The event emission uses block_size = spec.block_size * 1 = 32 - for Mamba (SSM state is not CP-sharded). The guard in - _process_store_completion detects len(token_ids) != block_size and emits - token_ids=[] instead of misleading data. When #49962 fixes the capture - path, the guard stops triggering and correct token_ids flow through. - """ +def test_mamba_align_skips_positional_event_metadata() -> None: + """Mamba align emits events only from explicit boundary handoffs.""" vbs = BLOCK_SIZE * 2 kv_cfg = _make_mixed_kv_cache_config( num_blocks=16, with_mamba=True, mamba_block_size=32 @@ -739,8 +730,6 @@ def test_mamba_dcp_metadata_guard() -> None: sched = fix.scheduler gpu_pool = fix.gpu_block_pool - # Mamba g_block_size = 32 * 2 = 64, so 2 Mamba blocks need 128 computed - # tokens to be "ready"; FA g_block_size = 16 * 2 = 32 needs only 64. req = _make_cp_request(num_blocks=4, virtual_block_size=vbs) fa_blocks = _allocate_cp_gpu_blocks(gpu_pool, req, 2, vbs, group_id=0) mamba_blocks = _allocate_cp_gpu_blocks(gpu_pool, req, 2, vbs, group_id=1) @@ -759,32 +748,12 @@ def test_mamba_dcp_metadata_guard() -> None: events = list(sched.take_events()) stored = [e for e in events if isinstance(e, BlockStored)] - assert len(stored) == 4, ( - f"expected 4 BlockStored (2 FA + 2 Mamba), got {len(stored)}" - ) - - by_group: dict[int, list[BlockStored]] = {} - for ev in stored: - by_group.setdefault(ev.group_idx, []).append(ev) - - # FA group: capture size (64) == emission size (64) → token_ids present. - fa_events = by_group[0] - assert len(fa_events) == 2 - for ev in fa_events: + assert len(stored) == 2 + assert {ev.group_idx for ev in stored} == {0} + for idx, ev in enumerate(stored): assert ev.block_size == vbs - assert len(ev.token_ids) == vbs, ( - f"FA token_ids should be {vbs}, got {len(ev.token_ids)}" + assert ev.token_ids == req.prompt_token_ids[idx * vbs : (idx + 1) * vbs] + expected_hash = maybe_convert_block_hash( + get_block_hash(make_block_hash_with_group_id(req.block_hashes[idx], 0)) ) - - # Mamba group: capture size (64) != emission size (32) → guard emits []. - mamba_events = by_group[1] - assert len(mamba_events) == 2 - for ev in mamba_events: - assert ev.block_size == 32 - assert ev.token_ids == [], ( - f"Mamba token_ids should be [] (guard), got {ev.token_ids}" - ) - assert ev.parent_block_hash is None, ( - "Mamba parent_block_hash should be None (guard)" - ) - assert ev.extra_keys is None, "Mamba extra_keys should be None (guard)" + assert ev.block_hashes == [expected_hash] diff --git a/tests/v1/simple_kv_offload/test_scheduler.py b/tests/v1/simple_kv_offload/test_scheduler.py index db7e8b7aa5..1952e3a4b0 100644 --- a/tests/v1/simple_kv_offload/test_scheduler.py +++ b/tests/v1/simple_kv_offload/test_scheduler.py @@ -4,6 +4,7 @@ from __future__ import annotations +import logging from dataclasses import dataclass import pytest @@ -18,6 +19,7 @@ from vllm.config import ( SchedulerConfig, VllmConfig, ) +from vllm.config.kv_events import KVEventsConfig from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import ( SimpleCPUOffloadConnector, ) @@ -31,6 +33,7 @@ from vllm.v1.core.kv_cache_utils import ( ) from vllm.v1.core.sched.output import ( CachedRequestData, + KVConnectorBlockState, NewRequestData, SchedulerOutput, ) @@ -42,6 +45,7 @@ from vllm.v1.kv_cache_interface import ( KVCacheConfig, KVCacheGroupSpec, KVCacheTensor, + MambaSpec, SlidingWindowSpec, ) from vllm.v1.outputs import KVConnectorOutput @@ -2128,3 +2132,671 @@ def test_cp_lazy_target_blocks_scaling(cp_world_size: int) -> None: f"cp_world_size={cp_world_size}: target_cp={target_cp} should be " f"less than target_base={target_base}" ) + + +def _make_hybrid_attention_mamba_scheduler( + *, + num_cpu_blocks: int = 8, + num_gpu_blocks: int = 16, + attention_block_size: int = BLOCK_SIZE, + block_size: int = 4 * BLOCK_SIZE, + scheduler_block_size: int | None = None, + hash_block_size: int | None = None, + dcp_world_size: int = 4, + lazy: bool = False, + mamba_cache_mode: str = "align", + enable_kv_cache_events: bool = False, +) -> SchedulerFixture: + """Build a scheduler for one attention group plus one Mamba group.""" + scheduler_block_size = scheduler_block_size or block_size + hash_block_size = hash_block_size or block_size + attention_spec = FullAttentionSpec( + block_size=attention_block_size, + num_kv_heads=NUM_KV_HEADS, + head_size=HEAD_SIZE, + dtype=DTYPE, + ) + mamba_spec = MambaSpec( + block_size=block_size, + shapes=((1, 1),), + dtypes=(torch.float32,), + mamba_cache_mode=mamba_cache_mode, + ) + groups = [ + KVCacheGroupSpec(["attention"], attention_spec), + KVCacheGroupSpec(["mamba"], mamba_spec), + ] + tensors = [ + KVCacheTensor( + size=spec.page_size_bytes * num_gpu_blocks, + layers=group.layer_names, + layer_stride=spec.page_size_bytes * num_gpu_blocks, + block_stride=spec.page_size_bytes, + ) + for group, spec in zip(groups, (attention_spec, mamba_spec)) + ] + kv_cache_config = KVCacheConfig( + num_blocks=num_gpu_blocks, + kv_cache_tensors=tensors, + kv_cache_groups=groups, + ) + vllm_config = _make_cp_vllm_config(dcp_world_size=dcp_world_size) + vllm_config.cache_config.prefix_cache_retention_interval = 0 + vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode + if enable_kv_cache_events: + vllm_config.kv_events_config = KVEventsConfig(enable_kv_cache_events=True) + # _derive_cpu_config() scales the requested capacity against + # kv_cache_tensors[0].size, so express it as a multiple of that tensor's + # per-block size to get exactly num_cpu_blocks. + cpu_capacity_bytes = tensors[0].size // num_gpu_blocks * num_cpu_blocks + sched = SimpleCPUOffloadScheduler( + vllm_config=vllm_config, + kv_cache_config=kv_cache_config, + cpu_capacity_bytes=cpu_capacity_bytes, + scheduler_block_size=scheduler_block_size, + hash_block_size=hash_block_size, + lazy_offload=lazy, + ) + gpu_block_pool = BlockPool( + num_gpu_blocks=num_gpu_blocks, + enable_caching=True, + hash_block_size=hash_block_size, + ) + sched.bind_gpu_block_pool(gpu_block_pool) + return SchedulerFixture( + scheduler=sched, + gpu_block_pool=gpu_block_pool, + vllm_config=vllm_config, + kv_cache_config=kv_cache_config, + ) + + +def test_hybrid_store_uses_resolved_group_block_sizes() -> None: + """Replicated groups must not be scaled by the DCP world size. + + ``dcp_world_size_for_kv_cache_spec`` gives full attention the process DCP + size and every other spec 1. Reconstructing a group's geometry as + ``spec.block_size * cp_world_size`` over-scales the replicated groups, so + the eager store scan believes far fewer of their blocks are ready and + silently offloads only a fraction of them. + """ + attention_block_size = BLOCK_SIZE + mamba_block_size = 4 * BLOCK_SIZE + dcp_world_size = 2 + fix = _make_hybrid_attention_mamba_scheduler( + num_cpu_blocks=32, + num_gpu_blocks=32, + attention_block_size=attention_block_size, + block_size=mamba_block_size, + # Every prefix-cacheable group's resolved block size must be a multiple + # of the hash block, so it cannot exceed the smallest of them. + hash_block_size=BLOCK_SIZE, + dcp_world_size=dcp_world_size, + mamba_cache_mode="all", + ) + sched = fix.scheduler + gpu_pool = fix.gpu_block_pool + attention_size, mamba_size = sched.group_block_sizes + assert attention_size == attention_block_size * dcp_world_size + assert mamba_size == mamba_block_size + + confirmed = 2 * sched.block_size + req = _make_cp_request(num_blocks=8, virtual_block_size=BLOCK_SIZE) + attention_blocks = _allocate_cp_gpu_blocks( + gpu_pool, req, confirmed // attention_size, attention_size, group_id=0 + ) + mamba_blocks = _allocate_cp_gpu_blocks( + gpu_pool, req, confirmed // mamba_size, mamba_size, group_id=1 + ) + kv_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks)) + sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) + sched.build_connector_meta( + make_scheduler_output( + {req.request_id: confirmed}, + new_reqs={req.request_id: kv_blocks.get_block_ids()}, + ) + ) + req.num_computed_tokens = confirmed + meta = sched.build_connector_meta( + make_scheduler_output( + {req.request_id: 1}, + cached_req_new_blocks={req.request_id: None}, + ) + ) + + # Both groups are positionally stable here, so every confirmed block of + # both must be offloaded. Over-scaling the replicated group halves its + # count. + assert sched._reqs_to_store[req.request_id].num_stored_blocks == [ + confirmed // attention_size, + confirmed // mamba_size, + ] + assert set(meta.store_gpu_blocks) == { + *(b.block_id for b in attention_blocks), + *(b.block_id for b in mamba_blocks), + } + + +def test_dcp_mixed_cache_loads_fine_grained_external_hit() -> None: + """Map a fine-grained external suffix after a nonzero local prefix.""" + hash_block_size = BLOCK_SIZE + mamba_block_size = 4 * BLOCK_SIZE + local_tokens = 4 * BLOCK_SIZE + # Span two Mamba blocks and end partially in the second one. This exercises + # both cdiv(external_tokens, mamba_block_size) > 1 and the partial tail. + external_tokens = 6 * BLOCK_SIZE + total_cached_tokens = local_tokens + external_tokens + fix = _make_hybrid_attention_mamba_scheduler( + num_cpu_blocks=32, + num_gpu_blocks=32, + block_size=mamba_block_size, + hash_block_size=hash_block_size, + dcp_world_size=2, + ) + sched = fix.scheduler + gpu_pool = fix.gpu_block_pool + + assert external_tokens % mamba_block_size != 0 + assert sched.cpu_coordinator.enable_partial_hash_hits + + producer = _make_cp_request(num_blocks=10, virtual_block_size=hash_block_size) + attention_blocks = _allocate_cp_gpu_blocks( + gpu_pool, + producer, + num_blocks=5, + virtual_block_size=2 * BLOCK_SIZE, + group_id=0, + ) + mamba_blocks = gpu_pool.get_new_blocks(3) + gpu_pool.cache_full_blocks( + request=producer, + blocks=mamba_blocks, + num_cached_blocks=0, + num_full_blocks=2, + block_size=mamba_block_size, + kv_cache_group_id=1, + ) + gpu_pool.cache_partial_block( + request=producer, + block=mamba_blocks[2], + num_tokens=total_cached_tokens, + kv_cache_group_id=1, + block_size=mamba_block_size, + ) + producer_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks)) + sched.update_state_after_alloc(producer, producer_blocks, num_external_tokens=0) + + # Positional scanning stores append-only FA blocks. Mamba align blocks use + # exact handoffs for every retained boundary, including the partial tail. + store_output = make_scheduler_output( + {producer.request_id: total_cached_tokens}, + new_reqs={producer.request_id: producer_blocks.get_block_ids()}, + ) + store_output.kv_connector_block_state = KVConnectorBlockState( + req_ids=set(), + resolve_block_ids=lambda _: (), + boundary_state_offloads={ + producer.request_id: [ + (1, mamba_blocks[0].block_id, mamba_block_size), + (1, mamba_blocks[1].block_id, 2 * mamba_block_size), + (1, mamba_blocks[2].block_id, total_cached_tokens), + ] + }, + ) + store_meta = sched.build_connector_meta(store_output) + # The exact handoffs land in the step they arrive; the append-only FA scan + # picks its blocks up once the scheduler has committed the tokens. + assert set(store_meta.store_gpu_blocks) == { + block.block_id for block in mamba_blocks + } + simulate_store_completion(sched, store_meta.store_event) + producer.num_computed_tokens = total_cached_tokens + fa_meta = sched.build_connector_meta( + make_scheduler_output( + {producer.request_id: 1}, + cached_req_new_blocks={producer.request_id: None}, + ) + ) + assert set(fa_meta.store_gpu_blocks) == { + block.block_id for block in attention_blocks + } + simulate_store_completion(sched, fa_meta.store_event) + + consumer = Request( + request_id="req-dcp-mixed-fine-load", + prompt_token_ids=producer.prompt_token_ids, + sampling_params=producer.sampling_params, + pooling_params=None, + mm_features=None, + block_hasher=producer._block_hasher, + ) + hit_tokens, is_async = sched.get_num_new_matched_tokens( + consumer, num_computed_tokens=local_tokens + ) + assert hit_tokens == external_tokens + assert is_async is True + pending_blocks, pending_tokens, _ = sched._pending_cpu_hits[consumer.request_id] + assert pending_tokens == external_tokens + assert len(pending_blocks[1]) == 2 + assert pending_blocks[1][0].is_null + assert not pending_blocks[1][1].is_null + + local_attention = attention_blocks[:2] + local_mamba = mamba_blocks[:1] + load_blocks = KVCacheBlocks( + blocks=( + local_attention + gpu_pool.get_new_blocks(3), + local_mamba + gpu_pool.get_new_blocks(2), + ) + ) + sched.update_state_after_alloc( + consumer, + load_blocks, + num_external_tokens=hit_tokens, + ) + load_meta = sched.build_connector_meta( + make_scheduler_output( + {consumer.request_id: 1}, + new_reqs={consumer.request_id: load_blocks.get_block_ids()}, + ) + ) + assert load_meta.load_gpu_blocks == [ + *(block.block_id for block in load_blocks.blocks[0][2:5]), + load_blocks.blocks[1][2].block_id, + ] + assert len(load_meta.load_cpu_blocks) == 4 + + +def test_finished_eager_store_caches_fa_partial_tail_for_hybrid_hit() -> None: + """A finished non-LCM prompt needs the FA partial tail in CPU too.""" + hash_block_size = BLOCK_SIZE + mamba_block_size = BLOCK_SIZE + attention_block_size = 4 * BLOCK_SIZE + dcp_world_size = 2 + scheduler_block_size = attention_block_size * dcp_world_size + total_cached_tokens = 5 * hash_block_size + fix = _make_hybrid_attention_mamba_scheduler( + num_cpu_blocks=32, + num_gpu_blocks=32, + attention_block_size=attention_block_size, + block_size=mamba_block_size, + scheduler_block_size=scheduler_block_size, + hash_block_size=hash_block_size, + dcp_world_size=dcp_world_size, + ) + sched = fix.scheduler + gpu_pool = fix.gpu_block_pool + assert sched.fa_block_size == scheduler_block_size + assert total_cached_tokens % sched.fa_block_size != 0 + + producer = _make_cp_request(num_blocks=5, virtual_block_size=hash_block_size) + attention_blocks = gpu_pool.get_new_blocks(1) + gpu_pool.cache_partial_block( + request=producer, + block=attention_blocks[0], + num_tokens=total_cached_tokens, + kv_cache_group_id=0, + block_size=scheduler_block_size, + ) + mamba_blocks = gpu_pool.get_new_blocks(5) + gpu_pool.cache_full_blocks( + request=producer, + blocks=mamba_blocks, + num_cached_blocks=0, + num_full_blocks=5, + kv_cache_group_id=1, + block_size=mamba_block_size, + ) + producer_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks)) + sched.update_state_after_alloc(producer, producer_blocks, num_external_tokens=0) + + store_output = make_scheduler_output( + {producer.request_id: total_cached_tokens}, + new_reqs={producer.request_id: producer_blocks.get_block_ids()}, + ) + store_output.kv_connector_block_state = KVConnectorBlockState( + req_ids=set(), + resolve_block_ids=lambda _: (), + boundary_state_offloads={ + producer.request_id: [ + (1, mamba_blocks[4].block_id, total_cached_tokens), + ] + }, + ) + store_meta = sched.build_connector_meta(store_output) + simulate_store_completion(sched, store_meta.store_event) + + producer.num_computed_tokens = total_cached_tokens + sched.request_finished_all_groups(producer, producer_blocks.get_block_ids()) + finish_meta = sched.build_connector_meta(make_scheduler_output({})) + assert attention_blocks[0].block_id in finish_meta.store_gpu_blocks + simulate_store_completion(sched, finish_meta.store_event) + + consumer = Request( + request_id="req-finished-partial-tail-load", + prompt_token_ids=producer.prompt_token_ids, + sampling_params=producer.sampling_params, + pooling_params=None, + mm_features=None, + block_hasher=producer._block_hasher, + ) + hit_tokens, is_async = sched.get_num_new_matched_tokens( + consumer, num_computed_tokens=0 + ) + assert hit_tokens == total_cached_tokens + assert is_async is True + + +def test_finished_eager_store_does_not_duplicate_completed_partial_tail() -> None: + """Decode can fill the boundary block; the scan then already covers it.""" + hash_block_size = BLOCK_SIZE + mamba_block_size = BLOCK_SIZE + attention_block_size = 4 * BLOCK_SIZE + dcp_world_size = 2 + scheduler_block_size = attention_block_size * dcp_world_size + prompt_tokens = 5 * hash_block_size + fix = _make_hybrid_attention_mamba_scheduler( + num_cpu_blocks=32, + num_gpu_blocks=32, + attention_block_size=attention_block_size, + block_size=mamba_block_size, + scheduler_block_size=scheduler_block_size, + hash_block_size=hash_block_size, + dcp_world_size=dcp_world_size, + ) + sched = fix.scheduler + gpu_pool = fix.gpu_block_pool + assert prompt_tokens % sched.fa_block_size != 0 + + producer = _make_cp_request(num_blocks=5, virtual_block_size=hash_block_size) + attention_blocks = gpu_pool.get_new_blocks(1) + gpu_pool.cache_partial_block( + request=producer, + block=attention_blocks[0], + num_tokens=prompt_tokens, + kv_cache_group_id=0, + block_size=scheduler_block_size, + ) + mamba_blocks = gpu_pool.get_new_blocks(5) + gpu_pool.cache_full_blocks( + request=producer, + blocks=mamba_blocks, + num_cached_blocks=0, + num_full_blocks=5, + kv_cache_group_id=1, + block_size=mamba_block_size, + ) + producer_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks)) + sched.update_state_after_alloc(producer, producer_blocks, num_external_tokens=0) + + # Decode carries the request past the physical attention block boundary, so + # the positional scan now considers the same block the partial tail names. + producer.num_computed_tokens = sched.fa_block_size + sched.request_finished_all_groups(producer, producer_blocks.get_block_ids()) + finish_meta = sched.build_connector_meta(make_scheduler_output({})) + + stored = finish_meta.store_gpu_blocks + assert attention_blocks[0].block_id in stored + assert len(stored) == len(set(stored)) + assert len(finish_meta.store_cpu_blocks) == len(stored) + + +def test_external_lookup_rejects_misaligned_local_prefix(caplog_vllm) -> None: + scheduler_block_size = 4 * BLOCK_SIZE + fix = _make_hybrid_attention_mamba_scheduler(block_size=scheduler_block_size) + + request = _make_cp_request(num_blocks=2, virtual_block_size=scheduler_block_size) + # The ``vllm`` logger sets propagate=False, so plain ``caplog`` can observe + # nothing; ``caplog_vllm`` re-enables propagation for the duration. + with caplog_vllm.at_level(logging.WARNING): + hit_tokens, is_async = fix.scheduler.get_num_new_matched_tokens( + request, num_computed_tokens=BLOCK_SIZE + ) + + assert (hit_tokens, is_async) == (0, False) + assert "requires scheduler-block-aligned local tokens" in caplog_vllm.text + + +def test_fine_grained_external_hits_require_eager_offload() -> None: + hash_block_size = BLOCK_SIZE + scheduler_block_size = 4 * BLOCK_SIZE + + eager = _make_hybrid_attention_mamba_scheduler( + block_size=scheduler_block_size, + hash_block_size=hash_block_size, + ) + lazy = _make_hybrid_attention_mamba_scheduler( + block_size=scheduler_block_size, + hash_block_size=hash_block_size, + lazy=True, + ) + + assert eager.scheduler.cpu_coordinator.enable_partial_hash_hits + assert not lazy.scheduler.cpu_coordinator.enable_partial_hash_hits + + +def test_eager_store_does_not_scan_mamba_blocks_positionally() -> None: + """Mamba stores must use exact handoffs, not historical block positions.""" + block_size = 4 * BLOCK_SIZE + fix = _make_hybrid_attention_mamba_scheduler( + num_cpu_blocks=16, num_gpu_blocks=24, block_size=block_size + ) + sched = fix.scheduler + gpu_pool = fix.gpu_block_pool + attention_manager, mamba_manager = sched.cpu_coordinator.single_type_managers + assert attention_manager.has_positionally_stable_blocks + assert not mamba_manager.has_positionally_stable_blocks + + req = _make_cp_request(num_blocks=2, virtual_block_size=block_size) + attn_blocks = _allocate_cp_gpu_blocks( + gpu_pool, req, 2, virtual_block_size=block_size, group_id=0 + ) + gpu_blocks = gpu_pool.get_new_blocks(2) + gpu_blocks[1].set_block_hash(make_block_hash_with_group_id(req.block_hashes[1], 1)) + kv_blocks = KVCacheBlocks(blocks=(attn_blocks, gpu_blocks)) + req.num_computed_tokens = 0 + sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) + + output = make_scheduler_output( + {req.request_id: 2 * block_size}, + new_reqs={req.request_id: kv_blocks.get_block_ids()}, + ) + output.kv_connector_block_state = KVConnectorBlockState( + req_ids=set(), + resolve_block_ids=lambda _: (), + boundary_state_offloads={ + req.request_id: [(1, gpu_blocks[1].block_id, 2 * block_size)] + }, + ) + meta1 = sched.build_connector_meta(output) + assert meta1.store_event >= 0 + # Only the exact handoff lands this step. build_connector_meta runs before + # the scheduler commits this step's tokens, so the positional scan has + # nothing confirmed yet. + assert set(meta1.store_gpu_blocks) == {gpu_blocks[1].block_id} + assert sched._reqs_to_store[req.request_id].num_stored_blocks == [0, 0] + simulate_store_completion(sched, meta1.store_event) + + # Next step the attention blocks are confirmed and scanned positionally. A + # later hash update still must not make the mamba align snapshot a valid + # source; the explicit boundary-handoff tests cover the supported path. + gpu_blocks[0].set_block_hash(make_block_hash_with_group_id(req.block_hashes[0], 1)) + req.num_computed_tokens = 2 * block_size + meta2 = sched.build_connector_meta( + make_scheduler_output( + {req.request_id: 1}, + cached_req_new_blocks={req.request_id: None}, + ) + ) + assert set(meta2.store_gpu_blocks) == {block.block_id for block in attn_blocks} + assert sched._reqs_to_store[req.request_id].num_stored_blocks == [2, 0] + simulate_store_completion(sched, meta2.store_event) + + # Nothing further: mamba align is never picked up positionally. + meta3 = sched.build_connector_meta( + make_scheduler_output( + {req.request_id: 1}, + cached_req_new_blocks={req.request_id: None}, + ) + ) + assert meta3.store_event == -1 + assert meta3.store_gpu_blocks == [] + assert sched._reqs_to_store[req.request_id].num_stored_blocks == [2, 0] + + +def test_eager_store_scans_mamba_all_blocks_positionally() -> None: + """Mamba all-mode retains positional identity and needs no handoff.""" + block_size = 4 * BLOCK_SIZE + fix = _make_hybrid_attention_mamba_scheduler( + num_cpu_blocks=16, + num_gpu_blocks=24, + block_size=block_size, + mamba_cache_mode="all", + ) + sched = fix.scheduler + gpu_pool = fix.gpu_block_pool + attention_manager, mamba_manager = sched.cpu_coordinator.single_type_managers + assert attention_manager.has_positionally_stable_blocks + assert mamba_manager.has_positionally_stable_blocks + + req = _make_cp_request(num_blocks=2, virtual_block_size=block_size) + attention_blocks = _allocate_cp_gpu_blocks( + gpu_pool, req, 2, virtual_block_size=block_size, group_id=0 + ) + mamba_blocks = _allocate_cp_gpu_blocks( + gpu_pool, req, 2, virtual_block_size=block_size, group_id=1 + ) + kv_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks)) + sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) + + # build_connector_meta runs before the scheduler commits this step's + # tokens, so the positional scan picks the blocks up on the next step. + first = sched.build_connector_meta( + make_scheduler_output( + {req.request_id: 2 * block_size}, + new_reqs={req.request_id: kv_blocks.get_block_ids()}, + ) + ) + assert first.store_gpu_blocks == [] + req.num_computed_tokens = 2 * block_size + meta = sched.build_connector_meta( + make_scheduler_output( + {req.request_id: 1}, + cached_req_new_blocks={req.request_id: None}, + ) + ) + + assert set(meta.store_gpu_blocks) == { + *(block.block_id for block in attention_blocks), + *(block.block_id for block in mamba_blocks), + } + assert sched._reqs_to_store[req.request_id].num_stored_blocks == [2, 2] + + +@pytest.mark.parametrize("gone_via", ["preempted", "finished", "unregistered"]) +def test_boundary_handoff_dropped_for_departing_request(gone_via: str) -> None: + """Handoffs for a request leaving this step must not be read. + + The scheduler drains boundary handoffs without filtering by request + liveness, so a preempted or finished request can still offer block IDs. + Those blocks are being released and may already back another request, so + copying them would publish unrelated KV under a valid hash. + """ + block_size = 4 * BLOCK_SIZE + fix = _make_hybrid_attention_mamba_scheduler( + num_cpu_blocks=16, num_gpu_blocks=24, block_size=block_size + ) + sched = fix.scheduler + gpu_pool = fix.gpu_block_pool + + req = _make_cp_request(num_blocks=2, virtual_block_size=block_size) + attn_blocks = _allocate_cp_gpu_blocks( + gpu_pool, req, 2, virtual_block_size=block_size, group_id=0 + ) + mamba_blocks = gpu_pool.get_new_blocks(2) + mamba_blocks[1].set_block_hash( + make_block_hash_with_group_id(req.block_hashes[1], 1) + ) + kv_blocks = KVCacheBlocks(blocks=(attn_blocks, mamba_blocks)) + sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) + + output = make_scheduler_output( + {req.request_id: 2 * block_size}, + new_reqs={req.request_id: kv_blocks.get_block_ids()}, + ) + output.kv_connector_block_state = KVConnectorBlockState( + req_ids=set(), + resolve_block_ids=lambda _: (), + boundary_state_offloads={ + req.request_id: [(1, mamba_blocks[1].block_id, 2 * block_size)] + }, + ) + if gone_via == "preempted": + output.preempted_req_ids = {req.request_id} + elif gone_via == "finished": + output.finished_req_ids = {req.request_id} + else: + sched._reqs_to_store.pop(req.request_id) + + meta = sched.build_connector_meta(output) + + # The exact-handoff block is never a store source for a departing request. + assert mamba_blocks[1].block_id not in meta.store_gpu_blocks + stats = sched.get_boundary_store_stats() + assert stats.dropped_request_gone == 1 + assert stats.stored == 0 + + +def test_boundary_handoff_keeps_block_meta_index_parallel() -> None: + """Boundary stores must extend block_meta alongside the block ids. + + ``_process_store_completion`` indexes ``block_meta`` by position, so a + boundary handoff that appends only block ids shortens the list, trips its + length assertion and shifts every following entry's metadata. + """ + block_size = 4 * BLOCK_SIZE + fix = _make_hybrid_attention_mamba_scheduler( + num_cpu_blocks=16, + num_gpu_blocks=24, + block_size=block_size, + enable_kv_cache_events=True, + ) + sched = fix.scheduler + gpu_pool = fix.gpu_block_pool + assert sched.enable_kv_cache_events + + req = _make_cp_request(num_blocks=2, virtual_block_size=block_size) + attn_blocks = _allocate_cp_gpu_blocks( + gpu_pool, req, 2, virtual_block_size=block_size, group_id=0 + ) + mamba_blocks = gpu_pool.get_new_blocks(2) + mamba_blocks[1].set_block_hash( + make_block_hash_with_group_id(req.block_hashes[1], 1) + ) + kv_blocks = KVCacheBlocks(blocks=(attn_blocks, mamba_blocks)) + sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) + # Make the FA blocks positionally eligible in the same event as the + # explicit Mamba boundary handoff. + req.num_computed_tokens = 2 * block_size + + output = make_scheduler_output( + {req.request_id: 2 * block_size}, + new_reqs={req.request_id: kv_blocks.get_block_ids()}, + ) + output.kv_connector_block_state = KVConnectorBlockState( + req_ids=set(), + resolve_block_ids=lambda _: (), + boundary_state_offloads={ + req.request_id: [(1, mamba_blocks[1].block_id, 2 * block_size)] + }, + ) + + gpu_ids, cpu_ids, req_ids, block_meta = sched.prepare_store_specs(output) + assert mamba_blocks[1].block_id in gpu_ids + assert set(gpu_ids) == { + *(block.block_id for block in attn_blocks), + mamba_blocks[1].block_id, + } + assert req_ids == [req.request_id] + assert block_meta is not None + assert len(block_meta) == len(gpu_ids) == len(cpu_ids) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py index 0589c83232..64bf8146c9 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py @@ -19,6 +19,7 @@ from vllm.logger import init_logger from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.outputs import KVConnectorOutput from vllm.v1.simple_kv_offload.manager import ( + BoundaryStoreStats, SimpleCPUOffloadScheduler, ) from vllm.v1.simple_kv_offload.metadata import ( @@ -300,6 +301,12 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA): return self.scheduler_manager.has_pending_stores() return False + def get_boundary_store_stats(self) -> BoundaryStoreStats | None: + """Return cumulative boundary-handoff store diagnostics.""" + if self.scheduler_manager is not None: + return self.scheduler_manager.get_boundary_store_stats() + return None + def take_events(self) -> Iterable[KVCacheEvent]: if self.scheduler_manager is not None: return self.scheduler_manager.take_events() diff --git a/vllm/v1/core/kv_cache_coordinator.py b/vllm/v1/core/kv_cache_coordinator.py index a495f997bf..5fa8812924 100644 --- a/vllm/v1/core/kv_cache_coordinator.py +++ b/vllm/v1/core/kv_cache_coordinator.py @@ -155,6 +155,9 @@ class KVCacheCoordinator(ABC): for manager in self.single_type_managers: if isinstance(manager, MambaManager): manager.drop_eagle_checkpoint_block = True + self.group_block_sizes = tuple( + manager.block_size for manager in self.single_type_managers + ) # A positive retention interval must be a multiple of the base hit granularity # (``scheduler_block_size``) to land on real cache-hit boundaries. @@ -605,6 +608,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): hash_block_size: int, metrics_collector: KVCacheMetricsCollector | None = None, num_prefill_lookahead: int = 0, + allow_partial_hash_hits: bool = True, ): super().__init__( kv_cache_config, @@ -629,7 +633,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): # Only groups that participate in prefix caching must satisfy the # divisibility constraint; groups that opt out (e.g. GLM-5.3-Flash kpool # tail, block_size=kpool) are scratch buffers and excluded. - group_block_sizes = [ + cacheable_block_sizes = [ manager.block_size for manager, group in zip( self.single_type_managers, kv_cache_config.kv_cache_groups @@ -637,10 +641,10 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): if group.kv_cache_spec.prefix_cacheable ] assert all( - block_size % hash_block_size == 0 for block_size in group_block_sizes + block_size % hash_block_size == 0 for block_size in cacheable_block_sizes ), ( "Each KV cache group's real block_size must be divisible by " - f"hash_block_size. block_sizes={group_block_sizes}, " + f"hash_block_size. block_sizes={cacheable_block_sizes}, " f"hash_block_size={hash_block_size}" ) assert pcp_world_size == 1, "PCP not support hybrid attn now." @@ -669,7 +673,9 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): ) for g in kv_cache_config.kv_cache_groups ) - self.enable_partial_hash_hits = has_partial_mamba_group + self.enable_partial_hash_hits = ( + allow_partial_hash_hits and has_partial_mamba_group + ) if self.enable_partial_hash_hits: unsupported_partial_hit_managers = { type(manager).__name__ @@ -987,6 +993,7 @@ def get_kv_cache_coordinator( hash_block_size: int, metrics_collector: KVCacheMetricsCollector | None = None, num_prefill_lookahead: int = 0, + allow_partial_hash_hits: bool = True, ) -> KVCacheCoordinator: if not enable_caching: return KVCacheCoordinatorNoPrefixCache( @@ -1030,4 +1037,5 @@ def get_kv_cache_coordinator( hash_block_size=hash_block_size, metrics_collector=metrics_collector, num_prefill_lookahead=num_prefill_lookahead, + allow_partial_hash_hits=allow_partial_hash_hits, ) diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index 15b35d9f7b..93475e5055 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -48,6 +48,16 @@ class SingleTypeKVCacheManager(ABC): supports_fine_grained_hash_lookup: ClassVar[bool] = False + @property + def has_positionally_stable_blocks(self) -> bool: + """Whether positional offload scans can follow this block table. + + True means already-scanned indices will not be reused for a different + token range. Managers may still null blocks that left their retention + window, as long as the cursor's positional history remains valid. + """ + return True + def __init__( self, kv_cache_spec: KVCacheSpec, @@ -1423,6 +1433,12 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager): class MambaManager(SingleTypeKVCacheManager): supports_fine_grained_hash_lookup: ClassVar[bool] = True + @property + def has_positionally_stable_blocks(self) -> bool: + # Align-mode Mamba can null interior states and relocate speculative + # blocks in place. Other modes retain positional identity. + return self.mamba_cache_mode != "align" + def __init__( self, kv_cache_spec: MambaSpec, block_pool: BlockPool, **kwargs ) -> None: diff --git a/vllm/v1/simple_kv_offload/manager.py b/vllm/v1/simple_kv_offload/manager.py index afb51fb1d3..9cae2f714c 100644 --- a/vllm/v1/simple_kv_offload/manager.py +++ b/vllm/v1/simple_kv_offload/manager.py @@ -5,6 +5,7 @@ import contextlib from collections.abc import Iterable from dataclasses import dataclass, field, replace +from enum import Enum, auto from typing import TYPE_CHECKING, Any from vllm.config import VllmConfig @@ -26,6 +27,7 @@ from vllm.v1.core.kv_cache_coordinator import ( from vllm.v1.core.kv_cache_utils import ( BlockHashWithGroupId, ExternalBlockHash, + dcp_world_size_for_kv_cache_spec, generate_block_hash_extra_keys, get_block_hash, get_group_id, @@ -99,6 +101,36 @@ class StoreRequestState: finished: bool = False +class _StoreAdmission(Enum): + READY = auto() + NULL_BLOCK = auto() + NOT_HASHED = auto() + IN_FLIGHT = auto() + ALREADY_CACHED = auto() + + +@dataclass +class BoundaryStoreStats: + """Cumulative counters for boundary hand-off stores. + + Every branch that declines a hand-off is silent by design: the boundary is + simply not offloaded and a later request misses. Without counters the only + symptom is a lower cache hit rate with nothing in the logs, so each decline + reason is counted and exposed through + ``SimpleCPUOffloadConnector.get_boundary_store_stats()``. Reset by + ``reset()``; not otherwise cleared. + """ + + published: int = 0 + stored: int = 0 + dropped_cpu_full: int = 0 + dropped_request_gone: int = 0 + skipped_already_cached: int = 0 + skipped_in_flight: int = 0 + dropped_null_block: int = 0 + dropped_not_hashed: int = 0 + + class SimpleCPUOffloadScheduler: """Scheduler-side manager for CPU offloading.""" @@ -141,16 +173,6 @@ class SimpleCPUOffloadScheduler: self.fa_gidx = g_idx break assert 0 <= self.fa_gidx < len(self.cpu_kv_cache_config.kv_cache_groups) - # FA group's own block_size; divides scheduler_block_size (the LCM) - # but is NOT assumed to equal it. - self.fa_block_size: int = ( - self.cpu_kv_cache_config.kv_cache_groups[ - self.fa_gidx - ].kv_cache_spec.block_size - * self.cp_world_size - ) - assert self.block_size % self.fa_block_size == 0 - logger.info( "SimpleCPUOffloadScheduler: Allocating %d offload blocks " "(%.2f GB, mode=%s, backend=%s)", @@ -175,7 +197,13 @@ class SimpleCPUOffloadScheduler: pcp_world_size=1, scheduler_block_size=self.block_size, hash_block_size=self.hash_block_size, + allow_partial_hash_hits=not lazy_offload, ) + self.group_block_sizes = self.cpu_coordinator.group_block_sizes + # FA group's own resolved block_size; divides scheduler_block_size (the + # LCM) but is NOT assumed to equal it. + self.fa_block_size: int = self.group_block_sizes[self.fa_gidx] + assert self.block_size % self.fa_block_size == 0 self.cpu_block_pool: BlockPool = self.cpu_coordinator.block_pool # GPU block pool reference - bound after scheduler builds kv_cache_manager self._gpu_block_pool: BlockPool | None = None @@ -186,10 +214,13 @@ class SimpleCPUOffloadScheduler: # the worker reports completions by event index, not request id. self._load_event_to_reqs: dict[int, list[str]] = {} - # Pending (cpu_hit_blocks, hit_length) tuples from find_longest_cache_hit, - # kept pinned via touch() while awaiting update_state_after_alloc(). + # Pending (cpu_hit_blocks, hit_length, num_computed_tokens) tuples from + # find_longest_cache_hit, kept pinned via touch() while awaiting + # update_state_after_alloc(). ``num_computed_tokens`` is the local + # prefix the hit was resolved against; the load path needs it to place + # external blocks and must not re-derive it from block metadata. self._pending_cpu_hits: dict[ - str, tuple[tuple[list[KVCacheBlock], ...], int] + str, tuple[tuple[list[KVCacheBlock], ...], int, int] ] = {} # Store metadata @@ -216,6 +247,7 @@ class SimpleCPUOffloadScheduler: # Event counters self._load_event_counter: int = 0 self._store_event_counter: int = 0 + self.boundary_store_stats = BoundaryStoreStats() # For TP/PP: track partial store completions across steps. # Events must be reported by all world_size workers before considered complete. @@ -267,7 +299,11 @@ class SimpleCPUOffloadScheduler: target = 0 for g in kv_cache_config.kv_cache_groups: spec = g.kv_cache_spec - block_size = spec.block_size * cp_world_size + # Only full attention is sharded across DCP ranks; replicated specs + # (mamba, sliding window, chunked-local) keep their own block size. + block_size = spec.block_size * dcp_world_size_for_kv_cache_spec( + spec, cp_world_size + ) if isinstance(spec, MambaSpec): target += 2 elif isinstance(spec, SlidingWindowSpec): @@ -296,6 +332,20 @@ class SimpleCPUOffloadScheduler: if request.skip_reading_prefix_cache: return 0, False + if num_computed_tokens % self.block_size != 0: + # Transfers are whole-block copies, so an external suffix cannot + # start in the middle of a destination block. When the GPU-side + # coordinator also serves fine-grained hits, the local prefix can + # land on a hash boundary that is not a scheduler-block boundary; + # such a request keeps its local hit and skips the external one. + logger.warning_once( + "SimpleCPU external lookup requires scheduler-block-aligned " + "local tokens, got %d tokens with scheduler block size %d.", + num_computed_tokens, + self.block_size, + ) + return 0, False + num_skipped_hashes = num_computed_tokens // self.hash_block_size remaining_hashes = request.block_hashes[num_skipped_hashes:] @@ -318,6 +368,7 @@ class SimpleCPUOffloadScheduler: self._pending_cpu_hits[request.request_id] = ( cpu_hit_blocks, hit_length, + num_computed_tokens, ) return hit_length, True return 0, False @@ -371,20 +422,19 @@ class SimpleCPUOffloadScheduler: ) return - cpu_hit_blocks_full, _ = pending + cpu_hit_blocks_full, _, num_computed_tokens = pending - # ``num_external_tokens`` is LCM-aligned (checked per-group below), - # so this counts whole scheduler-aligned chunks of incoming tokens. - num_blocks_to_load = num_external_tokens // self.block_size - assert num_blocks_to_load > 0 - num_cached_fa_blocks = sum( - blk.block_hash is not None for blk in blocks.blocks[self.fa_gidx] + assert num_computed_tokens % self.block_size == 0, ( + "SimpleCPU external loads must start at a scheduler-block boundary" + ) + # Fine-grained hits are hash-block aligned, not group-block aligned, so + # the per-group block count below rounds up instead of dividing exactly. + # Hash alignment is still required: it is what makes the accepted + # external suffix start on a boundary every group can address. + assert num_external_tokens % self.hash_block_size == 0, ( + f"num_external_tokens={num_external_tokens} is not aligned to " + f"hash_block_size={self.hash_block_size}" ) - num_computed_tokens = num_cached_fa_blocks * self.fa_block_size - - # Build transfer pairs across all groups. - total_computed_tokens = num_computed_tokens + num_external_tokens - kv_cache_groups = self.cpu_kv_cache_config.kv_cache_groups # The scheduler may have accepted fewer blocks than # get_num_new_matched_tokens() reported. @@ -393,14 +443,8 @@ class SimpleCPUOffloadScheduler: # the rest will be released along with the temp pin below. cpu_hit_blocks: list[list[KVCacheBlock]] = [] for g in range(num_groups): - g_block_size = ( - kv_cache_groups[g].kv_cache_spec.block_size * self.cp_world_size - ) - assert num_external_tokens % g_block_size == 0, ( - f"num_external_tokens={num_external_tokens} not aligned to " - f"group {g} block_size={g_block_size}" - ) - n_take_g = num_external_tokens // g_block_size + g_block_size = self.group_block_sizes[g] + n_take_g = cdiv(num_external_tokens, g_block_size) cpu_hit_blocks.append(cpu_hit_blocks_full[g][:n_take_g]) gpu_block_ids: list[int] = [] @@ -413,14 +457,8 @@ class SimpleCPUOffloadScheduler: if n_ext_g == 0: continue - # Number of blocks in the computed range for this group. - g_block_size = ( - kv_cache_groups[g].kv_cache_spec.block_size * self.cp_world_size - ) - n_computed_g = cdiv(total_computed_tokens, g_block_size) - - # Back-trace: ext blocks sit at the tail of the computed range. - gpu_ext_start = n_computed_g - n_ext_g + g_block_size = self.group_block_sizes[g] + gpu_ext_start = num_computed_tokens // g_block_size group_gpu_ids = block_ids_by_group[g] for i, cpu_blk in enumerate(cpu_blocks_g): @@ -614,6 +652,72 @@ class SimpleCPUOffloadScheduler: num_groups = len(self.cpu_kv_cache_config.kv_cache_groups) # Dedup against blocks already scheduled. in_flight = self._in_flight_store_gpu_blocks + num_free = cpu_block_pool.get_num_free_blocks() + + block_state = scheduler_output.kv_connector_block_state + boundary_offloads = ( + block_state.boundary_state_offloads if block_state is not None else {} + ) + stats = self.boundary_store_stats + preempted_req_ids = scheduler_output.preempted_req_ids or () + for req_id, entries in boundary_offloads.items(): + # The scheduler drains handoffs without filtering by request + # liveness. A request that finished or was preempted in this step + # is giving up its blocks, which may already have been reallocated + # to another request; reading them would publish unrelated KV under + # a valid hash. Drop conservatively, as the mooncake store does. + store_state = self._reqs_to_store.get(req_id) + if ( + store_state is None + or store_state.finished + or req_id in preempted_req_ids + or req_id in scheduler_output.finished_req_ids + ): + stats.published += len(entries) + stats.dropped_request_gone += len(entries) + continue + scheduled_for_req = False + # ``boundary_tokens`` is not needed here: the cache key comes from + # the handed-off block itself, not from the offered boundary. + for _, gpu_block_id, _ in entries: + stats.published += 1 + gpu_block = gpu_block_pool.blocks[gpu_block_id] + admission = self._classify_store_candidate(gpu_block) + if admission is _StoreAdmission.NULL_BLOCK: + stats.dropped_null_block += 1 + continue + if admission is _StoreAdmission.NOT_HASHED: + stats.dropped_not_hashed += 1 + continue + if admission is _StoreAdmission.IN_FLIGHT: + stats.skipped_in_flight += 1 + continue + if admission is _StoreAdmission.ALREADY_CACHED: + stats.skipped_already_cached += 1 + continue + if num_free <= 0: + stats.dropped_cpu_full += 1 + logger.warning_once( + "SimpleCPU dropped a boundary-state handoff because " + "the CPU block pool is full; this boundary will not be retried." + ) + continue + cpu_block = cpu_block_pool.get_new_blocks(1)[0] + merged_gpu_block_ids.append(gpu_block_id) + merged_cpu_block_ids.append(cpu_block.block_id) + if merged_block_meta is not None: + # Keep the metadata list index-parallel with the block ids. + # The hand-off block's own hashes are resolved at completion, + # so there is nothing to capture here; emit the event without + # a payload, as the size-mismatch fallback already does. + merged_block_meta.append({}) + in_flight.add(gpu_block_id) + gpu_block_pool.touch([gpu_block]) + num_free -= 1 + stats.stored += 1 + scheduled_for_req = True + if scheduled_for_req: + req_ids.append(req_id) for req_id, new_block_id_groups, preempted in yield_req_data(scheduler_output): state = self._reqs_to_store.get(req_id) @@ -641,7 +745,7 @@ class SimpleCPUOffloadScheduler: self._select_eager_blocks_to_store(state, block_ids_by_group) ) - # --- Phase 2: Batch allocate CPU blocks --- + # Batch allocate the CPU destinations. n_to_alloc = len(gpu_block_ids) if n_to_alloc > 0: cpu_blocks_alloc = cpu_block_pool.get_new_blocks(n_to_alloc) @@ -674,6 +778,9 @@ class SimpleCPUOffloadScheduler: for g in range(num_groups): state.num_stored_blocks[g] += advanced_per_group[g] + # A request can contribute both boundary and positional blocks to the + # same event. Completion tracking is set-based, so keep one request ID. + req_ids = list(dict.fromkeys(req_ids)) return merged_gpu_block_ids, merged_cpu_block_ids, req_ids, merged_block_meta def _select_eager_blocks_to_store( @@ -694,19 +801,31 @@ class SimpleCPUOffloadScheduler: ) request = state.request confirmed_tokens = request.num_computed_tokens - request.num_output_placeholders - aligned_tokens = confirmed_tokens // self.block_size * self.block_size + # Truncate to the granularity a lookup can actually land on. With + # fine-grained hits that is hash_block_size; otherwise hits only land on + # the scheduler block (the group LCM). Using the LCM unconditionally + # would drop whole blocks that sit between the last LCM boundary and a + # reachable fine-grained boundary, so the other groups would hold that + # boundary and this one would not, and the joint hybrid lookup would + # reconcile to zero. + store_alignment = ( + self.hash_block_size + if self.cpu_coordinator.enable_partial_hash_hits + else self.block_size + ) + aligned_tokens = confirmed_tokens // store_alignment * store_alignment num_free = self.cpu_block_pool.get_num_free_blocks() for g, group_gpu_ids in enumerate(block_ids_by_group): if len(gpu_block_ids) >= num_free: break + group_manager = self.cpu_coordinator.single_type_managers[g] + if not group_manager.has_positionally_stable_blocks: + continue # FIXME (yifan): handle CPU cache eviction, where # num_stored_blocks can be stale and omit evicted blocks in # the middle of the request. - group_size = ( - self.cpu_kv_cache_config.kv_cache_groups[g].kv_cache_spec.block_size - * self.cp_world_size - ) + group_size = self.group_block_sizes[g] ready = min(len(group_gpu_ids), aligned_tokens // group_size) resolved_hashes = ( resolve_block_hashes( @@ -720,20 +839,16 @@ class SimpleCPUOffloadScheduler: start = state.num_stored_blocks[g] for i, gpu_block_id in enumerate(group_gpu_ids[start:ready], start=start): gpu_block = self._gpu_block_pool.blocks[gpu_block_id] - if gpu_block.is_null or gpu_block.block_hash is None: - advanced_per_group[g] += 1 - continue if ( - gpu_block_id in self._in_flight_store_gpu_blocks - or self.cpu_block_pool.cached_block_hash_to_block.get_one_block( - gpu_block.block_hash - ) - is not None + self._classify_store_candidate(gpu_block) + is not _StoreAdmission.READY ): advanced_per_group[g] += 1 continue if len(gpu_block_ids) >= num_free: break + primary_block_hash = gpu_block.block_hash + assert primary_block_hash is not None gpu_block_ids.append(gpu_block_id) advanced_per_group[g] += 1 if block_meta is not None: @@ -750,7 +865,7 @@ class SimpleCPUOffloadScheduler: request, token_start, token_end, curr_mm_idx ) meta_by_hash = { - gpu_block.block_hash: BlockStoreMeta( + primary_block_hash: BlockStoreMeta( token_ids=list( request.all_token_ids[token_start:token_end] ), @@ -766,7 +881,7 @@ class SimpleCPUOffloadScheduler: block_hash = make_block_hash_with_group_id( request.block_hashes[hash_idx], g ) - if block_hash == gpu_block.block_hash: + if block_hash is None or block_hash == primary_block_hash: continue hash_start = hash_idx * self.hash_block_size hash_end = hash_start + self.hash_block_size @@ -796,6 +911,25 @@ class SimpleCPUOffloadScheduler: return gpu_block_ids, advanced_per_group, block_meta + def _classify_store_candidate(self, gpu_block: "KVCacheBlock") -> _StoreAdmission: + if gpu_block.is_null: + return _StoreAdmission.NULL_BLOCK + if gpu_block.block_hash is None: + return _StoreAdmission.NOT_HASHED + if gpu_block.block_id in self._in_flight_store_gpu_blocks: + return _StoreAdmission.IN_FLIGHT + if ( + self.cpu_block_pool.cached_block_hash_to_block.get_one_block( + gpu_block.block_hash + ) + is not None + ): + return _StoreAdmission.ALREADY_CACHED + return _StoreAdmission.READY + + def get_boundary_store_stats(self) -> BoundaryStoreStats: + return replace(self.boundary_store_stats) + def update_connector_output(self, connector_output: KVConnectorOutput) -> None: """Handle async transfer completions from worker. @@ -886,12 +1020,7 @@ class SimpleCPUOffloadScheduler: if self.enable_kv_cache_events: meta_by_hash = block_meta[i] if block_meta is not None else {} primary_group_idx = get_group_id(primary_hash) - primary_spec = self.cpu_kv_cache_config.kv_cache_groups[ - primary_group_idx - ].kv_cache_spec - primary_block_size = primary_spec.block_size * ( - self.cp_world_size if not isinstance(primary_spec, MambaSpec) else 1 - ) + primary_block_size = self.group_block_sizes[primary_group_idx] events_to_emit: list[ tuple[BlockHashWithGroupId, BlockStoreMeta | None, int] ] = [ @@ -1028,6 +1157,21 @@ class SimpleCPUOffloadScheduler: if state is None or gpu_pool is None: return gpu_ids, _, block_meta = self._select_eager_blocks_to_store(state, block_ids) + partial_tail = self._find_fa_partial_tail_source(request, block_ids) + if ( + partial_tail is not None + # Decode tokens can fill the boundary block, in which case the + # positional scan above already selected it. Appending again would + # allocate two CPU blocks for one GPU block. + and partial_tail not in gpu_ids + and len(gpu_ids) < self.cpu_block_pool.get_num_free_blocks() + ): + gpu_ids.append(partial_tail) + if block_meta is not None: + # The block's own hashes are registered at completion, so no + # capture is needed here; emit the event without a payload, as + # the size-mismatch fallback above already does. + block_meta.append({}) if not gpu_ids: return cpu_blocks = self.cpu_block_pool.get_new_blocks(len(gpu_ids)) @@ -1037,9 +1181,64 @@ class SimpleCPUOffloadScheduler: self._in_flight_store_gpu_blocks.update(gpu_ids) gpu_pool.touch([gpu_pool.blocks[bid] for bid in gpu_ids]) - def _free_pending_cpu_hit(self, pending: tuple) -> None: + def _find_fa_partial_tail_source( + self, request: "Request", block_ids: tuple[list[int], ...] + ) -> int | None: + """Locate the full-attention prompt-tail block for a fine-grained hit. + + Mirrors ``FullAttentionManager._cache_partial_tail_block``: only the + final prompt hash boundary is eligible, and boundaries that land on a + physical block edge are already covered by the positional scan. + + Attention block tables are append-only, so the block is located + positionally, as the mooncake store and offloading connector also do; + no new core hand-off is required. The boundary key is registered at + completion along with the block's other hashes, so nothing has to be + captured here. + """ + if not self.cpu_coordinator.enable_partial_hash_hits: + return None + assert self._gpu_block_pool is not None + boundary_tokens = ( + request.num_prompt_tokens // self.hash_block_size * self.hash_block_size + ) + if boundary_tokens == 0 or boundary_tokens > request.num_computed_tokens: + return None + if boundary_tokens % self.fa_block_size == 0: + return None + block_idx = boundary_tokens // self.fa_block_size + fa_block_ids = block_ids[self.fa_gidx] + if block_idx >= len(fa_block_ids): + return None + gpu_block_id = fa_block_ids[block_idx] + gpu_block = self._gpu_block_pool.blocks[gpu_block_id] + if gpu_block.is_null or gpu_block_id in self._in_flight_store_gpu_blocks: + return None + hash_idx = boundary_tokens // self.hash_block_size - 1 + if hash_idx >= len(request.block_hashes): + return None + block_hash = make_block_hash_with_group_id( + request.block_hashes[hash_idx], self.fa_gidx + ) + # Only worth copying when the boundary key is actually registered on + # the source block; completion replays the block's own hashes. + if ( + gpu_block.block_hash != block_hash + and block_hash + not in self._gpu_block_pool.cached_block_hashes_by_block.get( + gpu_block_id, () + ) + ): + return None + if self.cpu_block_pool.cached_block_hash_to_block.get_one_block(block_hash): + return None + return gpu_block_id + + def _free_pending_cpu_hit( + self, pending: tuple[tuple[list["KVCacheBlock"], ...], int, int] + ) -> None: """Release the temporary CPU block pin taken in get_num_new_matched_tokens().""" - cpu_hit_blocks, _ = pending + cpu_hit_blocks, _hit_length, _num_computed_tokens = pending blocks_to_free = [ blk for grp in cpu_hit_blocks for blk in grp if not blk.is_null ] @@ -1140,6 +1339,7 @@ class SimpleCPUOffloadScheduler: if event_idx in self._abandoned_store_event_to_blocks } self._cursor = None + self.boundary_store_stats = BoundaryStoreStats() # NOTE: _load_event_counter / _store_event_counter are not # reset as they are monotonic and must stay ahead of the workers # high-water marks to avoid event index collisions