diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index 5cbbef1df5..b782e34b01 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -25,14 +25,17 @@ from vllm.multimodal.inputs import ( from vllm.sampling_params import SamplingParams, StructuredOutputsParams from vllm.utils.hashing import sha256 from vllm.v1.core.encoder_cache_manager import EncoderCacheManager +from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput from vllm.v1.core.sched.scheduler import Scheduler +from vllm.v1.core.single_type_kv_cache_manager import register_all_kvcache_specs from vllm.v1.engine import FinishReason from vllm.v1.kv_cache_interface import ( FullAttentionSpec, KVCacheConfig, KVCacheGroupSpec, + MambaSpec, ) from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput from vllm.v1.request import Request, RequestStatus @@ -5464,3 +5467,197 @@ def test_async_load_reservation_prevents_wedge_e2e(): assert b.status == RequestStatus.WAITING assert b.num_preemptions == 0 assert b.request_id not in req_to_blocks + + +def _create_hybrid_mamba_connector_scheduler( + matched_tokens: int, + block_size: int = 16, + num_blocks: int = 100, +) -> Scheduler: + """FA + Mamba ("all" cache mode) scheduler with a MockKVConnector.""" + model_config = ModelConfig( + model="facebook/opt-125m", + trust_remote_code=True, + dtype="float16", + seed=42, + skip_tokenizer_init=True, + ) + vllm_config = VllmConfig( + scheduler_config=SchedulerConfig( + max_num_seqs=4, + max_num_batched_tokens=8192, + max_model_len=8192, + enable_chunked_prefill=True, + is_encoder_decoder=False, + watermark=0.0, + ), + model_config=model_config, + cache_config=CacheConfig( + block_size=block_size, + enable_prefix_caching=True, + mamba_cache_mode="all", + ), + kv_transfer_config=KVTransferConfig( + kv_connector="MockKVConnector", + kv_role="kv_both", + kv_connector_extra_config={ + "matched_tokens": matched_tokens, + "is_async": False, + }, + ), + ) + vllm_config.cache_config.num_gpu_blocks = num_blocks + kv_cache_config = KVCacheConfig( + num_blocks=num_blocks, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["fa"], + FullAttentionSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + ), + KVCacheGroupSpec( + ["mamba"], + MambaSpec( + block_size=block_size, + shapes=((1, 1),), + dtypes=(torch.float32,), + mamba_cache_mode="all", + ), + ), + ], + ) + register_all_kvcache_specs(vllm_config) + return Scheduler( + vllm_config=vllm_config, + kv_cache_config=kv_cache_config, + structured_output_manager=StructuredOutputManager(vllm_config), + block_size=block_size, + hash_block_size=block_size, + log_stats=True, + ) + + +@pytest.mark.parametrize( + "matched_tokens,expected_num_computed", + [ + # No external hit: resume on the deepest locally-consistent boundary + # (block 0's state survives for both groups). + (0, 16), + # One external block on top of the reconciled local boundary. + (16, 32), + ], +) +def test_hybrid_per_group_hit_divergence_with_connector( + matched_tokens: int, expected_num_computed: int +): + """Per-group prefix hits can diverge for hybrid models with a connector + (#46453): under block pressure the FA prefix tail is evicted while a + deeper Mamba state block survives. The scheduler must not report the + deeper hit as locally computed (evicted FA blocks are not resident -> + engine crash / dirty KV); it falls back to the reconciled boundary that + every group is consistent at. + """ + block_size = 16 + scheduler = _create_hybrid_mamba_connector_scheduler(matched_tokens) + manager = scheduler.kv_cache_manager + assert isinstance(manager.coordinator, HybridKVCacheCoordinator) + + # Seed a 4-block prefix so both groups cache all four boundaries + # (mamba cache mode "all" caches every block's state densely). + [fill] = create_requests( + num_requests=1, + num_tokens=4 * block_size, + max_tokens=1, + same_prompt=True, + block_size=block_size, + req_ids=["fill"], + ) + computed_blocks, num_computed, _ = manager.get_computed_blocks(fill) + blocks = manager.allocate_slots( + fill, fill.num_tokens, num_computed, computed_blocks + ) + fa_ids = [b.block_id for b in blocks.blocks[0]] + mamba_ids = [b.block_id for b in blocks.blocks[1]] + manager.free(fill) + + # Evict the FA tail and the middle mamba states; block 0 (both groups) + # and the deep mamba state at block 3 survive. + manager.block_pool.evict_blocks({fa_ids[2], fa_ids[3], mamba_ids[1], mamba_ids[2]}) + + # A replay of the prefix plus one extra block now sees diverged + # per-group hits: FA stops at the evicted tail, while the mamba lookup + # finds the deeper surviving state. + [replay] = create_requests( + num_requests=1, + num_tokens=5 * block_size, + max_tokens=1, + same_prompt=True, + block_size=block_size, + req_ids=["replay"], + ) + _, per_group_hits = manager.coordinator.find_longest_cache_hit_per_group( + replay.block_hashes, replay.num_tokens - 1 + ) + assert per_group_hits == (2 * block_size, 4 * block_size) # diverged + + scheduler.add_request(replay) + output = scheduler.schedule() + num_scheduled = output.num_scheduled_tokens[replay.request_id] + assert replay.num_tokens - num_scheduled == expected_num_computed + + +def test_hybrid_per_group_hit_divergence_fa_deeper_no_external(): + """The opposite divergence: the FA prefix survives deeper than the Mamba + state and the connector supplies nothing (ext == 0). Reporting the deep FA + hit as locally computed would resume with no valid Mamba state at that + boundary (silent bad output). The scheduler must fall back to the + convergent boundary that every group agrees on (block 0's surviving state). + """ + block_size = 16 + scheduler = _create_hybrid_mamba_connector_scheduler(matched_tokens=0) + manager = scheduler.kv_cache_manager + assert isinstance(manager.coordinator, HybridKVCacheCoordinator) + + # Seed a 4-block prefix in both groups. + [fill] = create_requests( + num_requests=1, + num_tokens=4 * block_size, + max_tokens=1, + same_prompt=True, + block_size=block_size, + req_ids=["fill"], + ) + computed_blocks, num_computed, _ = manager.get_computed_blocks(fill) + blocks = manager.allocate_slots( + fill, fill.num_tokens, num_computed, computed_blocks + ) + mamba_ids = [b.block_id for b in blocks.blocks[1]] + manager.free(fill) + + # Keep all FA blocks; evict every mamba state but block 0. FA reaches 4 + # blocks, the mamba hit only reaches 1 -> diverged (FA > Mamba). + manager.block_pool.evict_blocks({mamba_ids[1], mamba_ids[2], mamba_ids[3]}) + + [replay] = create_requests( + num_requests=1, + num_tokens=5 * block_size, + max_tokens=1, + same_prompt=True, + block_size=block_size, + req_ids=["replay"], + ) + _, per_group_hits = manager.coordinator.find_longest_cache_hit_per_group( + replay.block_hashes, replay.num_tokens - 1 + ) + assert per_group_hits == (4 * block_size, 1 * block_size) # FA deeper + + scheduler.add_request(replay) + output = scheduler.schedule() + num_scheduled = output.num_scheduled_tokens[replay.request_id] + # Must resume at the convergent boundary (block 0), not the deep FA hit. + assert replay.num_tokens - num_scheduled == block_size diff --git a/vllm/v1/core/kv_cache_coordinator.py b/vllm/v1/core/kv_cache_coordinator.py index df8769c3f3..70cd267506 100644 --- a/vllm/v1/core/kv_cache_coordinator.py +++ b/vllm/v1/core/kv_cache_coordinator.py @@ -634,6 +634,15 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): key=lambda g: not isinstance(g.spec, FullAttentionSpec) ) + # Dense reference group for per-group lookups (None when the model + # has no full-attention layers): full attention is downward-closed, + # so any group reporting a longer per-group hit implies the union of + # per-group hits is not consistent at a single boundary (#46453). + first = self.attention_groups[0] + self.full_attention_group_id: int | None = ( + first.group_ids[0] if isinstance(first.spec, FullAttentionSpec) else None + ) + # Propagate the eagle bit to each manager (default to ``use_eagle=False``). for group in self.attention_groups: if group.use_eagle: diff --git a/vllm/v1/core/kv_cache_manager.py b/vllm/v1/core/kv_cache_manager.py index cc269e6ff1..e93a1ce640 100644 --- a/vllm/v1/core/kv_cache_manager.py +++ b/vllm/v1/core/kv_cache_manager.py @@ -9,7 +9,10 @@ from typing import Literal, overload from vllm.distributed.kv_events import BlockStored, KVCacheEvent from vllm.logger import init_logger from vllm.utils.math_utils import cdiv -from vllm.v1.core.kv_cache_coordinator import get_kv_cache_coordinator +from vllm.v1.core.kv_cache_coordinator import ( + HybridKVCacheCoordinator, + get_kv_cache_coordinator, +) from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector from vllm.v1.core.kv_cache_utils import KVCacheBlock, KVCacheBlockCopy from vllm.v1.kv_cache_interface import ( @@ -287,6 +290,53 @@ class KVCacheManager: blocks = self.create_kv_cache_blocks(computed_blocks) return blocks, num_new_computed_tokens, shared_prefix_boundary + def get_computed_blocks_for_connector( + self, request: Request + ) -> tuple[KVCacheBlocks, int, int, bool]: + """Local prefix-cache lookup for a request scheduled with a KV connector. + + Hybrid (Mamba + full-attention) models can have per-group prefix hits + diverge under block pressure: the full-attention tail may be evicted + while a deeper Mamba state survives, or vice versa. Report the + full-attention hit as the local prefix - the connector transfers the + remaining suffix and the Mamba state is transferred unconditionally by + nixl's ``_apply_prefix_caching`` - and flag when that hit ran deeper + than a lagging group. Such a hit only has a valid Mamba state at its + boundary if the connector supplies it, so the caller must fall back to + ``get_computed_blocks`` to reconcile when no external tokens are found. + + Non-hybrid models and already-convergent hits use ``get_computed_blocks``. + + Returns: + The ``get_computed_blocks`` triple (blocks, number of local computed + tokens, shared-prefix boundary) plus ``hit_diverged``. + """ + coordinator = self.coordinator + if not ( + self.kv_cache_config.has_mamba_layers + and isinstance(coordinator, HybridKVCacheCoordinator) + and coordinator.full_attention_group_id is not None + ): + return *self.get_computed_blocks(request), False + + if not self.prefix_cache_lookup_enabled(request): + return self.empty_kv_cache_blocks, 0, 0, False + + fa_group_id = coordinator.full_attention_group_id + computed, per_group_hits = coordinator.find_longest_cache_hit_per_group( + request.block_hashes, request.num_tokens - 1 + ) + if any(hit > per_group_hits[fa_group_id] for hit in per_group_hits): + # A lagging group hit deeper than full attention means its + # full-attention blocks were evicted; use the reconciled boundary + # that every group agrees on. + return *self.get_computed_blocks(request), False + + num_local = per_group_hits[fa_group_id] + blocks = self.create_kv_cache_blocks(computed) + # Per-group lookups do not detect an uncached shared prefix (boundary 0). + return blocks, num_local, 0, min(per_group_hits) < num_local + def allocate_slots( self, request: Request, diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index a693061d51..fcd60804a3 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -35,7 +35,6 @@ from vllm.v1.core.encoder_cache_manager import ( EncoderCacheManager, EncoderDecoderCacheManager, ) -from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector from vllm.v1.core.kv_cache_utils import KVCacheBlock @@ -711,50 +710,24 @@ class Scheduler(SchedulerInterface): # Get already-cached tokens. if request.num_computed_tokens == 0: did_prefix_cache_lookup = True + hit_diverged = False # Get locally-cached tokens. - if ( - self.connector is not None - and self.has_mamba_layers - and isinstance( - self.kv_cache_manager.coordinator, HybridKVCacheCoordinator + if self.connector is not None: + # A KV connector transfers the missing suffix, which needs a + # hybrid-aware lookup that can diverge across groups. + ( + new_computed_blocks, + num_new_local_computed_tokens, + request.shared_prefix_boundary, + hit_diverged, + ) = self.kv_cache_manager.get_computed_blocks_for_connector( + request ) - ): - # The per-group lookup does not detect an uncached shared - # prefix, so there is no junction to pin in this path. - request.shared_prefix_boundary = 0 - kv_cache_manager = self.kv_cache_manager - if not kv_cache_manager.prefix_cache_lookup_enabled(request): - # Mirror the get_computed_blocks() early-out: the - # request must recompute its prompt. - new_computed_blocks = kv_cache_manager.empty_kv_cache_blocks - num_new_local_computed_tokens = 0 - else: - computed, per_group_hits = ( - self.kv_cache_manager.coordinator.find_longest_cache_hit_per_group( - request.block_hashes, request.num_tokens - 1 - ) - ) - new_computed_blocks = ( - self.kv_cache_manager.create_kv_cache_blocks(computed) - ) - # NOTE(ZhanqiuHu): For Mamba hybrid models, - # num_new_local_computed_tokens should be the FA hit - # length. This value is passed to the connector's - # get_num_new_matched_tokens which computes: - # external = total - local_computed. - # Using the FA hit skips re-transferring FA blocks - # already cached on D-side. The Mamba state (always - # the last block) is transferred unconditionally by - # _apply_prefix_caching in nixl/worker.py. - num_new_local_computed_tokens = max(per_group_hits) else: ( new_computed_blocks, num_new_local_computed_tokens, - # Junction to pin (Marconi-style APC) so its - # sparse-retention state (Mamba block / sliding-window - # tail) survives retention and serves a later hit; 0 - # if no uncached shared prefix was detected. + # Marconi shared-prefix junction to pin; 0 if none. request.shared_prefix_boundary, ) = self.kv_cache_manager.get_computed_blocks(request) @@ -776,6 +749,16 @@ class Scheduler(SchedulerInterface): num_external_computed_tokens = ext_tokens + if hit_diverged and num_external_computed_tokens == 0: + # No external tokens back the deeper local hit, so its + # resume boundary would have no valid Mamba state. + # Reconcile to the boundary every group agrees on. + ( + new_computed_blocks, + num_new_local_computed_tokens, + request.shared_prefix_boundary, + ) = self.kv_cache_manager.get_computed_blocks(request) + connector_prefix_cache_queries = ( request.num_tokens - num_new_local_computed_tokens )