From 58d3918e3ea0a544ffedadad2ba84559e9c51d8f Mon Sep 17 00:00:00 2001 From: "Jeff (Junze) Ma" <93145857+majunze2001@users.noreply.github.com> Date: Sat, 8 Aug 2026 05:20:16 -0700 Subject: [PATCH] [BugFix] Preserve divergent FA hits with external Mamba state (#51468) Signed-off-by: Roger Wang Signed-off-by: Yifan Qiao Signed-off-by: Jeff Ma Co-authored-by: Roger Wang Co-authored-by: Yifan Qiao Co-authored-by: OpenAI Codex --- .../test_partial_prefix_cache_hits.py | 64 +++++++++++++++++++ tests/v1/core/test_scheduler.py | 27 +++++--- vllm/v1/core/kv_cache_manager.py | 11 +++- 3 files changed, 90 insertions(+), 12 deletions(-) diff --git a/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py b/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py index ec3f293c0a..543991a874 100644 --- a/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py +++ b/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py @@ -590,6 +590,70 @@ def test_truncate_computed_blocks_preserves_sparse_prefix_positions(): assert [len(group) for group in blocks.blocks] == [3, 2] +def test_truncate_computed_blocks_allows_short_mamba_group_only(): + """External state may replace a short Mamba hit, but other groups must + cover the aligned local endpoint.""" + hash_block_size = 2 + kv_cache_config = KVCacheConfig( + num_blocks=24, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["full"], + FullAttentionSpec( + block_size=hash_block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + ), + KVCacheGroupSpec( + ["mamba"], + MambaSpec( + block_size=2 * hash_block_size, + shapes=(1, 1), + dtypes=(torch.float32,), + mamba_cache_mode="align", + ), + ), + ], + ) + manager = make_kv_cache_manager( + kv_cache_config=kv_cache_config, + max_model_len=8192, + enable_caching=True, + hash_block_size=hash_block_size, + ) + producer = make_request("producer", [0, 0, 1, 1, 2, 2], hash_block_size, sha256) + blocks, num_computed, _ = manager.get_computed_blocks(producer) + assert manager.allocate_slots(producer, 6, num_computed, blocks) is not None + manager.free(producer) + manager.new_step_starts() + + consumer = make_request( + "consumer", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256 + ) + blocks, num_computed, _ = manager.get_computed_blocks(consumer) + assert num_computed == 6 + assert [len(group) for group in blocks.blocks] == [3, 2] + + short_mamba = manager.create_kv_cache_blocks((list(blocks.blocks[0]), [])) + truncated = manager.truncate_computed_blocks(short_mamba, 4) + assert [len(group) for group in truncated.blocks] == [2, 0] + + short_full_attention = manager.create_kv_cache_blocks( + (list(blocks.blocks[0][:1]), list(blocks.blocks[1])) + ) + with pytest.raises(AssertionError): + manager.truncate_computed_blocks(short_full_attention, 4) + + with pytest.raises(AssertionError): + manager.truncate_computed_blocks(blocks, 6) + + # The lookup result itself is never mutated. + assert [len(group) for group in blocks.blocks] == [3, 2] + + def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit(): hash_block_size = 2 block_size = 2 * hash_block_size diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index 30c7625aa7..30ab5bc09d 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -5685,15 +5685,23 @@ def test_hybrid_per_group_hit_divergence_with_connector( assert replay.num_tokens - num_scheduled == expected_num_computed -def test_hybrid_per_group_hit_divergence_fa_deeper_no_external(): - """The opposite divergence: the FA prefix survives deeper than the Mamba - state and the connector supplies nothing (ext == 0). Reporting the deep FA - hit as locally computed would resume with no valid Mamba state at that - boundary (silent bad output). The scheduler must fall back to the - convergent boundary that every group agrees on (block 0's surviving state). +@pytest.mark.parametrize( + ("matched_tokens", "replay_blocks", "expected_num_computed"), + [ + (0, 5, 16), + (16, 6, 80), + ], +) +def test_hybrid_fa_deeper_hit_uses_external_mamba_state( + matched_tokens: int, + replay_blocks: int, + expected_num_computed: int, +): + """A positive external hit supplies the Mamba state at the resume + boundary; without one, the scheduler falls back to the common local hit. """ block_size = 16 - scheduler = _create_hybrid_mamba_connector_scheduler(matched_tokens=0) + scheduler = _create_hybrid_mamba_connector_scheduler(matched_tokens) manager = scheduler.kv_cache_manager assert isinstance(manager.coordinator, HybridKVCacheCoordinator) @@ -5719,7 +5727,7 @@ def test_hybrid_per_group_hit_divergence_fa_deeper_no_external(): [replay] = create_requests( num_requests=1, - num_tokens=5 * block_size, + num_tokens=replay_blocks * block_size, max_tokens=1, same_prompt=True, block_size=block_size, @@ -5733,8 +5741,7 @@ def test_hybrid_per_group_hit_divergence_fa_deeper_no_external(): scheduler.add_request(replay) output = scheduler.schedule() num_scheduled = output.num_scheduled_tokens[replay.request_id] - # Must resume at the convergent boundary (block 0), not the deep FA hit. - assert replay.num_tokens - num_scheduled == block_size + assert replay.num_tokens - num_scheduled == expected_num_computed def _make_encoder_instance_request(scheduler, text_prefix=8, image_tokens=16): diff --git a/vllm/v1/core/kv_cache_manager.py b/vllm/v1/core/kv_cache_manager.py index fcb67726a5..ca1fb73420 100644 --- a/vllm/v1/core/kv_cache_manager.py +++ b/vllm/v1/core/kv_cache_manager.py @@ -20,6 +20,7 @@ from vllm.v1.kv_cache_interface import ( CrossAttentionSpec, EncoderOnlyAttentionSpec, KVCacheConfig, + MambaSpec, get_kv_cache_spec_kind, get_kv_cache_spec_sliding_window, ) @@ -779,17 +780,23 @@ class KVCacheManager: ) -> KVCacheBlocks: """Return a lookup-result view truncated at an aligned token endpoint. + An external hit can supply the final Mamba state even when the local + Mamba group ends before this endpoint. Other groups must cover it. Pure slicing: refcounts are untouched and ``blocks`` is not mutated. """ truncated: list[list[KVCacheBlock]] = [] - for group_blocks, manager in zip( + for group_blocks, manager, group in zip( blocks.blocks, self.coordinator.single_type_managers, + self.kv_cache_config.kv_cache_groups, strict=True, ): assert num_computed_tokens % manager.block_size == 0 num_blocks = num_computed_tokens // manager.block_size - assert num_blocks <= len(group_blocks) + if isinstance(group.kv_cache_spec, MambaSpec): + num_blocks = min(num_blocks, len(group_blocks)) + else: + assert num_blocks <= len(group_blocks) truncated.append(list(group_blocks[:num_blocks])) return self.create_kv_cache_blocks(tuple(truncated))