[BugFix] Preserve divergent FA hits with external Mamba state (#51468)

Signed-off-by: Roger Wang <hey@rogerw.io>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Jeff Ma <jeffjma@umich.edu>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Jeff (Junze) Ma
2026-08-08 20:20:16 +08:00
committed by GitHub
co-authored by Roger Wang Yifan Qiao OpenAI Codex
parent 0fe9a916e8
commit 58d3918e3e
3 changed files with 90 additions and 12 deletions
@@ -590,6 +590,70 @@ def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
assert [len(group) for group in blocks.blocks] == [3, 2]
def test_truncate_computed_blocks_allows_short_mamba_group_only():
"""External state may replace a short Mamba hit, but other groups must
cover the aligned local endpoint."""
hash_block_size = 2
kv_cache_config = KVCacheConfig(
num_blocks=24,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=2 * hash_block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
producer = make_request("producer", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
blocks, num_computed, _ = manager.get_computed_blocks(producer)
assert manager.allocate_slots(producer, 6, num_computed, blocks) is not None
manager.free(producer)
manager.new_step_starts()
consumer = make_request(
"consumer", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256
)
blocks, num_computed, _ = manager.get_computed_blocks(consumer)
assert num_computed == 6
assert [len(group) for group in blocks.blocks] == [3, 2]
short_mamba = manager.create_kv_cache_blocks((list(blocks.blocks[0]), []))
truncated = manager.truncate_computed_blocks(short_mamba, 4)
assert [len(group) for group in truncated.blocks] == [2, 0]
short_full_attention = manager.create_kv_cache_blocks(
(list(blocks.blocks[0][:1]), list(blocks.blocks[1]))
)
with pytest.raises(AssertionError):
manager.truncate_computed_blocks(short_full_attention, 4)
with pytest.raises(AssertionError):
manager.truncate_computed_blocks(blocks, 6)
# The lookup result itself is never mutated.
assert [len(group) for group in blocks.blocks] == [3, 2]
def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit():
hash_block_size = 2
block_size = 2 * hash_block_size
+17 -10
View File
@@ -5685,15 +5685,23 @@ def test_hybrid_per_group_hit_divergence_with_connector(
assert replay.num_tokens - num_scheduled == expected_num_computed
def test_hybrid_per_group_hit_divergence_fa_deeper_no_external():
"""The opposite divergence: the FA prefix survives deeper than the Mamba
state and the connector supplies nothing (ext == 0). Reporting the deep FA
hit as locally computed would resume with no valid Mamba state at that
boundary (silent bad output). The scheduler must fall back to the
convergent boundary that every group agrees on (block 0's surviving state).
@pytest.mark.parametrize(
("matched_tokens", "replay_blocks", "expected_num_computed"),
[
(0, 5, 16),
(16, 6, 80),
],
)
def test_hybrid_fa_deeper_hit_uses_external_mamba_state(
matched_tokens: int,
replay_blocks: int,
expected_num_computed: int,
):
"""A positive external hit supplies the Mamba state at the resume
boundary; without one, the scheduler falls back to the common local hit.
"""
block_size = 16
scheduler = _create_hybrid_mamba_connector_scheduler(matched_tokens=0)
scheduler = _create_hybrid_mamba_connector_scheduler(matched_tokens)
manager = scheduler.kv_cache_manager
assert isinstance(manager.coordinator, HybridKVCacheCoordinator)
@@ -5719,7 +5727,7 @@ def test_hybrid_per_group_hit_divergence_fa_deeper_no_external():
[replay] = create_requests(
num_requests=1,
num_tokens=5 * block_size,
num_tokens=replay_blocks * block_size,
max_tokens=1,
same_prompt=True,
block_size=block_size,
@@ -5733,8 +5741,7 @@ def test_hybrid_per_group_hit_divergence_fa_deeper_no_external():
scheduler.add_request(replay)
output = scheduler.schedule()
num_scheduled = output.num_scheduled_tokens[replay.request_id]
# Must resume at the convergent boundary (block 0), not the deep FA hit.
assert replay.num_tokens - num_scheduled == block_size
assert replay.num_tokens - num_scheduled == expected_num_computed
def _make_encoder_instance_request(scheduler, text_prefix=8, image_tokens=16):
+9 -2
View File
@@ -20,6 +20,7 @@ from vllm.v1.kv_cache_interface import (
CrossAttentionSpec,
EncoderOnlyAttentionSpec,
KVCacheConfig,
MambaSpec,
get_kv_cache_spec_kind,
get_kv_cache_spec_sliding_window,
)
@@ -779,17 +780,23 @@ class KVCacheManager:
) -> KVCacheBlocks:
"""Return a lookup-result view truncated at an aligned token endpoint.
An external hit can supply the final Mamba state even when the local
Mamba group ends before this endpoint. Other groups must cover it.
Pure slicing: refcounts are untouched and ``blocks`` is not mutated.
"""
truncated: list[list[KVCacheBlock]] = []
for group_blocks, manager in zip(
for group_blocks, manager, group in zip(
blocks.blocks,
self.coordinator.single_type_managers,
self.kv_cache_config.kv_cache_groups,
strict=True,
):
assert num_computed_tokens % manager.block_size == 0
num_blocks = num_computed_tokens // manager.block_size
assert num_blocks <= len(group_blocks)
if isinstance(group.kv_cache_spec, MambaSpec):
num_blocks = min(num_blocks, len(group_blocks))
else:
assert num_blocks <= len(group_blocks)
truncated.append(list(group_blocks[:num_blocks]))
return self.create_kv_cache_blocks(tuple(truncated))