[BugFix] Preserve divergent FA hits with external Mamba state (#51468)
Signed-off-by: Roger Wang <hey@rogerw.io> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Jeff Ma <jeffjma@umich.edu> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Roger Wang
Yifan Qiao
OpenAI Codex
parent
0fe9a916e8
commit
58d3918e3e
@@ -590,6 +590,70 @@ def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
|
||||
assert [len(group) for group in blocks.blocks] == [3, 2]
|
||||
|
||||
|
||||
def test_truncate_computed_blocks_allows_short_mamba_group_only():
|
||||
"""External state may replace a short Mamba hit, but other groups must
|
||||
cover the aligned local endpoint."""
|
||||
hash_block_size = 2
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=24,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=hash_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=2 * hash_block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
producer = make_request("producer", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
blocks, num_computed, _ = manager.get_computed_blocks(producer)
|
||||
assert manager.allocate_slots(producer, 6, num_computed, blocks) is not None
|
||||
manager.free(producer)
|
||||
manager.new_step_starts()
|
||||
|
||||
consumer = make_request(
|
||||
"consumer", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256
|
||||
)
|
||||
blocks, num_computed, _ = manager.get_computed_blocks(consumer)
|
||||
assert num_computed == 6
|
||||
assert [len(group) for group in blocks.blocks] == [3, 2]
|
||||
|
||||
short_mamba = manager.create_kv_cache_blocks((list(blocks.blocks[0]), []))
|
||||
truncated = manager.truncate_computed_blocks(short_mamba, 4)
|
||||
assert [len(group) for group in truncated.blocks] == [2, 0]
|
||||
|
||||
short_full_attention = manager.create_kv_cache_blocks(
|
||||
(list(blocks.blocks[0][:1]), list(blocks.blocks[1]))
|
||||
)
|
||||
with pytest.raises(AssertionError):
|
||||
manager.truncate_computed_blocks(short_full_attention, 4)
|
||||
|
||||
with pytest.raises(AssertionError):
|
||||
manager.truncate_computed_blocks(blocks, 6)
|
||||
|
||||
# The lookup result itself is never mutated.
|
||||
assert [len(group) for group in blocks.blocks] == [3, 2]
|
||||
|
||||
|
||||
def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit():
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
|
||||
@@ -5685,15 +5685,23 @@ def test_hybrid_per_group_hit_divergence_with_connector(
|
||||
assert replay.num_tokens - num_scheduled == expected_num_computed
|
||||
|
||||
|
||||
def test_hybrid_per_group_hit_divergence_fa_deeper_no_external():
|
||||
"""The opposite divergence: the FA prefix survives deeper than the Mamba
|
||||
state and the connector supplies nothing (ext == 0). Reporting the deep FA
|
||||
hit as locally computed would resume with no valid Mamba state at that
|
||||
boundary (silent bad output). The scheduler must fall back to the
|
||||
convergent boundary that every group agrees on (block 0's surviving state).
|
||||
@pytest.mark.parametrize(
|
||||
("matched_tokens", "replay_blocks", "expected_num_computed"),
|
||||
[
|
||||
(0, 5, 16),
|
||||
(16, 6, 80),
|
||||
],
|
||||
)
|
||||
def test_hybrid_fa_deeper_hit_uses_external_mamba_state(
|
||||
matched_tokens: int,
|
||||
replay_blocks: int,
|
||||
expected_num_computed: int,
|
||||
):
|
||||
"""A positive external hit supplies the Mamba state at the resume
|
||||
boundary; without one, the scheduler falls back to the common local hit.
|
||||
"""
|
||||
block_size = 16
|
||||
scheduler = _create_hybrid_mamba_connector_scheduler(matched_tokens=0)
|
||||
scheduler = _create_hybrid_mamba_connector_scheduler(matched_tokens)
|
||||
manager = scheduler.kv_cache_manager
|
||||
assert isinstance(manager.coordinator, HybridKVCacheCoordinator)
|
||||
|
||||
@@ -5719,7 +5727,7 @@ def test_hybrid_per_group_hit_divergence_fa_deeper_no_external():
|
||||
|
||||
[replay] = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=5 * block_size,
|
||||
num_tokens=replay_blocks * block_size,
|
||||
max_tokens=1,
|
||||
same_prompt=True,
|
||||
block_size=block_size,
|
||||
@@ -5733,8 +5741,7 @@ def test_hybrid_per_group_hit_divergence_fa_deeper_no_external():
|
||||
scheduler.add_request(replay)
|
||||
output = scheduler.schedule()
|
||||
num_scheduled = output.num_scheduled_tokens[replay.request_id]
|
||||
# Must resume at the convergent boundary (block 0), not the deep FA hit.
|
||||
assert replay.num_tokens - num_scheduled == block_size
|
||||
assert replay.num_tokens - num_scheduled == expected_num_computed
|
||||
|
||||
|
||||
def _make_encoder_instance_request(scheduler, text_prefix=8, image_tokens=16):
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
CrossAttentionSpec,
|
||||
EncoderOnlyAttentionSpec,
|
||||
KVCacheConfig,
|
||||
MambaSpec,
|
||||
get_kv_cache_spec_kind,
|
||||
get_kv_cache_spec_sliding_window,
|
||||
)
|
||||
@@ -779,17 +780,23 @@ class KVCacheManager:
|
||||
) -> KVCacheBlocks:
|
||||
"""Return a lookup-result view truncated at an aligned token endpoint.
|
||||
|
||||
An external hit can supply the final Mamba state even when the local
|
||||
Mamba group ends before this endpoint. Other groups must cover it.
|
||||
Pure slicing: refcounts are untouched and ``blocks`` is not mutated.
|
||||
"""
|
||||
truncated: list[list[KVCacheBlock]] = []
|
||||
for group_blocks, manager in zip(
|
||||
for group_blocks, manager, group in zip(
|
||||
blocks.blocks,
|
||||
self.coordinator.single_type_managers,
|
||||
self.kv_cache_config.kv_cache_groups,
|
||||
strict=True,
|
||||
):
|
||||
assert num_computed_tokens % manager.block_size == 0
|
||||
num_blocks = num_computed_tokens // manager.block_size
|
||||
assert num_blocks <= len(group_blocks)
|
||||
if isinstance(group.kv_cache_spec, MambaSpec):
|
||||
num_blocks = min(num_blocks, len(group_blocks))
|
||||
else:
|
||||
assert num_blocks <= len(group_blocks)
|
||||
truncated.append(list(group_blocks[:num_blocks]))
|
||||
return self.create_kv_cache_blocks(tuple(truncated))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user