[BugFix] Preserve divergent FA hits with external Mamba state (#51468)

Signed-off-by: Roger Wang <hey@rogerw.io>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Jeff Ma <jeffjma@umich.edu>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Jeff (Junze) Ma
2026-08-08 20:20:16 +08:00
committed by GitHub
co-authored by Roger Wang Yifan Qiao OpenAI Codex
parent 0fe9a916e8
commit 58d3918e3e
3 changed files with 90 additions and 12 deletions
@@ -590,6 +590,70 @@ def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
assert [len(group) for group in blocks.blocks] == [3, 2]
def test_truncate_computed_blocks_allows_short_mamba_group_only():
"""External state may replace a short Mamba hit, but other groups must
cover the aligned local endpoint."""
hash_block_size = 2
kv_cache_config = KVCacheConfig(
num_blocks=24,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=2 * hash_block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
producer = make_request("producer", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
blocks, num_computed, _ = manager.get_computed_blocks(producer)
assert manager.allocate_slots(producer, 6, num_computed, blocks) is not None
manager.free(producer)
manager.new_step_starts()
consumer = make_request(
"consumer", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256
)
blocks, num_computed, _ = manager.get_computed_blocks(consumer)
assert num_computed == 6
assert [len(group) for group in blocks.blocks] == [3, 2]
short_mamba = manager.create_kv_cache_blocks((list(blocks.blocks[0]), []))
truncated = manager.truncate_computed_blocks(short_mamba, 4)
assert [len(group) for group in truncated.blocks] == [2, 0]
short_full_attention = manager.create_kv_cache_blocks(
(list(blocks.blocks[0][:1]), list(blocks.blocks[1]))
)
with pytest.raises(AssertionError):
manager.truncate_computed_blocks(short_full_attention, 4)
with pytest.raises(AssertionError):
manager.truncate_computed_blocks(blocks, 6)
# The lookup result itself is never mutated.
assert [len(group) for group in blocks.blocks] == [3, 2]
def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit():
hash_block_size = 2
block_size = 2 * hash_block_size