[BugFix] Preserve divergent FA hits with external Mamba state (#51468)
Signed-off-by: Roger Wang <hey@rogerw.io> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Jeff Ma <jeffjma@umich.edu> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Roger Wang
Yifan Qiao
OpenAI Codex
parent
0fe9a916e8
commit
58d3918e3e
@@ -590,6 +590,70 @@ def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
|
||||
assert [len(group) for group in blocks.blocks] == [3, 2]
|
||||
|
||||
|
||||
def test_truncate_computed_blocks_allows_short_mamba_group_only():
|
||||
"""External state may replace a short Mamba hit, but other groups must
|
||||
cover the aligned local endpoint."""
|
||||
hash_block_size = 2
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=24,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=hash_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=2 * hash_block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
producer = make_request("producer", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
blocks, num_computed, _ = manager.get_computed_blocks(producer)
|
||||
assert manager.allocate_slots(producer, 6, num_computed, blocks) is not None
|
||||
manager.free(producer)
|
||||
manager.new_step_starts()
|
||||
|
||||
consumer = make_request(
|
||||
"consumer", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256
|
||||
)
|
||||
blocks, num_computed, _ = manager.get_computed_blocks(consumer)
|
||||
assert num_computed == 6
|
||||
assert [len(group) for group in blocks.blocks] == [3, 2]
|
||||
|
||||
short_mamba = manager.create_kv_cache_blocks((list(blocks.blocks[0]), []))
|
||||
truncated = manager.truncate_computed_blocks(short_mamba, 4)
|
||||
assert [len(group) for group in truncated.blocks] == [2, 0]
|
||||
|
||||
short_full_attention = manager.create_kv_cache_blocks(
|
||||
(list(blocks.blocks[0][:1]), list(blocks.blocks[1]))
|
||||
)
|
||||
with pytest.raises(AssertionError):
|
||||
manager.truncate_computed_blocks(short_full_attention, 4)
|
||||
|
||||
with pytest.raises(AssertionError):
|
||||
manager.truncate_computed_blocks(blocks, 6)
|
||||
|
||||
# The lookup result itself is never mutated.
|
||||
assert [len(group) for group in blocks.blocks] == [3, 2]
|
||||
|
||||
|
||||
def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit():
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
|
||||
Reference in New Issue
Block a user