Signed-off-by: Rishi Puri <riship@nvidia.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
ed13deb376
commit
fd10e8946d
@@ -531,3 +531,39 @@ def test_load_mask_without_eagle_unchanged():
|
||||
assert hit == 64
|
||||
masks = coord.load_mask(hs, token_len=hit)
|
||||
assert masks[0] == [True, True, True, True]
|
||||
|
||||
|
||||
def _mamba(block_size=16):
|
||||
return MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
|
||||
|
||||
def test_lookup_with_eagle_hybrid_full_plus_mamba_no_overrun():
|
||||
"""Full+Mamba with eagle must not overrun the attention-verified hit.
|
||||
|
||||
``MambaManager`` ignores ``drop_eagle_block`` (a Mamba block at position
|
||||
p IS the recurrent state after (p + 1) * block_size tokens; there is
|
||||
nothing to recompute), so granting the Mamba group the one-block eagle
|
||||
peek margin lets it match one block PAST the eagle-pruned full-attention
|
||||
hit, and adopting that length resumes the recurrent state ahead of the
|
||||
verified token prefix (#43559). Gating the margin on ``not
|
||||
isinstance(spec, MambaSpec)`` pins the hit to the attention-verified 48.
|
||||
"""
|
||||
groups = [
|
||||
KVCacheGroupSpec(["L0"], _full(16)),
|
||||
KVCacheGroupSpec(["L1"], _mamba(16)),
|
||||
]
|
||||
coord = _make_coord(groups, hash_block_size=16, use_eagle=True)
|
||||
hs = _hashes(4)
|
||||
exists = {(g, bytes(h)) for g in (0, 1) for h in hs}
|
||||
cmap = ExternalCachedBlockPool(16, exists)
|
||||
_masks, hit = coord.find_longest_cache_hit(
|
||||
hs, max_length=64, cached_block_pool=cmap
|
||||
)
|
||||
# FullAttn matches 4 blocks, eagle pops 1 -> 48 verified tokens. The
|
||||
# Mamba group must serve its state@48 snapshot, not peek to state@64.
|
||||
assert hit == 48
|
||||
|
||||
@@ -338,6 +338,9 @@ class MooncakeStoreCoordinator:
|
||||
|
||||
drop_eagle_block = idx in eagle_indices and idx not in eagle_verified
|
||||
_max_length = curr_hit_length
|
||||
# No eagle peek margin for a recurrent (Mamba) group: its finder
|
||||
# never drops a block, so a widened bound would match past the
|
||||
# attention-verified hit and resume from speculative state (#43559).
|
||||
if drop_eagle_block and not isinstance(spec, MambaSpec):
|
||||
eagle_margin = (
|
||||
self.hash_block_size
|
||||
|
||||
Reference in New Issue
Block a user