Signed-off-by: Rishi Puri <riship@nvidia.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
ed13deb376
commit
fd10e8946d
@@ -531,3 +531,39 @@ def test_load_mask_without_eagle_unchanged():
|
|||||||
assert hit == 64
|
assert hit == 64
|
||||||
masks = coord.load_mask(hs, token_len=hit)
|
masks = coord.load_mask(hs, token_len=hit)
|
||||||
assert masks[0] == [True, True, True, True]
|
assert masks[0] == [True, True, True, True]
|
||||||
|
|
||||||
|
|
||||||
|
def _mamba(block_size=16):
|
||||||
|
return MambaSpec(
|
||||||
|
block_size=block_size,
|
||||||
|
shapes=((1, 1),),
|
||||||
|
dtypes=(torch.float32,),
|
||||||
|
mamba_cache_mode="align",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_lookup_with_eagle_hybrid_full_plus_mamba_no_overrun():
|
||||||
|
"""Full+Mamba with eagle must not overrun the attention-verified hit.
|
||||||
|
|
||||||
|
``MambaManager`` ignores ``drop_eagle_block`` (a Mamba block at position
|
||||||
|
p IS the recurrent state after (p + 1) * block_size tokens; there is
|
||||||
|
nothing to recompute), so granting the Mamba group the one-block eagle
|
||||||
|
peek margin lets it match one block PAST the eagle-pruned full-attention
|
||||||
|
hit, and adopting that length resumes the recurrent state ahead of the
|
||||||
|
verified token prefix (#43559). Gating the margin on ``not
|
||||||
|
isinstance(spec, MambaSpec)`` pins the hit to the attention-verified 48.
|
||||||
|
"""
|
||||||
|
groups = [
|
||||||
|
KVCacheGroupSpec(["L0"], _full(16)),
|
||||||
|
KVCacheGroupSpec(["L1"], _mamba(16)),
|
||||||
|
]
|
||||||
|
coord = _make_coord(groups, hash_block_size=16, use_eagle=True)
|
||||||
|
hs = _hashes(4)
|
||||||
|
exists = {(g, bytes(h)) for g in (0, 1) for h in hs}
|
||||||
|
cmap = ExternalCachedBlockPool(16, exists)
|
||||||
|
_masks, hit = coord.find_longest_cache_hit(
|
||||||
|
hs, max_length=64, cached_block_pool=cmap
|
||||||
|
)
|
||||||
|
# FullAttn matches 4 blocks, eagle pops 1 -> 48 verified tokens. The
|
||||||
|
# Mamba group must serve its state@48 snapshot, not peek to state@64.
|
||||||
|
assert hit == 48
|
||||||
|
|||||||
@@ -338,6 +338,9 @@ class MooncakeStoreCoordinator:
|
|||||||
|
|
||||||
drop_eagle_block = idx in eagle_indices and idx not in eagle_verified
|
drop_eagle_block = idx in eagle_indices and idx not in eagle_verified
|
||||||
_max_length = curr_hit_length
|
_max_length = curr_hit_length
|
||||||
|
# No eagle peek margin for a recurrent (Mamba) group: its finder
|
||||||
|
# never drops a block, so a widened bound would match past the
|
||||||
|
# attention-verified hit and resume from speculative state (#43559).
|
||||||
if drop_eagle_block and not isinstance(spec, MambaSpec):
|
if drop_eagle_block and not isinstance(spec, MambaSpec):
|
||||||
eagle_margin = (
|
eagle_margin = (
|
||||||
self.hash_block_size
|
self.hash_block_size
|
||||||
|
|||||||
Reference in New Issue
Block a user