[Bugfix] Cache the EAGLE/MTP lookahead block in the SWA prefix-cache mask (#44082)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
@@ -233,7 +233,7 @@ class MooncakeStoreCoordinator:
|
||||
kv_cache_group_ids=group_ids,
|
||||
block_pool=cast(BlockPool, cached_block_pool),
|
||||
kv_cache_spec=spec,
|
||||
use_eagle=(0 in eagle_indices),
|
||||
drop_eagle_block=(0 in eagle_indices),
|
||||
alignment_tokens=spec.block_size,
|
||||
)
|
||||
num_groups = len(self.kv_cache_groups)
|
||||
@@ -262,9 +262,9 @@ class MooncakeStoreCoordinator:
|
||||
)
|
||||
continue
|
||||
|
||||
use_eagle = idx in eagle_indices and idx not in eagle_verified
|
||||
drop_eagle_block = idx in eagle_indices and idx not in eagle_verified
|
||||
_max_length = curr_hit_length
|
||||
if use_eagle:
|
||||
if drop_eagle_block:
|
||||
_max_length = min(curr_hit_length + spec.block_size, max_length)
|
||||
hashes = self.block_hashes_for_spec(block_hashes, spec)
|
||||
hit_blocks = manager_cls.find_longest_cache_hit(
|
||||
@@ -273,11 +273,11 @@ class MooncakeStoreCoordinator:
|
||||
kv_cache_group_ids=group_ids,
|
||||
block_pool=cast(BlockPool, cached_block_pool),
|
||||
kv_cache_spec=spec,
|
||||
use_eagle=use_eagle,
|
||||
drop_eagle_block=drop_eagle_block,
|
||||
alignment_tokens=self.lcm_block_size,
|
||||
)
|
||||
_new_hit_length = len(hit_blocks[0]) * spec.block_size
|
||||
if use_eagle:
|
||||
if drop_eagle_block:
|
||||
eagle_verified.add(idx)
|
||||
elif _new_hit_length < curr_hit_length:
|
||||
eagle_verified.clear()
|
||||
|
||||
Reference in New Issue
Block a user