[Bugfix][Mooncake] Use the EAGLE replay boundary when retaining sliding-window tails (#61012)
Signed-off-by: Stu Cao <stu@primeintellect.ai> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
Yifan Qiao
Codex
parent
1789f09588
commit
2339bd39d6
@@ -11,6 +11,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_coordinator import replay_boundaries
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
KVCacheBlock,
|
||||
@@ -341,6 +342,15 @@ class MooncakeStoreCoordinator:
|
||||
f"aligned_token_len ({aligned_token_len}) must be a multiple of "
|
||||
f"{mask_alignment}"
|
||||
)
|
||||
reachable_boundaries = (
|
||||
()
|
||||
if num_prompt_tokens is None
|
||||
else replay_boundaries(
|
||||
num_prompt_tokens,
|
||||
self.lcm_block_size,
|
||||
use_eagle=bool(self.eagle_group_ids),
|
||||
)
|
||||
)
|
||||
masks: list[list[bool] | None] = []
|
||||
for g_idx, g in enumerate(self.kv_cache_groups):
|
||||
spec = _unwrap_spec(g.kv_cache_spec)
|
||||
@@ -352,9 +362,6 @@ class MooncakeStoreCoordinator:
|
||||
manager_cls = KVCacheSpecRegistry.get_manager_class(spec)
|
||||
assert manager_cls is not None
|
||||
use_eagle = g_idx in self.eagle_group_ids
|
||||
reachable_boundaries = (
|
||||
() if num_prompt_tokens is None else (num_prompt_tokens - 1,)
|
||||
)
|
||||
mask = manager_cls.reachable_block_mask(
|
||||
start_block=start_chunk,
|
||||
end_block=end_chunk,
|
||||
|
||||
Reference in New Issue
Block a user