[Bugfix][Mooncake] Use the EAGLE replay boundary when retaining sliding-window tails (#61012)

Signed-off-by: Stu Cao <stu@primeintellect.ai>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Stu Cao
2026-10-11 09:15:18 +00:00
committed by GitHub
co-authored by Claude Fable 5.1 Yifan Qiao Codex
parent 1789f09588
commit 2339bd39d6
4 changed files with 85 additions and 27 deletions
@@ -3,6 +3,7 @@
from math import lcm
import pytest
import torch
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
@@ -422,6 +423,49 @@ def test_store_mask_retention_interval_zero_keeps_only_replay_boundary():
assert masks[1] == [i == 11 for i in range(16)]
def test_store_mask_retention_interval_zero_eagle_keeps_lookup_tail():
"""With EAGLE the replay boundary sits one scheduler block lower, as in
core's ``replay_boundaries``: the lookup matches that block and drops
back from it. The retained SWA tail must span the ``need`` contiguous
blocks the lookup requires (cdiv(8 - 1, 8) + 1 = 2 here); the non-EAGLE
boundary leaves only one of them inside the saved prefix."""
coord = _make_coord(
_retention_groups(), hash_block_size=8, use_eagle=True, retention_interval=0
)
# num_prompt=100 -> boundary 64 -> chunks 7, 8 (8 is the EAGLE peek).
masks = coord.store_mask(96, num_prompt_tokens=100)
assert masks[0] is None
assert masks[1] == [i in (7, 8) for i in range(12)]
# Aligned num_prompt=96 -> boundaries 32 (resend) and 64 (longer sibling).
masks = coord.store_mask(96, num_prompt_tokens=96)
assert masks[1] == [i in (3, 4, 7, 8) for i in range(12)]
@pytest.mark.parametrize("num_prompt_tokens", [100, 96])
def test_store_mask_retention_interval_zero_eagle_round_trip(num_prompt_tokens):
"""Everything store_mask keeps is in the store; the EAGLE lookup of the
same prompt must then reach the full block below the one it drops (64 of
96). The unaligned prompt hit 0 before the fix; the aligned one already
hit."""
groups = _retention_groups()
coord = _make_coord(groups, hash_block_size=8, use_eagle=True, retention_interval=0)
hashes = _hashes(96 // coord.hash_block_size)
exists: set[tuple[int, bytes]] = set()
store_masks = coord.store_mask(96, num_prompt_tokens=num_prompt_tokens)
for gid, (group, mask) in enumerate(zip(groups, store_masks, strict=True)):
group_hashes = coord.block_hashes_for_spec(hashes, group.kv_cache_spec)
for chunk_id, block_hash in enumerate(group_hashes):
if mask is None or mask[chunk_id]:
exists.add((gid, bytes(block_hash)))
_masks, hit = coord.find_longest_cache_hit(
hashes,
max_length=96,
cached_block_pool=ExternalCachedBlockPool(coord.hash_block_size, exists),
)
assert hit == 64
def test_store_mask_retention_interval_keeps_segment_and_replay_tails():
"""Sparse segment tails (interval=64 -> chunks 7,15) plus the replay
boundary tail (num_prompt=100 -> chunk 11) coexist."""