[Bugfix][Mooncake] Save the EAGLE attention proof with hybrid Mamba partial tails (#60695)

Signed-off-by: Dao Le <Dao007forever@gmail.com>
Signed-off-by: Dao007forever <daole@inferact.ai>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
Dao007forever
2026-10-10 00:08:08 +00:00
committed by GitHub
co-authored by Wei Zhao Claude Opus 5.5 zjy0516
parent 03a7d3776f
commit f8e6e365ae
11 changed files with 1142 additions and 390 deletions
+114 -25
View File
@@ -4,7 +4,12 @@
import contextlib
import json
import os
import random
import shutil
import socket
import subprocess
import time
from collections.abc import Iterator
from dataclasses import dataclass
from math import lcm
@@ -60,6 +65,16 @@ SPEC_CONFIG = {
}
NIXL = {"kv_connector": "NixlConnector", "kv_role": "kv_both"}
MOONCAKE = {
"kv_connector": "MooncakeStoreConnector",
"kv_role": "kv_both",
"kv_load_failure_policy": "recompute",
"kv_connector_extra_config": {
"load_async": True,
"lookup_async": True,
"enable_cross_layers_blocks": False,
},
}
# SimpleCPUOffloadConnector as in the Kimi-K3 recipe.
OFFLOAD = {
"kv_connector": "SimpleCPUOffloadConnector",
@@ -243,10 +258,16 @@ def _tokens_text_logprobs(response: dict):
def _reset_gpu_prefix_cache(url: str) -> None:
response = requests.post(
f"{url}/reset_prefix_cache", params={"reset_external": "false"}, timeout=60
)
response.raise_for_status()
deadline = time.monotonic() + 60
while True:
response = requests.post(
f"{url}/reset_prefix_cache", params={"reset_external": "false"}, timeout=60
)
response.raise_for_status()
if response.json()["success"]:
return
assert time.monotonic() < deadline, "GPU cache reset timed out"
time.sleep(0.1)
@dataclass(frozen=True)
@@ -354,6 +375,35 @@ def _format(turns: list[Turn]) -> str:
return "\n".join(rows)
def _check_resend_reuses_prompt_checkpoint(
deployment: Deployment, prompt_len: int
) -> None:
"""Resends reuse the Mamba checkpoint and its companion EAGLE attention proof."""
mode = Mode(128, True)
salt = str(uuid4())
rng = random.Random(0)
prompt = [rng.randint(1000, 150000) for _ in range(prompt_len)]
with _serve(deployment, mode) as servers:
url = servers[0].url_root
cold = _complete(url, prompt, salt, max_tokens=1)
if deployment.offload:
_reset_gpu_prefix_cache(url)
resend = _complete(url, prompt, salt, max_tokens=1)
recompute = _complete(url, prompt, f"{salt}-recompute", max_tokens=1)
# EAGLE's dropped hash unit already leaves tokens to recompute on resend.
expected = (prompt_len // 128 - 1) * 128
actual = _cached(resend)
assert _cached(cold) == _cached(recompute) == 0
assert actual == expected, f"{prompt_len=}: {actual=}, {expected=}"
check_logprobs_close(
outputs_0_lst=[_tokens_text_logprobs(recompute)],
outputs_1_lst=[_tokens_text_logprobs(resend)],
name_0="recompute",
name_1="resend",
)
@pytest.mark.parametrize(
"prompt_len",
[
@@ -368,28 +418,67 @@ def _format(turns: list[Turn]) -> str:
],
)
def test_dspark_exact_resend_reuses_prompt_checkpoint(prompt_len: int) -> None:
"""Resends reuse the Mamba checkpoint and its companion EAGLE attention proof."""
mode = Mode(128, True)
salt = str(uuid4())
rng = random.Random(0)
prompt = [rng.randint(1000, 150000) for _ in range(prompt_len)]
deployment = DEPLOYMENTS["plain"]
with _serve(deployment, mode) as servers:
url = servers[0].url_root
cold = _complete(url, prompt, salt, max_tokens=1)
resend = _complete(url, prompt, salt, max_tokens=1)
recompute = _complete(url, prompt, f"{salt}-recompute", max_tokens=1)
_check_resend_reuses_prompt_checkpoint(DEPLOYMENTS["plain"], prompt_len)
# EAGLE's dropped hash unit already leaves tokens to recompute on resend.
expected = (prompt_len // 128 - 1) * 128
actual = _cached(resend)
assert _cached(cold) == _cached(recompute) == 0
assert actual == expected, f"{prompt_len=}: {actual=}, {expected=}"
check_logprobs_close(
outputs_0_lst=[_tokens_text_logprobs(recompute)],
outputs_1_lst=[_tokens_text_logprobs(resend)],
name_0="recompute",
name_1="resend",
@pytest.fixture
def mooncake_store(tmp_path, monkeypatch) -> Iterator[None]:
"""Run a Mooncake master and point the servers' store config at it."""
if shutil.which("mooncake_master") is None:
pytest.skip("mooncake_master is not installed")
port = get_open_port()
log_path = tmp_path / "mooncake_master.log"
with open(log_path, "w") as log:
master = subprocess.Popen(
[
"mooncake_master",
f"--port={port}",
f"--metrics_port={get_open_port()}",
"--enable_metric_reporting=false",
"--default_kv_lease_ttl=60000",
],
stdout=log,
stderr=subprocess.STDOUT,
)
try:
deadline = time.monotonic() + 60
while True:
assert master.poll() is None, log_path.read_text()
with (
contextlib.suppress(OSError),
socket.create_connection(("127.0.0.1", port)),
):
break
assert time.monotonic() < deadline, "Mooncake master did not start"
time.sleep(0.1)
config_path = tmp_path / "mooncake.json"
config_path.write_text(
json.dumps(
{
"metadata_server": "P2PHANDSHAKE",
"master_server_address": f"127.0.0.1:{port}",
"protocol": "rdma",
"device_name": os.getenv("MOONCAKE_DEVICE_NAME", "mlx5_12"),
"global_segment_size": "4GB",
"local_buffer_size": "1GB",
}
)
)
monkeypatch.setenv("MOONCAKE_CONFIG_PATH", str(config_path))
monkeypatch.setenv("MC_MAX_MR_SIZE", str(4 << 30))
yield
finally:
master.kill()
master.wait()
@pytest.mark.usefixtures("mooncake_store")
def test_mooncake_resend_reuses_prompt_checkpoint() -> None:
"""Blocks=6144, PMU=128. P=7449 needs Mamba state at 7296 and attention proof
at 7424, both beyond the normal Mooncake save boundary of 6144: ensure they
remain reusable from the store after a GPU cache reset."""
_check_resend_reuses_prompt_checkpoint(
Deployment(Instance(kv_config=MOONCAKE), offload=True), 7449
)
@@ -30,7 +30,13 @@ def _mamba_align(block_size=32):
)
def _make_coord(groups, hash_block_size, use_eagle=False, retention_interval=None):
def _make_coord(
groups,
hash_block_size,
use_eagle=False,
retention_interval=None,
enable_partial_hash_hits=False,
):
"""Construct a coordinator using the natural LCM of group block sizes as
the scheduler block size — mirrors ``resolve_kv_cache_block_sizes`` for
the test fixtures."""
@@ -42,6 +48,7 @@ def _make_coord(groups, hash_block_size, use_eagle=False, retention_interval=Non
hash_block_size=hash_block_size,
use_eagle=use_eagle,
retention_interval=retention_interval,
enable_partial_hash_hits=enable_partial_hash_hits,
)
@@ -219,8 +226,7 @@ def test_coordinator_fine_grained_partial_tail_hit():
KVCacheGroupSpec(["L0"], _full(32)),
KVCacheGroupSpec(["L1"], _mamba_align(32)),
]
coord = _make_coord(groups, hash_block_size=16)
assert coord.enable_partial_hash_hits
coord = _make_coord(groups, hash_block_size=16, enable_partial_hash_hits=True)
hs = _hashes(4) # 4 hash units of 16 = 64 tokens; block 0 = [0,32), etc.
# Both groups: full block 0 (key = last sub-hash hs[1]) + partial boundary
# at token 48 (key = hs[2]). No hs[3] -> block 1 is not full.
@@ -287,38 +293,6 @@ def test_store_mask_swa_only_window_around_each_lcm_boundary():
assert masks[1] == [False, False, False, True, False, False, False, True]
def test_store_mask_swa_does_not_double_scale_under_dcp():
"""The coordinator receives DCP-resolved specs (worker.py applies
resolve_dcp_kv_cache_spec) and indexes chunks in that already-scaled block
size. So dcp_world_size must NOT further scale reachable_block_mask; the
mask must be identical for any dcp_world_size given the same specs.
Regression: previously the coordinator forwarded its dcp_world_size, which
double-scaled the SWA block size and produced a wrong mask when dcp > 1.
"""
full = _full(32)
swa = _swa(block_size=8, sliding_window=8)
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
scheduler_block_size = lcm(32, 8)
def make(dcp_world_size):
return MooncakeStoreCoordinator(
groups,
scheduler_block_size=scheduler_block_size,
hash_block_size=8,
dcp_world_size=dcp_world_size,
)
mask_dcp1 = make(1).store_mask(64)
mask_dcp2 = make(2).store_mask(64)
# dcp must not change the mask; both match the correct single-block-size
# result (chunks ending at 32 and 64 -> blocks 3 and 7).
expected = [False, False, False, True, False, False, False, True]
assert mask_dcp1[1] == expected
assert mask_dcp2[1] == expected
def test_store_mask_swa_wider_window_covers_more_blocks_per_lcm():
"""Same hybrid layout but sliding_window=16 (= 2 SWA blocks). Each lcm
boundary should now span two SWA tail blocks."""
@@ -18,11 +18,15 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator imp
MooncakeStoreCoordinator,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
BoundaryPut,
ChunkedTokenDatabase,
KeyMetadata,
LoadSpec,
ReqMeta,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler import ( # noqa: E501
_partial_tail_non_mamba_puts,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import ( # noqa: E501
KVCacheStoreRecvingThread,
KVCacheStoreSendingThread,
@@ -378,6 +382,15 @@ def test_chunked_token_database_hash_block_size_smaller_than_block_size():
assert out[1][2].hex() == fine_hashes[6].hex()
def _resolve_partial_tail(thread, req: ReqMeta) -> ReqMeta:
"""Add the tail's non-Mamba puts as the scheduler does before the worker."""
req.boundary_puts = [BoundaryPut(*put) for put in req.boundary_puts or []]
req.boundary_puts[:0] = _partial_tail_non_mamba_puts(
thread.coord, req, [db.block_size for db in thread.token_databases]
)
return req
def test_sub_block_partial_tail_offload_reads_cow_block():
"""Sub-block prompt (the 900/128/1536 shape, scaled to 12/4/16): the
partial tail is offloaded for both groups under the boundary sub-hash. The
@@ -394,7 +407,12 @@ def test_sub_block_partial_tail_offload_reads_cow_block():
KVCacheGroupSpec(["L0"], full),
KVCacheGroupSpec(["L1"], mamba),
]
coord = MooncakeStoreCoordinator(groups, scheduler_block_size=16, hash_block_size=4)
coord = MooncakeStoreCoordinator(
groups,
scheduler_block_size=16,
hash_block_size=4,
enable_partial_hash_hits=True,
)
assert coord.enable_partial_hash_hits
class _RecordingStore(_DictStore):
@@ -441,11 +459,13 @@ def test_sub_block_partial_tail_offload_reads_cow_block():
block_ids=([1], [2]),
block_hashes=hs,
can_save=True,
num_prompt_tokens=20,
boundary_state_offloads=[(1, mamba_cow_block, 12)],
num_prompt_tokens=13,
prefill_end_tokens=20,
completed_token_len=20,
boundary_puts=[(1, mamba_cow_block, 12)],
)
send._maybe_offload_boundary_states(req)
send._maybe_offload_boundary_states(_resolve_partial_tail(send, req))
# boundary = 12 // 4 * 4 = 12 -> keyed by hs[12 // 4 - 1] = hs[2].
partial_hash = hs[2]
@@ -473,7 +493,12 @@ def test_offload_syncs_event_before_put():
KVCacheGroupSpec(["L0"], full),
KVCacheGroupSpec(["L1"], mamba),
]
coord = MooncakeStoreCoordinator(groups, scheduler_block_size=16, hash_block_size=4)
coord = MooncakeStoreCoordinator(
groups,
scheduler_block_size=16,
hash_block_size=4,
enable_partial_hash_hits=True,
)
event = MagicMock()
class _FencedStore(_DictStore):
@@ -513,12 +538,13 @@ def test_offload_syncs_event_before_put():
block_hashes=hs,
can_save=True,
num_prompt_tokens=12,
completed_token_len=12,
store_job_id=1,
boundary_state_offloads=[(1, 7, 12)],
boundary_puts=[(1, 7, 8)],
)
req.current_event = event
send.add_request(req)
send.add_request(_resolve_partial_tail(send, req))
send._handle_request(send.request_queue.get())
assert send.request_queue.qsize() == 0
assert store._data
@@ -532,7 +558,8 @@ def test_sub_block_partial_tail_offload_covers_smaller_group_blocks():
offload must persist every FA block up to the boundary — the normal save
floors to the lcm, so those blocks are otherwise never written and the
consumer's per-group lookup would miss. The mamba boundary block still
reads the core-provided CoW target."""
reads the core-provided CoW target. Without EAGLE, the 12-token prompt's
resend-safe checkpoint is 8, leaving two attention blocks to save."""
full = FullAttentionSpec(block_size=4, num_kv_heads=8, head_size=64, dtype=None)
mamba = MambaSpec(
block_size=16,
@@ -544,7 +571,12 @@ def test_sub_block_partial_tail_offload_covers_smaller_group_blocks():
KVCacheGroupSpec(["L0"], full),
KVCacheGroupSpec(["L1"], mamba),
]
coord = MooncakeStoreCoordinator(groups, scheduler_block_size=16, hash_block_size=4)
coord = MooncakeStoreCoordinator(
groups,
scheduler_block_size=16,
hash_block_size=4,
enable_partial_hash_hits=True,
)
assert coord.enable_partial_hash_hits
class _RecordingStore(_DictStore):
@@ -590,19 +622,19 @@ def test_sub_block_partial_tail_offload_covers_smaller_group_blocks():
block_hashes=hs,
can_save=True,
num_prompt_tokens=12,
boundary_state_offloads=[(1, mamba_cow_block, 12)],
completed_token_len=12,
boundary_puts=[(1, mamba_cow_block, 8)],
)
send._maybe_offload_boundary_states(req)
send._maybe_offload_boundary_states(_resolve_partial_tail(send, req))
# FA (block 4): full blocks ending at 4, 8 and 12, keyed by their normal
# FA (block 4): full blocks ending at 4 and 8, keyed by their normal
# block-end hashes; mamba (block 16): the partial boundary block under
# the boundary sub-hash, read from the CoW target.
expected = {
token_dbs[0].key_for(hs[0]): [1 * 512],
token_dbs[0].key_for(hs[1]): [2 * 512],
token_dbs[0].key_for(hs[2]): [3 * 512],
token_dbs[1].key_for(hs[2]): [10_000 + mamba_cow_block * 512],
token_dbs[1].key_for(hs[1]): [10_000 + mamba_cow_block * 512],
}
assert store.puts == expected
@@ -611,11 +643,10 @@ def test_worker_lookup_hits_sub_block_partial_tail():
"""worker.lookup must query sub-block keys when partial hash hits are on.
Regression test: ``lookup`` hard-coded ``fine_grained = False``, so the
sub-block keys persisted by ``_sub_block_tail_puts`` were never probed and
partial prefix hits silently returned 0 while the store side kept writing
them. Here the mamba block (16) exceeds the hash unit (4), so
``enable_partial_hash_hits`` is on and the lookup must find the stored
boundary at 12.
partial tail's sub-block keys were never probed and partial prefix hits
silently returned 0 while the store side kept writing them. Here the
mamba block (16) exceeds the hash unit (4), so ``enable_partial_hash_hits``
is on and the lookup must find the stored boundary at 8.
"""
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
mamba = MambaSpec(
@@ -644,6 +675,8 @@ def test_worker_lookup_hits_sub_block_partial_tail():
KVCacheGroupSpec(["L0"], full),
KVCacheGroupSpec(["L1"], mamba),
],
# The hit alignment the engine core resolves for this config.
cache_hit_alignment_tokens=4,
)
vllm_config = _minimal_vllm_config(cache_block_size=16)
# Hash unit 4 < mamba block 16 -> partial hash hits are enabled.
@@ -672,7 +705,7 @@ def test_worker_lookup_hits_sub_block_partial_tail():
replicate_config=MagicMock(),
)
# Persist the sub-block partial tail at boundary 12 (keyed by hs[12//4-1]).
# A 12-token prompt leaves a resend-safe sub-block checkpoint at 8.
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(5)]
req = ReqMeta(
req_id="r0",
@@ -680,15 +713,16 @@ def test_worker_lookup_hits_sub_block_partial_tail():
block_ids=([1], [2]),
block_hashes=hs,
can_save=True,
num_prompt_tokens=20,
boundary_state_offloads=[(1, 7, 12)],
num_prompt_tokens=12,
completed_token_len=12,
boundary_puts=[(1, 7, 8)],
)
send_thread._maybe_offload_boundary_states(req)
send_thread._maybe_offload_boundary_states(_resolve_partial_tail(send_thread, req))
worker.store = store
# A 13-token prompt sharing the prefix must hit the stored boundary at 12.
assert worker.lookup(num_tokens=13, block_hashes=hs).hit_length == 12
# A 13-token prompt sharing the prefix must hit the stored boundary at 8.
assert worker.lookup(num_tokens=13, block_hashes=hs).hit_length == 8
def test_worker_setup_tolerates_finer_scratch_group():
@@ -744,6 +778,8 @@ def test_worker_setup_tolerates_finer_scratch_group():
KVCacheGroupSpec(["L1"], mamba),
KVCacheGroupSpec(["L2"], scratch),
],
# The hit alignment the engine core resolves for this config.
cache_hit_alignment_tokens=8,
)
vllm_config = _minimal_vllm_config(cache_block_size=16)
# Hash unit 8 divides the participating groups (16) but not the scratch
@@ -777,7 +813,7 @@ def test_worker_setup_tolerates_finer_scratch_group():
group_participates=[True, True],
)
# Persist the sub-block partial tail at boundary 12 (keyed by hs[12//8-1]).
# Persist the sub-block partial tail at boundary 8 (keyed by hs[8//8-1]).
hs = [BlockHash(bytes([i + 1]) * 8) for i in range(3)]
req = ReqMeta(
req_id="r0",
@@ -785,10 +821,11 @@ def test_worker_setup_tolerates_finer_scratch_group():
block_ids=([1], [2]),
block_hashes=hs,
can_save=True,
num_prompt_tokens=20,
boundary_state_offloads=[(1, 7, 12)],
num_prompt_tokens=12,
completed_token_len=12,
boundary_puts=[(1, 7, 8)],
)
send_thread._maybe_offload_boundary_states(req)
send_thread._maybe_offload_boundary_states(_resolve_partial_tail(send_thread, req))
worker.store = store
@@ -7,7 +7,11 @@ from unittest.mock import patch
import pytest
import torch
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import (
MooncakeStoreCoordinator,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
BoundaryPut,
LoadSpec,
MooncakeLookupResult,
MooncakeStoreConnectorMetadata,
@@ -17,6 +21,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler import (
MooncakeStoreScheduler,
_partial_tail_non_mamba_puts,
)
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
@@ -62,6 +67,31 @@ def _make_bare_scheduler(
scheduler._next_store_job_id = 0
scheduler._pinned_saves = {}
scheduler._boundary_state_group_ids = frozenset({1})
scheduler._store_coord = MooncakeStoreCoordinator(
[
KVCacheGroupSpec(
["attention"],
FullAttentionSpec(
block_size=max(16, hash_block_size),
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=max(16, hash_block_size),
shapes=((1,),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
max(16, hash_block_size),
hash_block_size,
enable_partial_hash_hits=enable_partial_hash_hits,
)
return scheduler
@@ -121,6 +151,7 @@ def _make_new_scheduler_output() -> SimpleNamespace:
req_id="req-0",
num_computed_tokens=0,
prompt_token_ids=list(range(32)),
num_prompt_tokens=32,
prefill_token_ids=None,
block_ids=([0, 1],),
block_hashes=[b"h0", b"h1"],
@@ -199,6 +230,7 @@ def _add_unfinished_request(
) -> None:
request = SimpleNamespace(
all_token_ids=token_ids,
num_prompt_tokens=prefill_end_tokens,
block_hashes=block_hashes,
num_output_placeholders=0,
)
@@ -268,6 +300,7 @@ def _make_qsa_hybrid_cache_config():
def test_scheduler_projects_nonprefix_groups_and_mamba_ids():
vllm_config = SimpleNamespace(
speculative_config=None,
kv_transfer_config=SimpleNamespace(
kv_role="kv_both", kv_connector_extra_config={}
),
@@ -528,6 +561,7 @@ def _make_pending_load_unfinished_request(
) -> None:
request = SimpleNamespace(
num_tokens=num_tokens,
num_prompt_tokens=num_tokens,
block_hashes=block_hashes,
num_output_placeholders=0,
)
@@ -594,6 +628,7 @@ def _make_resumed_unfinished_request(
) -> None:
request = SimpleNamespace(
all_token_ids=token_ids,
num_prompt_tokens=32,
block_hashes=block_hashes,
num_computed_tokens=num_computed_tokens,
num_output_placeholders=0,
@@ -796,13 +831,24 @@ def test_from_request_tracker_load_with_can_load_false_still_saves():
assert tracker.num_saved_tokens == 48
def test_from_request_tracker_no_load_saves_normally():
@pytest.mark.parametrize("token_len,save_partial_tail", [(48, False), (13, True)])
def test_from_request_tracker_no_load_saves_normally(token_len, save_partial_tail):
tracker = RequestTracker(
req_id="req-0",
token_len=48,
token_len=token_len,
allocated_block_ids=([0, 1, 2],),
num_saved_tokens=0,
prefill_end_tokens=token_len,
)
if save_partial_tail:
tracker.token_len = 8
assert (
ReqMeta.from_request_tracker(
tracker, 16, save_partial_tail=True, num_prompt_tokens=token_len
)
is None
)
tracker.token_len = token_len
req_meta = ReqMeta.from_request_tracker(
tracker,
@@ -810,12 +856,70 @@ def test_from_request_tracker_no_load_saves_normally():
load_spec=None,
skip_save=False,
block_hashes=[b"h0", b"h1", b"h2"],
save_partial_tail=save_partial_tail,
num_prompt_tokens=token_len,
)
assert req_meta is not None
assert req_meta.can_save is True
assert req_meta.load_spec is None
assert tracker.num_saved_tokens == 48
assert req_meta.token_len_chunk == token_len // 16 * 16
assert req_meta.completed_token_len == token_len
assert req_meta.boundary_puts is None
assert tracker.num_saved_tokens == token_len // 16 * 16
assert (
ReqMeta.from_request_tracker(
tracker,
16,
save_partial_tail=save_partial_tail,
num_prompt_tokens=token_len,
)
is None
)
def test_partial_tail_is_resolved_once_on_the_prompt_completing_save():
scheduler = _make_bare_scheduler(
hash_block_size=4, enable_partial_hash_hits=True, save_decode_cache=True
)
scheduler._store_group_ids = (0, 1)
_add_unfinished_request(
scheduler,
token_ids=list(range(48)),
block_hashes=[bytes([i]) for i in range(12)],
prefill_end_tokens=45,
)
block_ids = ([0, 1, 2], [50])
scheduler._request_trackers["req-0"].allocated_block_ids = block_ids
def step(num_computed_tokens: int, num_scheduled_tokens: int):
return scheduler.build_connector_meta(
SimpleNamespace(
finished_req_ids=set(),
preempted_req_ids=set(),
scheduled_new_reqs=[],
scheduled_cached_reqs=SimpleNamespace(
req_ids=["req-0"],
new_block_ids=[()],
num_computed_tokens=[num_computed_tokens],
resumed_req_ids=set(),
),
num_scheduled_tokens={"req-0": num_scheduled_tokens},
scheduled_spec_decode_tokens={},
kv_connector_block_state=_make_connector_block_state(block_ids),
)
).requests
# The save that completes the prompt carries the tail for the worker.
(req_meta,) = step(44, 1)
assert req_meta.publish_partial_tail
assert req_meta.boundary_puts == [(0, 2, 44)]
# A later save does not recompute or resend it.
(req_meta,) = step(45, 3)
assert req_meta.token_len_chunk == 48
assert not req_meta.publish_partial_tail
assert req_meta.boundary_puts is None
class _StubLookupClient:
@@ -1018,7 +1122,7 @@ def _add_pending_partial_tail_request(
all_token_ids=list(range(num_tokens)),
block_hashes=block_hashes,
num_output_placeholders=0,
num_prompt_tokens=12,
num_prompt_tokens=13,
)
scheduler._unfinished_requests["req-0"] = (request, block_ids)
scheduler._request_trackers["req-0"] = RequestTracker(
@@ -1057,7 +1161,7 @@ def test_pending_partial_tail_emits_offload_only_reqmeta():
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
out = _add_pending_partial_tail_request(
scheduler,
num_tokens=12,
num_tokens=13,
block_hashes=[b"h0", b"h1", b"h2"],
block_ids=([0],),
)
@@ -1069,8 +1173,8 @@ def test_pending_partial_tail_emits_offload_only_reqmeta():
assert req_meta.req_id == "req-0"
assert req_meta.can_save is True
assert req_meta.token_len_chunk == 0
assert req_meta.boundary_state_offloads == [(1, 7, 12)]
assert req_meta.num_prompt_tokens == 12
assert req_meta.boundary_puts == [(1, 7, 12)]
assert req_meta.num_prompt_tokens == 13
assert req_meta.block_ids == ([0],)
store_job_id = req_meta.store_job_id
assert scheduler._pinned_saves[store_job_id][0] == [7]
@@ -1081,34 +1185,71 @@ def test_pending_partial_tail_emits_offload_only_reqmeta():
assert tracker.has_pending_offload is True
def test_finished_partial_tail_is_pre_pinned_as_store_job():
@pytest.mark.parametrize("attention_block_size", [4, 8, 16])
@pytest.mark.parametrize("use_eagle", [False, True])
@pytest.mark.parametrize("num_workers", [1, 2])
def test_finished_partial_tail_is_pre_pinned_as_store_job(
attention_block_size, use_eagle, num_workers
):
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
scheduler._store_group_ids = (0, 1)
scheduler.client = SimpleNamespace(discard=lambda *_: None)
scheduler._num_workers = num_workers
groups = list(scheduler._store_coord.kv_cache_groups)
groups[0] = KVCacheGroupSpec(
["attention"],
FullAttentionSpec(
block_size=attention_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
is_eagle_group=use_eagle,
)
scheduler._store_coord = MooncakeStoreCoordinator(
groups, 16, 4, use_eagle, enable_partial_hash_hits=True
)
prompt_tokens = 49 if use_eagle else 45
attention_ids = list(
range(1, (prompt_tokens + attention_block_size - 1) // attention_block_size + 1)
)
block_ids = (attention_ids, [50])
request = SimpleNamespace(
request_id="req-0",
block_hashes=[b"h0", b"h1", b"h2"],
block_hashes=[bytes([i]) for i in range(prompt_tokens // 4)],
num_computed_tokens=prompt_tokens,
num_prompt_tokens=prompt_tokens,
)
scheduler._request_trackers["req-0"] = RequestTracker(
req_id="req-0",
token_len=12,
allocated_block_ids=([3], [9]),
token_ids=list(range(12)),
prefill_end_tokens=12,
token_len=prompt_tokens,
allocated_block_ids=block_ids,
token_ids=list(range(prompt_tokens)),
prefill_end_tokens=prompt_tokens,
)
block_ids = ([3], [9])
pool = scheduler._gpu_block_pool
owned_ids = attention_ids + [50]
pool.touch([pool.blocks[i] for i in owned_ids])
delay_free = scheduler.register_finished_partial_tail(
request,
block_ids,
[(1, 9, 12)],
[(1, 50, 44)],
)
# The exact source is pinned immediately, so the request can be freed
# before the next connector metadata build.
assert delay_free is False
assert scheduler._gpu_block_pool.blocks[9].ref_cnt == 1
assert scheduler._gpu_block_pool.blocks[3].ref_cnt == 0
proof_end = 48 if use_eagle else 44
expected = attention_ids[
32 // attention_block_size : (proof_end + attention_block_size - 1)
// attention_block_size
]
pool.free_blocks([pool.blocks[i] for i in owned_ids])
assert all(
pool.blocks[i].ref_cnt == (1 if i in expected else 0) for i in attention_ids
)
assert pool.blocks[50].ref_cnt == 1
out = SimpleNamespace(
finished_req_ids={"req-0"},
@@ -1132,13 +1273,116 @@ def test_finished_partial_tail_is_pre_pinned_as_store_job():
assert req_meta.token_len_chunk == 0
assert req_meta.block_ids == block_ids
assert req_meta.block_hashes == request.block_hashes
assert req_meta.boundary_state_offloads == [(1, 9, 12)]
assert scheduler._pinned_saves[req_meta.store_job_id][0] == [9]
assert scheduler._gpu_block_pool.blocks[9].ref_cnt == 1
assert req_meta.boundary_puts == [
(0, attention_ids[idx], min((idx + 1) * attention_block_size, proof_end))
for idx in range(
32 // attention_block_size,
(proof_end + attention_block_size - 1) // attention_block_size,
)
] + [(1, 50, 44)]
assert set(scheduler._pinned_saves[req_meta.store_job_id][0]) == {50, *expected}
assert all(pool.blocks[i].ref_cnt == 1 for i in expected + [50])
assert scheduler._finished_partial_tail_metas == {}
for _ in range(num_workers - 1):
scheduler.update_connector_output(
_make_worker_output({req_meta.store_job_id: 1})
)
assert all(pool.blocks[i].ref_cnt == 1 for i in expected + [50])
scheduler.update_connector_output(_make_worker_output({req_meta.store_job_id: 1}))
assert scheduler._gpu_block_pool.blocks[9].ref_cnt == 0
assert all(pool.blocks[i].ref_cnt == 0 for i in owned_ids)
def test_partial_tail_boundary_follows_eagle_block_drop_not_group_flags():
"""Core drops the Mamba checkpoint whenever EAGLE block drop is on, even if
only the Mamba group carries the eagle flag."""
groups = [
KVCacheGroupSpec(
["attention"],
FullAttentionSpec(
block_size=16, num_kv_heads=1, head_size=1, dtype=torch.float32
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=16,
shapes=((1,),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
is_eagle_group=True,
),
]
coord = MooncakeStoreCoordinator(groups, 16, 4, use_eagle=True)
# Core's checkpoint for a 49-token prompt: 48 minus one EAGLE hash unit.
req_meta = ReqMeta(
req_id="req-0",
token_len_chunk=0,
block_ids=([1, 2, 3, 4], [50]),
block_hashes=[bytes([i]) for i in range(12)],
num_prompt_tokens=49,
completed_token_len=49,
boundary_puts=[BoundaryPut(1, 50, 44)],
)
assert _partial_tail_non_mamba_puts(coord, req_meta, [16, 16]) == [
BoundaryPut(0, 3, 44)
]
@pytest.mark.parametrize(
"completed, mamba_tails, publish, expected",
[
# The chunk ending at the junction (40) hands off its Mamba state. Its
# attention KV comes from the normal saves, so nothing is added here.
(40, [40], False, []),
# The prompt-completing save: blocks from the LCM floor 64 up to the
# checkpoint 68 plus the EAGLE proof unit, whether or not the junction
# hand-off is in the same save.
(75, [68], True, [BoundaryPut(0, 17, 68), BoundaryPut(0, 18, 72)]),
(75, [40, 68], True, [BoundaryPut(0, 17, 68), BoundaryPut(0, 18, 72)]),
],
)
def test_shared_prefix_junction_handoff_adds_no_attention_tail(
completed, mamba_tails, publish, expected
):
"""``--enable-mamba-shared-prefix-checkpoint`` also hands off a sub-block
Mamba state at the shared-prefix junction, below the prompt checkpoint."""
groups = [
KVCacheGroupSpec(
["attention"],
FullAttentionSpec(
block_size=4, num_kv_heads=1, head_size=1, dtype=torch.float32
),
is_eagle_group=True,
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=16,
shapes=((1,),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
]
coord = MooncakeStoreCoordinator(
groups, 16, 4, use_eagle=True, enable_partial_hash_hits=True
)
# 75-token prompt: checkpoint 72 - 4 = 68.
req_meta = ReqMeta(
req_id="req-0",
token_len_chunk=0,
block_ids=(list(range(1, 20)), [50, 51, 52, 53, 54]),
block_hashes=[bytes([i]) for i in range(18)],
num_prompt_tokens=75,
completed_token_len=completed,
boundary_puts=[BoundaryPut(1, 60 + i, n) for i, n in enumerate(mamba_tails)],
publish_partial_tail=publish,
)
assert _partial_tail_non_mamba_puts(coord, req_meta, [4, 16]) == expected
def test_decode_boundary_state_offload_dropped_unclaimed():
@@ -1235,7 +1479,7 @@ def test_boundary_state_group_ids_are_remapped_to_store_projection():
scheduler._handle_boundary_state_offloads({"req-0": [(2, 7, 800)]}, meta)
assert meta.requests[0].boundary_state_offloads == [(1, 7, 800)]
assert meta.requests[0].boundary_puts == [(1, 7, 800)]
def test_resumed_prefill_claims_boundaries_past_prompt_length():
@@ -1245,13 +1489,13 @@ def test_resumed_prefill_claims_boundaries_past_prompt_length():
# missing for boundaries full attention does store and no joint hit could
# complete there.
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
_register_offload_request(scheduler, prefill_end_tokens=20, num_prompt_tokens=12)
out = _make_offload_only_output([(1, 7, 16), (1, 9, 20), (1, 11, 24)])
_register_offload_request(scheduler, prefill_end_tokens=37, num_prompt_tokens=13)
out = _make_offload_only_output([(1, 7, 16), (1, 9, 32), (1, 11, 48)])
meta = scheduler.build_connector_meta(out)
# 16 and 20 are inside the resumed prefill; 24 is past it.
assert meta.requests[0].boundary_state_offloads == [(1, 7, 16), (1, 9, 20)]
# Aligned states at 16 and 32 are inside the resumed prefill; 48 is past it.
assert meta.requests[0].boundary_puts == [(1, 7, 16), (1, 9, 32)]
store_job_id = meta.requests[0].store_job_id
assert scheduler._pinned_saves[store_job_id][0] == [7, 9]
@@ -1267,7 +1511,7 @@ def test_boundary_state_job_pins_exact_blocks_once():
meta = scheduler.build_connector_meta(out)
req_meta = meta.requests[0]
assert req_meta.boundary_state_offloads == [
assert req_meta.boundary_puts == [
(1, 7, 16),
(1, 8, 32),
(1, 9, 48),
@@ -1294,6 +1538,7 @@ def test_store_job_pins_current_non_null_non_mamba_blocks():
request = SimpleNamespace(
all_token_ids=list(range(48)),
block_hashes=[bytes([i]) for i in range(12)],
num_prompt_tokens=48,
num_output_placeholders=0,
)
scheduler._unfinished_requests["req-0"] = (request, ([7, 2, 0], [21, 0, 22]))
@@ -1379,7 +1624,7 @@ def test_preemption_and_request_id_reuse_do_not_release_inflight_job():
_register_offload_request(scheduler, prefill_end_tokens=64, num_prompt_tokens=64)
second = scheduler.build_connector_meta(_make_offload_only_output([(1, 8, 32)]))
second_job_id = second.requests[0].store_job_id
assert second.requests[0].boundary_state_offloads == [(1, 8, 32)]
assert second.requests[0].boundary_puts == [(1, 8, 32)]
scheduler.update_connector_output(_make_worker_output({first_job_id: 1}))
assert scheduler._gpu_block_pool.blocks[7].ref_cnt == 0
@@ -1412,9 +1657,9 @@ def test_resumed_partial_tail_uses_exact_boundary():
meta = scheduler.build_connector_meta(out)
assert len(meta.requests) == 1
assert meta.requests[0].boundary_state_offloads == [(1, 7, 12)]
# Ordinary metadata retains the full resumed prefill range.
assert meta.requests[0].num_prompt_tokens == 20
assert meta.requests[0].boundary_puts == [(1, 7, 12)]
assert meta.requests[0].num_prompt_tokens == 13
assert meta.requests[0].prefill_end_tokens == 20
tracker = scheduler._request_trackers["req-0"]
assert tracker.num_saved_tokens == 0
assert tracker.has_pending_offload is True
@@ -1427,7 +1672,7 @@ def test_resumed_partial_tail_attached_to_save_keeps_exact_boundary():
all_token_ids=list(range(48)),
block_hashes=[b"h0", b"h1", b"h2"],
num_output_placeholders=0,
num_prompt_tokens=36,
num_prompt_tokens=37,
)
scheduler._unfinished_requests["req-0"] = (request, ([0, 1],))
scheduler._request_trackers["req-0"] = RequestTracker(
@@ -1445,8 +1690,9 @@ def test_resumed_partial_tail_attached_to_save_keeps_exact_boundary():
assert len(meta.requests) == 1
assert meta.requests[0].can_save is True
assert meta.requests[0].boundary_state_offloads == [(0, 7, 36)]
assert meta.requests[0].num_prompt_tokens == 48
assert meta.requests[0].boundary_puts == [(0, 7, 36)]
assert meta.requests[0].num_prompt_tokens == 37
assert meta.requests[0].prefill_end_tokens == 48
# Ordinary saving still covers the full resumed prefill range.
tracker = scheduler._request_trackers["req-0"]
assert tracker.num_saved_tokens == 48
@@ -1460,7 +1706,7 @@ def test_partial_tail_cow_block_is_referenced_for_the_job():
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
out = _add_pending_partial_tail_request(
scheduler,
num_tokens=12,
num_tokens=13,
block_hashes=[b"h0", b"h1", b"h2"],
block_ids=([0],),
)
@@ -30,6 +30,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
BlobBlockHashes,
BoundaryPut,
ChunkedTokenDatabase,
KeyMetadata,
LBHNCStoreLayout,
@@ -44,6 +45,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import (
MooncakeStoreConnectorStats,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler import ( # noqa: E501
_partial_tail_non_mamba_puts,
)
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.core.kv_cache_utils import BlockHash, maybe_convert_block_hash
from vllm.v1.kv_cache_interface import (
@@ -942,6 +946,15 @@ def test_store_sending_thread_retries_skipped_range_after_pressure():
assert store.batch_put_from_multi_buffers.call_args.args[0] == keys
def _resolve_partial_tail(thread, req: ReqMeta) -> ReqMeta:
"""Add the tail's non-Mamba puts as the scheduler does before the worker."""
req.boundary_puts = [BoundaryPut(*put) for put in req.boundary_puts or []]
req.boundary_puts[:0] = _partial_tail_non_mamba_puts(
thread.coord, req, [db.block_size for db in thread.token_databases]
)
return req
def _make_partial_tail_send_thread(
store,
*,
@@ -954,6 +967,8 @@ def _make_partial_tail_send_thread(
hash_block_size=4,
lcm_block_size=16,
mamba_group_ids={1},
use_eagle=False,
eagle_proof_margin_by_group={},
)
db = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0),
@@ -987,17 +1002,74 @@ def _make_partial_tail_req(block_ids: list[int]) -> ReqMeta:
block_ids=(block_ids, [1]),
block_hashes=[b"a0", b"a1", b"a2"],
can_save=True,
boundary_state_offloads=[(1, 7, 12)],
boundary_puts=[(1, 7, 12)],
num_prompt_tokens=13,
completed_token_len=13,
)
@pytest.mark.parametrize("use_eagle", [False, True])
def test_partial_tail_ignores_handoff_off_prompt_boundary(use_eagle):
"""A Mamba hand-off away from the prompt checkpoint (a shared-prefix
junction) gets no attention tail; only its own Mamba put remains."""
store = MagicMock()
thread = _make_partial_tail_send_thread(store)
thread.coord.use_eagle = use_eagle
thread.coord.eagle_proof_margin_by_group = {0: 4} if use_eagle else {}
req = _make_partial_tail_req([1, 2, 3])
req.num_prompt_tokens = 17 if use_eagle else 13
req.boundary_puts = [(1, 7, 8)]
assert _resolve_partial_tail(thread, req).boundary_puts == [(1, 7, 8)]
def test_eagle_attention_proof_published_after_checkpoint_handoff():
store = MagicMock()
stored = set()
store.batch_is_exist.side_effect = lambda keys: [int(k in stored) for k in keys]
def put(keys, *args):
stored.update(keys)
return [256] * len(keys)
store.batch_put_from_multi_buffers.side_effect = put
thread = _make_partial_tail_send_thread(store)
thread.coord.use_eagle = True
thread.coord.eagle_proof_margin_by_group = {0: 4}
metadata = _make_partial_tail_req([1, 2, 3])
metadata.num_prompt_tokens = 13
metadata.completed_token_len = 8
metadata.boundary_puts = [(1, 7, 8)]
assert thread._maybe_offload_boundary_states(
_resolve_partial_tail(thread, metadata)
)
mamba_key = thread.token_databases[1].key_for(b"a1")
attention_key = thread.token_databases[0].key_for(b"a2")
assert mamba_key in stored
assert attention_key not in stored
metadata.completed_token_len = 13
metadata.boundary_puts = None
metadata.publish_partial_tail = True
assert thread._maybe_offload_boundary_states(
_resolve_partial_tail(thread, metadata)
)
assert attention_key in stored
keys, addrs, *_ = store.batch_put_from_multi_buffers.call_args.args
assert addrs[keys.index(attention_key)] == [0x1000 + 3 * 256]
assert mamba_key not in keys
def test_partial_tail_offload_skips_null_source_blocks():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
thread = _make_partial_tail_send_thread(store)
assert thread._maybe_offload_boundary_states(_make_partial_tail_req([0, 2, 3]))
assert thread._maybe_offload_boundary_states(
_resolve_partial_tail(thread, _make_partial_tail_req([0, 2, 3]))
)
keys, addrs, _sizes, _replicate_config = (
store.batch_put_from_multi_buffers.call_args.args
@@ -1131,9 +1203,11 @@ def test_partial_tail_offload_skips_cap_omitted_mamba_group():
block_hashes=[b"a0", b"a1", b"a2"],
can_save=True,
# The scheduler accepted group 1 and omitted group 2 at boundary 12.
boundary_state_offloads=[(1, 7, 12)],
boundary_puts=[(1, 7, 12)],
num_prompt_tokens=13,
completed_token_len=13,
)
assert thread._maybe_offload_boundary_states(req)
assert thread._maybe_offload_boundary_states(_resolve_partial_tail(thread, req))
keys, addrs, _sizes, _replicate_config = (
store.batch_put_from_multi_buffers.call_args.args
@@ -1156,7 +1230,9 @@ def test_partial_tail_offload_replaces_stale_group_ids_after_filtering():
supports_group_ids=True,
)
assert thread._maybe_offload_boundary_states(_make_partial_tail_req([1, 2, 3]))
assert thread._maybe_offload_boundary_states(
_resolve_partial_tail(thread, _make_partial_tail_req([1, 2, 3]))
)
keys, _addrs, _sizes, config = store.batch_put_from_multi_buffers.call_args.args
assert keys == [
@@ -1190,13 +1266,17 @@ def test_partial_tail_put_failure_activates_pressure_gate():
store.batch_put_from_multi_buffers.return_value = [256, -200, 256]
thread = _make_partial_tail_send_thread(store)
_run_store_req(thread, _make_partial_tail_req([1, 2, 3]))
_run_store_req(
thread, _resolve_partial_tail(thread, _make_partial_tail_req([1, 2, 3]))
)
assert thread._store_pressure_active is True
assert thread._skip_store_requests == {"req-a"}
assert thread._saved_offset.get("req-a", 0) == 0
_run_store_req(thread, _make_partial_tail_req([1, 2, 3]))
_run_store_req(
thread, _resolve_partial_tail(thread, _make_partial_tail_req([1, 2, 3]))
)
assert store.batch_put_from_multi_buffers.call_count == 1
@@ -1277,9 +1357,9 @@ def test_block_aligned_snapshot_offload_uses_provided_block():
block_ids=([1, 2, 3], [5]),
block_hashes=hs,
can_save=True,
boundary_state_offloads=[(1, 7, 32)],
boundary_puts=[(1, 7, 32)],
)
assert thread._maybe_offload_boundary_states(req)
assert thread._maybe_offload_boundary_states(_resolve_partial_tail(thread, req))
keys, addrs, _sizes, _ = store.batch_put_from_multi_buffers.call_args.args
# boundary 32 is block-aligned for the mamba group (block 16): one key,
@@ -1289,7 +1369,9 @@ def test_block_aligned_snapshot_offload_uses_provided_block():
assert addrs == [[0x2000 + 7 * 256]]
def test_mixed_snapshot_and_sub_block_offloads():
@pytest.mark.parametrize("saved_tokens", [0, 32])
@pytest.mark.parametrize("use_eagle", [False, True])
def test_mixed_snapshot_and_sub_block_offloads(saved_tokens, use_eagle):
"""A retention snapshot and the prompt-end sub-block CoW tail can arrive
in one hand-off; the sub-block path covers FA gap blocks but reads the
mamba boundary only from the CoW block, never positionally."""
@@ -1297,39 +1379,150 @@ def test_mixed_snapshot_and_sub_block_offloads():
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
thread = _make_partial_tail_send_thread(store)
thread.coord.use_eagle = use_eagle
thread.coord.eagle_proof_margin_by_group = {0: 4} if use_eagle else {}
thread._saved_offset["req-a"] = saved_tokens
hs = [bytes([i + 1]) * 4 for i in range(11)] # 11 hash units = 44 tokens
hs = [bytes([i + 1]) * 4 for i in range(12)]
req = ReqMeta(
req_id="req-a",
token_len_chunk=0,
block_ids=([1, 2, 3], [0, 0, 0]),
block_ids=(list(range(1, 14)), [0, 0, 0]),
block_hashes=hs,
can_save=True,
boundary_state_offloads=[(1, 9, 32), (1, 7, 44)],
boundary_puts=[(1, 9, 32), (1, 7, 44)],
num_prompt_tokens=49 if use_eagle else 45,
completed_token_len=49 if use_eagle else 45,
)
assert thread._maybe_offload_boundary_states(req)
assert thread._maybe_offload_boundary_states(_resolve_partial_tail(thread, req))
keys, addrs, _sizes, _ = store.batch_put_from_multi_buffers.call_args.args
db_full, db_mamba = thread.token_databases
proof_end = 48 if use_eagle else 44
assert keys == [
# The tail reads only the LCM gap, even when normal saves lag.
*[db_full.key_for(BlockHash(hs[i])) for i in range(8, proof_end // 4)],
# Aligned snapshot: boundary 32 from the handed-off block 9.
db_mamba.key_for(BlockHash(hs[7])),
# Sub-block boundary 44: FA gap blocks ending at 4, 8, 12.
db_full.key_for(BlockHash(hs[0])),
db_full.key_for(BlockHash(hs[1])),
db_full.key_for(BlockHash(hs[2])),
# Mamba boundary block from the CoW hand-off (block 7).
db_mamba.key_for(BlockHash(hs[10])),
]
assert addrs == [
*[[0x1000 + (i + 1) * 256] for i in range(8, proof_end // 4)],
[0x2000 + 9 * 256],
[0x1000 + 1 * 256],
[0x1000 + 2 * 256],
[0x1000 + 3 * 256],
[0x2000 + 7 * 256],
]
@pytest.mark.parametrize("tp_rank", [0, 1])
def test_partial_tail_with_smaller_mamba_blocks_writes_one_mamba_key(tp_rank):
"""Under DCP2 the attention block spans the LCM (16) and Mamba blocks are
half that. The prompt-completing save writes one attention block and one
Mamba key: the boundary state from the CoW block, never the interior
Mamba blocks."""
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
coord = SimpleNamespace(
enable_partial_hash_hits=True,
hash_block_size=4,
lcm_block_size=16,
mamba_group_ids={1},
use_eagle=False,
eagle_proof_margin_by_group={},
)
db_full = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0), block_size=16, hash_block_size=4
)
db_full.set_kv_caches_base_addr([0x1000])
db_full.set_block_len([256])
db_mamba = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
block_size=8,
hash_block_size=4,
)
db_mamba.set_kv_caches_base_addr([0x2000])
db_mamba.set_block_len([256])
thread = _make_store_sending_thread(
store, coord=coord, token_databases=[db_full, db_mamba], tp_rank=tp_rank
)
# Mamba is replicated across two ranks, which split its keys by block.
thread.group_put_steps = [1, 2]
hs = [bytes([i + 1]) * 4 for i in range(11)]
req = ReqMeta(
req_id="req-a",
token_len_chunk=0,
block_ids=([1, 2, 3], list(range(20, 26))),
block_hashes=hs,
can_save=True,
boundary_puts=[(1, 7, 44)],
num_prompt_tokens=45,
completed_token_len=45,
publish_partial_tail=True,
)
assert thread._maybe_offload_boundary_states(_resolve_partial_tail(thread, req))
assert req.boundary_puts == [(0, 3, 44), (1, 7, 44)]
keys, addrs, _sizes, _ = store.batch_put_from_multi_buffers.call_args.args
# Mamba block cdiv(44, 8) - 1 = 5 belongs to put_step_rank 5 % 2 = 1,
# which is tp_rank 0 for group 1.
mamba_puts = [db_mamba.key_for(BlockHash(hs[10]))] if tp_rank == 0 else []
assert keys == [db_full.key_for(BlockHash(hs[10])), *mamba_puts]
assert addrs == [[0x1000 + 3 * 256], *([[0x2000 + 7 * 256]] * len(mamba_puts))]
def test_shared_prefix_junction_handoff_writes_only_its_mamba_key():
"""A junction hand-off (``--enable-mamba-shared-prefix-checkpoint``) below
the prompt checkpoint stores its Mamba state under the junction hash and
no attention tail."""
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
coord = SimpleNamespace(
enable_partial_hash_hits=True,
hash_block_size=4,
lcm_block_size=16,
mamba_group_ids={1},
use_eagle=True,
eagle_proof_margin_by_group={0: 4},
)
db_full = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0), block_size=4, hash_block_size=4
)
db_full.set_kv_caches_base_addr([0x1000])
db_full.set_block_len([256])
db_mamba = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
block_size=16,
hash_block_size=4,
)
db_mamba.set_kv_caches_base_addr([0x2000])
db_mamba.set_block_len([256])
thread = _make_store_sending_thread(
store, coord=coord, token_databases=[db_full, db_mamba]
)
hs = [bytes([i + 1]) * 4 for i in range(18)]
# 75-token prompt (checkpoint 68), chunk cut at the junction 40.
req = ReqMeta(
req_id="req-b",
token_len_chunk=0,
block_ids=(list(range(1, 11)), [20, 21, 22]),
block_hashes=hs,
can_save=True,
boundary_puts=[(1, 7, 40)],
num_prompt_tokens=75,
completed_token_len=40,
)
assert thread._maybe_offload_boundary_states(_resolve_partial_tail(thread, req))
assert req.boundary_puts == [(1, 7, 40)]
keys, addrs, _sizes, _ = store.batch_put_from_multi_buffers.call_args.args
assert keys == [db_mamba.key_for(BlockHash(hs[9]))]
assert addrs == [[0x2000 + 7 * 256]]
def test_snapshot_offload_skips_null_handoff_block():
"""A hand-off the core could not materialize (null block) carries no
committed state; persisting it would poison the boundary key."""
@@ -1338,13 +1531,16 @@ def test_snapshot_offload_skips_null_handoff_block():
hs = [bytes([i + 1]) * 4 for i in range(8)]
assert thread._maybe_offload_boundary_states(
ReqMeta(
req_id="req-a",
token_len_chunk=0,
block_ids=([1, 2, 3], [5]),
block_hashes=hs,
can_save=True,
boundary_state_offloads=[(1, NULL_BLOCK_ID, 32)],
_resolve_partial_tail(
thread,
ReqMeta(
req_id="req-a",
token_len_chunk=0,
block_ids=([1, 2, 3], [5]),
block_hashes=hs,
can_save=True,
boundary_puts=[(1, NULL_BLOCK_ID, 32)],
),
)
)
store.batch_is_exist.assert_not_called()
@@ -1398,6 +1594,7 @@ def test_store_sending_thread_delta_saves_only_new_masked_chunks():
[True, False],
),
)
coord.store_mask = MagicMock(side_effect=coord.store_mask)
db_full = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0),
@@ -1427,6 +1624,8 @@ def test_store_sending_thread_delta_saves_only_new_masked_chunks():
block_ids=([0, 1, 2, 3], [0, 1, 2, 3]),
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
can_save=True,
num_prompt_tokens=33,
prefill_end_tokens=64,
),
)
@@ -1436,6 +1635,7 @@ def test_store_sending_thread_delta_saves_only_new_masked_chunks():
assert full_hashes == [b"a2".hex(), b"a3".hex()]
assert masked_hashes == [b"a2".hex()]
coord.store_mask.assert_called_once_with(64, 32, num_prompt_tokens=64)
def test_store_sending_thread_prepares_missing_chunks_once_per_group():
@@ -3580,6 +3780,139 @@ def _make_bare_worker(
return worker
@pytest.mark.parametrize("use_eagle", [False, True])
@pytest.mark.parametrize("prefix_match_unit", [None, 8])
@pytest.mark.parametrize("dcp_world_size,mamba_block_size", [(1, 32), (8, 16)])
def test_mooncake_lookup_reuses_resolved_hash_checkpoint(
use_eagle, prefix_match_unit, dcp_world_size, mamba_block_size
):
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
coordinator as mooncake_coordinator,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler import (
MooncakeStoreScheduler,
)
groups = [
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=16, num_kv_heads=1, head_size=1, dtype=torch.float32
),
is_eagle_group=use_eagle,
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=mamba_block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
]
config = SimpleNamespace(
speculative_config=None,
kv_transfer_config=SimpleNamespace(
kv_role="kv_both", kv_connector_extra_config={}
),
kv_events_config=None,
cache_config=SimpleNamespace(
block_size=16,
enable_prefix_caching=True,
prefix_match_unit=prefix_match_unit,
),
parallel_config=SimpleNamespace(
decode_context_parallel_size=dcp_world_size, world_size=dcp_world_size
),
)
with patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"scheduler.LookupKeyClient"
):
scheduler = MooncakeStoreScheduler(
config,
KVCacheConfig(
num_blocks=128,
kv_cache_tensors=[],
kv_cache_groups=groups,
# The hit alignment the engine core resolves for this config.
cache_hit_alignment_tokens=prefix_match_unit or 16,
),
)
assert scheduler.enable_partial_hash_hits
hash_block_size = prefix_match_unit or 16
assert scheduler._hash_block_size == hash_block_size
coordinator = mooncake_coordinator.MooncakeStoreCoordinator(
groups,
scheduler_block_size=math.lcm(16 * dcp_world_size, mamba_block_size),
hash_block_size=hash_block_size,
enable_partial_hash_hits=True,
)
hashes = [BlockHash(bytes([i]) * 16) for i in range(512 // hash_block_size)]
pool = mooncake_coordinator.ExternalCachedBlockPool(
hash_block_size=hash_block_size,
exists={(group, bytes(h)) for group in range(2) for h in hashes},
)
_, hit = coordinator.find_longest_cache_hit(hashes, 511, pool)
assert hit == 512 - hash_block_size
@pytest.mark.parametrize("alignment, expected", [(8, True), (32, False), (None, False)])
def test_scheduler_follows_core_hit_alignment(alignment, expected):
"""The store takes partial hits from core's resolved alignment, not from its
own groups: these Mamba specs alone would allow them, but core may still
keep hits on the block (e.g. for a sliding-window group it never sees)."""
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler import (
MooncakeStoreScheduler,
)
groups = [
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=32, num_kv_heads=1, head_size=1, dtype=torch.float32
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=32,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
]
config = SimpleNamespace(
speculative_config=None,
kv_transfer_config=SimpleNamespace(
kv_role="kv_both", kv_connector_extra_config={}
),
kv_events_config=None,
cache_config=SimpleNamespace(
block_size=32, enable_prefix_caching=True, prefix_match_unit=8
),
parallel_config=SimpleNamespace(decode_context_parallel_size=1, world_size=1),
)
with patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"scheduler.LookupKeyClient"
):
scheduler = MooncakeStoreScheduler(
config,
KVCacheConfig(
num_blocks=128,
kv_cache_tensors=[],
kv_cache_groups=groups,
cache_hit_alignment_tokens=alignment,
),
)
assert scheduler.enable_partial_hash_hits is expected
assert scheduler._store_coord.enable_partial_hash_hits is expected
def test_lookup_key_prefixes_cover_dcp_rank_namespaces():
worker = _make_bare_worker()
worker.tp_size = 4
@@ -3873,6 +4206,7 @@ def test_lookup_partial_tail_uses_hash_alignment():
worker._kv_cache_groups,
scheduler_block_size=16,
hash_block_size=4,
enable_partial_hash_hits=True,
)
_refresh_group_tp_replication_factors(worker)
worker.store.batch_is_exist.return_value = [0, 0, 1, 0, 0, 1]
@@ -3946,6 +4280,7 @@ def test_lookup_plan_resolves_group_tail_keys_from_existing_hashes():
scheduler_block_size=16,
hash_block_size=4,
use_eagle=True,
enable_partial_hash_hits=True,
)
_refresh_group_tp_replication_factors(worker)
hashes = [BlockHash(f"h{i}".encode()) for i in range(6)]
@@ -4015,6 +4350,7 @@ def test_lookup_plan_recovers_tail_key_after_multi_chunk_convergence():
worker._kv_cache_groups,
scheduler_block_size=16,
hash_block_size=4,
enable_partial_hash_hits=True,
)
_refresh_group_tp_replication_factors(worker)
hashes = [BlockHash(f"h{i}".encode()) for i in range(12)]