[Bugfix][Mooncake] Save the EAGLE attention proof with hybrid Mamba partial tails (#60695)

Signed-off-by: Dao Le <Dao007forever@gmail.com>
Signed-off-by: Dao007forever <daole@inferact.ai>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
Dao007forever
2026-10-10 00:08:08 +00:00
committed by GitHub
co-authored by Wei Zhao Claude Opus 5.5 zjy0516
parent 03a7d3776f
commit f8e6e365ae
11 changed files with 1142 additions and 390 deletions
+114 -25
View File
@@ -4,7 +4,12 @@
import contextlib
import json
import os
import random
import shutil
import socket
import subprocess
import time
from collections.abc import Iterator
from dataclasses import dataclass
from math import lcm
@@ -60,6 +65,16 @@ SPEC_CONFIG = {
}
NIXL = {"kv_connector": "NixlConnector", "kv_role": "kv_both"}
MOONCAKE = {
"kv_connector": "MooncakeStoreConnector",
"kv_role": "kv_both",
"kv_load_failure_policy": "recompute",
"kv_connector_extra_config": {
"load_async": True,
"lookup_async": True,
"enable_cross_layers_blocks": False,
},
}
# SimpleCPUOffloadConnector as in the Kimi-K3 recipe.
OFFLOAD = {
"kv_connector": "SimpleCPUOffloadConnector",
@@ -243,10 +258,16 @@ def _tokens_text_logprobs(response: dict):
def _reset_gpu_prefix_cache(url: str) -> None:
response = requests.post(
f"{url}/reset_prefix_cache", params={"reset_external": "false"}, timeout=60
)
response.raise_for_status()
deadline = time.monotonic() + 60
while True:
response = requests.post(
f"{url}/reset_prefix_cache", params={"reset_external": "false"}, timeout=60
)
response.raise_for_status()
if response.json()["success"]:
return
assert time.monotonic() < deadline, "GPU cache reset timed out"
time.sleep(0.1)
@dataclass(frozen=True)
@@ -354,6 +375,35 @@ def _format(turns: list[Turn]) -> str:
return "\n".join(rows)
def _check_resend_reuses_prompt_checkpoint(
deployment: Deployment, prompt_len: int
) -> None:
"""Resends reuse the Mamba checkpoint and its companion EAGLE attention proof."""
mode = Mode(128, True)
salt = str(uuid4())
rng = random.Random(0)
prompt = [rng.randint(1000, 150000) for _ in range(prompt_len)]
with _serve(deployment, mode) as servers:
url = servers[0].url_root
cold = _complete(url, prompt, salt, max_tokens=1)
if deployment.offload:
_reset_gpu_prefix_cache(url)
resend = _complete(url, prompt, salt, max_tokens=1)
recompute = _complete(url, prompt, f"{salt}-recompute", max_tokens=1)
# EAGLE's dropped hash unit already leaves tokens to recompute on resend.
expected = (prompt_len // 128 - 1) * 128
actual = _cached(resend)
assert _cached(cold) == _cached(recompute) == 0
assert actual == expected, f"{prompt_len=}: {actual=}, {expected=}"
check_logprobs_close(
outputs_0_lst=[_tokens_text_logprobs(recompute)],
outputs_1_lst=[_tokens_text_logprobs(resend)],
name_0="recompute",
name_1="resend",
)
@pytest.mark.parametrize(
"prompt_len",
[
@@ -368,28 +418,67 @@ def _format(turns: list[Turn]) -> str:
],
)
def test_dspark_exact_resend_reuses_prompt_checkpoint(prompt_len: int) -> None:
"""Resends reuse the Mamba checkpoint and its companion EAGLE attention proof."""
mode = Mode(128, True)
salt = str(uuid4())
rng = random.Random(0)
prompt = [rng.randint(1000, 150000) for _ in range(prompt_len)]
deployment = DEPLOYMENTS["plain"]
with _serve(deployment, mode) as servers:
url = servers[0].url_root
cold = _complete(url, prompt, salt, max_tokens=1)
resend = _complete(url, prompt, salt, max_tokens=1)
recompute = _complete(url, prompt, f"{salt}-recompute", max_tokens=1)
_check_resend_reuses_prompt_checkpoint(DEPLOYMENTS["plain"], prompt_len)
# EAGLE's dropped hash unit already leaves tokens to recompute on resend.
expected = (prompt_len // 128 - 1) * 128
actual = _cached(resend)
assert _cached(cold) == _cached(recompute) == 0
assert actual == expected, f"{prompt_len=}: {actual=}, {expected=}"
check_logprobs_close(
outputs_0_lst=[_tokens_text_logprobs(recompute)],
outputs_1_lst=[_tokens_text_logprobs(resend)],
name_0="recompute",
name_1="resend",
@pytest.fixture
def mooncake_store(tmp_path, monkeypatch) -> Iterator[None]:
"""Run a Mooncake master and point the servers' store config at it."""
if shutil.which("mooncake_master") is None:
pytest.skip("mooncake_master is not installed")
port = get_open_port()
log_path = tmp_path / "mooncake_master.log"
with open(log_path, "w") as log:
master = subprocess.Popen(
[
"mooncake_master",
f"--port={port}",
f"--metrics_port={get_open_port()}",
"--enable_metric_reporting=false",
"--default_kv_lease_ttl=60000",
],
stdout=log,
stderr=subprocess.STDOUT,
)
try:
deadline = time.monotonic() + 60
while True:
assert master.poll() is None, log_path.read_text()
with (
contextlib.suppress(OSError),
socket.create_connection(("127.0.0.1", port)),
):
break
assert time.monotonic() < deadline, "Mooncake master did not start"
time.sleep(0.1)
config_path = tmp_path / "mooncake.json"
config_path.write_text(
json.dumps(
{
"metadata_server": "P2PHANDSHAKE",
"master_server_address": f"127.0.0.1:{port}",
"protocol": "rdma",
"device_name": os.getenv("MOONCAKE_DEVICE_NAME", "mlx5_12"),
"global_segment_size": "4GB",
"local_buffer_size": "1GB",
}
)
)
monkeypatch.setenv("MOONCAKE_CONFIG_PATH", str(config_path))
monkeypatch.setenv("MC_MAX_MR_SIZE", str(4 << 30))
yield
finally:
master.kill()
master.wait()
@pytest.mark.usefixtures("mooncake_store")
def test_mooncake_resend_reuses_prompt_checkpoint() -> None:
"""Blocks=6144, PMU=128. P=7449 needs Mamba state at 7296 and attention proof
at 7424, both beyond the normal Mooncake save boundary of 6144: ensure they
remain reusable from the store after a GPU cache reset."""
_check_resend_reuses_prompt_checkpoint(
Deployment(Instance(kv_config=MOONCAKE), offload=True), 7449
)