[Bugfix][Mooncake] Save the EAGLE attention proof with hybrid Mamba partial tails (#60695)
Signed-off-by: Dao Le <Dao007forever@gmail.com> Signed-off-by: Dao007forever <daole@inferact.ai> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
co-authored by
Wei Zhao
Claude Opus 5.5
zjy0516
parent
03a7d3776f
commit
f8e6e365ae
@@ -4,7 +4,12 @@
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from math import lcm
|
||||
@@ -60,6 +65,16 @@ SPEC_CONFIG = {
|
||||
}
|
||||
|
||||
NIXL = {"kv_connector": "NixlConnector", "kv_role": "kv_both"}
|
||||
MOONCAKE = {
|
||||
"kv_connector": "MooncakeStoreConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_load_failure_policy": "recompute",
|
||||
"kv_connector_extra_config": {
|
||||
"load_async": True,
|
||||
"lookup_async": True,
|
||||
"enable_cross_layers_blocks": False,
|
||||
},
|
||||
}
|
||||
# SimpleCPUOffloadConnector as in the Kimi-K3 recipe.
|
||||
OFFLOAD = {
|
||||
"kv_connector": "SimpleCPUOffloadConnector",
|
||||
@@ -243,10 +258,16 @@ def _tokens_text_logprobs(response: dict):
|
||||
|
||||
|
||||
def _reset_gpu_prefix_cache(url: str) -> None:
|
||||
response = requests.post(
|
||||
f"{url}/reset_prefix_cache", params={"reset_external": "false"}, timeout=60
|
||||
)
|
||||
response.raise_for_status()
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
response = requests.post(
|
||||
f"{url}/reset_prefix_cache", params={"reset_external": "false"}, timeout=60
|
||||
)
|
||||
response.raise_for_status()
|
||||
if response.json()["success"]:
|
||||
return
|
||||
assert time.monotonic() < deadline, "GPU cache reset timed out"
|
||||
time.sleep(0.1)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -354,6 +375,35 @@ def _format(turns: list[Turn]) -> str:
|
||||
return "\n".join(rows)
|
||||
|
||||
|
||||
def _check_resend_reuses_prompt_checkpoint(
|
||||
deployment: Deployment, prompt_len: int
|
||||
) -> None:
|
||||
"""Resends reuse the Mamba checkpoint and its companion EAGLE attention proof."""
|
||||
mode = Mode(128, True)
|
||||
salt = str(uuid4())
|
||||
rng = random.Random(0)
|
||||
prompt = [rng.randint(1000, 150000) for _ in range(prompt_len)]
|
||||
with _serve(deployment, mode) as servers:
|
||||
url = servers[0].url_root
|
||||
cold = _complete(url, prompt, salt, max_tokens=1)
|
||||
if deployment.offload:
|
||||
_reset_gpu_prefix_cache(url)
|
||||
resend = _complete(url, prompt, salt, max_tokens=1)
|
||||
recompute = _complete(url, prompt, f"{salt}-recompute", max_tokens=1)
|
||||
|
||||
# EAGLE's dropped hash unit already leaves tokens to recompute on resend.
|
||||
expected = (prompt_len // 128 - 1) * 128
|
||||
actual = _cached(resend)
|
||||
assert _cached(cold) == _cached(recompute) == 0
|
||||
assert actual == expected, f"{prompt_len=}: {actual=}, {expected=}"
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=[_tokens_text_logprobs(recompute)],
|
||||
outputs_1_lst=[_tokens_text_logprobs(resend)],
|
||||
name_0="recompute",
|
||||
name_1="resend",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"prompt_len",
|
||||
[
|
||||
@@ -368,28 +418,67 @@ def _format(turns: list[Turn]) -> str:
|
||||
],
|
||||
)
|
||||
def test_dspark_exact_resend_reuses_prompt_checkpoint(prompt_len: int) -> None:
|
||||
"""Resends reuse the Mamba checkpoint and its companion EAGLE attention proof."""
|
||||
mode = Mode(128, True)
|
||||
salt = str(uuid4())
|
||||
rng = random.Random(0)
|
||||
prompt = [rng.randint(1000, 150000) for _ in range(prompt_len)]
|
||||
deployment = DEPLOYMENTS["plain"]
|
||||
with _serve(deployment, mode) as servers:
|
||||
url = servers[0].url_root
|
||||
cold = _complete(url, prompt, salt, max_tokens=1)
|
||||
resend = _complete(url, prompt, salt, max_tokens=1)
|
||||
recompute = _complete(url, prompt, f"{salt}-recompute", max_tokens=1)
|
||||
_check_resend_reuses_prompt_checkpoint(DEPLOYMENTS["plain"], prompt_len)
|
||||
|
||||
# EAGLE's dropped hash unit already leaves tokens to recompute on resend.
|
||||
expected = (prompt_len // 128 - 1) * 128
|
||||
actual = _cached(resend)
|
||||
assert _cached(cold) == _cached(recompute) == 0
|
||||
assert actual == expected, f"{prompt_len=}: {actual=}, {expected=}"
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=[_tokens_text_logprobs(recompute)],
|
||||
outputs_1_lst=[_tokens_text_logprobs(resend)],
|
||||
name_0="recompute",
|
||||
name_1="resend",
|
||||
|
||||
@pytest.fixture
|
||||
def mooncake_store(tmp_path, monkeypatch) -> Iterator[None]:
|
||||
"""Run a Mooncake master and point the servers' store config at it."""
|
||||
if shutil.which("mooncake_master") is None:
|
||||
pytest.skip("mooncake_master is not installed")
|
||||
port = get_open_port()
|
||||
log_path = tmp_path / "mooncake_master.log"
|
||||
with open(log_path, "w") as log:
|
||||
master = subprocess.Popen(
|
||||
[
|
||||
"mooncake_master",
|
||||
f"--port={port}",
|
||||
f"--metrics_port={get_open_port()}",
|
||||
"--enable_metric_reporting=false",
|
||||
"--default_kv_lease_ttl=60000",
|
||||
],
|
||||
stdout=log,
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
try:
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
assert master.poll() is None, log_path.read_text()
|
||||
with (
|
||||
contextlib.suppress(OSError),
|
||||
socket.create_connection(("127.0.0.1", port)),
|
||||
):
|
||||
break
|
||||
assert time.monotonic() < deadline, "Mooncake master did not start"
|
||||
time.sleep(0.1)
|
||||
config_path = tmp_path / "mooncake.json"
|
||||
config_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"metadata_server": "P2PHANDSHAKE",
|
||||
"master_server_address": f"127.0.0.1:{port}",
|
||||
"protocol": "rdma",
|
||||
"device_name": os.getenv("MOONCAKE_DEVICE_NAME", "mlx5_12"),
|
||||
"global_segment_size": "4GB",
|
||||
"local_buffer_size": "1GB",
|
||||
}
|
||||
)
|
||||
)
|
||||
monkeypatch.setenv("MOONCAKE_CONFIG_PATH", str(config_path))
|
||||
monkeypatch.setenv("MC_MAX_MR_SIZE", str(4 << 30))
|
||||
yield
|
||||
finally:
|
||||
master.kill()
|
||||
master.wait()
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("mooncake_store")
|
||||
def test_mooncake_resend_reuses_prompt_checkpoint() -> None:
|
||||
"""Blocks=6144, PMU=128. P=7449 needs Mamba state at 7296 and attention proof
|
||||
at 7424, both beyond the normal Mooncake save boundary of 6144: ensure they
|
||||
remain reusable from the store after a GPU cache reset."""
|
||||
_check_resend_reuses_prompt_checkpoint(
|
||||
Deployment(Instance(kv_config=MOONCAKE), offload=True), 7449
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user