This commit is contained in:
@@ -46,6 +46,13 @@ Now you can send requests to the proxy server through port 8000.
|
||||
- For headless instances, must be the same as the master instance
|
||||
- Each instance needs a unique port on its host; using the same port number across different hosts is fine
|
||||
|
||||
- `VLLM_MOONCAKE_CONNECTOR_TIMEOUT`: Per-request timeout (in seconds) for MooncakeConnector HTTP calls to the bootstrap server, including prefiller worker registration at startup. (Optional)
|
||||
- Default: 30.0
|
||||
- Global rank 0 hosts the bootstrap server in the same process that mounts the Mooncake transfer engine's host segment, so a large `MOONCAKE_GLOBAL_SEGMENT_SIZE` can delay responses for several seconds during startup
|
||||
- Raise this if bootstrap server timeout warnings appear on hosts with very large host-memory segments
|
||||
- Bootstrap registration uses up to 3 attempts internally, with exponential backoff capped at 10 seconds; connection errors and timeouts are both retried
|
||||
- Once the attempts are exhausted the caller raises instead of blocking, so failures surface rather than hanging
|
||||
|
||||
- `WITH_NVIDIA_PEERMEM`: Selects how mooncake registers GPU memory for RDMA. Read by mooncake, not vLLM.
|
||||
- Default: 1, which uses `ibv_reg_mr()` and requires the `nvidia-peermem` kernel module to be loaded
|
||||
- Set to 0 to use the DMA-BUF path, which does not need that module. Required on hosts where `nvidia-peermem` is not loaded, such as GB200
|
||||
|
||||
@@ -3,10 +3,12 @@
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import threading
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import torch
|
||||
import zmq.asyncio
|
||||
@@ -14,6 +16,7 @@ import zmq.asyncio
|
||||
from tests.v1.attention.utils import dense_kv_cache_views
|
||||
from vllm import envs
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import mooncake_connector
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import (
|
||||
KVConnectorRole,
|
||||
MooncakeConnector,
|
||||
@@ -33,6 +36,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector im
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import (
|
||||
MooncakeBootstrapServer,
|
||||
RegisterWorkerPayload,
|
||||
)
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -538,6 +542,46 @@ def bootstrap_server():
|
||||
server.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_register_worker_recovers_from_slow_bootstrap_server(monkeypatch):
|
||||
"""End-to-end: a real HTTP server that responds slower than the configured
|
||||
timeout must be retried on the wire, not treated as fatal."""
|
||||
original = MooncakeBootstrapServer.register_worker
|
||||
state = {"done": False, "calls": 0}
|
||||
|
||||
async def slow_register(self, payload: RegisterWorkerPayload):
|
||||
state["calls"] += 1
|
||||
if not state["done"]:
|
||||
state["done"] = True
|
||||
# Record the worker first, then stall past the client timeout, so
|
||||
# the retry exercises the duplicate-registration path.
|
||||
response = await original(self, payload)
|
||||
await asyncio.sleep(1.5)
|
||||
return response
|
||||
return await original(self, payload)
|
||||
|
||||
monkeypatch.setattr(MooncakeBootstrapServer, "register_worker", slow_register)
|
||||
|
||||
port = get_open_port()
|
||||
monkeypatch.setenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", str(port))
|
||||
monkeypatch.setenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "0.5")
|
||||
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 5)
|
||||
|
||||
server = MooncakeBootstrapServer("127.0.0.1", port)
|
||||
server.start()
|
||||
try:
|
||||
worker = _make_local_register_worker_stub()
|
||||
await MooncakeConnectorWorker.register_worker_with_bootstrap(worker)
|
||||
assert state["calls"] == 2
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get(f"http://127.0.0.1:{port}/query")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["0"]["engine_id"] == "eng-1"
|
||||
finally:
|
||||
server.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer):
|
||||
"""Tests the bootstrap server's api for worker registration and querying.
|
||||
@@ -589,9 +633,16 @@ async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer):
|
||||
assert data["0"]["worker_addr"]["0"]["0"] == "tcp://1.1.1.1:1111"
|
||||
assert data["0"]["worker_addr"]["0"]["1"] == "tcp://2.2.2.2:2222"
|
||||
|
||||
# Test failure: re-registering the same worker
|
||||
# Re-registering the identical payload is idempotent.
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.post(f"{base_url}/register", json=payload1)
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"status": "ok"}
|
||||
|
||||
# Test failure: same ranks, conflicting address
|
||||
payload1_conflict = dict(payload1, addr="tcp://9.9.9.9:9999")
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.post(f"{base_url}/register", json=payload1_conflict)
|
||||
assert response.status_code == 400
|
||||
assert "is already registered" in response.text
|
||||
|
||||
@@ -628,6 +679,137 @@ def _make_bootstrap_vllm_config(
|
||||
)
|
||||
|
||||
|
||||
def _make_local_register_worker_stub():
|
||||
return SimpleNamespace(
|
||||
vllm_config=SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
local_engines_only=False,
|
||||
data_parallel_rank_local=0,
|
||||
data_parallel_index=0,
|
||||
nnodes_within_dp=1,
|
||||
master_addr="127.0.0.1",
|
||||
data_parallel_master_ip="127.0.0.1",
|
||||
)
|
||||
),
|
||||
hostname="127.0.0.1",
|
||||
side_channel_port=1234,
|
||||
engine_id="eng-1",
|
||||
dp_rank=0,
|
||||
tp_rank=0,
|
||||
pp_rank=0,
|
||||
)
|
||||
|
||||
|
||||
def _make_register_worker_stub():
|
||||
return SimpleNamespace(
|
||||
vllm_config=_make_bootstrap_vllm_config(),
|
||||
hostname="127.0.0.1",
|
||||
side_channel_port=1234,
|
||||
engine_id="eng-1",
|
||||
dp_rank=0,
|
||||
tp_rank=0,
|
||||
pp_rank=0,
|
||||
)
|
||||
|
||||
|
||||
class _FlakyAsyncClient:
|
||||
"""httpx.AsyncClient stub that read-times-out `failures` times first."""
|
||||
|
||||
def __init__(self, failures: int, calls: list[int], **kwargs):
|
||||
self.failures = failures
|
||||
self.calls = calls
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc_info):
|
||||
return False
|
||||
|
||||
async def post(self, url, json=None):
|
||||
self.calls.append(1)
|
||||
if len(self.calls) <= self.failures:
|
||||
raise httpx.ReadTimeout("simulated slow bootstrap server")
|
||||
response = MagicMock()
|
||||
response.raise_for_status = MagicMock()
|
||||
return response
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_register_worker_retries_on_read_timeout(monkeypatch):
|
||||
"""A slow rank-0 bootstrap response must be retried, not treated as fatal."""
|
||||
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 5)
|
||||
monkeypatch.setattr(asyncio, "sleep", AsyncMock())
|
||||
|
||||
calls: list[int] = []
|
||||
monkeypatch.setattr(
|
||||
httpx,
|
||||
"AsyncClient",
|
||||
lambda **kwargs: _FlakyAsyncClient(2, calls),
|
||||
)
|
||||
|
||||
worker = _make_register_worker_stub()
|
||||
await MooncakeConnectorWorker.register_worker_with_bootstrap(worker)
|
||||
|
||||
assert len(calls) == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_register_worker_raises_after_max_attempts(monkeypatch):
|
||||
"""Terminal registration failure must raise, not loop forever."""
|
||||
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 3)
|
||||
monkeypatch.setattr(asyncio, "sleep", AsyncMock())
|
||||
|
||||
calls: list[int] = []
|
||||
monkeypatch.setattr(
|
||||
httpx,
|
||||
"AsyncClient",
|
||||
lambda **kwargs: _FlakyAsyncClient(99, calls),
|
||||
)
|
||||
|
||||
worker = _make_register_worker_stub()
|
||||
with pytest.raises(RuntimeError, match="after 3 attempts"):
|
||||
await MooncakeConnectorWorker.register_worker_with_bootstrap(worker)
|
||||
|
||||
assert len(calls) == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sender_listener_failure_propagates_to_caller(monkeypatch):
|
||||
"""A terminal registration failure must surface via the Future rather than
|
||||
leaving register_kv_caches blocked on its ready event."""
|
||||
monkeypatch.setenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "0.1")
|
||||
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 1)
|
||||
|
||||
loop = asyncio.new_event_loop()
|
||||
thread = threading.Thread(target=loop.run_forever, daemon=True)
|
||||
thread.start()
|
||||
|
||||
async def failing_listener(ready_event):
|
||||
raise RuntimeError("simulated terminal registration failure")
|
||||
|
||||
try:
|
||||
ready_event = threading.Event()
|
||||
fut = asyncio.run_coroutine_threadsafe(failing_listener(ready_event), loop)
|
||||
|
||||
deadline = time.monotonic() + 10.0
|
||||
raised = None
|
||||
while not ready_event.wait(timeout=0.1):
|
||||
if fut.done():
|
||||
try:
|
||||
fut.result()
|
||||
except RuntimeError as e:
|
||||
raised = e
|
||||
break
|
||||
assert time.monotonic() < deadline, "caller blocked past its deadline"
|
||||
|
||||
assert raised is not None
|
||||
assert "simulated terminal registration failure" in str(raised)
|
||||
finally:
|
||||
loop.call_soon_threadsafe(loop.stop)
|
||||
thread.join(timeout=5)
|
||||
loop.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"tp_rank",
|
||||
|
||||
@@ -10,7 +10,7 @@ from collections.abc import Collection
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from enum import IntEnum
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import msgspec
|
||||
import numpy as np
|
||||
@@ -70,6 +70,8 @@ from vllm.v1.worker.utils import select_common_block_size
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_BOOTSTRAP_MAX_ATTEMPTS: Final[int] = 3
|
||||
|
||||
try:
|
||||
from mooncake.engine import TransferEngine
|
||||
except ImportError:
|
||||
@@ -1147,24 +1149,44 @@ class MooncakeConnectorWorker:
|
||||
pp_rank=self.pp_rank,
|
||||
addr=worker_addr,
|
||||
)
|
||||
while True:
|
||||
timeout = envs.VLLM_MOONCAKE_CONNECTOR_TIMEOUT
|
||||
max_attempts = _BOOTSTRAP_MAX_ATTEMPTS
|
||||
backoff = 1.0
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
response = await client.post(url, json=payload.model_dump())
|
||||
response.raise_for_status()
|
||||
logger.debug("Successfully registered with bootstrap server at %s", url)
|
||||
break
|
||||
except httpx.ConnectError:
|
||||
# Bootstrap server not ready, wait for a while and retry.
|
||||
await asyncio.sleep(1)
|
||||
return
|
||||
except (httpx.ConnectError, httpx.TimeoutException) as e:
|
||||
# Rank 0 may be busy registering a large memory segment.
|
||||
if attempt == max_attempts:
|
||||
break
|
||||
logger.warning(
|
||||
"Bootstrap registration attempt %d/%d for %s failed with %s: %s. "
|
||||
"Retrying in %.1fs.",
|
||||
attempt,
|
||||
max_attempts,
|
||||
payload,
|
||||
type(e).__name__,
|
||||
e,
|
||||
backoff,
|
||||
)
|
||||
await asyncio.sleep(backoff)
|
||||
backoff = min(backoff * 2, 10.0)
|
||||
except Exception as e:
|
||||
err_msg = (
|
||||
e.response.text if isinstance(e, httpx.HTTPStatusError) else str(e)
|
||||
)
|
||||
logger.error(
|
||||
logger.exception(
|
||||
"Error registering %s with bootstrap server: %s", payload, err_msg
|
||||
)
|
||||
raise e
|
||||
raise
|
||||
raise RuntimeError(
|
||||
f"Failed to register {payload} with the Mooncake bootstrap server at "
|
||||
f"{url} after {max_attempts} attempts."
|
||||
)
|
||||
|
||||
async def _mooncake_sender_listener(self, ready_event: threading.Event):
|
||||
"""Background thread that listens for Mooncake requests, dispatches them
|
||||
@@ -1800,11 +1822,29 @@ class MooncakeConnectorWorker:
|
||||
if self.is_kv_consumer:
|
||||
return
|
||||
|
||||
# httpx applies the timeout to each phase (connect, read, write, pool)
|
||||
# rather than to the whole request, so budget for all four.
|
||||
register_timeout = envs.VLLM_MOONCAKE_CONNECTOR_TIMEOUT
|
||||
max_attempts = _BOOTSTRAP_MAX_ATTEMPTS
|
||||
backoff_total = sum(min(2.0**i, 10.0) for i in range(max_attempts - 1))
|
||||
ready_timeout = max_attempts * 4 * register_timeout + backoff_total + 30.0
|
||||
ready_event = threading.Event()
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
fut = asyncio.run_coroutine_threadsafe(
|
||||
self._mooncake_sender_listener(ready_event), self.sender_loop
|
||||
)
|
||||
ready_event.wait() # Wait for listener ZMQ socket to be ready.
|
||||
deadline = time.monotonic() + ready_timeout
|
||||
while not ready_event.wait(timeout=1.0):
|
||||
if fut.done():
|
||||
fut.result()
|
||||
raise RuntimeError(
|
||||
"Mooncake sender listener exited before becoming ready."
|
||||
)
|
||||
if time.monotonic() > deadline:
|
||||
fut.cancel()
|
||||
raise RuntimeError(
|
||||
"Mooncake sender listener did not become ready within "
|
||||
f"{ready_timeout:.0f}s."
|
||||
)
|
||||
|
||||
async def fetch_finished_recving_reqs(self) -> set[ReqId]:
|
||||
finished_recving_reqs = self.finished_recving_reqs
|
||||
|
||||
@@ -97,14 +97,19 @@ class MooncakeBootstrapServer:
|
||||
dp_entry.worker_addr[payload.tp_rank] = {}
|
||||
|
||||
tp_entry = dp_entry.worker_addr[payload.tp_rank]
|
||||
if payload.pp_rank in tp_entry:
|
||||
existing = tp_entry.get(payload.pp_rank)
|
||||
if existing is not None:
|
||||
# A client timeout can fire after the server recorded the
|
||||
# registration so an identical retry must not be an error.
|
||||
if existing == payload.addr:
|
||||
return {"status": "ok"}
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
f"Worker with dp_rank={payload.dp_rank}, "
|
||||
f"tp_rank={payload.tp_rank}, pp_rank={payload.pp_rank} "
|
||||
f"is already registered at "
|
||||
f"{tp_entry[payload.pp_rank]}, "
|
||||
f"{existing}, "
|
||||
f"but still want to register at {payload.addr}"
|
||||
),
|
||||
)
|
||||
|
||||
@@ -260,6 +260,7 @@ if TYPE_CHECKING:
|
||||
VLLM_ROCM_QUICK_REDUCE_MIN_SIZE_BYTES_MB: int | None = None
|
||||
VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB: int | None = None
|
||||
VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT: int = 480
|
||||
VLLM_MOONCAKE_CONNECTOR_TIMEOUT: float = 30.0
|
||||
VLLM_ENABLE_CUDAGRAPH_GC: bool = False
|
||||
VLLM_LOOPBACK_IP: str = ""
|
||||
VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = True
|
||||
@@ -1713,6 +1714,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_EC_SIDE_CHANNEL_PORT": lambda: int(
|
||||
os.getenv("VLLM_EC_SIDE_CHANNEL_PORT", "5601")
|
||||
),
|
||||
# Per-request timeout (seconds) when a prefiller worker registers with
|
||||
# the Mooncake bootstrap server.
|
||||
"VLLM_MOONCAKE_CONNECTOR_TIMEOUT": lambda: float(
|
||||
os.getenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "30.0")
|
||||
),
|
||||
# Port used for Mooncake handshake between remote agents.
|
||||
"VLLM_MOONCAKE_BOOTSTRAP_PORT": lambda: int(
|
||||
os.getenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", "8998")
|
||||
|
||||
Reference in New Issue
Block a user