[P/D] Mooncake: Add unit tests and minor fixes for mooncake connector (#36946)

Signed-off-by: Tianchen Ding <dtcccc@linux.alibaba.com>
This commit is contained in:
dtc
2026-03-27 09:26:40 +01:00
committed by GitHub
parent 84e439a9cb
commit 6287e7fa20
4 changed files with 774 additions and 13 deletions
@@ -47,14 +47,17 @@ from vllm.v1.attention.backends.utils import get_kv_cache_layout
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.request import RequestStatus
logger = init_logger(__name__)
try:
from mooncake.engine import TransferEngine
except ImportError as e:
raise ImportError(
except ImportError:
logger.warning(
"Please install mooncake by following the instructions at "
"https://github.com/kvcache-ai/Mooncake/blob/main/doc/en/build.md "
"to run VLLM with MooncakeTransferEngine."
) from e
)
TransferEngine = None
if TYPE_CHECKING:
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
@@ -64,8 +67,6 @@ if TYPE_CHECKING:
ReqId = str # Internal scheduler request ID
TransferId = str # KV transfer coordination ID (shared by P/D)
logger = init_logger(__name__)
@dataclass(frozen=True)
class TransferRegion:
@@ -638,6 +639,9 @@ class MooncakeConnectorWorker:
"""Implementation of Worker side methods"""
def __init__(self, vllm_config: VllmConfig, engine_id: str):
if TransferEngine is None:
logger.error("Mooncake is not available")
raise RuntimeError("Mooncake is not available")
logger.info("Initializing Mooncake Transfer Engine worker %s", engine_id)
self.vllm_config = vllm_config
@@ -721,9 +725,7 @@ class MooncakeConnectorWorker:
# Start bootstrap server on global rank 0.
if should_launch_bootstrap_server(vllm_config):
_, port = get_mooncake_bootstrap_addr(vllm_config)
self.bootstrap_server = MooncakeBootstrapServer(
vllm_config, "0.0.0.0", port
)
self.bootstrap_server = MooncakeBootstrapServer("0.0.0.0", port)
self.bootstrap_server.start()
if not self.is_kv_producer:
@@ -778,7 +780,9 @@ class MooncakeConnectorWorker:
if self.sender_loop.is_running():
self.sender_loop.call_soon_threadsafe(self.sender_loop.stop)
self._sender_listener_t.join()
if should_launch_bootstrap_server(self.vllm_config):
if should_launch_bootstrap_server(self.vllm_config) and hasattr(
self, "bootstrap_server"
):
self.bootstrap_server.shutdown()
if not self.is_kv_producer and self.receiver_loop.is_running():
self.receiver_loop.call_soon_threadsafe(self.receiver_loop.stop)
@@ -8,7 +8,6 @@ import uvicorn
from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.utils import EngineId
from vllm.logger import init_logger
@@ -38,7 +37,7 @@ class MooncakeBootstrapServer:
Prefiller workers register their connection info (IP, port, ranks) here.
"""
def __init__(self, vllm_config: VllmConfig, host: str, port: int):
def __init__(self, host: str, port: int):
self.workers: dict[int, EngineEntry] = {}
self.host = host