[EPD] Add ECMooncakeConnector for encoder cache over Mooncake TransferEngine (#41567)

Signed-off-by: Teng Ma <sima.mt@alibaba-inc.com>
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
Signed-off-by: Zhou ziheng <jiaranran2@gmail.com>
Signed-off-by: jiangkuaixue123 <jiangxiaozhou111@163.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Tianyu Guo <guoty@inferact.ai>
Co-authored-by: Zhou ziheng <jiaranran2@gmail.com>
Co-authored-by: jiangkuaixue123 <jiangxiaozhou111@163.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Teng Ma
2026-09-07 06:21:43 +00:00
committed by GitHub
co-authored by Cursor Tianyu Guo Zhou ziheng jiangkuaixue123 OpenAI Codex
parent f7f060d253
commit 6fbb00b188
30 changed files with 5774 additions and 301 deletions
@@ -19,3 +19,57 @@ steps:
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/mooncake_integration/config_sweep_accuracy_test.sh
- label: ":nvidia: (L4) Mooncake EC TCP E2E"
key: mooncake-ec-tcp-e2e-2-gpus
timeout_in_minutes: 30
working_dir: "/vllm-workspace"
device: l4
num_devices: 2
source_file_dependencies:
- vllm/distributed/ec_transfer/
- vllm/config/ec_transfer.py
- vllm/config/multimodal.py
- vllm/config/vllm.py
- vllm/multimodal/
- vllm/v1/core/encoder_cache_manager.py
- vllm/v1/core/sched/
- vllm/v1/engine/
- vllm/v1/worker/ec_connector_model_runner_mixin.py
- vllm/v1/worker/gpu_model_runner.py
- vllm/v1/worker/gpu_worker.py
- vllm/v1/worker/gpu/ec_connector.py
- vllm/v1/worker/gpu/model_runner.py
- vllm/v1/worker/gpu/mm/
- examples/disaggregated/disaggregated_encoder/
- tests/v1/ec_connector/
- requirements/kv_connectors.txt
env:
PYTHON_BIN: "/vllm-workspace/.venv/bin/python"
PYTHONUNBUFFERED: "1"
VLLM_HOST_IP: "127.0.0.1"
VLLM_USE_V2_MODEL_RUNNER: "1"
MOONCAKE_EC_PROTOCOL: "tcp"
USE_MM_PROMPTS: "1"
SKIP_BASELINE: "0"
CONCURRENCY: "3"
REPEAT: "2"
LOG_PATH: "/tmp/mooncake-ec-e2e"
BASELINE_FILE: "/tmp/mooncake-ec-e2e/baseline.json"
commands:
- uv venv --system-site-packages --python 3.12 .venv
- |
uv pip install --python .venv/bin/python "$(.venv/bin/python - <<'PY'
from pathlib import Path
import torch
requirement = next(
line for line in Path("requirements/kv_connectors.txt").read_text().splitlines()
if line.startswith("mooncake-transfer-engine ")
)
if torch.version.cuda.split(".")[0] == "13":
requirement = requirement.replace("mooncake-transfer-engine", "mooncake-transfer-engine-cuda13")
print(requirement)
PY
)"
- bash tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh
@@ -23,6 +23,7 @@ import argparse
import asyncio
import hashlib
import io
import itertools
import json
import logging
import os
@@ -30,6 +31,7 @@ import random
import time
import uuid
from collections.abc import AsyncIterator
from typing import Any
import aiohttp
import pybase64 as base64
@@ -41,9 +43,7 @@ from fastapi.responses import JSONResponse, StreamingResponse
# FastAPI app & global state
###############################################################################
logging.basicConfig(
level=logging.DEBUG, format="%(asctime)s %(levelname)s: %(message)s"
)
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s: %(message)s")
logger = logging.getLogger("proxy")
app = FastAPI()
@@ -87,9 +87,24 @@ def encoder_rr_assignment(
return urls, next_start
def validate_ec_consumer_routing(
prefill_urls: list[str], consumer_addrs: list[str]
) -> None:
"""Reject the topology whose EC destination cannot be routed safely."""
if prefill_urls and consumer_addrs:
raise ValueError(
"Mooncake EC consumer routing supports E+PD only; disable independent "
"prefill or omit --ec-consumer-zmq-addrs."
)
# Diagnostic switch: forward the original request to the decoder so the
# only difference from the rewrite path is the rewrite itself.
NO_REWRITE = False
# Decode-side retries for a retryable internal error (`finish_reason="error"`,
# e.g. an encoder embedding the connector could not deliver). Re-issuing runs
# the encode again, which produces a fresh transfer.
DECODE_RETRIES = 1
# Grid metadata reported by the encoder instance, keyed by item index.
@@ -136,6 +151,7 @@ def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict:
here -- a second derivation could disagree with the encoder's.
"""
rewritten = 0
transfer_items = []
idx = 0
new_messages = []
for msg in req_data.get("messages", []):
@@ -151,26 +167,42 @@ def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict:
meta = dict(item_meta.get(idx) or {})
idx += 1
item_uuid = meta.pop("mm_hash", None)
ec_mm_hash = meta.pop("ec_mm_hash", None) or item_uuid
transfer_id = meta.pop("transfer_id", None)
# Whatever keys the encoder reported are the metadata its model
# declared as needed to size the placeholder range; the proxy does
# not need to know their names.
metadata = {k: _b64_tensor(v) for k, v in meta.items()}
if not metadata or not item_uuid:
# The encoder reported no metadata (e.g. the item came from its
# processor cache); let the decoder process the media itself.
# Nothing to size the placeholder range with. A processor cache
# hit is not a cause on its own: with the default `lru` type the
# engine restores the item before the scheduler reports it. It
# goes missing when the encode request failed, or under
# `--mm-processor-cache-type shm`, where a hit replaces the item
# with its shared-memory address and only the worker restores
# it. Send the media so the decoder can derive the grid itself.
new_content.append(item)
continue
embeds_type = EMBEDS_TYPES[item["type"]]
new_content.append(
{"type": embeds_type, embeds_type: metadata, "uuid": item_uuid}
)
if transfer_id is not None:
transfer_items.append(
{"mm_hash": ec_mm_hash, "transfer_id": transfer_id}
)
rewritten += 1
new_messages.append({**msg, "content": new_content})
if not rewritten:
return req_data
logger.info("Rewrote %d media item(s) as metadata references", rewritten)
return {**req_data, "messages": new_messages}
rewritten_request = {**req_data, "messages": new_messages}
if transfer_items:
ec_transfer_params = dict(req_data.get("ec_transfer_params") or {})
ec_transfer_params["ec_items"] = transfer_items
rewritten_request["ec_transfer_params"] = ec_transfer_params
return rewritten_request
def extract_mm_items(request_data: dict) -> list[dict]:
@@ -196,7 +228,8 @@ async def fanout_encoder_primer(
orig_request: dict,
e_urls: list[str],
req_id: str,
) -> dict[int, dict]:
consumer_zmq: str | None = None,
) -> tuple[dict[int, dict], dict[str, Any]]:
"""
1. Build one request *per MM item* with all text removed.
2. Send them concurrently to the encode cluster.
@@ -206,19 +239,25 @@ async def fanout_encoder_primer(
`ec_transfer_params`: its EC cache key and the grid its processor produced.
The proxy still supplies the uuid so both sides key the cache the same way;
the grid can only come from the encoder, which is the side that computed it.
Also returns the connector handles to put on the decode body, as a fresh
mapping. `orig_request` is left untouched so a retry re-encodes from the
original request instead of carrying the previous attempt's handles.
"""
logger.info("[%s] Processing multimodal items...", req_id)
mm_items = extract_mm_items(orig_request)
if not mm_items:
logger.info("[%s] No multimodal items, skipping encoder", req_id)
return {} # nothing to do
return {}, {} # nothing to do
logger.info("[%s] got %d multimodal items...", req_id, len(mm_items))
tasks = []
item_uuids: dict[int, str] = {}
item_transfer_ids: dict[int, str] = {}
item_meta: dict[int, dict] = {}
ec_params: dict[str, Any] = {}
# Round-robin over encode servers to distribute load a bit. The cursor
# persists across requests so fan-out doesn't restart at e_urls[0] every
@@ -241,6 +280,8 @@ async def fanout_encoder_primer(
item_uuid = None if NO_REWRITE else content_uuid(item)
if item_uuid is not None:
item_uuids[idx] = item_uuid
transfer_id = uuid.uuid4().hex
item_transfer_ids[idx] = transfer_id
encoder_req = {
# You *may* need to keep additional fields
@@ -257,6 +298,19 @@ async def fanout_encoder_primer(
# once the prompt is encoded and its embeddings are published.
"stream": False,
}
if consumer_zmq is not None:
# No mm_hash here on purpose. The encoder's own
# `mm_features[i].identifier` is derived from the uuid *and* the
# engine's media_io_kwargs / mm_processor_kwargs, so this proxy
# cannot know it before the encoder runs. Sending the bare uuid
# would fail the connector's hash match and make the producer
# invent its own transfer id, which the consumer could then never
# cancel. Omitting it lets the connector match by position, which
# is exact: one encoder request carries exactly one item.
encoder_req["ec_transfer_params"] = {
"consumer_zmq": consumer_zmq,
"ec_items": [{"transfer_id": transfer_id}],
}
tasks.append(
encode_session.post(
f"{target_url}/v1/chat/completions",
@@ -304,24 +358,42 @@ async def fanout_encoder_primer(
except Exception:
logger.warning("[%s] Could not read encoder metadata #%d", req_id, idx)
params = {}
if idx in item_uuids:
reported = params.get(item_uuids[idx])
if params:
# One encoder request carries exactly one item, so there is a
# single reported entry. Do not key it by this proxy's uuid: when
# media_io_kwargs or mm_processor_kwargs are set the engine
# re-hashes the uuid together with them, so the encoder's own
# `mm_features[i].identifier` is a derived value this proxy cannot
# predict. Fall back to the sole entry, and carry the key the
# encoder actually used through as `ec_mm_hash`.
ec_mm_hash = item_uuids.get(idx)
reported = params.get(ec_mm_hash)
if reported is None and len(params) == 1:
((ec_mm_hash, reported),) = params.items()
if reported:
metadata = reported.get("metadata") or {}
if metadata:
item_meta[idx] = {**metadata, "mm_hash": item_uuids[idx]}
item_meta[idx] = {
**metadata,
"mm_hash": item_uuids.get(idx, ec_mm_hash),
"ec_mm_hash": ec_mm_hash,
}
if idx in item_transfer_ids:
item_meta[idx]["transfer_id"] = item_transfer_ids[idx]
# Whatever the encoder reported alongside `metadata` is the
# connector's own handle on the published embedding (for NIXL,
# peer_host/peer_port/size_bytes). The decoder's connector
# looks it up by mm_hash on the request, so carry it through.
orig_request.setdefault("ec_transfer_params", {})[item_uuids[idx]] = (
reported
)
ec_params[item_uuids.get(idx, ec_mm_hash)] = reported
if NO_REWRITE and consumer_zmq is not None:
ec_params.setdefault("ec_items", []).append(
{"mm_hash": ec_mm_hash, "transfer_id": item_transfer_ids[idx]}
)
logger.info(
"[%s] All %d encoder requests completed successfully", req_id, len(mm_items)
)
return item_meta
return item_meta, ec_params
async def maybe_prefill(
@@ -331,8 +403,11 @@ async def maybe_prefill(
) -> dict:
"""
- Do prefill-only task if p_url exist;
- Return modified request data with kv transfer params (for nixl connector)
- Return a new body carrying kv transfer params (for nixl connector)
- Else, skip and return the original request data for decode
`req_data` is never mutated: a decode retry re-enters this function with the
same body, and one attempt's `remote_block_ids` must not reach the next.
"""
if p_url:
logger.info("[%s] Processing through prefill: %s", req_id, p_url)
@@ -342,11 +417,9 @@ async def maybe_prefill(
prefill_response_json = await prefill_response.json()
kv_transfer_params = prefill_response_json.get("kv_transfer_params", {})
if kv_transfer_params:
req_data["kv_transfer_params"] = kv_transfer_params
return {**req_data, "kv_transfer_params": kv_transfer_params}
return req_data
else:
return req_data
return req_data
async def process_prefill_stage(
@@ -409,7 +482,6 @@ async def process_prefill_stage(
###############################################################################
@app.middleware("http")
async def log_requests(request: Request, call_next):
"""Middleware to log all incoming requests and responses"""
req_id = request.headers.get("x-request-id", str(uuid.uuid4()))
@@ -458,7 +530,20 @@ async def log_requests(request: Request, call_next):
async def on_startup() -> None:
global encode_session, prefill_session, decode_session
timeout = aiohttp.ClientTimeout(total=100_000)
connector = aiohttp.TCPConnector(limit=0, force_close=False)
# vLLM closes an idle keep-alive connection after
# VLLM_HTTP_TIMEOUT_KEEP_ALIVE seconds (5 by default), while aiohttp keeps
# pooling it for 15. Reusing one it has already closed fails the request
# with ServerDisconnectedError, and the server logs nothing at all: it
# closed the socket before the request arrived. Retire ours first.
server_keep_alive = float(os.getenv("VLLM_HTTP_TIMEOUT_KEEP_ALIVE", "5"))
connector = aiohttp.TCPConnector(
limit=0,
**(
{"keepalive_timeout": server_keep_alive / 2}
if server_keep_alive > 0
else {"force_close": True}
),
)
encode_session = aiohttp.ClientSession(timeout=timeout, connector=connector)
if app.state.p_urls:
# only setup if prefill instance(s) exist
@@ -482,40 +567,97 @@ async def on_shutdown() -> None:
###############################################################################
async def prepare_for_decode(
req_data: dict,
req_id: str,
e_urls: list[str],
p_url: str,
consumer_zmq: str | None,
) -> tuple[dict, float, float]:
"""Encode, rewrite and prefill, returning the body to send to decode.
`req_data` is left untouched so a retry starts from the original media
rather than from a body whose images are already metadata references.
"""
_t0 = time.perf_counter()
item_meta, ec_params = await fanout_encoder_primer(
req_data, e_urls, req_id, consumer_zmq
)
_t1 = time.perf_counter()
prepared = req_data if NO_REWRITE else rewrite_for_decode(req_data, item_meta)
if ec_params:
# A fresh body every time: `rewrite_for_decode` hands back `req_data`
# itself when it rewrote nothing, and this attempt's handles must not
# outlive it into a retry.
handles = dict(prepared.get("ec_transfer_params") or {})
handles.update(ec_params)
prepared = {**prepared, "ec_transfer_params": handles}
_t2 = time.perf_counter()
prepared = await maybe_prefill(prepared, p_url, req_id)
return prepared, _t1 - _t0, _t2 - _t1
async def forward_non_stream(
req_data: dict, req_id: str, e_urls: list[str], p_url: str, d_url: str
req_data: dict,
req_id: str,
e_urls: list[str],
p_url: str,
d_url: str,
consumer_zmq: str | None,
dp_rank: int | None = None,
) -> dict:
try:
# Step 1: Process through Encoder instance (if has MM input)
_t0 = time.perf_counter()
item_meta = await fanout_encoder_primer(req_data, e_urls, req_id)
_t1 = time.perf_counter()
req_data = req_data if NO_REWRITE else rewrite_for_decode(req_data, item_meta)
_t2 = time.perf_counter()
# Step 2: Process through Prefill instance
req_data = await maybe_prefill(req_data, p_url, req_id)
# Step 3: Process through Decode instance
logger.info("[%s] Forwarding to decode: %s", req_id, d_url)
headers = {"x-request-id": req_id}
# Non-streaming response
async with decode_session.post(
f"{d_url}/v1/chat/completions", json=req_data, headers=headers
) as resp:
resp.raise_for_status()
out = await resp.json()
_t3 = time.perf_counter()
logger.info(
"STAGE %s encode=%.1f rewrite=%.1f decode=%.1f total=%.1f",
"no-rewrite" if NO_REWRITE else "rewrite",
(_t1 - _t0) * 1e3,
(_t2 - _t1) * 1e3,
(_t3 - _t2) * 1e3,
(_t3 - _t0) * 1e3,
for attempt in range(DECODE_RETRIES + 1):
_t0 = time.perf_counter()
prepared, encode_s, rewrite_s = await prepare_for_decode(
req_data, req_id, e_urls, p_url, consumer_zmq
)
return out
_t2 = time.perf_counter()
logger.info("[%s] Forwarding to decode: %s", req_id, d_url)
headers = {"x-request-id": req_id}
if dp_rank is not None:
headers["X-data-parallel-rank"] = str(dp_rank)
async with decode_session.post(
f"{d_url}/v1/chat/completions", json=prepared, headers=headers
) as resp:
if resp.status >= 400:
detail = await resp.text()
# 500 is the decoder's retryable internal error, which
# includes an encoder embedding it could not obtain. Redoing
# the encode publishes the item again.
if resp.status == 500 and attempt < DECODE_RETRIES:
logger.warning(
"[%s] Decode returned 500, re-encoding and retrying "
"(attempt %d/%d): %s",
req_id,
attempt + 1,
DECODE_RETRIES,
detail[:200],
)
continue
logger.error(
"[%s] Decode request returned status %s: %s",
req_id,
resp.status,
detail,
)
raise HTTPException(status_code=resp.status, detail=detail)
out = await resp.json()
_t3 = time.perf_counter()
logger.info(
"STAGE %s encode=%.1f rewrite=%.1f decode=%.1f total=%.1f "
"attempt=%d",
"no-rewrite" if NO_REWRITE else "rewrite",
encode_s * 1e3,
rewrite_s * 1e3,
(_t3 - _t2) * 1e3,
(_t3 - _t0) * 1e3,
attempt,
)
return out
raise HTTPException(status_code=500, detail="Decode failed after re-encoding")
except HTTPException:
raise
@@ -525,47 +667,66 @@ async def forward_non_stream(
async def forward_stream(
req_data: dict, req_id: str, e_urls: list[str], p_url: str, d_url: str
req_data: dict,
req_id: str,
e_urls: list[str],
p_url: str,
d_url: str,
consumer_zmq: str | None,
dp_rank: int | None = None,
) -> AsyncIterator[str]:
try:
# Step 1: Process through Encoder instance (if has MM input)
_t0 = time.perf_counter()
item_meta = await fanout_encoder_primer(req_data, e_urls, req_id)
_t1 = time.perf_counter()
req_data = req_data if NO_REWRITE else rewrite_for_decode(req_data, item_meta)
_t2 = time.perf_counter()
for attempt in range(DECODE_RETRIES + 1):
_t0 = time.perf_counter()
prepared, encode_s, rewrite_s = await prepare_for_decode(
req_data, req_id, e_urls, p_url, consumer_zmq
)
_t2 = time.perf_counter()
# Step 2: Process through Prefill instance
req_data = await maybe_prefill(req_data, p_url, req_id)
logger.info("[%s] Starting streaming from decode: %s", req_id, d_url)
headers = {"x-request-id": req_id}
if dp_rank is not None:
headers["X-data-parallel-rank"] = str(dp_rank)
# Step 3: Process through Decode instance
logger.info("[%s] Starting streaming from decode: %s", req_id, d_url)
headers = {"x-request-id": req_id}
_first = None
async with decode_session.post(
f"{d_url}/v1/chat/completions",
json=prepared,
headers=headers,
) as resp:
# Retry only before the first chunk: once anything reached the
# client the response cannot be replaced.
if resp.status == 500 and attempt < DECODE_RETRIES:
detail = await resp.text()
logger.warning(
"[%s] Decode returned 500 before streaming, re-encoding "
"and retrying (attempt %d/%d): %s",
req_id,
attempt + 1,
DECODE_RETRIES,
detail[:200],
)
continue
resp.raise_for_status()
async for chunk in resp.content.iter_chunked(1024):
if chunk:
if _first is None:
_first = time.perf_counter()
yield chunk.decode("utf-8", errors="ignore")
_t3 = time.perf_counter()
# Streaming response
_first = None
async with decode_session.post(
f"{d_url}/v1/chat/completions",
json=req_data,
headers=headers,
) as resp:
resp.raise_for_status()
async for chunk in resp.content.iter_chunked(1024):
if chunk:
if _first is None:
_first = time.perf_counter()
yield chunk.decode("utf-8", errors="ignore")
_t3 = time.perf_counter()
logger.info(
"STAGE %s encode=%.1f rewrite=%.2f decode_ttfb=%.1f decode_total=%.1f",
"no-rewrite" if NO_REWRITE else "rewrite",
(_t1 - _t0) * 1e3,
(_t2 - _t1) * 1e3,
((_first or _t3) - _t2) * 1e3,
(_t3 - _t2) * 1e3,
)
logger.info("[%s] Streaming completed", req_id)
logger.info(
"STAGE %s encode=%.1f rewrite=%.2f decode_ttfb=%.1f "
"decode_total=%.1f attempt=%d",
"no-rewrite" if NO_REWRITE else "rewrite",
encode_s * 1e3,
rewrite_s * 1e3,
((_first or _t3) - _t2) * 1e3,
(_t3 - _t2) * 1e3,
attempt,
)
logger.info("[%s] Streaming completed", req_id)
return
except HTTPException:
logger.exception("[%s] HTTPException in forward_stream", req_id)
@@ -590,16 +751,29 @@ async def chat_completions(request: Request):
e_urls = app.state.e_urls # we want the full list for fan-out
p_url = random.choice(app.state.p_urls) if app.state.p_urls else None
d_url = random.choice(app.state.d_urls)
decode_index = random.randrange(len(app.state.d_urls))
d_url = app.state.d_urls[decode_index]
dp_size = app.state.ec_consumer_dp_size
# Round-robin the replica, then name it to both halves: the decoder
# honours the rank header instead of its own balancer, and the encoder
# pushes to that replica's control channel. Choosing once here means a
# decode retry re-encodes to the same replica.
dp_rank = next(app.state.replica_counter) % dp_size if dp_size > 1 else None
ec_index = decode_index * dp_size + (dp_rank or 0)
consumer_zmq = app.state.d_ec_urls[ec_index] if app.state.d_ec_urls else None
is_streaming = req_data.get("stream", False)
if is_streaming:
return StreamingResponse(
forward_stream(req_data, req_id, e_urls, p_url, d_url),
forward_stream(
req_data, req_id, e_urls, p_url, d_url, consumer_zmq, dp_rank
),
media_type="text/event-stream",
)
result = await forward_non_stream(req_data, req_id, e_urls, p_url, d_url)
result = await forward_non_stream(
req_data, req_id, e_urls, p_url, d_url, consumer_zmq, dp_rank
)
return JSONResponse(content=result)
except HTTPException:
@@ -750,6 +924,14 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--host", default="0.0.0.0")
parser.add_argument("--port", type=int, default=8000)
parser.add_argument(
"--log-requests",
action="store_true",
help=(
"Log every request in and out, and raise the log level to DEBUG. "
"Off by default: the proxy is on the request path."
),
)
parser.add_argument(
"--no-rewrite",
action="store_true",
@@ -764,8 +946,8 @@ if __name__ == "__main__":
"--prefill-servers-urls",
required=True,
help=(
'Comma-separated prefill URLs ("http://p1:8003,http://p2:8004") ',
'to enable E->P->D, set "disable" or "none" to enable E->PD',
'Comma-separated prefill URLs ("http://p1:8003,http://p2:8004") '
'to enable E->P->D, set "disable" or "none" to enable E->PD'
),
)
parser.add_argument(
@@ -773,15 +955,64 @@ if __name__ == "__main__":
required=True,
help='Comma-separated decode URLs ("http://d1:8005,http://d2:8006")',
)
parser.add_argument(
"--decode-retries",
type=int,
default=1,
help=(
"Re-encode and re-send when decode returns 500, which is its "
"retryable internal error (an undeliverable encoder embedding "
"among them). 0 disables."
),
)
parser.add_argument(
"--ec-consumer-zmq-addrs",
default="",
help=(
"Comma-separated Mooncake EC consumer control addresses, aligned "
"with --decode-servers-urls. Required for Mooncake EC consumers and "
"supported only in E+PD mode. With --ec-consumer-dp-size > 1, list "
"each server's replicas consecutively: s0r0,s0r1,s1r0,s1r1."
),
)
parser.add_argument(
"--ec-consumer-dp-size",
type=int,
default=1,
help=(
"Data-parallel replicas per EC consumer. The proxy picks a replica "
"round-robin and names it to both halves of the request, because an "
"encoder push has to land where the request will run."
),
)
args = parser.parse_args()
if args.log_requests:
logging.getLogger().setLevel(logging.DEBUG)
app.middleware("http")(log_requests)
NO_REWRITE = args.no_rewrite
DECODE_RETRIES = max(0, args.decode_retries)
app.state.e_urls = [
u.strip() for u in args.encode_servers_urls.split(",") if u.strip()
]
app.state.d_urls = [
u.strip() for u in args.decode_servers_urls.split(",") if u.strip()
]
app.state.d_ec_urls = [
u.strip() for u in args.ec_consumer_zmq_addrs.split(",") if u.strip()
]
if args.ec_consumer_dp_size < 1:
parser.error("--ec-consumer-dp-size must be at least 1")
app.state.ec_consumer_dp_size = args.ec_consumer_dp_size
app.state.replica_counter = itertools.count()
expected = len(app.state.d_urls) * args.ec_consumer_dp_size
if app.state.d_ec_urls and len(app.state.d_ec_urls) != expected:
parser.error(
"--ec-consumer-zmq-addrs must contain one address per consumer "
f"replica: expected {expected} "
f"({len(app.state.d_urls)} servers x {args.ec_consumer_dp_size} replicas), "
f"got {len(app.state.d_ec_urls)}"
)
# handle prefill instances
if args.prefill_servers_urls.lower() in ("disable", "none", ""):
app.state.p_urls = []
@@ -793,11 +1024,21 @@ if __name__ == "__main__":
u.strip() for u in args.prefill_servers_urls.split(",") if u.strip()
]
logger.info("Disaggregated prefill phase is enabled. Running E + P + D...")
try:
validate_ec_consumer_routing(app.state.p_urls, app.state.d_ec_urls)
except ValueError as exc:
parser.error(str(exc))
logger.info("Proxy listening on %s:%s", args.host, args.port)
logger.info("Encode servers: %s", app.state.e_urls)
logger.info("Prefill instances %s", app.state.p_urls)
logger.info("Decode servers: %s", app.state.d_urls)
if app.state.ec_consumer_dp_size > 1:
logger.info(
"EC consumer replicas per server: %d (control addresses: %s)",
app.state.ec_consumer_dp_size,
app.state.d_ec_urls,
)
uvicorn.run(
app,
@@ -167,6 +167,38 @@ def test_get_freed_mm_hashes_clears_freed_list():
assert manager.get_freed_mm_hashes() == []
def test_referencing_a_freeable_entry_protects_it_from_eviction():
"""A waiting request can hold an entry it is not scheduled for yet.
Entries with no referent are evictable. A request that is deferred (e.g.
by an EC connector waiting on another item) still needs the items it
already has, and a connector that hands out one transfer per request
cannot always produce the same item a second time.
"""
manager = EncoderCacheManager(cache_size=10)
owner = MockRequest("owner", ["a"], [5])
waiter = MockRequest("waiter", ["a"], [5])
newcomer = MockRequest("newcomer", ["b"], [6])
manager.allocate(owner, 0)
manager.free_encoder_input(owner, 0)
assert "a" in manager.freeable
# The waiting request references it; it is no longer reclaimable.
assert manager.check_and_update_cache(waiter, 0)
assert "a" not in manager.freeable
# 'b' no longer fits, and 'a' must not be taken away to make room.
assert not manager.can_allocate(newcomer, 0, int(1e9), 0)
assert "a" in manager.cached
assert manager.get_freed_mm_hashes() == []
# Once the waiter is done with it, the entry is reclaimable again.
manager.free_encoder_input(waiter, 0)
assert manager.can_allocate(newcomer, 0, int(1e9), 0)
assert manager.get_freed_mm_hashes() == ["a"]
def test_reallocated_hash_is_not_reported_as_freed():
manager = EncoderCacheManager(cache_size=8)
req_a = MockRequest("reqA", ["a"], [4])
+89 -3
View File
@@ -5643,6 +5643,68 @@ def test_free_encoder_inputs_respects_unconfirmed_placeholders():
assert manager.get_cached_input_ids(request) == set()
def test_free_encoder_inputs_notifies_the_ec_connector():
"""The connector learns the item is consumed, not just that the request ended.
Connectors hold per-item transfer state (remote buffers, reservations);
waiting for `request_finished` pins it for the whole generation.
"""
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
mm_start_pos, mm_length = 50, 100
request = create_requests(
num_requests=1,
num_tokens=mm_start_pos + mm_length + 10,
mm_positions=[[PlaceholderRange(offset=mm_start_pos, length=mm_length)]],
)[0]
scheduler.encoder_cache_manager.allocate(request, 0)
scheduler.ec_connector = Mock()
request.num_computed_tokens = mm_start_pos + mm_length - 1
scheduler._free_encoder_inputs(request)
scheduler.ec_connector.update_state_after_free.assert_not_called()
request.num_computed_tokens = mm_start_pos + mm_length
scheduler._free_encoder_inputs(request)
scheduler.ec_connector.update_state_after_free.assert_called_once_with(request, 0)
def test_unavailable_encoder_input_fails_the_request_as_retryable():
"""An encoder input the connector gave up on must end the request.
`FinishReason.ERROR` is the retryable channel the KV connector already uses
for load failures, so the caller can re-issue; deferring instead parked the
request until the client timed out.
"""
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
request = create_requests(
num_requests=1,
num_tokens=160,
mm_positions=[[PlaceholderRange(offset=50, length=100)]],
)[0]
scheduler.add_request(request)
scheduler.ec_connector = Mock()
scheduler.ec_connector.take_unavailable_requests.return_value = {request.request_id}
# `request_finished` reports (delay_free, params) for the request teardown.
scheduler.ec_connector.request_finished.return_value = (False, None)
scheduler_output = scheduler.schedule()
outputs = scheduler.update_from_output(
scheduler_output,
ModelRunnerOutput(
req_ids=[request.request_id],
req_id_to_index={request.request_id: 0},
sampled_token_ids=[[]],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
),
)
assert request.status == RequestStatus.FINISHED_ERROR
engine_outputs = outputs[0].outputs
assert [o.finish_reason for o in engine_outputs] == [FinishReason.ERROR]
def test_free_encoder_inputs_defers_for_eagle_lookahead():
"""With EAGLE speculative decoding, the encoder input is retained one extra
position so the drafter's +1 look-ahead mm-embedding gather (which reads one
@@ -5840,9 +5902,8 @@ def test_ec_connector_ensure_cache_available_defers_request(use_kv_connector):
# ensure_cache_available must have been called with (request, num_computed_tokens=0)
# for a brand-new request that has no cached tokens yet.
scheduler.ec_connector.ensure_cache_available.assert_called_once_with(
request_deferred, 0
)
ensure_call = scheduler.ec_connector.ensure_cache_available.call_args
assert ensure_call.args == (request_deferred, 0)
# Deferred request must NOT be scheduled
assert request_deferred.request_id not in output.num_scheduled_tokens
_assert_right_encoder_cache_allocated(scheduler, expected_total_allocated=0)
@@ -5874,6 +5935,31 @@ def test_ec_connector_ensure_cache_available_defers_request(use_kv_connector):
_assert_right_encoder_inputs(output, expected_total_reqs=0)
def test_ec_connector_defers_running_request_for_async_reload():
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
max_num_batched_tokens=32,
use_ec_connector=True,
ec_role="ec_consumer",
)
request = create_requests(
num_requests=1,
num_tokens=128,
mm_positions=[[PlaceholderRange(offset=48, length=32)]],
req_ids=["request"],
)[0]
scheduler.ec_connector.ensure_cache_available = Mock(side_effect=[True, False])
scheduler.add_request(request)
first_output = scheduler.schedule()
assert first_output.num_scheduled_tokens[request.request_id] == 32
second_output = scheduler.schedule()
assert request.request_id not in second_output.num_scheduled_tokens
ensure_call = scheduler.ec_connector.ensure_cache_available.call_args
assert ensure_call.args[:2] == (request, 32)
def test_ec_connector_pending_prefetch_only_checks_future_mm_features():
"""Test that future mm feature filtering only yields features beyond
the computed token frontier.
+212 -170
View File
@@ -1,171 +1,213 @@
# EPD Correctness Test
This test verifies that EPD (Encoder-Prefill-Decode) disaggregation produces identical outputs to a baseline single instance.
## What It Tests
- **Baseline**: Single vLLM instance serving a multimodal model
- **EPD (1E+1PD)**: 1 Encoder + 1 Prefill-Decode instance
- **Baseline (1P+1D)**: 1 Prefill + 1 Decode instance
- **EPD (1E+1P+1D)**: 1 Encoder + 1 Prefill + 1 Decode instance
The test ensures that disaggregated encoding produces **identical** outputs to the baseline.
Note that currently PD disaggregation set up may give slightly different results from a single instance. Therefore, we need the result from 1P+1D as the baseline for 1E+1P+1D
# EPD Correctness Test
This test verifies that EPD (Encoder-Prefill-Decode) disaggregation produces identical outputs to a baseline single instance.
## What It Tests
- **Baseline**: Single vLLM instance serving a multimodal model
- **EPD (1E+1PD)**: 1 Encoder + 1 Prefill-Decode instance
- **Baseline (1P+1D)**: 1 Prefill + 1 Decode instance
- **EPD (1E+1P+1D)**: 1 Encoder + 1 Prefill + 1 Decode instance
The test ensures that disaggregated encoding produces **identical** outputs to the baseline.
Note that currently PD disaggregation set up may give slightly different results from a single instance. Therefore, we need the result from 1P+1D as the baseline for 1E+1P+1D
Please refer to [Disaggregated Encoder Feature](../../../../docs/features/disagg_encoder.md) for the detailed explanation for the EPD features.
## Files
- `run_epd_correctness_test.sh` - Main test script (starts all instances and runs tests)
- `test_epd_correctness.py` - Python test script (compares outputs)
## Usage
### Multimodal Prompts (Default)
```bash
cd vllm
./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
```
This runs the test with actual multimodal (image) prompts.
### Text-Only Prompts
```bash
cd vllm
USE_MM_PROMPTS=0 ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
```
This runs a quick test with text-only prompts to verify the setup works.
### Custom Configuration
```bash
# Use specific GPUs
GPU_E=0 GPU_PD=1 GPU_P=1 GPU_D=2 bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
# Use specific ports
ENDPOINT_PORT=10001 bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
# Use specific model
MODEL="Qwen/Qwen2.5-VL-3B-Instruct" bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
# Use specific storage path
EC_SHARED_STORAGE_PATH="/tmp/my_ec_cache" bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
```
## How It Works
### Step 1: Baseline
1. Start single vLLM instance on GPU
2. Run test prompts (multimodal or text-only)
3. Save outputs to `.vllm_epd_baseline.txt`
4. Shutdown instance
### Step 2: EPD (1E + 1PD)
1. Clear encoder cache storage
2. Start instances and proxy
3. Run same test prompts
4. Assert outputs match baseline exactly
5. Shutdown instances
### Step 3: EPD (1E + 1P + 1D)
1. Clear encoder cache storage
2. Start instances and proxy
3. Run same test prompts
4. Assert outputs match baseline exactly
5. Shutdown instances
## Test Scenarios
### Multimodal Prompts (--use_mm_prompts)
Tests encoder cache transfer:
- Single image query
- Multiple images in one request
- Mixed image and text
- Image with detailed questions
### Text-Only Prompts (default)
Quick sanity check:
- Simple text queries
- Text-only explanations
- Verifies proxy routing works
## Expected Behavior
### ✅ Test Passes When
- All disagg outputs match baseline outputs exactly
- No errors during instance startup
- Encoder cache is properly saved and loaded
- Proxy correctly routes requests
### ❌ Test Fails When
- Outputs differ between baseline and disagg
- Server startup fails
- Encoder cache not found (should fall back to local execution)
- Proxy routing errors
## Notes
- The test uses deterministic generation (`temperature=0.0`, `seed=42`)
- Encoder cache should enable exact output reproduction
- Test cleans up all instances and cache files after completion
- Safe to run multiple times (idempotent)
- We setup the PD disagg part with NixlConnector. Please read details about EPD in `examples/disaggregated/disaggregated_encoder/README.md`
## Requirements
- Multiple GPUs (3 for 1E+1P+1D, 2 for 1E+1PD, 1 for baseline)
- 1E+1P+1D is runnable with 2 GPU by assign E and P on the same GPU now.
- Multimodal model (e.g., Qwen2.5-VL-3B-Instruct)
- Internet access (for accessing vllm test images)
## Debugging
### Check Logs
Logs and baseline output are saved in `/tmp/` by default.
Can be customized by changing the environment variables.
### Check Encoder Cache
```bash
# Verify cache files are created
ls -la $EC_SHARED_STORAGE_PATH/
# Should see directories with mm_hash names
# Each containing encoder_cache.safetensors
```
### Manual Testing
Run individual components:
```bash
# Baseline only
python test_epd_correctness.py \
--service_url http://localhost:8000 \
--model_name Qwen/Qwen2.5-VL-3B-Instruct \
--mode baseline \
--baseline_file test_output.txt \
--use_mm_prompts
# Disagg only (requires baseline output file!)
python test_epd_correctness.py \
--service_url http://localhost:8000 \
--model_name Qwen/Qwen2.5-VL-3B-Instruct \
--mode disagg \
--baseline_file test_output.txt \
--use_mm_prompts
```
## Files
- `run_epd_correctness_test.sh` - Main test script (starts all instances and runs tests)
- `test_epd_correctness.py` - Python test script (compares outputs)
## Usage
### Multimodal Prompts (Default)
```bash
cd vllm
./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
```
This runs the test with actual multimodal (image) prompts.
### Text-Only Prompts
```bash
cd vllm
USE_MM_PROMPTS=0 ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
```
This runs a quick test with text-only prompts to verify the setup works.
### Custom Configuration
```bash
# Use specific GPUs
GPU_E=0 GPU_PD=1 GPU_P=1 GPU_D=2 bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
# Use specific ports
ENDPOINT_PORT=10001 bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
# Use specific model
MODEL="Qwen/Qwen2.5-VL-3B-Instruct" bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
# Use specific storage path
EC_SHARED_STORAGE_PATH="/tmp/my_ec_cache" bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
```
## How It Works
### Mooncake EC (1E + 1PD)
```bash
PYTHON_BIN="$PWD/.venv/bin/python" \
bash tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh
```
Requires two GPUs and Mooncake TransferEngine. TCP is the default transport;
no RDMA-capable network hardware is required.
The script sets `MC_FORCE_TCP=1` in TCP mode so Mooncake cannot auto-select RDMA.
The baseline runs first on GPU 0, followed by E on GPU 0 and PD on GPU 1.
`MOONCAKE_EC_PROTOCOL=rdma` selects RDMA instead; host-specific transport
environment variables should be set by the caller.
Three black-box cases check fixed short answers and compare with the baseline:
- One image: read the STOP sign.
- Two different images (including a local file): identify flowers and birds.
- The same image twice: read both STOP signs.
By default, all three requests run concurrently for two rounds, exercising
shared hashes across requests and reuse after completion. Every response is
compared, not just the final round. Set `CONCURRENCY` and `REPEAT` to override.
Prefix caching is disabled and CUDA graphs remain enabled. This is a small
correctness suite, not a performance benchmark or failure-injection suite.
`LOG_PATH` and `BASELINE_FILE` select the log directory and reference output.
`SKIP_BASELINE=1` reuses a reference generated with the same model and test
configuration. `USE_MM_PROMPTS=0` only checks text routing, not EC transfer.
Buildkite runs this script as `mooncake-ec-tcp-e2e-2-gpus` on two L4 GPUs.
The job is defined in `.buildkite/test_areas/disaggregated_mooncake.yaml` and
selected for changes to EC, multimodal processing, scheduler/model-runner
integration, the proxy, or these tests (subject to normal PR CI approval).
It reuses the CI image's Python packages through a system-site-packages venv
and installs the CUDA-compatible Mooncake wheel. It uses loopback networking
and TCP only, without RDMA devices or peer-memory setup.
The job fails on startup errors, request errors, or answer mismatches. On
failure, the script prints the last 100 lines of each server/proxy log to
the CI job log; full files remain under `LOG_PATH` while the container exists.
### Step 1: Baseline
1. Start single vLLM instance on GPU
2. Run test prompts (multimodal or text-only)
3. Save outputs to `.vllm_epd_baseline.txt`
4. Shutdown instance
### Step 2: EPD (1E + 1PD)
1. Clear encoder cache storage
2. Start instances and proxy
3. Run same test prompts
4. Assert outputs match baseline exactly
5. Shutdown instances
### Step 3: EPD (1E + 1P + 1D)
1. Clear encoder cache storage
2. Start instances and proxy
3. Run same test prompts
4. Assert outputs match baseline exactly
5. Shutdown instances
## Test Scenarios
### Multimodal Prompts (--use_mm_prompts)
Tests encoder cache transfer:
- Single image query
- Multiple images in one request
- Mixed image and text
- Image with detailed questions
### Text-Only Prompts (default)
Quick sanity check:
- Simple text queries
- Text-only explanations
- Verifies proxy routing works
## Expected Behavior
### ✅ Test Passes When
- All disagg outputs match baseline outputs exactly
- No errors during instance startup
- Encoder cache is properly saved and loaded
- Proxy correctly routes requests
### ❌ Test Fails When
- Outputs differ between baseline and disagg
- Server startup fails
- Encoder cache not found (should fall back to local execution)
- Proxy routing errors
## Notes
- The test uses deterministic generation (`temperature=0.0`, `seed=42`)
- Encoder cache should enable exact output reproduction
- Test cleans up all instances and cache files after completion
- Safe to run multiple times (idempotent)
- We setup the PD disagg part with NixlConnector. Please read details about EPD in `examples/disaggregated/disaggregated_encoder/README.md`
## Requirements
- Multiple GPUs (3 for 1E+1P+1D, 2 for 1E+1PD, 1 for baseline)
- 1E+1P+1D is runnable with 2 GPU by assign E and P on the same GPU now.
- Multimodal model (e.g., Qwen2.5-VL-3B-Instruct)
- Internet access (for accessing vllm test images)
## Debugging
### Check Logs
Logs and baseline output are saved in `/tmp/` by default.
Can be customized by changing the environment variables.
### Check Encoder Cache
```bash
# Verify cache files are created
ls -la $EC_SHARED_STORAGE_PATH/
# Should see directories with mm_hash names
# Each containing encoder_cache.safetensors
```
### Manual Testing
Run individual components:
```bash
# Baseline only
python test_epd_correctness.py \
--service_url http://localhost:8000 \
--model_name Qwen/Qwen2.5-VL-3B-Instruct \
--mode baseline \
--baseline_file test_output.txt \
--use_mm_prompts
# Disagg only (requires baseline output file!)
python test_epd_correctness.py \
--service_url http://localhost:8000 \
--model_name Qwen/Qwen2.5-VL-3B-Instruct \
--mode disagg \
--baseline_file test_output.txt \
--use_mm_prompts
```
@@ -0,0 +1,289 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Full-stack EPD validation with ECMooncakeConnector (Mooncake TransferEngine):
# 1) Single-GPU baseline (multimodal) -> saves baseline JSON
# 2) 1 Encoder + 1 PD with Mooncake EC -> compare outputs to baseline
#
# Usage (from repo root):
# ./tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh
#
# Env:
# MODEL HF model id (default: Qwen/Qwen2.5-VL-3B-Instruct)
# GPU_SINGLE / GPU_E / GPU_PD GPU ids (defaults 0 / 0 / 1)
# ENDPOINT_PORT, ENCODE_PORT, PREFILL_DECODE_PORT
# EC_MOONCAKE_RESERVATION_HOST consumer control host (default 127.0.0.1)
# MOONCAKE_EC_PROTOCOL tcp | rdma (default tcp)
# USE_MM_PROMPTS 1 (default) or 0 for text-only quick sanity
# TIMEOUT_SECONDS wait_for_server timeout (default 1200)
# SKIP_BASELINE set to 1 to reuse existing BASELINE_FILE
# CONCURRENCY / REPEAT concurrent requests / rounds (defaults 3 / 2)
# MAX_MODEL_LEN context length (default 16384)
set -euo pipefail
GIT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)
cd "$GIT_ROOT" || exit 1
export PYTHONPATH="${GIT_ROOT}:${PYTHONPATH:-}"
PYTHON_BIN="${PYTHON_BIN:-${GIT_ROOT}/.venv/bin/python}"
MODEL="${MODEL:-Qwen/Qwen2.5-VL-3B-Instruct}"
USE_MM_PROMPTS="${USE_MM_PROMPTS:-1}"
TEST_ARGS=(--concurrency "${CONCURRENCY:-3}" --repeat "${REPEAT:-2}")
if [[ "$USE_MM_PROMPTS" == "1" ]]; then
TEST_ARGS+=(--use_mm_prompts --mm_smoke_test)
fi
MAX_MODEL_LEN="${MAX_MODEL_LEN:-16384}"
GPU_SINGLE="${GPU_SINGLE:-0}"
GPU_E="${GPU_E:-0}"
GPU_PD="${GPU_PD:-1}"
ENCODE_PORT="${ENCODE_PORT:-19534}"
PREFILL_DECODE_PORT="${PREFILL_DECODE_PORT:-19537}"
ENDPOINT_PORT="${ENDPOINT_PORT:-10002}"
BASELINE_PORT="${BASELINE_PORT:-10003}"
EC_MOONCAKE_RESERVATION_HOST="${EC_MOONCAKE_RESERVATION_HOST:-127.0.0.1}"
EC_MOONCAKE_RESERVATION_PORT="${EC_MOONCAKE_RESERVATION_PORT:-19019}"
MOONCAKE_EC_PROTOCOL="${MOONCAKE_EC_PROTOCOL:-tcp}"
export EC_MOONCAKE_RESERVATION_HOST
export EC_MOONCAKE_RESERVATION_PORT
export MOONCAKE_EC_PROTOCOL
if [[ "$MOONCAKE_EC_PROTOCOL" == "tcp" ]]; then
# TransferEngine may otherwise auto-select RDMA on hosts with an HCA.
export MC_FORCE_TCP=1
else
unset MC_FORCE_TCP
fi
LOG_PATH="${LOG_PATH:-/tmp}"
BASELINE_FILE="${BASELINE_FILE:-/tmp/vllm_epd_mooncake_baseline.txt}"
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-1200}"
PIDS=()
mkdir -p "$LOG_PATH"
VLLM_SERVE=("$PYTHON_BIN" -m vllm.entrypoints.cli.main serve)
ENC_EC_JSON=$("$PYTHON_BIN" <<PY
import json, os
print(json.dumps({
"ec_connector": "ECMooncakeConnector",
"ec_role": "ec_producer",
"ec_connector_extra_config": {
"mooncake_protocol": os.environ.get("MOONCAKE_EC_PROTOCOL", "tcp"),
},
}, separators=(",", ":")))
PY
)
PD_EC_JSON=$("$PYTHON_BIN" <<PY
import json, os
print(json.dumps({
"ec_connector": "ECMooncakeConnector",
"ec_role": "ec_consumer",
"ec_ip": os.environ["EC_MOONCAKE_RESERVATION_HOST"],
"ec_port": int(os.environ.get("EC_MOONCAKE_RESERVATION_PORT", "19019")),
"ec_connector_extra_config": {
"mooncake_protocol": os.environ.get("MOONCAKE_EC_PROTOCOL", "tcp"),
},
}, separators=(",", ":")))
PY
)
EC_MOONCAKE_RESERVATION_ADDR=$("$PYTHON_BIN" <<'PY'
import os
from vllm.utils.network_utils import make_zmq_path
print(make_zmq_path(
"tcp", os.environ["EC_MOONCAKE_RESERVATION_HOST"],
int(os.environ["EC_MOONCAKE_RESERVATION_PORT"]),
))
PY
)
wait_for_server() {
local port=$1
local pid=$2
local deadline=$((SECONDS + TIMEOUT_SECONDS))
while ((SECONDS < deadline)); do
kill -0 "$pid" 2>/dev/null || return 1
if curl --max-time 2 -fsS "http://localhost:${port}/health" >/dev/null 2>&1; then
return 0
fi
sleep 2
done
return 1
}
cleanup_instances() {
if ((${#PIDS[@]} == 0)); then
return
fi
echo "Cleaning up tracked vLLM / proxy processes..."
for pid in "${PIDS[@]}"; do
kill "$pid" 2>/dev/null || true
done
for _ in {1..10}; do
local alive=0
for pid in "${PIDS[@]}"; do
if kill -0 "$pid" 2>/dev/null; then
alive=1
fi
done
((alive == 0)) && break
sleep 1
done
for pid in "${PIDS[@]}"; do
if kill -0 "$pid" 2>/dev/null; then
kill -KILL "$pid" 2>/dev/null || true
fi
wait "$pid" 2>/dev/null || true
done
PIDS=()
}
finish() {
local status=$?
cleanup_instances
if ((status != 0)); then
for log in "${LOG_PATH}"/mooncake_epd_*.log; do
[[ -f "$log" ]] || continue
echo "=== $log (last 100 lines) ==="
tail -100 "$log"
done
fi
exit "$status"
}
trap finish EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
run_baseline() {
echo "================================"
echo "BASELINE (single vLLM, MM if enabled)"
echo "================================"
cleanup_instances
local PORT=$BASELINE_PORT
echo "Starting baseline on GPU $GPU_SINGLE port $PORT"
CUDA_VISIBLE_DEVICES="$GPU_SINGLE" "${VLLM_SERVE[@]}" "$MODEL" \
--port "$PORT" \
--gpu-memory-utilization 0.75 \
--max-num-seqs 32 \
--max-model-len "$MAX_MODEL_LEN" \
--no-enable-prefix-caching \
--limit-mm-per-prompt '{"image":2,"video":0}' \
--allowed-local-media-path "${GIT_ROOT}/tests/v1/ec_connector/integration" \
>"${LOG_PATH}/mooncake_epd_baseline.log" 2>&1 &
local BASELINE_PID=$!
PIDS+=("$BASELINE_PID")
echo "Waiting for baseline..."
wait_for_server "$PORT" "$BASELINE_PID" || { echo "Baseline failed to start"; return 1; }
curl -s "http://127.0.0.1:${PORT}/v1/models" | head -c 200 || true
echo ""
"$PYTHON_BIN" "${GIT_ROOT}/tests/v1/ec_connector/integration/test_epd_correctness.py" \
--service_url "http://localhost:$PORT" \
--model_name "$MODEL" \
--mode baseline \
--baseline_file "$BASELINE_FILE" \
"${TEST_ARGS[@]}"
cleanup_instances
}
run_epd_mooncake() {
echo "================================"
echo "EPD 1E + 1PD with ECMooncakeConnector"
echo "Reservation port on consumer: $EC_MOONCAKE_RESERVATION_PORT"
echo "Mooncake protocol: $MOONCAKE_EC_PROTOCOL"
echo "================================"
cleanup_instances
echo "Starting ENCODER on GPU $GPU_E port $ENCODE_PORT"
CUDA_VISIBLE_DEVICES="$GPU_E" "${VLLM_SERVE[@]}" "$MODEL" \
--port "$ENCODE_PORT" \
--gpu-memory-utilization 0.35 \
--mm-tensor-ipc torch_shm \
--enable-request-id-headers \
--no-enable-prefix-caching \
--max-num-batched-tokens "$MAX_MODEL_LEN" \
--max-num-seqs 32 \
--max-model-len "$MAX_MODEL_LEN" \
--limit-mm-per-prompt '{"image":2,"video":0}' \
--allowed-local-media-path "${GIT_ROOT}/tests/v1/ec_connector/integration" \
--ec-transfer-config "$ENC_EC_JSON" \
>"${LOG_PATH}/mooncake_epd_encoder.log" 2>&1 &
local ENCODER_PID=$!
PIDS+=("$ENCODER_PID")
echo "Starting PD on GPU $GPU_PD port $PREFILL_DECODE_PORT"
CUDA_VISIBLE_DEVICES="$GPU_PD" "${VLLM_SERVE[@]}" "$MODEL" \
--port "$PREFILL_DECODE_PORT" \
--gpu-memory-utilization 0.75 \
--mm-tensor-ipc torch_shm \
--enable-mm-embeds \
--enable-request-id-headers \
--max-num-seqs 32 \
--max-model-len "$MAX_MODEL_LEN" \
--no-enable-prefix-caching \
--limit-mm-per-prompt '{"image":2,"video":0}' \
--allowed-local-media-path "${GIT_ROOT}/tests/v1/ec_connector/integration" \
--ec-transfer-config "$PD_EC_JSON" \
>"${LOG_PATH}/mooncake_epd_pd.log" 2>&1 &
local PD_PID=$!
PIDS+=("$PD_PID")
echo "Waiting for encoder..."
wait_for_server "$ENCODE_PORT" "$ENCODER_PID" || { echo "Encoder failed to start"; return 1; }
echo "Waiting for PD..."
wait_for_server "$PREFILL_DECODE_PORT" "$PD_PID" || { echo "PD failed to start"; return 1; }
echo "Starting EPD proxy on $ENDPOINT_PORT"
"$PYTHON_BIN" "${GIT_ROOT}/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py" \
--host "0.0.0.0" \
--port "$ENDPOINT_PORT" \
--encode-servers-urls "http://localhost:$ENCODE_PORT" \
--prefill-servers-urls "disable" \
--decode-servers-urls "http://localhost:$PREFILL_DECODE_PORT" \
--ec-consumer-zmq-addrs \
"$EC_MOONCAKE_RESERVATION_ADDR" \
>"${LOG_PATH}/mooncake_epd_proxy.log" 2>&1 &
local PROXY_PID=$!
PIDS+=("$PROXY_PID")
echo "Waiting for proxy..."
wait_for_server "$ENDPOINT_PORT" "$PROXY_PID" || { echo "Proxy failed to start"; return 1; }
curl -s "http://127.0.0.1:${ENDPOINT_PORT}/health" || true
echo ""
"$PYTHON_BIN" "${GIT_ROOT}/tests/v1/ec_connector/integration/test_epd_correctness.py" \
--service_url "http://localhost:$ENDPOINT_PORT" \
--model_name "$MODEL" \
--mode disagg \
--baseline_file "$BASELINE_FILE" \
"${TEST_ARGS[@]}"
cleanup_instances
}
echo "================================"
echo "1E + 1PD ECMooncake end-to-end correctness"
echo "MODEL=$MODEL"
echo "================================"
if [[ "${SKIP_BASELINE:-0}" != "1" ]]; then
run_baseline
else
echo "SKIP_BASELINE=1 -> using existing $BASELINE_FILE"
[[ -f "$BASELINE_FILE" ]] || { echo "Missing baseline file"; exit 1; }
fi
run_epd_mooncake
echo "================================"
echo "PASS: Mooncake EC EPD matches baseline"
echo "Logs: ${LOG_PATH}/mooncake_epd_*.log"
echo "================================"
@@ -26,6 +26,7 @@ import argparse
import json
import os
import time
from concurrent.futures import ThreadPoolExecutor
import openai
import requests
@@ -138,17 +139,20 @@ def run_chat_completion(
Returns:
Generated text content
"""
client = openai.OpenAI(api_key="EMPTY", base_url=base_url)
with openai.OpenAI(
api_key="EMPTY", base_url=base_url, timeout=120, max_retries=0
) as client:
completion = client.chat.completions.create(
model=model_name,
messages=messages,
max_tokens=max_tokens,
temperature=0.0,
seed=42,
)
completion = client.chat.completions.create(
model=model_name,
messages=messages,
max_tokens=max_tokens,
temperature=0.0,
seed=42,
)
return completion.choices[0].message.content
content = completion.choices[0].message.content
assert content, "Expected a nonempty completion"
return content
def main():
@@ -198,7 +202,19 @@ def main():
help="Skip the two-image multimodal prompt",
)
parser.add_argument(
"--mm_smoke_test",
action="store_true",
help="Use three short, fixed-answer image cases, including duplicate images",
)
parser.add_argument("--concurrency", type=int, default=1)
parser.add_argument("--repeat", type=int, default=1)
args = parser.parse_args()
if args.concurrency < 1 or args.repeat < 1:
parser.error("--concurrency and --repeat must be positive")
if args.mm_smoke_test and (not args.use_mm_prompts or args.skip_two_image_prompt):
parser.error("--mm_smoke_test requires all multimodal prompts")
print(f"Service URL: {args.service_url}")
print(f"Model: {args.model_name}")
@@ -235,27 +251,76 @@ def main():
test_prompts = SAMPLE_PROMPTS_TEXT
print("Using text-only prompts for quick testing")
if args.mm_smoke_test:
image = SAMPLE_PROMPTS_MM[0]["messages"][0]["content"][0]
image_pair = SAMPLE_PROMPTS_MM[1]["messages"][0]["content"][:2]
cases = [
(
"Single image",
[image],
"What word is on the red road sign? Reply with only that word "
"in uppercase.",
"STOP",
),
(
"Two different images",
image_pair,
"Do these pictures contain both flowers and birds? "
"Reply with only YES or NO.",
"YES",
),
(
"Same image twice",
[image, image],
"Read the red road sign in each image, in image order. Reply "
"with only the two uppercase words separated by a comma and a space.",
"STOP, STOP",
),
]
test_prompts = [
{
"description": description,
"expected": expected,
"messages": [
{
"role": "user",
"content": [*images, {"type": "text", "text": question}],
}
],
}
for description, images, question, expected in cases
]
# Run completions
service_url = f"{args.service_url}/v1"
output_strs = {}
for i, prompt_data in enumerate(test_prompts):
print(
f"\nRunning prompt {i + 1}/{len(test_prompts)}: "
f"{prompt_data['description']}"
)
output_str = run_chat_completion(
def complete(prompt_data):
output = run_chat_completion(
base_url=service_url,
model_name=args.model_name,
messages=prompt_data["messages"],
max_tokens=MAX_OUTPUT_LEN,
max_tokens=16 if args.mm_smoke_test else MAX_OUTPUT_LEN,
)
if args.mm_smoke_test:
output = output.strip()
assert output == prompt_data["expected"], (
f"{prompt_data['description']}: expected {prompt_data['expected']!r}, "
f"got {output!r}"
)
return output
# Use description as key for comparison
key = prompt_data["description"]
output_strs[key] = output_str
print(f"Output: {output_str}")
# Each round includes concurrent requests sharing image hashes; later rounds
# exercise reuse after previous requests have finished.
with ThreadPoolExecutor(max_workers=args.concurrency) as executor:
for repeat in range(args.repeat):
outputs = executor.map(complete, test_prompts)
for prompt_data, output_str in zip(test_prompts, outputs):
key = prompt_data["description"]
if args.repeat > 1:
key = f"{key} (round {repeat + 1})"
output_strs[key] = output_str
print(f"{key}: {output_str}")
if args.mode in ("baseline", "baseline_pd"):
# Baseline mode: Save outputs
@@ -0,0 +1,172 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Body handling across the EPD proxy's decode retries.
Exercises the REAL helpers loaded from the ``examples/`` proxy, so a future
change to them is what these tests catch.
"""
import asyncio
import importlib.util
from pathlib import Path
import pytest
PROXY_REL = "examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py"
@pytest.fixture(scope="module")
def proxy():
path = Path(__file__).parents[4] / PROXY_REL
spec = importlib.util.spec_from_file_location("disagg_epd_proxy_retry", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
class _Response:
def __init__(self, params):
self._params = params
async def json(self):
return {"kv_transfer_params": self._params}
def test_maybe_prefill_leaves_the_caller_body_untouched(proxy, monkeypatch):
"""A decode retry re-enters this function with the body it was given.
Mutating that body in place let one attempt's `remote_block_ids` survive
into the next, so a retry whose prefill returns nothing sent decode blocks
the prefiller may already have freed.
"""
served = [{"remote_block_ids": [1, 2]}, {}]
async def _stage(req_data, p_url, req_id):
assert "kv_transfer_params" not in req_data
return _Response(served.pop(0))
monkeypatch.setattr(proxy, "process_prefill_stage", _stage)
body = {"messages": [], "stream": False}
first = asyncio.run(proxy.maybe_prefill(body, "http://prefill", "r1"))
assert first["kv_transfer_params"] == {"remote_block_ids": [1, 2]}
assert "kv_transfer_params" not in body
second = asyncio.run(proxy.maybe_prefill(body, "http://prefill", "r1"))
assert "kv_transfer_params" not in second
class _EncoderResponse:
def __init__(self, params):
self.status = 200
self._params = params
async def json(self):
return {"ec_transfer_params": self._params}
async def text(self):
return ""
class _EncoderSession:
"""Serve one canned encoder reply per attempt."""
def __init__(self, replies):
self._replies = list(replies)
async def post(self, url, json=None, headers=None):
return _EncoderResponse(self._replies.pop(0))
def test_a_decode_retry_does_not_inherit_the_previous_handles(proxy, monkeypatch):
"""The retry loop re-enters `prepare_for_decode` with the same body.
Recording the encoder's connector handles on that body in place let
attempt 1's handle survive into attempt 2, so a second encode that
reported nothing still sent decode a handle on an embedding the encoder
no longer publishes -- the exact state the retry exists to leave behind.
"""
handle = {"metadata": {"image_grid_thw": [1, 2, 2]}, "peer_port": 1234}
monkeypatch.setattr(
proxy,
"encode_session",
_EncoderSession([{"encoder-side-hash": handle}, {}]),
)
async def _no_prefill(req_data, p_url, req_id):
return req_data
monkeypatch.setattr(proxy, "maybe_prefill", _no_prefill)
body = {
"messages": [
{
"role": "user",
"content": [{"type": "image_url", "image_url": {"url": "image"}}],
}
],
"stream": False,
}
args = ("r1", ["http://encoder"], "http://prefill", None)
first, _, _ = asyncio.run(proxy.prepare_for_decode(body, *args))
reported = first["ec_transfer_params"]
assert [handle] == [value for key, value in reported.items() if key != "ec_items"]
assert "ec_transfer_params" not in body
# Attempt 2's encode reports nothing: decode must be told nothing.
second, _, _ = asyncio.run(proxy.prepare_for_decode(body, *args))
assert "ec_transfer_params" not in second
def test_raw_media_keeps_encoder_transfer_identity(proxy, monkeypatch):
handle = {"metadata": {"image_grid_thw": [1, 2, 2]}}
monkeypatch.setattr(proxy, "NO_REWRITE", True)
monkeypatch.setattr(
proxy, "encode_session", _EncoderSession([{"encoded-hash": handle}])
)
body = {
"messages": [
{
"role": "user",
"content": [{"type": "image_url", "image_url": {"url": "image"}}],
}
]
}
prepared, _, _ = asyncio.run(
proxy.prepare_for_decode(
body, "request", ["http://encoder"], "", "tcp://consumer:1"
)
)
assert prepared["messages"] == body["messages"]
assert "ec_transfer_params" not in body
params = prepared["ec_transfer_params"]
assert params["encoded-hash"] == handle
assert params["ec_items"][0]["mm_hash"] == "encoded-hash"
assert params["ec_items"][0]["transfer_id"]
@pytest.mark.parametrize("server_keep_alive", ["0.1", "1", "5", "2", "30"])
def test_pooled_connections_are_retired_before_the_server_closes_them(
proxy, monkeypatch, server_keep_alive
):
"""The proxy must not hand a request a connection the server has dropped.
vLLM closes idle keep-alive connections at `VLLM_HTTP_TIMEOUT_KEEP_ALIVE`
seconds while aiohttp pools them for 15, so a slow hop leaves a dead
connection in the pool. The next request fails with
ServerDisconnectedError and the server logs nothing, because it closed the
socket before the request arrived.
"""
monkeypatch.setenv("VLLM_HTTP_TIMEOUT_KEEP_ALIVE", server_keep_alive)
monkeypatch.setattr(proxy.app.state, "p_urls", [], raising=False)
asyncio.run(proxy.on_startup())
try:
# No public accessor for the pool's idle timeout.
pooled_for = proxy.encode_session.connector._keepalive_timeout
assert pooled_for < float(server_keep_alive)
assert proxy.decode_session.connector._keepalive_timeout == pooled_for
finally:
asyncio.run(proxy.on_shutdown())
@@ -32,8 +32,13 @@ def _load_proxy_module():
@pytest.fixture(scope="module")
def assign():
return _load_proxy_module().encoder_rr_assignment
def proxy():
return _load_proxy_module()
@pytest.fixture(scope="module")
def assign(proxy):
return proxy.encoder_rr_assignment
def _drive(assign, e_urls, counts):
@@ -80,3 +85,44 @@ def test_single_encoder_always_resolves_to_it(assign):
urls, next_cursor = assign(["E0"], 0, 4)
assert urls == ["E0"] * 4
assert next_cursor == 0
def test_generic_e_p_d_routing_without_mooncake_is_supported(proxy):
proxy.validate_ec_consumer_routing(["http://prefill"], [])
def test_mooncake_e_pd_routing_is_supported(proxy):
proxy.validate_ec_consumer_routing([], ["tcp://decode:19019"])
def test_mooncake_independent_prefill_routing_fails_fast(proxy):
with pytest.raises(ValueError, match=r"supports E\+PD only"):
proxy.validate_ec_consumer_routing(["http://prefill"], ["tcp://decode:19019"])
def test_decode_rewrite_preserves_engine_reported_ec_hash(proxy):
request = {
"messages": [
{
"role": "user",
"content": [{"type": "image_url", "image_url": {"url": "image"}}],
}
]
}
rewritten = proxy.rewrite_for_decode(
request,
{
0: {
"mm_hash": "proxy-uuid",
"ec_mm_hash": "engine-derived-hash",
"transfer_id": "transfer",
"image_grid_thw": [1, 2, 3],
}
},
)
assert rewritten["messages"][0]["content"][0]["uuid"] == "proxy-uuid"
assert rewritten["ec_transfer_params"]["ec_items"] == [
{"mm_hash": "engine-derived-hash", "transfer_id": "transfer"}
]
@@ -12,6 +12,9 @@ from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorMetadata,
)
from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT
from vllm.v1.worker.ec_connector_model_runner_mixin import (
ECConnectorModelRunnerMixin,
)
from vllm.v1.worker.gpu.ec_connector import NO_OP_EC_CONNECTOR, ActiveECConnector
pytestmark = pytest.mark.cpu_test
@@ -52,6 +55,9 @@ def test_saves_newly_added_caches_for_every_producer(is_producer, is_consumer):
saved = [call.kwargs["mm_hash"] for call in fake.save_caches.call_args_list]
assert saved == (["mm_new"] if is_producer else [])
assert fake.start_save_caches.called == is_producer
if is_producer:
assert fake.start_save_caches.call_args.kwargs["encoder_cache"] is encoder_cache
assert fake.start_load_caches.called == is_consumer
@@ -66,6 +72,26 @@ def test_worker_meta_is_reported_on_context_exit():
assert fake.clear_connector_metadata.called
def test_v1_producer_receives_existing_encoder_cache():
encoder_cache = {"mm_cached": None}
fake = MagicMock(spec=ECConnectorBase)
fake.is_producer = True
fake.is_consumer = False
fake.get_finished.return_value = (None, None)
module = "vllm.v1.worker.ec_connector_model_runner_mixin"
with (
patch(f"{module}.has_ec_transfer", return_value=True),
patch(f"{module}.get_ec_transfer", return_value=fake),
ECConnectorModelRunnerMixin.maybe_get_ec_connector_output(
_scheduler_output(), encoder_cache
),
):
pass
assert fake.start_save_caches.call_args.kwargs["encoder_cache"] is encoder_cache
def test_no_forward_reports_without_running_the_model():
connector, _ = _connector()
+5
View File
@@ -18,6 +18,11 @@ class ECTransferConfig:
ec_connector: str | None = None
"""The EC connector for vLLM to transmit EC caches between vLLM instances.
Built-in options include ``ECExampleConnector`` (shared filesystem via
safetensors) and ``ECMooncakeConnector`` (Mooncake TransferEngine RDMA;
requires ``mooncake-transfer-engine`` and matching producer/consumer
``ec_connector_extra_config``; see ``mooncake_ec_connector`` module docstring).
"""
engine_id: str | None = None
@@ -261,6 +261,25 @@ class ECConnectorBase(ABC):
"""
return True
def start_save_caches(self, **kwargs: Any) -> None:
"""Prepare this step's outbound pushes before the model runs."""
return
def start_worker_services(self) -> None:
"""Start Worker-side services once the model is resident."""
return
def take_unavailable_requests(self) -> set[str]:
"""Request IDs whose encoder inputs the connector can no longer obtain.
The Scheduler fails these; re-issuing the request re-runs the encode.
"""
return set()
def update_state_after_free(self, request: "Request", index: int):
"""Notify the connector that an encoder cache entry was released."""
return
@abstractmethod
def update_state_after_alloc(self, request: "Request", index: int):
"""
@@ -7,6 +7,7 @@ Owns the mmap region and the embedding cache, and handles the producer
for the ECCPUConnector.
"""
from collections.abc import Collection
from typing import TYPE_CHECKING, Any
import torch
@@ -184,7 +185,10 @@ class ECCPUScheduler:
return entry is not None and entry.ready
def ensure_cache_available(
self, request: "Request", num_computed_tokens: int
self,
request: "Request",
num_computed_tokens: int,
local_cache_hashes: Collection[str] | None = None,
) -> bool:
if not self._nixl_enabled:
return True # CPU offload never blocks.
@@ -89,3 +89,9 @@ ECConnectorFactory.register_connector(
"vllm.distributed.ec_transfer.ec_connector.cpu.connector",
"ECCPUConnector",
)
ECConnectorFactory.register_connector(
"ECMooncakeConnector",
"vllm.distributed.ec_transfer.ec_connector.mooncake_ec_connector",
"ECMooncakeConnector",
)
@@ -0,0 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Internal building blocks for the Mooncake encoder-cache connector.
The public connector remains in :mod:`mooncake_ec_connector`. This package
separates configuration, control-plane messaging, transfer state, registered
memory, and worker/scheduler orchestration so that each component has one
owner and can be tested independently.
"""
@@ -0,0 +1,123 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import math
from dataclasses import dataclass
from typing import TYPE_CHECKING
from vllm.utils.network_utils import make_zmq_path
if TYPE_CHECKING:
from vllm.config import VllmConfig
# Scheduler and Worker must agree on when a remote reservation becomes stale.
_RESERVATION_TTL_SECONDS = 300
def _positive_int(name: str, value: object) -> int:
message = f"ECMooncakeConnector requires {name} to be a positive integer."
if isinstance(value, bool):
raise ValueError(message)
if isinstance(value, int):
result = value
elif isinstance(value, float) and math.isfinite(value) and value.is_integer():
result = int(value)
elif isinstance(value, str):
try:
result = int(value)
except ValueError as error:
raise ValueError(message) from error
else:
raise ValueError(message)
if result <= 0:
raise ValueError(f"ECMooncakeConnector requires {name} > 0.")
return result
def _positive_float(name: str, value: object) -> float:
message = f"ECMooncakeConnector requires {name} > 0."
if isinstance(value, bool):
raise ValueError(message)
try:
result = float(value) # type: ignore[arg-type]
except (TypeError, ValueError) as error:
raise ValueError(message) from error
if not math.isfinite(result) or result <= 0:
raise ValueError(message)
return result
@dataclass(frozen=True)
class MooncakeECConfig:
"""Validated settings shared by the Scheduler and Worker roles.
``control_port`` is the first TP-shard port after the DP offset;
``control_addr`` targets that shard, which advertises the full topology.
``control_host`` is the address this instance both advertises and binds,
so reaching a Consumer means being routed to it rather than finding it on
every interface.
"""
is_producer: bool
is_consumer: bool
protocol: str
buffer_device: str
control_host: str
control_port: int
control_addr: str
control_timeout_ms: int
push_wait_timeout_s: float
pool_size: int
@classmethod
def from_vllm_config(cls, vllm_config: VllmConfig) -> MooncakeECConfig:
parallel_config = vllm_config.parallel_config
ec_config = vllm_config.ec_transfer_config
assert ec_config is not None
if ec_config.is_ec_producer:
if parallel_config.tensor_parallel_size > 1:
raise ValueError(
"ECMooncakeConnector producers require tensor_parallel_size=1."
)
if parallel_config.pipeline_parallel_size > 1:
raise ValueError(
"ECMooncakeConnector producers do not support pipeline parallelism."
)
if parallel_config.data_parallel_size > 1:
raise ValueError(
"ECMooncakeConnector producers require data_parallel_size=1."
)
registered_buffer_size = _positive_int(
"ec_buffer_size", ec_config.ec_buffer_size
)
get = ec_config.get_from_extra_config
control_port = int(ec_config.ec_port) + (
parallel_config.data_parallel_index * parallel_config.tensor_parallel_size
)
highest_port = control_port + parallel_config.tensor_parallel_size - 1
if not 1 <= control_port <= highest_port <= 65535:
raise ValueError("ECMooncakeConnector ec_port must be in 1..65535.")
return cls(
is_producer=ec_config.is_ec_producer,
is_consumer=ec_config.is_ec_consumer,
protocol=str(get("mooncake_protocol", "rdma")),
buffer_device=str(ec_config.ec_buffer_device or "cuda").lower(),
control_host=str(ec_config.ec_ip),
control_port=control_port,
control_addr=make_zmq_path("tcp", ec_config.ec_ip, control_port),
control_timeout_ms=max(
1,
math.ceil(
_positive_float("control_timeout_s", get("control_timeout_s", 30))
* 1000
),
),
push_wait_timeout_s=_positive_float(
"push_wait_timeout_s", get("push_wait_timeout_s", 60)
),
pool_size=registered_buffer_size,
)
@@ -0,0 +1,460 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ZMQ control plane for Mooncake encoder-cache reservations and events.
Tensor bytes never travel through this module. It coordinates destination
reservations, completion/cancellation, TP-shard discovery, and readiness
notifications while :mod:`transfer` owns the Mooncake data plane.
"""
from __future__ import annotations
import threading
import time
from collections import deque
from collections.abc import Callable
from concurrent.futures import Executor, Future
from typing import Any
import torch
import zmq
from vllm.logger import init_logger
from vllm.utils.network_utils import make_zmq_path
logger = init_logger(__name__)
_MAX_PENDING_EVENTS = 4096
_RESERVATION_REAP_INTERVAL_SECONDS = 1
ControlRequest = dict[str, Any]
ControlResponse = dict[str, Any]
class ControlClient:
"""Send control requests through reusable, thread-local REQ sockets."""
def __init__(self, timeout_ms: int) -> None:
self._context = zmq.Context()
self._timeout_ms = timeout_ms
self._local = threading.local()
self._topologies: dict[str, list[str]] = {}
self._closed = False
def _sockets(self) -> dict[str, zmq.Socket]:
sockets = getattr(self._local, "sockets", None)
if sockets is None:
sockets = {}
self._local.sockets = sockets
return sockets
def _discard(self, addr: str) -> None:
socket = self._sockets().pop(addr, None)
if socket is not None:
socket.close(linger=0)
def _exchange(self, addr: str, payload: ControlRequest) -> ControlResponse:
sockets = self._sockets()
socket = sockets.get(addr)
if socket is None:
socket = self._context.socket(zmq.REQ)
socket.setsockopt(zmq.IPV6, 1)
socket.setsockopt(zmq.RCVTIMEO, self._timeout_ms)
socket.setsockopt(zmq.SNDTIMEO, self._timeout_ms)
socket.setsockopt(zmq.LINGER, 0)
socket.connect(addr)
sockets[addr] = socket
try:
socket.send_json(payload)
response = socket.recv_json()
except Exception:
self._discard(addr)
raise
assert isinstance(response, dict)
return response
def request(self, addr: str, payload: ControlRequest) -> Any:
response = self._exchange(addr, payload)
if not response.get("ok"):
raise RuntimeError(response.get("error", "EC control request failed"))
return response.get("result")
def discover_shards(self, base_addr: str) -> list[str] | None:
if base_addr in self._topologies:
return self._topologies[base_addr]
try:
reply = self.request(base_addr, {"op": "peers"})
ports = reply.get("ports") if isinstance(reply, dict) else None
if not isinstance(ports, list) or not ports:
raise ValueError("invalid or empty peer list")
prefix = base_addr.rsplit(":", 1)[0]
shards = [f"{prefix}:{int(port)}" for port in ports]
except Exception:
logger.warning(
"EC Mooncake consumer at %s did not report its shards",
base_addr,
exc_info=True,
)
return None
self._topologies[base_addr] = shards
return shards
def close(self) -> None:
if self._closed:
return
self._closed = True
self._context.destroy(linger=0)
def make_cancel_request(
transfer_id: str,
reservation_id: str = "",
*,
abandon: bool = False,
refresh: bool = False,
) -> ControlRequest:
request: ControlRequest = {
"op": "cancel",
"transfer_id": transfer_id,
"reservation_id": reservation_id,
}
if abandon:
request["abandon"] = True
if refresh:
request["refresh"] = True
return request
class EventInbox:
"""Receive Consumer readiness events without blocking the Scheduler.
Subscribing needs one blocking control request per shard, so it runs on
`executor` when one is given: an unreachable Consumer must cost the
Scheduler nothing, not one control timeout per drain.
"""
def __init__(self, client: ControlClient, executor: Executor | None = None) -> None:
self._client = client
self._executor = executor
self._discovery: Future[list[str] | None] | None = None
self._context: zmq.Context | None = None
self._socket: zmq.Socket | None = None
self._closed = False
self.shard_count = 1
def _endpoints(self, base_addr: str) -> list[str] | None:
"""Ask every shard where it publishes readiness events.
Blocking: called on `self._executor` unless there is none.
"""
shards = self._client.discover_shards(base_addr)
if shards is None:
return None
endpoints = []
for addr in shards:
try:
event_port = self._client.request(addr, {"op": "event_port"})
address, _ = addr.rsplit(":", 1)
endpoints.append(f"{address}:{int(event_port)}")
except Exception:
logger.warning(
"EC Mooncake could not subscribe to the event channel of "
"consumer shard %s; retrying the complete topology later.",
addr,
)
return None
return endpoints
def _connect(self, base_addr: str) -> None:
if self._socket is not None or self._closed:
return
if self._executor is None:
self._install(self._endpoints(base_addr))
return
if self._discovery is None:
self._discovery = self._executor.submit(self._endpoints, base_addr)
return
if not self._discovery.done():
return
discovery, self._discovery = self._discovery, None
try:
self._install(discovery.result())
except Exception:
logger.warning(
"EC Mooncake event-channel discovery for %s failed; retrying.",
base_addr,
exc_info=True,
)
def _install(self, endpoints: list[str] | None) -> None:
"""Adopt a discovered topology.
Runs on the caller's thread so the socket is only ever touched there.
"""
if not endpoints or self._closed:
return
context = zmq.Context()
socket = context.socket(zmq.PULL)
socket.setsockopt(zmq.IPV6, 1)
for endpoint in endpoints:
socket.connect(endpoint)
self._context = context
self._socket = socket
self.shard_count = len(endpoints)
def drain(self, base_addr: str) -> list[dict[str, Any]]:
self._connect(base_addr)
if self._socket is None:
return []
events = []
while True:
try:
events.append(self._socket.recv_json(flags=zmq.DONTWAIT))
except zmq.Again:
return events
except zmq.ZMQError:
logger.warning("EC Mooncake event channel failed", exc_info=True)
return events
except (ValueError, UnicodeError):
# The event channel is a plain PULL socket: an undecodable
# frame must cost one frame, not the engine.
logger.warning(
"Discarding an undecodable EC Mooncake event.", exc_info=True
)
def close(self) -> None:
if self._closed:
return
self._closed = True
if self._discovery is not None:
self._discovery.cancel()
self._discovery = None
if self._socket is not None:
self._socket.close(linger=0)
self._socket = None
if self._context is not None:
self._context.term()
class ConsumerControlServer:
"""Expose Consumer reservations and readiness events over ZMQ.
One server runs on every receiving TP rank. The REP channel handles
reservation operations, while a PUSH channel publishes newly ready items
to the Scheduler.
"""
def __init__(
self,
host: str,
port: int,
reserve: Callable[[dict[str, Any]], dict[str, Any]],
status: Callable[[str], dict[str, Any] | None],
complete: Callable[[str, str], tuple[bool, bool]],
cancel: Callable[[str, str, bool, bool], bool],
reap: Callable[[], int],
peer_ports: list[int] | None = None,
device: torch.device | None = None,
drain_ready: Callable[[], list[str]] = lambda: [],
) -> None:
self.host = host
self.port = port
self.peer_ports = peer_ports or [port]
self._device = device
self.event_port: int | None = None
self._reserve = reserve
self._status = status
self._complete = complete
self._cancel = cancel
self._reap = reap
self._drain_ready = drain_ready
self._stop = threading.Event()
self._started = threading.Event()
self._thread: threading.Thread | None = None
self._startup_error: Exception | None = None
def start(self) -> None:
def loop() -> None:
if self._device is not None and self._device.type == "cuda":
torch.accelerator.set_device_index(self._device.index or 0)
context = zmq.Context()
socket = context.socket(zmq.REP)
event_socket = context.socket(zmq.PUSH)
socket.setsockopt(zmq.IPV6, 1)
event_socket.setsockopt(zmq.IPV6, 1)
pending_events: deque[dict[str, Any]] = deque()
def queue_event(event: dict[str, Any]) -> None:
event["shard"] = self.port
if len(pending_events) >= _MAX_PENDING_EVENTS:
dropped = pending_events.popleft()
logger.warning(
"EC Mooncake event backlog full on port %d; dropping "
"readiness for transfer_id=%s",
self.port,
dropped.get("transfer_id"),
)
pending_events.append(event)
def queue_ready(transfer_id: str) -> None:
status = self._status(transfer_id)
if status is not None:
queue_event({"transfer_id": transfer_id, **status})
last_reap_at = time.monotonic()
socket.setsockopt(zmq.RCVTIMEO, 100)
try:
socket.bind(make_zmq_path("tcp", self.host, self.port))
event_socket.bind(make_zmq_path("tcp", self.host, 0))
self.event_port = int(
event_socket.getsockopt(zmq.LAST_ENDPOINT)
.decode()
.rsplit(":", 1)[1]
)
except Exception as e:
self._startup_error = e
self._started.set()
socket.close(linger=0)
event_socket.close(linger=0)
context.term()
return
self._started.set()
try:
while not self._stop.is_set():
while pending_events:
try:
event_socket.send_json(
pending_events[0], flags=zmq.DONTWAIT
)
except zmq.Again:
break
pending_events.popleft()
now = time.monotonic()
if now - last_reap_at >= _RESERVATION_REAP_INTERVAL_SECONDS:
self._reap()
last_reap_at = now
try:
request = socket.recv_json()
except zmq.Again:
continue
except Exception:
# The frame arrived but did not decode. REP still owes a
# reply, so answer before returning to the loop.
logger.exception(
"EC Mooncake control channel on port %d received an "
"undecodable request",
self.port,
)
socket.send_json(
{"ok": False, "error": "malformed control request"}
)
continue
try:
op = request.get("op")
result: Any = None
if op in ("reserve", "reserve_batch"):
items = (
request["items"] if op == "reserve_batch" else [request]
)
results = []
for item in items:
try:
reserved = self._reserve(item)
results.append({"ok": True, "result": reserved})
if reserved.get("ready"):
queue_ready(str(item["transfer_id"]))
except Exception as exc:
results.append({"ok": False, "error": str(exc)})
if op == "reserve_batch":
result = {"items": results}
elif not results[0]["ok"]:
raise RuntimeError(results[0]["error"])
else:
result = results[0]["result"]
elif op == "status":
result = self._status(str(request["transfer_id"]))
elif op == "event_port":
result = self.event_port
elif op == "peers":
result = {"ports": self.peer_ports}
elif op in ("complete", "complete_batch"):
items = (
request["items"]
if op == "complete_batch"
else [request]
)
completions = []
for item in items:
transfer_id = str(item["transfer_id"])
accepted, became_ready = self._complete(
transfer_id, str(item["reservation_id"])
)
for ready_id in self._drain_ready():
queue_ready(ready_id)
completions.append(
{
"completed": accepted,
"became_ready": became_ready,
}
)
if not became_ready:
continue
queue_ready(transfer_id)
result = (
{"items": completions}
if op == "complete_batch"
else completions[0]
)
elif op == "cancel":
result = {
"cancelled": self._cancel(
str(request["transfer_id"]),
str(request.get("reservation_id", "")),
bool(request.get("abandon", False)),
bool(request.get("refresh", False)),
)
}
else:
raise ValueError(f"unknown control op: {op!r}")
socket.send_json({"ok": True, "result": result})
except Exception as e:
socket.send_json({"ok": False, "error": str(e)})
except Exception:
# `finally` closes the sockets and the thread ends, so without
# this every later reserve against this shard would surface
# only as a control timeout with nothing to attribute it to.
logger.exception(
"EC Mooncake control channel on port %d stopped serving",
self.port,
)
raise
finally:
socket.close(linger=0)
event_socket.close(linger=0)
context.term()
self._thread = threading.Thread(
target=loop, name="ec-mooncake-control", daemon=True
)
self._thread.start()
if not self._started.wait(timeout=5):
raise RuntimeError("EC Mooncake control channel failed to start")
if self._startup_error is not None:
raise RuntimeError("EC Mooncake control channel failed to bind") from (
self._startup_error
)
logger.info(
"EC Mooncake control channel listening on tcp://%s:%d (events tcp://%s:%d)",
self.host,
self.port,
self.host,
self.event_port,
)
def close(self) -> None:
if self._thread is None:
return
self._stop.set()
self._thread.join()
self._thread = None
@@ -0,0 +1,504 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Registered-memory allocation and residency for Mooncake transfers.
The Producer pool stages source tensors in a registered slab. The Consumer
pool owns destination allocations from reservation through publication,
resident reuse, CUDA-safe retirement, and pressure-driven reclamation.
"""
from __future__ import annotations
import bisect
import threading
from collections import OrderedDict
from collections.abc import Callable
from dataclasses import dataclass
from typing import Generic, TypeVar
import torch
from vllm.distributed.ec_transfer.ec_connector.mooncake.transfer import (
MooncakeTransfer,
)
from vllm.logger import init_logger
logger = init_logger(__name__)
_T = TypeVar("_T")
@dataclass
class MemoryAllocation:
"""Describe a tensor view carved from the Consumer receive slab."""
offset: int
size: int
tensor: torch.Tensor
@dataclass
class _ResidentEntry(Generic[_T]):
"""Store one resident value and its ownership accounting."""
value: _T
pinned: bool = True
leases: int = 0
@dataclass
class ResidentLease(Generic[_T]):
"""Represent a borrow of one resident entry."""
key: str
_entry: _ResidentEntry[_T]
_active: bool = True
@property
def value(self) -> _T:
return self._entry.value
class ContiguousAllocator:
"""Own a registered slab and its aligned regions; callers serialize access."""
def __init__(self, capacity: int, alignment: int = 256):
self.alignment = alignment
self._capacity = capacity
self._free = [(0, capacity)]
self.tensor: torch.Tensor | None = None
self._disabled = False
def prepare(self, device: torch.device, transfer: MooncakeTransfer) -> None:
if self.tensor is not None or self._disabled:
return
try:
tensor = torch.empty(self._capacity, dtype=torch.uint8, device=device)
ret = transfer.register_memory(tensor)
if ret != 0:
raise RuntimeError(f"Mooncake returned {ret}")
except (RuntimeError, torch.OutOfMemoryError) as error:
self._disabled = True
logger.warning("Could not initialize the EC registered buffer: %s", error)
return
self.tensor = tensor
self._free = [(0, tensor.nbytes)]
logger.info("Registered %d-byte buffer for Mooncake EC", tensor.nbytes)
def view(
self, offset: int, nbytes: int, shape: tuple[int, ...], dtype: torch.dtype
) -> torch.Tensor:
assert self.tensor is not None
return self.tensor.narrow(0, offset, nbytes).view(dtype).view(shape)
def close(self, transfer: MooncakeTransfer) -> bool:
if self.tensor is not None and not transfer.unregister_memory(self.tensor):
return False
self.tensor = None
self._free.clear()
return True
def allocate(self, nbytes: int) -> tuple[int, int] | None:
size = (nbytes + self.alignment - 1) // self.alignment * self.alignment
for index, (offset, available) in enumerate(self._free):
if size > available:
continue
if size == available:
self._free.pop(index)
else:
self._free[index] = (offset + size, available - size)
return offset, size
return None
def free(self, offset: int, size: int) -> None:
index = bisect.bisect_left(self._free, (offset, size))
self._free.insert(index, (offset, size))
if index + 1 < len(self._free):
next_offset, next_size = self._free[index + 1]
if offset + size == next_offset:
self._free[index] = (offset, size + next_size)
self._free.pop(index + 1)
if index > 0:
previous_offset, previous_size = self._free[index - 1]
current_offset, current_size = self._free[index]
if previous_offset + previous_size == current_offset:
self._free[index - 1] = (
previous_offset,
previous_size + current_size,
)
self._free.pop(index)
class ResidentPool(Generic[_T]):
"""Track resident values, active leases, and LRU eviction eligibility."""
def __init__(self):
self._entries: dict[str, _ResidentEntry[_T]] = {}
self._evictable: OrderedDict[str, None] = OrderedDict()
def referenced(self) -> list[str]:
return [key for key, entry in self._entries.items() if entry.pinned]
def get(self, key: str) -> _T | None:
entry = self._entries.get(key)
return entry.value if entry is not None else None
def insert(
self,
key: str,
value: _T,
) -> _T | None:
"""Pin an entry and return a displaced value that has no owners."""
previous = self._entries.get(key)
entry = _ResidentEntry(value)
self._entries[key] = entry
self._evictable.pop(key, None)
if previous is not None and previous.leases == 0:
return previous.value
return None
def pin(self, key: str) -> _T | None:
entry = self._entries.get(key)
if entry is None:
return None
self._evictable.pop(key, None)
entry.pinned = True
return entry.value
def acquire(self, key: str) -> ResidentLease[_T] | None:
entry = self._entries.get(key)
if entry is None:
return None
entry.leases += 1
self._evictable.pop(key, None)
return ResidentLease(key, entry)
def release(self, lease: ResidentLease[_T]) -> _T | None:
if not lease._active:
return None
lease._active = False
entry = lease._entry
entry.leases -= 1
current = self._entries.get(lease.key)
if current is not entry:
if entry.leases == 0:
return entry.value
return None
if not entry.pinned and entry.leases == 0:
self._evictable[lease.key] = None
return None
def consume(self, lease: ResidentLease[_T]) -> tuple[_T, _T | None]:
current = self._entries[lease.key]
current.pinned = True
self._evictable.pop(lease.key, None)
released = self.release(lease)
return current.value, released
def retire(self, key: str) -> None:
if key not in self._entries:
return
entry = self._entries[key]
entry.pinned = False
if entry.leases == 0:
self._evictable[key] = None
def evict_lru(self, evict: Callable[[str, _T], bool]) -> str | None:
for key in list(self._evictable):
entry = self._entries[key]
if not evict(key, entry.value):
continue
self._evictable.pop(key, None)
del self._entries[key]
return key
return None
def clear(self) -> None:
self._entries.clear()
self._evictable.clear()
@dataclass
class StagedSources:
"""Own Producer tensor views and their staging-slab regions."""
tensors: list[torch.Tensor]
regions: list[tuple[int, int]]
class ProducerMemoryPool:
"""Own the Producer staging slab and regions carved from it."""
def __init__(self, capacity: int, transfer: MooncakeTransfer) -> None:
self._transfer = transfer
self._allocator = ContiguousAllocator(capacity)
self._lock = threading.Lock()
self._local = threading.local()
@property
def tensor(self) -> torch.Tensor | None:
return self._allocator.tensor
def _free_regions(self, regions: list[tuple[int, int]]) -> None:
for offset, size in regions:
self._allocator.free(offset, size)
def stage(self, tensors: list[torch.Tensor]) -> StagedSources | None:
"""Copy tensors into one registered slab, or return None for fallback."""
if not tensors:
return StagedSources([], [])
allocator = self._allocator
staged: list[torch.Tensor] = []
regions: list[tuple[int, int]] = []
with self._lock:
allocator.prepare(tensors[0].device, self._transfer)
pool = allocator.tensor
if pool is None:
return None
for tensor in tensors:
region = allocator.allocate(tensor.nbytes)
if region is None:
self._free_regions(regions)
return None
regions.append(region)
staged.append(
allocator.view(
region[0], tensor.nbytes, tuple(tensor.shape), tensor.dtype
)
)
if pool.device.type == "cuda":
stream = getattr(self._local, "stream", None)
if stream is None:
stream = torch.cuda.Stream(device=pool.device)
self._local.stream = stream
with torch.cuda.stream(stream):
for destination, source in zip(staged, tensors):
destination.copy_(source, non_blocking=True)
stream.synchronize()
else:
for destination, source in zip(staged, tensors):
destination.copy_(source)
return StagedSources(staged, regions)
def release(self, staged: StagedSources) -> None:
if not staged.regions:
return
with self._lock:
self._free_regions(staged.regions)
def close(self) -> None:
with self._lock:
self._allocator.close(self._transfer)
class ConsumerMemoryPool:
"""Own the registered receive slab and resident allocation lifecycle."""
def __init__(
self,
capacity: int,
transfer: MooncakeTransfer,
) -> None:
self._transfer = transfer
self._allocator = ContiguousAllocator(capacity)
self._residents: ResidentPool[MemoryAllocation] = ResidentPool()
self._retire_events: dict[str, torch.Event] = {}
self._pending_frees: list[tuple[torch.Event, MemoryAllocation]] = []
self._reclaimed: set[str] = set()
self.lock = threading.RLock()
@property
def tensor(self) -> torch.Tensor | None:
return self._allocator.tensor
def prepare(
self,
device: torch.device,
) -> None:
with self.lock:
self._allocator.prepare(device, self._transfer)
def _free(self, allocation: MemoryAllocation) -> None:
self._allocator.free(allocation.offset, allocation.size)
def free(self, allocation: MemoryAllocation) -> None:
with self.lock:
self._free(allocation)
def _defer_or_free(
self, allocation: MemoryAllocation, event: torch.Event | None
) -> None:
if event is None or event.query():
self._free(allocation)
else:
self._pending_frees.append((event, allocation))
def _poll_frees_locked(self) -> None:
pending = []
for event, allocation in self._pending_frees:
if event.query():
self._free(allocation)
else:
pending.append((event, allocation))
self._pending_frees = pending
def _reclaim_locked(self, nbytes: int) -> tuple[int, int] | None:
def evict(mm_hash: str, allocation: MemoryAllocation) -> bool:
event = self._retire_events.pop(mm_hash, None)
self._defer_or_free(allocation, event)
self._reclaimed.add(mm_hash)
return True
while self._residents.evict_lru(evict) is not None:
# Eviction defers any free whose CUDA event is still pending, so
# those bytes reach the allocator only once the event is polled.
self._poll_frees_locked()
region = self._allocator.allocate(nbytes)
if region is not None:
return region
self._poll_frees_locked()
return self._allocator.allocate(nbytes)
def _make_allocation(
self,
region: tuple[int, int],
nbytes: int,
shape: tuple[int, ...],
dtype: torch.dtype,
) -> MemoryAllocation:
offset, size = region
tensor = self._allocator.view(offset, nbytes, shape, dtype)
return MemoryAllocation(offset, size, tensor)
def try_allocate(
self, nbytes: int, shape: tuple[int, ...], dtype: torch.dtype
) -> MemoryAllocation | None:
with self.lock:
allocator = self._allocator
assert allocator.tensor is not None
self._poll_frees_locked()
region = allocator.allocate(nbytes)
if region is None:
return None
return self._make_allocation(region, nbytes, shape, dtype)
def reclaim_and_allocate(
self, nbytes: int, shape: tuple[int, ...], dtype: torch.dtype
) -> MemoryAllocation | None:
with self.lock:
region = self._reclaim_locked(nbytes)
if region is None:
return None
return self._make_allocation(region, nbytes, shape, dtype)
def acquire_cached(
self, mm_hash: str, shape: tuple[int, ...], dtype: torch.dtype
) -> ResidentLease[MemoryAllocation] | None:
with self.lock:
allocation = self._residents.get(mm_hash)
if allocation is None:
return None
if (
tuple(allocation.tensor.shape) != shape
or allocation.tensor.dtype != dtype
):
raise ValueError("conflicting cached tensor for mm_hash")
return self._residents.acquire(mm_hash)
def take_resident(
self, mm_hash: str, shape: tuple[int, ...], dtype_name: str
) -> torch.Tensor | None:
with self.lock:
allocation = self._residents.get(mm_hash)
if allocation is None:
return None
tensor = allocation.tensor
if (
tuple(tensor.shape) != shape
or str(tensor.dtype).split(".")[-1] != dtype_name
):
return None
self._residents.pin(mm_hash)
self._retire_events.pop(mm_hash, None)
return tensor
def _record_release_event(self) -> torch.Event | None:
pool = self.tensor
if pool is None or pool.device.type != "cuda":
return None
event = torch.Event()
event.record(torch.accelerator.current_stream(pool.device))
return event
def release_cached(self, lease: ResidentLease[MemoryAllocation]) -> None:
with self.lock:
released = self._residents.release(lease)
if released is not None:
self._defer_or_free(released, self._record_release_event())
def publish(
self,
mm_hash: str,
allocation: MemoryAllocation,
lease: ResidentLease[MemoryAllocation] | None = None,
*,
pin: bool = True,
) -> MemoryAllocation:
with self.lock:
if lease is not None:
canonical, released = self._residents.consume(lease)
self._retire_events.pop(mm_hash, None)
if released is not None:
self._defer_or_free(released, self._record_release_event())
return canonical
previous = self._residents.get(mm_hash)
displaced = self._residents.insert(mm_hash, allocation)
if not pin:
self._residents.retire(mm_hash)
event = None
if previous is not None and previous is not allocation:
event = self._retire_events.pop(mm_hash, None)
if displaced is None or displaced is allocation:
return allocation
if event is None:
event = self._record_release_event()
self._defer_or_free(displaced, event)
return allocation
def retire_stale(
self,
encoder_cache: dict[str, torch.Tensor],
reserved_hashes: set[str] | None = None,
*,
freed: list[str] | None = None,
) -> None:
if self.tensor is None:
return
with self.lock:
for mm_hash in self._residents.referenced() if freed is None else freed:
allocation = self._residents.get(mm_hash)
if allocation is None:
continue
if encoder_cache.get(mm_hash) is allocation.tensor:
continue
if reserved_hashes and mm_hash in reserved_hashes:
continue
event = self._record_release_event()
if event is not None:
self._retire_events[mm_hash] = event
self._residents.retire(mm_hash)
self._poll_frees_locked()
def drain_reclaimed(self) -> set[str]:
with self.lock:
reclaimed = self._reclaimed
self._reclaimed = set()
return reclaimed
def close(self) -> None:
with self.lock:
if not self._allocator.close(self._transfer):
return
self._residents.clear()
self._retire_events.clear()
self._pending_frees.clear()
@@ -0,0 +1,75 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Metadata exchanged by the Mooncake encoder-cache connector."""
from __future__ import annotations
from dataclasses import dataclass, field
from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorMetadata,
ECConnectorWorkerMetadata,
)
@dataclass
class ECMooncakeLoadSpec:
"""Describe a remote reservation or resident allocation to load."""
mm_hash: str
nbytes: int
shape: tuple[int, ...]
dtype: str
transfer_id: str
# The consumer pool still holds this item, so the load is a local handoff:
# no transfer, no producer.
local: bool = False
@dataclass
class ECMooncakePushSpec:
"""Describe a destination reservation prepared before a tensor is ready."""
mm_hash: str
nbytes: int
shape: tuple[int, ...]
dtype: str
consumer_zmq: str
transfer_id: str
request_id: str = ""
@dataclass
class ECMooncakeConnectorMetadata(ECConnectorMetadata):
"""Worker operations emitted for one Scheduler step."""
loads: list[ECMooncakeLoadSpec] = field(default_factory=list)
pushes: list[ECMooncakePushSpec] = field(default_factory=list)
freed: list[str] | None = None
@dataclass
class ECMooncakeWorkerMetadata(ECConnectorWorkerMetadata):
"""Completion state reported from Workers to the Scheduler."""
loaded: set[str] = field(default_factory=set)
failed_loads: set[str] = field(default_factory=set)
# Items the receive pool dropped under pressure. The scheduler assumes an
# evicted item stays resident until told otherwise.
reclaimed: set[str] = field(default_factory=set)
pending_saves: bool = False
failed_saves: set[str] = field(default_factory=set)
def aggregate(self, other: ECConnectorWorkerMetadata) -> ECMooncakeWorkerMetadata:
assert isinstance(other, ECMooncakeWorkerMetadata)
return ECMooncakeWorkerMetadata(
# Every tensor-parallel rank gathers the embedding from its own
# cache, so an item counts as loaded only where all of them have
# it; one rank falling short must fail the load rather than leave
# the scheduler believing it is ready.
loaded=self.loaded & other.loaded,
failed_loads=self.failed_loads | other.failed_loads,
reclaimed=self.reclaimed | other.reclaimed,
pending_saves=self.pending_saves or other.pending_saves,
failed_saves=self.failed_saves | other.failed_saves,
)
@@ -0,0 +1,418 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Producer-side push lifecycle, futures, and source-tensor ownership.
This module records when a push may advance or release its source. Worker
orchestration performs the actual control exchanges and Mooncake writes.
"""
from __future__ import annotations
import threading
from collections import OrderedDict
from collections.abc import Callable
from concurrent.futures import Future, ThreadPoolExecutor
from contextlib import suppress
from dataclasses import dataclass, field
from enum import Enum, auto
from typing import Any
import torch
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
ECMooncakePushSpec,
)
def _same_destination(
existing: ECMooncakePushSpec, incoming: ECMooncakePushSpec
) -> bool:
"""Whether two specs describe one push, ignoring which request asked.
The same encoding requested twice shares a transfer id legitimately; only
a different payload or destination is a collision.
"""
return (
existing.mm_hash == incoming.mm_hash
and existing.nbytes == incoming.nbytes
and existing.shape == incoming.shape
and existing.dtype == incoming.dtype
and existing.consumer_zmq == incoming.consumer_zmq
)
class ProducerPushState(Enum):
"""Lifecycle of one Producer push from reservation to terminal state."""
# The reservation reply and source tensor may arrive in either order.
WAITING_INPUTS = auto()
WRITING = auto()
NOTIFYING = auto()
DONE = auto()
CANCEL_PENDING = auto()
CANCELLED = auto()
FAILED = auto()
@dataclass
class ProducerPushRecord:
"""Collect all asynchronous state for one Producer push."""
spec: ECMooncakePushSpec
state: ProducerPushState
reservation_future: Future[list[dict[str, Any]]]
reservations: list[dict[str, Any]] = field(default_factory=list)
shard_futures: list[Future[Any]] = field(default_factory=list)
source_tensor: torch.Tensor | None = None
source_event: torch.Event | None = None
batch_future: Future[None] | None = None
error: str | None = None
_ALLOWED_TRANSITIONS = {
ProducerPushState.WAITING_INPUTS: {
ProducerPushState.WRITING,
ProducerPushState.CANCEL_PENDING,
ProducerPushState.FAILED,
},
ProducerPushState.WRITING: {
ProducerPushState.NOTIFYING,
ProducerPushState.FAILED,
},
ProducerPushState.NOTIFYING: {
ProducerPushState.DONE,
ProducerPushState.FAILED,
},
ProducerPushState.CANCEL_PENDING: {
ProducerPushState.CANCELLED,
ProducerPushState.FAILED,
},
ProducerPushState.DONE: set(),
ProducerPushState.CANCELLED: set(),
ProducerPushState.FAILED: set(),
}
_TERMINAL_STATES = {
ProducerPushState.DONE,
ProducerPushState.CANCELLED,
ProducerPushState.FAILED,
}
_TERMINAL_LIMIT = 1 << 16
class ProducerPushManager:
"""Own Producer push records, transitions, and source tensor leases.
Terminal records are retained for duplicate-metadata idempotency. Separate
ordered indexes keep hot polling paths from scanning those tombstones.
"""
def __init__(self, wake: Callable[[], None] = lambda: None) -> None:
self._records: OrderedDict[str, ProducerPushRecord] = OrderedDict()
self._active_ids: OrderedDict[str, None] = OrderedDict()
self._reapable_terminal_ids: OrderedDict[str, None] = OrderedDict()
self._unreported_ids: OrderedDict[str, None] = OrderedDict()
self._batch_ids: OrderedDict[str, None] = OrderedDict()
self._source_waiters: dict[str, OrderedDict[str, None]] = {}
self._lock = threading.RLock()
self._wake = wake
def reserve(
self,
spec: ECMooncakePushSpec,
submit: Callable[[], Future[list[dict[str, Any]]]],
) -> tuple[ProducerPushRecord, bool]:
with self._lock:
existing = self._records.get(spec.transfer_id)
if existing is not None:
if not _same_destination(existing.spec, spec):
raise ValueError(
f"Conflicting EC destination for transfer_id={spec.transfer_id}"
)
return existing, False
record = ProducerPushRecord(
spec=spec,
state=ProducerPushState.WAITING_INPUTS,
reservation_future=submit(),
)
self._records[spec.transfer_id] = record
self._active_ids[spec.transfer_id] = None
self._source_waiters.setdefault(spec.mm_hash, OrderedDict())[
spec.transfer_id
] = None
record.reservation_future.add_done_callback(
lambda future: self._reservation_done(record, future)
)
return record, True
def bind_source(
self,
mm_hash: str,
tensor: torch.Tensor,
ready_event: torch.Event | None,
) -> None:
with self._lock:
waiters = self._source_waiters.pop(mm_hash, OrderedDict())
for transfer_id in waiters:
record = self._records[transfer_id]
if (
record.source_tensor is not None
or record.state is not ProducerPushState.WAITING_INPUTS
):
continue
record.source_tensor = tensor
record.source_event = ready_event
self._wake()
def submit_batches(
self,
executor: ThreadPoolExecutor,
run_batch: Callable[[list[ProducerPushRecord]], None],
*,
wait: bool = False,
) -> bool:
pending_event = False
with self._lock:
grouped: dict[str, list[ProducerPushRecord]] = {}
for transfer_id in list(self._active_ids):
record = self._records[transfer_id]
if (
record.source_tensor is not None
and record.batch_future is None
and record.state is ProducerPushState.WAITING_INPUTS
):
if not record.reservation_future.done():
continue
if record.source_event is not None:
if wait:
record.source_event.synchronize()
elif not record.source_event.query():
pending_event = True
continue
grouped.setdefault(record.spec.consumer_zmq, []).append(record)
batches = list(grouped.values())
for records in batches:
future = executor.submit(run_batch, records)
for record in records:
record.batch_future = future
self._batch_ids[record.spec.transfer_id] = None
return pending_event
def resolve_reservations(self, record: ProducerPushRecord) -> list[dict[str, Any]]:
results = record.reservation_future.result()
with self._lock:
record.reservations = results
return list(results)
def finish_reservations(
self,
outcomes: list[tuple[ProducerPushRecord, list[dict[str, Any]] | Exception]],
) -> None:
with self._lock:
for record, result in outcomes:
if isinstance(result, Exception):
record.reservation_future.set_exception(result)
else:
record.reservation_future.set_result(result)
def _reservation_done(
self,
record: ProducerPushRecord,
future: Future[list[dict[str, Any]]],
) -> None:
try:
future.result()
except Exception as exc:
with self._lock:
self._set_error(record, exc)
if (
record.state is ProducerPushState.WAITING_INPUTS
and record.source_tensor is None
):
self._transition(record, ProducerPushState.FAILED)
finally:
self._wake()
def settle_all(self, records: list[ProducerPushRecord]) -> None:
for record in records:
with suppress(Exception):
record.reservation_future.result()
def replace_reservations(
self,
record: ProducerPushRecord,
reservations: list[dict[str, Any]],
) -> None:
with self._lock:
record.reservations = reservations
def track_shard_futures(
self,
records: list[ProducerPushRecord],
futures: list[Future[Any]],
) -> None:
with self._lock:
for record in records:
record.shard_futures.extend(futures)
def begin_writing(self, record: ProducerPushRecord) -> None:
with self._lock:
if record.source_tensor is None:
raise RuntimeError(
f"Producer push {record.spec.transfer_id!r} has no source tensor"
)
self._transition(record, ProducerPushState.WRITING)
def begin_notifying(self, records: list[ProducerPushRecord]) -> None:
with self._lock:
for record in records:
self._transition(record, ProducerPushState.NOTIFYING)
def complete(self, records: list[ProducerPushRecord]) -> None:
with self._lock:
self._check_sources_releasable(records)
for record in records:
self._transition(record, ProducerPushState.DONE)
self._release_source(record)
def fail(self, records: list[ProducerPushRecord], error: Exception) -> None:
with self._lock:
self._check_sources_releasable(records)
for record in records:
if record.state not in _TERMINAL_STATES:
self._set_error(record, error)
self._transition(record, ProducerPushState.FAILED)
self._release_source(record)
def cancel_requests(self, request_ids: set[str] | None) -> list[ProducerPushRecord]:
"""Cancel source-less waiters for requests, or all waiters at shutdown."""
cancelled = []
with self._lock:
for transfer_id in list(self._active_ids):
record = self._records[transfer_id]
if (
(
request_ids is not None
and record.spec.request_id not in request_ids
)
or record.source_tensor is not None
or record.batch_future is not None
):
continue
if record.state is not ProducerPushState.WAITING_INPUTS:
continue
self._transition(record, ProducerPushState.CANCEL_PENDING)
cancelled.append(record)
return cancelled
def finish_cancel(self, record: ProducerPushRecord) -> None:
with self._lock:
if record.state is ProducerPushState.CANCEL_PENDING:
self._transition(record, ProducerPushState.CANCELLED)
def submit_cancel(
self,
record: ProducerPushRecord,
executor: ThreadPoolExecutor,
run_cancel: Callable[[ProducerPushRecord], None],
) -> None:
with self._lock:
future = executor.submit(run_cancel, record)
record.batch_future = future
transfer_id = record.spec.transfer_id
self._batch_ids[transfer_id] = None
if record.state in _TERMINAL_STATES:
self._reapable_terminal_ids.pop(transfer_id, None)
def poll(self) -> list[tuple[str, str]]:
failures = []
with self._lock:
for transfer_id in list(self._batch_ids):
record = self._records[transfer_id]
future = record.batch_future
assert future is not None
if not future.done():
continue
try:
future.result()
except Exception as exc:
self._set_error(record, exc)
self._batch_ids.pop(transfer_id)
self._mark_reapable(record)
for transfer_id in list(self._unreported_ids):
if transfer_id in self._batch_ids:
continue
record = self._records[transfer_id]
assert record.error is not None
failures.append((record.spec.mm_hash, record.error))
self._unreported_ids.pop(transfer_id)
self._mark_reapable(record)
self._reap_terminals()
return failures
@property
def pending(self) -> bool:
with self._lock:
return bool(self._active_ids or self._batch_ids)
def _release_source(self, record: ProducerPushRecord) -> None:
record.source_tensor = None
record.source_event = None
def _set_error(self, record: ProducerPushRecord, error: BaseException) -> None:
if record.error is None:
record.error = str(error)
if record.state in _TERMINAL_STATES:
transfer_id = record.spec.transfer_id
self._unreported_ids[transfer_id] = None
self._reapable_terminal_ids.pop(transfer_id, None)
def _reap_terminals(self) -> None:
while len(self._reapable_terminal_ids) > _TERMINAL_LIMIT:
transfer_id, _ = self._reapable_terminal_ids.popitem(last=False)
self._records.pop(transfer_id)
def _mark_reapable(self, record: ProducerPushRecord) -> None:
transfer_id = record.spec.transfer_id
if (
record.state in _TERMINAL_STATES
and transfer_id not in self._batch_ids
and transfer_id not in self._unreported_ids
):
self._reapable_terminal_ids[transfer_id] = None
def _drop_source_waiter(self, record: ProducerPushRecord) -> None:
waiters = self._source_waiters.get(record.spec.mm_hash)
if waiters is None:
return
waiters.pop(record.spec.transfer_id, None)
if not waiters:
self._source_waiters.pop(record.spec.mm_hash)
@staticmethod
def _check_sources_releasable(records: list[ProducerPushRecord]) -> None:
for record in records:
futures = [record.reservation_future, *record.shard_futures]
if record.source_tensor is not None and not all(
future.done() for future in futures
):
raise RuntimeError(
f"Producer push {record.spec.transfer_id!r} released its "
"source too early"
)
def _transition(self, record: ProducerPushRecord, state: ProducerPushState) -> None:
if state not in _ALLOWED_TRANSITIONS[record.state]:
raise RuntimeError(
f"Cannot transition producer push {record.spec.transfer_id!r} from "
f"{record.state.name} to {state.name}"
)
record.state = state
if state is not ProducerPushState.WAITING_INPUTS:
self._drop_source_waiter(record)
if state in _TERMINAL_STATES:
transfer_id = record.spec.transfer_id
self._active_ids.pop(transfer_id, None)
if record.error is not None:
self._unreported_ids[transfer_id] = None
self._mark_reapable(record)
@@ -0,0 +1,487 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Consumer-side reservation lifecycle and destination-memory ownership.
Reservations make remote writes idempotent and ensure cancellation or expiry
cannot free a destination while Mooncake may still be writing into it.
"""
from __future__ import annotations
import threading
import time
import uuid
from collections import OrderedDict
from dataclasses import dataclass
from enum import Enum, auto
import torch
from vllm.distributed.ec_transfer.ec_connector.mooncake.memory import (
ConsumerMemoryPool,
MemoryAllocation,
ResidentLease,
)
class ConsumerReservationState(Enum):
"""Lifecycle of one Consumer destination reservation."""
WRITING = auto()
READY = auto()
CANCEL_PENDING = auto()
EXPIRE_PENDING = auto()
CANCELLED = auto()
EXPIRED = auto()
_ALLOWED_TRANSITIONS = {
ConsumerReservationState.WRITING: {
ConsumerReservationState.READY,
ConsumerReservationState.CANCEL_PENDING,
ConsumerReservationState.EXPIRE_PENDING,
ConsumerReservationState.CANCELLED,
},
ConsumerReservationState.READY: {
ConsumerReservationState.CANCELLED,
ConsumerReservationState.EXPIRED,
},
ConsumerReservationState.CANCEL_PENDING: {ConsumerReservationState.CANCELLED},
ConsumerReservationState.EXPIRE_PENDING: {
ConsumerReservationState.CANCELLED,
ConsumerReservationState.EXPIRED,
},
ConsumerReservationState.CANCELLED: set(),
ConsumerReservationState.EXPIRED: set(),
}
_ACTIVE_STATES = {
ConsumerReservationState.WRITING,
ConsumerReservationState.READY,
ConsumerReservationState.CANCEL_PENDING,
ConsumerReservationState.EXPIRE_PENDING,
}
_DEFERRED_STATES = {
ConsumerReservationState.CANCEL_PENDING,
ConsumerReservationState.EXPIRE_PENDING,
}
_WRITER_OWNED_STATES = {
ConsumerReservationState.WRITING,
ConsumerReservationState.CANCEL_PENDING,
ConsumerReservationState.EXPIRE_PENDING,
}
@dataclass
class ConsumerReservation:
"""Own the identity and destination allocation of one remote push."""
transfer_id: str
mm_hash: str
reservation_id: str
state: ConsumerReservationState
shape: tuple[int, ...] = ()
dtype: str = ""
allocation: MemoryAllocation | None = None
lease: ResidentLease[MemoryAllocation] | None = None
expires_at: float = 0
writer_id: str = ""
class ConsumerReservationManager:
"""Own reservation transitions and destination allocation releases.
A remote writer owns a ``WRITING`` allocation. Cancellation and expiry are
therefore deferred until completion, unless refresh explicitly abandons
that writer before replacing its reservation.
"""
def __init__(
self,
memory: ConsumerMemoryPool,
lease_ttl: float,
tombstone_limit: int,
) -> None:
self._memory = memory
self._lease_ttl = lease_ttl
self._tombstone_limit = tombstone_limit
self._records: dict[str, ConsumerReservation] = {}
self._active_ids: dict[str, None] = {}
self._tombstones: OrderedDict[str, None] = OrderedDict()
self._writers: dict[str, ConsumerReservation] = {}
self._followers: dict[str, dict[str, None]] = {}
self._ready: list[str] = []
self._condition = threading.Condition(memory.lock)
self._shutting_down = False
def reserve(
self,
transfer_id: str,
mm_hash: str,
nbytes: int,
shape: tuple[int, ...],
dtype_name: str,
dtype: torch.dtype,
) -> tuple[ConsumerReservation | None, bool]:
with self._condition:
if self._shutting_down:
raise RuntimeError("Consumer reservation manager is shutting down")
existing = self._records.get(transfer_id)
if (
existing is not None
and existing.state is ConsumerReservationState.CANCELLED
):
return existing, False
if (
existing is not None
and existing.state is ConsumerReservationState.EXPIRED
):
self._remove(transfer_id)
existing = None
if (
existing is not None
and existing.state is ConsumerReservationState.EXPIRE_PENDING
):
raise RuntimeError(
f"Transfer {transfer_id!r} still has an active writer"
)
if existing is not None:
if (
existing.mm_hash != mm_hash
or existing.shape != shape
or existing.dtype != dtype_name
):
raise ValueError("conflicting reservation for transfer_id")
if existing.state not in _ACTIVE_STATES:
raise RuntimeError(
f"Cannot reserve transfer {transfer_id!r} in "
f"{existing.state.name}"
)
if existing.state is ConsumerReservationState.WRITING:
existing.expires_at = time.monotonic() + self._lease_ttl
return existing, False
lease = self._memory.acquire_cached(mm_hash, shape, dtype)
now = time.monotonic()
if lease is not None:
record = ConsumerReservation(
transfer_id,
mm_hash,
uuid.uuid4().hex,
ConsumerReservationState.READY,
shape,
dtype_name,
lease.value,
lease,
now + self._lease_ttl,
)
self._insert(record)
return record, False
writer = self._writers.get(mm_hash)
if writer is not None and writer.state is ConsumerReservationState.WRITING:
if writer.shape != shape or writer.dtype != dtype_name:
raise ValueError("conflicting in-flight tensor for mm_hash")
record = ConsumerReservation(
transfer_id,
mm_hash,
uuid.uuid4().hex,
ConsumerReservationState.WRITING,
shape,
dtype_name,
writer.allocation,
expires_at=now + self._lease_ttl,
writer_id=writer.transfer_id,
)
self._insert(record)
self._followers.setdefault(writer.transfer_id, {})[transfer_id] = None
return record, False
allocation = self._memory.try_allocate(nbytes, shape, dtype)
if allocation is None:
self._expire_locked(time.monotonic())
allocation = self._memory.try_allocate(nbytes, shape, dtype)
if allocation is None:
allocation = self._memory.reclaim_and_allocate(nbytes, shape, dtype)
if allocation is None:
return None, False
record = ConsumerReservation(
transfer_id,
mm_hash,
uuid.uuid4().hex,
ConsumerReservationState.WRITING,
shape,
dtype_name,
allocation,
None,
now + self._lease_ttl,
)
self._insert(record)
self._writers[mm_hash] = record
return record, True
def status(self, transfer_id: str) -> ConsumerReservation | None:
with self._memory.lock:
record = self._records.get(transfer_id)
if record is None or record.state not in _ACTIVE_STATES:
return None
return record
def complete(self, transfer_id: str, reservation_id: str) -> tuple[bool, bool]:
with self._condition:
try:
record = self._records.get(transfer_id)
if record is None or record.reservation_id != reservation_id:
return False, False
if record.state is ConsumerReservationState.READY:
return True, False
if record.writer_id:
return False, False
if record.state in _WRITER_OWNED_STATES:
self._publish_followers(record)
if record.state in _DEFERRED_STATES:
terminal = (
ConsumerReservationState.CANCELLED
if record.state is ConsumerReservationState.CANCEL_PENDING
else ConsumerReservationState.EXPIRED
)
self._terminate(record, terminal)
return True, False
if record.state is not ConsumerReservationState.WRITING:
return False, False
self._transition(record, ConsumerReservationState.READY)
record.expires_at = time.monotonic() + self._lease_ttl
return True, True
finally:
self._condition.notify_all()
def _publish_followers(self, writer: ConsumerReservation) -> None:
if self._writers.get(writer.mm_hash) is writer:
self._writers.pop(writer.mm_hash)
followers = self._followers.pop(writer.transfer_id, {})
if not followers:
return
assert writer.allocation is not None
allocation = self._memory.publish(writer.mm_hash, writer.allocation, pin=False)
for record in [writer, *(self._records[key] for key in followers)]:
record.allocation = allocation
record.lease = self._memory.acquire_cached(
record.mm_hash, record.shape, allocation.tensor.dtype
)
assert record.lease is not None
if record is not writer:
record.writer_id = ""
self._transition(record, ConsumerReservationState.READY)
record.expires_at = time.monotonic() + self._lease_ttl
self._ready.append(record.transfer_id)
def drain_ready(self) -> list[str]:
with self._memory.lock:
ready, self._ready = self._ready, []
return ready
def begin_shutdown(self) -> None:
"""Stop new reservations and cancel everything without a remote writer."""
with self._condition:
if self._shutting_down:
return
self._shutting_down = True
for transfer_id in list(self._active_ids):
record = self._records[transfer_id]
if record.writer_id or record.state is ConsumerReservationState.READY:
self._terminate(record, ConsumerReservationState.CANCELLED)
elif record.state is ConsumerReservationState.WRITING:
self._defer(record, ConsumerReservationState.CANCEL_PENDING)
self._condition.notify_all()
def wait_for_writers(self, timeout: float) -> bool:
"""Wait until no reservation is still owned by a remote writer."""
def writers_finished() -> bool:
return not any(
self._records[transfer_id].state in _WRITER_OWNED_STATES
for transfer_id in self._active_ids
)
with self._condition:
return self._condition.wait_for(writers_finished, timeout=max(0, timeout))
def cancel(
self,
transfer_id: str,
reservation_id: str,
abandon: bool = False,
refresh: bool = False,
) -> bool:
with self._memory.lock:
record = self._records.get(transfer_id)
if (
record is not None
and reservation_id
and record.reservation_id != reservation_id
):
return False
if record is None:
record = ConsumerReservation(
transfer_id,
"",
"",
ConsumerReservationState.CANCELLED,
)
self._insert(record)
self._set_tombstone_deadline(record)
self._reap_tombstones(time.monotonic())
return True
if record.state is ConsumerReservationState.CANCELLED:
self._set_tombstone_deadline(record)
self._reap_tombstones(time.monotonic())
return True
if refresh:
if not abandon or record.state not in {
ConsumerReservationState.WRITING,
ConsumerReservationState.EXPIRE_PENDING,
}:
return False
if record.state is ConsumerReservationState.WRITING:
self._transition(record, ConsumerReservationState.EXPIRE_PENDING)
self._terminate(record, ConsumerReservationState.EXPIRED)
self._condition.notify_all()
self._reap_tombstones(time.monotonic())
return True
if record.writer_id:
self._terminate(record, ConsumerReservationState.CANCELLED)
return True
if record.state in _DEFERRED_STATES and not abandon:
return True
if record.state is ConsumerReservationState.WRITING and not abandon:
self._defer(record, ConsumerReservationState.CANCEL_PENDING)
return True
if record.state not in _ACTIVE_STATES:
return False
self._terminate(record, ConsumerReservationState.CANCELLED)
if self._shutting_down:
self._condition.notify_all()
self._reap_tombstones(time.monotonic())
return True
def take(self, transfer_id: str, mm_hash: str) -> MemoryAllocation:
with self._memory.lock:
record = self._records.get(transfer_id)
if (
record is None
or record.state is not ConsumerReservationState.READY
or record.mm_hash != mm_hash
or record.allocation is None
):
raise RuntimeError(
f"Pushed EC tensor is not ready for mm_hash={mm_hash}"
)
allocation = self._memory.publish(mm_hash, record.allocation, record.lease)
record.allocation = None
record.lease = None
self._remove(transfer_id)
return allocation
def expire(self) -> int:
with self._memory.lock:
return self._expire_locked(time.monotonic())
def retire_stale(
self, encoder_cache: dict[str, torch.Tensor], freed: list[str] | None = None
) -> None:
with self._memory.lock:
self._memory.retire_stale(encoder_cache, freed=freed)
def _expire_locked(self, now: float) -> int:
expired = 0
for transfer_id in list(self._active_ids):
record = self._records[transfer_id]
if record.expires_at > now:
continue
if record.state is ConsumerReservationState.READY:
self._terminate(record, ConsumerReservationState.EXPIRED)
expired += 1
elif record.state is ConsumerReservationState.WRITING:
if record.writer_id:
self._terminate(record, ConsumerReservationState.CANCELLED)
else:
self._defer(record, ConsumerReservationState.EXPIRE_PENDING)
self._reap_tombstones(now)
return expired
def _defer(
self, record: ConsumerReservation, state: ConsumerReservationState
) -> None:
"""Keep the destination until the writer completes or abandons it."""
self._transition(record, state)
record.expires_at = float("inf")
def _terminate(
self, record: ConsumerReservation, state: ConsumerReservationState
) -> None:
self._transition(record, state)
self._release(record)
self._set_tombstone_deadline(record)
def _release(self, record: ConsumerReservation) -> None:
if record.writer_id:
followers = self._followers.get(record.writer_id, {})
followers.pop(record.transfer_id, None)
record.allocation = None
return
if self._writers.get(record.mm_hash) is record:
self._writers.pop(record.mm_hash)
for transfer_id in list(self._followers.pop(record.transfer_id, {})):
self._terminate(
self._records[transfer_id], ConsumerReservationState.CANCELLED
)
allocation = record.allocation
if allocation is None:
return
if record.lease is not None:
self._memory.release_cached(record.lease)
else:
self._memory.free(allocation)
record.allocation = None
record.lease = None
def _set_tombstone_deadline(self, record: ConsumerReservation) -> None:
record.expires_at = time.monotonic() + self._lease_ttl
self._tombstones[record.transfer_id] = None
self._tombstones.move_to_end(record.transfer_id)
def _reap_tombstones(self, now: float) -> int:
dropped = 0
while self._tombstones:
transfer_id = next(iter(self._tombstones))
record = self._records[transfer_id]
if (
record.expires_at > now
and len(self._tombstones) <= self._tombstone_limit
):
break
self._remove(transfer_id)
dropped += 1
return dropped
def _insert(self, record: ConsumerReservation) -> None:
self._records[record.transfer_id] = record
if record.state in _ACTIVE_STATES:
self._active_ids[record.transfer_id] = None
def _remove(self, transfer_id: str) -> None:
self._records.pop(transfer_id, None)
self._active_ids.pop(transfer_id, None)
self._tombstones.pop(transfer_id, None)
def _transition(
self, record: ConsumerReservation, state: ConsumerReservationState
) -> None:
if state not in _ALLOWED_TRANSITIONS[record.state]:
raise RuntimeError(
f"Cannot transition {record.transfer_id!r} from "
f"{record.state.name} to {state.name}"
)
record.state = state
if state in _ACTIVE_STATES:
self._active_ids[record.transfer_id] = None
else:
self._active_ids.pop(record.transfer_id, None)
@@ -0,0 +1,537 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Scheduler-side planning and observation for Mooncake cache transfers.
The Scheduler prepares Producer reservations, consumes Consumer readiness
events, emits per-step Worker metadata, and converts Worker results back into
request availability without owning tensor memory or running data transfers.
"""
from __future__ import annotations
import math
import time
from collections import OrderedDict
from concurrent.futures import Future, ThreadPoolExecutor
from typing import TYPE_CHECKING, Any
import torch
from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
_get_encoder_cache_hidden_dim,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.config import (
_RESERVATION_TTL_SECONDS,
MooncakeECConfig,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.control import (
ControlClient,
EventInbox,
make_cancel_request,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
ECMooncakeConnectorMetadata,
ECMooncakeLoadSpec,
ECMooncakePushSpec,
ECMooncakeWorkerMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.state import (
SchedulerTransferState,
SchedulerTransferTable,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.transfer import (
ensure_mooncake_available,
)
from vllm.distributed.ec_transfer.ec_connector.utils import (
PlaceholderMetadataResolver,
collect_ec_item_metadata,
)
from vllm.logger import init_logger
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import ECConnectorOutput
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
# Unmatched `ec_items` entries are a protocol desync, not a per-request
# event: warn on the first few, then only once in a while.
_MAX_UNRESOLVED_TRANSFER_ID_WARNINGS = 5
_DRAIN_MIN_INTERVAL = 0.005
_MAX_PENDING_EVENTS = 4096
_MAX_TERMINAL_TRANSFER_RECORDS = 1 << 16
_CANCEL_ATTEMPTS = 2
_CONTROL_WORKERS = 8
class ECMooncakeScheduler:
"""Coordinate Mooncake transfers from the vLLM Scheduler process."""
def __init__(self, vllm_config: VllmConfig) -> None:
ensure_mooncake_available()
config = MooncakeECConfig.from_vllm_config(vllm_config)
self._is_producer = config.is_producer
self._is_consumer = config.is_consumer
self._control_addr = config.control_addr
self._push_wait_timeout = config.push_wait_timeout_s
self._encoder_cache_hidden_dim = (
_get_encoder_cache_hidden_dim(vllm_config) if config.is_producer else None
)
self._model_config = vllm_config.model_config
self._control_client = ControlClient(config.control_timeout_ms)
self._control_executor = ThreadPoolExecutor(
max_workers=_CONTROL_WORKERS,
thread_name_prefix="ec-mooncake-control",
)
self._event_inbox = EventInbox(self._control_client, self._control_executor)
self._metadata_resolver = PlaceholderMetadataResolver(vllm_config.model_config)
self._unresolved_transfer_ids = 0
self._drain_pending = True
self._drained_at = 0.0
self._pending_cancels: dict[str, Future[Any]] = {}
self._transfers = SchedulerTransferTable(
config.pool_size, _RESERVATION_TTL_SECONDS
)
self._scheduler_pending_work = False
self._pushes_to_prepare: dict[str, ECMooncakePushSpec] = {}
self._prepared_push_transfer_ids: set[str] = set()
self._event_ready_shards: OrderedDict[str, set[int]] = OrderedDict()
# Mirror of the engine's encoder cache, maintained from the alloc and
# free notifications the Scheduler already sends. Tracking it here
# keeps `ensure_cache_available` on the upstream two-argument shape.
self._local_cache: set[str] = set()
self._failed_saves: set[str] = set()
def _cancel_remote(self, consumer_zmq: str, transfer_id: str) -> bool:
pending = None
for _ in range(_CANCEL_ATTEMPTS):
pending = self._control_client.discover_shards(consumer_zmq)
if pending is not None:
break
if pending is None:
raise RuntimeError(
f"Could not discover every EC consumer shard at {consumer_zmq}"
)
cancelled = False
error: BaseException | None = None
for _ in range(_CANCEL_ATTEMPTS):
failed = []
for addr in pending:
try:
result = self._control_client.request(
addr,
make_cancel_request(transfer_id),
)
except Exception as exc:
if error is None:
error = exc
failed.append(addr)
continue
cancelled |= isinstance(result, dict) and bool(result.get("cancelled"))
if not failed:
return cancelled
pending = failed
if error is not None:
raise error
return cancelled
def _note_awaiting_push(
self,
mm_hash: str,
transfer_id: str,
request_id: str,
now: float,
) -> None:
record = self._transfers.wait_for_event(
transfer_id,
request_id,
mm_hash,
now + self._push_wait_timeout,
)
if record is None:
# The id names another encoding; the request is already failed.
return
if record.state is not SchedulerTransferState.WAITING_EVENT:
return
assert record.deadline is not None
if now < record.deadline:
return
elapsed = now - record.deadline + self._push_wait_timeout
self._transfers.mark_unavailable(transfer_id, now)
logger.warning(
"EC Mooncake waited %.1fs for a push of mm_hash=%s "
"(transfer_id=%s) that never arrived; "
"requests needing it fail with a retryable error.",
elapsed,
mm_hash,
transfer_id,
)
def take_unavailable_requests(self) -> set[str]:
failed, self._failed_saves = self._failed_saves, set()
return failed | self._transfers.take_unavailable_requests()
def _poll_pending_cancels(self) -> None:
pending = {}
for transfer_id, future in self._pending_cancels.items():
if not future.done():
pending[transfer_id] = future
continue
try:
future.result()
except Exception:
logger.warning(
"EC Mooncake reservation cancellation failed", exc_info=True
)
self._pending_cancels = pending
def _note_shard_ready(self, data: dict[str, Any]) -> bool:
shard_count = self._event_inbox.shard_count
if shard_count <= 1:
return True
transfer_id = str(data["transfer_id"])
record = self._transfers.get(transfer_id)
if (
record is not None
and record.state is not SchedulerTransferState.WAITING_EVENT
):
return False
shard = data.get("shard")
shards = self._event_ready_shards.setdefault(transfer_id, set())
self._event_ready_shards.move_to_end(transfer_id)
shards.add(int(shard) if shard is not None else len(shards))
if len(shards) < shard_count:
while len(self._event_ready_shards) > _MAX_PENDING_EVENTS:
self._event_ready_shards.popitem(last=False)
return False
self._event_ready_shards.pop(transfer_id, None)
return True
def _forget_shard_readiness(self, transfer_id: str) -> None:
self._event_ready_shards.pop(transfer_id, None)
def _store_pushed_spec(self, data: dict[str, Any]) -> bool:
transfer_id = str(data["transfer_id"])
identifier = str(data["mm_hash"])
_, accepted = self._transfers.observe_ready(
ECMooncakeLoadSpec(
mm_hash=identifier,
nbytes=int(data["nbytes"]),
shape=tuple(int(value) for value in data["shape"]),
dtype=str(data["dtype"]),
transfer_id=transfer_id,
),
time.monotonic() + _RESERVATION_TTL_SECONDS,
)
return accepted
def _queue_cancel(
self,
transfer_id: str,
mm_hash: str = "",
request_id: str = "",
) -> None:
if not self._transfers.cancel(
transfer_id,
time.monotonic(),
mm_hash=mm_hash,
request_id=request_id,
):
return
self._forget_shard_readiness(transfer_id)
self._pending_cancels[transfer_id] = self._control_executor.submit(
self._cancel_remote,
self._control_addr,
transfer_id,
)
def _expire_transfers(self) -> None:
now = time.monotonic()
expired = self._transfers.expire(now, _MAX_TERMINAL_TRANSFER_RECORDS)
for record in expired:
self._queue_cancel(record.transfer_id)
def _drain_push_notifications(self) -> None:
now = time.monotonic()
if not self._drain_pending and now - self._drained_at < _DRAIN_MIN_INTERVAL:
return
self._drain_pending = False
self._drained_at = now
self._poll_pending_cancels()
self._expire_transfers()
events = self._event_inbox.drain(self._control_addr)
for data in events:
try:
if not isinstance(data, dict):
raise TypeError("EC readiness event must be an object")
if not data.get("ready"):
continue
self._accept_ready_event(data)
except (KeyError, TypeError, ValueError):
# Readiness events cross a plain PULL socket, so a malformed
# one costs that event and not the engine.
logger.warning(
"Discarding a malformed EC readiness event.", exc_info=True
)
def _accept_ready_event(self, data: dict[str, Any]) -> None:
transfer_id = str(data["transfer_id"])
record = self._transfers.get(transfer_id)
if record is not None and record.state in {
SchedulerTransferState.CANCELLED,
SchedulerTransferState.UNAVAILABLE,
SchedulerTransferState.EXPIRED,
SchedulerTransferState.FAILED,
}:
return
if not self._note_shard_ready(data):
return
self._store_pushed_spec(data)
def has_cache_item(self, identifier: str) -> bool:
if not self._is_consumer:
return False
self._drain_push_notifications()
return self._transfers.has_state(
identifier,
(
SchedulerTransferState.READY,
SchedulerTransferState.RESIDENT,
SchedulerTransferState.AVAILABLE,
),
)
def _warn_unresolved_transfer_id(
self, request: Any, index: int, where: str
) -> None:
"""Report an `ec_items` entry that cannot be matched to a feature.
A caller that cannot resolve a transfer id has to fall back to a
locally invented one or skip its bookkeeping entirely, and either way
the two sides of a transfer stop agreeing on its name. That desyncs
silently -- reservations are never released and only surface minutes
later as a full buffer pool -- so say it out loud the first time.
`mm_hash` in `ec_items` must be a value some engine reported, never one
the caller derived itself: `mm_features[i].identifier` folds in the
engine's media_io_kwargs and mm_processor_kwargs, so a media uuid alone
no longer equals it. Omit the field to match by position instead.
"""
self._unresolved_transfer_ids += 1
count = self._unresolved_transfer_ids
if count > _MAX_UNRESOLVED_TRANSFER_ID_WARNINGS and count % 1000:
return
params = getattr(request, "ec_transfer_params", None)
items = (params or {}).get("ec_items") or []
logger.warning(
"EC Mooncake could not resolve a transfer id at %s for req=%s "
"index=%d: feature identifier=%s does not match any of the %d "
"ec_items %s (ec_transfer_params present=%s). Occurrence %d. "
"ec_items[].mm_hash must echo an engine-reported identifier, or "
"be omitted so the entry is matched by position.",
where,
request.request_id,
index,
request.mm_features[index].identifier[:16],
len(items),
[str(item.get("mm_hash"))[:16] for item in items[:4]],
params is not None,
count,
)
@staticmethod
def _request_transfer_id(request: Any, index: int) -> str | None:
params = getattr(request, "ec_transfer_params", None) or {}
items = params.get("ec_items") or []
mm_hash = request.mm_features[index].identifier
if index < len(items):
item = items[index]
if item.get("mm_hash") in (None, mm_hash) and item.get("transfer_id"):
return str(item["transfer_id"])
for item in items:
if item.get("mm_hash") == mm_hash and item.get("transfer_id"):
return str(item["transfer_id"])
return None
def ensure_cache_available(self, request: Any, num_computed_tokens: int) -> bool:
if self._is_producer:
for index, feature in enumerate(request.mm_features):
if (
feature.mm_position.offset + feature.mm_position.length
> num_computed_tokens
):
self._prepare_push_spec(request, index)
if not self._is_consumer:
return True
self._drain_push_notifications()
# One timestamp for the whole decision, so the deadlines it hands out
# cannot disagree between features.
now = time.monotonic()
all_ready = True
for index, feature in enumerate(request.mm_features):
if (
feature.mm_position.offset + feature.mm_position.length
<= num_computed_tokens
):
continue
mm_hash = feature.identifier
transfer_id = self._request_transfer_id(request, index)
if transfer_id is not None:
self._transfers.touch_available(
transfer_id, now + _RESERVATION_TTL_SECONDS
)
if mm_hash in self._local_cache:
continue
if self._transfers.has_state(mm_hash, (SchedulerTransferState.READY,)):
continue
if self._transfers.has_state(mm_hash, (SchedulerTransferState.LOADING,)):
all_ready = False
continue
record = self._transfers.begin_load(
mm_hash,
transfer_id,
request.request_id,
now + self._push_wait_timeout,
)
if record is not None:
self._scheduler_pending_work = True
all_ready = False
else:
waiting_id = transfer_id or f"{request.request_id}:{index}"
self._note_awaiting_push(mm_hash, waiting_id, request.request_id, now)
all_ready = False
return all_ready
def _prepare_push_spec(self, request: Any, index: int) -> None:
params = getattr(request, "ec_transfer_params", None) or {}
consumer_zmq = params.get("consumer_zmq")
mm_hash = request.mm_features[index].identifier
transfer_id = self._request_transfer_id(request, index)
if transfer_id is None:
# Still push: after the proxy has rewritten the item to embeds the
# consumer has no media left to fall back on, so a nameless push
# beats none. But the consumer knows this transfer by the id it
# sent, not by the one invented here, so its cancel will never
# reach the reservation this push is about to take.
if consumer_zmq:
self._warn_unresolved_transfer_id(request, index, "push prepare")
transfer_id = f"{request.request_id}:{index}"
if not consumer_zmq or transfer_id in self._prepared_push_transfer_ids:
return
num_tokens = request.get_num_encoder_embeds(index)
dtype = self._model_config.dtype
assert isinstance(dtype, torch.dtype)
assert self._encoder_cache_hidden_dim is not None
dtype_name = str(dtype).split(".")[-1]
shape = (num_tokens, self._encoder_cache_hidden_dim)
nbytes = math.prod(shape) * dtype.itemsize
self._pushes_to_prepare[transfer_id] = ECMooncakePushSpec(
mm_hash=mm_hash,
nbytes=nbytes,
shape=shape,
dtype=dtype_name,
consumer_zmq=str(consumer_zmq),
transfer_id=transfer_id,
request_id=request.request_id,
)
self._prepared_push_transfer_ids.add(transfer_id)
def update_state_after_alloc(self, request: Any, index: int) -> None:
self._local_cache.add(request.mm_features[index].identifier)
if self._is_producer:
self._prepare_push_spec(request, index)
def update_state_after_free(self, request: Any, index: int) -> None:
if not self._is_consumer:
return
transfer_id = self._request_transfer_id(request, index)
if transfer_id is None:
self._warn_unresolved_transfer_id(request, index, "encoder-cache free")
return
self._queue_cancel(
transfer_id,
mm_hash=request.mm_features[index].identifier,
request_id=request.request_id,
)
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> ECConnectorMetadata:
for mm_hash in scheduler_output.free_encoder_mm_hashes:
self._local_cache.discard(mm_hash)
self._transfers.release_ready(mm_hash, time.monotonic())
for transfer_id in self._transfers.drain_orphaned():
self._queue_cancel(transfer_id)
meta = ECMooncakeConnectorMetadata(
freed=scheduler_output.free_encoder_mm_hashes
)
for push_spec in self._pushes_to_prepare.values():
meta.pushes.append(push_spec)
self._pushes_to_prepare.clear()
for record in self._transfers.take_loads_to_dispatch():
assert record.spec is not None
meta.loads.append(record.spec)
self._poll_pending_cancels()
self._drain_pending = True
return meta
def update_connector_output(self, connector_output: ECConnectorOutput) -> None:
meta = connector_output.ec_connector_worker_meta
if not isinstance(meta, ECMooncakeWorkerMetadata):
return
for mm_hash in meta.loaded:
self._transfers.complete_load(mm_hash)
for mm_hash in meta.failed_loads:
self._transfers.fail_load(mm_hash, time.monotonic())
for mm_hash in meta.reclaimed:
self._transfers.reclaim(mm_hash, time.monotonic())
self._scheduler_pending_work = meta.pending_saves
self._failed_saves.update(meta.failed_saves)
def has_pending_push_work(self) -> bool:
return self._scheduler_pending_work
def request_finished(self, request: Any) -> tuple[bool, dict[str, Any] | None]:
if self._is_consumer:
for index in range(len(request.mm_features)):
transfer_id = self._request_transfer_id(request, index)
if transfer_id is None:
self._warn_unresolved_transfer_id(request, index, "request finish")
continue
self._queue_cancel(
transfer_id,
mm_hash=request.mm_features[index].identifier,
request_id=request.request_id,
)
if self._is_producer and self._prepared_push_transfer_ids:
for index in range(len(request.mm_features)):
transfer_id = self._request_transfer_id(request, index)
if transfer_id is None:
transfer_id = f"{request.request_id}:{index}"
self._prepared_push_transfer_ids.discard(transfer_id)
if not self._is_producer:
return False, None
items = collect_ec_item_metadata(request.mm_features, self._metadata_resolver)
for index, feature in enumerate(request.mm_features):
transfer_id = self._request_transfer_id(request, index)
if transfer_id is not None:
items[feature.identifier]["transfer_id"] = transfer_id
if not items:
return False, None
return False, items
def close(self) -> None:
self._control_executor.shutdown(wait=True, cancel_futures=True)
self._control_client.close()
self._event_inbox.close()
@@ -0,0 +1,448 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Scheduler-owned lifecycle and indexes for Consumer-bound transfers."""
from __future__ import annotations
from collections import OrderedDict, deque
from collections.abc import Iterable
from dataclasses import dataclass, field, replace
from enum import Enum, auto
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
ECMooncakeLoadSpec,
)
from vllm.logger import init_logger
logger = init_logger(__name__)
class SchedulerTransferState(Enum):
"""Lifecycle states visible to the Scheduler.
The states cover waiting for a Consumer event, dispatching a Worker load,
retaining a locally reusable tensor, and terminal failure conditions.
"""
WAITING_EVENT = auto()
AVAILABLE = auto()
LOADING = auto()
READY = auto()
RESIDENT = auto()
UNAVAILABLE = auto()
EXPIRED = auto()
FAILED = auto()
CANCELLED = auto()
@dataclass
class SchedulerTransfer:
"""Track one transfer as observed by the Scheduler."""
transfer_id: str
request_id: str
mm_hash: str
state: SchedulerTransferState
spec: ECMooncakeLoadSpec | None
deadline: float | None
notified_requests: set[str] = field(default_factory=set, repr=False)
_TERMINAL_STATES = {
SchedulerTransferState.UNAVAILABLE,
SchedulerTransferState.EXPIRED,
SchedulerTransferState.FAILED,
SchedulerTransferState.CANCELLED,
}
class SchedulerTransferTable:
"""Own Scheduler transfer state, lookup indexes, and dispatch queues.
The Scheduler is single-threaded, so direct transitions are sufficient;
the Producer and Consumer managers retain stricter transition matrices
because callbacks and control requests can race there.
"""
def __init__(self, resident_capacity: int, tombstone_ttl: float) -> None:
self._resident_capacity = resident_capacity
self._tombstone_ttl = tombstone_ttl
self._records: OrderedDict[str, SchedulerTransfer] = OrderedDict()
self._active_ids: dict[str, None] = {}
self._terminal_ids: OrderedDict[str, None] = OrderedDict()
self._resident_ids: OrderedDict[str, None] = OrderedDict()
self._resident_bytes = 0
self._hash_index: dict[str, deque[str]] = {}
self._loads_to_dispatch: OrderedDict[str, None] = OrderedDict()
self._unavailable_requests: set[str] = set()
# Records expired straight out of READY/RESIDENT: the consumer worker
# still holds their push reservation, and nothing else tells it to let
# go, so the destination buffer would sit pinned until the lease TTL.
self._orphaned: list[str] = []
def get(self, transfer_id: str) -> SchedulerTransfer | None:
return self._records.get(transfer_id)
def drain_orphaned(self) -> list[str]:
"""Transfer IDs expired out of READY/RESIDENT since the last drain.
The caller must release the consumer worker's reservation for each:
`cancel()` still accepts an EXPIRED record, so the usual cancel path
applies.
"""
orphaned = self._orphaned
self._orphaned = []
return orphaned
def records_for_hash(
self,
mm_hash: str,
states: Iterable[SchedulerTransferState],
) -> list[SchedulerTransfer]:
wanted = set(states)
return [
record
for transfer_id in self._hash_index.get(mm_hash, ())
if (record := self._records.get(transfer_id)) is not None
and record.state in wanted
]
def first_for_hash(
self,
mm_hash: str,
states: Iterable[SchedulerTransferState],
) -> SchedulerTransfer | None:
states = tuple(states)
if len(states) == 1:
wanted_state = states[0]
for transfer_id in self._hash_index.get(mm_hash, ()):
record = self._records.get(transfer_id)
if record is not None and record.state is wanted_state:
return record
return None
wanted_states = set(states)
for transfer_id in self._hash_index.get(mm_hash, ()):
record = self._records.get(transfer_id)
if record is not None and record.state in wanted_states:
return record
return None
def has_state(self, mm_hash: str, states: Iterable[SchedulerTransferState]) -> bool:
return self.first_for_hash(mm_hash, states) is not None
def wait_for_event(
self,
transfer_id: str,
request_id: str,
mm_hash: str,
deadline: float,
) -> SchedulerTransfer | None:
"""Track a request waiting for a push.
Returns None when `transfer_id` already names a different encoding.
The id comes from the request, so a collision has to fail that one
request; re-issuing it re-runs the encode under a fresh id.
"""
record = self._records.get(transfer_id)
if record is None:
record = SchedulerTransfer(
transfer_id=transfer_id,
request_id=request_id,
mm_hash=mm_hash,
state=SchedulerTransferState.WAITING_EVENT,
spec=None,
deadline=deadline,
)
self._insert(record)
elif not self._identity_matches(record, mm_hash):
self._refuse_colliding_id(record, mm_hash, request_id)
return None
else:
if not record.request_id:
record.request_id = request_id
if record.state in _TERMINAL_STATES and request_id:
self._notify_unavailable(record, request_id)
return record
def observe_ready(
self, spec: ECMooncakeLoadSpec, deadline: float
) -> tuple[SchedulerTransfer, bool]:
transfer_id = spec.transfer_id
record = self._records.get(transfer_id)
if record is None:
record = SchedulerTransfer(
transfer_id=transfer_id,
request_id="",
mm_hash=spec.mm_hash,
state=SchedulerTransferState.WAITING_EVENT,
spec=None,
deadline=None,
)
self._insert(record)
elif not self._identity_matches(record, spec.mm_hash):
self._refuse_colliding_id(record, spec.mm_hash, record.request_id)
return record, False
if record.state is not SchedulerTransferState.WAITING_EVENT:
return record, False
record.spec = spec
record.deadline = deadline
self._transition(record, SchedulerTransferState.AVAILABLE)
return record, True
def touch_available(self, transfer_id: str, deadline: float) -> None:
record = self._records.get(transfer_id)
if record is not None and record.state is SchedulerTransferState.AVAILABLE:
record.deadline = deadline
def begin_load(
self,
mm_hash: str,
transfer_id: str | None = None,
request_id: str = "",
deadline: float | None = None,
) -> SchedulerTransfer | None:
record = self._records.get(transfer_id) if transfer_id else None
if record is not None and (
record.mm_hash != mm_hash
or record.state is not SchedulerTransferState.AVAILABLE
):
record = None
if record is None:
record = self.first_for_hash(
mm_hash,
(
SchedulerTransferState.AVAILABLE,
SchedulerTransferState.RESIDENT,
),
)
if record is None or record.spec is None:
return None
if not record.request_id:
record.request_id = request_id
# A dispatched load that never reports back would otherwise hold this
# hash in LOADING for good, deferring every later request for it.
record.deadline = deadline
self._transition(record, SchedulerTransferState.LOADING)
self._loads_to_dispatch[record.transfer_id] = None
return record
def take_loads_to_dispatch(self) -> list[SchedulerTransfer]:
records = [
record
for transfer_id in self._loads_to_dispatch
if (record := self._records.get(transfer_id)) is not None
and record.state is SchedulerTransferState.LOADING
]
self._loads_to_dispatch.clear()
return records
def complete_load(self, mm_hash: str) -> bool:
record = self.first_for_hash(mm_hash, (SchedulerTransferState.LOADING,))
if record is None:
return self.has_state(mm_hash, (SchedulerTransferState.READY,))
self._transition(record, SchedulerTransferState.READY)
return True
def fail_load(self, mm_hash: str, now: float) -> bool:
record = self.first_for_hash(mm_hash, (SchedulerTransferState.LOADING,))
if record is None:
return self.has_state(mm_hash, (SchedulerTransferState.FAILED,))
self._transition(record, SchedulerTransferState.FAILED, now=now)
return True
def release_ready(self, mm_hash: str, now: float) -> None:
ready = self.records_for_hash(mm_hash, (SchedulerTransferState.READY,))
if ready:
canonical = ready[-1]
for record in self.records_for_hash(
mm_hash,
(SchedulerTransferState.READY, SchedulerTransferState.RESIDENT),
):
if record is not canonical:
self._transition(record, SchedulerTransferState.EXPIRED, now=now)
if canonical.spec is None:
self._transition(canonical, SchedulerTransferState.EXPIRED, now=now)
else:
canonical.spec = replace(canonical.spec, local=True)
self._transition(canonical, SchedulerTransferState.RESIDENT)
self._evict_residents(now)
def reclaim(self, mm_hash: str, now: float) -> None:
for record in self.records_for_hash(
mm_hash,
(SchedulerTransferState.READY, SchedulerTransferState.RESIDENT),
):
if record.state is SchedulerTransferState.READY:
record.spec = None
else:
self._transition(record, SchedulerTransferState.EXPIRED, now=now)
def mark_unavailable(self, transfer_id: str, now: float) -> None:
record = self._records[transfer_id]
self._transition(record, SchedulerTransferState.UNAVAILABLE, now=now)
if record.request_id:
self._notify_unavailable(record, record.request_id)
def cancel(
self,
transfer_id: str,
now: float,
mm_hash: str = "",
request_id: str = "",
) -> bool:
record = self._records.get(transfer_id)
if (
record is not None
and mm_hash
and not self._identity_matches(record, mm_hash)
):
return False
if record is None:
record = SchedulerTransfer(
transfer_id=transfer_id,
request_id=request_id,
mm_hash=mm_hash,
state=SchedulerTransferState.WAITING_EVENT,
spec=None,
deadline=None,
)
self._insert(record)
if record.state in {
SchedulerTransferState.CANCELLED,
SchedulerTransferState.READY,
SchedulerTransferState.RESIDENT,
SchedulerTransferState.FAILED,
}:
return False
self._transition(record, SchedulerTransferState.CANCELLED, now=now)
self._loads_to_dispatch.pop(transfer_id, None)
return True
def expire(self, now: float, terminal_limit: int) -> list[SchedulerTransfer]:
if terminal_limit < 0:
raise ValueError("terminal_limit must be non-negative")
expired = []
for transfer_id in list(self._active_ids):
record = self._records[transfer_id]
if record.deadline is None or record.deadline > now:
continue
if record.state is SchedulerTransferState.AVAILABLE:
self._transition(
record,
SchedulerTransferState.EXPIRED,
now=now,
)
expired.append(record)
elif record.state is SchedulerTransferState.LOADING:
self._transition(
record,
SchedulerTransferState.UNAVAILABLE,
now=now,
)
if record.request_id:
self._notify_unavailable(record, record.request_id)
expired.append(record)
while self._terminal_ids:
transfer_id = next(iter(self._terminal_ids))
record = self._records[transfer_id]
if (
record.deadline is not None
and record.deadline > now
and len(self._terminal_ids) <= terminal_limit
):
break
self._remove(transfer_id)
return expired
def take_unavailable_requests(self) -> set[str]:
unavailable = self._unavailable_requests
self._unavailable_requests = set()
return unavailable
def _notify_unavailable(self, record: SchedulerTransfer, request_id: str) -> None:
if request_id not in record.notified_requests:
record.notified_requests.add(request_id)
self._unavailable_requests.add(request_id)
def _insert(self, record: SchedulerTransfer) -> None:
self._records[record.transfer_id] = record
self._active_ids[record.transfer_id] = None
if record.mm_hash:
self._hash_index.setdefault(record.mm_hash, deque()).append(
record.transfer_id
)
@staticmethod
def _identity_matches(record: SchedulerTransfer, mm_hash: str) -> bool:
return not record.mm_hash or record.mm_hash == mm_hash
def _refuse_colliding_id(
self, record: SchedulerTransfer, mm_hash: str, request_id: str
) -> None:
"""Report a transfer id that already names another encoding.
Transfer ids arrive on the request, so a collision is reachable from
outside and must not reach the engine as an exception.
"""
logger.warning(
"EC Mooncake transfer_id=%s already names mm_hash=%s; refusing "
"mm_hash=%s for request %s",
record.transfer_id,
record.mm_hash[:16],
mm_hash[:16],
request_id or "<unknown>",
)
if request_id:
self._unavailable_requests.add(request_id)
def _transition(
self,
record: SchedulerTransfer,
state: SchedulerTransferState,
now: float | None = None,
) -> None:
if record.state is SchedulerTransferState.RESIDENT:
self._resident_ids.pop(record.transfer_id, None)
if record.spec is not None:
self._resident_bytes -= record.spec.nbytes
if state is SchedulerTransferState.RESIDENT:
self._resident_ids[record.transfer_id] = None
if record.spec is not None:
self._resident_bytes += record.spec.nbytes
if state in _TERMINAL_STATES:
if now is None:
raise ValueError("Terminal transition requires a timestamp")
# Keep a bounded tombstone so late events and repeat request IDs
# remain idempotent instead of reviving a finished transfer.
record.deadline = now + self._tombstone_ttl
self._active_ids.pop(record.transfer_id, None)
self._terminal_ids[record.transfer_id] = None
self._terminal_ids.move_to_end(record.transfer_id)
if state is SchedulerTransferState.EXPIRED and record.state in (
SchedulerTransferState.READY,
SchedulerTransferState.RESIDENT,
):
self._orphaned.append(record.transfer_id)
record.state = state
self._records.move_to_end(record.transfer_id)
def _evict_residents(self, now: float) -> None:
while self._resident_bytes > self._resident_capacity and self._resident_ids:
record = self._records[next(iter(self._resident_ids))]
self._transition(record, SchedulerTransferState.EXPIRED, now=now)
def _remove(self, transfer_id: str) -> None:
record = self._records.pop(transfer_id, None)
self._active_ids.pop(transfer_id, None)
self._terminal_ids.pop(transfer_id, None)
if transfer_id in self._resident_ids:
self._resident_ids.pop(transfer_id)
if record is not None and record.spec is not None:
self._resident_bytes -= record.spec.nbytes
self._loads_to_dispatch.pop(transfer_id, None)
if record is None or not record.mm_hash:
return
transfer_ids = self._hash_index[record.mm_hash]
transfer_ids.remove(transfer_id)
if not transfer_ids:
self._hash_index.pop(record.mm_hash)
@@ -0,0 +1,209 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Mooncake data-plane engine and memory-registration ownership.
The wrapper isolates optional-engine initialization, session addressing,
synchronous batched writes, and the lifetime of transient source registrations.
"""
from __future__ import annotations
import threading
from dataclasses import dataclass
import torch
from vllm.logger import init_logger
logger = init_logger(__name__)
_MOONCAKE_IMPORT_ERROR: ImportError | None = None
try:
from mooncake.engine import TransferEngine
except ImportError as error:
TransferEngine = None # type: ignore[misc, assignment]
_MOONCAKE_IMPORT_ERROR = error
def ensure_mooncake_available() -> None:
if _MOONCAKE_IMPORT_ERROR is not None:
raise ImportError(
"Install mooncake-transfer-engine to use ECMooncakeConnector."
) from _MOONCAKE_IMPORT_ERROR
@dataclass
class _SourceRegistration:
"""Retain one transient source range while batches reference it."""
tensor: torch.Tensor
nbytes: int
users: int = 1
class MooncakeTransfer:
"""Own a lazy Mooncake engine and transient memory registrations."""
def __init__(self, hostname: str, protocol: str) -> None:
self._hostname = hostname
self._protocol = protocol
self._engine: TransferEngine | None = None
self._engine_lock = threading.Lock()
self._source_registrations: dict[int, _SourceRegistration] = {}
self._pending_unregister: dict[int, torch.Tensor] = {}
self._registration_lock = threading.Lock()
self._closed = False
def _ensure_engine(self) -> TransferEngine:
if self._engine is not None:
return self._engine
with self._engine_lock:
if self._engine is not None:
return self._engine
engine = TransferEngine()
ret = engine.initialize(self._hostname, "P2PHANDSHAKE", self._protocol, "")
if ret != 0:
raise RuntimeError("Mooncake TransferEngine initialization failed.")
self._engine = engine
logger.info(
"ECMooncakeConnector TransferEngine ready at %s:%d",
self._hostname,
engine.get_rpc_port(),
)
return self._engine
def ensure_ready(self) -> None:
self._ensure_engine()
def local_session(self) -> str:
engine = self._ensure_engine()
return f"{self._hostname}:{engine.get_rpc_port()}"
def register_memory(self, tensor: torch.Tensor) -> int:
return self._ensure_engine().batch_register_memory(
[tensor.data_ptr()], [tensor.nbytes]
)
def unregister_memory(self, tensor: torch.Tensor) -> bool:
engine = self._ensure_engine()
address = tensor.data_ptr()
ret = engine.unregister_memory(address)
with self._registration_lock:
if ret != 0:
logger.error(
"Mooncake EC memory unregistration failed for address %d: %d",
address,
ret,
)
self._pending_unregister[address] = tensor
return False
self._pending_unregister.pop(address, None)
return True
@staticmethod
def _source_range(tensor: torch.Tensor) -> tuple[int, int]:
# Encoder batches commonly split one storage into sibling tensor views.
# Registering each view's exact bytes avoids overlapping memory regions.
return tensor.data_ptr(), tensor.nbytes
def acquire_sources(self, tensors: list[torch.Tensor]) -> list[int]:
ranges: dict[int, tuple[int, torch.Tensor]] = {}
for tensor in tensors:
address, nbytes = self._source_range(tensor)
ranges.setdefault(address, (nbytes, tensor))
engine = self._ensure_engine()
acquired: list[int] = []
new_addresses: list[int] = []
new_lengths: list[int] = []
with self._registration_lock:
for address, (nbytes, tensor) in ranges.items():
entry = self._source_registrations.get(address)
if entry is not None:
if entry.nbytes != nbytes:
raise RuntimeError(
"Mooncake EC source storage changed size while registered"
)
entry.users += 1
acquired.append(address)
continue
new_addresses.append(address)
new_lengths.append(nbytes)
self._source_registrations[address] = _SourceRegistration(
tensor=tensor,
nbytes=nbytes,
)
acquired.append(address)
if new_addresses:
ret = engine.batch_register_memory(new_addresses, new_lengths)
if ret != 0:
for address in acquired:
entry = self._source_registrations[address]
entry.users -= 1
if entry.users == 0:
del self._source_registrations[address]
raise RuntimeError("Mooncake EC source registration failed")
return acquired
def release_sources(self, addresses: list[int]) -> bool:
if not addresses:
return True
with self._registration_lock:
unused = []
for address in addresses:
entry = self._source_registrations.get(address)
if entry is None:
continue
entry.users -= 1
if entry.users == 0:
unused.append(address)
if not unused:
return True
ret = self._ensure_engine().batch_unregister_memory(unused)
if ret != 0:
logger.warning(
"Keeping %d EC source tensors registered after Mooncake "
"unregistration failure",
len(unused),
)
return False
for address in unused:
del self._source_registrations[address]
self._pending_unregister.pop(address, None)
return True
def write(
self,
session: str,
sources: list[int],
destinations: list[int],
lengths: list[int],
) -> None:
"""Write one synchronous batch, returning only at terminal status."""
ret = self._ensure_engine().batch_transfer_sync_write(
session, sources, destinations, lengths
)
if ret != 0:
raise RuntimeError(
f"Mooncake EC push to {session} failed with status {ret}"
)
def close(self) -> None:
if self._closed:
return
self._closed = True
engine = self._engine
if engine is None:
return
with self._registration_lock:
addresses = list(self._source_registrations)
addresses.extend(self._pending_unregister)
if not addresses:
return
ret = engine.batch_unregister_memory(list(dict.fromkeys(addresses)))
if ret != 0:
logger.error("Mooncake EC batch memory unregistration failed: %d", ret)
return
self._source_registrations.clear()
self._pending_unregister.clear()
@@ -0,0 +1,940 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Worker-side orchestration of Mooncake control, memory, and data planes.
Consumer Workers expose rank-local reservations and publish received tensors.
Producer Workers reserve every destination shard, bind computed sources, run
batched Mooncake writes, and report asynchronous completion to the Scheduler.
"""
from __future__ import annotations
import math
import threading
import time
from collections.abc import Callable
from concurrent.futures import Future, ThreadPoolExecutor
from functools import partial
from typing import TYPE_CHECKING, Any, TypeVar, cast
import torch
from vllm.distributed.ec_transfer.ec_connector.mooncake.config import (
_RESERVATION_TTL_SECONDS,
MooncakeECConfig,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.control import (
ConsumerControlServer,
ControlClient,
make_cancel_request,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.memory import (
ConsumerMemoryPool,
ProducerMemoryPool,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
ECMooncakeConnectorMetadata,
ECMooncakePushSpec,
ECMooncakeWorkerMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.producer import (
ProducerPushManager,
ProducerPushRecord,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.reservation import (
ConsumerReservationManager,
ConsumerReservationState,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.transfer import (
MooncakeTransfer,
ensure_mooncake_available,
)
from vllm.logger import init_logger
from vllm.utils.network_utils import get_ip
logger = init_logger(__name__)
_T = TypeVar("_T")
if TYPE_CHECKING:
from vllm.config import VllmConfig
_RESERVATION_REFRESH_SECONDS = _RESERVATION_TTL_SECONDS / 2
_MAX_CANCELLED_TRANSFER_IDS = 1 << 16
_CANCEL_ATTEMPTS = 2
_TRANSFER_WORKERS = 4
_CONTROL_WORKERS = 8
class _FanoutError(RuntimeError):
"""Retain shard outcomes after every started task settles."""
def __init__(self, error: BaseException, results: list[Any | None]) -> None:
self.results = results
super().__init__(str(error))
class ECMooncakeWorker:
"""Orchestrate consumer reservations and producer push batches.
Consumers may use TP, PP, and DP. Only the first PP stage receives encoder
outputs, and each TP rank exposes a consecutive control port and receives
the same source concurrently. Producers remain unsharded and unreplicated.
With DP, the caller must route both halves of a request to the same replica
and pass that replica's control address to the producer.
"""
def __init__(self, vllm_config: VllmConfig) -> None:
ensure_mooncake_available()
config = MooncakeECConfig.from_vllm_config(vllm_config)
self.is_producer = config.is_producer
self.is_consumer = config.is_consumer
self._buffer_device = config.buffer_device
self._control_host = config.control_host
self._control_port = config.control_port
self._transfer = MooncakeTransfer(get_ip(), config.protocol)
self._consumer_memory = ConsumerMemoryPool(
config.pool_size,
self._transfer,
)
self._reservations = ConsumerReservationManager(
self._consumer_memory,
_RESERVATION_TTL_SECONDS,
_MAX_CANCELLED_TRANSFER_IDS,
)
self._consumer_rank_resolved = False
self._is_receiving_rank = True
self._tp_rank = 0
self._tp_size = 1
self._control_server: ConsumerControlServer | None = None
# Worker producer
self._producer_memory = ProducerMemoryPool(
config.pool_size,
self._transfer,
)
self._control_client = ControlClient(config.control_timeout_ms)
self._shutdown_drain_timeout_s = config.control_timeout_ms / 1000
self._io_executor = ThreadPoolExecutor(
max_workers=_TRANSFER_WORKERS,
thread_name_prefix="ec-mooncake-transfer",
)
self._control_executor = ThreadPoolExecutor(
max_workers=_CONTROL_WORKERS,
thread_name_prefix="ec-mooncake-control",
)
self._shard_pool: ThreadPoolExecutor | None = None
self._shard_pool_lock = threading.Lock()
self._push_ready = threading.Event()
self._producer_pushes = ProducerPushManager(self._push_ready.set)
self._dispatch_stop = threading.Event()
self._dispatcher: threading.Thread | None = None
self._failed_saves: set[str] = set()
self._collecting_sources = False
self._completed_loads: set[str] = set()
self._failed_loads: set[str] = set()
self._shutdown = False
if self.is_producer:
self._dispatcher = threading.Thread(
target=self._dispatch_pushes, name="ec-mooncake-ready", daemon=True
)
self._dispatcher.start()
def _dispatch_pushes(self) -> None:
pending_event = False
while not self._dispatch_stop.is_set():
self._push_ready.wait(timeout=0.001 if pending_event else None)
self._push_ready.clear()
if self._dispatch_stop.is_set():
break
if self._collecting_sources:
pending_event = False
continue
pending_event = self._producer_pushes.submit_batches(
self._io_executor, self._push_batch
)
def _resolve_consumer_rank(self) -> None:
"""Place this worker in the consumer receive topology."""
if self._consumer_rank_resolved:
return
self._consumer_rank_resolved = True
try:
from vllm.distributed.parallel_state import get_pp_group, get_tp_group
tp_group = get_tp_group()
self._tp_rank = tp_group.rank_in_group
self._tp_size = tp_group.world_size
self._is_receiving_rank = get_pp_group().is_first_rank
except AssertionError:
# Groups are only absent outside a distributed run, where this
# worker is the whole consumer.
self._tp_rank = 0
self._tp_size = 1
self._is_receiving_rank = True
def start_services(self) -> None:
if not self.is_consumer or self._control_server is not None:
return
self._resolve_consumer_rank()
if not self._is_receiving_rank:
# Later pipeline stages hold no encoder outputs, so they need
# neither a receive pool nor a control channel.
return
self._consumer_memory.prepare(torch.device(self._buffer_device))
consumer_pool = self._consumer_memory.tensor
if consumer_pool is None:
raise RuntimeError(
"Mooncake push mode requires a registered consumer buffer pool."
)
base_port = self._control_port
self._control_server = ConsumerControlServer(
# The reservation channel hands out registered-memory addresses
# and takes cancellations, so it listens where the Consumer
# advertises itself (`ec_ip`), not on every interface.
self._control_host,
base_port + self._tp_rank,
self._reserve_push_destination,
self._push_status,
self._reservations.complete,
self._reservations.cancel,
self._reservations.expire,
peer_ports=[base_port + rank for rank in range(self._tp_size)],
device=consumer_pool.device,
drain_ready=self._reservations.drain_ready,
)
try:
self._control_server.start()
except Exception:
self._control_server.close()
self._control_server = None
raise
def _reserve_push_destination(self, payload: dict[str, Any]) -> dict[str, Any]:
transfer_id = str(payload["transfer_id"])
mm_hash = str(payload["mm_hash"])
nbytes = int(payload["nbytes"])
shape = tuple(int(value) for value in payload["shape"])
dtype_name = str(payload["dtype"])
dtype = getattr(torch, dtype_name, None)
if not isinstance(dtype, torch.dtype):
raise ValueError(f"Unsupported torch dtype string: {dtype_name!r}")
expected_nbytes = math.prod(shape) * dtype.itemsize
if expected_nbytes != nbytes:
raise ValueError("shape and dtype do not match nbytes")
self._reservations.expire()
reservation, should_write = self._reservations.reserve(
transfer_id, mm_hash, nbytes, shape, dtype_name, dtype
)
if reservation is None:
raise RuntimeError("EC consumer buffer pool is full")
if reservation.state in {
ConsumerReservationState.CANCEL_PENDING,
ConsumerReservationState.CANCELLED,
}:
return {
"reservation_id": "",
"dst_session": "",
"dst_ptr": 0,
"nbytes": nbytes,
"write": False,
"ready": False,
"cancelled": True,
}
assert reservation.allocation is not None
return {
"reservation_id": reservation.reservation_id,
"dst_session": self._transfer.local_session(),
"dst_ptr": reservation.allocation.tensor.data_ptr(),
"nbytes": reservation.allocation.tensor.nbytes,
"write": should_write,
"ready": reservation.state is ConsumerReservationState.READY,
"cached": reservation.lease is not None,
}
def _push_status(self, transfer_id: str) -> dict[str, Any] | None:
reservation = self._reservations.status(transfer_id)
if reservation is None:
return None
assert reservation.allocation is not None
return {
"mm_hash": reservation.mm_hash,
"ready": reservation.state is ConsumerReservationState.READY,
"reservation_id": reservation.reservation_id,
"nbytes": reservation.allocation.tensor.nbytes,
"shape": list(reservation.shape),
"dtype": reservation.dtype,
}
def _shard_executor(self) -> ThreadPoolExecutor:
"""Use a separate pool so nested shard fan-out cannot deadlock."""
with self._shard_pool_lock:
if self._shard_pool is None:
self._shard_pool = ThreadPoolExecutor(
max_workers=32, thread_name_prefix="ec-mooncake-shard"
)
return self._shard_pool
def _reserve_one(self, addr: str, spec: ECMooncakePushSpec) -> dict[str, Any]:
result = self._control_client.request(
addr,
{
"op": "reserve",
"transfer_id": spec.transfer_id,
"mm_hash": spec.mm_hash,
"nbytes": spec.nbytes,
"shape": list(spec.shape),
"dtype": spec.dtype,
},
)
if not isinstance(result, dict):
raise RuntimeError("Invalid EC reservation response")
result["_received_at"] = time.monotonic()
result["addr"] = addr
return result
def _run_fanout(
self,
tasks: list[Callable[[], _T]],
on_submit: Callable[[int, Future[_T]], None] | None = None,
) -> list[_T]:
"""Run every started shard task and retain partial results on failure.
Waiting for all submitted tasks is what makes source-memory release and
partial-reservation cleanup safe after one shard fails.
"""
if not tasks:
return []
futures: list[tuple[int, Future[_T]]] = []
results: list[_T | None] = [None] * len(tasks)
error: BaseException | None = None
for index, task in enumerate(tasks[1:], 1):
try:
future = self._shard_executor().submit(task)
except Exception as exc:
error = exc
break
futures.append((index, future))
if on_submit is not None:
on_submit(index, future)
if error is None:
try:
results[0] = tasks[0]()
except Exception as exc:
error = exc
for index, future in futures:
try:
results[index] = future.result()
except Exception as exc:
if error is None:
error = exc
if error is not None:
raise _FanoutError(error, results)
return cast(list[_T], results)
def _cancel_reservations(
self,
spec: ECMooncakePushSpec,
reservations: list[dict[str, Any]],
*,
refresh: bool = False,
record: ProducerPushRecord | None = None,
) -> None:
reservations = [
shard for shard in reservations if not shard.get("cancelled", False)
]
if not reservations:
return
def cancel(shard: dict[str, Any]) -> dict[str, Any]:
addr = shard.get("addr")
if not isinstance(addr, str) or not addr:
raise RuntimeError(
"EC reservation is missing a confirmed shard address"
)
result = self._control_client.request(
addr,
make_cancel_request(
spec.transfer_id,
str(shard.get("reservation_id", "")),
abandon=True,
refresh=refresh,
),
)
if not isinstance(result, dict) or not result.get("cancelled"):
raise RuntimeError(
f"Could not cancel EC reservation for mm_hash={spec.mm_hash}"
)
return shard
def track(_index: int, future: Future[dict[str, Any]]) -> None:
if record is not None:
self._producer_pushes.track_shard_futures([record], [future])
self._run_fanout([partial(cancel, shard) for shard in reservations], track)
def _retry_cancel_reservations(
self,
spec: ECMooncakePushSpec,
reservations: list[dict[str, Any]],
*,
record: ProducerPushRecord | None = None,
) -> None:
pending = [shard for shard in reservations if not shard.get("cancelled", False)]
error: _FanoutError | None = None
for _ in range(_CANCEL_ATTEMPTS):
try:
self._cancel_reservations(spec, pending, record=record)
except _FanoutError as exc:
error = exc
pending = [
shard
for index, shard in enumerate(pending)
if exc.results[index] is None
]
continue
return
assert error is not None
raise error
def _reserve_remote(self, spec: ECMooncakePushSpec) -> list[dict[str, Any]]:
"""Reserve a destination on every shard of the consumer."""
result = self._reserve_remote_many([spec])[0]
if isinstance(result, Exception):
raise result
return result
def _refresh_remote_reservations(
self,
spec: ECMooncakePushSpec,
reservations: list[dict[str, Any]],
record: ProducerPushRecord | None = None,
) -> list[dict[str, Any]]:
stale = [
shard
for shard in reservations
if not shard.get("ready", False)
and not shard.get("cached", False)
and not shard.get("cancelled", False)
]
try:
self._cancel_reservations(spec, stale, refresh=True, record=record)
except _FanoutError as exc:
pending = [
shard for index, shard in enumerate(stale) if exc.results[index] is None
]
try:
self._retry_cancel_reservations(spec, pending, record=record)
except _FanoutError as cleanup_error:
raise exc from cleanup_error
raise
return self._reserve_remote(spec)
@staticmethod
def _validate_push_source(push: ProducerPushRecord) -> None:
tensor = push.source_tensor
assert tensor is not None
spec = push.spec
if tuple(tensor.shape) != tuple(spec.shape):
raise ValueError(f"EC source shape mismatch for mm_hash={spec.mm_hash}")
if str(tensor.dtype).split(".")[-1] != spec.dtype:
raise ValueError(f"EC source dtype mismatch for mm_hash={spec.mm_hash}")
if not tensor.is_contiguous():
raise ValueError(f"EC source must be contiguous for mm_hash={spec.mm_hash}")
if tensor.nbytes != spec.nbytes:
raise ValueError(f"EC source size mismatch for mm_hash={spec.mm_hash}")
def start_save_caches(
self,
metadata: ECMooncakeConnectorMetadata,
encoder_cache: dict[str, torch.Tensor] | None = None,
**kwargs: Any,
) -> None:
self._collecting_sources = True
new: dict[str, list[ProducerPushRecord]] = {}
for spec in metadata.pushes:
try:
record, created = self._producer_pushes.reserve(spec, Future)
if created:
new.setdefault(spec.consumer_zmq, []).append(record)
except ValueError:
logger.warning("Rejected conflicting EC push", exc_info=True)
self._failed_saves.add(spec.request_id)
for records in new.values():
future = self._control_executor.submit(self._reserve_batch, records)
future.add_done_callback(partial(self._reservation_batch_done, records))
if not isinstance(encoder_cache, dict):
return
for mm_hash in dict.fromkeys(spec.mm_hash for spec in metadata.pushes):
tensor = encoder_cache.get(mm_hash)
if tensor is not None:
self._bind_push_source(tensor, mm_hash)
def _reserve_batch(self, records: list[ProducerPushRecord]) -> None:
"""Resolve independent item futures with one reserve RPC per shard."""
if len(records) == 1:
record = records[0]
try:
record.reservation_future.set_result(self._reserve_remote(record.spec))
except Exception as exc:
record.reservation_future.set_exception(exc)
return
outcomes = self._reserve_remote_many([record.spec for record in records])
self._producer_pushes.finish_reservations(list(zip(records, outcomes)))
def _reserve_remote_many(
self, specs: list[ECMooncakePushSpec]
) -> list[list[dict[str, Any]] | Exception]:
shards = None
for _ in range(_CANCEL_ATTEMPTS):
shards = self._control_client.discover_shards(specs[0].consumer_zmq)
if shards is not None:
break
if shards is None:
raise RuntimeError(
f"Could not discover every EC consumer shard at {specs[0].consumer_zmq}"
)
def reserve(addr: str) -> list[dict[str, Any]]:
if len(specs) == 1:
return [{"ok": True, "result": self._reserve_one(addr, specs[0])}]
response = self._control_client.request(
addr,
{
"op": "reserve_batch",
"items": [
{
"transfer_id": spec.transfer_id,
"mm_hash": spec.mm_hash,
"nbytes": spec.nbytes,
"shape": list(spec.shape),
"dtype": spec.dtype,
}
for spec in specs
],
},
)
items = response["items"]
if len(items) != len(specs):
raise RuntimeError("Malformed EC reservation batch response")
for item in items:
if item.get("ok"):
item["result"]["_received_at"] = time.monotonic()
return items
fanout_error = None
results: list[Any]
try:
results = self._run_fanout([partial(reserve, addr) for addr in shards])
except _FanoutError as exc:
results = exc.results
fanout_error = exc
outcomes: list[list[dict[str, Any]] | Exception] = []
for index, spec in enumerate(specs):
reservations: list[dict[str, Any]] = []
error = None
for addr, items in zip(shards, results):
item = items[index] if items is not None else {}
if item.get("ok"):
reservations.append({**item["result"], "addr": addr})
else:
error = fanout_error or RuntimeError(item["error"])
reservations.append({"addr": addr, "reservation_id": ""})
if error is None:
outcomes.append(reservations)
else:
failure = _FanoutError(error, list(reservations))
try:
self._retry_cancel_reservations(spec, reservations)
except Exception as cleanup_error:
failure.__cause__ = cleanup_error
logger.exception("Failed to release partial EC reservation batch")
outcomes.append(failure)
return outcomes
@staticmethod
def _reservation_batch_done(
records: list[ProducerPushRecord], future: Future[None]
) -> None:
error = future.exception()
if error is not None:
for record in records:
if not record.reservation_future.done():
record.reservation_future.set_exception(error)
def start_load_caches(
self,
metadata: ECMooncakeConnectorMetadata,
encoder_cache: dict[str, torch.Tensor],
**kwargs: Any,
) -> None:
self._resolve_consumer_rank()
if not self._is_receiving_rank:
# Later pipeline stages never gather multimodal embeddings.
return
self._transfer.ensure_ready()
if self._buffer_device == "cuda" and not torch.accelerator.is_available():
raise RuntimeError(
"ECMooncakeConnector requires CUDA for ec_buffer_device=cuda"
)
self._reservations.retire_stale(encoder_cache, metadata.freed)
for spec in metadata.loads:
if spec.mm_hash in encoder_cache:
if not spec.local:
# The spec's id is one shard's; cancel by transfer.
self._reservations.cancel(spec.transfer_id, "")
self._completed_loads.add(spec.mm_hash)
continue
if spec.local:
tensor = self._consumer_memory.take_resident(
spec.mm_hash, tuple(spec.shape), spec.dtype
)
else:
try:
allocation = self._reservations.take(spec.transfer_id, spec.mm_hash)
tensor = allocation.tensor
except RuntimeError as e:
logger.warning("EC Mooncake pushed load failed: %s", e)
tensor = None
if tensor is None:
self._failed_loads.add(spec.mm_hash)
else:
encoder_cache[spec.mm_hash] = tensor
self._completed_loads.add(spec.mm_hash)
def _push_batch(self, pushes: list[ProducerPushRecord]) -> None:
started_at = time.monotonic()
ready: list[tuple[ProducerPushRecord, dict[str, Any]]] = []
written_pushes: dict[str, ProducerPushRecord] = {}
failure: Exception | None = None
try:
for push in pushes:
self._validate_push_source(push)
reservations = self._producer_pushes.resolve_reservations(push)
stale = [
index
for index, shard in enumerate(reservations)
if not shard.get("ready", False)
and not shard.get("cancelled", False)
and time.monotonic() - float(shard.get("_received_at", started_at))
>= _RESERVATION_REFRESH_SECONDS
]
if stale:
reservations = self._refresh_remote_reservations(
push.spec, reservations, push
)
self._producer_pushes.replace_reservations(push, reservations)
self._producer_pushes.begin_writing(push)
writable = [
shard
for shard in reservations
if not shard.get("cached", False)
and not shard.get("cancelled", False)
and shard.get("write", True)
]
source = push.source_tensor
assert source is not None
for shard in writable:
if int(shard["nbytes"]) != source.nbytes:
raise RuntimeError(
"Reserved EC size does not match tensor for "
f"mm_hash={push.spec.mm_hash}"
)
ready.append((push, shard))
written_pushes.setdefault(push.spec.transfer_id, push)
if ready:
# Stage each source once, then write it to every destination.
source_index = {
push.spec.transfer_id: index
for index, push in enumerate(written_pushes.values())
}
tensors = [
cast(torch.Tensor, push.source_tensor)
for push in written_pushes.values()
if push.source_tensor is not None
]
lengths = [tensor.nbytes for tensor in tensors]
staged = self._producer_memory.stage(tensors)
registered_sources: list[int] = []
if staged is not None:
sources = staged.tensors
else:
sources = tensors
registered_sources = self._transfer.acquire_sources(tensors)
addresses = [tensor.data_ptr() for tensor in sources]
try:
by_session: dict[str, list[tuple[int, int]]] = {}
session_records: dict[str, dict[str, ProducerPushRecord]] = {}
for push, shard in ready:
session = str(shard["dst_session"])
by_session.setdefault(session, []).append(
(source_index[push.spec.transfer_id], int(shard["dst_ptr"]))
)
session_records.setdefault(session, {})[
push.spec.transfer_id
] = push
def write(session: str, items: list[tuple[int, int]]) -> None:
self._transfer.write(
session,
[addresses[index] for index, _ in items],
[dst for _, dst in items],
[lengths[index] for index, _ in items],
)
sessions = list(by_session.items())
# Write shards concurrently to avoid serial TP latency.
def track_write(index: int, future: Future[None]) -> None:
session = sessions[index][0]
self._producer_pushes.track_shard_futures(
list(session_records[session].values()), [future]
)
writes: list[Callable[[], None]] = [
partial(write, *session) for session in sessions
]
self._run_fanout(writes, track_write)
finally:
if staged is not None:
self._producer_memory.release(staged)
self._transfer.release_sources(registered_sources)
self._producer_pushes.begin_notifying(pushes)
self._notify_completions(ready)
self._producer_pushes.complete(pushes)
except Exception as exc:
# Report asynchronously; raising here would fail EngineCore.
failure = exc
logger.exception(
"EC Mooncake push batch failed for mm_hashes=%s",
[push.spec.mm_hash for push in pushes],
)
self._producer_pushes.settle_all(pushes)
self._abandon_pushes(pushes)
finally:
if failure is not None:
self._producer_pushes.fail(pushes, failure)
def _notify_completions(
self, notifications: list[tuple[ProducerPushRecord, dict[str, Any]]]
) -> None:
"""Tell the consumer, in one message per destination, what landed."""
if not notifications:
return
by_destination: dict[str, list[tuple[ProducerPushRecord, dict[str, Any]]]] = {}
for push, reservation in notifications:
by_destination.setdefault(
str(reservation.get("addr", push.spec.consumer_zmq)), []
).append((push, reservation))
destinations = list(by_destination.items())
def notify(
consumer_zmq: str,
items: list[tuple[ProducerPushRecord, dict[str, Any]]],
) -> None:
result = self._control_client.request(
consumer_zmq,
{
"op": "complete_batch",
"items": [
{
"transfer_id": push.spec.transfer_id,
"reservation_id": reservation["reservation_id"],
}
for push, reservation in items
],
},
)
completions = result.get("items", []) if isinstance(result, dict) else []
if len(completions) != len(items):
raise RuntimeError("Malformed EC completion response")
for (push, _), completion in zip(items, completions):
if not completion.get("completed"):
raise RuntimeError(
f"Unknown EC reservation for mm_hash={push.spec.mm_hash}"
)
def track(index: int, future: Future[None]) -> None:
records = {
push.spec.transfer_id: push for push, _ in destinations[index][1]
}
self._producer_pushes.track_shard_futures(list(records.values()), [future])
self._run_fanout(
[
partial(notify, destination, items)
for destination, items in destinations
],
track,
)
@staticmethod
def _known_reservations(record: ProducerPushRecord) -> list[dict[str, Any]]:
if record.reservations:
return list(record.reservations)
try:
return list(record.reservation_future.result())
except _FanoutError as exc:
return [result for result in exc.results if isinstance(result, dict)]
except Exception:
return []
def _abandon_pushes(self, pushes: list[ProducerPushRecord]) -> None:
"""Release the consumer-side reservations of a batch that failed."""
for push in pushes:
shards = self._known_reservations(push)
if not shards:
# No confirmed topology means there is no safe address to
# cancel. The original push failure remains observable via
# ProducerPushManager.fail below.
continue
try:
self._retry_cancel_reservations(push.spec, shards, record=push)
except _FanoutError:
logger.exception(
"Failed to abandon EC reservations for transfer_id=%s",
push.spec.transfer_id,
)
def _flush_pending_pushes(self) -> None:
self._producer_pushes.submit_batches(
self._io_executor,
self._push_batch,
)
def _bind_push_source(self, tensor: torch.Tensor, mm_hash: str) -> None:
ready_event = None
if tensor.device.type == "cuda":
ready_event = torch.Event()
ready_event.record(torch.accelerator.current_stream(tensor.device))
self._producer_pushes.bind_source(mm_hash, tensor, ready_event)
def _cancel_orphaned_reservation(self, record: ProducerPushRecord) -> None:
resolution_error: Exception | None = None
try:
reservations = self._producer_pushes.resolve_reservations(record)
except Exception as exc:
resolution_error = exc
reservations = self._known_reservations(record)
reservations = [
shard for shard in reservations if not shard.get("cancelled", False)
]
if not reservations:
if resolution_error is not None:
self._producer_pushes.fail([record], resolution_error)
else:
self._producer_pushes.finish_cancel(record)
return
try:
self._retry_cancel_reservations(record.spec, reservations, record=record)
except Exception as cleanup_error:
if resolution_error is not None:
combined_error = RuntimeError(
f"EC reservation resolution failed ({resolution_error}); "
f"cleanup also failed ({cleanup_error})"
)
combined_error.__cause__ = resolution_error
cleanup_error = combined_error
self._producer_pushes.fail([record], cleanup_error)
return
if resolution_error is not None:
self._producer_pushes.fail([record], resolution_error)
else:
self._producer_pushes.finish_cancel(record)
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
if not self.is_producer:
return None, None
for record in self._producer_pushes.cancel_requests(finished_req_ids):
self._producer_pushes.submit_cancel(
record,
self._io_executor,
self._cancel_orphaned_reservation,
)
return None, None
def save_caches(
self, encoder_cache: dict[str, torch.Tensor], mm_hash: str, **kwargs: Any
) -> None:
if not self.is_producer:
return
tensor = encoder_cache[mm_hash]
self._bind_push_source(tensor, mm_hash)
def build_connector_worker_meta(self) -> ECMooncakeWorkerMetadata | None:
if self.is_consumer and not self._is_receiving_rank:
# `loaded` is intersected across reporting ranks, so a stage that
# never loads must not report at all rather than report nothing.
return None
self._collecting_sources = False
self._push_ready.set()
self._flush_pending_pushes()
failures = self._producer_pushes.poll()
for mm_hash, error in failures:
logger.error(
"EC Mooncake async save failed for mm_hash=%s: %s",
mm_hash,
error,
)
reclaimed = self._consumer_memory.drain_reclaimed()
meta = ECMooncakeWorkerMetadata(
loaded=self._completed_loads,
failed_loads=self._failed_loads,
reclaimed=reclaimed,
pending_saves=self._producer_pushes.pending,
failed_saves=self._failed_saves,
)
self._failed_saves = set()
self._completed_loads = set()
self._failed_loads = set()
return meta
def close(self) -> None:
if self._shutdown:
return
self._shutdown = True
dispatcher = getattr(self, "_dispatcher", None)
if dispatcher is not None:
self._dispatch_stop.set()
self._push_ready.set()
dispatcher.join()
if self._control_server is not None:
self._reservations.begin_shutdown()
self._flush_pending_pushes()
for record in self._producer_pushes.cancel_requests(None):
self._producer_pushes.submit_cancel(
record,
self._io_executor,
self._cancel_orphaned_reservation,
)
self._control_executor.shutdown(wait=True)
self._producer_pushes.submit_batches(
self._io_executor, self._push_batch, wait=True
)
self._io_executor.shutdown(wait=True)
if self._shard_pool is not None:
self._shard_pool.shutdown(wait=True)
# Every producer-side thread that could hold a control socket is stopped.
self._control_client.close()
drained = True
if self._control_server is not None:
drained = self._reservations.wait_for_writers(
self._shutdown_drain_timeout_s
)
self._control_server.close()
if drained:
self._consumer_memory.close()
else:
logger.error(
"Timed out waiting for Mooncake EC writers; keeping the consumer "
"receive pool registered"
)
self._producer_memory.close()
self._transfer.close()
@@ -0,0 +1,130 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Encoder-cache connector backed by Mooncake TransferEngine."""
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorBase,
ECConnectorMetadata,
ECConnectorRole,
ECConnectorWorkerMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
ECMooncakeConnectorMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.scheduler import (
ECMooncakeScheduler,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.worker import ECMooncakeWorker
if TYPE_CHECKING:
import torch
from vllm.config import VllmConfig
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import ECConnectorOutput
from vllm.v1.request import Request
class ECMooncakeConnector(ECConnectorBase):
"""Preserve the public API while delegating to one process-role component."""
def __init__(self, vllm_config: VllmConfig, role: ECConnectorRole):
super().__init__(vllm_config=vllm_config, role=role)
self._scheduler: ECMooncakeScheduler | None = None
self._worker: ECMooncakeWorker | None = None
self._closed = False
if role == ECConnectorRole.SCHEDULER:
self._scheduler = ECMooncakeScheduler(vllm_config)
elif role == ECConnectorRole.WORKER:
self._worker = ECMooncakeWorker(vllm_config)
else:
raise ValueError(f"Unknown EC connector role: {role}")
def start_worker_services(self) -> None:
assert self._worker is not None
self._worker.start_services()
def start_save_caches(self, **kwargs: Any) -> None:
assert self._worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, ECMooncakeConnectorMetadata)
self._worker.start_save_caches(metadata, **kwargs)
def start_load_caches(
self, encoder_cache: dict[str, torch.Tensor], **kwargs: Any
) -> None:
assert self._worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, ECMooncakeConnectorMetadata)
self._worker.start_load_caches(metadata, encoder_cache, **kwargs)
def save_caches(
self, encoder_cache: dict[str, torch.Tensor], mm_hash: str, **kwargs: Any
) -> None:
assert self._worker is not None
self._worker.save_caches(encoder_cache, mm_hash, **kwargs)
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
assert self._worker is not None
return self._worker.get_finished(finished_req_ids)
def build_connector_worker_meta(self) -> ECConnectorWorkerMetadata | None:
assert self._worker is not None
return self._worker.build_connector_worker_meta()
def take_unavailable_requests(self) -> set[str]:
assert self._scheduler is not None
return self._scheduler.take_unavailable_requests()
def has_cache_item(self, identifier: str) -> bool:
assert self._scheduler is not None
return self._scheduler.has_cache_item(identifier)
def ensure_cache_available(
self, request: Request, num_computed_tokens: int
) -> bool:
assert self._scheduler is not None
return self._scheduler.ensure_cache_available(request, num_computed_tokens)
def update_state_after_alloc(self, request: Request, index: int) -> None:
assert self._scheduler is not None
self._scheduler.update_state_after_alloc(request, index)
def update_state_after_free(self, request: Request, index: int) -> None:
assert self._scheduler is not None
self._scheduler.update_state_after_free(request, index)
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> ECConnectorMetadata:
assert self._scheduler is not None
return self._scheduler.build_connector_meta(scheduler_output)
def update_connector_output(self, connector_output: ECConnectorOutput) -> None:
assert self._scheduler is not None
self._scheduler.update_connector_output(connector_output)
def has_pending_push_work(self) -> bool:
assert self._scheduler is not None
return self._scheduler.has_pending_push_work()
def request_finished(self, request: Request) -> tuple[bool, dict[str, Any] | None]:
assert self._scheduler is not None
return self._scheduler.request_finished(request)
def shutdown(self) -> None:
if self._closed:
return
self._closed = True
if self._scheduler is not None:
self._scheduler.close()
if self._worker is not None:
self._worker.close()
+41 -10
View File
@@ -616,6 +616,17 @@ class Scheduler(SchedulerInterface):
req_index += 1
continue
if (
self.ec_connector is not None
and request.mm_features
and not self.ec_connector.ensure_cache_available(
request,
request.num_computed_tokens - request.num_output_placeholders,
)
):
req_index += 1
continue
num_new_tokens = (
request.num_tokens_with_spec
+ request.num_output_placeholders
@@ -945,13 +956,7 @@ class Scheduler(SchedulerInterface):
assert num_computed_tokens <= request.num_tokens
# Skip request with pending mm encoding prefetches
if (
self.ec_connector is not None
and request.mm_features
and not self.ec_connector.ensure_cache_available(
request, num_computed_tokens
)
):
if self._ec_transfer_pending(request, num_computed_tokens):
request_queue.pop_request()
step_skipped_waiting.prepend_request(request)
continue
@@ -966,11 +971,18 @@ class Scheduler(SchedulerInterface):
)
else:
# KVTransfer: WAITING reqs have num_computed_tokens > 0
# after async KV recvs are completed.
# after async KV recvs are completed. A streaming-input
# session resumes here too, carrying whatever media its
# latest chunk added, so this branch needs the same gate.
new_computed_blocks = self.kv_cache_manager.empty_kv_cache_blocks
num_new_local_computed_tokens = 0
num_computed_tokens = request.num_computed_tokens
if self._ec_transfer_pending(request, num_computed_tokens):
request_queue.pop_request()
step_skipped_waiting.prepend_request(request)
continue
encoder_inputs_to_schedule = None
external_load_encoder_input = []
new_encoder_compute_budget = encoder_compute_budget
@@ -2162,6 +2174,10 @@ class Scheduler(SchedulerInterface):
self.grammar_compile_error_reqs.clear()
if failed_kv_load_req_ids and not self.recompute_kv_load_failures:
error_req_ids.update(failed_kv_load_req_ids)
if self.ec_connector is not None:
# An encoder input the connector can no longer obtain. Failing is
# retryable: re-issuing the request re-runs the encode.
error_req_ids.update(self.ec_connector.take_unavailable_requests())
if error_req_ids:
error_reqs = self.finish_requests(
@@ -2258,6 +2274,16 @@ class Scheduler(SchedulerInterface):
return engine_core_outputs
def _ec_transfer_pending(self, request: Request, num_computed_tokens: int) -> bool:
"""Whether an encoder input this request needs is still in transit."""
return (
self.ec_connector is not None
and bool(request.mm_features)
and not self.ec_connector.ensure_cache_available(
request, num_computed_tokens
)
)
@staticmethod
def _is_blocked_waiting_status(status: RequestStatus) -> bool:
return status in (
@@ -2344,7 +2370,7 @@ class Scheduler(SchedulerInterface):
# With Whisper, as soon as we've generated a single token,
# we know we're done with the encoder input. Cross Attention
# KVs have been calculated and cached already.
self.encoder_cache_manager.free_encoder_input(request, input_id)
self._free_encoder_input(request, input_id)
elif (
start_pos + num_tokens + spec_lookahead
<= request.num_computed_tokens - request.num_output_placeholders
@@ -2352,7 +2378,12 @@ class Scheduler(SchedulerInterface):
# Processed, stored in the decoder KV cache, and far enough past
# the placeholder range (plus the drafter's look-ahead) that no
# rejection or drafter gather can reference it.
self.encoder_cache_manager.free_encoder_input(request, input_id)
self._free_encoder_input(request, input_id)
def _free_encoder_input(self, request: Request, input_id: int) -> None:
self.encoder_cache_manager.free_encoder_input(request, input_id)
if self.ec_connector is not None:
self.ec_connector.update_state_after_free(request, input_id)
def update_draft_token_ids(self, draft_token_ids: DraftTokenIds) -> None:
for req_id, spec_token_ids in zip(
@@ -64,6 +64,12 @@ class ECConnectorModelRunnerMixin:
assert scheduler_output.ec_connector_metadata is not None
ec_connector.bind_connector_metadata(scheduler_output.ec_connector_metadata)
if ec_connector.is_producer:
# Pass the cache explicitly: a producer needs it to publish an item
# that is already cached from an earlier step, which this step will
# not re-encode -- `save_caches` never fires for those.
ec_connector.start_save_caches(encoder_cache=encoder_cache, **kwargs)
# Load caches for consumer or both roles
if ec_connector.is_consumer:
ec_connector.start_load_caches(encoder_cache, **kwargs)
@@ -74,5 +80,6 @@ class ECConnectorModelRunnerMixin:
output.finished_sending, output.finished_recving = (
ec_connector.get_finished(scheduler_output.finished_req_ids)
)
output.ec_connector_worker_meta = ec_connector.build_connector_worker_meta()
ec_connector.clear_connector_metadata()
+3
View File
@@ -62,6 +62,9 @@ class ActiveECConnector(ECConnector):
assert scheduler_output.ec_connector_metadata is not None
ec_connector.bind_connector_metadata(scheduler_output.ec_connector_metadata)
if ec_connector.is_producer:
ec_connector.start_save_caches(encoder_cache=self.encoder_cache)
if ec_connector.is_consumer:
ec_connector.start_load_caches(self.encoder_cache)
+5
View File
@@ -28,6 +28,8 @@ from vllm.distributed import (
from vllm.distributed.ec_transfer import (
ensure_ec_transfer_initialized,
ensure_ec_transfer_shutdown,
get_ec_transfer,
has_ec_transfer,
)
from vllm.distributed.eplb.eplb_utils import override_envs_for_eplb
from vllm.distributed.kv_transfer import (
@@ -501,6 +503,9 @@ class Worker(WorkerBase):
):
self.model_runner.load_model(load_dummy_weights=load_dummy_weights)
if has_ec_transfer():
get_ec_transfer().start_worker_services()
if self.vllm_config.weight_transfer_config is not None:
self.weight_transfer_engine = WeightTransferEngineFactory.create_engine(
self.vllm_config.weight_transfer_config,