[EPD] Add ECMooncakeConnector for encoder cache over Mooncake TransferEngine (#41567)
Signed-off-by: Teng Ma <sima.mt@alibaba-inc.com> Signed-off-by: Tianyu Guo <guoty@inferact.ai> Signed-off-by: Zhou ziheng <jiaranran2@gmail.com> Signed-off-by: jiangkuaixue123 <jiangxiaozhou111@163.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Tianyu Guo <guoty@inferact.ai> Co-authored-by: Zhou ziheng <jiaranran2@gmail.com> Co-authored-by: jiangkuaixue123 <jiangxiaozhou111@163.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Cursor
Tianyu Guo
Zhou ziheng
jiangkuaixue123
OpenAI Codex
parent
f7f060d253
commit
6fbb00b188
@@ -19,3 +19,57 @@ steps:
|
||||
commands:
|
||||
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
|
||||
- bash v1/kv_connector/mooncake_integration/config_sweep_accuracy_test.sh
|
||||
|
||||
- label: ":nvidia: (L4) Mooncake EC TCP E2E"
|
||||
key: mooncake-ec-tcp-e2e-2-gpus
|
||||
timeout_in_minutes: 30
|
||||
working_dir: "/vllm-workspace"
|
||||
device: l4
|
||||
num_devices: 2
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/ec_transfer/
|
||||
- vllm/config/ec_transfer.py
|
||||
- vllm/config/multimodal.py
|
||||
- vllm/config/vllm.py
|
||||
- vllm/multimodal/
|
||||
- vllm/v1/core/encoder_cache_manager.py
|
||||
- vllm/v1/core/sched/
|
||||
- vllm/v1/engine/
|
||||
- vllm/v1/worker/ec_connector_model_runner_mixin.py
|
||||
- vllm/v1/worker/gpu_model_runner.py
|
||||
- vllm/v1/worker/gpu_worker.py
|
||||
- vllm/v1/worker/gpu/ec_connector.py
|
||||
- vllm/v1/worker/gpu/model_runner.py
|
||||
- vllm/v1/worker/gpu/mm/
|
||||
- examples/disaggregated/disaggregated_encoder/
|
||||
- tests/v1/ec_connector/
|
||||
- requirements/kv_connectors.txt
|
||||
env:
|
||||
PYTHON_BIN: "/vllm-workspace/.venv/bin/python"
|
||||
PYTHONUNBUFFERED: "1"
|
||||
VLLM_HOST_IP: "127.0.0.1"
|
||||
VLLM_USE_V2_MODEL_RUNNER: "1"
|
||||
MOONCAKE_EC_PROTOCOL: "tcp"
|
||||
USE_MM_PROMPTS: "1"
|
||||
SKIP_BASELINE: "0"
|
||||
CONCURRENCY: "3"
|
||||
REPEAT: "2"
|
||||
LOG_PATH: "/tmp/mooncake-ec-e2e"
|
||||
BASELINE_FILE: "/tmp/mooncake-ec-e2e/baseline.json"
|
||||
commands:
|
||||
- uv venv --system-site-packages --python 3.12 .venv
|
||||
- |
|
||||
uv pip install --python .venv/bin/python "$(.venv/bin/python - <<'PY'
|
||||
from pathlib import Path
|
||||
import torch
|
||||
|
||||
requirement = next(
|
||||
line for line in Path("requirements/kv_connectors.txt").read_text().splitlines()
|
||||
if line.startswith("mooncake-transfer-engine ")
|
||||
)
|
||||
if torch.version.cuda.split(".")[0] == "13":
|
||||
requirement = requirement.replace("mooncake-transfer-engine", "mooncake-transfer-engine-cuda13")
|
||||
print(requirement)
|
||||
PY
|
||||
)"
|
||||
- bash tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh
|
||||
|
||||
@@ -23,6 +23,7 @@ import argparse
|
||||
import asyncio
|
||||
import hashlib
|
||||
import io
|
||||
import itertools
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -30,6 +31,7 @@ import random
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
import pybase64 as base64
|
||||
@@ -41,9 +43,7 @@ from fastapi.responses import JSONResponse, StreamingResponse
|
||||
# FastAPI app & global state
|
||||
###############################################################################
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG, format="%(asctime)s %(levelname)s: %(message)s"
|
||||
)
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s: %(message)s")
|
||||
logger = logging.getLogger("proxy")
|
||||
|
||||
app = FastAPI()
|
||||
@@ -87,9 +87,24 @@ def encoder_rr_assignment(
|
||||
return urls, next_start
|
||||
|
||||
|
||||
def validate_ec_consumer_routing(
|
||||
prefill_urls: list[str], consumer_addrs: list[str]
|
||||
) -> None:
|
||||
"""Reject the topology whose EC destination cannot be routed safely."""
|
||||
if prefill_urls and consumer_addrs:
|
||||
raise ValueError(
|
||||
"Mooncake EC consumer routing supports E+PD only; disable independent "
|
||||
"prefill or omit --ec-consumer-zmq-addrs."
|
||||
)
|
||||
|
||||
|
||||
# Diagnostic switch: forward the original request to the decoder so the
|
||||
# only difference from the rewrite path is the rewrite itself.
|
||||
NO_REWRITE = False
|
||||
# Decode-side retries for a retryable internal error (`finish_reason="error"`,
|
||||
# e.g. an encoder embedding the connector could not deliver). Re-issuing runs
|
||||
# the encode again, which produces a fresh transfer.
|
||||
DECODE_RETRIES = 1
|
||||
|
||||
|
||||
# Grid metadata reported by the encoder instance, keyed by item index.
|
||||
@@ -136,6 +151,7 @@ def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict:
|
||||
here -- a second derivation could disagree with the encoder's.
|
||||
"""
|
||||
rewritten = 0
|
||||
transfer_items = []
|
||||
idx = 0
|
||||
new_messages = []
|
||||
for msg in req_data.get("messages", []):
|
||||
@@ -151,26 +167,42 @@ def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict:
|
||||
meta = dict(item_meta.get(idx) or {})
|
||||
idx += 1
|
||||
item_uuid = meta.pop("mm_hash", None)
|
||||
ec_mm_hash = meta.pop("ec_mm_hash", None) or item_uuid
|
||||
transfer_id = meta.pop("transfer_id", None)
|
||||
# Whatever keys the encoder reported are the metadata its model
|
||||
# declared as needed to size the placeholder range; the proxy does
|
||||
# not need to know their names.
|
||||
metadata = {k: _b64_tensor(v) for k, v in meta.items()}
|
||||
if not metadata or not item_uuid:
|
||||
# The encoder reported no metadata (e.g. the item came from its
|
||||
# processor cache); let the decoder process the media itself.
|
||||
# Nothing to size the placeholder range with. A processor cache
|
||||
# hit is not a cause on its own: with the default `lru` type the
|
||||
# engine restores the item before the scheduler reports it. It
|
||||
# goes missing when the encode request failed, or under
|
||||
# `--mm-processor-cache-type shm`, where a hit replaces the item
|
||||
# with its shared-memory address and only the worker restores
|
||||
# it. Send the media so the decoder can derive the grid itself.
|
||||
new_content.append(item)
|
||||
continue
|
||||
embeds_type = EMBEDS_TYPES[item["type"]]
|
||||
new_content.append(
|
||||
{"type": embeds_type, embeds_type: metadata, "uuid": item_uuid}
|
||||
)
|
||||
if transfer_id is not None:
|
||||
transfer_items.append(
|
||||
{"mm_hash": ec_mm_hash, "transfer_id": transfer_id}
|
||||
)
|
||||
rewritten += 1
|
||||
new_messages.append({**msg, "content": new_content})
|
||||
|
||||
if not rewritten:
|
||||
return req_data
|
||||
logger.info("Rewrote %d media item(s) as metadata references", rewritten)
|
||||
return {**req_data, "messages": new_messages}
|
||||
rewritten_request = {**req_data, "messages": new_messages}
|
||||
if transfer_items:
|
||||
ec_transfer_params = dict(req_data.get("ec_transfer_params") or {})
|
||||
ec_transfer_params["ec_items"] = transfer_items
|
||||
rewritten_request["ec_transfer_params"] = ec_transfer_params
|
||||
return rewritten_request
|
||||
|
||||
|
||||
def extract_mm_items(request_data: dict) -> list[dict]:
|
||||
@@ -196,7 +228,8 @@ async def fanout_encoder_primer(
|
||||
orig_request: dict,
|
||||
e_urls: list[str],
|
||||
req_id: str,
|
||||
) -> dict[int, dict]:
|
||||
consumer_zmq: str | None = None,
|
||||
) -> tuple[dict[int, dict], dict[str, Any]]:
|
||||
"""
|
||||
1. Build one request *per MM item* with all text removed.
|
||||
2. Send them concurrently to the encode cluster.
|
||||
@@ -206,19 +239,25 @@ async def fanout_encoder_primer(
|
||||
`ec_transfer_params`: its EC cache key and the grid its processor produced.
|
||||
The proxy still supplies the uuid so both sides key the cache the same way;
|
||||
the grid can only come from the encoder, which is the side that computed it.
|
||||
|
||||
Also returns the connector handles to put on the decode body, as a fresh
|
||||
mapping. `orig_request` is left untouched so a retry re-encodes from the
|
||||
original request instead of carrying the previous attempt's handles.
|
||||
"""
|
||||
logger.info("[%s] Processing multimodal items...", req_id)
|
||||
|
||||
mm_items = extract_mm_items(orig_request)
|
||||
if not mm_items:
|
||||
logger.info("[%s] No multimodal items, skipping encoder", req_id)
|
||||
return {} # nothing to do
|
||||
return {}, {} # nothing to do
|
||||
|
||||
logger.info("[%s] got %d multimodal items...", req_id, len(mm_items))
|
||||
|
||||
tasks = []
|
||||
item_uuids: dict[int, str] = {}
|
||||
item_transfer_ids: dict[int, str] = {}
|
||||
item_meta: dict[int, dict] = {}
|
||||
ec_params: dict[str, Any] = {}
|
||||
|
||||
# Round-robin over encode servers to distribute load a bit. The cursor
|
||||
# persists across requests so fan-out doesn't restart at e_urls[0] every
|
||||
@@ -241,6 +280,8 @@ async def fanout_encoder_primer(
|
||||
item_uuid = None if NO_REWRITE else content_uuid(item)
|
||||
if item_uuid is not None:
|
||||
item_uuids[idx] = item_uuid
|
||||
transfer_id = uuid.uuid4().hex
|
||||
item_transfer_ids[idx] = transfer_id
|
||||
|
||||
encoder_req = {
|
||||
# You *may* need to keep additional fields
|
||||
@@ -257,6 +298,19 @@ async def fanout_encoder_primer(
|
||||
# once the prompt is encoded and its embeddings are published.
|
||||
"stream": False,
|
||||
}
|
||||
if consumer_zmq is not None:
|
||||
# No mm_hash here on purpose. The encoder's own
|
||||
# `mm_features[i].identifier` is derived from the uuid *and* the
|
||||
# engine's media_io_kwargs / mm_processor_kwargs, so this proxy
|
||||
# cannot know it before the encoder runs. Sending the bare uuid
|
||||
# would fail the connector's hash match and make the producer
|
||||
# invent its own transfer id, which the consumer could then never
|
||||
# cancel. Omitting it lets the connector match by position, which
|
||||
# is exact: one encoder request carries exactly one item.
|
||||
encoder_req["ec_transfer_params"] = {
|
||||
"consumer_zmq": consumer_zmq,
|
||||
"ec_items": [{"transfer_id": transfer_id}],
|
||||
}
|
||||
tasks.append(
|
||||
encode_session.post(
|
||||
f"{target_url}/v1/chat/completions",
|
||||
@@ -304,24 +358,42 @@ async def fanout_encoder_primer(
|
||||
except Exception:
|
||||
logger.warning("[%s] Could not read encoder metadata #%d", req_id, idx)
|
||||
params = {}
|
||||
if idx in item_uuids:
|
||||
reported = params.get(item_uuids[idx])
|
||||
if params:
|
||||
# One encoder request carries exactly one item, so there is a
|
||||
# single reported entry. Do not key it by this proxy's uuid: when
|
||||
# media_io_kwargs or mm_processor_kwargs are set the engine
|
||||
# re-hashes the uuid together with them, so the encoder's own
|
||||
# `mm_features[i].identifier` is a derived value this proxy cannot
|
||||
# predict. Fall back to the sole entry, and carry the key the
|
||||
# encoder actually used through as `ec_mm_hash`.
|
||||
ec_mm_hash = item_uuids.get(idx)
|
||||
reported = params.get(ec_mm_hash)
|
||||
if reported is None and len(params) == 1:
|
||||
((ec_mm_hash, reported),) = params.items()
|
||||
if reported:
|
||||
metadata = reported.get("metadata") or {}
|
||||
if metadata:
|
||||
item_meta[idx] = {**metadata, "mm_hash": item_uuids[idx]}
|
||||
item_meta[idx] = {
|
||||
**metadata,
|
||||
"mm_hash": item_uuids.get(idx, ec_mm_hash),
|
||||
"ec_mm_hash": ec_mm_hash,
|
||||
}
|
||||
if idx in item_transfer_ids:
|
||||
item_meta[idx]["transfer_id"] = item_transfer_ids[idx]
|
||||
# Whatever the encoder reported alongside `metadata` is the
|
||||
# connector's own handle on the published embedding (for NIXL,
|
||||
# peer_host/peer_port/size_bytes). The decoder's connector
|
||||
# looks it up by mm_hash on the request, so carry it through.
|
||||
orig_request.setdefault("ec_transfer_params", {})[item_uuids[idx]] = (
|
||||
reported
|
||||
)
|
||||
ec_params[item_uuids.get(idx, ec_mm_hash)] = reported
|
||||
if NO_REWRITE and consumer_zmq is not None:
|
||||
ec_params.setdefault("ec_items", []).append(
|
||||
{"mm_hash": ec_mm_hash, "transfer_id": item_transfer_ids[idx]}
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"[%s] All %d encoder requests completed successfully", req_id, len(mm_items)
|
||||
)
|
||||
return item_meta
|
||||
return item_meta, ec_params
|
||||
|
||||
|
||||
async def maybe_prefill(
|
||||
@@ -331,8 +403,11 @@ async def maybe_prefill(
|
||||
) -> dict:
|
||||
"""
|
||||
- Do prefill-only task if p_url exist;
|
||||
- Return modified request data with kv transfer params (for nixl connector)
|
||||
- Return a new body carrying kv transfer params (for nixl connector)
|
||||
- Else, skip and return the original request data for decode
|
||||
|
||||
`req_data` is never mutated: a decode retry re-enters this function with the
|
||||
same body, and one attempt's `remote_block_ids` must not reach the next.
|
||||
"""
|
||||
if p_url:
|
||||
logger.info("[%s] Processing through prefill: %s", req_id, p_url)
|
||||
@@ -342,11 +417,9 @@ async def maybe_prefill(
|
||||
prefill_response_json = await prefill_response.json()
|
||||
kv_transfer_params = prefill_response_json.get("kv_transfer_params", {})
|
||||
if kv_transfer_params:
|
||||
req_data["kv_transfer_params"] = kv_transfer_params
|
||||
return {**req_data, "kv_transfer_params": kv_transfer_params}
|
||||
|
||||
return req_data
|
||||
else:
|
||||
return req_data
|
||||
return req_data
|
||||
|
||||
|
||||
async def process_prefill_stage(
|
||||
@@ -409,7 +482,6 @@ async def process_prefill_stage(
|
||||
###############################################################################
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def log_requests(request: Request, call_next):
|
||||
"""Middleware to log all incoming requests and responses"""
|
||||
req_id = request.headers.get("x-request-id", str(uuid.uuid4()))
|
||||
@@ -458,7 +530,20 @@ async def log_requests(request: Request, call_next):
|
||||
async def on_startup() -> None:
|
||||
global encode_session, prefill_session, decode_session
|
||||
timeout = aiohttp.ClientTimeout(total=100_000)
|
||||
connector = aiohttp.TCPConnector(limit=0, force_close=False)
|
||||
# vLLM closes an idle keep-alive connection after
|
||||
# VLLM_HTTP_TIMEOUT_KEEP_ALIVE seconds (5 by default), while aiohttp keeps
|
||||
# pooling it for 15. Reusing one it has already closed fails the request
|
||||
# with ServerDisconnectedError, and the server logs nothing at all: it
|
||||
# closed the socket before the request arrived. Retire ours first.
|
||||
server_keep_alive = float(os.getenv("VLLM_HTTP_TIMEOUT_KEEP_ALIVE", "5"))
|
||||
connector = aiohttp.TCPConnector(
|
||||
limit=0,
|
||||
**(
|
||||
{"keepalive_timeout": server_keep_alive / 2}
|
||||
if server_keep_alive > 0
|
||||
else {"force_close": True}
|
||||
),
|
||||
)
|
||||
encode_session = aiohttp.ClientSession(timeout=timeout, connector=connector)
|
||||
if app.state.p_urls:
|
||||
# only setup if prefill instance(s) exist
|
||||
@@ -482,40 +567,97 @@ async def on_shutdown() -> None:
|
||||
###############################################################################
|
||||
|
||||
|
||||
async def prepare_for_decode(
|
||||
req_data: dict,
|
||||
req_id: str,
|
||||
e_urls: list[str],
|
||||
p_url: str,
|
||||
consumer_zmq: str | None,
|
||||
) -> tuple[dict, float, float]:
|
||||
"""Encode, rewrite and prefill, returning the body to send to decode.
|
||||
|
||||
`req_data` is left untouched so a retry starts from the original media
|
||||
rather than from a body whose images are already metadata references.
|
||||
"""
|
||||
_t0 = time.perf_counter()
|
||||
item_meta, ec_params = await fanout_encoder_primer(
|
||||
req_data, e_urls, req_id, consumer_zmq
|
||||
)
|
||||
_t1 = time.perf_counter()
|
||||
prepared = req_data if NO_REWRITE else rewrite_for_decode(req_data, item_meta)
|
||||
if ec_params:
|
||||
# A fresh body every time: `rewrite_for_decode` hands back `req_data`
|
||||
# itself when it rewrote nothing, and this attempt's handles must not
|
||||
# outlive it into a retry.
|
||||
handles = dict(prepared.get("ec_transfer_params") or {})
|
||||
handles.update(ec_params)
|
||||
prepared = {**prepared, "ec_transfer_params": handles}
|
||||
_t2 = time.perf_counter()
|
||||
prepared = await maybe_prefill(prepared, p_url, req_id)
|
||||
return prepared, _t1 - _t0, _t2 - _t1
|
||||
|
||||
|
||||
async def forward_non_stream(
|
||||
req_data: dict, req_id: str, e_urls: list[str], p_url: str, d_url: str
|
||||
req_data: dict,
|
||||
req_id: str,
|
||||
e_urls: list[str],
|
||||
p_url: str,
|
||||
d_url: str,
|
||||
consumer_zmq: str | None,
|
||||
dp_rank: int | None = None,
|
||||
) -> dict:
|
||||
try:
|
||||
# Step 1: Process through Encoder instance (if has MM input)
|
||||
_t0 = time.perf_counter()
|
||||
item_meta = await fanout_encoder_primer(req_data, e_urls, req_id)
|
||||
_t1 = time.perf_counter()
|
||||
req_data = req_data if NO_REWRITE else rewrite_for_decode(req_data, item_meta)
|
||||
_t2 = time.perf_counter()
|
||||
|
||||
# Step 2: Process through Prefill instance
|
||||
req_data = await maybe_prefill(req_data, p_url, req_id)
|
||||
|
||||
# Step 3: Process through Decode instance
|
||||
logger.info("[%s] Forwarding to decode: %s", req_id, d_url)
|
||||
headers = {"x-request-id": req_id}
|
||||
|
||||
# Non-streaming response
|
||||
async with decode_session.post(
|
||||
f"{d_url}/v1/chat/completions", json=req_data, headers=headers
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
out = await resp.json()
|
||||
_t3 = time.perf_counter()
|
||||
logger.info(
|
||||
"STAGE %s encode=%.1f rewrite=%.1f decode=%.1f total=%.1f",
|
||||
"no-rewrite" if NO_REWRITE else "rewrite",
|
||||
(_t1 - _t0) * 1e3,
|
||||
(_t2 - _t1) * 1e3,
|
||||
(_t3 - _t2) * 1e3,
|
||||
(_t3 - _t0) * 1e3,
|
||||
for attempt in range(DECODE_RETRIES + 1):
|
||||
_t0 = time.perf_counter()
|
||||
prepared, encode_s, rewrite_s = await prepare_for_decode(
|
||||
req_data, req_id, e_urls, p_url, consumer_zmq
|
||||
)
|
||||
return out
|
||||
_t2 = time.perf_counter()
|
||||
|
||||
logger.info("[%s] Forwarding to decode: %s", req_id, d_url)
|
||||
headers = {"x-request-id": req_id}
|
||||
if dp_rank is not None:
|
||||
headers["X-data-parallel-rank"] = str(dp_rank)
|
||||
|
||||
async with decode_session.post(
|
||||
f"{d_url}/v1/chat/completions", json=prepared, headers=headers
|
||||
) as resp:
|
||||
if resp.status >= 400:
|
||||
detail = await resp.text()
|
||||
# 500 is the decoder's retryable internal error, which
|
||||
# includes an encoder embedding it could not obtain. Redoing
|
||||
# the encode publishes the item again.
|
||||
if resp.status == 500 and attempt < DECODE_RETRIES:
|
||||
logger.warning(
|
||||
"[%s] Decode returned 500, re-encoding and retrying "
|
||||
"(attempt %d/%d): %s",
|
||||
req_id,
|
||||
attempt + 1,
|
||||
DECODE_RETRIES,
|
||||
detail[:200],
|
||||
)
|
||||
continue
|
||||
logger.error(
|
||||
"[%s] Decode request returned status %s: %s",
|
||||
req_id,
|
||||
resp.status,
|
||||
detail,
|
||||
)
|
||||
raise HTTPException(status_code=resp.status, detail=detail)
|
||||
out = await resp.json()
|
||||
_t3 = time.perf_counter()
|
||||
logger.info(
|
||||
"STAGE %s encode=%.1f rewrite=%.1f decode=%.1f total=%.1f "
|
||||
"attempt=%d",
|
||||
"no-rewrite" if NO_REWRITE else "rewrite",
|
||||
encode_s * 1e3,
|
||||
rewrite_s * 1e3,
|
||||
(_t3 - _t2) * 1e3,
|
||||
(_t3 - _t0) * 1e3,
|
||||
attempt,
|
||||
)
|
||||
return out
|
||||
raise HTTPException(status_code=500, detail="Decode failed after re-encoding")
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
@@ -525,47 +667,66 @@ async def forward_non_stream(
|
||||
|
||||
|
||||
async def forward_stream(
|
||||
req_data: dict, req_id: str, e_urls: list[str], p_url: str, d_url: str
|
||||
req_data: dict,
|
||||
req_id: str,
|
||||
e_urls: list[str],
|
||||
p_url: str,
|
||||
d_url: str,
|
||||
consumer_zmq: str | None,
|
||||
dp_rank: int | None = None,
|
||||
) -> AsyncIterator[str]:
|
||||
try:
|
||||
# Step 1: Process through Encoder instance (if has MM input)
|
||||
_t0 = time.perf_counter()
|
||||
item_meta = await fanout_encoder_primer(req_data, e_urls, req_id)
|
||||
_t1 = time.perf_counter()
|
||||
req_data = req_data if NO_REWRITE else rewrite_for_decode(req_data, item_meta)
|
||||
_t2 = time.perf_counter()
|
||||
for attempt in range(DECODE_RETRIES + 1):
|
||||
_t0 = time.perf_counter()
|
||||
prepared, encode_s, rewrite_s = await prepare_for_decode(
|
||||
req_data, req_id, e_urls, p_url, consumer_zmq
|
||||
)
|
||||
_t2 = time.perf_counter()
|
||||
|
||||
# Step 2: Process through Prefill instance
|
||||
req_data = await maybe_prefill(req_data, p_url, req_id)
|
||||
logger.info("[%s] Starting streaming from decode: %s", req_id, d_url)
|
||||
headers = {"x-request-id": req_id}
|
||||
if dp_rank is not None:
|
||||
headers["X-data-parallel-rank"] = str(dp_rank)
|
||||
|
||||
# Step 3: Process through Decode instance
|
||||
logger.info("[%s] Starting streaming from decode: %s", req_id, d_url)
|
||||
headers = {"x-request-id": req_id}
|
||||
_first = None
|
||||
async with decode_session.post(
|
||||
f"{d_url}/v1/chat/completions",
|
||||
json=prepared,
|
||||
headers=headers,
|
||||
) as resp:
|
||||
# Retry only before the first chunk: once anything reached the
|
||||
# client the response cannot be replaced.
|
||||
if resp.status == 500 and attempt < DECODE_RETRIES:
|
||||
detail = await resp.text()
|
||||
logger.warning(
|
||||
"[%s] Decode returned 500 before streaming, re-encoding "
|
||||
"and retrying (attempt %d/%d): %s",
|
||||
req_id,
|
||||
attempt + 1,
|
||||
DECODE_RETRIES,
|
||||
detail[:200],
|
||||
)
|
||||
continue
|
||||
resp.raise_for_status()
|
||||
async for chunk in resp.content.iter_chunked(1024):
|
||||
if chunk:
|
||||
if _first is None:
|
||||
_first = time.perf_counter()
|
||||
yield chunk.decode("utf-8", errors="ignore")
|
||||
_t3 = time.perf_counter()
|
||||
|
||||
# Streaming response
|
||||
_first = None
|
||||
async with decode_session.post(
|
||||
f"{d_url}/v1/chat/completions",
|
||||
json=req_data,
|
||||
headers=headers,
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
async for chunk in resp.content.iter_chunked(1024):
|
||||
if chunk:
|
||||
if _first is None:
|
||||
_first = time.perf_counter()
|
||||
yield chunk.decode("utf-8", errors="ignore")
|
||||
_t3 = time.perf_counter()
|
||||
|
||||
logger.info(
|
||||
"STAGE %s encode=%.1f rewrite=%.2f decode_ttfb=%.1f decode_total=%.1f",
|
||||
"no-rewrite" if NO_REWRITE else "rewrite",
|
||||
(_t1 - _t0) * 1e3,
|
||||
(_t2 - _t1) * 1e3,
|
||||
((_first or _t3) - _t2) * 1e3,
|
||||
(_t3 - _t2) * 1e3,
|
||||
)
|
||||
logger.info("[%s] Streaming completed", req_id)
|
||||
logger.info(
|
||||
"STAGE %s encode=%.1f rewrite=%.2f decode_ttfb=%.1f "
|
||||
"decode_total=%.1f attempt=%d",
|
||||
"no-rewrite" if NO_REWRITE else "rewrite",
|
||||
encode_s * 1e3,
|
||||
rewrite_s * 1e3,
|
||||
((_first or _t3) - _t2) * 1e3,
|
||||
(_t3 - _t2) * 1e3,
|
||||
attempt,
|
||||
)
|
||||
logger.info("[%s] Streaming completed", req_id)
|
||||
return
|
||||
|
||||
except HTTPException:
|
||||
logger.exception("[%s] HTTPException in forward_stream", req_id)
|
||||
@@ -590,16 +751,29 @@ async def chat_completions(request: Request):
|
||||
|
||||
e_urls = app.state.e_urls # we want the full list for fan-out
|
||||
p_url = random.choice(app.state.p_urls) if app.state.p_urls else None
|
||||
d_url = random.choice(app.state.d_urls)
|
||||
decode_index = random.randrange(len(app.state.d_urls))
|
||||
d_url = app.state.d_urls[decode_index]
|
||||
dp_size = app.state.ec_consumer_dp_size
|
||||
# Round-robin the replica, then name it to both halves: the decoder
|
||||
# honours the rank header instead of its own balancer, and the encoder
|
||||
# pushes to that replica's control channel. Choosing once here means a
|
||||
# decode retry re-encodes to the same replica.
|
||||
dp_rank = next(app.state.replica_counter) % dp_size if dp_size > 1 else None
|
||||
ec_index = decode_index * dp_size + (dp_rank or 0)
|
||||
consumer_zmq = app.state.d_ec_urls[ec_index] if app.state.d_ec_urls else None
|
||||
|
||||
is_streaming = req_data.get("stream", False)
|
||||
|
||||
if is_streaming:
|
||||
return StreamingResponse(
|
||||
forward_stream(req_data, req_id, e_urls, p_url, d_url),
|
||||
forward_stream(
|
||||
req_data, req_id, e_urls, p_url, d_url, consumer_zmq, dp_rank
|
||||
),
|
||||
media_type="text/event-stream",
|
||||
)
|
||||
result = await forward_non_stream(req_data, req_id, e_urls, p_url, d_url)
|
||||
result = await forward_non_stream(
|
||||
req_data, req_id, e_urls, p_url, d_url, consumer_zmq, dp_rank
|
||||
)
|
||||
return JSONResponse(content=result)
|
||||
|
||||
except HTTPException:
|
||||
@@ -750,6 +924,14 @@ if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--host", default="0.0.0.0")
|
||||
parser.add_argument("--port", type=int, default=8000)
|
||||
parser.add_argument(
|
||||
"--log-requests",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Log every request in and out, and raise the log level to DEBUG. "
|
||||
"Off by default: the proxy is on the request path."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-rewrite",
|
||||
action="store_true",
|
||||
@@ -764,8 +946,8 @@ if __name__ == "__main__":
|
||||
"--prefill-servers-urls",
|
||||
required=True,
|
||||
help=(
|
||||
'Comma-separated prefill URLs ("http://p1:8003,http://p2:8004") ',
|
||||
'to enable E->P->D, set "disable" or "none" to enable E->PD',
|
||||
'Comma-separated prefill URLs ("http://p1:8003,http://p2:8004") '
|
||||
'to enable E->P->D, set "disable" or "none" to enable E->PD'
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
@@ -773,15 +955,64 @@ if __name__ == "__main__":
|
||||
required=True,
|
||||
help='Comma-separated decode URLs ("http://d1:8005,http://d2:8006")',
|
||||
)
|
||||
parser.add_argument(
|
||||
"--decode-retries",
|
||||
type=int,
|
||||
default=1,
|
||||
help=(
|
||||
"Re-encode and re-send when decode returns 500, which is its "
|
||||
"retryable internal error (an undeliverable encoder embedding "
|
||||
"among them). 0 disables."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ec-consumer-zmq-addrs",
|
||||
default="",
|
||||
help=(
|
||||
"Comma-separated Mooncake EC consumer control addresses, aligned "
|
||||
"with --decode-servers-urls. Required for Mooncake EC consumers and "
|
||||
"supported only in E+PD mode. With --ec-consumer-dp-size > 1, list "
|
||||
"each server's replicas consecutively: s0r0,s0r1,s1r0,s1r1."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ec-consumer-dp-size",
|
||||
type=int,
|
||||
default=1,
|
||||
help=(
|
||||
"Data-parallel replicas per EC consumer. The proxy picks a replica "
|
||||
"round-robin and names it to both halves of the request, because an "
|
||||
"encoder push has to land where the request will run."
|
||||
),
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
if args.log_requests:
|
||||
logging.getLogger().setLevel(logging.DEBUG)
|
||||
app.middleware("http")(log_requests)
|
||||
NO_REWRITE = args.no_rewrite
|
||||
DECODE_RETRIES = max(0, args.decode_retries)
|
||||
app.state.e_urls = [
|
||||
u.strip() for u in args.encode_servers_urls.split(",") if u.strip()
|
||||
]
|
||||
app.state.d_urls = [
|
||||
u.strip() for u in args.decode_servers_urls.split(",") if u.strip()
|
||||
]
|
||||
app.state.d_ec_urls = [
|
||||
u.strip() for u in args.ec_consumer_zmq_addrs.split(",") if u.strip()
|
||||
]
|
||||
if args.ec_consumer_dp_size < 1:
|
||||
parser.error("--ec-consumer-dp-size must be at least 1")
|
||||
app.state.ec_consumer_dp_size = args.ec_consumer_dp_size
|
||||
app.state.replica_counter = itertools.count()
|
||||
expected = len(app.state.d_urls) * args.ec_consumer_dp_size
|
||||
if app.state.d_ec_urls and len(app.state.d_ec_urls) != expected:
|
||||
parser.error(
|
||||
"--ec-consumer-zmq-addrs must contain one address per consumer "
|
||||
f"replica: expected {expected} "
|
||||
f"({len(app.state.d_urls)} servers x {args.ec_consumer_dp_size} replicas), "
|
||||
f"got {len(app.state.d_ec_urls)}"
|
||||
)
|
||||
# handle prefill instances
|
||||
if args.prefill_servers_urls.lower() in ("disable", "none", ""):
|
||||
app.state.p_urls = []
|
||||
@@ -793,11 +1024,21 @@ if __name__ == "__main__":
|
||||
u.strip() for u in args.prefill_servers_urls.split(",") if u.strip()
|
||||
]
|
||||
logger.info("Disaggregated prefill phase is enabled. Running E + P + D...")
|
||||
try:
|
||||
validate_ec_consumer_routing(app.state.p_urls, app.state.d_ec_urls)
|
||||
except ValueError as exc:
|
||||
parser.error(str(exc))
|
||||
|
||||
logger.info("Proxy listening on %s:%s", args.host, args.port)
|
||||
logger.info("Encode servers: %s", app.state.e_urls)
|
||||
logger.info("Prefill instances %s", app.state.p_urls)
|
||||
logger.info("Decode servers: %s", app.state.d_urls)
|
||||
if app.state.ec_consumer_dp_size > 1:
|
||||
logger.info(
|
||||
"EC consumer replicas per server: %d (control addresses: %s)",
|
||||
app.state.ec_consumer_dp_size,
|
||||
app.state.d_ec_urls,
|
||||
)
|
||||
|
||||
uvicorn.run(
|
||||
app,
|
||||
|
||||
@@ -167,6 +167,38 @@ def test_get_freed_mm_hashes_clears_freed_list():
|
||||
assert manager.get_freed_mm_hashes() == []
|
||||
|
||||
|
||||
def test_referencing_a_freeable_entry_protects_it_from_eviction():
|
||||
"""A waiting request can hold an entry it is not scheduled for yet.
|
||||
|
||||
Entries with no referent are evictable. A request that is deferred (e.g.
|
||||
by an EC connector waiting on another item) still needs the items it
|
||||
already has, and a connector that hands out one transfer per request
|
||||
cannot always produce the same item a second time.
|
||||
"""
|
||||
manager = EncoderCacheManager(cache_size=10)
|
||||
owner = MockRequest("owner", ["a"], [5])
|
||||
waiter = MockRequest("waiter", ["a"], [5])
|
||||
newcomer = MockRequest("newcomer", ["b"], [6])
|
||||
|
||||
manager.allocate(owner, 0)
|
||||
manager.free_encoder_input(owner, 0)
|
||||
assert "a" in manager.freeable
|
||||
|
||||
# The waiting request references it; it is no longer reclaimable.
|
||||
assert manager.check_and_update_cache(waiter, 0)
|
||||
assert "a" not in manager.freeable
|
||||
|
||||
# 'b' no longer fits, and 'a' must not be taken away to make room.
|
||||
assert not manager.can_allocate(newcomer, 0, int(1e9), 0)
|
||||
assert "a" in manager.cached
|
||||
assert manager.get_freed_mm_hashes() == []
|
||||
|
||||
# Once the waiter is done with it, the entry is reclaimable again.
|
||||
manager.free_encoder_input(waiter, 0)
|
||||
assert manager.can_allocate(newcomer, 0, int(1e9), 0)
|
||||
assert manager.get_freed_mm_hashes() == ["a"]
|
||||
|
||||
|
||||
def test_reallocated_hash_is_not_reported_as_freed():
|
||||
manager = EncoderCacheManager(cache_size=8)
|
||||
req_a = MockRequest("reqA", ["a"], [4])
|
||||
|
||||
@@ -5643,6 +5643,68 @@ def test_free_encoder_inputs_respects_unconfirmed_placeholders():
|
||||
assert manager.get_cached_input_ids(request) == set()
|
||||
|
||||
|
||||
def test_free_encoder_inputs_notifies_the_ec_connector():
|
||||
"""The connector learns the item is consumed, not just that the request ended.
|
||||
|
||||
Connectors hold per-item transfer state (remote buffers, reservations);
|
||||
waiting for `request_finished` pins it for the whole generation.
|
||||
"""
|
||||
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
|
||||
mm_start_pos, mm_length = 50, 100
|
||||
request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=mm_start_pos + mm_length + 10,
|
||||
mm_positions=[[PlaceholderRange(offset=mm_start_pos, length=mm_length)]],
|
||||
)[0]
|
||||
scheduler.encoder_cache_manager.allocate(request, 0)
|
||||
scheduler.ec_connector = Mock()
|
||||
|
||||
request.num_computed_tokens = mm_start_pos + mm_length - 1
|
||||
scheduler._free_encoder_inputs(request)
|
||||
scheduler.ec_connector.update_state_after_free.assert_not_called()
|
||||
|
||||
request.num_computed_tokens = mm_start_pos + mm_length
|
||||
scheduler._free_encoder_inputs(request)
|
||||
scheduler.ec_connector.update_state_after_free.assert_called_once_with(request, 0)
|
||||
|
||||
|
||||
def test_unavailable_encoder_input_fails_the_request_as_retryable():
|
||||
"""An encoder input the connector gave up on must end the request.
|
||||
|
||||
`FinishReason.ERROR` is the retryable channel the KV connector already uses
|
||||
for load failures, so the caller can re-issue; deferring instead parked the
|
||||
request until the client timed out.
|
||||
"""
|
||||
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
|
||||
request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=160,
|
||||
mm_positions=[[PlaceholderRange(offset=50, length=100)]],
|
||||
)[0]
|
||||
scheduler.add_request(request)
|
||||
scheduler.ec_connector = Mock()
|
||||
scheduler.ec_connector.take_unavailable_requests.return_value = {request.request_id}
|
||||
# `request_finished` reports (delay_free, params) for the request teardown.
|
||||
scheduler.ec_connector.request_finished.return_value = (False, None)
|
||||
|
||||
scheduler_output = scheduler.schedule()
|
||||
outputs = scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
ModelRunnerOutput(
|
||||
req_ids=[request.request_id],
|
||||
req_id_to_index={request.request_id: 0},
|
||||
sampled_token_ids=[[]],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
),
|
||||
)
|
||||
|
||||
assert request.status == RequestStatus.FINISHED_ERROR
|
||||
engine_outputs = outputs[0].outputs
|
||||
assert [o.finish_reason for o in engine_outputs] == [FinishReason.ERROR]
|
||||
|
||||
|
||||
def test_free_encoder_inputs_defers_for_eagle_lookahead():
|
||||
"""With EAGLE speculative decoding, the encoder input is retained one extra
|
||||
position so the drafter's +1 look-ahead mm-embedding gather (which reads one
|
||||
@@ -5840,9 +5902,8 @@ def test_ec_connector_ensure_cache_available_defers_request(use_kv_connector):
|
||||
|
||||
# ensure_cache_available must have been called with (request, num_computed_tokens=0)
|
||||
# for a brand-new request that has no cached tokens yet.
|
||||
scheduler.ec_connector.ensure_cache_available.assert_called_once_with(
|
||||
request_deferred, 0
|
||||
)
|
||||
ensure_call = scheduler.ec_connector.ensure_cache_available.call_args
|
||||
assert ensure_call.args == (request_deferred, 0)
|
||||
# Deferred request must NOT be scheduled
|
||||
assert request_deferred.request_id not in output.num_scheduled_tokens
|
||||
_assert_right_encoder_cache_allocated(scheduler, expected_total_allocated=0)
|
||||
@@ -5874,6 +5935,31 @@ def test_ec_connector_ensure_cache_available_defers_request(use_kv_connector):
|
||||
_assert_right_encoder_inputs(output, expected_total_reqs=0)
|
||||
|
||||
|
||||
def test_ec_connector_defers_running_request_for_async_reload():
|
||||
scheduler = create_scheduler(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
max_num_batched_tokens=32,
|
||||
use_ec_connector=True,
|
||||
ec_role="ec_consumer",
|
||||
)
|
||||
request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=128,
|
||||
mm_positions=[[PlaceholderRange(offset=48, length=32)]],
|
||||
req_ids=["request"],
|
||||
)[0]
|
||||
scheduler.ec_connector.ensure_cache_available = Mock(side_effect=[True, False])
|
||||
|
||||
scheduler.add_request(request)
|
||||
first_output = scheduler.schedule()
|
||||
assert first_output.num_scheduled_tokens[request.request_id] == 32
|
||||
|
||||
second_output = scheduler.schedule()
|
||||
assert request.request_id not in second_output.num_scheduled_tokens
|
||||
ensure_call = scheduler.ec_connector.ensure_cache_available.call_args
|
||||
assert ensure_call.args[:2] == (request, 32)
|
||||
|
||||
|
||||
def test_ec_connector_pending_prefetch_only_checks_future_mm_features():
|
||||
"""Test that future mm feature filtering only yields features beyond
|
||||
the computed token frontier.
|
||||
|
||||
@@ -1,171 +1,213 @@
|
||||
# EPD Correctness Test
|
||||
|
||||
This test verifies that EPD (Encoder-Prefill-Decode) disaggregation produces identical outputs to a baseline single instance.
|
||||
|
||||
## What It Tests
|
||||
|
||||
- **Baseline**: Single vLLM instance serving a multimodal model
|
||||
- **EPD (1E+1PD)**: 1 Encoder + 1 Prefill-Decode instance
|
||||
- **Baseline (1P+1D)**: 1 Prefill + 1 Decode instance
|
||||
- **EPD (1E+1P+1D)**: 1 Encoder + 1 Prefill + 1 Decode instance
|
||||
|
||||
The test ensures that disaggregated encoding produces **identical** outputs to the baseline.
|
||||
|
||||
Note that currently PD disaggregation set up may give slightly different results from a single instance. Therefore, we need the result from 1P+1D as the baseline for 1E+1P+1D
|
||||
|
||||
# EPD Correctness Test
|
||||
|
||||
This test verifies that EPD (Encoder-Prefill-Decode) disaggregation produces identical outputs to a baseline single instance.
|
||||
|
||||
## What It Tests
|
||||
|
||||
- **Baseline**: Single vLLM instance serving a multimodal model
|
||||
- **EPD (1E+1PD)**: 1 Encoder + 1 Prefill-Decode instance
|
||||
- **Baseline (1P+1D)**: 1 Prefill + 1 Decode instance
|
||||
- **EPD (1E+1P+1D)**: 1 Encoder + 1 Prefill + 1 Decode instance
|
||||
|
||||
The test ensures that disaggregated encoding produces **identical** outputs to the baseline.
|
||||
|
||||
Note that currently PD disaggregation set up may give slightly different results from a single instance. Therefore, we need the result from 1P+1D as the baseline for 1E+1P+1D
|
||||
|
||||
Please refer to [Disaggregated Encoder Feature](../../../../docs/features/disagg_encoder.md) for the detailed explanation for the EPD features.
|
||||
|
||||
## Files
|
||||
|
||||
- `run_epd_correctness_test.sh` - Main test script (starts all instances and runs tests)
|
||||
- `test_epd_correctness.py` - Python test script (compares outputs)
|
||||
|
||||
## Usage
|
||||
|
||||
### Multimodal Prompts (Default)
|
||||
|
||||
```bash
|
||||
cd vllm
|
||||
./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
```
|
||||
|
||||
This runs the test with actual multimodal (image) prompts.
|
||||
|
||||
### Text-Only Prompts
|
||||
|
||||
```bash
|
||||
cd vllm
|
||||
USE_MM_PROMPTS=0 ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
```
|
||||
|
||||
This runs a quick test with text-only prompts to verify the setup works.
|
||||
|
||||
### Custom Configuration
|
||||
|
||||
```bash
|
||||
# Use specific GPUs
|
||||
GPU_E=0 GPU_PD=1 GPU_P=1 GPU_D=2 bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
|
||||
# Use specific ports
|
||||
ENDPOINT_PORT=10001 bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
|
||||
# Use specific model
|
||||
MODEL="Qwen/Qwen2.5-VL-3B-Instruct" bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
|
||||
# Use specific storage path
|
||||
EC_SHARED_STORAGE_PATH="/tmp/my_ec_cache" bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
### Step 1: Baseline
|
||||
|
||||
1. Start single vLLM instance on GPU
|
||||
2. Run test prompts (multimodal or text-only)
|
||||
3. Save outputs to `.vllm_epd_baseline.txt`
|
||||
4. Shutdown instance
|
||||
|
||||
### Step 2: EPD (1E + 1PD)
|
||||
|
||||
1. Clear encoder cache storage
|
||||
2. Start instances and proxy
|
||||
3. Run same test prompts
|
||||
4. Assert outputs match baseline exactly
|
||||
5. Shutdown instances
|
||||
|
||||
### Step 3: EPD (1E + 1P + 1D)
|
||||
|
||||
1. Clear encoder cache storage
|
||||
2. Start instances and proxy
|
||||
3. Run same test prompts
|
||||
4. Assert outputs match baseline exactly
|
||||
5. Shutdown instances
|
||||
|
||||
## Test Scenarios
|
||||
|
||||
### Multimodal Prompts (--use_mm_prompts)
|
||||
|
||||
Tests encoder cache transfer:
|
||||
|
||||
- Single image query
|
||||
- Multiple images in one request
|
||||
- Mixed image and text
|
||||
- Image with detailed questions
|
||||
|
||||
### Text-Only Prompts (default)
|
||||
|
||||
Quick sanity check:
|
||||
|
||||
- Simple text queries
|
||||
- Text-only explanations
|
||||
- Verifies proxy routing works
|
||||
|
||||
## Expected Behavior
|
||||
|
||||
### ✅ Test Passes When
|
||||
|
||||
- All disagg outputs match baseline outputs exactly
|
||||
- No errors during instance startup
|
||||
- Encoder cache is properly saved and loaded
|
||||
- Proxy correctly routes requests
|
||||
|
||||
### ❌ Test Fails When
|
||||
|
||||
- Outputs differ between baseline and disagg
|
||||
- Server startup fails
|
||||
- Encoder cache not found (should fall back to local execution)
|
||||
- Proxy routing errors
|
||||
|
||||
## Notes
|
||||
|
||||
- The test uses deterministic generation (`temperature=0.0`, `seed=42`)
|
||||
- Encoder cache should enable exact output reproduction
|
||||
- Test cleans up all instances and cache files after completion
|
||||
- Safe to run multiple times (idempotent)
|
||||
- We setup the PD disagg part with NixlConnector. Please read details about EPD in `examples/disaggregated/disaggregated_encoder/README.md`
|
||||
|
||||
## Requirements
|
||||
|
||||
- Multiple GPUs (3 for 1E+1P+1D, 2 for 1E+1PD, 1 for baseline)
|
||||
- 1E+1P+1D is runnable with 2 GPU by assign E and P on the same GPU now.
|
||||
- Multimodal model (e.g., Qwen2.5-VL-3B-Instruct)
|
||||
- Internet access (for accessing vllm test images)
|
||||
|
||||
## Debugging
|
||||
|
||||
### Check Logs
|
||||
|
||||
Logs and baseline output are saved in `/tmp/` by default.
|
||||
Can be customized by changing the environment variables.
|
||||
|
||||
### Check Encoder Cache
|
||||
|
||||
```bash
|
||||
# Verify cache files are created
|
||||
ls -la $EC_SHARED_STORAGE_PATH/
|
||||
|
||||
# Should see directories with mm_hash names
|
||||
# Each containing encoder_cache.safetensors
|
||||
```
|
||||
|
||||
### Manual Testing
|
||||
|
||||
Run individual components:
|
||||
|
||||
```bash
|
||||
# Baseline only
|
||||
python test_epd_correctness.py \
|
||||
--service_url http://localhost:8000 \
|
||||
--model_name Qwen/Qwen2.5-VL-3B-Instruct \
|
||||
--mode baseline \
|
||||
--baseline_file test_output.txt \
|
||||
--use_mm_prompts
|
||||
|
||||
# Disagg only (requires baseline output file!)
|
||||
python test_epd_correctness.py \
|
||||
--service_url http://localhost:8000 \
|
||||
--model_name Qwen/Qwen2.5-VL-3B-Instruct \
|
||||
--mode disagg \
|
||||
--baseline_file test_output.txt \
|
||||
--use_mm_prompts
|
||||
```
|
||||
|
||||
## Files
|
||||
|
||||
- `run_epd_correctness_test.sh` - Main test script (starts all instances and runs tests)
|
||||
- `test_epd_correctness.py` - Python test script (compares outputs)
|
||||
|
||||
## Usage
|
||||
|
||||
### Multimodal Prompts (Default)
|
||||
|
||||
```bash
|
||||
cd vllm
|
||||
./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
```
|
||||
|
||||
This runs the test with actual multimodal (image) prompts.
|
||||
|
||||
### Text-Only Prompts
|
||||
|
||||
```bash
|
||||
cd vllm
|
||||
USE_MM_PROMPTS=0 ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
```
|
||||
|
||||
This runs a quick test with text-only prompts to verify the setup works.
|
||||
|
||||
### Custom Configuration
|
||||
|
||||
```bash
|
||||
# Use specific GPUs
|
||||
GPU_E=0 GPU_PD=1 GPU_P=1 GPU_D=2 bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
|
||||
# Use specific ports
|
||||
ENDPOINT_PORT=10001 bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
|
||||
# Use specific model
|
||||
MODEL="Qwen/Qwen2.5-VL-3B-Instruct" bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
|
||||
# Use specific storage path
|
||||
EC_SHARED_STORAGE_PATH="/tmp/my_ec_cache" bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
### Mooncake EC (1E + 1PD)
|
||||
|
||||
```bash
|
||||
PYTHON_BIN="$PWD/.venv/bin/python" \
|
||||
bash tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh
|
||||
```
|
||||
|
||||
Requires two GPUs and Mooncake TransferEngine. TCP is the default transport;
|
||||
no RDMA-capable network hardware is required.
|
||||
The script sets `MC_FORCE_TCP=1` in TCP mode so Mooncake cannot auto-select RDMA.
|
||||
The baseline runs first on GPU 0, followed by E on GPU 0 and PD on GPU 1.
|
||||
`MOONCAKE_EC_PROTOCOL=rdma` selects RDMA instead; host-specific transport
|
||||
environment variables should be set by the caller.
|
||||
|
||||
Three black-box cases check fixed short answers and compare with the baseline:
|
||||
|
||||
- One image: read the STOP sign.
|
||||
- Two different images (including a local file): identify flowers and birds.
|
||||
- The same image twice: read both STOP signs.
|
||||
|
||||
By default, all three requests run concurrently for two rounds, exercising
|
||||
shared hashes across requests and reuse after completion. Every response is
|
||||
compared, not just the final round. Set `CONCURRENCY` and `REPEAT` to override.
|
||||
Prefix caching is disabled and CUDA graphs remain enabled. This is a small
|
||||
correctness suite, not a performance benchmark or failure-injection suite.
|
||||
|
||||
`LOG_PATH` and `BASELINE_FILE` select the log directory and reference output.
|
||||
`SKIP_BASELINE=1` reuses a reference generated with the same model and test
|
||||
configuration. `USE_MM_PROMPTS=0` only checks text routing, not EC transfer.
|
||||
|
||||
Buildkite runs this script as `mooncake-ec-tcp-e2e-2-gpus` on two L4 GPUs.
|
||||
The job is defined in `.buildkite/test_areas/disaggregated_mooncake.yaml` and
|
||||
selected for changes to EC, multimodal processing, scheduler/model-runner
|
||||
integration, the proxy, or these tests (subject to normal PR CI approval).
|
||||
It reuses the CI image's Python packages through a system-site-packages venv
|
||||
and installs the CUDA-compatible Mooncake wheel. It uses loopback networking
|
||||
and TCP only, without RDMA devices or peer-memory setup.
|
||||
|
||||
The job fails on startup errors, request errors, or answer mismatches. On
|
||||
failure, the script prints the last 100 lines of each server/proxy log to
|
||||
the CI job log; full files remain under `LOG_PATH` while the container exists.
|
||||
|
||||
### Step 1: Baseline
|
||||
|
||||
1. Start single vLLM instance on GPU
|
||||
2. Run test prompts (multimodal or text-only)
|
||||
3. Save outputs to `.vllm_epd_baseline.txt`
|
||||
4. Shutdown instance
|
||||
|
||||
### Step 2: EPD (1E + 1PD)
|
||||
|
||||
1. Clear encoder cache storage
|
||||
2. Start instances and proxy
|
||||
3. Run same test prompts
|
||||
4. Assert outputs match baseline exactly
|
||||
5. Shutdown instances
|
||||
|
||||
### Step 3: EPD (1E + 1P + 1D)
|
||||
|
||||
1. Clear encoder cache storage
|
||||
2. Start instances and proxy
|
||||
3. Run same test prompts
|
||||
4. Assert outputs match baseline exactly
|
||||
5. Shutdown instances
|
||||
|
||||
## Test Scenarios
|
||||
|
||||
### Multimodal Prompts (--use_mm_prompts)
|
||||
|
||||
Tests encoder cache transfer:
|
||||
|
||||
- Single image query
|
||||
- Multiple images in one request
|
||||
- Mixed image and text
|
||||
- Image with detailed questions
|
||||
|
||||
### Text-Only Prompts (default)
|
||||
|
||||
Quick sanity check:
|
||||
|
||||
- Simple text queries
|
||||
- Text-only explanations
|
||||
- Verifies proxy routing works
|
||||
|
||||
## Expected Behavior
|
||||
|
||||
### ✅ Test Passes When
|
||||
|
||||
- All disagg outputs match baseline outputs exactly
|
||||
- No errors during instance startup
|
||||
- Encoder cache is properly saved and loaded
|
||||
- Proxy correctly routes requests
|
||||
|
||||
### ❌ Test Fails When
|
||||
|
||||
- Outputs differ between baseline and disagg
|
||||
- Server startup fails
|
||||
- Encoder cache not found (should fall back to local execution)
|
||||
- Proxy routing errors
|
||||
|
||||
## Notes
|
||||
|
||||
- The test uses deterministic generation (`temperature=0.0`, `seed=42`)
|
||||
- Encoder cache should enable exact output reproduction
|
||||
- Test cleans up all instances and cache files after completion
|
||||
- Safe to run multiple times (idempotent)
|
||||
- We setup the PD disagg part with NixlConnector. Please read details about EPD in `examples/disaggregated/disaggregated_encoder/README.md`
|
||||
|
||||
## Requirements
|
||||
|
||||
- Multiple GPUs (3 for 1E+1P+1D, 2 for 1E+1PD, 1 for baseline)
|
||||
- 1E+1P+1D is runnable with 2 GPU by assign E and P on the same GPU now.
|
||||
- Multimodal model (e.g., Qwen2.5-VL-3B-Instruct)
|
||||
- Internet access (for accessing vllm test images)
|
||||
|
||||
## Debugging
|
||||
|
||||
### Check Logs
|
||||
|
||||
Logs and baseline output are saved in `/tmp/` by default.
|
||||
Can be customized by changing the environment variables.
|
||||
|
||||
### Check Encoder Cache
|
||||
|
||||
```bash
|
||||
# Verify cache files are created
|
||||
ls -la $EC_SHARED_STORAGE_PATH/
|
||||
|
||||
# Should see directories with mm_hash names
|
||||
# Each containing encoder_cache.safetensors
|
||||
```
|
||||
|
||||
### Manual Testing
|
||||
|
||||
Run individual components:
|
||||
|
||||
```bash
|
||||
# Baseline only
|
||||
python test_epd_correctness.py \
|
||||
--service_url http://localhost:8000 \
|
||||
--model_name Qwen/Qwen2.5-VL-3B-Instruct \
|
||||
--mode baseline \
|
||||
--baseline_file test_output.txt \
|
||||
--use_mm_prompts
|
||||
|
||||
# Disagg only (requires baseline output file!)
|
||||
python test_epd_correctness.py \
|
||||
--service_url http://localhost:8000 \
|
||||
--model_name Qwen/Qwen2.5-VL-3B-Instruct \
|
||||
--mode disagg \
|
||||
--baseline_file test_output.txt \
|
||||
--use_mm_prompts
|
||||
```
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Full-stack EPD validation with ECMooncakeConnector (Mooncake TransferEngine):
|
||||
# 1) Single-GPU baseline (multimodal) -> saves baseline JSON
|
||||
# 2) 1 Encoder + 1 PD with Mooncake EC -> compare outputs to baseline
|
||||
#
|
||||
# Usage (from repo root):
|
||||
# ./tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh
|
||||
#
|
||||
# Env:
|
||||
# MODEL HF model id (default: Qwen/Qwen2.5-VL-3B-Instruct)
|
||||
# GPU_SINGLE / GPU_E / GPU_PD GPU ids (defaults 0 / 0 / 1)
|
||||
# ENDPOINT_PORT, ENCODE_PORT, PREFILL_DECODE_PORT
|
||||
# EC_MOONCAKE_RESERVATION_HOST consumer control host (default 127.0.0.1)
|
||||
# MOONCAKE_EC_PROTOCOL tcp | rdma (default tcp)
|
||||
# USE_MM_PROMPTS 1 (default) or 0 for text-only quick sanity
|
||||
# TIMEOUT_SECONDS wait_for_server timeout (default 1200)
|
||||
# SKIP_BASELINE set to 1 to reuse existing BASELINE_FILE
|
||||
# CONCURRENCY / REPEAT concurrent requests / rounds (defaults 3 / 2)
|
||||
# MAX_MODEL_LEN context length (default 16384)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
GIT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)
|
||||
cd "$GIT_ROOT" || exit 1
|
||||
export PYTHONPATH="${GIT_ROOT}:${PYTHONPATH:-}"
|
||||
PYTHON_BIN="${PYTHON_BIN:-${GIT_ROOT}/.venv/bin/python}"
|
||||
|
||||
MODEL="${MODEL:-Qwen/Qwen2.5-VL-3B-Instruct}"
|
||||
USE_MM_PROMPTS="${USE_MM_PROMPTS:-1}"
|
||||
TEST_ARGS=(--concurrency "${CONCURRENCY:-3}" --repeat "${REPEAT:-2}")
|
||||
if [[ "$USE_MM_PROMPTS" == "1" ]]; then
|
||||
TEST_ARGS+=(--use_mm_prompts --mm_smoke_test)
|
||||
fi
|
||||
MAX_MODEL_LEN="${MAX_MODEL_LEN:-16384}"
|
||||
|
||||
GPU_SINGLE="${GPU_SINGLE:-0}"
|
||||
GPU_E="${GPU_E:-0}"
|
||||
GPU_PD="${GPU_PD:-1}"
|
||||
|
||||
ENCODE_PORT="${ENCODE_PORT:-19534}"
|
||||
PREFILL_DECODE_PORT="${PREFILL_DECODE_PORT:-19537}"
|
||||
ENDPOINT_PORT="${ENDPOINT_PORT:-10002}"
|
||||
BASELINE_PORT="${BASELINE_PORT:-10003}"
|
||||
|
||||
EC_MOONCAKE_RESERVATION_HOST="${EC_MOONCAKE_RESERVATION_HOST:-127.0.0.1}"
|
||||
EC_MOONCAKE_RESERVATION_PORT="${EC_MOONCAKE_RESERVATION_PORT:-19019}"
|
||||
MOONCAKE_EC_PROTOCOL="${MOONCAKE_EC_PROTOCOL:-tcp}"
|
||||
export EC_MOONCAKE_RESERVATION_HOST
|
||||
export EC_MOONCAKE_RESERVATION_PORT
|
||||
export MOONCAKE_EC_PROTOCOL
|
||||
if [[ "$MOONCAKE_EC_PROTOCOL" == "tcp" ]]; then
|
||||
# TransferEngine may otherwise auto-select RDMA on hosts with an HCA.
|
||||
export MC_FORCE_TCP=1
|
||||
else
|
||||
unset MC_FORCE_TCP
|
||||
fi
|
||||
|
||||
LOG_PATH="${LOG_PATH:-/tmp}"
|
||||
BASELINE_FILE="${BASELINE_FILE:-/tmp/vllm_epd_mooncake_baseline.txt}"
|
||||
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-1200}"
|
||||
PIDS=()
|
||||
|
||||
mkdir -p "$LOG_PATH"
|
||||
|
||||
VLLM_SERVE=("$PYTHON_BIN" -m vllm.entrypoints.cli.main serve)
|
||||
|
||||
ENC_EC_JSON=$("$PYTHON_BIN" <<PY
|
||||
import json, os
|
||||
print(json.dumps({
|
||||
"ec_connector": "ECMooncakeConnector",
|
||||
"ec_role": "ec_producer",
|
||||
"ec_connector_extra_config": {
|
||||
"mooncake_protocol": os.environ.get("MOONCAKE_EC_PROTOCOL", "tcp"),
|
||||
},
|
||||
}, separators=(",", ":")))
|
||||
PY
|
||||
)
|
||||
|
||||
PD_EC_JSON=$("$PYTHON_BIN" <<PY
|
||||
import json, os
|
||||
print(json.dumps({
|
||||
"ec_connector": "ECMooncakeConnector",
|
||||
"ec_role": "ec_consumer",
|
||||
"ec_ip": os.environ["EC_MOONCAKE_RESERVATION_HOST"],
|
||||
"ec_port": int(os.environ.get("EC_MOONCAKE_RESERVATION_PORT", "19019")),
|
||||
"ec_connector_extra_config": {
|
||||
"mooncake_protocol": os.environ.get("MOONCAKE_EC_PROTOCOL", "tcp"),
|
||||
},
|
||||
}, separators=(",", ":")))
|
||||
PY
|
||||
)
|
||||
|
||||
EC_MOONCAKE_RESERVATION_ADDR=$("$PYTHON_BIN" <<'PY'
|
||||
import os
|
||||
|
||||
from vllm.utils.network_utils import make_zmq_path
|
||||
|
||||
print(make_zmq_path(
|
||||
"tcp", os.environ["EC_MOONCAKE_RESERVATION_HOST"],
|
||||
int(os.environ["EC_MOONCAKE_RESERVATION_PORT"]),
|
||||
))
|
||||
PY
|
||||
)
|
||||
|
||||
wait_for_server() {
|
||||
local port=$1
|
||||
local pid=$2
|
||||
local deadline=$((SECONDS + TIMEOUT_SECONDS))
|
||||
while ((SECONDS < deadline)); do
|
||||
kill -0 "$pid" 2>/dev/null || return 1
|
||||
if curl --max-time 2 -fsS "http://localhost:${port}/health" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
cleanup_instances() {
|
||||
if ((${#PIDS[@]} == 0)); then
|
||||
return
|
||||
fi
|
||||
echo "Cleaning up tracked vLLM / proxy processes..."
|
||||
for pid in "${PIDS[@]}"; do
|
||||
kill "$pid" 2>/dev/null || true
|
||||
done
|
||||
for _ in {1..10}; do
|
||||
local alive=0
|
||||
for pid in "${PIDS[@]}"; do
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
alive=1
|
||||
fi
|
||||
done
|
||||
((alive == 0)) && break
|
||||
sleep 1
|
||||
done
|
||||
for pid in "${PIDS[@]}"; do
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
kill -KILL "$pid" 2>/dev/null || true
|
||||
fi
|
||||
wait "$pid" 2>/dev/null || true
|
||||
done
|
||||
PIDS=()
|
||||
}
|
||||
|
||||
finish() {
|
||||
local status=$?
|
||||
cleanup_instances
|
||||
if ((status != 0)); then
|
||||
for log in "${LOG_PATH}"/mooncake_epd_*.log; do
|
||||
[[ -f "$log" ]] || continue
|
||||
echo "=== $log (last 100 lines) ==="
|
||||
tail -100 "$log"
|
||||
done
|
||||
fi
|
||||
exit "$status"
|
||||
}
|
||||
|
||||
trap finish EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
run_baseline() {
|
||||
echo "================================"
|
||||
echo "BASELINE (single vLLM, MM if enabled)"
|
||||
echo "================================"
|
||||
cleanup_instances
|
||||
local PORT=$BASELINE_PORT
|
||||
echo "Starting baseline on GPU $GPU_SINGLE port $PORT"
|
||||
CUDA_VISIBLE_DEVICES="$GPU_SINGLE" "${VLLM_SERVE[@]}" "$MODEL" \
|
||||
--port "$PORT" \
|
||||
--gpu-memory-utilization 0.75 \
|
||||
--max-num-seqs 32 \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--no-enable-prefix-caching \
|
||||
--limit-mm-per-prompt '{"image":2,"video":0}' \
|
||||
--allowed-local-media-path "${GIT_ROOT}/tests/v1/ec_connector/integration" \
|
||||
>"${LOG_PATH}/mooncake_epd_baseline.log" 2>&1 &
|
||||
local BASELINE_PID=$!
|
||||
PIDS+=("$BASELINE_PID")
|
||||
echo "Waiting for baseline..."
|
||||
wait_for_server "$PORT" "$BASELINE_PID" || { echo "Baseline failed to start"; return 1; }
|
||||
curl -s "http://127.0.0.1:${PORT}/v1/models" | head -c 200 || true
|
||||
echo ""
|
||||
"$PYTHON_BIN" "${GIT_ROOT}/tests/v1/ec_connector/integration/test_epd_correctness.py" \
|
||||
--service_url "http://localhost:$PORT" \
|
||||
--model_name "$MODEL" \
|
||||
--mode baseline \
|
||||
--baseline_file "$BASELINE_FILE" \
|
||||
"${TEST_ARGS[@]}"
|
||||
cleanup_instances
|
||||
}
|
||||
|
||||
run_epd_mooncake() {
|
||||
echo "================================"
|
||||
echo "EPD 1E + 1PD with ECMooncakeConnector"
|
||||
echo "Reservation port on consumer: $EC_MOONCAKE_RESERVATION_PORT"
|
||||
echo "Mooncake protocol: $MOONCAKE_EC_PROTOCOL"
|
||||
echo "================================"
|
||||
cleanup_instances
|
||||
|
||||
echo "Starting ENCODER on GPU $GPU_E port $ENCODE_PORT"
|
||||
CUDA_VISIBLE_DEVICES="$GPU_E" "${VLLM_SERVE[@]}" "$MODEL" \
|
||||
--port "$ENCODE_PORT" \
|
||||
--gpu-memory-utilization 0.35 \
|
||||
--mm-tensor-ipc torch_shm \
|
||||
--enable-request-id-headers \
|
||||
--no-enable-prefix-caching \
|
||||
--max-num-batched-tokens "$MAX_MODEL_LEN" \
|
||||
--max-num-seqs 32 \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--limit-mm-per-prompt '{"image":2,"video":0}' \
|
||||
--allowed-local-media-path "${GIT_ROOT}/tests/v1/ec_connector/integration" \
|
||||
--ec-transfer-config "$ENC_EC_JSON" \
|
||||
>"${LOG_PATH}/mooncake_epd_encoder.log" 2>&1 &
|
||||
local ENCODER_PID=$!
|
||||
PIDS+=("$ENCODER_PID")
|
||||
|
||||
echo "Starting PD on GPU $GPU_PD port $PREFILL_DECODE_PORT"
|
||||
CUDA_VISIBLE_DEVICES="$GPU_PD" "${VLLM_SERVE[@]}" "$MODEL" \
|
||||
--port "$PREFILL_DECODE_PORT" \
|
||||
--gpu-memory-utilization 0.75 \
|
||||
--mm-tensor-ipc torch_shm \
|
||||
--enable-mm-embeds \
|
||||
--enable-request-id-headers \
|
||||
--max-num-seqs 32 \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--no-enable-prefix-caching \
|
||||
--limit-mm-per-prompt '{"image":2,"video":0}' \
|
||||
--allowed-local-media-path "${GIT_ROOT}/tests/v1/ec_connector/integration" \
|
||||
--ec-transfer-config "$PD_EC_JSON" \
|
||||
>"${LOG_PATH}/mooncake_epd_pd.log" 2>&1 &
|
||||
local PD_PID=$!
|
||||
PIDS+=("$PD_PID")
|
||||
|
||||
echo "Waiting for encoder..."
|
||||
wait_for_server "$ENCODE_PORT" "$ENCODER_PID" || { echo "Encoder failed to start"; return 1; }
|
||||
echo "Waiting for PD..."
|
||||
wait_for_server "$PREFILL_DECODE_PORT" "$PD_PID" || { echo "PD failed to start"; return 1; }
|
||||
|
||||
echo "Starting EPD proxy on $ENDPOINT_PORT"
|
||||
"$PYTHON_BIN" "${GIT_ROOT}/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py" \
|
||||
--host "0.0.0.0" \
|
||||
--port "$ENDPOINT_PORT" \
|
||||
--encode-servers-urls "http://localhost:$ENCODE_PORT" \
|
||||
--prefill-servers-urls "disable" \
|
||||
--decode-servers-urls "http://localhost:$PREFILL_DECODE_PORT" \
|
||||
--ec-consumer-zmq-addrs \
|
||||
"$EC_MOONCAKE_RESERVATION_ADDR" \
|
||||
>"${LOG_PATH}/mooncake_epd_proxy.log" 2>&1 &
|
||||
local PROXY_PID=$!
|
||||
PIDS+=("$PROXY_PID")
|
||||
|
||||
echo "Waiting for proxy..."
|
||||
wait_for_server "$ENDPOINT_PORT" "$PROXY_PID" || { echo "Proxy failed to start"; return 1; }
|
||||
curl -s "http://127.0.0.1:${ENDPOINT_PORT}/health" || true
|
||||
echo ""
|
||||
|
||||
"$PYTHON_BIN" "${GIT_ROOT}/tests/v1/ec_connector/integration/test_epd_correctness.py" \
|
||||
--service_url "http://localhost:$ENDPOINT_PORT" \
|
||||
--model_name "$MODEL" \
|
||||
--mode disagg \
|
||||
--baseline_file "$BASELINE_FILE" \
|
||||
"${TEST_ARGS[@]}"
|
||||
|
||||
cleanup_instances
|
||||
}
|
||||
|
||||
echo "================================"
|
||||
echo "1E + 1PD ECMooncake end-to-end correctness"
|
||||
echo "MODEL=$MODEL"
|
||||
echo "================================"
|
||||
|
||||
if [[ "${SKIP_BASELINE:-0}" != "1" ]]; then
|
||||
run_baseline
|
||||
else
|
||||
echo "SKIP_BASELINE=1 -> using existing $BASELINE_FILE"
|
||||
[[ -f "$BASELINE_FILE" ]] || { echo "Missing baseline file"; exit 1; }
|
||||
fi
|
||||
|
||||
run_epd_mooncake
|
||||
|
||||
echo "================================"
|
||||
echo "PASS: Mooncake EC EPD matches baseline"
|
||||
echo "Logs: ${LOG_PATH}/mooncake_epd_*.log"
|
||||
echo "================================"
|
||||
@@ -26,6 +26,7 @@ import argparse
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import openai
|
||||
import requests
|
||||
@@ -138,17 +139,20 @@ def run_chat_completion(
|
||||
Returns:
|
||||
Generated text content
|
||||
"""
|
||||
client = openai.OpenAI(api_key="EMPTY", base_url=base_url)
|
||||
with openai.OpenAI(
|
||||
api_key="EMPTY", base_url=base_url, timeout=120, max_retries=0
|
||||
) as client:
|
||||
completion = client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_tokens=max_tokens,
|
||||
temperature=0.0,
|
||||
seed=42,
|
||||
)
|
||||
|
||||
completion = client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_tokens=max_tokens,
|
||||
temperature=0.0,
|
||||
seed=42,
|
||||
)
|
||||
|
||||
return completion.choices[0].message.content
|
||||
content = completion.choices[0].message.content
|
||||
assert content, "Expected a nonempty completion"
|
||||
return content
|
||||
|
||||
|
||||
def main():
|
||||
@@ -198,7 +202,19 @@ def main():
|
||||
help="Skip the two-image multimodal prompt",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--mm_smoke_test",
|
||||
action="store_true",
|
||||
help="Use three short, fixed-answer image cases, including duplicate images",
|
||||
)
|
||||
parser.add_argument("--concurrency", type=int, default=1)
|
||||
parser.add_argument("--repeat", type=int, default=1)
|
||||
|
||||
args = parser.parse_args()
|
||||
if args.concurrency < 1 or args.repeat < 1:
|
||||
parser.error("--concurrency and --repeat must be positive")
|
||||
if args.mm_smoke_test and (not args.use_mm_prompts or args.skip_two_image_prompt):
|
||||
parser.error("--mm_smoke_test requires all multimodal prompts")
|
||||
|
||||
print(f"Service URL: {args.service_url}")
|
||||
print(f"Model: {args.model_name}")
|
||||
@@ -235,27 +251,76 @@ def main():
|
||||
test_prompts = SAMPLE_PROMPTS_TEXT
|
||||
print("Using text-only prompts for quick testing")
|
||||
|
||||
if args.mm_smoke_test:
|
||||
image = SAMPLE_PROMPTS_MM[0]["messages"][0]["content"][0]
|
||||
image_pair = SAMPLE_PROMPTS_MM[1]["messages"][0]["content"][:2]
|
||||
cases = [
|
||||
(
|
||||
"Single image",
|
||||
[image],
|
||||
"What word is on the red road sign? Reply with only that word "
|
||||
"in uppercase.",
|
||||
"STOP",
|
||||
),
|
||||
(
|
||||
"Two different images",
|
||||
image_pair,
|
||||
"Do these pictures contain both flowers and birds? "
|
||||
"Reply with only YES or NO.",
|
||||
"YES",
|
||||
),
|
||||
(
|
||||
"Same image twice",
|
||||
[image, image],
|
||||
"Read the red road sign in each image, in image order. Reply "
|
||||
"with only the two uppercase words separated by a comma and a space.",
|
||||
"STOP, STOP",
|
||||
),
|
||||
]
|
||||
test_prompts = [
|
||||
{
|
||||
"description": description,
|
||||
"expected": expected,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [*images, {"type": "text", "text": question}],
|
||||
}
|
||||
],
|
||||
}
|
||||
for description, images, question, expected in cases
|
||||
]
|
||||
|
||||
# Run completions
|
||||
service_url = f"{args.service_url}/v1"
|
||||
output_strs = {}
|
||||
|
||||
for i, prompt_data in enumerate(test_prompts):
|
||||
print(
|
||||
f"\nRunning prompt {i + 1}/{len(test_prompts)}: "
|
||||
f"{prompt_data['description']}"
|
||||
)
|
||||
|
||||
output_str = run_chat_completion(
|
||||
def complete(prompt_data):
|
||||
output = run_chat_completion(
|
||||
base_url=service_url,
|
||||
model_name=args.model_name,
|
||||
messages=prompt_data["messages"],
|
||||
max_tokens=MAX_OUTPUT_LEN,
|
||||
max_tokens=16 if args.mm_smoke_test else MAX_OUTPUT_LEN,
|
||||
)
|
||||
if args.mm_smoke_test:
|
||||
output = output.strip()
|
||||
assert output == prompt_data["expected"], (
|
||||
f"{prompt_data['description']}: expected {prompt_data['expected']!r}, "
|
||||
f"got {output!r}"
|
||||
)
|
||||
return output
|
||||
|
||||
# Use description as key for comparison
|
||||
key = prompt_data["description"]
|
||||
output_strs[key] = output_str
|
||||
print(f"Output: {output_str}")
|
||||
# Each round includes concurrent requests sharing image hashes; later rounds
|
||||
# exercise reuse after previous requests have finished.
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as executor:
|
||||
for repeat in range(args.repeat):
|
||||
outputs = executor.map(complete, test_prompts)
|
||||
for prompt_data, output_str in zip(test_prompts, outputs):
|
||||
key = prompt_data["description"]
|
||||
if args.repeat > 1:
|
||||
key = f"{key} (round {repeat + 1})"
|
||||
output_strs[key] = output_str
|
||||
print(f"{key}: {output_str}")
|
||||
|
||||
if args.mode in ("baseline", "baseline_pd"):
|
||||
# Baseline mode: Save outputs
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Body handling across the EPD proxy's decode retries.
|
||||
|
||||
Exercises the REAL helpers loaded from the ``examples/`` proxy, so a future
|
||||
change to them is what these tests catch.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
PROXY_REL = "examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def proxy():
|
||||
path = Path(__file__).parents[4] / PROXY_REL
|
||||
spec = importlib.util.spec_from_file_location("disagg_epd_proxy_retry", path)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
class _Response:
|
||||
def __init__(self, params):
|
||||
self._params = params
|
||||
|
||||
async def json(self):
|
||||
return {"kv_transfer_params": self._params}
|
||||
|
||||
|
||||
def test_maybe_prefill_leaves_the_caller_body_untouched(proxy, monkeypatch):
|
||||
"""A decode retry re-enters this function with the body it was given.
|
||||
|
||||
Mutating that body in place let one attempt's `remote_block_ids` survive
|
||||
into the next, so a retry whose prefill returns nothing sent decode blocks
|
||||
the prefiller may already have freed.
|
||||
"""
|
||||
served = [{"remote_block_ids": [1, 2]}, {}]
|
||||
|
||||
async def _stage(req_data, p_url, req_id):
|
||||
assert "kv_transfer_params" not in req_data
|
||||
return _Response(served.pop(0))
|
||||
|
||||
monkeypatch.setattr(proxy, "process_prefill_stage", _stage)
|
||||
|
||||
body = {"messages": [], "stream": False}
|
||||
first = asyncio.run(proxy.maybe_prefill(body, "http://prefill", "r1"))
|
||||
assert first["kv_transfer_params"] == {"remote_block_ids": [1, 2]}
|
||||
assert "kv_transfer_params" not in body
|
||||
|
||||
second = asyncio.run(proxy.maybe_prefill(body, "http://prefill", "r1"))
|
||||
assert "kv_transfer_params" not in second
|
||||
|
||||
|
||||
class _EncoderResponse:
|
||||
def __init__(self, params):
|
||||
self.status = 200
|
||||
self._params = params
|
||||
|
||||
async def json(self):
|
||||
return {"ec_transfer_params": self._params}
|
||||
|
||||
async def text(self):
|
||||
return ""
|
||||
|
||||
|
||||
class _EncoderSession:
|
||||
"""Serve one canned encoder reply per attempt."""
|
||||
|
||||
def __init__(self, replies):
|
||||
self._replies = list(replies)
|
||||
|
||||
async def post(self, url, json=None, headers=None):
|
||||
return _EncoderResponse(self._replies.pop(0))
|
||||
|
||||
|
||||
def test_a_decode_retry_does_not_inherit_the_previous_handles(proxy, monkeypatch):
|
||||
"""The retry loop re-enters `prepare_for_decode` with the same body.
|
||||
|
||||
Recording the encoder's connector handles on that body in place let
|
||||
attempt 1's handle survive into attempt 2, so a second encode that
|
||||
reported nothing still sent decode a handle on an embedding the encoder
|
||||
no longer publishes -- the exact state the retry exists to leave behind.
|
||||
"""
|
||||
handle = {"metadata": {"image_grid_thw": [1, 2, 2]}, "peer_port": 1234}
|
||||
monkeypatch.setattr(
|
||||
proxy,
|
||||
"encode_session",
|
||||
_EncoderSession([{"encoder-side-hash": handle}, {}]),
|
||||
)
|
||||
|
||||
async def _no_prefill(req_data, p_url, req_id):
|
||||
return req_data
|
||||
|
||||
monkeypatch.setattr(proxy, "maybe_prefill", _no_prefill)
|
||||
|
||||
body = {
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "image_url", "image_url": {"url": "image"}}],
|
||||
}
|
||||
],
|
||||
"stream": False,
|
||||
}
|
||||
args = ("r1", ["http://encoder"], "http://prefill", None)
|
||||
|
||||
first, _, _ = asyncio.run(proxy.prepare_for_decode(body, *args))
|
||||
reported = first["ec_transfer_params"]
|
||||
assert [handle] == [value for key, value in reported.items() if key != "ec_items"]
|
||||
assert "ec_transfer_params" not in body
|
||||
|
||||
# Attempt 2's encode reports nothing: decode must be told nothing.
|
||||
second, _, _ = asyncio.run(proxy.prepare_for_decode(body, *args))
|
||||
assert "ec_transfer_params" not in second
|
||||
|
||||
|
||||
def test_raw_media_keeps_encoder_transfer_identity(proxy, monkeypatch):
|
||||
handle = {"metadata": {"image_grid_thw": [1, 2, 2]}}
|
||||
monkeypatch.setattr(proxy, "NO_REWRITE", True)
|
||||
monkeypatch.setattr(
|
||||
proxy, "encode_session", _EncoderSession([{"encoded-hash": handle}])
|
||||
)
|
||||
body = {
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "image_url", "image_url": {"url": "image"}}],
|
||||
}
|
||||
]
|
||||
}
|
||||
prepared, _, _ = asyncio.run(
|
||||
proxy.prepare_for_decode(
|
||||
body, "request", ["http://encoder"], "", "tcp://consumer:1"
|
||||
)
|
||||
)
|
||||
assert prepared["messages"] == body["messages"]
|
||||
assert "ec_transfer_params" not in body
|
||||
params = prepared["ec_transfer_params"]
|
||||
assert params["encoded-hash"] == handle
|
||||
assert params["ec_items"][0]["mm_hash"] == "encoded-hash"
|
||||
assert params["ec_items"][0]["transfer_id"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("server_keep_alive", ["0.1", "1", "5", "2", "30"])
|
||||
def test_pooled_connections_are_retired_before_the_server_closes_them(
|
||||
proxy, monkeypatch, server_keep_alive
|
||||
):
|
||||
"""The proxy must not hand a request a connection the server has dropped.
|
||||
|
||||
vLLM closes idle keep-alive connections at `VLLM_HTTP_TIMEOUT_KEEP_ALIVE`
|
||||
seconds while aiohttp pools them for 15, so a slow hop leaves a dead
|
||||
connection in the pool. The next request fails with
|
||||
ServerDisconnectedError and the server logs nothing, because it closed the
|
||||
socket before the request arrived.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_HTTP_TIMEOUT_KEEP_ALIVE", server_keep_alive)
|
||||
monkeypatch.setattr(proxy.app.state, "p_urls", [], raising=False)
|
||||
|
||||
asyncio.run(proxy.on_startup())
|
||||
try:
|
||||
# No public accessor for the pool's idle timeout.
|
||||
pooled_for = proxy.encode_session.connector._keepalive_timeout
|
||||
assert pooled_for < float(server_keep_alive)
|
||||
assert proxy.decode_session.connector._keepalive_timeout == pooled_for
|
||||
finally:
|
||||
asyncio.run(proxy.on_shutdown())
|
||||
@@ -32,8 +32,13 @@ def _load_proxy_module():
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def assign():
|
||||
return _load_proxy_module().encoder_rr_assignment
|
||||
def proxy():
|
||||
return _load_proxy_module()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def assign(proxy):
|
||||
return proxy.encoder_rr_assignment
|
||||
|
||||
|
||||
def _drive(assign, e_urls, counts):
|
||||
@@ -80,3 +85,44 @@ def test_single_encoder_always_resolves_to_it(assign):
|
||||
urls, next_cursor = assign(["E0"], 0, 4)
|
||||
assert urls == ["E0"] * 4
|
||||
assert next_cursor == 0
|
||||
|
||||
|
||||
def test_generic_e_p_d_routing_without_mooncake_is_supported(proxy):
|
||||
proxy.validate_ec_consumer_routing(["http://prefill"], [])
|
||||
|
||||
|
||||
def test_mooncake_e_pd_routing_is_supported(proxy):
|
||||
proxy.validate_ec_consumer_routing([], ["tcp://decode:19019"])
|
||||
|
||||
|
||||
def test_mooncake_independent_prefill_routing_fails_fast(proxy):
|
||||
with pytest.raises(ValueError, match=r"supports E\+PD only"):
|
||||
proxy.validate_ec_consumer_routing(["http://prefill"], ["tcp://decode:19019"])
|
||||
|
||||
|
||||
def test_decode_rewrite_preserves_engine_reported_ec_hash(proxy):
|
||||
request = {
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "image_url", "image_url": {"url": "image"}}],
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
rewritten = proxy.rewrite_for_decode(
|
||||
request,
|
||||
{
|
||||
0: {
|
||||
"mm_hash": "proxy-uuid",
|
||||
"ec_mm_hash": "engine-derived-hash",
|
||||
"transfer_id": "transfer",
|
||||
"image_grid_thw": [1, 2, 3],
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
assert rewritten["messages"][0]["content"][0]["uuid"] == "proxy-uuid"
|
||||
assert rewritten["ec_transfer_params"]["ec_items"] == [
|
||||
{"mm_hash": "engine-derived-hash", "transfer_id": "transfer"}
|
||||
]
|
||||
|
||||
@@ -12,6 +12,9 @@ from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorMetadata,
|
||||
)
|
||||
from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT
|
||||
from vllm.v1.worker.ec_connector_model_runner_mixin import (
|
||||
ECConnectorModelRunnerMixin,
|
||||
)
|
||||
from vllm.v1.worker.gpu.ec_connector import NO_OP_EC_CONNECTOR, ActiveECConnector
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
@@ -52,6 +55,9 @@ def test_saves_newly_added_caches_for_every_producer(is_producer, is_consumer):
|
||||
|
||||
saved = [call.kwargs["mm_hash"] for call in fake.save_caches.call_args_list]
|
||||
assert saved == (["mm_new"] if is_producer else [])
|
||||
assert fake.start_save_caches.called == is_producer
|
||||
if is_producer:
|
||||
assert fake.start_save_caches.call_args.kwargs["encoder_cache"] is encoder_cache
|
||||
assert fake.start_load_caches.called == is_consumer
|
||||
|
||||
|
||||
@@ -66,6 +72,26 @@ def test_worker_meta_is_reported_on_context_exit():
|
||||
assert fake.clear_connector_metadata.called
|
||||
|
||||
|
||||
def test_v1_producer_receives_existing_encoder_cache():
|
||||
encoder_cache = {"mm_cached": None}
|
||||
fake = MagicMock(spec=ECConnectorBase)
|
||||
fake.is_producer = True
|
||||
fake.is_consumer = False
|
||||
fake.get_finished.return_value = (None, None)
|
||||
|
||||
module = "vllm.v1.worker.ec_connector_model_runner_mixin"
|
||||
with (
|
||||
patch(f"{module}.has_ec_transfer", return_value=True),
|
||||
patch(f"{module}.get_ec_transfer", return_value=fake),
|
||||
ECConnectorModelRunnerMixin.maybe_get_ec_connector_output(
|
||||
_scheduler_output(), encoder_cache
|
||||
),
|
||||
):
|
||||
pass
|
||||
|
||||
assert fake.start_save_caches.call_args.kwargs["encoder_cache"] is encoder_cache
|
||||
|
||||
|
||||
def test_no_forward_reports_without_running_the_model():
|
||||
connector, _ = _connector()
|
||||
|
||||
|
||||
@@ -18,6 +18,11 @@ class ECTransferConfig:
|
||||
|
||||
ec_connector: str | None = None
|
||||
"""The EC connector for vLLM to transmit EC caches between vLLM instances.
|
||||
|
||||
Built-in options include ``ECExampleConnector`` (shared filesystem via
|
||||
safetensors) and ``ECMooncakeConnector`` (Mooncake TransferEngine RDMA;
|
||||
requires ``mooncake-transfer-engine`` and matching producer/consumer
|
||||
``ec_connector_extra_config``; see ``mooncake_ec_connector`` module docstring).
|
||||
"""
|
||||
|
||||
engine_id: str | None = None
|
||||
|
||||
@@ -261,6 +261,25 @@ class ECConnectorBase(ABC):
|
||||
"""
|
||||
return True
|
||||
|
||||
def start_save_caches(self, **kwargs: Any) -> None:
|
||||
"""Prepare this step's outbound pushes before the model runs."""
|
||||
return
|
||||
|
||||
def start_worker_services(self) -> None:
|
||||
"""Start Worker-side services once the model is resident."""
|
||||
return
|
||||
|
||||
def take_unavailable_requests(self) -> set[str]:
|
||||
"""Request IDs whose encoder inputs the connector can no longer obtain.
|
||||
|
||||
The Scheduler fails these; re-issuing the request re-runs the encode.
|
||||
"""
|
||||
return set()
|
||||
|
||||
def update_state_after_free(self, request: "Request", index: int):
|
||||
"""Notify the connector that an encoder cache entry was released."""
|
||||
return
|
||||
|
||||
@abstractmethod
|
||||
def update_state_after_alloc(self, request: "Request", index: int):
|
||||
"""
|
||||
|
||||
@@ -7,6 +7,7 @@ Owns the mmap region and the embedding cache, and handles the producer
|
||||
for the ECCPUConnector.
|
||||
"""
|
||||
|
||||
from collections.abc import Collection
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
@@ -184,7 +185,10 @@ class ECCPUScheduler:
|
||||
return entry is not None and entry.ready
|
||||
|
||||
def ensure_cache_available(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
self,
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
local_cache_hashes: Collection[str] | None = None,
|
||||
) -> bool:
|
||||
if not self._nixl_enabled:
|
||||
return True # CPU offload never blocks.
|
||||
|
||||
@@ -89,3 +89,9 @@ ECConnectorFactory.register_connector(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.connector",
|
||||
"ECCPUConnector",
|
||||
)
|
||||
|
||||
ECConnectorFactory.register_connector(
|
||||
"ECMooncakeConnector",
|
||||
"vllm.distributed.ec_transfer.ec_connector.mooncake_ec_connector",
|
||||
"ECMooncakeConnector",
|
||||
)
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Internal building blocks for the Mooncake encoder-cache connector.
|
||||
|
||||
The public connector remains in :mod:`mooncake_ec_connector`. This package
|
||||
separates configuration, control-plane messaging, transfer state, registered
|
||||
memory, and worker/scheduler orchestration so that each component has one
|
||||
owner and can be tested independently.
|
||||
"""
|
||||
@@ -0,0 +1,123 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.utils.network_utils import make_zmq_path
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
# Scheduler and Worker must agree on when a remote reservation becomes stale.
|
||||
_RESERVATION_TTL_SECONDS = 300
|
||||
|
||||
|
||||
def _positive_int(name: str, value: object) -> int:
|
||||
message = f"ECMooncakeConnector requires {name} to be a positive integer."
|
||||
if isinstance(value, bool):
|
||||
raise ValueError(message)
|
||||
if isinstance(value, int):
|
||||
result = value
|
||||
elif isinstance(value, float) and math.isfinite(value) and value.is_integer():
|
||||
result = int(value)
|
||||
elif isinstance(value, str):
|
||||
try:
|
||||
result = int(value)
|
||||
except ValueError as error:
|
||||
raise ValueError(message) from error
|
||||
else:
|
||||
raise ValueError(message)
|
||||
if result <= 0:
|
||||
raise ValueError(f"ECMooncakeConnector requires {name} > 0.")
|
||||
return result
|
||||
|
||||
|
||||
def _positive_float(name: str, value: object) -> float:
|
||||
message = f"ECMooncakeConnector requires {name} > 0."
|
||||
if isinstance(value, bool):
|
||||
raise ValueError(message)
|
||||
try:
|
||||
result = float(value) # type: ignore[arg-type]
|
||||
except (TypeError, ValueError) as error:
|
||||
raise ValueError(message) from error
|
||||
if not math.isfinite(result) or result <= 0:
|
||||
raise ValueError(message)
|
||||
return result
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MooncakeECConfig:
|
||||
"""Validated settings shared by the Scheduler and Worker roles.
|
||||
|
||||
``control_port`` is the first TP-shard port after the DP offset;
|
||||
``control_addr`` targets that shard, which advertises the full topology.
|
||||
``control_host`` is the address this instance both advertises and binds,
|
||||
so reaching a Consumer means being routed to it rather than finding it on
|
||||
every interface.
|
||||
"""
|
||||
|
||||
is_producer: bool
|
||||
is_consumer: bool
|
||||
protocol: str
|
||||
buffer_device: str
|
||||
control_host: str
|
||||
control_port: int
|
||||
control_addr: str
|
||||
control_timeout_ms: int
|
||||
push_wait_timeout_s: float
|
||||
pool_size: int
|
||||
|
||||
@classmethod
|
||||
def from_vllm_config(cls, vllm_config: VllmConfig) -> MooncakeECConfig:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
ec_config = vllm_config.ec_transfer_config
|
||||
assert ec_config is not None
|
||||
|
||||
if ec_config.is_ec_producer:
|
||||
if parallel_config.tensor_parallel_size > 1:
|
||||
raise ValueError(
|
||||
"ECMooncakeConnector producers require tensor_parallel_size=1."
|
||||
)
|
||||
if parallel_config.pipeline_parallel_size > 1:
|
||||
raise ValueError(
|
||||
"ECMooncakeConnector producers do not support pipeline parallelism."
|
||||
)
|
||||
if parallel_config.data_parallel_size > 1:
|
||||
raise ValueError(
|
||||
"ECMooncakeConnector producers require data_parallel_size=1."
|
||||
)
|
||||
|
||||
registered_buffer_size = _positive_int(
|
||||
"ec_buffer_size", ec_config.ec_buffer_size
|
||||
)
|
||||
get = ec_config.get_from_extra_config
|
||||
control_port = int(ec_config.ec_port) + (
|
||||
parallel_config.data_parallel_index * parallel_config.tensor_parallel_size
|
||||
)
|
||||
highest_port = control_port + parallel_config.tensor_parallel_size - 1
|
||||
if not 1 <= control_port <= highest_port <= 65535:
|
||||
raise ValueError("ECMooncakeConnector ec_port must be in 1..65535.")
|
||||
|
||||
return cls(
|
||||
is_producer=ec_config.is_ec_producer,
|
||||
is_consumer=ec_config.is_ec_consumer,
|
||||
protocol=str(get("mooncake_protocol", "rdma")),
|
||||
buffer_device=str(ec_config.ec_buffer_device or "cuda").lower(),
|
||||
control_host=str(ec_config.ec_ip),
|
||||
control_port=control_port,
|
||||
control_addr=make_zmq_path("tcp", ec_config.ec_ip, control_port),
|
||||
control_timeout_ms=max(
|
||||
1,
|
||||
math.ceil(
|
||||
_positive_float("control_timeout_s", get("control_timeout_s", 30))
|
||||
* 1000
|
||||
),
|
||||
),
|
||||
push_wait_timeout_s=_positive_float(
|
||||
"push_wait_timeout_s", get("push_wait_timeout_s", 60)
|
||||
),
|
||||
pool_size=registered_buffer_size,
|
||||
)
|
||||
@@ -0,0 +1,460 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ZMQ control plane for Mooncake encoder-cache reservations and events.
|
||||
|
||||
Tensor bytes never travel through this module. It coordinates destination
|
||||
reservations, completion/cancellation, TP-shard discovery, and readiness
|
||||
notifications while :mod:`transfer` owns the Mooncake data plane.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import Executor, Future
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import zmq
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.network_utils import make_zmq_path
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_MAX_PENDING_EVENTS = 4096
|
||||
_RESERVATION_REAP_INTERVAL_SECONDS = 1
|
||||
|
||||
|
||||
ControlRequest = dict[str, Any]
|
||||
ControlResponse = dict[str, Any]
|
||||
|
||||
|
||||
class ControlClient:
|
||||
"""Send control requests through reusable, thread-local REQ sockets."""
|
||||
|
||||
def __init__(self, timeout_ms: int) -> None:
|
||||
self._context = zmq.Context()
|
||||
self._timeout_ms = timeout_ms
|
||||
self._local = threading.local()
|
||||
self._topologies: dict[str, list[str]] = {}
|
||||
self._closed = False
|
||||
|
||||
def _sockets(self) -> dict[str, zmq.Socket]:
|
||||
sockets = getattr(self._local, "sockets", None)
|
||||
if sockets is None:
|
||||
sockets = {}
|
||||
self._local.sockets = sockets
|
||||
return sockets
|
||||
|
||||
def _discard(self, addr: str) -> None:
|
||||
socket = self._sockets().pop(addr, None)
|
||||
if socket is not None:
|
||||
socket.close(linger=0)
|
||||
|
||||
def _exchange(self, addr: str, payload: ControlRequest) -> ControlResponse:
|
||||
sockets = self._sockets()
|
||||
socket = sockets.get(addr)
|
||||
if socket is None:
|
||||
socket = self._context.socket(zmq.REQ)
|
||||
socket.setsockopt(zmq.IPV6, 1)
|
||||
socket.setsockopt(zmq.RCVTIMEO, self._timeout_ms)
|
||||
socket.setsockopt(zmq.SNDTIMEO, self._timeout_ms)
|
||||
socket.setsockopt(zmq.LINGER, 0)
|
||||
socket.connect(addr)
|
||||
sockets[addr] = socket
|
||||
try:
|
||||
socket.send_json(payload)
|
||||
response = socket.recv_json()
|
||||
except Exception:
|
||||
self._discard(addr)
|
||||
raise
|
||||
assert isinstance(response, dict)
|
||||
return response
|
||||
|
||||
def request(self, addr: str, payload: ControlRequest) -> Any:
|
||||
response = self._exchange(addr, payload)
|
||||
if not response.get("ok"):
|
||||
raise RuntimeError(response.get("error", "EC control request failed"))
|
||||
return response.get("result")
|
||||
|
||||
def discover_shards(self, base_addr: str) -> list[str] | None:
|
||||
if base_addr in self._topologies:
|
||||
return self._topologies[base_addr]
|
||||
try:
|
||||
reply = self.request(base_addr, {"op": "peers"})
|
||||
ports = reply.get("ports") if isinstance(reply, dict) else None
|
||||
if not isinstance(ports, list) or not ports:
|
||||
raise ValueError("invalid or empty peer list")
|
||||
prefix = base_addr.rsplit(":", 1)[0]
|
||||
shards = [f"{prefix}:{int(port)}" for port in ports]
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"EC Mooncake consumer at %s did not report its shards",
|
||||
base_addr,
|
||||
exc_info=True,
|
||||
)
|
||||
return None
|
||||
self._topologies[base_addr] = shards
|
||||
return shards
|
||||
|
||||
def close(self) -> None:
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
self._context.destroy(linger=0)
|
||||
|
||||
|
||||
def make_cancel_request(
|
||||
transfer_id: str,
|
||||
reservation_id: str = "",
|
||||
*,
|
||||
abandon: bool = False,
|
||||
refresh: bool = False,
|
||||
) -> ControlRequest:
|
||||
request: ControlRequest = {
|
||||
"op": "cancel",
|
||||
"transfer_id": transfer_id,
|
||||
"reservation_id": reservation_id,
|
||||
}
|
||||
if abandon:
|
||||
request["abandon"] = True
|
||||
if refresh:
|
||||
request["refresh"] = True
|
||||
return request
|
||||
|
||||
|
||||
class EventInbox:
|
||||
"""Receive Consumer readiness events without blocking the Scheduler.
|
||||
|
||||
Subscribing needs one blocking control request per shard, so it runs on
|
||||
`executor` when one is given: an unreachable Consumer must cost the
|
||||
Scheduler nothing, not one control timeout per drain.
|
||||
"""
|
||||
|
||||
def __init__(self, client: ControlClient, executor: Executor | None = None) -> None:
|
||||
self._client = client
|
||||
self._executor = executor
|
||||
self._discovery: Future[list[str] | None] | None = None
|
||||
self._context: zmq.Context | None = None
|
||||
self._socket: zmq.Socket | None = None
|
||||
self._closed = False
|
||||
self.shard_count = 1
|
||||
|
||||
def _endpoints(self, base_addr: str) -> list[str] | None:
|
||||
"""Ask every shard where it publishes readiness events.
|
||||
|
||||
Blocking: called on `self._executor` unless there is none.
|
||||
"""
|
||||
shards = self._client.discover_shards(base_addr)
|
||||
if shards is None:
|
||||
return None
|
||||
endpoints = []
|
||||
for addr in shards:
|
||||
try:
|
||||
event_port = self._client.request(addr, {"op": "event_port"})
|
||||
address, _ = addr.rsplit(":", 1)
|
||||
endpoints.append(f"{address}:{int(event_port)}")
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"EC Mooncake could not subscribe to the event channel of "
|
||||
"consumer shard %s; retrying the complete topology later.",
|
||||
addr,
|
||||
)
|
||||
return None
|
||||
return endpoints
|
||||
|
||||
def _connect(self, base_addr: str) -> None:
|
||||
if self._socket is not None or self._closed:
|
||||
return
|
||||
if self._executor is None:
|
||||
self._install(self._endpoints(base_addr))
|
||||
return
|
||||
if self._discovery is None:
|
||||
self._discovery = self._executor.submit(self._endpoints, base_addr)
|
||||
return
|
||||
if not self._discovery.done():
|
||||
return
|
||||
discovery, self._discovery = self._discovery, None
|
||||
try:
|
||||
self._install(discovery.result())
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"EC Mooncake event-channel discovery for %s failed; retrying.",
|
||||
base_addr,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def _install(self, endpoints: list[str] | None) -> None:
|
||||
"""Adopt a discovered topology.
|
||||
|
||||
Runs on the caller's thread so the socket is only ever touched there.
|
||||
"""
|
||||
if not endpoints or self._closed:
|
||||
return
|
||||
context = zmq.Context()
|
||||
socket = context.socket(zmq.PULL)
|
||||
socket.setsockopt(zmq.IPV6, 1)
|
||||
for endpoint in endpoints:
|
||||
socket.connect(endpoint)
|
||||
self._context = context
|
||||
self._socket = socket
|
||||
self.shard_count = len(endpoints)
|
||||
|
||||
def drain(self, base_addr: str) -> list[dict[str, Any]]:
|
||||
self._connect(base_addr)
|
||||
if self._socket is None:
|
||||
return []
|
||||
events = []
|
||||
while True:
|
||||
try:
|
||||
events.append(self._socket.recv_json(flags=zmq.DONTWAIT))
|
||||
except zmq.Again:
|
||||
return events
|
||||
except zmq.ZMQError:
|
||||
logger.warning("EC Mooncake event channel failed", exc_info=True)
|
||||
return events
|
||||
except (ValueError, UnicodeError):
|
||||
# The event channel is a plain PULL socket: an undecodable
|
||||
# frame must cost one frame, not the engine.
|
||||
logger.warning(
|
||||
"Discarding an undecodable EC Mooncake event.", exc_info=True
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
if self._discovery is not None:
|
||||
self._discovery.cancel()
|
||||
self._discovery = None
|
||||
if self._socket is not None:
|
||||
self._socket.close(linger=0)
|
||||
self._socket = None
|
||||
if self._context is not None:
|
||||
self._context.term()
|
||||
|
||||
|
||||
class ConsumerControlServer:
|
||||
"""Expose Consumer reservations and readiness events over ZMQ.
|
||||
|
||||
One server runs on every receiving TP rank. The REP channel handles
|
||||
reservation operations, while a PUSH channel publishes newly ready items
|
||||
to the Scheduler.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
host: str,
|
||||
port: int,
|
||||
reserve: Callable[[dict[str, Any]], dict[str, Any]],
|
||||
status: Callable[[str], dict[str, Any] | None],
|
||||
complete: Callable[[str, str], tuple[bool, bool]],
|
||||
cancel: Callable[[str, str, bool, bool], bool],
|
||||
reap: Callable[[], int],
|
||||
peer_ports: list[int] | None = None,
|
||||
device: torch.device | None = None,
|
||||
drain_ready: Callable[[], list[str]] = lambda: [],
|
||||
) -> None:
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.peer_ports = peer_ports or [port]
|
||||
self._device = device
|
||||
self.event_port: int | None = None
|
||||
self._reserve = reserve
|
||||
self._status = status
|
||||
self._complete = complete
|
||||
self._cancel = cancel
|
||||
self._reap = reap
|
||||
self._drain_ready = drain_ready
|
||||
self._stop = threading.Event()
|
||||
self._started = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
self._startup_error: Exception | None = None
|
||||
|
||||
def start(self) -> None:
|
||||
def loop() -> None:
|
||||
if self._device is not None and self._device.type == "cuda":
|
||||
torch.accelerator.set_device_index(self._device.index or 0)
|
||||
context = zmq.Context()
|
||||
socket = context.socket(zmq.REP)
|
||||
event_socket = context.socket(zmq.PUSH)
|
||||
socket.setsockopt(zmq.IPV6, 1)
|
||||
event_socket.setsockopt(zmq.IPV6, 1)
|
||||
pending_events: deque[dict[str, Any]] = deque()
|
||||
|
||||
def queue_event(event: dict[str, Any]) -> None:
|
||||
event["shard"] = self.port
|
||||
if len(pending_events) >= _MAX_PENDING_EVENTS:
|
||||
dropped = pending_events.popleft()
|
||||
logger.warning(
|
||||
"EC Mooncake event backlog full on port %d; dropping "
|
||||
"readiness for transfer_id=%s",
|
||||
self.port,
|
||||
dropped.get("transfer_id"),
|
||||
)
|
||||
pending_events.append(event)
|
||||
|
||||
def queue_ready(transfer_id: str) -> None:
|
||||
status = self._status(transfer_id)
|
||||
if status is not None:
|
||||
queue_event({"transfer_id": transfer_id, **status})
|
||||
|
||||
last_reap_at = time.monotonic()
|
||||
socket.setsockopt(zmq.RCVTIMEO, 100)
|
||||
try:
|
||||
socket.bind(make_zmq_path("tcp", self.host, self.port))
|
||||
event_socket.bind(make_zmq_path("tcp", self.host, 0))
|
||||
self.event_port = int(
|
||||
event_socket.getsockopt(zmq.LAST_ENDPOINT)
|
||||
.decode()
|
||||
.rsplit(":", 1)[1]
|
||||
)
|
||||
except Exception as e:
|
||||
self._startup_error = e
|
||||
self._started.set()
|
||||
socket.close(linger=0)
|
||||
event_socket.close(linger=0)
|
||||
context.term()
|
||||
return
|
||||
self._started.set()
|
||||
try:
|
||||
while not self._stop.is_set():
|
||||
while pending_events:
|
||||
try:
|
||||
event_socket.send_json(
|
||||
pending_events[0], flags=zmq.DONTWAIT
|
||||
)
|
||||
except zmq.Again:
|
||||
break
|
||||
pending_events.popleft()
|
||||
now = time.monotonic()
|
||||
if now - last_reap_at >= _RESERVATION_REAP_INTERVAL_SECONDS:
|
||||
self._reap()
|
||||
last_reap_at = now
|
||||
try:
|
||||
request = socket.recv_json()
|
||||
except zmq.Again:
|
||||
continue
|
||||
except Exception:
|
||||
# The frame arrived but did not decode. REP still owes a
|
||||
# reply, so answer before returning to the loop.
|
||||
logger.exception(
|
||||
"EC Mooncake control channel on port %d received an "
|
||||
"undecodable request",
|
||||
self.port,
|
||||
)
|
||||
socket.send_json(
|
||||
{"ok": False, "error": "malformed control request"}
|
||||
)
|
||||
continue
|
||||
try:
|
||||
op = request.get("op")
|
||||
result: Any = None
|
||||
if op in ("reserve", "reserve_batch"):
|
||||
items = (
|
||||
request["items"] if op == "reserve_batch" else [request]
|
||||
)
|
||||
results = []
|
||||
for item in items:
|
||||
try:
|
||||
reserved = self._reserve(item)
|
||||
results.append({"ok": True, "result": reserved})
|
||||
if reserved.get("ready"):
|
||||
queue_ready(str(item["transfer_id"]))
|
||||
except Exception as exc:
|
||||
results.append({"ok": False, "error": str(exc)})
|
||||
if op == "reserve_batch":
|
||||
result = {"items": results}
|
||||
elif not results[0]["ok"]:
|
||||
raise RuntimeError(results[0]["error"])
|
||||
else:
|
||||
result = results[0]["result"]
|
||||
elif op == "status":
|
||||
result = self._status(str(request["transfer_id"]))
|
||||
elif op == "event_port":
|
||||
result = self.event_port
|
||||
elif op == "peers":
|
||||
result = {"ports": self.peer_ports}
|
||||
elif op in ("complete", "complete_batch"):
|
||||
items = (
|
||||
request["items"]
|
||||
if op == "complete_batch"
|
||||
else [request]
|
||||
)
|
||||
completions = []
|
||||
for item in items:
|
||||
transfer_id = str(item["transfer_id"])
|
||||
accepted, became_ready = self._complete(
|
||||
transfer_id, str(item["reservation_id"])
|
||||
)
|
||||
for ready_id in self._drain_ready():
|
||||
queue_ready(ready_id)
|
||||
completions.append(
|
||||
{
|
||||
"completed": accepted,
|
||||
"became_ready": became_ready,
|
||||
}
|
||||
)
|
||||
if not became_ready:
|
||||
continue
|
||||
queue_ready(transfer_id)
|
||||
result = (
|
||||
{"items": completions}
|
||||
if op == "complete_batch"
|
||||
else completions[0]
|
||||
)
|
||||
elif op == "cancel":
|
||||
result = {
|
||||
"cancelled": self._cancel(
|
||||
str(request["transfer_id"]),
|
||||
str(request.get("reservation_id", "")),
|
||||
bool(request.get("abandon", False)),
|
||||
bool(request.get("refresh", False)),
|
||||
)
|
||||
}
|
||||
else:
|
||||
raise ValueError(f"unknown control op: {op!r}")
|
||||
socket.send_json({"ok": True, "result": result})
|
||||
except Exception as e:
|
||||
socket.send_json({"ok": False, "error": str(e)})
|
||||
except Exception:
|
||||
# `finally` closes the sockets and the thread ends, so without
|
||||
# this every later reserve against this shard would surface
|
||||
# only as a control timeout with nothing to attribute it to.
|
||||
logger.exception(
|
||||
"EC Mooncake control channel on port %d stopped serving",
|
||||
self.port,
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
socket.close(linger=0)
|
||||
event_socket.close(linger=0)
|
||||
context.term()
|
||||
|
||||
self._thread = threading.Thread(
|
||||
target=loop, name="ec-mooncake-control", daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
if not self._started.wait(timeout=5):
|
||||
raise RuntimeError("EC Mooncake control channel failed to start")
|
||||
if self._startup_error is not None:
|
||||
raise RuntimeError("EC Mooncake control channel failed to bind") from (
|
||||
self._startup_error
|
||||
)
|
||||
logger.info(
|
||||
"EC Mooncake control channel listening on tcp://%s:%d (events tcp://%s:%d)",
|
||||
self.host,
|
||||
self.port,
|
||||
self.host,
|
||||
self.event_port,
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
if self._thread is None:
|
||||
return
|
||||
self._stop.set()
|
||||
self._thread.join()
|
||||
self._thread = None
|
||||
@@ -0,0 +1,504 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Registered-memory allocation and residency for Mooncake transfers.
|
||||
|
||||
The Producer pool stages source tensors in a registered slab. The Consumer
|
||||
pool owns destination allocations from reservation through publication,
|
||||
resident reuse, CUDA-safe retirement, and pressure-driven reclamation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import bisect
|
||||
import threading
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Generic, TypeVar
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.transfer import (
|
||||
MooncakeTransfer,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
@dataclass
|
||||
class MemoryAllocation:
|
||||
"""Describe a tensor view carved from the Consumer receive slab."""
|
||||
|
||||
offset: int
|
||||
size: int
|
||||
tensor: torch.Tensor
|
||||
|
||||
|
||||
@dataclass
|
||||
class _ResidentEntry(Generic[_T]):
|
||||
"""Store one resident value and its ownership accounting."""
|
||||
|
||||
value: _T
|
||||
pinned: bool = True
|
||||
leases: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class ResidentLease(Generic[_T]):
|
||||
"""Represent a borrow of one resident entry."""
|
||||
|
||||
key: str
|
||||
_entry: _ResidentEntry[_T]
|
||||
_active: bool = True
|
||||
|
||||
@property
|
||||
def value(self) -> _T:
|
||||
return self._entry.value
|
||||
|
||||
|
||||
class ContiguousAllocator:
|
||||
"""Own a registered slab and its aligned regions; callers serialize access."""
|
||||
|
||||
def __init__(self, capacity: int, alignment: int = 256):
|
||||
self.alignment = alignment
|
||||
self._capacity = capacity
|
||||
self._free = [(0, capacity)]
|
||||
self.tensor: torch.Tensor | None = None
|
||||
self._disabled = False
|
||||
|
||||
def prepare(self, device: torch.device, transfer: MooncakeTransfer) -> None:
|
||||
if self.tensor is not None or self._disabled:
|
||||
return
|
||||
try:
|
||||
tensor = torch.empty(self._capacity, dtype=torch.uint8, device=device)
|
||||
ret = transfer.register_memory(tensor)
|
||||
if ret != 0:
|
||||
raise RuntimeError(f"Mooncake returned {ret}")
|
||||
except (RuntimeError, torch.OutOfMemoryError) as error:
|
||||
self._disabled = True
|
||||
logger.warning("Could not initialize the EC registered buffer: %s", error)
|
||||
return
|
||||
self.tensor = tensor
|
||||
self._free = [(0, tensor.nbytes)]
|
||||
logger.info("Registered %d-byte buffer for Mooncake EC", tensor.nbytes)
|
||||
|
||||
def view(
|
||||
self, offset: int, nbytes: int, shape: tuple[int, ...], dtype: torch.dtype
|
||||
) -> torch.Tensor:
|
||||
assert self.tensor is not None
|
||||
return self.tensor.narrow(0, offset, nbytes).view(dtype).view(shape)
|
||||
|
||||
def close(self, transfer: MooncakeTransfer) -> bool:
|
||||
if self.tensor is not None and not transfer.unregister_memory(self.tensor):
|
||||
return False
|
||||
self.tensor = None
|
||||
self._free.clear()
|
||||
return True
|
||||
|
||||
def allocate(self, nbytes: int) -> tuple[int, int] | None:
|
||||
size = (nbytes + self.alignment - 1) // self.alignment * self.alignment
|
||||
for index, (offset, available) in enumerate(self._free):
|
||||
if size > available:
|
||||
continue
|
||||
if size == available:
|
||||
self._free.pop(index)
|
||||
else:
|
||||
self._free[index] = (offset + size, available - size)
|
||||
return offset, size
|
||||
return None
|
||||
|
||||
def free(self, offset: int, size: int) -> None:
|
||||
index = bisect.bisect_left(self._free, (offset, size))
|
||||
self._free.insert(index, (offset, size))
|
||||
if index + 1 < len(self._free):
|
||||
next_offset, next_size = self._free[index + 1]
|
||||
if offset + size == next_offset:
|
||||
self._free[index] = (offset, size + next_size)
|
||||
self._free.pop(index + 1)
|
||||
if index > 0:
|
||||
previous_offset, previous_size = self._free[index - 1]
|
||||
current_offset, current_size = self._free[index]
|
||||
if previous_offset + previous_size == current_offset:
|
||||
self._free[index - 1] = (
|
||||
previous_offset,
|
||||
previous_size + current_size,
|
||||
)
|
||||
self._free.pop(index)
|
||||
|
||||
|
||||
class ResidentPool(Generic[_T]):
|
||||
"""Track resident values, active leases, and LRU eviction eligibility."""
|
||||
|
||||
def __init__(self):
|
||||
self._entries: dict[str, _ResidentEntry[_T]] = {}
|
||||
self._evictable: OrderedDict[str, None] = OrderedDict()
|
||||
|
||||
def referenced(self) -> list[str]:
|
||||
return [key for key, entry in self._entries.items() if entry.pinned]
|
||||
|
||||
def get(self, key: str) -> _T | None:
|
||||
entry = self._entries.get(key)
|
||||
return entry.value if entry is not None else None
|
||||
|
||||
def insert(
|
||||
self,
|
||||
key: str,
|
||||
value: _T,
|
||||
) -> _T | None:
|
||||
"""Pin an entry and return a displaced value that has no owners."""
|
||||
previous = self._entries.get(key)
|
||||
entry = _ResidentEntry(value)
|
||||
self._entries[key] = entry
|
||||
self._evictable.pop(key, None)
|
||||
if previous is not None and previous.leases == 0:
|
||||
return previous.value
|
||||
return None
|
||||
|
||||
def pin(self, key: str) -> _T | None:
|
||||
entry = self._entries.get(key)
|
||||
if entry is None:
|
||||
return None
|
||||
self._evictable.pop(key, None)
|
||||
entry.pinned = True
|
||||
return entry.value
|
||||
|
||||
def acquire(self, key: str) -> ResidentLease[_T] | None:
|
||||
entry = self._entries.get(key)
|
||||
if entry is None:
|
||||
return None
|
||||
entry.leases += 1
|
||||
self._evictable.pop(key, None)
|
||||
return ResidentLease(key, entry)
|
||||
|
||||
def release(self, lease: ResidentLease[_T]) -> _T | None:
|
||||
if not lease._active:
|
||||
return None
|
||||
lease._active = False
|
||||
entry = lease._entry
|
||||
entry.leases -= 1
|
||||
current = self._entries.get(lease.key)
|
||||
if current is not entry:
|
||||
if entry.leases == 0:
|
||||
return entry.value
|
||||
return None
|
||||
if not entry.pinned and entry.leases == 0:
|
||||
self._evictable[lease.key] = None
|
||||
return None
|
||||
|
||||
def consume(self, lease: ResidentLease[_T]) -> tuple[_T, _T | None]:
|
||||
current = self._entries[lease.key]
|
||||
current.pinned = True
|
||||
self._evictable.pop(lease.key, None)
|
||||
released = self.release(lease)
|
||||
return current.value, released
|
||||
|
||||
def retire(self, key: str) -> None:
|
||||
if key not in self._entries:
|
||||
return
|
||||
entry = self._entries[key]
|
||||
entry.pinned = False
|
||||
if entry.leases == 0:
|
||||
self._evictable[key] = None
|
||||
|
||||
def evict_lru(self, evict: Callable[[str, _T], bool]) -> str | None:
|
||||
for key in list(self._evictable):
|
||||
entry = self._entries[key]
|
||||
if not evict(key, entry.value):
|
||||
continue
|
||||
self._evictable.pop(key, None)
|
||||
del self._entries[key]
|
||||
return key
|
||||
return None
|
||||
|
||||
def clear(self) -> None:
|
||||
self._entries.clear()
|
||||
self._evictable.clear()
|
||||
|
||||
|
||||
@dataclass
|
||||
class StagedSources:
|
||||
"""Own Producer tensor views and their staging-slab regions."""
|
||||
|
||||
tensors: list[torch.Tensor]
|
||||
regions: list[tuple[int, int]]
|
||||
|
||||
|
||||
class ProducerMemoryPool:
|
||||
"""Own the Producer staging slab and regions carved from it."""
|
||||
|
||||
def __init__(self, capacity: int, transfer: MooncakeTransfer) -> None:
|
||||
self._transfer = transfer
|
||||
self._allocator = ContiguousAllocator(capacity)
|
||||
self._lock = threading.Lock()
|
||||
self._local = threading.local()
|
||||
|
||||
@property
|
||||
def tensor(self) -> torch.Tensor | None:
|
||||
return self._allocator.tensor
|
||||
|
||||
def _free_regions(self, regions: list[tuple[int, int]]) -> None:
|
||||
for offset, size in regions:
|
||||
self._allocator.free(offset, size)
|
||||
|
||||
def stage(self, tensors: list[torch.Tensor]) -> StagedSources | None:
|
||||
"""Copy tensors into one registered slab, or return None for fallback."""
|
||||
if not tensors:
|
||||
return StagedSources([], [])
|
||||
allocator = self._allocator
|
||||
staged: list[torch.Tensor] = []
|
||||
regions: list[tuple[int, int]] = []
|
||||
with self._lock:
|
||||
allocator.prepare(tensors[0].device, self._transfer)
|
||||
pool = allocator.tensor
|
||||
if pool is None:
|
||||
return None
|
||||
for tensor in tensors:
|
||||
region = allocator.allocate(tensor.nbytes)
|
||||
if region is None:
|
||||
self._free_regions(regions)
|
||||
return None
|
||||
regions.append(region)
|
||||
staged.append(
|
||||
allocator.view(
|
||||
region[0], tensor.nbytes, tuple(tensor.shape), tensor.dtype
|
||||
)
|
||||
)
|
||||
if pool.device.type == "cuda":
|
||||
stream = getattr(self._local, "stream", None)
|
||||
if stream is None:
|
||||
stream = torch.cuda.Stream(device=pool.device)
|
||||
self._local.stream = stream
|
||||
with torch.cuda.stream(stream):
|
||||
for destination, source in zip(staged, tensors):
|
||||
destination.copy_(source, non_blocking=True)
|
||||
stream.synchronize()
|
||||
else:
|
||||
for destination, source in zip(staged, tensors):
|
||||
destination.copy_(source)
|
||||
return StagedSources(staged, regions)
|
||||
|
||||
def release(self, staged: StagedSources) -> None:
|
||||
if not staged.regions:
|
||||
return
|
||||
with self._lock:
|
||||
self._free_regions(staged.regions)
|
||||
|
||||
def close(self) -> None:
|
||||
with self._lock:
|
||||
self._allocator.close(self._transfer)
|
||||
|
||||
|
||||
class ConsumerMemoryPool:
|
||||
"""Own the registered receive slab and resident allocation lifecycle."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
capacity: int,
|
||||
transfer: MooncakeTransfer,
|
||||
) -> None:
|
||||
self._transfer = transfer
|
||||
self._allocator = ContiguousAllocator(capacity)
|
||||
self._residents: ResidentPool[MemoryAllocation] = ResidentPool()
|
||||
self._retire_events: dict[str, torch.Event] = {}
|
||||
self._pending_frees: list[tuple[torch.Event, MemoryAllocation]] = []
|
||||
self._reclaimed: set[str] = set()
|
||||
self.lock = threading.RLock()
|
||||
|
||||
@property
|
||||
def tensor(self) -> torch.Tensor | None:
|
||||
return self._allocator.tensor
|
||||
|
||||
def prepare(
|
||||
self,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
with self.lock:
|
||||
self._allocator.prepare(device, self._transfer)
|
||||
|
||||
def _free(self, allocation: MemoryAllocation) -> None:
|
||||
self._allocator.free(allocation.offset, allocation.size)
|
||||
|
||||
def free(self, allocation: MemoryAllocation) -> None:
|
||||
with self.lock:
|
||||
self._free(allocation)
|
||||
|
||||
def _defer_or_free(
|
||||
self, allocation: MemoryAllocation, event: torch.Event | None
|
||||
) -> None:
|
||||
if event is None or event.query():
|
||||
self._free(allocation)
|
||||
else:
|
||||
self._pending_frees.append((event, allocation))
|
||||
|
||||
def _poll_frees_locked(self) -> None:
|
||||
pending = []
|
||||
for event, allocation in self._pending_frees:
|
||||
if event.query():
|
||||
self._free(allocation)
|
||||
else:
|
||||
pending.append((event, allocation))
|
||||
self._pending_frees = pending
|
||||
|
||||
def _reclaim_locked(self, nbytes: int) -> tuple[int, int] | None:
|
||||
def evict(mm_hash: str, allocation: MemoryAllocation) -> bool:
|
||||
event = self._retire_events.pop(mm_hash, None)
|
||||
self._defer_or_free(allocation, event)
|
||||
self._reclaimed.add(mm_hash)
|
||||
return True
|
||||
|
||||
while self._residents.evict_lru(evict) is not None:
|
||||
# Eviction defers any free whose CUDA event is still pending, so
|
||||
# those bytes reach the allocator only once the event is polled.
|
||||
self._poll_frees_locked()
|
||||
region = self._allocator.allocate(nbytes)
|
||||
if region is not None:
|
||||
return region
|
||||
self._poll_frees_locked()
|
||||
return self._allocator.allocate(nbytes)
|
||||
|
||||
def _make_allocation(
|
||||
self,
|
||||
region: tuple[int, int],
|
||||
nbytes: int,
|
||||
shape: tuple[int, ...],
|
||||
dtype: torch.dtype,
|
||||
) -> MemoryAllocation:
|
||||
offset, size = region
|
||||
tensor = self._allocator.view(offset, nbytes, shape, dtype)
|
||||
return MemoryAllocation(offset, size, tensor)
|
||||
|
||||
def try_allocate(
|
||||
self, nbytes: int, shape: tuple[int, ...], dtype: torch.dtype
|
||||
) -> MemoryAllocation | None:
|
||||
with self.lock:
|
||||
allocator = self._allocator
|
||||
assert allocator.tensor is not None
|
||||
self._poll_frees_locked()
|
||||
region = allocator.allocate(nbytes)
|
||||
if region is None:
|
||||
return None
|
||||
return self._make_allocation(region, nbytes, shape, dtype)
|
||||
|
||||
def reclaim_and_allocate(
|
||||
self, nbytes: int, shape: tuple[int, ...], dtype: torch.dtype
|
||||
) -> MemoryAllocation | None:
|
||||
with self.lock:
|
||||
region = self._reclaim_locked(nbytes)
|
||||
if region is None:
|
||||
return None
|
||||
return self._make_allocation(region, nbytes, shape, dtype)
|
||||
|
||||
def acquire_cached(
|
||||
self, mm_hash: str, shape: tuple[int, ...], dtype: torch.dtype
|
||||
) -> ResidentLease[MemoryAllocation] | None:
|
||||
with self.lock:
|
||||
allocation = self._residents.get(mm_hash)
|
||||
if allocation is None:
|
||||
return None
|
||||
if (
|
||||
tuple(allocation.tensor.shape) != shape
|
||||
or allocation.tensor.dtype != dtype
|
||||
):
|
||||
raise ValueError("conflicting cached tensor for mm_hash")
|
||||
return self._residents.acquire(mm_hash)
|
||||
|
||||
def take_resident(
|
||||
self, mm_hash: str, shape: tuple[int, ...], dtype_name: str
|
||||
) -> torch.Tensor | None:
|
||||
with self.lock:
|
||||
allocation = self._residents.get(mm_hash)
|
||||
if allocation is None:
|
||||
return None
|
||||
tensor = allocation.tensor
|
||||
if (
|
||||
tuple(tensor.shape) != shape
|
||||
or str(tensor.dtype).split(".")[-1] != dtype_name
|
||||
):
|
||||
return None
|
||||
self._residents.pin(mm_hash)
|
||||
self._retire_events.pop(mm_hash, None)
|
||||
return tensor
|
||||
|
||||
def _record_release_event(self) -> torch.Event | None:
|
||||
pool = self.tensor
|
||||
if pool is None or pool.device.type != "cuda":
|
||||
return None
|
||||
event = torch.Event()
|
||||
event.record(torch.accelerator.current_stream(pool.device))
|
||||
return event
|
||||
|
||||
def release_cached(self, lease: ResidentLease[MemoryAllocation]) -> None:
|
||||
with self.lock:
|
||||
released = self._residents.release(lease)
|
||||
if released is not None:
|
||||
self._defer_or_free(released, self._record_release_event())
|
||||
|
||||
def publish(
|
||||
self,
|
||||
mm_hash: str,
|
||||
allocation: MemoryAllocation,
|
||||
lease: ResidentLease[MemoryAllocation] | None = None,
|
||||
*,
|
||||
pin: bool = True,
|
||||
) -> MemoryAllocation:
|
||||
with self.lock:
|
||||
if lease is not None:
|
||||
canonical, released = self._residents.consume(lease)
|
||||
self._retire_events.pop(mm_hash, None)
|
||||
if released is not None:
|
||||
self._defer_or_free(released, self._record_release_event())
|
||||
return canonical
|
||||
previous = self._residents.get(mm_hash)
|
||||
displaced = self._residents.insert(mm_hash, allocation)
|
||||
if not pin:
|
||||
self._residents.retire(mm_hash)
|
||||
event = None
|
||||
if previous is not None and previous is not allocation:
|
||||
event = self._retire_events.pop(mm_hash, None)
|
||||
if displaced is None or displaced is allocation:
|
||||
return allocation
|
||||
if event is None:
|
||||
event = self._record_release_event()
|
||||
self._defer_or_free(displaced, event)
|
||||
return allocation
|
||||
|
||||
def retire_stale(
|
||||
self,
|
||||
encoder_cache: dict[str, torch.Tensor],
|
||||
reserved_hashes: set[str] | None = None,
|
||||
*,
|
||||
freed: list[str] | None = None,
|
||||
) -> None:
|
||||
if self.tensor is None:
|
||||
return
|
||||
with self.lock:
|
||||
for mm_hash in self._residents.referenced() if freed is None else freed:
|
||||
allocation = self._residents.get(mm_hash)
|
||||
if allocation is None:
|
||||
continue
|
||||
if encoder_cache.get(mm_hash) is allocation.tensor:
|
||||
continue
|
||||
if reserved_hashes and mm_hash in reserved_hashes:
|
||||
continue
|
||||
event = self._record_release_event()
|
||||
if event is not None:
|
||||
self._retire_events[mm_hash] = event
|
||||
self._residents.retire(mm_hash)
|
||||
self._poll_frees_locked()
|
||||
|
||||
def drain_reclaimed(self) -> set[str]:
|
||||
with self.lock:
|
||||
reclaimed = self._reclaimed
|
||||
self._reclaimed = set()
|
||||
return reclaimed
|
||||
|
||||
def close(self) -> None:
|
||||
with self.lock:
|
||||
if not self._allocator.close(self._transfer):
|
||||
return
|
||||
self._residents.clear()
|
||||
self._retire_events.clear()
|
||||
self._pending_frees.clear()
|
||||
@@ -0,0 +1,75 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Metadata exchanged by the Mooncake encoder-cache connector."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorMetadata,
|
||||
ECConnectorWorkerMetadata,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ECMooncakeLoadSpec:
|
||||
"""Describe a remote reservation or resident allocation to load."""
|
||||
|
||||
mm_hash: str
|
||||
nbytes: int
|
||||
shape: tuple[int, ...]
|
||||
dtype: str
|
||||
transfer_id: str
|
||||
# The consumer pool still holds this item, so the load is a local handoff:
|
||||
# no transfer, no producer.
|
||||
local: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class ECMooncakePushSpec:
|
||||
"""Describe a destination reservation prepared before a tensor is ready."""
|
||||
|
||||
mm_hash: str
|
||||
nbytes: int
|
||||
shape: tuple[int, ...]
|
||||
dtype: str
|
||||
consumer_zmq: str
|
||||
transfer_id: str
|
||||
request_id: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class ECMooncakeConnectorMetadata(ECConnectorMetadata):
|
||||
"""Worker operations emitted for one Scheduler step."""
|
||||
|
||||
loads: list[ECMooncakeLoadSpec] = field(default_factory=list)
|
||||
pushes: list[ECMooncakePushSpec] = field(default_factory=list)
|
||||
freed: list[str] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ECMooncakeWorkerMetadata(ECConnectorWorkerMetadata):
|
||||
"""Completion state reported from Workers to the Scheduler."""
|
||||
|
||||
loaded: set[str] = field(default_factory=set)
|
||||
failed_loads: set[str] = field(default_factory=set)
|
||||
# Items the receive pool dropped under pressure. The scheduler assumes an
|
||||
# evicted item stays resident until told otherwise.
|
||||
reclaimed: set[str] = field(default_factory=set)
|
||||
pending_saves: bool = False
|
||||
failed_saves: set[str] = field(default_factory=set)
|
||||
|
||||
def aggregate(self, other: ECConnectorWorkerMetadata) -> ECMooncakeWorkerMetadata:
|
||||
assert isinstance(other, ECMooncakeWorkerMetadata)
|
||||
return ECMooncakeWorkerMetadata(
|
||||
# Every tensor-parallel rank gathers the embedding from its own
|
||||
# cache, so an item counts as loaded only where all of them have
|
||||
# it; one rank falling short must fail the load rather than leave
|
||||
# the scheduler believing it is ready.
|
||||
loaded=self.loaded & other.loaded,
|
||||
failed_loads=self.failed_loads | other.failed_loads,
|
||||
reclaimed=self.reclaimed | other.reclaimed,
|
||||
pending_saves=self.pending_saves or other.pending_saves,
|
||||
failed_saves=self.failed_saves | other.failed_saves,
|
||||
)
|
||||
@@ -0,0 +1,418 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Producer-side push lifecycle, futures, and source-tensor ownership.
|
||||
|
||||
This module records when a push may advance or release its source. Worker
|
||||
orchestration performs the actual control exchanges and Mooncake writes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum, auto
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
|
||||
ECMooncakePushSpec,
|
||||
)
|
||||
|
||||
|
||||
def _same_destination(
|
||||
existing: ECMooncakePushSpec, incoming: ECMooncakePushSpec
|
||||
) -> bool:
|
||||
"""Whether two specs describe one push, ignoring which request asked.
|
||||
|
||||
The same encoding requested twice shares a transfer id legitimately; only
|
||||
a different payload or destination is a collision.
|
||||
"""
|
||||
return (
|
||||
existing.mm_hash == incoming.mm_hash
|
||||
and existing.nbytes == incoming.nbytes
|
||||
and existing.shape == incoming.shape
|
||||
and existing.dtype == incoming.dtype
|
||||
and existing.consumer_zmq == incoming.consumer_zmq
|
||||
)
|
||||
|
||||
|
||||
class ProducerPushState(Enum):
|
||||
"""Lifecycle of one Producer push from reservation to terminal state."""
|
||||
|
||||
# The reservation reply and source tensor may arrive in either order.
|
||||
WAITING_INPUTS = auto()
|
||||
WRITING = auto()
|
||||
NOTIFYING = auto()
|
||||
DONE = auto()
|
||||
CANCEL_PENDING = auto()
|
||||
CANCELLED = auto()
|
||||
FAILED = auto()
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProducerPushRecord:
|
||||
"""Collect all asynchronous state for one Producer push."""
|
||||
|
||||
spec: ECMooncakePushSpec
|
||||
state: ProducerPushState
|
||||
reservation_future: Future[list[dict[str, Any]]]
|
||||
reservations: list[dict[str, Any]] = field(default_factory=list)
|
||||
shard_futures: list[Future[Any]] = field(default_factory=list)
|
||||
source_tensor: torch.Tensor | None = None
|
||||
source_event: torch.Event | None = None
|
||||
batch_future: Future[None] | None = None
|
||||
error: str | None = None
|
||||
|
||||
|
||||
_ALLOWED_TRANSITIONS = {
|
||||
ProducerPushState.WAITING_INPUTS: {
|
||||
ProducerPushState.WRITING,
|
||||
ProducerPushState.CANCEL_PENDING,
|
||||
ProducerPushState.FAILED,
|
||||
},
|
||||
ProducerPushState.WRITING: {
|
||||
ProducerPushState.NOTIFYING,
|
||||
ProducerPushState.FAILED,
|
||||
},
|
||||
ProducerPushState.NOTIFYING: {
|
||||
ProducerPushState.DONE,
|
||||
ProducerPushState.FAILED,
|
||||
},
|
||||
ProducerPushState.CANCEL_PENDING: {
|
||||
ProducerPushState.CANCELLED,
|
||||
ProducerPushState.FAILED,
|
||||
},
|
||||
ProducerPushState.DONE: set(),
|
||||
ProducerPushState.CANCELLED: set(),
|
||||
ProducerPushState.FAILED: set(),
|
||||
}
|
||||
|
||||
_TERMINAL_STATES = {
|
||||
ProducerPushState.DONE,
|
||||
ProducerPushState.CANCELLED,
|
||||
ProducerPushState.FAILED,
|
||||
}
|
||||
_TERMINAL_LIMIT = 1 << 16
|
||||
|
||||
|
||||
class ProducerPushManager:
|
||||
"""Own Producer push records, transitions, and source tensor leases.
|
||||
|
||||
Terminal records are retained for duplicate-metadata idempotency. Separate
|
||||
ordered indexes keep hot polling paths from scanning those tombstones.
|
||||
"""
|
||||
|
||||
def __init__(self, wake: Callable[[], None] = lambda: None) -> None:
|
||||
self._records: OrderedDict[str, ProducerPushRecord] = OrderedDict()
|
||||
self._active_ids: OrderedDict[str, None] = OrderedDict()
|
||||
self._reapable_terminal_ids: OrderedDict[str, None] = OrderedDict()
|
||||
self._unreported_ids: OrderedDict[str, None] = OrderedDict()
|
||||
self._batch_ids: OrderedDict[str, None] = OrderedDict()
|
||||
self._source_waiters: dict[str, OrderedDict[str, None]] = {}
|
||||
self._lock = threading.RLock()
|
||||
self._wake = wake
|
||||
|
||||
def reserve(
|
||||
self,
|
||||
spec: ECMooncakePushSpec,
|
||||
submit: Callable[[], Future[list[dict[str, Any]]]],
|
||||
) -> tuple[ProducerPushRecord, bool]:
|
||||
with self._lock:
|
||||
existing = self._records.get(spec.transfer_id)
|
||||
if existing is not None:
|
||||
if not _same_destination(existing.spec, spec):
|
||||
raise ValueError(
|
||||
f"Conflicting EC destination for transfer_id={spec.transfer_id}"
|
||||
)
|
||||
return existing, False
|
||||
record = ProducerPushRecord(
|
||||
spec=spec,
|
||||
state=ProducerPushState.WAITING_INPUTS,
|
||||
reservation_future=submit(),
|
||||
)
|
||||
self._records[spec.transfer_id] = record
|
||||
self._active_ids[spec.transfer_id] = None
|
||||
self._source_waiters.setdefault(spec.mm_hash, OrderedDict())[
|
||||
spec.transfer_id
|
||||
] = None
|
||||
record.reservation_future.add_done_callback(
|
||||
lambda future: self._reservation_done(record, future)
|
||||
)
|
||||
return record, True
|
||||
|
||||
def bind_source(
|
||||
self,
|
||||
mm_hash: str,
|
||||
tensor: torch.Tensor,
|
||||
ready_event: torch.Event | None,
|
||||
) -> None:
|
||||
with self._lock:
|
||||
waiters = self._source_waiters.pop(mm_hash, OrderedDict())
|
||||
for transfer_id in waiters:
|
||||
record = self._records[transfer_id]
|
||||
if (
|
||||
record.source_tensor is not None
|
||||
or record.state is not ProducerPushState.WAITING_INPUTS
|
||||
):
|
||||
continue
|
||||
record.source_tensor = tensor
|
||||
record.source_event = ready_event
|
||||
self._wake()
|
||||
|
||||
def submit_batches(
|
||||
self,
|
||||
executor: ThreadPoolExecutor,
|
||||
run_batch: Callable[[list[ProducerPushRecord]], None],
|
||||
*,
|
||||
wait: bool = False,
|
||||
) -> bool:
|
||||
pending_event = False
|
||||
with self._lock:
|
||||
grouped: dict[str, list[ProducerPushRecord]] = {}
|
||||
for transfer_id in list(self._active_ids):
|
||||
record = self._records[transfer_id]
|
||||
if (
|
||||
record.source_tensor is not None
|
||||
and record.batch_future is None
|
||||
and record.state is ProducerPushState.WAITING_INPUTS
|
||||
):
|
||||
if not record.reservation_future.done():
|
||||
continue
|
||||
if record.source_event is not None:
|
||||
if wait:
|
||||
record.source_event.synchronize()
|
||||
elif not record.source_event.query():
|
||||
pending_event = True
|
||||
continue
|
||||
grouped.setdefault(record.spec.consumer_zmq, []).append(record)
|
||||
batches = list(grouped.values())
|
||||
for records in batches:
|
||||
future = executor.submit(run_batch, records)
|
||||
for record in records:
|
||||
record.batch_future = future
|
||||
self._batch_ids[record.spec.transfer_id] = None
|
||||
return pending_event
|
||||
|
||||
def resolve_reservations(self, record: ProducerPushRecord) -> list[dict[str, Any]]:
|
||||
results = record.reservation_future.result()
|
||||
with self._lock:
|
||||
record.reservations = results
|
||||
return list(results)
|
||||
|
||||
def finish_reservations(
|
||||
self,
|
||||
outcomes: list[tuple[ProducerPushRecord, list[dict[str, Any]] | Exception]],
|
||||
) -> None:
|
||||
with self._lock:
|
||||
for record, result in outcomes:
|
||||
if isinstance(result, Exception):
|
||||
record.reservation_future.set_exception(result)
|
||||
else:
|
||||
record.reservation_future.set_result(result)
|
||||
|
||||
def _reservation_done(
|
||||
self,
|
||||
record: ProducerPushRecord,
|
||||
future: Future[list[dict[str, Any]]],
|
||||
) -> None:
|
||||
try:
|
||||
future.result()
|
||||
except Exception as exc:
|
||||
with self._lock:
|
||||
self._set_error(record, exc)
|
||||
if (
|
||||
record.state is ProducerPushState.WAITING_INPUTS
|
||||
and record.source_tensor is None
|
||||
):
|
||||
self._transition(record, ProducerPushState.FAILED)
|
||||
finally:
|
||||
self._wake()
|
||||
|
||||
def settle_all(self, records: list[ProducerPushRecord]) -> None:
|
||||
for record in records:
|
||||
with suppress(Exception):
|
||||
record.reservation_future.result()
|
||||
|
||||
def replace_reservations(
|
||||
self,
|
||||
record: ProducerPushRecord,
|
||||
reservations: list[dict[str, Any]],
|
||||
) -> None:
|
||||
with self._lock:
|
||||
record.reservations = reservations
|
||||
|
||||
def track_shard_futures(
|
||||
self,
|
||||
records: list[ProducerPushRecord],
|
||||
futures: list[Future[Any]],
|
||||
) -> None:
|
||||
with self._lock:
|
||||
for record in records:
|
||||
record.shard_futures.extend(futures)
|
||||
|
||||
def begin_writing(self, record: ProducerPushRecord) -> None:
|
||||
with self._lock:
|
||||
if record.source_tensor is None:
|
||||
raise RuntimeError(
|
||||
f"Producer push {record.spec.transfer_id!r} has no source tensor"
|
||||
)
|
||||
self._transition(record, ProducerPushState.WRITING)
|
||||
|
||||
def begin_notifying(self, records: list[ProducerPushRecord]) -> None:
|
||||
with self._lock:
|
||||
for record in records:
|
||||
self._transition(record, ProducerPushState.NOTIFYING)
|
||||
|
||||
def complete(self, records: list[ProducerPushRecord]) -> None:
|
||||
with self._lock:
|
||||
self._check_sources_releasable(records)
|
||||
for record in records:
|
||||
self._transition(record, ProducerPushState.DONE)
|
||||
self._release_source(record)
|
||||
|
||||
def fail(self, records: list[ProducerPushRecord], error: Exception) -> None:
|
||||
with self._lock:
|
||||
self._check_sources_releasable(records)
|
||||
for record in records:
|
||||
if record.state not in _TERMINAL_STATES:
|
||||
self._set_error(record, error)
|
||||
self._transition(record, ProducerPushState.FAILED)
|
||||
self._release_source(record)
|
||||
|
||||
def cancel_requests(self, request_ids: set[str] | None) -> list[ProducerPushRecord]:
|
||||
"""Cancel source-less waiters for requests, or all waiters at shutdown."""
|
||||
cancelled = []
|
||||
with self._lock:
|
||||
for transfer_id in list(self._active_ids):
|
||||
record = self._records[transfer_id]
|
||||
if (
|
||||
(
|
||||
request_ids is not None
|
||||
and record.spec.request_id not in request_ids
|
||||
)
|
||||
or record.source_tensor is not None
|
||||
or record.batch_future is not None
|
||||
):
|
||||
continue
|
||||
if record.state is not ProducerPushState.WAITING_INPUTS:
|
||||
continue
|
||||
self._transition(record, ProducerPushState.CANCEL_PENDING)
|
||||
cancelled.append(record)
|
||||
return cancelled
|
||||
|
||||
def finish_cancel(self, record: ProducerPushRecord) -> None:
|
||||
with self._lock:
|
||||
if record.state is ProducerPushState.CANCEL_PENDING:
|
||||
self._transition(record, ProducerPushState.CANCELLED)
|
||||
|
||||
def submit_cancel(
|
||||
self,
|
||||
record: ProducerPushRecord,
|
||||
executor: ThreadPoolExecutor,
|
||||
run_cancel: Callable[[ProducerPushRecord], None],
|
||||
) -> None:
|
||||
with self._lock:
|
||||
future = executor.submit(run_cancel, record)
|
||||
record.batch_future = future
|
||||
transfer_id = record.spec.transfer_id
|
||||
self._batch_ids[transfer_id] = None
|
||||
if record.state in _TERMINAL_STATES:
|
||||
self._reapable_terminal_ids.pop(transfer_id, None)
|
||||
|
||||
def poll(self) -> list[tuple[str, str]]:
|
||||
failures = []
|
||||
with self._lock:
|
||||
for transfer_id in list(self._batch_ids):
|
||||
record = self._records[transfer_id]
|
||||
future = record.batch_future
|
||||
assert future is not None
|
||||
if not future.done():
|
||||
continue
|
||||
try:
|
||||
future.result()
|
||||
except Exception as exc:
|
||||
self._set_error(record, exc)
|
||||
self._batch_ids.pop(transfer_id)
|
||||
self._mark_reapable(record)
|
||||
for transfer_id in list(self._unreported_ids):
|
||||
if transfer_id in self._batch_ids:
|
||||
continue
|
||||
record = self._records[transfer_id]
|
||||
assert record.error is not None
|
||||
failures.append((record.spec.mm_hash, record.error))
|
||||
self._unreported_ids.pop(transfer_id)
|
||||
self._mark_reapable(record)
|
||||
self._reap_terminals()
|
||||
return failures
|
||||
|
||||
@property
|
||||
def pending(self) -> bool:
|
||||
with self._lock:
|
||||
return bool(self._active_ids or self._batch_ids)
|
||||
|
||||
def _release_source(self, record: ProducerPushRecord) -> None:
|
||||
record.source_tensor = None
|
||||
record.source_event = None
|
||||
|
||||
def _set_error(self, record: ProducerPushRecord, error: BaseException) -> None:
|
||||
if record.error is None:
|
||||
record.error = str(error)
|
||||
if record.state in _TERMINAL_STATES:
|
||||
transfer_id = record.spec.transfer_id
|
||||
self._unreported_ids[transfer_id] = None
|
||||
self._reapable_terminal_ids.pop(transfer_id, None)
|
||||
|
||||
def _reap_terminals(self) -> None:
|
||||
while len(self._reapable_terminal_ids) > _TERMINAL_LIMIT:
|
||||
transfer_id, _ = self._reapable_terminal_ids.popitem(last=False)
|
||||
self._records.pop(transfer_id)
|
||||
|
||||
def _mark_reapable(self, record: ProducerPushRecord) -> None:
|
||||
transfer_id = record.spec.transfer_id
|
||||
if (
|
||||
record.state in _TERMINAL_STATES
|
||||
and transfer_id not in self._batch_ids
|
||||
and transfer_id not in self._unreported_ids
|
||||
):
|
||||
self._reapable_terminal_ids[transfer_id] = None
|
||||
|
||||
def _drop_source_waiter(self, record: ProducerPushRecord) -> None:
|
||||
waiters = self._source_waiters.get(record.spec.mm_hash)
|
||||
if waiters is None:
|
||||
return
|
||||
waiters.pop(record.spec.transfer_id, None)
|
||||
if not waiters:
|
||||
self._source_waiters.pop(record.spec.mm_hash)
|
||||
|
||||
@staticmethod
|
||||
def _check_sources_releasable(records: list[ProducerPushRecord]) -> None:
|
||||
for record in records:
|
||||
futures = [record.reservation_future, *record.shard_futures]
|
||||
if record.source_tensor is not None and not all(
|
||||
future.done() for future in futures
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"Producer push {record.spec.transfer_id!r} released its "
|
||||
"source too early"
|
||||
)
|
||||
|
||||
def _transition(self, record: ProducerPushRecord, state: ProducerPushState) -> None:
|
||||
if state not in _ALLOWED_TRANSITIONS[record.state]:
|
||||
raise RuntimeError(
|
||||
f"Cannot transition producer push {record.spec.transfer_id!r} from "
|
||||
f"{record.state.name} to {state.name}"
|
||||
)
|
||||
record.state = state
|
||||
if state is not ProducerPushState.WAITING_INPUTS:
|
||||
self._drop_source_waiter(record)
|
||||
if state in _TERMINAL_STATES:
|
||||
transfer_id = record.spec.transfer_id
|
||||
self._active_ids.pop(transfer_id, None)
|
||||
if record.error is not None:
|
||||
self._unreported_ids[transfer_id] = None
|
||||
self._mark_reapable(record)
|
||||
@@ -0,0 +1,487 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Consumer-side reservation lifecycle and destination-memory ownership.
|
||||
|
||||
Reservations make remote writes idempotent and ensure cancellation or expiry
|
||||
cannot free a destination while Mooncake may still be writing into it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.memory import (
|
||||
ConsumerMemoryPool,
|
||||
MemoryAllocation,
|
||||
ResidentLease,
|
||||
)
|
||||
|
||||
|
||||
class ConsumerReservationState(Enum):
|
||||
"""Lifecycle of one Consumer destination reservation."""
|
||||
|
||||
WRITING = auto()
|
||||
READY = auto()
|
||||
CANCEL_PENDING = auto()
|
||||
EXPIRE_PENDING = auto()
|
||||
CANCELLED = auto()
|
||||
EXPIRED = auto()
|
||||
|
||||
|
||||
_ALLOWED_TRANSITIONS = {
|
||||
ConsumerReservationState.WRITING: {
|
||||
ConsumerReservationState.READY,
|
||||
ConsumerReservationState.CANCEL_PENDING,
|
||||
ConsumerReservationState.EXPIRE_PENDING,
|
||||
ConsumerReservationState.CANCELLED,
|
||||
},
|
||||
ConsumerReservationState.READY: {
|
||||
ConsumerReservationState.CANCELLED,
|
||||
ConsumerReservationState.EXPIRED,
|
||||
},
|
||||
ConsumerReservationState.CANCEL_PENDING: {ConsumerReservationState.CANCELLED},
|
||||
ConsumerReservationState.EXPIRE_PENDING: {
|
||||
ConsumerReservationState.CANCELLED,
|
||||
ConsumerReservationState.EXPIRED,
|
||||
},
|
||||
ConsumerReservationState.CANCELLED: set(),
|
||||
ConsumerReservationState.EXPIRED: set(),
|
||||
}
|
||||
|
||||
_ACTIVE_STATES = {
|
||||
ConsumerReservationState.WRITING,
|
||||
ConsumerReservationState.READY,
|
||||
ConsumerReservationState.CANCEL_PENDING,
|
||||
ConsumerReservationState.EXPIRE_PENDING,
|
||||
}
|
||||
_DEFERRED_STATES = {
|
||||
ConsumerReservationState.CANCEL_PENDING,
|
||||
ConsumerReservationState.EXPIRE_PENDING,
|
||||
}
|
||||
_WRITER_OWNED_STATES = {
|
||||
ConsumerReservationState.WRITING,
|
||||
ConsumerReservationState.CANCEL_PENDING,
|
||||
ConsumerReservationState.EXPIRE_PENDING,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConsumerReservation:
|
||||
"""Own the identity and destination allocation of one remote push."""
|
||||
|
||||
transfer_id: str
|
||||
mm_hash: str
|
||||
reservation_id: str
|
||||
state: ConsumerReservationState
|
||||
shape: tuple[int, ...] = ()
|
||||
dtype: str = ""
|
||||
allocation: MemoryAllocation | None = None
|
||||
lease: ResidentLease[MemoryAllocation] | None = None
|
||||
expires_at: float = 0
|
||||
writer_id: str = ""
|
||||
|
||||
|
||||
class ConsumerReservationManager:
|
||||
"""Own reservation transitions and destination allocation releases.
|
||||
|
||||
A remote writer owns a ``WRITING`` allocation. Cancellation and expiry are
|
||||
therefore deferred until completion, unless refresh explicitly abandons
|
||||
that writer before replacing its reservation.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
memory: ConsumerMemoryPool,
|
||||
lease_ttl: float,
|
||||
tombstone_limit: int,
|
||||
) -> None:
|
||||
self._memory = memory
|
||||
self._lease_ttl = lease_ttl
|
||||
self._tombstone_limit = tombstone_limit
|
||||
self._records: dict[str, ConsumerReservation] = {}
|
||||
self._active_ids: dict[str, None] = {}
|
||||
self._tombstones: OrderedDict[str, None] = OrderedDict()
|
||||
self._writers: dict[str, ConsumerReservation] = {}
|
||||
self._followers: dict[str, dict[str, None]] = {}
|
||||
self._ready: list[str] = []
|
||||
self._condition = threading.Condition(memory.lock)
|
||||
self._shutting_down = False
|
||||
|
||||
def reserve(
|
||||
self,
|
||||
transfer_id: str,
|
||||
mm_hash: str,
|
||||
nbytes: int,
|
||||
shape: tuple[int, ...],
|
||||
dtype_name: str,
|
||||
dtype: torch.dtype,
|
||||
) -> tuple[ConsumerReservation | None, bool]:
|
||||
with self._condition:
|
||||
if self._shutting_down:
|
||||
raise RuntimeError("Consumer reservation manager is shutting down")
|
||||
existing = self._records.get(transfer_id)
|
||||
if (
|
||||
existing is not None
|
||||
and existing.state is ConsumerReservationState.CANCELLED
|
||||
):
|
||||
return existing, False
|
||||
if (
|
||||
existing is not None
|
||||
and existing.state is ConsumerReservationState.EXPIRED
|
||||
):
|
||||
self._remove(transfer_id)
|
||||
existing = None
|
||||
if (
|
||||
existing is not None
|
||||
and existing.state is ConsumerReservationState.EXPIRE_PENDING
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"Transfer {transfer_id!r} still has an active writer"
|
||||
)
|
||||
if existing is not None:
|
||||
if (
|
||||
existing.mm_hash != mm_hash
|
||||
or existing.shape != shape
|
||||
or existing.dtype != dtype_name
|
||||
):
|
||||
raise ValueError("conflicting reservation for transfer_id")
|
||||
if existing.state not in _ACTIVE_STATES:
|
||||
raise RuntimeError(
|
||||
f"Cannot reserve transfer {transfer_id!r} in "
|
||||
f"{existing.state.name}"
|
||||
)
|
||||
if existing.state is ConsumerReservationState.WRITING:
|
||||
existing.expires_at = time.monotonic() + self._lease_ttl
|
||||
return existing, False
|
||||
|
||||
lease = self._memory.acquire_cached(mm_hash, shape, dtype)
|
||||
now = time.monotonic()
|
||||
if lease is not None:
|
||||
record = ConsumerReservation(
|
||||
transfer_id,
|
||||
mm_hash,
|
||||
uuid.uuid4().hex,
|
||||
ConsumerReservationState.READY,
|
||||
shape,
|
||||
dtype_name,
|
||||
lease.value,
|
||||
lease,
|
||||
now + self._lease_ttl,
|
||||
)
|
||||
self._insert(record)
|
||||
return record, False
|
||||
writer = self._writers.get(mm_hash)
|
||||
if writer is not None and writer.state is ConsumerReservationState.WRITING:
|
||||
if writer.shape != shape or writer.dtype != dtype_name:
|
||||
raise ValueError("conflicting in-flight tensor for mm_hash")
|
||||
record = ConsumerReservation(
|
||||
transfer_id,
|
||||
mm_hash,
|
||||
uuid.uuid4().hex,
|
||||
ConsumerReservationState.WRITING,
|
||||
shape,
|
||||
dtype_name,
|
||||
writer.allocation,
|
||||
expires_at=now + self._lease_ttl,
|
||||
writer_id=writer.transfer_id,
|
||||
)
|
||||
self._insert(record)
|
||||
self._followers.setdefault(writer.transfer_id, {})[transfer_id] = None
|
||||
return record, False
|
||||
allocation = self._memory.try_allocate(nbytes, shape, dtype)
|
||||
if allocation is None:
|
||||
self._expire_locked(time.monotonic())
|
||||
allocation = self._memory.try_allocate(nbytes, shape, dtype)
|
||||
if allocation is None:
|
||||
allocation = self._memory.reclaim_and_allocate(nbytes, shape, dtype)
|
||||
if allocation is None:
|
||||
return None, False
|
||||
record = ConsumerReservation(
|
||||
transfer_id,
|
||||
mm_hash,
|
||||
uuid.uuid4().hex,
|
||||
ConsumerReservationState.WRITING,
|
||||
shape,
|
||||
dtype_name,
|
||||
allocation,
|
||||
None,
|
||||
now + self._lease_ttl,
|
||||
)
|
||||
self._insert(record)
|
||||
self._writers[mm_hash] = record
|
||||
return record, True
|
||||
|
||||
def status(self, transfer_id: str) -> ConsumerReservation | None:
|
||||
with self._memory.lock:
|
||||
record = self._records.get(transfer_id)
|
||||
if record is None or record.state not in _ACTIVE_STATES:
|
||||
return None
|
||||
return record
|
||||
|
||||
def complete(self, transfer_id: str, reservation_id: str) -> tuple[bool, bool]:
|
||||
with self._condition:
|
||||
try:
|
||||
record = self._records.get(transfer_id)
|
||||
if record is None or record.reservation_id != reservation_id:
|
||||
return False, False
|
||||
if record.state is ConsumerReservationState.READY:
|
||||
return True, False
|
||||
if record.writer_id:
|
||||
return False, False
|
||||
if record.state in _WRITER_OWNED_STATES:
|
||||
self._publish_followers(record)
|
||||
if record.state in _DEFERRED_STATES:
|
||||
terminal = (
|
||||
ConsumerReservationState.CANCELLED
|
||||
if record.state is ConsumerReservationState.CANCEL_PENDING
|
||||
else ConsumerReservationState.EXPIRED
|
||||
)
|
||||
self._terminate(record, terminal)
|
||||
return True, False
|
||||
if record.state is not ConsumerReservationState.WRITING:
|
||||
return False, False
|
||||
self._transition(record, ConsumerReservationState.READY)
|
||||
record.expires_at = time.monotonic() + self._lease_ttl
|
||||
return True, True
|
||||
finally:
|
||||
self._condition.notify_all()
|
||||
|
||||
def _publish_followers(self, writer: ConsumerReservation) -> None:
|
||||
if self._writers.get(writer.mm_hash) is writer:
|
||||
self._writers.pop(writer.mm_hash)
|
||||
followers = self._followers.pop(writer.transfer_id, {})
|
||||
if not followers:
|
||||
return
|
||||
assert writer.allocation is not None
|
||||
allocation = self._memory.publish(writer.mm_hash, writer.allocation, pin=False)
|
||||
for record in [writer, *(self._records[key] for key in followers)]:
|
||||
record.allocation = allocation
|
||||
record.lease = self._memory.acquire_cached(
|
||||
record.mm_hash, record.shape, allocation.tensor.dtype
|
||||
)
|
||||
assert record.lease is not None
|
||||
if record is not writer:
|
||||
record.writer_id = ""
|
||||
self._transition(record, ConsumerReservationState.READY)
|
||||
record.expires_at = time.monotonic() + self._lease_ttl
|
||||
self._ready.append(record.transfer_id)
|
||||
|
||||
def drain_ready(self) -> list[str]:
|
||||
with self._memory.lock:
|
||||
ready, self._ready = self._ready, []
|
||||
return ready
|
||||
|
||||
def begin_shutdown(self) -> None:
|
||||
"""Stop new reservations and cancel everything without a remote writer."""
|
||||
with self._condition:
|
||||
if self._shutting_down:
|
||||
return
|
||||
self._shutting_down = True
|
||||
for transfer_id in list(self._active_ids):
|
||||
record = self._records[transfer_id]
|
||||
if record.writer_id or record.state is ConsumerReservationState.READY:
|
||||
self._terminate(record, ConsumerReservationState.CANCELLED)
|
||||
elif record.state is ConsumerReservationState.WRITING:
|
||||
self._defer(record, ConsumerReservationState.CANCEL_PENDING)
|
||||
self._condition.notify_all()
|
||||
|
||||
def wait_for_writers(self, timeout: float) -> bool:
|
||||
"""Wait until no reservation is still owned by a remote writer."""
|
||||
|
||||
def writers_finished() -> bool:
|
||||
return not any(
|
||||
self._records[transfer_id].state in _WRITER_OWNED_STATES
|
||||
for transfer_id in self._active_ids
|
||||
)
|
||||
|
||||
with self._condition:
|
||||
return self._condition.wait_for(writers_finished, timeout=max(0, timeout))
|
||||
|
||||
def cancel(
|
||||
self,
|
||||
transfer_id: str,
|
||||
reservation_id: str,
|
||||
abandon: bool = False,
|
||||
refresh: bool = False,
|
||||
) -> bool:
|
||||
with self._memory.lock:
|
||||
record = self._records.get(transfer_id)
|
||||
if (
|
||||
record is not None
|
||||
and reservation_id
|
||||
and record.reservation_id != reservation_id
|
||||
):
|
||||
return False
|
||||
if record is None:
|
||||
record = ConsumerReservation(
|
||||
transfer_id,
|
||||
"",
|
||||
"",
|
||||
ConsumerReservationState.CANCELLED,
|
||||
)
|
||||
self._insert(record)
|
||||
self._set_tombstone_deadline(record)
|
||||
self._reap_tombstones(time.monotonic())
|
||||
return True
|
||||
if record.state is ConsumerReservationState.CANCELLED:
|
||||
self._set_tombstone_deadline(record)
|
||||
self._reap_tombstones(time.monotonic())
|
||||
return True
|
||||
if refresh:
|
||||
if not abandon or record.state not in {
|
||||
ConsumerReservationState.WRITING,
|
||||
ConsumerReservationState.EXPIRE_PENDING,
|
||||
}:
|
||||
return False
|
||||
if record.state is ConsumerReservationState.WRITING:
|
||||
self._transition(record, ConsumerReservationState.EXPIRE_PENDING)
|
||||
self._terminate(record, ConsumerReservationState.EXPIRED)
|
||||
self._condition.notify_all()
|
||||
self._reap_tombstones(time.monotonic())
|
||||
return True
|
||||
if record.writer_id:
|
||||
self._terminate(record, ConsumerReservationState.CANCELLED)
|
||||
return True
|
||||
if record.state in _DEFERRED_STATES and not abandon:
|
||||
return True
|
||||
if record.state is ConsumerReservationState.WRITING and not abandon:
|
||||
self._defer(record, ConsumerReservationState.CANCEL_PENDING)
|
||||
return True
|
||||
if record.state not in _ACTIVE_STATES:
|
||||
return False
|
||||
self._terminate(record, ConsumerReservationState.CANCELLED)
|
||||
if self._shutting_down:
|
||||
self._condition.notify_all()
|
||||
self._reap_tombstones(time.monotonic())
|
||||
return True
|
||||
|
||||
def take(self, transfer_id: str, mm_hash: str) -> MemoryAllocation:
|
||||
with self._memory.lock:
|
||||
record = self._records.get(transfer_id)
|
||||
if (
|
||||
record is None
|
||||
or record.state is not ConsumerReservationState.READY
|
||||
or record.mm_hash != mm_hash
|
||||
or record.allocation is None
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"Pushed EC tensor is not ready for mm_hash={mm_hash}"
|
||||
)
|
||||
allocation = self._memory.publish(mm_hash, record.allocation, record.lease)
|
||||
record.allocation = None
|
||||
record.lease = None
|
||||
self._remove(transfer_id)
|
||||
return allocation
|
||||
|
||||
def expire(self) -> int:
|
||||
with self._memory.lock:
|
||||
return self._expire_locked(time.monotonic())
|
||||
|
||||
def retire_stale(
|
||||
self, encoder_cache: dict[str, torch.Tensor], freed: list[str] | None = None
|
||||
) -> None:
|
||||
with self._memory.lock:
|
||||
self._memory.retire_stale(encoder_cache, freed=freed)
|
||||
|
||||
def _expire_locked(self, now: float) -> int:
|
||||
expired = 0
|
||||
for transfer_id in list(self._active_ids):
|
||||
record = self._records[transfer_id]
|
||||
if record.expires_at > now:
|
||||
continue
|
||||
if record.state is ConsumerReservationState.READY:
|
||||
self._terminate(record, ConsumerReservationState.EXPIRED)
|
||||
expired += 1
|
||||
elif record.state is ConsumerReservationState.WRITING:
|
||||
if record.writer_id:
|
||||
self._terminate(record, ConsumerReservationState.CANCELLED)
|
||||
else:
|
||||
self._defer(record, ConsumerReservationState.EXPIRE_PENDING)
|
||||
self._reap_tombstones(now)
|
||||
return expired
|
||||
|
||||
def _defer(
|
||||
self, record: ConsumerReservation, state: ConsumerReservationState
|
||||
) -> None:
|
||||
"""Keep the destination until the writer completes or abandons it."""
|
||||
self._transition(record, state)
|
||||
record.expires_at = float("inf")
|
||||
|
||||
def _terminate(
|
||||
self, record: ConsumerReservation, state: ConsumerReservationState
|
||||
) -> None:
|
||||
self._transition(record, state)
|
||||
self._release(record)
|
||||
self._set_tombstone_deadline(record)
|
||||
|
||||
def _release(self, record: ConsumerReservation) -> None:
|
||||
if record.writer_id:
|
||||
followers = self._followers.get(record.writer_id, {})
|
||||
followers.pop(record.transfer_id, None)
|
||||
record.allocation = None
|
||||
return
|
||||
if self._writers.get(record.mm_hash) is record:
|
||||
self._writers.pop(record.mm_hash)
|
||||
for transfer_id in list(self._followers.pop(record.transfer_id, {})):
|
||||
self._terminate(
|
||||
self._records[transfer_id], ConsumerReservationState.CANCELLED
|
||||
)
|
||||
allocation = record.allocation
|
||||
if allocation is None:
|
||||
return
|
||||
if record.lease is not None:
|
||||
self._memory.release_cached(record.lease)
|
||||
else:
|
||||
self._memory.free(allocation)
|
||||
record.allocation = None
|
||||
record.lease = None
|
||||
|
||||
def _set_tombstone_deadline(self, record: ConsumerReservation) -> None:
|
||||
record.expires_at = time.monotonic() + self._lease_ttl
|
||||
self._tombstones[record.transfer_id] = None
|
||||
self._tombstones.move_to_end(record.transfer_id)
|
||||
|
||||
def _reap_tombstones(self, now: float) -> int:
|
||||
dropped = 0
|
||||
while self._tombstones:
|
||||
transfer_id = next(iter(self._tombstones))
|
||||
record = self._records[transfer_id]
|
||||
if (
|
||||
record.expires_at > now
|
||||
and len(self._tombstones) <= self._tombstone_limit
|
||||
):
|
||||
break
|
||||
self._remove(transfer_id)
|
||||
dropped += 1
|
||||
return dropped
|
||||
|
||||
def _insert(self, record: ConsumerReservation) -> None:
|
||||
self._records[record.transfer_id] = record
|
||||
if record.state in _ACTIVE_STATES:
|
||||
self._active_ids[record.transfer_id] = None
|
||||
|
||||
def _remove(self, transfer_id: str) -> None:
|
||||
self._records.pop(transfer_id, None)
|
||||
self._active_ids.pop(transfer_id, None)
|
||||
self._tombstones.pop(transfer_id, None)
|
||||
|
||||
def _transition(
|
||||
self, record: ConsumerReservation, state: ConsumerReservationState
|
||||
) -> None:
|
||||
if state not in _ALLOWED_TRANSITIONS[record.state]:
|
||||
raise RuntimeError(
|
||||
f"Cannot transition {record.transfer_id!r} from "
|
||||
f"{record.state.name} to {state.name}"
|
||||
)
|
||||
record.state = state
|
||||
if state in _ACTIVE_STATES:
|
||||
self._active_ids[record.transfer_id] = None
|
||||
else:
|
||||
self._active_ids.pop(record.transfer_id, None)
|
||||
@@ -0,0 +1,537 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Scheduler-side planning and observation for Mooncake cache transfers.
|
||||
|
||||
The Scheduler prepares Producer reservations, consumes Consumer readiness
|
||||
events, emits per-step Worker metadata, and converts Worker results back into
|
||||
request availability without owning tensor memory or running data transfers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorMetadata,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
|
||||
_get_encoder_cache_hidden_dim,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.config import (
|
||||
_RESERVATION_TTL_SECONDS,
|
||||
MooncakeECConfig,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.control import (
|
||||
ControlClient,
|
||||
EventInbox,
|
||||
make_cancel_request,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
|
||||
ECMooncakeConnectorMetadata,
|
||||
ECMooncakeLoadSpec,
|
||||
ECMooncakePushSpec,
|
||||
ECMooncakeWorkerMetadata,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.state import (
|
||||
SchedulerTransferState,
|
||||
SchedulerTransferTable,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.transfer import (
|
||||
ensure_mooncake_available,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.utils import (
|
||||
PlaceholderMetadataResolver,
|
||||
collect_ec_item_metadata,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import ECConnectorOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Unmatched `ec_items` entries are a protocol desync, not a per-request
|
||||
# event: warn on the first few, then only once in a while.
|
||||
_MAX_UNRESOLVED_TRANSFER_ID_WARNINGS = 5
|
||||
|
||||
_DRAIN_MIN_INTERVAL = 0.005
|
||||
_MAX_PENDING_EVENTS = 4096
|
||||
_MAX_TERMINAL_TRANSFER_RECORDS = 1 << 16
|
||||
_CANCEL_ATTEMPTS = 2
|
||||
_CONTROL_WORKERS = 8
|
||||
|
||||
|
||||
class ECMooncakeScheduler:
|
||||
"""Coordinate Mooncake transfers from the vLLM Scheduler process."""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig) -> None:
|
||||
ensure_mooncake_available()
|
||||
config = MooncakeECConfig.from_vllm_config(vllm_config)
|
||||
|
||||
self._is_producer = config.is_producer
|
||||
self._is_consumer = config.is_consumer
|
||||
self._control_addr = config.control_addr
|
||||
self._push_wait_timeout = config.push_wait_timeout_s
|
||||
self._encoder_cache_hidden_dim = (
|
||||
_get_encoder_cache_hidden_dim(vllm_config) if config.is_producer else None
|
||||
)
|
||||
self._model_config = vllm_config.model_config
|
||||
self._control_client = ControlClient(config.control_timeout_ms)
|
||||
self._control_executor = ThreadPoolExecutor(
|
||||
max_workers=_CONTROL_WORKERS,
|
||||
thread_name_prefix="ec-mooncake-control",
|
||||
)
|
||||
self._event_inbox = EventInbox(self._control_client, self._control_executor)
|
||||
|
||||
self._metadata_resolver = PlaceholderMetadataResolver(vllm_config.model_config)
|
||||
self._unresolved_transfer_ids = 0
|
||||
self._drain_pending = True
|
||||
self._drained_at = 0.0
|
||||
self._pending_cancels: dict[str, Future[Any]] = {}
|
||||
self._transfers = SchedulerTransferTable(
|
||||
config.pool_size, _RESERVATION_TTL_SECONDS
|
||||
)
|
||||
self._scheduler_pending_work = False
|
||||
self._pushes_to_prepare: dict[str, ECMooncakePushSpec] = {}
|
||||
self._prepared_push_transfer_ids: set[str] = set()
|
||||
self._event_ready_shards: OrderedDict[str, set[int]] = OrderedDict()
|
||||
# Mirror of the engine's encoder cache, maintained from the alloc and
|
||||
# free notifications the Scheduler already sends. Tracking it here
|
||||
# keeps `ensure_cache_available` on the upstream two-argument shape.
|
||||
self._local_cache: set[str] = set()
|
||||
self._failed_saves: set[str] = set()
|
||||
|
||||
def _cancel_remote(self, consumer_zmq: str, transfer_id: str) -> bool:
|
||||
pending = None
|
||||
for _ in range(_CANCEL_ATTEMPTS):
|
||||
pending = self._control_client.discover_shards(consumer_zmq)
|
||||
if pending is not None:
|
||||
break
|
||||
if pending is None:
|
||||
raise RuntimeError(
|
||||
f"Could not discover every EC consumer shard at {consumer_zmq}"
|
||||
)
|
||||
|
||||
cancelled = False
|
||||
error: BaseException | None = None
|
||||
for _ in range(_CANCEL_ATTEMPTS):
|
||||
failed = []
|
||||
for addr in pending:
|
||||
try:
|
||||
result = self._control_client.request(
|
||||
addr,
|
||||
make_cancel_request(transfer_id),
|
||||
)
|
||||
except Exception as exc:
|
||||
if error is None:
|
||||
error = exc
|
||||
failed.append(addr)
|
||||
continue
|
||||
cancelled |= isinstance(result, dict) and bool(result.get("cancelled"))
|
||||
if not failed:
|
||||
return cancelled
|
||||
pending = failed
|
||||
if error is not None:
|
||||
raise error
|
||||
return cancelled
|
||||
|
||||
def _note_awaiting_push(
|
||||
self,
|
||||
mm_hash: str,
|
||||
transfer_id: str,
|
||||
request_id: str,
|
||||
now: float,
|
||||
) -> None:
|
||||
record = self._transfers.wait_for_event(
|
||||
transfer_id,
|
||||
request_id,
|
||||
mm_hash,
|
||||
now + self._push_wait_timeout,
|
||||
)
|
||||
if record is None:
|
||||
# The id names another encoding; the request is already failed.
|
||||
return
|
||||
if record.state is not SchedulerTransferState.WAITING_EVENT:
|
||||
return
|
||||
assert record.deadline is not None
|
||||
if now < record.deadline:
|
||||
return
|
||||
elapsed = now - record.deadline + self._push_wait_timeout
|
||||
self._transfers.mark_unavailable(transfer_id, now)
|
||||
logger.warning(
|
||||
"EC Mooncake waited %.1fs for a push of mm_hash=%s "
|
||||
"(transfer_id=%s) that never arrived; "
|
||||
"requests needing it fail with a retryable error.",
|
||||
elapsed,
|
||||
mm_hash,
|
||||
transfer_id,
|
||||
)
|
||||
|
||||
def take_unavailable_requests(self) -> set[str]:
|
||||
failed, self._failed_saves = self._failed_saves, set()
|
||||
return failed | self._transfers.take_unavailable_requests()
|
||||
|
||||
def _poll_pending_cancels(self) -> None:
|
||||
pending = {}
|
||||
for transfer_id, future in self._pending_cancels.items():
|
||||
if not future.done():
|
||||
pending[transfer_id] = future
|
||||
continue
|
||||
try:
|
||||
future.result()
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"EC Mooncake reservation cancellation failed", exc_info=True
|
||||
)
|
||||
self._pending_cancels = pending
|
||||
|
||||
def _note_shard_ready(self, data: dict[str, Any]) -> bool:
|
||||
shard_count = self._event_inbox.shard_count
|
||||
if shard_count <= 1:
|
||||
return True
|
||||
transfer_id = str(data["transfer_id"])
|
||||
record = self._transfers.get(transfer_id)
|
||||
if (
|
||||
record is not None
|
||||
and record.state is not SchedulerTransferState.WAITING_EVENT
|
||||
):
|
||||
return False
|
||||
shard = data.get("shard")
|
||||
shards = self._event_ready_shards.setdefault(transfer_id, set())
|
||||
self._event_ready_shards.move_to_end(transfer_id)
|
||||
shards.add(int(shard) if shard is not None else len(shards))
|
||||
if len(shards) < shard_count:
|
||||
while len(self._event_ready_shards) > _MAX_PENDING_EVENTS:
|
||||
self._event_ready_shards.popitem(last=False)
|
||||
return False
|
||||
self._event_ready_shards.pop(transfer_id, None)
|
||||
return True
|
||||
|
||||
def _forget_shard_readiness(self, transfer_id: str) -> None:
|
||||
self._event_ready_shards.pop(transfer_id, None)
|
||||
|
||||
def _store_pushed_spec(self, data: dict[str, Any]) -> bool:
|
||||
transfer_id = str(data["transfer_id"])
|
||||
identifier = str(data["mm_hash"])
|
||||
_, accepted = self._transfers.observe_ready(
|
||||
ECMooncakeLoadSpec(
|
||||
mm_hash=identifier,
|
||||
nbytes=int(data["nbytes"]),
|
||||
shape=tuple(int(value) for value in data["shape"]),
|
||||
dtype=str(data["dtype"]),
|
||||
transfer_id=transfer_id,
|
||||
),
|
||||
time.monotonic() + _RESERVATION_TTL_SECONDS,
|
||||
)
|
||||
return accepted
|
||||
|
||||
def _queue_cancel(
|
||||
self,
|
||||
transfer_id: str,
|
||||
mm_hash: str = "",
|
||||
request_id: str = "",
|
||||
) -> None:
|
||||
if not self._transfers.cancel(
|
||||
transfer_id,
|
||||
time.monotonic(),
|
||||
mm_hash=mm_hash,
|
||||
request_id=request_id,
|
||||
):
|
||||
return
|
||||
self._forget_shard_readiness(transfer_id)
|
||||
self._pending_cancels[transfer_id] = self._control_executor.submit(
|
||||
self._cancel_remote,
|
||||
self._control_addr,
|
||||
transfer_id,
|
||||
)
|
||||
|
||||
def _expire_transfers(self) -> None:
|
||||
now = time.monotonic()
|
||||
expired = self._transfers.expire(now, _MAX_TERMINAL_TRANSFER_RECORDS)
|
||||
for record in expired:
|
||||
self._queue_cancel(record.transfer_id)
|
||||
|
||||
def _drain_push_notifications(self) -> None:
|
||||
now = time.monotonic()
|
||||
if not self._drain_pending and now - self._drained_at < _DRAIN_MIN_INTERVAL:
|
||||
return
|
||||
self._drain_pending = False
|
||||
self._drained_at = now
|
||||
self._poll_pending_cancels()
|
||||
self._expire_transfers()
|
||||
events = self._event_inbox.drain(self._control_addr)
|
||||
for data in events:
|
||||
try:
|
||||
if not isinstance(data, dict):
|
||||
raise TypeError("EC readiness event must be an object")
|
||||
if not data.get("ready"):
|
||||
continue
|
||||
self._accept_ready_event(data)
|
||||
except (KeyError, TypeError, ValueError):
|
||||
# Readiness events cross a plain PULL socket, so a malformed
|
||||
# one costs that event and not the engine.
|
||||
logger.warning(
|
||||
"Discarding a malformed EC readiness event.", exc_info=True
|
||||
)
|
||||
|
||||
def _accept_ready_event(self, data: dict[str, Any]) -> None:
|
||||
transfer_id = str(data["transfer_id"])
|
||||
record = self._transfers.get(transfer_id)
|
||||
if record is not None and record.state in {
|
||||
SchedulerTransferState.CANCELLED,
|
||||
SchedulerTransferState.UNAVAILABLE,
|
||||
SchedulerTransferState.EXPIRED,
|
||||
SchedulerTransferState.FAILED,
|
||||
}:
|
||||
return
|
||||
if not self._note_shard_ready(data):
|
||||
return
|
||||
self._store_pushed_spec(data)
|
||||
|
||||
def has_cache_item(self, identifier: str) -> bool:
|
||||
if not self._is_consumer:
|
||||
return False
|
||||
self._drain_push_notifications()
|
||||
return self._transfers.has_state(
|
||||
identifier,
|
||||
(
|
||||
SchedulerTransferState.READY,
|
||||
SchedulerTransferState.RESIDENT,
|
||||
SchedulerTransferState.AVAILABLE,
|
||||
),
|
||||
)
|
||||
|
||||
def _warn_unresolved_transfer_id(
|
||||
self, request: Any, index: int, where: str
|
||||
) -> None:
|
||||
"""Report an `ec_items` entry that cannot be matched to a feature.
|
||||
|
||||
A caller that cannot resolve a transfer id has to fall back to a
|
||||
locally invented one or skip its bookkeeping entirely, and either way
|
||||
the two sides of a transfer stop agreeing on its name. That desyncs
|
||||
silently -- reservations are never released and only surface minutes
|
||||
later as a full buffer pool -- so say it out loud the first time.
|
||||
|
||||
`mm_hash` in `ec_items` must be a value some engine reported, never one
|
||||
the caller derived itself: `mm_features[i].identifier` folds in the
|
||||
engine's media_io_kwargs and mm_processor_kwargs, so a media uuid alone
|
||||
no longer equals it. Omit the field to match by position instead.
|
||||
"""
|
||||
self._unresolved_transfer_ids += 1
|
||||
count = self._unresolved_transfer_ids
|
||||
if count > _MAX_UNRESOLVED_TRANSFER_ID_WARNINGS and count % 1000:
|
||||
return
|
||||
params = getattr(request, "ec_transfer_params", None)
|
||||
items = (params or {}).get("ec_items") or []
|
||||
logger.warning(
|
||||
"EC Mooncake could not resolve a transfer id at %s for req=%s "
|
||||
"index=%d: feature identifier=%s does not match any of the %d "
|
||||
"ec_items %s (ec_transfer_params present=%s). Occurrence %d. "
|
||||
"ec_items[].mm_hash must echo an engine-reported identifier, or "
|
||||
"be omitted so the entry is matched by position.",
|
||||
where,
|
||||
request.request_id,
|
||||
index,
|
||||
request.mm_features[index].identifier[:16],
|
||||
len(items),
|
||||
[str(item.get("mm_hash"))[:16] for item in items[:4]],
|
||||
params is not None,
|
||||
count,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _request_transfer_id(request: Any, index: int) -> str | None:
|
||||
params = getattr(request, "ec_transfer_params", None) or {}
|
||||
items = params.get("ec_items") or []
|
||||
mm_hash = request.mm_features[index].identifier
|
||||
if index < len(items):
|
||||
item = items[index]
|
||||
if item.get("mm_hash") in (None, mm_hash) and item.get("transfer_id"):
|
||||
return str(item["transfer_id"])
|
||||
for item in items:
|
||||
if item.get("mm_hash") == mm_hash and item.get("transfer_id"):
|
||||
return str(item["transfer_id"])
|
||||
return None
|
||||
|
||||
def ensure_cache_available(self, request: Any, num_computed_tokens: int) -> bool:
|
||||
if self._is_producer:
|
||||
for index, feature in enumerate(request.mm_features):
|
||||
if (
|
||||
feature.mm_position.offset + feature.mm_position.length
|
||||
> num_computed_tokens
|
||||
):
|
||||
self._prepare_push_spec(request, index)
|
||||
if not self._is_consumer:
|
||||
return True
|
||||
|
||||
self._drain_push_notifications()
|
||||
# One timestamp for the whole decision, so the deadlines it hands out
|
||||
# cannot disagree between features.
|
||||
now = time.monotonic()
|
||||
all_ready = True
|
||||
for index, feature in enumerate(request.mm_features):
|
||||
if (
|
||||
feature.mm_position.offset + feature.mm_position.length
|
||||
<= num_computed_tokens
|
||||
):
|
||||
continue
|
||||
mm_hash = feature.identifier
|
||||
transfer_id = self._request_transfer_id(request, index)
|
||||
if transfer_id is not None:
|
||||
self._transfers.touch_available(
|
||||
transfer_id, now + _RESERVATION_TTL_SECONDS
|
||||
)
|
||||
if mm_hash in self._local_cache:
|
||||
continue
|
||||
if self._transfers.has_state(mm_hash, (SchedulerTransferState.READY,)):
|
||||
continue
|
||||
if self._transfers.has_state(mm_hash, (SchedulerTransferState.LOADING,)):
|
||||
all_ready = False
|
||||
continue
|
||||
record = self._transfers.begin_load(
|
||||
mm_hash,
|
||||
transfer_id,
|
||||
request.request_id,
|
||||
now + self._push_wait_timeout,
|
||||
)
|
||||
if record is not None:
|
||||
self._scheduler_pending_work = True
|
||||
all_ready = False
|
||||
else:
|
||||
waiting_id = transfer_id or f"{request.request_id}:{index}"
|
||||
self._note_awaiting_push(mm_hash, waiting_id, request.request_id, now)
|
||||
all_ready = False
|
||||
return all_ready
|
||||
|
||||
def _prepare_push_spec(self, request: Any, index: int) -> None:
|
||||
params = getattr(request, "ec_transfer_params", None) or {}
|
||||
consumer_zmq = params.get("consumer_zmq")
|
||||
mm_hash = request.mm_features[index].identifier
|
||||
transfer_id = self._request_transfer_id(request, index)
|
||||
if transfer_id is None:
|
||||
# Still push: after the proxy has rewritten the item to embeds the
|
||||
# consumer has no media left to fall back on, so a nameless push
|
||||
# beats none. But the consumer knows this transfer by the id it
|
||||
# sent, not by the one invented here, so its cancel will never
|
||||
# reach the reservation this push is about to take.
|
||||
if consumer_zmq:
|
||||
self._warn_unresolved_transfer_id(request, index, "push prepare")
|
||||
transfer_id = f"{request.request_id}:{index}"
|
||||
if not consumer_zmq or transfer_id in self._prepared_push_transfer_ids:
|
||||
return
|
||||
num_tokens = request.get_num_encoder_embeds(index)
|
||||
dtype = self._model_config.dtype
|
||||
assert isinstance(dtype, torch.dtype)
|
||||
assert self._encoder_cache_hidden_dim is not None
|
||||
dtype_name = str(dtype).split(".")[-1]
|
||||
shape = (num_tokens, self._encoder_cache_hidden_dim)
|
||||
nbytes = math.prod(shape) * dtype.itemsize
|
||||
self._pushes_to_prepare[transfer_id] = ECMooncakePushSpec(
|
||||
mm_hash=mm_hash,
|
||||
nbytes=nbytes,
|
||||
shape=shape,
|
||||
dtype=dtype_name,
|
||||
consumer_zmq=str(consumer_zmq),
|
||||
transfer_id=transfer_id,
|
||||
request_id=request.request_id,
|
||||
)
|
||||
self._prepared_push_transfer_ids.add(transfer_id)
|
||||
|
||||
def update_state_after_alloc(self, request: Any, index: int) -> None:
|
||||
self._local_cache.add(request.mm_features[index].identifier)
|
||||
if self._is_producer:
|
||||
self._prepare_push_spec(request, index)
|
||||
|
||||
def update_state_after_free(self, request: Any, index: int) -> None:
|
||||
if not self._is_consumer:
|
||||
return
|
||||
transfer_id = self._request_transfer_id(request, index)
|
||||
if transfer_id is None:
|
||||
self._warn_unresolved_transfer_id(request, index, "encoder-cache free")
|
||||
return
|
||||
self._queue_cancel(
|
||||
transfer_id,
|
||||
mm_hash=request.mm_features[index].identifier,
|
||||
request_id=request.request_id,
|
||||
)
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> ECConnectorMetadata:
|
||||
for mm_hash in scheduler_output.free_encoder_mm_hashes:
|
||||
self._local_cache.discard(mm_hash)
|
||||
self._transfers.release_ready(mm_hash, time.monotonic())
|
||||
for transfer_id in self._transfers.drain_orphaned():
|
||||
self._queue_cancel(transfer_id)
|
||||
meta = ECMooncakeConnectorMetadata(
|
||||
freed=scheduler_output.free_encoder_mm_hashes
|
||||
)
|
||||
for push_spec in self._pushes_to_prepare.values():
|
||||
meta.pushes.append(push_spec)
|
||||
self._pushes_to_prepare.clear()
|
||||
for record in self._transfers.take_loads_to_dispatch():
|
||||
assert record.spec is not None
|
||||
meta.loads.append(record.spec)
|
||||
self._poll_pending_cancels()
|
||||
self._drain_pending = True
|
||||
return meta
|
||||
|
||||
def update_connector_output(self, connector_output: ECConnectorOutput) -> None:
|
||||
meta = connector_output.ec_connector_worker_meta
|
||||
if not isinstance(meta, ECMooncakeWorkerMetadata):
|
||||
return
|
||||
for mm_hash in meta.loaded:
|
||||
self._transfers.complete_load(mm_hash)
|
||||
for mm_hash in meta.failed_loads:
|
||||
self._transfers.fail_load(mm_hash, time.monotonic())
|
||||
for mm_hash in meta.reclaimed:
|
||||
self._transfers.reclaim(mm_hash, time.monotonic())
|
||||
self._scheduler_pending_work = meta.pending_saves
|
||||
self._failed_saves.update(meta.failed_saves)
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
return self._scheduler_pending_work
|
||||
|
||||
def request_finished(self, request: Any) -> tuple[bool, dict[str, Any] | None]:
|
||||
if self._is_consumer:
|
||||
for index in range(len(request.mm_features)):
|
||||
transfer_id = self._request_transfer_id(request, index)
|
||||
if transfer_id is None:
|
||||
self._warn_unresolved_transfer_id(request, index, "request finish")
|
||||
continue
|
||||
self._queue_cancel(
|
||||
transfer_id,
|
||||
mm_hash=request.mm_features[index].identifier,
|
||||
request_id=request.request_id,
|
||||
)
|
||||
if self._is_producer and self._prepared_push_transfer_ids:
|
||||
for index in range(len(request.mm_features)):
|
||||
transfer_id = self._request_transfer_id(request, index)
|
||||
if transfer_id is None:
|
||||
transfer_id = f"{request.request_id}:{index}"
|
||||
self._prepared_push_transfer_ids.discard(transfer_id)
|
||||
if not self._is_producer:
|
||||
return False, None
|
||||
|
||||
items = collect_ec_item_metadata(request.mm_features, self._metadata_resolver)
|
||||
for index, feature in enumerate(request.mm_features):
|
||||
transfer_id = self._request_transfer_id(request, index)
|
||||
if transfer_id is not None:
|
||||
items[feature.identifier]["transfer_id"] = transfer_id
|
||||
|
||||
if not items:
|
||||
return False, None
|
||||
return False, items
|
||||
|
||||
def close(self) -> None:
|
||||
self._control_executor.shutdown(wait=True, cancel_futures=True)
|
||||
self._control_client.close()
|
||||
self._event_inbox.close()
|
||||
@@ -0,0 +1,448 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Scheduler-owned lifecycle and indexes for Consumer-bound transfers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import OrderedDict, deque
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass, field, replace
|
||||
from enum import Enum, auto
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
|
||||
ECMooncakeLoadSpec,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class SchedulerTransferState(Enum):
|
||||
"""Lifecycle states visible to the Scheduler.
|
||||
|
||||
The states cover waiting for a Consumer event, dispatching a Worker load,
|
||||
retaining a locally reusable tensor, and terminal failure conditions.
|
||||
"""
|
||||
|
||||
WAITING_EVENT = auto()
|
||||
AVAILABLE = auto()
|
||||
LOADING = auto()
|
||||
READY = auto()
|
||||
RESIDENT = auto()
|
||||
UNAVAILABLE = auto()
|
||||
EXPIRED = auto()
|
||||
FAILED = auto()
|
||||
CANCELLED = auto()
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchedulerTransfer:
|
||||
"""Track one transfer as observed by the Scheduler."""
|
||||
|
||||
transfer_id: str
|
||||
request_id: str
|
||||
mm_hash: str
|
||||
state: SchedulerTransferState
|
||||
spec: ECMooncakeLoadSpec | None
|
||||
deadline: float | None
|
||||
notified_requests: set[str] = field(default_factory=set, repr=False)
|
||||
|
||||
|
||||
_TERMINAL_STATES = {
|
||||
SchedulerTransferState.UNAVAILABLE,
|
||||
SchedulerTransferState.EXPIRED,
|
||||
SchedulerTransferState.FAILED,
|
||||
SchedulerTransferState.CANCELLED,
|
||||
}
|
||||
|
||||
|
||||
class SchedulerTransferTable:
|
||||
"""Own Scheduler transfer state, lookup indexes, and dispatch queues.
|
||||
|
||||
The Scheduler is single-threaded, so direct transitions are sufficient;
|
||||
the Producer and Consumer managers retain stricter transition matrices
|
||||
because callbacks and control requests can race there.
|
||||
"""
|
||||
|
||||
def __init__(self, resident_capacity: int, tombstone_ttl: float) -> None:
|
||||
self._resident_capacity = resident_capacity
|
||||
self._tombstone_ttl = tombstone_ttl
|
||||
self._records: OrderedDict[str, SchedulerTransfer] = OrderedDict()
|
||||
self._active_ids: dict[str, None] = {}
|
||||
self._terminal_ids: OrderedDict[str, None] = OrderedDict()
|
||||
self._resident_ids: OrderedDict[str, None] = OrderedDict()
|
||||
self._resident_bytes = 0
|
||||
self._hash_index: dict[str, deque[str]] = {}
|
||||
self._loads_to_dispatch: OrderedDict[str, None] = OrderedDict()
|
||||
self._unavailable_requests: set[str] = set()
|
||||
# Records expired straight out of READY/RESIDENT: the consumer worker
|
||||
# still holds their push reservation, and nothing else tells it to let
|
||||
# go, so the destination buffer would sit pinned until the lease TTL.
|
||||
self._orphaned: list[str] = []
|
||||
|
||||
def get(self, transfer_id: str) -> SchedulerTransfer | None:
|
||||
return self._records.get(transfer_id)
|
||||
|
||||
def drain_orphaned(self) -> list[str]:
|
||||
"""Transfer IDs expired out of READY/RESIDENT since the last drain.
|
||||
|
||||
The caller must release the consumer worker's reservation for each:
|
||||
`cancel()` still accepts an EXPIRED record, so the usual cancel path
|
||||
applies.
|
||||
"""
|
||||
orphaned = self._orphaned
|
||||
self._orphaned = []
|
||||
return orphaned
|
||||
|
||||
def records_for_hash(
|
||||
self,
|
||||
mm_hash: str,
|
||||
states: Iterable[SchedulerTransferState],
|
||||
) -> list[SchedulerTransfer]:
|
||||
wanted = set(states)
|
||||
return [
|
||||
record
|
||||
for transfer_id in self._hash_index.get(mm_hash, ())
|
||||
if (record := self._records.get(transfer_id)) is not None
|
||||
and record.state in wanted
|
||||
]
|
||||
|
||||
def first_for_hash(
|
||||
self,
|
||||
mm_hash: str,
|
||||
states: Iterable[SchedulerTransferState],
|
||||
) -> SchedulerTransfer | None:
|
||||
states = tuple(states)
|
||||
if len(states) == 1:
|
||||
wanted_state = states[0]
|
||||
for transfer_id in self._hash_index.get(mm_hash, ()):
|
||||
record = self._records.get(transfer_id)
|
||||
if record is not None and record.state is wanted_state:
|
||||
return record
|
||||
return None
|
||||
wanted_states = set(states)
|
||||
for transfer_id in self._hash_index.get(mm_hash, ()):
|
||||
record = self._records.get(transfer_id)
|
||||
if record is not None and record.state in wanted_states:
|
||||
return record
|
||||
return None
|
||||
|
||||
def has_state(self, mm_hash: str, states: Iterable[SchedulerTransferState]) -> bool:
|
||||
return self.first_for_hash(mm_hash, states) is not None
|
||||
|
||||
def wait_for_event(
|
||||
self,
|
||||
transfer_id: str,
|
||||
request_id: str,
|
||||
mm_hash: str,
|
||||
deadline: float,
|
||||
) -> SchedulerTransfer | None:
|
||||
"""Track a request waiting for a push.
|
||||
|
||||
Returns None when `transfer_id` already names a different encoding.
|
||||
The id comes from the request, so a collision has to fail that one
|
||||
request; re-issuing it re-runs the encode under a fresh id.
|
||||
"""
|
||||
record = self._records.get(transfer_id)
|
||||
if record is None:
|
||||
record = SchedulerTransfer(
|
||||
transfer_id=transfer_id,
|
||||
request_id=request_id,
|
||||
mm_hash=mm_hash,
|
||||
state=SchedulerTransferState.WAITING_EVENT,
|
||||
spec=None,
|
||||
deadline=deadline,
|
||||
)
|
||||
self._insert(record)
|
||||
elif not self._identity_matches(record, mm_hash):
|
||||
self._refuse_colliding_id(record, mm_hash, request_id)
|
||||
return None
|
||||
else:
|
||||
if not record.request_id:
|
||||
record.request_id = request_id
|
||||
if record.state in _TERMINAL_STATES and request_id:
|
||||
self._notify_unavailable(record, request_id)
|
||||
return record
|
||||
|
||||
def observe_ready(
|
||||
self, spec: ECMooncakeLoadSpec, deadline: float
|
||||
) -> tuple[SchedulerTransfer, bool]:
|
||||
transfer_id = spec.transfer_id
|
||||
record = self._records.get(transfer_id)
|
||||
if record is None:
|
||||
record = SchedulerTransfer(
|
||||
transfer_id=transfer_id,
|
||||
request_id="",
|
||||
mm_hash=spec.mm_hash,
|
||||
state=SchedulerTransferState.WAITING_EVENT,
|
||||
spec=None,
|
||||
deadline=None,
|
||||
)
|
||||
self._insert(record)
|
||||
elif not self._identity_matches(record, spec.mm_hash):
|
||||
self._refuse_colliding_id(record, spec.mm_hash, record.request_id)
|
||||
return record, False
|
||||
if record.state is not SchedulerTransferState.WAITING_EVENT:
|
||||
return record, False
|
||||
record.spec = spec
|
||||
record.deadline = deadline
|
||||
self._transition(record, SchedulerTransferState.AVAILABLE)
|
||||
return record, True
|
||||
|
||||
def touch_available(self, transfer_id: str, deadline: float) -> None:
|
||||
record = self._records.get(transfer_id)
|
||||
if record is not None and record.state is SchedulerTransferState.AVAILABLE:
|
||||
record.deadline = deadline
|
||||
|
||||
def begin_load(
|
||||
self,
|
||||
mm_hash: str,
|
||||
transfer_id: str | None = None,
|
||||
request_id: str = "",
|
||||
deadline: float | None = None,
|
||||
) -> SchedulerTransfer | None:
|
||||
record = self._records.get(transfer_id) if transfer_id else None
|
||||
if record is not None and (
|
||||
record.mm_hash != mm_hash
|
||||
or record.state is not SchedulerTransferState.AVAILABLE
|
||||
):
|
||||
record = None
|
||||
if record is None:
|
||||
record = self.first_for_hash(
|
||||
mm_hash,
|
||||
(
|
||||
SchedulerTransferState.AVAILABLE,
|
||||
SchedulerTransferState.RESIDENT,
|
||||
),
|
||||
)
|
||||
if record is None or record.spec is None:
|
||||
return None
|
||||
if not record.request_id:
|
||||
record.request_id = request_id
|
||||
# A dispatched load that never reports back would otherwise hold this
|
||||
# hash in LOADING for good, deferring every later request for it.
|
||||
record.deadline = deadline
|
||||
self._transition(record, SchedulerTransferState.LOADING)
|
||||
self._loads_to_dispatch[record.transfer_id] = None
|
||||
return record
|
||||
|
||||
def take_loads_to_dispatch(self) -> list[SchedulerTransfer]:
|
||||
records = [
|
||||
record
|
||||
for transfer_id in self._loads_to_dispatch
|
||||
if (record := self._records.get(transfer_id)) is not None
|
||||
and record.state is SchedulerTransferState.LOADING
|
||||
]
|
||||
self._loads_to_dispatch.clear()
|
||||
return records
|
||||
|
||||
def complete_load(self, mm_hash: str) -> bool:
|
||||
record = self.first_for_hash(mm_hash, (SchedulerTransferState.LOADING,))
|
||||
if record is None:
|
||||
return self.has_state(mm_hash, (SchedulerTransferState.READY,))
|
||||
self._transition(record, SchedulerTransferState.READY)
|
||||
return True
|
||||
|
||||
def fail_load(self, mm_hash: str, now: float) -> bool:
|
||||
record = self.first_for_hash(mm_hash, (SchedulerTransferState.LOADING,))
|
||||
if record is None:
|
||||
return self.has_state(mm_hash, (SchedulerTransferState.FAILED,))
|
||||
self._transition(record, SchedulerTransferState.FAILED, now=now)
|
||||
return True
|
||||
|
||||
def release_ready(self, mm_hash: str, now: float) -> None:
|
||||
ready = self.records_for_hash(mm_hash, (SchedulerTransferState.READY,))
|
||||
if ready:
|
||||
canonical = ready[-1]
|
||||
for record in self.records_for_hash(
|
||||
mm_hash,
|
||||
(SchedulerTransferState.READY, SchedulerTransferState.RESIDENT),
|
||||
):
|
||||
if record is not canonical:
|
||||
self._transition(record, SchedulerTransferState.EXPIRED, now=now)
|
||||
if canonical.spec is None:
|
||||
self._transition(canonical, SchedulerTransferState.EXPIRED, now=now)
|
||||
else:
|
||||
canonical.spec = replace(canonical.spec, local=True)
|
||||
self._transition(canonical, SchedulerTransferState.RESIDENT)
|
||||
self._evict_residents(now)
|
||||
|
||||
def reclaim(self, mm_hash: str, now: float) -> None:
|
||||
for record in self.records_for_hash(
|
||||
mm_hash,
|
||||
(SchedulerTransferState.READY, SchedulerTransferState.RESIDENT),
|
||||
):
|
||||
if record.state is SchedulerTransferState.READY:
|
||||
record.spec = None
|
||||
else:
|
||||
self._transition(record, SchedulerTransferState.EXPIRED, now=now)
|
||||
|
||||
def mark_unavailable(self, transfer_id: str, now: float) -> None:
|
||||
record = self._records[transfer_id]
|
||||
self._transition(record, SchedulerTransferState.UNAVAILABLE, now=now)
|
||||
if record.request_id:
|
||||
self._notify_unavailable(record, record.request_id)
|
||||
|
||||
def cancel(
|
||||
self,
|
||||
transfer_id: str,
|
||||
now: float,
|
||||
mm_hash: str = "",
|
||||
request_id: str = "",
|
||||
) -> bool:
|
||||
record = self._records.get(transfer_id)
|
||||
if (
|
||||
record is not None
|
||||
and mm_hash
|
||||
and not self._identity_matches(record, mm_hash)
|
||||
):
|
||||
return False
|
||||
if record is None:
|
||||
record = SchedulerTransfer(
|
||||
transfer_id=transfer_id,
|
||||
request_id=request_id,
|
||||
mm_hash=mm_hash,
|
||||
state=SchedulerTransferState.WAITING_EVENT,
|
||||
spec=None,
|
||||
deadline=None,
|
||||
)
|
||||
self._insert(record)
|
||||
if record.state in {
|
||||
SchedulerTransferState.CANCELLED,
|
||||
SchedulerTransferState.READY,
|
||||
SchedulerTransferState.RESIDENT,
|
||||
SchedulerTransferState.FAILED,
|
||||
}:
|
||||
return False
|
||||
self._transition(record, SchedulerTransferState.CANCELLED, now=now)
|
||||
self._loads_to_dispatch.pop(transfer_id, None)
|
||||
return True
|
||||
|
||||
def expire(self, now: float, terminal_limit: int) -> list[SchedulerTransfer]:
|
||||
if terminal_limit < 0:
|
||||
raise ValueError("terminal_limit must be non-negative")
|
||||
expired = []
|
||||
for transfer_id in list(self._active_ids):
|
||||
record = self._records[transfer_id]
|
||||
if record.deadline is None or record.deadline > now:
|
||||
continue
|
||||
if record.state is SchedulerTransferState.AVAILABLE:
|
||||
self._transition(
|
||||
record,
|
||||
SchedulerTransferState.EXPIRED,
|
||||
now=now,
|
||||
)
|
||||
expired.append(record)
|
||||
elif record.state is SchedulerTransferState.LOADING:
|
||||
self._transition(
|
||||
record,
|
||||
SchedulerTransferState.UNAVAILABLE,
|
||||
now=now,
|
||||
)
|
||||
if record.request_id:
|
||||
self._notify_unavailable(record, record.request_id)
|
||||
expired.append(record)
|
||||
while self._terminal_ids:
|
||||
transfer_id = next(iter(self._terminal_ids))
|
||||
record = self._records[transfer_id]
|
||||
if (
|
||||
record.deadline is not None
|
||||
and record.deadline > now
|
||||
and len(self._terminal_ids) <= terminal_limit
|
||||
):
|
||||
break
|
||||
self._remove(transfer_id)
|
||||
return expired
|
||||
|
||||
def take_unavailable_requests(self) -> set[str]:
|
||||
unavailable = self._unavailable_requests
|
||||
self._unavailable_requests = set()
|
||||
return unavailable
|
||||
|
||||
def _notify_unavailable(self, record: SchedulerTransfer, request_id: str) -> None:
|
||||
if request_id not in record.notified_requests:
|
||||
record.notified_requests.add(request_id)
|
||||
self._unavailable_requests.add(request_id)
|
||||
|
||||
def _insert(self, record: SchedulerTransfer) -> None:
|
||||
self._records[record.transfer_id] = record
|
||||
self._active_ids[record.transfer_id] = None
|
||||
if record.mm_hash:
|
||||
self._hash_index.setdefault(record.mm_hash, deque()).append(
|
||||
record.transfer_id
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _identity_matches(record: SchedulerTransfer, mm_hash: str) -> bool:
|
||||
return not record.mm_hash or record.mm_hash == mm_hash
|
||||
|
||||
def _refuse_colliding_id(
|
||||
self, record: SchedulerTransfer, mm_hash: str, request_id: str
|
||||
) -> None:
|
||||
"""Report a transfer id that already names another encoding.
|
||||
|
||||
Transfer ids arrive on the request, so a collision is reachable from
|
||||
outside and must not reach the engine as an exception.
|
||||
"""
|
||||
logger.warning(
|
||||
"EC Mooncake transfer_id=%s already names mm_hash=%s; refusing "
|
||||
"mm_hash=%s for request %s",
|
||||
record.transfer_id,
|
||||
record.mm_hash[:16],
|
||||
mm_hash[:16],
|
||||
request_id or "<unknown>",
|
||||
)
|
||||
if request_id:
|
||||
self._unavailable_requests.add(request_id)
|
||||
|
||||
def _transition(
|
||||
self,
|
||||
record: SchedulerTransfer,
|
||||
state: SchedulerTransferState,
|
||||
now: float | None = None,
|
||||
) -> None:
|
||||
if record.state is SchedulerTransferState.RESIDENT:
|
||||
self._resident_ids.pop(record.transfer_id, None)
|
||||
if record.spec is not None:
|
||||
self._resident_bytes -= record.spec.nbytes
|
||||
if state is SchedulerTransferState.RESIDENT:
|
||||
self._resident_ids[record.transfer_id] = None
|
||||
if record.spec is not None:
|
||||
self._resident_bytes += record.spec.nbytes
|
||||
if state in _TERMINAL_STATES:
|
||||
if now is None:
|
||||
raise ValueError("Terminal transition requires a timestamp")
|
||||
# Keep a bounded tombstone so late events and repeat request IDs
|
||||
# remain idempotent instead of reviving a finished transfer.
|
||||
record.deadline = now + self._tombstone_ttl
|
||||
self._active_ids.pop(record.transfer_id, None)
|
||||
self._terminal_ids[record.transfer_id] = None
|
||||
self._terminal_ids.move_to_end(record.transfer_id)
|
||||
if state is SchedulerTransferState.EXPIRED and record.state in (
|
||||
SchedulerTransferState.READY,
|
||||
SchedulerTransferState.RESIDENT,
|
||||
):
|
||||
self._orphaned.append(record.transfer_id)
|
||||
record.state = state
|
||||
self._records.move_to_end(record.transfer_id)
|
||||
|
||||
def _evict_residents(self, now: float) -> None:
|
||||
while self._resident_bytes > self._resident_capacity and self._resident_ids:
|
||||
record = self._records[next(iter(self._resident_ids))]
|
||||
self._transition(record, SchedulerTransferState.EXPIRED, now=now)
|
||||
|
||||
def _remove(self, transfer_id: str) -> None:
|
||||
record = self._records.pop(transfer_id, None)
|
||||
self._active_ids.pop(transfer_id, None)
|
||||
self._terminal_ids.pop(transfer_id, None)
|
||||
if transfer_id in self._resident_ids:
|
||||
self._resident_ids.pop(transfer_id)
|
||||
if record is not None and record.spec is not None:
|
||||
self._resident_bytes -= record.spec.nbytes
|
||||
self._loads_to_dispatch.pop(transfer_id, None)
|
||||
if record is None or not record.mm_hash:
|
||||
return
|
||||
transfer_ids = self._hash_index[record.mm_hash]
|
||||
transfer_ids.remove(transfer_id)
|
||||
if not transfer_ids:
|
||||
self._hash_index.pop(record.mm_hash)
|
||||
@@ -0,0 +1,209 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Mooncake data-plane engine and memory-registration ownership.
|
||||
|
||||
The wrapper isolates optional-engine initialization, session addressing,
|
||||
synchronous batched writes, and the lifetime of transient source registrations.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_MOONCAKE_IMPORT_ERROR: ImportError | None = None
|
||||
try:
|
||||
from mooncake.engine import TransferEngine
|
||||
except ImportError as error:
|
||||
TransferEngine = None # type: ignore[misc, assignment]
|
||||
_MOONCAKE_IMPORT_ERROR = error
|
||||
|
||||
|
||||
def ensure_mooncake_available() -> None:
|
||||
if _MOONCAKE_IMPORT_ERROR is not None:
|
||||
raise ImportError(
|
||||
"Install mooncake-transfer-engine to use ECMooncakeConnector."
|
||||
) from _MOONCAKE_IMPORT_ERROR
|
||||
|
||||
|
||||
@dataclass
|
||||
class _SourceRegistration:
|
||||
"""Retain one transient source range while batches reference it."""
|
||||
|
||||
tensor: torch.Tensor
|
||||
nbytes: int
|
||||
users: int = 1
|
||||
|
||||
|
||||
class MooncakeTransfer:
|
||||
"""Own a lazy Mooncake engine and transient memory registrations."""
|
||||
|
||||
def __init__(self, hostname: str, protocol: str) -> None:
|
||||
self._hostname = hostname
|
||||
self._protocol = protocol
|
||||
self._engine: TransferEngine | None = None
|
||||
self._engine_lock = threading.Lock()
|
||||
self._source_registrations: dict[int, _SourceRegistration] = {}
|
||||
self._pending_unregister: dict[int, torch.Tensor] = {}
|
||||
self._registration_lock = threading.Lock()
|
||||
self._closed = False
|
||||
|
||||
def _ensure_engine(self) -> TransferEngine:
|
||||
if self._engine is not None:
|
||||
return self._engine
|
||||
with self._engine_lock:
|
||||
if self._engine is not None:
|
||||
return self._engine
|
||||
engine = TransferEngine()
|
||||
ret = engine.initialize(self._hostname, "P2PHANDSHAKE", self._protocol, "")
|
||||
if ret != 0:
|
||||
raise RuntimeError("Mooncake TransferEngine initialization failed.")
|
||||
self._engine = engine
|
||||
logger.info(
|
||||
"ECMooncakeConnector TransferEngine ready at %s:%d",
|
||||
self._hostname,
|
||||
engine.get_rpc_port(),
|
||||
)
|
||||
return self._engine
|
||||
|
||||
def ensure_ready(self) -> None:
|
||||
self._ensure_engine()
|
||||
|
||||
def local_session(self) -> str:
|
||||
engine = self._ensure_engine()
|
||||
return f"{self._hostname}:{engine.get_rpc_port()}"
|
||||
|
||||
def register_memory(self, tensor: torch.Tensor) -> int:
|
||||
return self._ensure_engine().batch_register_memory(
|
||||
[tensor.data_ptr()], [tensor.nbytes]
|
||||
)
|
||||
|
||||
def unregister_memory(self, tensor: torch.Tensor) -> bool:
|
||||
engine = self._ensure_engine()
|
||||
address = tensor.data_ptr()
|
||||
ret = engine.unregister_memory(address)
|
||||
with self._registration_lock:
|
||||
if ret != 0:
|
||||
logger.error(
|
||||
"Mooncake EC memory unregistration failed for address %d: %d",
|
||||
address,
|
||||
ret,
|
||||
)
|
||||
self._pending_unregister[address] = tensor
|
||||
return False
|
||||
self._pending_unregister.pop(address, None)
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _source_range(tensor: torch.Tensor) -> tuple[int, int]:
|
||||
# Encoder batches commonly split one storage into sibling tensor views.
|
||||
# Registering each view's exact bytes avoids overlapping memory regions.
|
||||
return tensor.data_ptr(), tensor.nbytes
|
||||
|
||||
def acquire_sources(self, tensors: list[torch.Tensor]) -> list[int]:
|
||||
ranges: dict[int, tuple[int, torch.Tensor]] = {}
|
||||
for tensor in tensors:
|
||||
address, nbytes = self._source_range(tensor)
|
||||
ranges.setdefault(address, (nbytes, tensor))
|
||||
|
||||
engine = self._ensure_engine()
|
||||
acquired: list[int] = []
|
||||
new_addresses: list[int] = []
|
||||
new_lengths: list[int] = []
|
||||
with self._registration_lock:
|
||||
for address, (nbytes, tensor) in ranges.items():
|
||||
entry = self._source_registrations.get(address)
|
||||
if entry is not None:
|
||||
if entry.nbytes != nbytes:
|
||||
raise RuntimeError(
|
||||
"Mooncake EC source storage changed size while registered"
|
||||
)
|
||||
entry.users += 1
|
||||
acquired.append(address)
|
||||
continue
|
||||
new_addresses.append(address)
|
||||
new_lengths.append(nbytes)
|
||||
self._source_registrations[address] = _SourceRegistration(
|
||||
tensor=tensor,
|
||||
nbytes=nbytes,
|
||||
)
|
||||
acquired.append(address)
|
||||
|
||||
if new_addresses:
|
||||
ret = engine.batch_register_memory(new_addresses, new_lengths)
|
||||
if ret != 0:
|
||||
for address in acquired:
|
||||
entry = self._source_registrations[address]
|
||||
entry.users -= 1
|
||||
if entry.users == 0:
|
||||
del self._source_registrations[address]
|
||||
raise RuntimeError("Mooncake EC source registration failed")
|
||||
return acquired
|
||||
|
||||
def release_sources(self, addresses: list[int]) -> bool:
|
||||
if not addresses:
|
||||
return True
|
||||
with self._registration_lock:
|
||||
unused = []
|
||||
for address in addresses:
|
||||
entry = self._source_registrations.get(address)
|
||||
if entry is None:
|
||||
continue
|
||||
entry.users -= 1
|
||||
if entry.users == 0:
|
||||
unused.append(address)
|
||||
if not unused:
|
||||
return True
|
||||
ret = self._ensure_engine().batch_unregister_memory(unused)
|
||||
if ret != 0:
|
||||
logger.warning(
|
||||
"Keeping %d EC source tensors registered after Mooncake "
|
||||
"unregistration failure",
|
||||
len(unused),
|
||||
)
|
||||
return False
|
||||
for address in unused:
|
||||
del self._source_registrations[address]
|
||||
self._pending_unregister.pop(address, None)
|
||||
return True
|
||||
|
||||
def write(
|
||||
self,
|
||||
session: str,
|
||||
sources: list[int],
|
||||
destinations: list[int],
|
||||
lengths: list[int],
|
||||
) -> None:
|
||||
"""Write one synchronous batch, returning only at terminal status."""
|
||||
ret = self._ensure_engine().batch_transfer_sync_write(
|
||||
session, sources, destinations, lengths
|
||||
)
|
||||
if ret != 0:
|
||||
raise RuntimeError(
|
||||
f"Mooncake EC push to {session} failed with status {ret}"
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
engine = self._engine
|
||||
if engine is None:
|
||||
return
|
||||
with self._registration_lock:
|
||||
addresses = list(self._source_registrations)
|
||||
addresses.extend(self._pending_unregister)
|
||||
if not addresses:
|
||||
return
|
||||
ret = engine.batch_unregister_memory(list(dict.fromkeys(addresses)))
|
||||
if ret != 0:
|
||||
logger.error("Mooncake EC batch memory unregistration failed: %d", ret)
|
||||
return
|
||||
self._source_registrations.clear()
|
||||
self._pending_unregister.clear()
|
||||
@@ -0,0 +1,940 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Worker-side orchestration of Mooncake control, memory, and data planes.
|
||||
|
||||
Consumer Workers expose rank-local reservations and publish received tensors.
|
||||
Producer Workers reserve every destination shard, bind computed sources, run
|
||||
batched Mooncake writes, and report asynchronous completion to the Scheduler.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Any, TypeVar, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.config import (
|
||||
_RESERVATION_TTL_SECONDS,
|
||||
MooncakeECConfig,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.control import (
|
||||
ConsumerControlServer,
|
||||
ControlClient,
|
||||
make_cancel_request,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.memory import (
|
||||
ConsumerMemoryPool,
|
||||
ProducerMemoryPool,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
|
||||
ECMooncakeConnectorMetadata,
|
||||
ECMooncakePushSpec,
|
||||
ECMooncakeWorkerMetadata,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.producer import (
|
||||
ProducerPushManager,
|
||||
ProducerPushRecord,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.reservation import (
|
||||
ConsumerReservationManager,
|
||||
ConsumerReservationState,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.transfer import (
|
||||
MooncakeTransfer,
|
||||
ensure_mooncake_available,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.network_utils import get_ip
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_T = TypeVar("_T")
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
_RESERVATION_REFRESH_SECONDS = _RESERVATION_TTL_SECONDS / 2
|
||||
_MAX_CANCELLED_TRANSFER_IDS = 1 << 16
|
||||
_CANCEL_ATTEMPTS = 2
|
||||
_TRANSFER_WORKERS = 4
|
||||
_CONTROL_WORKERS = 8
|
||||
|
||||
|
||||
class _FanoutError(RuntimeError):
|
||||
"""Retain shard outcomes after every started task settles."""
|
||||
|
||||
def __init__(self, error: BaseException, results: list[Any | None]) -> None:
|
||||
self.results = results
|
||||
super().__init__(str(error))
|
||||
|
||||
|
||||
class ECMooncakeWorker:
|
||||
"""Orchestrate consumer reservations and producer push batches.
|
||||
|
||||
Consumers may use TP, PP, and DP. Only the first PP stage receives encoder
|
||||
outputs, and each TP rank exposes a consecutive control port and receives
|
||||
the same source concurrently. Producers remain unsharded and unreplicated.
|
||||
With DP, the caller must route both halves of a request to the same replica
|
||||
and pass that replica's control address to the producer.
|
||||
"""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig) -> None:
|
||||
ensure_mooncake_available()
|
||||
config = MooncakeECConfig.from_vllm_config(vllm_config)
|
||||
self.is_producer = config.is_producer
|
||||
self.is_consumer = config.is_consumer
|
||||
self._buffer_device = config.buffer_device
|
||||
self._control_host = config.control_host
|
||||
self._control_port = config.control_port
|
||||
self._transfer = MooncakeTransfer(get_ip(), config.protocol)
|
||||
self._consumer_memory = ConsumerMemoryPool(
|
||||
config.pool_size,
|
||||
self._transfer,
|
||||
)
|
||||
self._reservations = ConsumerReservationManager(
|
||||
self._consumer_memory,
|
||||
_RESERVATION_TTL_SECONDS,
|
||||
_MAX_CANCELLED_TRANSFER_IDS,
|
||||
)
|
||||
self._consumer_rank_resolved = False
|
||||
self._is_receiving_rank = True
|
||||
self._tp_rank = 0
|
||||
self._tp_size = 1
|
||||
self._control_server: ConsumerControlServer | None = None
|
||||
# Worker producer
|
||||
self._producer_memory = ProducerMemoryPool(
|
||||
config.pool_size,
|
||||
self._transfer,
|
||||
)
|
||||
self._control_client = ControlClient(config.control_timeout_ms)
|
||||
self._shutdown_drain_timeout_s = config.control_timeout_ms / 1000
|
||||
self._io_executor = ThreadPoolExecutor(
|
||||
max_workers=_TRANSFER_WORKERS,
|
||||
thread_name_prefix="ec-mooncake-transfer",
|
||||
)
|
||||
self._control_executor = ThreadPoolExecutor(
|
||||
max_workers=_CONTROL_WORKERS,
|
||||
thread_name_prefix="ec-mooncake-control",
|
||||
)
|
||||
self._shard_pool: ThreadPoolExecutor | None = None
|
||||
self._shard_pool_lock = threading.Lock()
|
||||
self._push_ready = threading.Event()
|
||||
self._producer_pushes = ProducerPushManager(self._push_ready.set)
|
||||
self._dispatch_stop = threading.Event()
|
||||
self._dispatcher: threading.Thread | None = None
|
||||
self._failed_saves: set[str] = set()
|
||||
self._collecting_sources = False
|
||||
self._completed_loads: set[str] = set()
|
||||
self._failed_loads: set[str] = set()
|
||||
self._shutdown = False
|
||||
if self.is_producer:
|
||||
self._dispatcher = threading.Thread(
|
||||
target=self._dispatch_pushes, name="ec-mooncake-ready", daemon=True
|
||||
)
|
||||
self._dispatcher.start()
|
||||
|
||||
def _dispatch_pushes(self) -> None:
|
||||
pending_event = False
|
||||
while not self._dispatch_stop.is_set():
|
||||
self._push_ready.wait(timeout=0.001 if pending_event else None)
|
||||
self._push_ready.clear()
|
||||
if self._dispatch_stop.is_set():
|
||||
break
|
||||
if self._collecting_sources:
|
||||
pending_event = False
|
||||
continue
|
||||
pending_event = self._producer_pushes.submit_batches(
|
||||
self._io_executor, self._push_batch
|
||||
)
|
||||
|
||||
def _resolve_consumer_rank(self) -> None:
|
||||
"""Place this worker in the consumer receive topology."""
|
||||
if self._consumer_rank_resolved:
|
||||
return
|
||||
self._consumer_rank_resolved = True
|
||||
try:
|
||||
from vllm.distributed.parallel_state import get_pp_group, get_tp_group
|
||||
|
||||
tp_group = get_tp_group()
|
||||
self._tp_rank = tp_group.rank_in_group
|
||||
self._tp_size = tp_group.world_size
|
||||
self._is_receiving_rank = get_pp_group().is_first_rank
|
||||
except AssertionError:
|
||||
# Groups are only absent outside a distributed run, where this
|
||||
# worker is the whole consumer.
|
||||
self._tp_rank = 0
|
||||
self._tp_size = 1
|
||||
self._is_receiving_rank = True
|
||||
|
||||
def start_services(self) -> None:
|
||||
if not self.is_consumer or self._control_server is not None:
|
||||
return
|
||||
self._resolve_consumer_rank()
|
||||
if not self._is_receiving_rank:
|
||||
# Later pipeline stages hold no encoder outputs, so they need
|
||||
# neither a receive pool nor a control channel.
|
||||
return
|
||||
self._consumer_memory.prepare(torch.device(self._buffer_device))
|
||||
consumer_pool = self._consumer_memory.tensor
|
||||
if consumer_pool is None:
|
||||
raise RuntimeError(
|
||||
"Mooncake push mode requires a registered consumer buffer pool."
|
||||
)
|
||||
base_port = self._control_port
|
||||
self._control_server = ConsumerControlServer(
|
||||
# The reservation channel hands out registered-memory addresses
|
||||
# and takes cancellations, so it listens where the Consumer
|
||||
# advertises itself (`ec_ip`), not on every interface.
|
||||
self._control_host,
|
||||
base_port + self._tp_rank,
|
||||
self._reserve_push_destination,
|
||||
self._push_status,
|
||||
self._reservations.complete,
|
||||
self._reservations.cancel,
|
||||
self._reservations.expire,
|
||||
peer_ports=[base_port + rank for rank in range(self._tp_size)],
|
||||
device=consumer_pool.device,
|
||||
drain_ready=self._reservations.drain_ready,
|
||||
)
|
||||
try:
|
||||
self._control_server.start()
|
||||
except Exception:
|
||||
self._control_server.close()
|
||||
self._control_server = None
|
||||
raise
|
||||
|
||||
def _reserve_push_destination(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
transfer_id = str(payload["transfer_id"])
|
||||
mm_hash = str(payload["mm_hash"])
|
||||
nbytes = int(payload["nbytes"])
|
||||
shape = tuple(int(value) for value in payload["shape"])
|
||||
dtype_name = str(payload["dtype"])
|
||||
dtype = getattr(torch, dtype_name, None)
|
||||
if not isinstance(dtype, torch.dtype):
|
||||
raise ValueError(f"Unsupported torch dtype string: {dtype_name!r}")
|
||||
expected_nbytes = math.prod(shape) * dtype.itemsize
|
||||
if expected_nbytes != nbytes:
|
||||
raise ValueError("shape and dtype do not match nbytes")
|
||||
|
||||
self._reservations.expire()
|
||||
reservation, should_write = self._reservations.reserve(
|
||||
transfer_id, mm_hash, nbytes, shape, dtype_name, dtype
|
||||
)
|
||||
if reservation is None:
|
||||
raise RuntimeError("EC consumer buffer pool is full")
|
||||
if reservation.state in {
|
||||
ConsumerReservationState.CANCEL_PENDING,
|
||||
ConsumerReservationState.CANCELLED,
|
||||
}:
|
||||
return {
|
||||
"reservation_id": "",
|
||||
"dst_session": "",
|
||||
"dst_ptr": 0,
|
||||
"nbytes": nbytes,
|
||||
"write": False,
|
||||
"ready": False,
|
||||
"cancelled": True,
|
||||
}
|
||||
assert reservation.allocation is not None
|
||||
|
||||
return {
|
||||
"reservation_id": reservation.reservation_id,
|
||||
"dst_session": self._transfer.local_session(),
|
||||
"dst_ptr": reservation.allocation.tensor.data_ptr(),
|
||||
"nbytes": reservation.allocation.tensor.nbytes,
|
||||
"write": should_write,
|
||||
"ready": reservation.state is ConsumerReservationState.READY,
|
||||
"cached": reservation.lease is not None,
|
||||
}
|
||||
|
||||
def _push_status(self, transfer_id: str) -> dict[str, Any] | None:
|
||||
reservation = self._reservations.status(transfer_id)
|
||||
if reservation is None:
|
||||
return None
|
||||
assert reservation.allocation is not None
|
||||
return {
|
||||
"mm_hash": reservation.mm_hash,
|
||||
"ready": reservation.state is ConsumerReservationState.READY,
|
||||
"reservation_id": reservation.reservation_id,
|
||||
"nbytes": reservation.allocation.tensor.nbytes,
|
||||
"shape": list(reservation.shape),
|
||||
"dtype": reservation.dtype,
|
||||
}
|
||||
|
||||
def _shard_executor(self) -> ThreadPoolExecutor:
|
||||
"""Use a separate pool so nested shard fan-out cannot deadlock."""
|
||||
with self._shard_pool_lock:
|
||||
if self._shard_pool is None:
|
||||
self._shard_pool = ThreadPoolExecutor(
|
||||
max_workers=32, thread_name_prefix="ec-mooncake-shard"
|
||||
)
|
||||
return self._shard_pool
|
||||
|
||||
def _reserve_one(self, addr: str, spec: ECMooncakePushSpec) -> dict[str, Any]:
|
||||
result = self._control_client.request(
|
||||
addr,
|
||||
{
|
||||
"op": "reserve",
|
||||
"transfer_id": spec.transfer_id,
|
||||
"mm_hash": spec.mm_hash,
|
||||
"nbytes": spec.nbytes,
|
||||
"shape": list(spec.shape),
|
||||
"dtype": spec.dtype,
|
||||
},
|
||||
)
|
||||
if not isinstance(result, dict):
|
||||
raise RuntimeError("Invalid EC reservation response")
|
||||
result["_received_at"] = time.monotonic()
|
||||
result["addr"] = addr
|
||||
return result
|
||||
|
||||
def _run_fanout(
|
||||
self,
|
||||
tasks: list[Callable[[], _T]],
|
||||
on_submit: Callable[[int, Future[_T]], None] | None = None,
|
||||
) -> list[_T]:
|
||||
"""Run every started shard task and retain partial results on failure.
|
||||
|
||||
Waiting for all submitted tasks is what makes source-memory release and
|
||||
partial-reservation cleanup safe after one shard fails.
|
||||
"""
|
||||
if not tasks:
|
||||
return []
|
||||
futures: list[tuple[int, Future[_T]]] = []
|
||||
results: list[_T | None] = [None] * len(tasks)
|
||||
error: BaseException | None = None
|
||||
for index, task in enumerate(tasks[1:], 1):
|
||||
try:
|
||||
future = self._shard_executor().submit(task)
|
||||
except Exception as exc:
|
||||
error = exc
|
||||
break
|
||||
futures.append((index, future))
|
||||
if on_submit is not None:
|
||||
on_submit(index, future)
|
||||
if error is None:
|
||||
try:
|
||||
results[0] = tasks[0]()
|
||||
except Exception as exc:
|
||||
error = exc
|
||||
for index, future in futures:
|
||||
try:
|
||||
results[index] = future.result()
|
||||
except Exception as exc:
|
||||
if error is None:
|
||||
error = exc
|
||||
if error is not None:
|
||||
raise _FanoutError(error, results)
|
||||
return cast(list[_T], results)
|
||||
|
||||
def _cancel_reservations(
|
||||
self,
|
||||
spec: ECMooncakePushSpec,
|
||||
reservations: list[dict[str, Any]],
|
||||
*,
|
||||
refresh: bool = False,
|
||||
record: ProducerPushRecord | None = None,
|
||||
) -> None:
|
||||
reservations = [
|
||||
shard for shard in reservations if not shard.get("cancelled", False)
|
||||
]
|
||||
if not reservations:
|
||||
return
|
||||
|
||||
def cancel(shard: dict[str, Any]) -> dict[str, Any]:
|
||||
addr = shard.get("addr")
|
||||
if not isinstance(addr, str) or not addr:
|
||||
raise RuntimeError(
|
||||
"EC reservation is missing a confirmed shard address"
|
||||
)
|
||||
result = self._control_client.request(
|
||||
addr,
|
||||
make_cancel_request(
|
||||
spec.transfer_id,
|
||||
str(shard.get("reservation_id", "")),
|
||||
abandon=True,
|
||||
refresh=refresh,
|
||||
),
|
||||
)
|
||||
if not isinstance(result, dict) or not result.get("cancelled"):
|
||||
raise RuntimeError(
|
||||
f"Could not cancel EC reservation for mm_hash={spec.mm_hash}"
|
||||
)
|
||||
return shard
|
||||
|
||||
def track(_index: int, future: Future[dict[str, Any]]) -> None:
|
||||
if record is not None:
|
||||
self._producer_pushes.track_shard_futures([record], [future])
|
||||
|
||||
self._run_fanout([partial(cancel, shard) for shard in reservations], track)
|
||||
|
||||
def _retry_cancel_reservations(
|
||||
self,
|
||||
spec: ECMooncakePushSpec,
|
||||
reservations: list[dict[str, Any]],
|
||||
*,
|
||||
record: ProducerPushRecord | None = None,
|
||||
) -> None:
|
||||
pending = [shard for shard in reservations if not shard.get("cancelled", False)]
|
||||
error: _FanoutError | None = None
|
||||
for _ in range(_CANCEL_ATTEMPTS):
|
||||
try:
|
||||
self._cancel_reservations(spec, pending, record=record)
|
||||
except _FanoutError as exc:
|
||||
error = exc
|
||||
pending = [
|
||||
shard
|
||||
for index, shard in enumerate(pending)
|
||||
if exc.results[index] is None
|
||||
]
|
||||
continue
|
||||
return
|
||||
assert error is not None
|
||||
raise error
|
||||
|
||||
def _reserve_remote(self, spec: ECMooncakePushSpec) -> list[dict[str, Any]]:
|
||||
"""Reserve a destination on every shard of the consumer."""
|
||||
result = self._reserve_remote_many([spec])[0]
|
||||
if isinstance(result, Exception):
|
||||
raise result
|
||||
return result
|
||||
|
||||
def _refresh_remote_reservations(
|
||||
self,
|
||||
spec: ECMooncakePushSpec,
|
||||
reservations: list[dict[str, Any]],
|
||||
record: ProducerPushRecord | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
stale = [
|
||||
shard
|
||||
for shard in reservations
|
||||
if not shard.get("ready", False)
|
||||
and not shard.get("cached", False)
|
||||
and not shard.get("cancelled", False)
|
||||
]
|
||||
try:
|
||||
self._cancel_reservations(spec, stale, refresh=True, record=record)
|
||||
except _FanoutError as exc:
|
||||
pending = [
|
||||
shard for index, shard in enumerate(stale) if exc.results[index] is None
|
||||
]
|
||||
try:
|
||||
self._retry_cancel_reservations(spec, pending, record=record)
|
||||
except _FanoutError as cleanup_error:
|
||||
raise exc from cleanup_error
|
||||
raise
|
||||
return self._reserve_remote(spec)
|
||||
|
||||
@staticmethod
|
||||
def _validate_push_source(push: ProducerPushRecord) -> None:
|
||||
tensor = push.source_tensor
|
||||
assert tensor is not None
|
||||
spec = push.spec
|
||||
if tuple(tensor.shape) != tuple(spec.shape):
|
||||
raise ValueError(f"EC source shape mismatch for mm_hash={spec.mm_hash}")
|
||||
if str(tensor.dtype).split(".")[-1] != spec.dtype:
|
||||
raise ValueError(f"EC source dtype mismatch for mm_hash={spec.mm_hash}")
|
||||
if not tensor.is_contiguous():
|
||||
raise ValueError(f"EC source must be contiguous for mm_hash={spec.mm_hash}")
|
||||
if tensor.nbytes != spec.nbytes:
|
||||
raise ValueError(f"EC source size mismatch for mm_hash={spec.mm_hash}")
|
||||
|
||||
def start_save_caches(
|
||||
self,
|
||||
metadata: ECMooncakeConnectorMetadata,
|
||||
encoder_cache: dict[str, torch.Tensor] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
self._collecting_sources = True
|
||||
new: dict[str, list[ProducerPushRecord]] = {}
|
||||
for spec in metadata.pushes:
|
||||
try:
|
||||
record, created = self._producer_pushes.reserve(spec, Future)
|
||||
if created:
|
||||
new.setdefault(spec.consumer_zmq, []).append(record)
|
||||
except ValueError:
|
||||
logger.warning("Rejected conflicting EC push", exc_info=True)
|
||||
self._failed_saves.add(spec.request_id)
|
||||
for records in new.values():
|
||||
future = self._control_executor.submit(self._reserve_batch, records)
|
||||
future.add_done_callback(partial(self._reservation_batch_done, records))
|
||||
if not isinstance(encoder_cache, dict):
|
||||
return
|
||||
for mm_hash in dict.fromkeys(spec.mm_hash for spec in metadata.pushes):
|
||||
tensor = encoder_cache.get(mm_hash)
|
||||
if tensor is not None:
|
||||
self._bind_push_source(tensor, mm_hash)
|
||||
|
||||
def _reserve_batch(self, records: list[ProducerPushRecord]) -> None:
|
||||
"""Resolve independent item futures with one reserve RPC per shard."""
|
||||
if len(records) == 1:
|
||||
record = records[0]
|
||||
try:
|
||||
record.reservation_future.set_result(self._reserve_remote(record.spec))
|
||||
except Exception as exc:
|
||||
record.reservation_future.set_exception(exc)
|
||||
return
|
||||
outcomes = self._reserve_remote_many([record.spec for record in records])
|
||||
self._producer_pushes.finish_reservations(list(zip(records, outcomes)))
|
||||
|
||||
def _reserve_remote_many(
|
||||
self, specs: list[ECMooncakePushSpec]
|
||||
) -> list[list[dict[str, Any]] | Exception]:
|
||||
shards = None
|
||||
for _ in range(_CANCEL_ATTEMPTS):
|
||||
shards = self._control_client.discover_shards(specs[0].consumer_zmq)
|
||||
if shards is not None:
|
||||
break
|
||||
if shards is None:
|
||||
raise RuntimeError(
|
||||
f"Could not discover every EC consumer shard at {specs[0].consumer_zmq}"
|
||||
)
|
||||
|
||||
def reserve(addr: str) -> list[dict[str, Any]]:
|
||||
if len(specs) == 1:
|
||||
return [{"ok": True, "result": self._reserve_one(addr, specs[0])}]
|
||||
response = self._control_client.request(
|
||||
addr,
|
||||
{
|
||||
"op": "reserve_batch",
|
||||
"items": [
|
||||
{
|
||||
"transfer_id": spec.transfer_id,
|
||||
"mm_hash": spec.mm_hash,
|
||||
"nbytes": spec.nbytes,
|
||||
"shape": list(spec.shape),
|
||||
"dtype": spec.dtype,
|
||||
}
|
||||
for spec in specs
|
||||
],
|
||||
},
|
||||
)
|
||||
items = response["items"]
|
||||
if len(items) != len(specs):
|
||||
raise RuntimeError("Malformed EC reservation batch response")
|
||||
for item in items:
|
||||
if item.get("ok"):
|
||||
item["result"]["_received_at"] = time.monotonic()
|
||||
return items
|
||||
|
||||
fanout_error = None
|
||||
results: list[Any]
|
||||
try:
|
||||
results = self._run_fanout([partial(reserve, addr) for addr in shards])
|
||||
except _FanoutError as exc:
|
||||
results = exc.results
|
||||
fanout_error = exc
|
||||
outcomes: list[list[dict[str, Any]] | Exception] = []
|
||||
for index, spec in enumerate(specs):
|
||||
reservations: list[dict[str, Any]] = []
|
||||
error = None
|
||||
for addr, items in zip(shards, results):
|
||||
item = items[index] if items is not None else {}
|
||||
if item.get("ok"):
|
||||
reservations.append({**item["result"], "addr": addr})
|
||||
else:
|
||||
error = fanout_error or RuntimeError(item["error"])
|
||||
reservations.append({"addr": addr, "reservation_id": ""})
|
||||
if error is None:
|
||||
outcomes.append(reservations)
|
||||
else:
|
||||
failure = _FanoutError(error, list(reservations))
|
||||
try:
|
||||
self._retry_cancel_reservations(spec, reservations)
|
||||
except Exception as cleanup_error:
|
||||
failure.__cause__ = cleanup_error
|
||||
logger.exception("Failed to release partial EC reservation batch")
|
||||
outcomes.append(failure)
|
||||
return outcomes
|
||||
|
||||
@staticmethod
|
||||
def _reservation_batch_done(
|
||||
records: list[ProducerPushRecord], future: Future[None]
|
||||
) -> None:
|
||||
error = future.exception()
|
||||
if error is not None:
|
||||
for record in records:
|
||||
if not record.reservation_future.done():
|
||||
record.reservation_future.set_exception(error)
|
||||
|
||||
def start_load_caches(
|
||||
self,
|
||||
metadata: ECMooncakeConnectorMetadata,
|
||||
encoder_cache: dict[str, torch.Tensor],
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
self._resolve_consumer_rank()
|
||||
if not self._is_receiving_rank:
|
||||
# Later pipeline stages never gather multimodal embeddings.
|
||||
return
|
||||
self._transfer.ensure_ready()
|
||||
if self._buffer_device == "cuda" and not torch.accelerator.is_available():
|
||||
raise RuntimeError(
|
||||
"ECMooncakeConnector requires CUDA for ec_buffer_device=cuda"
|
||||
)
|
||||
self._reservations.retire_stale(encoder_cache, metadata.freed)
|
||||
|
||||
for spec in metadata.loads:
|
||||
if spec.mm_hash in encoder_cache:
|
||||
if not spec.local:
|
||||
# The spec's id is one shard's; cancel by transfer.
|
||||
self._reservations.cancel(spec.transfer_id, "")
|
||||
self._completed_loads.add(spec.mm_hash)
|
||||
continue
|
||||
if spec.local:
|
||||
tensor = self._consumer_memory.take_resident(
|
||||
spec.mm_hash, tuple(spec.shape), spec.dtype
|
||||
)
|
||||
else:
|
||||
try:
|
||||
allocation = self._reservations.take(spec.transfer_id, spec.mm_hash)
|
||||
tensor = allocation.tensor
|
||||
except RuntimeError as e:
|
||||
logger.warning("EC Mooncake pushed load failed: %s", e)
|
||||
tensor = None
|
||||
if tensor is None:
|
||||
self._failed_loads.add(spec.mm_hash)
|
||||
else:
|
||||
encoder_cache[spec.mm_hash] = tensor
|
||||
self._completed_loads.add(spec.mm_hash)
|
||||
|
||||
def _push_batch(self, pushes: list[ProducerPushRecord]) -> None:
|
||||
started_at = time.monotonic()
|
||||
ready: list[tuple[ProducerPushRecord, dict[str, Any]]] = []
|
||||
written_pushes: dict[str, ProducerPushRecord] = {}
|
||||
failure: Exception | None = None
|
||||
try:
|
||||
for push in pushes:
|
||||
self._validate_push_source(push)
|
||||
reservations = self._producer_pushes.resolve_reservations(push)
|
||||
stale = [
|
||||
index
|
||||
for index, shard in enumerate(reservations)
|
||||
if not shard.get("ready", False)
|
||||
and not shard.get("cancelled", False)
|
||||
and time.monotonic() - float(shard.get("_received_at", started_at))
|
||||
>= _RESERVATION_REFRESH_SECONDS
|
||||
]
|
||||
if stale:
|
||||
reservations = self._refresh_remote_reservations(
|
||||
push.spec, reservations, push
|
||||
)
|
||||
self._producer_pushes.replace_reservations(push, reservations)
|
||||
self._producer_pushes.begin_writing(push)
|
||||
writable = [
|
||||
shard
|
||||
for shard in reservations
|
||||
if not shard.get("cached", False)
|
||||
and not shard.get("cancelled", False)
|
||||
and shard.get("write", True)
|
||||
]
|
||||
source = push.source_tensor
|
||||
assert source is not None
|
||||
for shard in writable:
|
||||
if int(shard["nbytes"]) != source.nbytes:
|
||||
raise RuntimeError(
|
||||
"Reserved EC size does not match tensor for "
|
||||
f"mm_hash={push.spec.mm_hash}"
|
||||
)
|
||||
ready.append((push, shard))
|
||||
written_pushes.setdefault(push.spec.transfer_id, push)
|
||||
if ready:
|
||||
# Stage each source once, then write it to every destination.
|
||||
source_index = {
|
||||
push.spec.transfer_id: index
|
||||
for index, push in enumerate(written_pushes.values())
|
||||
}
|
||||
tensors = [
|
||||
cast(torch.Tensor, push.source_tensor)
|
||||
for push in written_pushes.values()
|
||||
if push.source_tensor is not None
|
||||
]
|
||||
lengths = [tensor.nbytes for tensor in tensors]
|
||||
staged = self._producer_memory.stage(tensors)
|
||||
registered_sources: list[int] = []
|
||||
if staged is not None:
|
||||
sources = staged.tensors
|
||||
else:
|
||||
sources = tensors
|
||||
registered_sources = self._transfer.acquire_sources(tensors)
|
||||
addresses = [tensor.data_ptr() for tensor in sources]
|
||||
try:
|
||||
by_session: dict[str, list[tuple[int, int]]] = {}
|
||||
session_records: dict[str, dict[str, ProducerPushRecord]] = {}
|
||||
for push, shard in ready:
|
||||
session = str(shard["dst_session"])
|
||||
by_session.setdefault(session, []).append(
|
||||
(source_index[push.spec.transfer_id], int(shard["dst_ptr"]))
|
||||
)
|
||||
session_records.setdefault(session, {})[
|
||||
push.spec.transfer_id
|
||||
] = push
|
||||
|
||||
def write(session: str, items: list[tuple[int, int]]) -> None:
|
||||
self._transfer.write(
|
||||
session,
|
||||
[addresses[index] for index, _ in items],
|
||||
[dst for _, dst in items],
|
||||
[lengths[index] for index, _ in items],
|
||||
)
|
||||
|
||||
sessions = list(by_session.items())
|
||||
|
||||
# Write shards concurrently to avoid serial TP latency.
|
||||
def track_write(index: int, future: Future[None]) -> None:
|
||||
session = sessions[index][0]
|
||||
self._producer_pushes.track_shard_futures(
|
||||
list(session_records[session].values()), [future]
|
||||
)
|
||||
|
||||
writes: list[Callable[[], None]] = [
|
||||
partial(write, *session) for session in sessions
|
||||
]
|
||||
self._run_fanout(writes, track_write)
|
||||
finally:
|
||||
if staged is not None:
|
||||
self._producer_memory.release(staged)
|
||||
self._transfer.release_sources(registered_sources)
|
||||
|
||||
self._producer_pushes.begin_notifying(pushes)
|
||||
self._notify_completions(ready)
|
||||
self._producer_pushes.complete(pushes)
|
||||
except Exception as exc:
|
||||
# Report asynchronously; raising here would fail EngineCore.
|
||||
failure = exc
|
||||
logger.exception(
|
||||
"EC Mooncake push batch failed for mm_hashes=%s",
|
||||
[push.spec.mm_hash for push in pushes],
|
||||
)
|
||||
self._producer_pushes.settle_all(pushes)
|
||||
self._abandon_pushes(pushes)
|
||||
finally:
|
||||
if failure is not None:
|
||||
self._producer_pushes.fail(pushes, failure)
|
||||
|
||||
def _notify_completions(
|
||||
self, notifications: list[tuple[ProducerPushRecord, dict[str, Any]]]
|
||||
) -> None:
|
||||
"""Tell the consumer, in one message per destination, what landed."""
|
||||
if not notifications:
|
||||
return
|
||||
by_destination: dict[str, list[tuple[ProducerPushRecord, dict[str, Any]]]] = {}
|
||||
for push, reservation in notifications:
|
||||
by_destination.setdefault(
|
||||
str(reservation.get("addr", push.spec.consumer_zmq)), []
|
||||
).append((push, reservation))
|
||||
destinations = list(by_destination.items())
|
||||
|
||||
def notify(
|
||||
consumer_zmq: str,
|
||||
items: list[tuple[ProducerPushRecord, dict[str, Any]]],
|
||||
) -> None:
|
||||
result = self._control_client.request(
|
||||
consumer_zmq,
|
||||
{
|
||||
"op": "complete_batch",
|
||||
"items": [
|
||||
{
|
||||
"transfer_id": push.spec.transfer_id,
|
||||
"reservation_id": reservation["reservation_id"],
|
||||
}
|
||||
for push, reservation in items
|
||||
],
|
||||
},
|
||||
)
|
||||
completions = result.get("items", []) if isinstance(result, dict) else []
|
||||
if len(completions) != len(items):
|
||||
raise RuntimeError("Malformed EC completion response")
|
||||
for (push, _), completion in zip(items, completions):
|
||||
if not completion.get("completed"):
|
||||
raise RuntimeError(
|
||||
f"Unknown EC reservation for mm_hash={push.spec.mm_hash}"
|
||||
)
|
||||
|
||||
def track(index: int, future: Future[None]) -> None:
|
||||
records = {
|
||||
push.spec.transfer_id: push for push, _ in destinations[index][1]
|
||||
}
|
||||
self._producer_pushes.track_shard_futures(list(records.values()), [future])
|
||||
|
||||
self._run_fanout(
|
||||
[
|
||||
partial(notify, destination, items)
|
||||
for destination, items in destinations
|
||||
],
|
||||
track,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _known_reservations(record: ProducerPushRecord) -> list[dict[str, Any]]:
|
||||
if record.reservations:
|
||||
return list(record.reservations)
|
||||
try:
|
||||
return list(record.reservation_future.result())
|
||||
except _FanoutError as exc:
|
||||
return [result for result in exc.results if isinstance(result, dict)]
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
def _abandon_pushes(self, pushes: list[ProducerPushRecord]) -> None:
|
||||
"""Release the consumer-side reservations of a batch that failed."""
|
||||
for push in pushes:
|
||||
shards = self._known_reservations(push)
|
||||
if not shards:
|
||||
# No confirmed topology means there is no safe address to
|
||||
# cancel. The original push failure remains observable via
|
||||
# ProducerPushManager.fail below.
|
||||
continue
|
||||
try:
|
||||
self._retry_cancel_reservations(push.spec, shards, record=push)
|
||||
except _FanoutError:
|
||||
logger.exception(
|
||||
"Failed to abandon EC reservations for transfer_id=%s",
|
||||
push.spec.transfer_id,
|
||||
)
|
||||
|
||||
def _flush_pending_pushes(self) -> None:
|
||||
self._producer_pushes.submit_batches(
|
||||
self._io_executor,
|
||||
self._push_batch,
|
||||
)
|
||||
|
||||
def _bind_push_source(self, tensor: torch.Tensor, mm_hash: str) -> None:
|
||||
ready_event = None
|
||||
if tensor.device.type == "cuda":
|
||||
ready_event = torch.Event()
|
||||
ready_event.record(torch.accelerator.current_stream(tensor.device))
|
||||
self._producer_pushes.bind_source(mm_hash, tensor, ready_event)
|
||||
|
||||
def _cancel_orphaned_reservation(self, record: ProducerPushRecord) -> None:
|
||||
resolution_error: Exception | None = None
|
||||
try:
|
||||
reservations = self._producer_pushes.resolve_reservations(record)
|
||||
except Exception as exc:
|
||||
resolution_error = exc
|
||||
reservations = self._known_reservations(record)
|
||||
reservations = [
|
||||
shard for shard in reservations if not shard.get("cancelled", False)
|
||||
]
|
||||
if not reservations:
|
||||
if resolution_error is not None:
|
||||
self._producer_pushes.fail([record], resolution_error)
|
||||
else:
|
||||
self._producer_pushes.finish_cancel(record)
|
||||
return
|
||||
try:
|
||||
self._retry_cancel_reservations(record.spec, reservations, record=record)
|
||||
except Exception as cleanup_error:
|
||||
if resolution_error is not None:
|
||||
combined_error = RuntimeError(
|
||||
f"EC reservation resolution failed ({resolution_error}); "
|
||||
f"cleanup also failed ({cleanup_error})"
|
||||
)
|
||||
combined_error.__cause__ = resolution_error
|
||||
cleanup_error = combined_error
|
||||
self._producer_pushes.fail([record], cleanup_error)
|
||||
return
|
||||
if resolution_error is not None:
|
||||
self._producer_pushes.fail([record], resolution_error)
|
||||
else:
|
||||
self._producer_pushes.finish_cancel(record)
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
if not self.is_producer:
|
||||
return None, None
|
||||
|
||||
for record in self._producer_pushes.cancel_requests(finished_req_ids):
|
||||
self._producer_pushes.submit_cancel(
|
||||
record,
|
||||
self._io_executor,
|
||||
self._cancel_orphaned_reservation,
|
||||
)
|
||||
return None, None
|
||||
|
||||
def save_caches(
|
||||
self, encoder_cache: dict[str, torch.Tensor], mm_hash: str, **kwargs: Any
|
||||
) -> None:
|
||||
if not self.is_producer:
|
||||
return
|
||||
tensor = encoder_cache[mm_hash]
|
||||
self._bind_push_source(tensor, mm_hash)
|
||||
|
||||
def build_connector_worker_meta(self) -> ECMooncakeWorkerMetadata | None:
|
||||
if self.is_consumer and not self._is_receiving_rank:
|
||||
# `loaded` is intersected across reporting ranks, so a stage that
|
||||
# never loads must not report at all rather than report nothing.
|
||||
return None
|
||||
|
||||
self._collecting_sources = False
|
||||
self._push_ready.set()
|
||||
self._flush_pending_pushes()
|
||||
failures = self._producer_pushes.poll()
|
||||
for mm_hash, error in failures:
|
||||
logger.error(
|
||||
"EC Mooncake async save failed for mm_hash=%s: %s",
|
||||
mm_hash,
|
||||
error,
|
||||
)
|
||||
reclaimed = self._consumer_memory.drain_reclaimed()
|
||||
meta = ECMooncakeWorkerMetadata(
|
||||
loaded=self._completed_loads,
|
||||
failed_loads=self._failed_loads,
|
||||
reclaimed=reclaimed,
|
||||
pending_saves=self._producer_pushes.pending,
|
||||
failed_saves=self._failed_saves,
|
||||
)
|
||||
self._failed_saves = set()
|
||||
self._completed_loads = set()
|
||||
self._failed_loads = set()
|
||||
return meta
|
||||
|
||||
def close(self) -> None:
|
||||
if self._shutdown:
|
||||
return
|
||||
self._shutdown = True
|
||||
dispatcher = getattr(self, "_dispatcher", None)
|
||||
if dispatcher is not None:
|
||||
self._dispatch_stop.set()
|
||||
self._push_ready.set()
|
||||
dispatcher.join()
|
||||
if self._control_server is not None:
|
||||
self._reservations.begin_shutdown()
|
||||
self._flush_pending_pushes()
|
||||
for record in self._producer_pushes.cancel_requests(None):
|
||||
self._producer_pushes.submit_cancel(
|
||||
record,
|
||||
self._io_executor,
|
||||
self._cancel_orphaned_reservation,
|
||||
)
|
||||
self._control_executor.shutdown(wait=True)
|
||||
self._producer_pushes.submit_batches(
|
||||
self._io_executor, self._push_batch, wait=True
|
||||
)
|
||||
self._io_executor.shutdown(wait=True)
|
||||
if self._shard_pool is not None:
|
||||
self._shard_pool.shutdown(wait=True)
|
||||
# Every producer-side thread that could hold a control socket is stopped.
|
||||
self._control_client.close()
|
||||
drained = True
|
||||
if self._control_server is not None:
|
||||
drained = self._reservations.wait_for_writers(
|
||||
self._shutdown_drain_timeout_s
|
||||
)
|
||||
self._control_server.close()
|
||||
if drained:
|
||||
self._consumer_memory.close()
|
||||
else:
|
||||
logger.error(
|
||||
"Timed out waiting for Mooncake EC writers; keeping the consumer "
|
||||
"receive pool registered"
|
||||
)
|
||||
self._producer_memory.close()
|
||||
self._transfer.close()
|
||||
@@ -0,0 +1,130 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Encoder-cache connector backed by Mooncake TransferEngine."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorBase,
|
||||
ECConnectorMetadata,
|
||||
ECConnectorRole,
|
||||
ECConnectorWorkerMetadata,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
|
||||
ECMooncakeConnectorMetadata,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.scheduler import (
|
||||
ECMooncakeScheduler,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.worker import ECMooncakeWorker
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import ECConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
|
||||
class ECMooncakeConnector(ECConnectorBase):
|
||||
"""Preserve the public API while delegating to one process-role component."""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, role: ECConnectorRole):
|
||||
super().__init__(vllm_config=vllm_config, role=role)
|
||||
|
||||
self._scheduler: ECMooncakeScheduler | None = None
|
||||
self._worker: ECMooncakeWorker | None = None
|
||||
self._closed = False
|
||||
|
||||
if role == ECConnectorRole.SCHEDULER:
|
||||
self._scheduler = ECMooncakeScheduler(vllm_config)
|
||||
elif role == ECConnectorRole.WORKER:
|
||||
self._worker = ECMooncakeWorker(vllm_config)
|
||||
else:
|
||||
raise ValueError(f"Unknown EC connector role: {role}")
|
||||
|
||||
def start_worker_services(self) -> None:
|
||||
assert self._worker is not None
|
||||
self._worker.start_services()
|
||||
|
||||
def start_save_caches(self, **kwargs: Any) -> None:
|
||||
assert self._worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, ECMooncakeConnectorMetadata)
|
||||
self._worker.start_save_caches(metadata, **kwargs)
|
||||
|
||||
def start_load_caches(
|
||||
self, encoder_cache: dict[str, torch.Tensor], **kwargs: Any
|
||||
) -> None:
|
||||
assert self._worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, ECMooncakeConnectorMetadata)
|
||||
self._worker.start_load_caches(metadata, encoder_cache, **kwargs)
|
||||
|
||||
def save_caches(
|
||||
self, encoder_cache: dict[str, torch.Tensor], mm_hash: str, **kwargs: Any
|
||||
) -> None:
|
||||
assert self._worker is not None
|
||||
self._worker.save_caches(encoder_cache, mm_hash, **kwargs)
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
assert self._worker is not None
|
||||
return self._worker.get_finished(finished_req_ids)
|
||||
|
||||
def build_connector_worker_meta(self) -> ECConnectorWorkerMetadata | None:
|
||||
assert self._worker is not None
|
||||
return self._worker.build_connector_worker_meta()
|
||||
|
||||
def take_unavailable_requests(self) -> set[str]:
|
||||
assert self._scheduler is not None
|
||||
return self._scheduler.take_unavailable_requests()
|
||||
|
||||
def has_cache_item(self, identifier: str) -> bool:
|
||||
assert self._scheduler is not None
|
||||
return self._scheduler.has_cache_item(identifier)
|
||||
|
||||
def ensure_cache_available(
|
||||
self, request: Request, num_computed_tokens: int
|
||||
) -> bool:
|
||||
assert self._scheduler is not None
|
||||
return self._scheduler.ensure_cache_available(request, num_computed_tokens)
|
||||
|
||||
def update_state_after_alloc(self, request: Request, index: int) -> None:
|
||||
assert self._scheduler is not None
|
||||
self._scheduler.update_state_after_alloc(request, index)
|
||||
|
||||
def update_state_after_free(self, request: Request, index: int) -> None:
|
||||
assert self._scheduler is not None
|
||||
self._scheduler.update_state_after_free(request, index)
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> ECConnectorMetadata:
|
||||
assert self._scheduler is not None
|
||||
return self._scheduler.build_connector_meta(scheduler_output)
|
||||
|
||||
def update_connector_output(self, connector_output: ECConnectorOutput) -> None:
|
||||
assert self._scheduler is not None
|
||||
self._scheduler.update_connector_output(connector_output)
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
assert self._scheduler is not None
|
||||
return self._scheduler.has_pending_push_work()
|
||||
|
||||
def request_finished(self, request: Request) -> tuple[bool, dict[str, Any] | None]:
|
||||
assert self._scheduler is not None
|
||||
return self._scheduler.request_finished(request)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
if self._scheduler is not None:
|
||||
self._scheduler.close()
|
||||
if self._worker is not None:
|
||||
self._worker.close()
|
||||
@@ -616,6 +616,17 @@ class Scheduler(SchedulerInterface):
|
||||
req_index += 1
|
||||
continue
|
||||
|
||||
if (
|
||||
self.ec_connector is not None
|
||||
and request.mm_features
|
||||
and not self.ec_connector.ensure_cache_available(
|
||||
request,
|
||||
request.num_computed_tokens - request.num_output_placeholders,
|
||||
)
|
||||
):
|
||||
req_index += 1
|
||||
continue
|
||||
|
||||
num_new_tokens = (
|
||||
request.num_tokens_with_spec
|
||||
+ request.num_output_placeholders
|
||||
@@ -945,13 +956,7 @@ class Scheduler(SchedulerInterface):
|
||||
assert num_computed_tokens <= request.num_tokens
|
||||
|
||||
# Skip request with pending mm encoding prefetches
|
||||
if (
|
||||
self.ec_connector is not None
|
||||
and request.mm_features
|
||||
and not self.ec_connector.ensure_cache_available(
|
||||
request, num_computed_tokens
|
||||
)
|
||||
):
|
||||
if self._ec_transfer_pending(request, num_computed_tokens):
|
||||
request_queue.pop_request()
|
||||
step_skipped_waiting.prepend_request(request)
|
||||
continue
|
||||
@@ -966,11 +971,18 @@ class Scheduler(SchedulerInterface):
|
||||
)
|
||||
else:
|
||||
# KVTransfer: WAITING reqs have num_computed_tokens > 0
|
||||
# after async KV recvs are completed.
|
||||
# after async KV recvs are completed. A streaming-input
|
||||
# session resumes here too, carrying whatever media its
|
||||
# latest chunk added, so this branch needs the same gate.
|
||||
new_computed_blocks = self.kv_cache_manager.empty_kv_cache_blocks
|
||||
num_new_local_computed_tokens = 0
|
||||
num_computed_tokens = request.num_computed_tokens
|
||||
|
||||
if self._ec_transfer_pending(request, num_computed_tokens):
|
||||
request_queue.pop_request()
|
||||
step_skipped_waiting.prepend_request(request)
|
||||
continue
|
||||
|
||||
encoder_inputs_to_schedule = None
|
||||
external_load_encoder_input = []
|
||||
new_encoder_compute_budget = encoder_compute_budget
|
||||
@@ -2162,6 +2174,10 @@ class Scheduler(SchedulerInterface):
|
||||
self.grammar_compile_error_reqs.clear()
|
||||
if failed_kv_load_req_ids and not self.recompute_kv_load_failures:
|
||||
error_req_ids.update(failed_kv_load_req_ids)
|
||||
if self.ec_connector is not None:
|
||||
# An encoder input the connector can no longer obtain. Failing is
|
||||
# retryable: re-issuing the request re-runs the encode.
|
||||
error_req_ids.update(self.ec_connector.take_unavailable_requests())
|
||||
|
||||
if error_req_ids:
|
||||
error_reqs = self.finish_requests(
|
||||
@@ -2258,6 +2274,16 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
return engine_core_outputs
|
||||
|
||||
def _ec_transfer_pending(self, request: Request, num_computed_tokens: int) -> bool:
|
||||
"""Whether an encoder input this request needs is still in transit."""
|
||||
return (
|
||||
self.ec_connector is not None
|
||||
and bool(request.mm_features)
|
||||
and not self.ec_connector.ensure_cache_available(
|
||||
request, num_computed_tokens
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_blocked_waiting_status(status: RequestStatus) -> bool:
|
||||
return status in (
|
||||
@@ -2344,7 +2370,7 @@ class Scheduler(SchedulerInterface):
|
||||
# With Whisper, as soon as we've generated a single token,
|
||||
# we know we're done with the encoder input. Cross Attention
|
||||
# KVs have been calculated and cached already.
|
||||
self.encoder_cache_manager.free_encoder_input(request, input_id)
|
||||
self._free_encoder_input(request, input_id)
|
||||
elif (
|
||||
start_pos + num_tokens + spec_lookahead
|
||||
<= request.num_computed_tokens - request.num_output_placeholders
|
||||
@@ -2352,7 +2378,12 @@ class Scheduler(SchedulerInterface):
|
||||
# Processed, stored in the decoder KV cache, and far enough past
|
||||
# the placeholder range (plus the drafter's look-ahead) that no
|
||||
# rejection or drafter gather can reference it.
|
||||
self.encoder_cache_manager.free_encoder_input(request, input_id)
|
||||
self._free_encoder_input(request, input_id)
|
||||
|
||||
def _free_encoder_input(self, request: Request, input_id: int) -> None:
|
||||
self.encoder_cache_manager.free_encoder_input(request, input_id)
|
||||
if self.ec_connector is not None:
|
||||
self.ec_connector.update_state_after_free(request, input_id)
|
||||
|
||||
def update_draft_token_ids(self, draft_token_ids: DraftTokenIds) -> None:
|
||||
for req_id, spec_token_ids in zip(
|
||||
|
||||
@@ -64,6 +64,12 @@ class ECConnectorModelRunnerMixin:
|
||||
assert scheduler_output.ec_connector_metadata is not None
|
||||
ec_connector.bind_connector_metadata(scheduler_output.ec_connector_metadata)
|
||||
|
||||
if ec_connector.is_producer:
|
||||
# Pass the cache explicitly: a producer needs it to publish an item
|
||||
# that is already cached from an earlier step, which this step will
|
||||
# not re-encode -- `save_caches` never fires for those.
|
||||
ec_connector.start_save_caches(encoder_cache=encoder_cache, **kwargs)
|
||||
|
||||
# Load caches for consumer or both roles
|
||||
if ec_connector.is_consumer:
|
||||
ec_connector.start_load_caches(encoder_cache, **kwargs)
|
||||
@@ -74,5 +80,6 @@ class ECConnectorModelRunnerMixin:
|
||||
output.finished_sending, output.finished_recving = (
|
||||
ec_connector.get_finished(scheduler_output.finished_req_ids)
|
||||
)
|
||||
output.ec_connector_worker_meta = ec_connector.build_connector_worker_meta()
|
||||
|
||||
ec_connector.clear_connector_metadata()
|
||||
|
||||
@@ -62,6 +62,9 @@ class ActiveECConnector(ECConnector):
|
||||
assert scheduler_output.ec_connector_metadata is not None
|
||||
ec_connector.bind_connector_metadata(scheduler_output.ec_connector_metadata)
|
||||
|
||||
if ec_connector.is_producer:
|
||||
ec_connector.start_save_caches(encoder_cache=self.encoder_cache)
|
||||
|
||||
if ec_connector.is_consumer:
|
||||
ec_connector.start_load_caches(self.encoder_cache)
|
||||
|
||||
|
||||
@@ -28,6 +28,8 @@ from vllm.distributed import (
|
||||
from vllm.distributed.ec_transfer import (
|
||||
ensure_ec_transfer_initialized,
|
||||
ensure_ec_transfer_shutdown,
|
||||
get_ec_transfer,
|
||||
has_ec_transfer,
|
||||
)
|
||||
from vllm.distributed.eplb.eplb_utils import override_envs_for_eplb
|
||||
from vllm.distributed.kv_transfer import (
|
||||
@@ -501,6 +503,9 @@ class Worker(WorkerBase):
|
||||
):
|
||||
self.model_runner.load_model(load_dummy_weights=load_dummy_weights)
|
||||
|
||||
if has_ec_transfer():
|
||||
get_ec_transfer().start_worker_services()
|
||||
|
||||
if self.vllm_config.weight_transfer_config is not None:
|
||||
self.weight_transfer_engine = WeightTransferEngineFactory.create_engine(
|
||||
self.vllm_config.weight_transfer_config,
|
||||
|
||||
Reference in New Issue
Block a user