diff --git a/.buildkite/test_areas/disaggregated_mooncake.yaml b/.buildkite/test_areas/disaggregated_mooncake.yaml index 0911507eb2..8c061a95ad 100644 --- a/.buildkite/test_areas/disaggregated_mooncake.yaml +++ b/.buildkite/test_areas/disaggregated_mooncake.yaml @@ -19,3 +19,57 @@ steps: commands: - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh - bash v1/kv_connector/mooncake_integration/config_sweep_accuracy_test.sh + +- label: ":nvidia: (L4) Mooncake EC TCP E2E" + key: mooncake-ec-tcp-e2e-2-gpus + timeout_in_minutes: 30 + working_dir: "/vllm-workspace" + device: l4 + num_devices: 2 + source_file_dependencies: + - vllm/distributed/ec_transfer/ + - vllm/config/ec_transfer.py + - vllm/config/multimodal.py + - vllm/config/vllm.py + - vllm/multimodal/ + - vllm/v1/core/encoder_cache_manager.py + - vllm/v1/core/sched/ + - vllm/v1/engine/ + - vllm/v1/worker/ec_connector_model_runner_mixin.py + - vllm/v1/worker/gpu_model_runner.py + - vllm/v1/worker/gpu_worker.py + - vllm/v1/worker/gpu/ec_connector.py + - vllm/v1/worker/gpu/model_runner.py + - vllm/v1/worker/gpu/mm/ + - examples/disaggregated/disaggregated_encoder/ + - tests/v1/ec_connector/ + - requirements/kv_connectors.txt + env: + PYTHON_BIN: "/vllm-workspace/.venv/bin/python" + PYTHONUNBUFFERED: "1" + VLLM_HOST_IP: "127.0.0.1" + VLLM_USE_V2_MODEL_RUNNER: "1" + MOONCAKE_EC_PROTOCOL: "tcp" + USE_MM_PROMPTS: "1" + SKIP_BASELINE: "0" + CONCURRENCY: "3" + REPEAT: "2" + LOG_PATH: "/tmp/mooncake-ec-e2e" + BASELINE_FILE: "/tmp/mooncake-ec-e2e/baseline.json" + commands: + - uv venv --system-site-packages --python 3.12 .venv + - | + uv pip install --python .venv/bin/python "$(.venv/bin/python - <<'PY' + from pathlib import Path + import torch + + requirement = next( + line for line in Path("requirements/kv_connectors.txt").read_text().splitlines() + if line.startswith("mooncake-transfer-engine ") + ) + if torch.version.cuda.split(".")[0] == "13": + requirement = requirement.replace("mooncake-transfer-engine", "mooncake-transfer-engine-cuda13") + print(requirement) + PY + )" + - bash tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh diff --git a/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py b/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py index cc7704d049..1581e21748 100644 --- a/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py +++ b/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py @@ -23,6 +23,7 @@ import argparse import asyncio import hashlib import io +import itertools import json import logging import os @@ -30,6 +31,7 @@ import random import time import uuid from collections.abc import AsyncIterator +from typing import Any import aiohttp import pybase64 as base64 @@ -41,9 +43,7 @@ from fastapi.responses import JSONResponse, StreamingResponse # FastAPI app & global state ############################################################################### -logging.basicConfig( - level=logging.DEBUG, format="%(asctime)s %(levelname)s: %(message)s" -) +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s: %(message)s") logger = logging.getLogger("proxy") app = FastAPI() @@ -87,9 +87,24 @@ def encoder_rr_assignment( return urls, next_start +def validate_ec_consumer_routing( + prefill_urls: list[str], consumer_addrs: list[str] +) -> None: + """Reject the topology whose EC destination cannot be routed safely.""" + if prefill_urls and consumer_addrs: + raise ValueError( + "Mooncake EC consumer routing supports E+PD only; disable independent " + "prefill or omit --ec-consumer-zmq-addrs." + ) + + # Diagnostic switch: forward the original request to the decoder so the # only difference from the rewrite path is the rewrite itself. NO_REWRITE = False +# Decode-side retries for a retryable internal error (`finish_reason="error"`, +# e.g. an encoder embedding the connector could not deliver). Re-issuing runs +# the encode again, which produces a fresh transfer. +DECODE_RETRIES = 1 # Grid metadata reported by the encoder instance, keyed by item index. @@ -136,6 +151,7 @@ def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict: here -- a second derivation could disagree with the encoder's. """ rewritten = 0 + transfer_items = [] idx = 0 new_messages = [] for msg in req_data.get("messages", []): @@ -151,26 +167,42 @@ def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict: meta = dict(item_meta.get(idx) or {}) idx += 1 item_uuid = meta.pop("mm_hash", None) + ec_mm_hash = meta.pop("ec_mm_hash", None) or item_uuid + transfer_id = meta.pop("transfer_id", None) # Whatever keys the encoder reported are the metadata its model # declared as needed to size the placeholder range; the proxy does # not need to know their names. metadata = {k: _b64_tensor(v) for k, v in meta.items()} if not metadata or not item_uuid: - # The encoder reported no metadata (e.g. the item came from its - # processor cache); let the decoder process the media itself. + # Nothing to size the placeholder range with. A processor cache + # hit is not a cause on its own: with the default `lru` type the + # engine restores the item before the scheduler reports it. It + # goes missing when the encode request failed, or under + # `--mm-processor-cache-type shm`, where a hit replaces the item + # with its shared-memory address and only the worker restores + # it. Send the media so the decoder can derive the grid itself. new_content.append(item) continue embeds_type = EMBEDS_TYPES[item["type"]] new_content.append( {"type": embeds_type, embeds_type: metadata, "uuid": item_uuid} ) + if transfer_id is not None: + transfer_items.append( + {"mm_hash": ec_mm_hash, "transfer_id": transfer_id} + ) rewritten += 1 new_messages.append({**msg, "content": new_content}) if not rewritten: return req_data logger.info("Rewrote %d media item(s) as metadata references", rewritten) - return {**req_data, "messages": new_messages} + rewritten_request = {**req_data, "messages": new_messages} + if transfer_items: + ec_transfer_params = dict(req_data.get("ec_transfer_params") or {}) + ec_transfer_params["ec_items"] = transfer_items + rewritten_request["ec_transfer_params"] = ec_transfer_params + return rewritten_request def extract_mm_items(request_data: dict) -> list[dict]: @@ -196,7 +228,8 @@ async def fanout_encoder_primer( orig_request: dict, e_urls: list[str], req_id: str, -) -> dict[int, dict]: + consumer_zmq: str | None = None, +) -> tuple[dict[int, dict], dict[str, Any]]: """ 1. Build one request *per MM item* with all text removed. 2. Send them concurrently to the encode cluster. @@ -206,19 +239,25 @@ async def fanout_encoder_primer( `ec_transfer_params`: its EC cache key and the grid its processor produced. The proxy still supplies the uuid so both sides key the cache the same way; the grid can only come from the encoder, which is the side that computed it. + + Also returns the connector handles to put on the decode body, as a fresh + mapping. `orig_request` is left untouched so a retry re-encodes from the + original request instead of carrying the previous attempt's handles. """ logger.info("[%s] Processing multimodal items...", req_id) mm_items = extract_mm_items(orig_request) if not mm_items: logger.info("[%s] No multimodal items, skipping encoder", req_id) - return {} # nothing to do + return {}, {} # nothing to do logger.info("[%s] got %d multimodal items...", req_id, len(mm_items)) tasks = [] item_uuids: dict[int, str] = {} + item_transfer_ids: dict[int, str] = {} item_meta: dict[int, dict] = {} + ec_params: dict[str, Any] = {} # Round-robin over encode servers to distribute load a bit. The cursor # persists across requests so fan-out doesn't restart at e_urls[0] every @@ -241,6 +280,8 @@ async def fanout_encoder_primer( item_uuid = None if NO_REWRITE else content_uuid(item) if item_uuid is not None: item_uuids[idx] = item_uuid + transfer_id = uuid.uuid4().hex + item_transfer_ids[idx] = transfer_id encoder_req = { # You *may* need to keep additional fields @@ -257,6 +298,19 @@ async def fanout_encoder_primer( # once the prompt is encoded and its embeddings are published. "stream": False, } + if consumer_zmq is not None: + # No mm_hash here on purpose. The encoder's own + # `mm_features[i].identifier` is derived from the uuid *and* the + # engine's media_io_kwargs / mm_processor_kwargs, so this proxy + # cannot know it before the encoder runs. Sending the bare uuid + # would fail the connector's hash match and make the producer + # invent its own transfer id, which the consumer could then never + # cancel. Omitting it lets the connector match by position, which + # is exact: one encoder request carries exactly one item. + encoder_req["ec_transfer_params"] = { + "consumer_zmq": consumer_zmq, + "ec_items": [{"transfer_id": transfer_id}], + } tasks.append( encode_session.post( f"{target_url}/v1/chat/completions", @@ -304,24 +358,42 @@ async def fanout_encoder_primer( except Exception: logger.warning("[%s] Could not read encoder metadata #%d", req_id, idx) params = {} - if idx in item_uuids: - reported = params.get(item_uuids[idx]) + if params: + # One encoder request carries exactly one item, so there is a + # single reported entry. Do not key it by this proxy's uuid: when + # media_io_kwargs or mm_processor_kwargs are set the engine + # re-hashes the uuid together with them, so the encoder's own + # `mm_features[i].identifier` is a derived value this proxy cannot + # predict. Fall back to the sole entry, and carry the key the + # encoder actually used through as `ec_mm_hash`. + ec_mm_hash = item_uuids.get(idx) + reported = params.get(ec_mm_hash) + if reported is None and len(params) == 1: + ((ec_mm_hash, reported),) = params.items() if reported: metadata = reported.get("metadata") or {} if metadata: - item_meta[idx] = {**metadata, "mm_hash": item_uuids[idx]} + item_meta[idx] = { + **metadata, + "mm_hash": item_uuids.get(idx, ec_mm_hash), + "ec_mm_hash": ec_mm_hash, + } + if idx in item_transfer_ids: + item_meta[idx]["transfer_id"] = item_transfer_ids[idx] # Whatever the encoder reported alongside `metadata` is the # connector's own handle on the published embedding (for NIXL, # peer_host/peer_port/size_bytes). The decoder's connector # looks it up by mm_hash on the request, so carry it through. - orig_request.setdefault("ec_transfer_params", {})[item_uuids[idx]] = ( - reported - ) + ec_params[item_uuids.get(idx, ec_mm_hash)] = reported + if NO_REWRITE and consumer_zmq is not None: + ec_params.setdefault("ec_items", []).append( + {"mm_hash": ec_mm_hash, "transfer_id": item_transfer_ids[idx]} + ) logger.info( "[%s] All %d encoder requests completed successfully", req_id, len(mm_items) ) - return item_meta + return item_meta, ec_params async def maybe_prefill( @@ -331,8 +403,11 @@ async def maybe_prefill( ) -> dict: """ - Do prefill-only task if p_url exist; - - Return modified request data with kv transfer params (for nixl connector) + - Return a new body carrying kv transfer params (for nixl connector) - Else, skip and return the original request data for decode + + `req_data` is never mutated: a decode retry re-enters this function with the + same body, and one attempt's `remote_block_ids` must not reach the next. """ if p_url: logger.info("[%s] Processing through prefill: %s", req_id, p_url) @@ -342,11 +417,9 @@ async def maybe_prefill( prefill_response_json = await prefill_response.json() kv_transfer_params = prefill_response_json.get("kv_transfer_params", {}) if kv_transfer_params: - req_data["kv_transfer_params"] = kv_transfer_params + return {**req_data, "kv_transfer_params": kv_transfer_params} - return req_data - else: - return req_data + return req_data async def process_prefill_stage( @@ -409,7 +482,6 @@ async def process_prefill_stage( ############################################################################### -@app.middleware("http") async def log_requests(request: Request, call_next): """Middleware to log all incoming requests and responses""" req_id = request.headers.get("x-request-id", str(uuid.uuid4())) @@ -458,7 +530,20 @@ async def log_requests(request: Request, call_next): async def on_startup() -> None: global encode_session, prefill_session, decode_session timeout = aiohttp.ClientTimeout(total=100_000) - connector = aiohttp.TCPConnector(limit=0, force_close=False) + # vLLM closes an idle keep-alive connection after + # VLLM_HTTP_TIMEOUT_KEEP_ALIVE seconds (5 by default), while aiohttp keeps + # pooling it for 15. Reusing one it has already closed fails the request + # with ServerDisconnectedError, and the server logs nothing at all: it + # closed the socket before the request arrived. Retire ours first. + server_keep_alive = float(os.getenv("VLLM_HTTP_TIMEOUT_KEEP_ALIVE", "5")) + connector = aiohttp.TCPConnector( + limit=0, + **( + {"keepalive_timeout": server_keep_alive / 2} + if server_keep_alive > 0 + else {"force_close": True} + ), + ) encode_session = aiohttp.ClientSession(timeout=timeout, connector=connector) if app.state.p_urls: # only setup if prefill instance(s) exist @@ -482,40 +567,97 @@ async def on_shutdown() -> None: ############################################################################### +async def prepare_for_decode( + req_data: dict, + req_id: str, + e_urls: list[str], + p_url: str, + consumer_zmq: str | None, +) -> tuple[dict, float, float]: + """Encode, rewrite and prefill, returning the body to send to decode. + + `req_data` is left untouched so a retry starts from the original media + rather than from a body whose images are already metadata references. + """ + _t0 = time.perf_counter() + item_meta, ec_params = await fanout_encoder_primer( + req_data, e_urls, req_id, consumer_zmq + ) + _t1 = time.perf_counter() + prepared = req_data if NO_REWRITE else rewrite_for_decode(req_data, item_meta) + if ec_params: + # A fresh body every time: `rewrite_for_decode` hands back `req_data` + # itself when it rewrote nothing, and this attempt's handles must not + # outlive it into a retry. + handles = dict(prepared.get("ec_transfer_params") or {}) + handles.update(ec_params) + prepared = {**prepared, "ec_transfer_params": handles} + _t2 = time.perf_counter() + prepared = await maybe_prefill(prepared, p_url, req_id) + return prepared, _t1 - _t0, _t2 - _t1 + + async def forward_non_stream( - req_data: dict, req_id: str, e_urls: list[str], p_url: str, d_url: str + req_data: dict, + req_id: str, + e_urls: list[str], + p_url: str, + d_url: str, + consumer_zmq: str | None, + dp_rank: int | None = None, ) -> dict: try: - # Step 1: Process through Encoder instance (if has MM input) - _t0 = time.perf_counter() - item_meta = await fanout_encoder_primer(req_data, e_urls, req_id) - _t1 = time.perf_counter() - req_data = req_data if NO_REWRITE else rewrite_for_decode(req_data, item_meta) - _t2 = time.perf_counter() - - # Step 2: Process through Prefill instance - req_data = await maybe_prefill(req_data, p_url, req_id) - - # Step 3: Process through Decode instance - logger.info("[%s] Forwarding to decode: %s", req_id, d_url) - headers = {"x-request-id": req_id} - - # Non-streaming response - async with decode_session.post( - f"{d_url}/v1/chat/completions", json=req_data, headers=headers - ) as resp: - resp.raise_for_status() - out = await resp.json() - _t3 = time.perf_counter() - logger.info( - "STAGE %s encode=%.1f rewrite=%.1f decode=%.1f total=%.1f", - "no-rewrite" if NO_REWRITE else "rewrite", - (_t1 - _t0) * 1e3, - (_t2 - _t1) * 1e3, - (_t3 - _t2) * 1e3, - (_t3 - _t0) * 1e3, + for attempt in range(DECODE_RETRIES + 1): + _t0 = time.perf_counter() + prepared, encode_s, rewrite_s = await prepare_for_decode( + req_data, req_id, e_urls, p_url, consumer_zmq ) - return out + _t2 = time.perf_counter() + + logger.info("[%s] Forwarding to decode: %s", req_id, d_url) + headers = {"x-request-id": req_id} + if dp_rank is not None: + headers["X-data-parallel-rank"] = str(dp_rank) + + async with decode_session.post( + f"{d_url}/v1/chat/completions", json=prepared, headers=headers + ) as resp: + if resp.status >= 400: + detail = await resp.text() + # 500 is the decoder's retryable internal error, which + # includes an encoder embedding it could not obtain. Redoing + # the encode publishes the item again. + if resp.status == 500 and attempt < DECODE_RETRIES: + logger.warning( + "[%s] Decode returned 500, re-encoding and retrying " + "(attempt %d/%d): %s", + req_id, + attempt + 1, + DECODE_RETRIES, + detail[:200], + ) + continue + logger.error( + "[%s] Decode request returned status %s: %s", + req_id, + resp.status, + detail, + ) + raise HTTPException(status_code=resp.status, detail=detail) + out = await resp.json() + _t3 = time.perf_counter() + logger.info( + "STAGE %s encode=%.1f rewrite=%.1f decode=%.1f total=%.1f " + "attempt=%d", + "no-rewrite" if NO_REWRITE else "rewrite", + encode_s * 1e3, + rewrite_s * 1e3, + (_t3 - _t2) * 1e3, + (_t3 - _t0) * 1e3, + attempt, + ) + return out + raise HTTPException(status_code=500, detail="Decode failed after re-encoding") except HTTPException: raise @@ -525,47 +667,66 @@ async def forward_non_stream( async def forward_stream( - req_data: dict, req_id: str, e_urls: list[str], p_url: str, d_url: str + req_data: dict, + req_id: str, + e_urls: list[str], + p_url: str, + d_url: str, + consumer_zmq: str | None, + dp_rank: int | None = None, ) -> AsyncIterator[str]: try: - # Step 1: Process through Encoder instance (if has MM input) - _t0 = time.perf_counter() - item_meta = await fanout_encoder_primer(req_data, e_urls, req_id) - _t1 = time.perf_counter() - req_data = req_data if NO_REWRITE else rewrite_for_decode(req_data, item_meta) - _t2 = time.perf_counter() + for attempt in range(DECODE_RETRIES + 1): + _t0 = time.perf_counter() + prepared, encode_s, rewrite_s = await prepare_for_decode( + req_data, req_id, e_urls, p_url, consumer_zmq + ) + _t2 = time.perf_counter() - # Step 2: Process through Prefill instance - req_data = await maybe_prefill(req_data, p_url, req_id) + logger.info("[%s] Starting streaming from decode: %s", req_id, d_url) + headers = {"x-request-id": req_id} + if dp_rank is not None: + headers["X-data-parallel-rank"] = str(dp_rank) - # Step 3: Process through Decode instance - logger.info("[%s] Starting streaming from decode: %s", req_id, d_url) - headers = {"x-request-id": req_id} + _first = None + async with decode_session.post( + f"{d_url}/v1/chat/completions", + json=prepared, + headers=headers, + ) as resp: + # Retry only before the first chunk: once anything reached the + # client the response cannot be replaced. + if resp.status == 500 and attempt < DECODE_RETRIES: + detail = await resp.text() + logger.warning( + "[%s] Decode returned 500 before streaming, re-encoding " + "and retrying (attempt %d/%d): %s", + req_id, + attempt + 1, + DECODE_RETRIES, + detail[:200], + ) + continue + resp.raise_for_status() + async for chunk in resp.content.iter_chunked(1024): + if chunk: + if _first is None: + _first = time.perf_counter() + yield chunk.decode("utf-8", errors="ignore") + _t3 = time.perf_counter() - # Streaming response - _first = None - async with decode_session.post( - f"{d_url}/v1/chat/completions", - json=req_data, - headers=headers, - ) as resp: - resp.raise_for_status() - async for chunk in resp.content.iter_chunked(1024): - if chunk: - if _first is None: - _first = time.perf_counter() - yield chunk.decode("utf-8", errors="ignore") - _t3 = time.perf_counter() - - logger.info( - "STAGE %s encode=%.1f rewrite=%.2f decode_ttfb=%.1f decode_total=%.1f", - "no-rewrite" if NO_REWRITE else "rewrite", - (_t1 - _t0) * 1e3, - (_t2 - _t1) * 1e3, - ((_first or _t3) - _t2) * 1e3, - (_t3 - _t2) * 1e3, - ) - logger.info("[%s] Streaming completed", req_id) + logger.info( + "STAGE %s encode=%.1f rewrite=%.2f decode_ttfb=%.1f " + "decode_total=%.1f attempt=%d", + "no-rewrite" if NO_REWRITE else "rewrite", + encode_s * 1e3, + rewrite_s * 1e3, + ((_first or _t3) - _t2) * 1e3, + (_t3 - _t2) * 1e3, + attempt, + ) + logger.info("[%s] Streaming completed", req_id) + return except HTTPException: logger.exception("[%s] HTTPException in forward_stream", req_id) @@ -590,16 +751,29 @@ async def chat_completions(request: Request): e_urls = app.state.e_urls # we want the full list for fan-out p_url = random.choice(app.state.p_urls) if app.state.p_urls else None - d_url = random.choice(app.state.d_urls) + decode_index = random.randrange(len(app.state.d_urls)) + d_url = app.state.d_urls[decode_index] + dp_size = app.state.ec_consumer_dp_size + # Round-robin the replica, then name it to both halves: the decoder + # honours the rank header instead of its own balancer, and the encoder + # pushes to that replica's control channel. Choosing once here means a + # decode retry re-encodes to the same replica. + dp_rank = next(app.state.replica_counter) % dp_size if dp_size > 1 else None + ec_index = decode_index * dp_size + (dp_rank or 0) + consumer_zmq = app.state.d_ec_urls[ec_index] if app.state.d_ec_urls else None is_streaming = req_data.get("stream", False) if is_streaming: return StreamingResponse( - forward_stream(req_data, req_id, e_urls, p_url, d_url), + forward_stream( + req_data, req_id, e_urls, p_url, d_url, consumer_zmq, dp_rank + ), media_type="text/event-stream", ) - result = await forward_non_stream(req_data, req_id, e_urls, p_url, d_url) + result = await forward_non_stream( + req_data, req_id, e_urls, p_url, d_url, consumer_zmq, dp_rank + ) return JSONResponse(content=result) except HTTPException: @@ -750,6 +924,14 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--host", default="0.0.0.0") parser.add_argument("--port", type=int, default=8000) + parser.add_argument( + "--log-requests", + action="store_true", + help=( + "Log every request in and out, and raise the log level to DEBUG. " + "Off by default: the proxy is on the request path." + ), + ) parser.add_argument( "--no-rewrite", action="store_true", @@ -764,8 +946,8 @@ if __name__ == "__main__": "--prefill-servers-urls", required=True, help=( - 'Comma-separated prefill URLs ("http://p1:8003,http://p2:8004") ', - 'to enable E->P->D, set "disable" or "none" to enable E->PD', + 'Comma-separated prefill URLs ("http://p1:8003,http://p2:8004") ' + 'to enable E->P->D, set "disable" or "none" to enable E->PD' ), ) parser.add_argument( @@ -773,15 +955,64 @@ if __name__ == "__main__": required=True, help='Comma-separated decode URLs ("http://d1:8005,http://d2:8006")', ) + parser.add_argument( + "--decode-retries", + type=int, + default=1, + help=( + "Re-encode and re-send when decode returns 500, which is its " + "retryable internal error (an undeliverable encoder embedding " + "among them). 0 disables." + ), + ) + parser.add_argument( + "--ec-consumer-zmq-addrs", + default="", + help=( + "Comma-separated Mooncake EC consumer control addresses, aligned " + "with --decode-servers-urls. Required for Mooncake EC consumers and " + "supported only in E+PD mode. With --ec-consumer-dp-size > 1, list " + "each server's replicas consecutively: s0r0,s0r1,s1r0,s1r1." + ), + ) + parser.add_argument( + "--ec-consumer-dp-size", + type=int, + default=1, + help=( + "Data-parallel replicas per EC consumer. The proxy picks a replica " + "round-robin and names it to both halves of the request, because an " + "encoder push has to land where the request will run." + ), + ) args = parser.parse_args() + if args.log_requests: + logging.getLogger().setLevel(logging.DEBUG) + app.middleware("http")(log_requests) NO_REWRITE = args.no_rewrite + DECODE_RETRIES = max(0, args.decode_retries) app.state.e_urls = [ u.strip() for u in args.encode_servers_urls.split(",") if u.strip() ] app.state.d_urls = [ u.strip() for u in args.decode_servers_urls.split(",") if u.strip() ] + app.state.d_ec_urls = [ + u.strip() for u in args.ec_consumer_zmq_addrs.split(",") if u.strip() + ] + if args.ec_consumer_dp_size < 1: + parser.error("--ec-consumer-dp-size must be at least 1") + app.state.ec_consumer_dp_size = args.ec_consumer_dp_size + app.state.replica_counter = itertools.count() + expected = len(app.state.d_urls) * args.ec_consumer_dp_size + if app.state.d_ec_urls and len(app.state.d_ec_urls) != expected: + parser.error( + "--ec-consumer-zmq-addrs must contain one address per consumer " + f"replica: expected {expected} " + f"({len(app.state.d_urls)} servers x {args.ec_consumer_dp_size} replicas), " + f"got {len(app.state.d_ec_urls)}" + ) # handle prefill instances if args.prefill_servers_urls.lower() in ("disable", "none", ""): app.state.p_urls = [] @@ -793,11 +1024,21 @@ if __name__ == "__main__": u.strip() for u in args.prefill_servers_urls.split(",") if u.strip() ] logger.info("Disaggregated prefill phase is enabled. Running E + P + D...") + try: + validate_ec_consumer_routing(app.state.p_urls, app.state.d_ec_urls) + except ValueError as exc: + parser.error(str(exc)) logger.info("Proxy listening on %s:%s", args.host, args.port) logger.info("Encode servers: %s", app.state.e_urls) logger.info("Prefill instances %s", app.state.p_urls) logger.info("Decode servers: %s", app.state.d_urls) + if app.state.ec_consumer_dp_size > 1: + logger.info( + "EC consumer replicas per server: %d (control addresses: %s)", + app.state.ec_consumer_dp_size, + app.state.d_ec_urls, + ) uvicorn.run( app, diff --git a/tests/v1/core/test_encoder_cache_manager.py b/tests/v1/core/test_encoder_cache_manager.py index 331a31ce66..3948f7c6e3 100644 --- a/tests/v1/core/test_encoder_cache_manager.py +++ b/tests/v1/core/test_encoder_cache_manager.py @@ -167,6 +167,38 @@ def test_get_freed_mm_hashes_clears_freed_list(): assert manager.get_freed_mm_hashes() == [] +def test_referencing_a_freeable_entry_protects_it_from_eviction(): + """A waiting request can hold an entry it is not scheduled for yet. + + Entries with no referent are evictable. A request that is deferred (e.g. + by an EC connector waiting on another item) still needs the items it + already has, and a connector that hands out one transfer per request + cannot always produce the same item a second time. + """ + manager = EncoderCacheManager(cache_size=10) + owner = MockRequest("owner", ["a"], [5]) + waiter = MockRequest("waiter", ["a"], [5]) + newcomer = MockRequest("newcomer", ["b"], [6]) + + manager.allocate(owner, 0) + manager.free_encoder_input(owner, 0) + assert "a" in manager.freeable + + # The waiting request references it; it is no longer reclaimable. + assert manager.check_and_update_cache(waiter, 0) + assert "a" not in manager.freeable + + # 'b' no longer fits, and 'a' must not be taken away to make room. + assert not manager.can_allocate(newcomer, 0, int(1e9), 0) + assert "a" in manager.cached + assert manager.get_freed_mm_hashes() == [] + + # Once the waiter is done with it, the entry is reclaimable again. + manager.free_encoder_input(waiter, 0) + assert manager.can_allocate(newcomer, 0, int(1e9), 0) + assert manager.get_freed_mm_hashes() == ["a"] + + def test_reallocated_hash_is_not_reported_as_freed(): manager = EncoderCacheManager(cache_size=8) req_a = MockRequest("reqA", ["a"], [4]) diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index 1ef63cbb04..7a6d9f7887 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -5643,6 +5643,68 @@ def test_free_encoder_inputs_respects_unconfirmed_placeholders(): assert manager.get_cached_input_ids(request) == set() +def test_free_encoder_inputs_notifies_the_ec_connector(): + """The connector learns the item is consumed, not just that the request ended. + + Connectors hold per-item transfer state (remote buffers, reservations); + waiting for `request_finished` pins it for the whole generation. + """ + scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf") + mm_start_pos, mm_length = 50, 100 + request = create_requests( + num_requests=1, + num_tokens=mm_start_pos + mm_length + 10, + mm_positions=[[PlaceholderRange(offset=mm_start_pos, length=mm_length)]], + )[0] + scheduler.encoder_cache_manager.allocate(request, 0) + scheduler.ec_connector = Mock() + + request.num_computed_tokens = mm_start_pos + mm_length - 1 + scheduler._free_encoder_inputs(request) + scheduler.ec_connector.update_state_after_free.assert_not_called() + + request.num_computed_tokens = mm_start_pos + mm_length + scheduler._free_encoder_inputs(request) + scheduler.ec_connector.update_state_after_free.assert_called_once_with(request, 0) + + +def test_unavailable_encoder_input_fails_the_request_as_retryable(): + """An encoder input the connector gave up on must end the request. + + `FinishReason.ERROR` is the retryable channel the KV connector already uses + for load failures, so the caller can re-issue; deferring instead parked the + request until the client timed out. + """ + scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf") + request = create_requests( + num_requests=1, + num_tokens=160, + mm_positions=[[PlaceholderRange(offset=50, length=100)]], + )[0] + scheduler.add_request(request) + scheduler.ec_connector = Mock() + scheduler.ec_connector.take_unavailable_requests.return_value = {request.request_id} + # `request_finished` reports (delay_free, params) for the request teardown. + scheduler.ec_connector.request_finished.return_value = (False, None) + + scheduler_output = scheduler.schedule() + outputs = scheduler.update_from_output( + scheduler_output, + ModelRunnerOutput( + req_ids=[request.request_id], + req_id_to_index={request.request_id: 0}, + sampled_token_ids=[[]], + logprobs=None, + prompt_logprobs_dict={}, + pooler_output=[], + ), + ) + + assert request.status == RequestStatus.FINISHED_ERROR + engine_outputs = outputs[0].outputs + assert [o.finish_reason for o in engine_outputs] == [FinishReason.ERROR] + + def test_free_encoder_inputs_defers_for_eagle_lookahead(): """With EAGLE speculative decoding, the encoder input is retained one extra position so the drafter's +1 look-ahead mm-embedding gather (which reads one @@ -5840,9 +5902,8 @@ def test_ec_connector_ensure_cache_available_defers_request(use_kv_connector): # ensure_cache_available must have been called with (request, num_computed_tokens=0) # for a brand-new request that has no cached tokens yet. - scheduler.ec_connector.ensure_cache_available.assert_called_once_with( - request_deferred, 0 - ) + ensure_call = scheduler.ec_connector.ensure_cache_available.call_args + assert ensure_call.args == (request_deferred, 0) # Deferred request must NOT be scheduled assert request_deferred.request_id not in output.num_scheduled_tokens _assert_right_encoder_cache_allocated(scheduler, expected_total_allocated=0) @@ -5874,6 +5935,31 @@ def test_ec_connector_ensure_cache_available_defers_request(use_kv_connector): _assert_right_encoder_inputs(output, expected_total_reqs=0) +def test_ec_connector_defers_running_request_for_async_reload(): + scheduler = create_scheduler( + model="llava-hf/llava-1.5-7b-hf", + max_num_batched_tokens=32, + use_ec_connector=True, + ec_role="ec_consumer", + ) + request = create_requests( + num_requests=1, + num_tokens=128, + mm_positions=[[PlaceholderRange(offset=48, length=32)]], + req_ids=["request"], + )[0] + scheduler.ec_connector.ensure_cache_available = Mock(side_effect=[True, False]) + + scheduler.add_request(request) + first_output = scheduler.schedule() + assert first_output.num_scheduled_tokens[request.request_id] == 32 + + second_output = scheduler.schedule() + assert request.request_id not in second_output.num_scheduled_tokens + ensure_call = scheduler.ec_connector.ensure_cache_available.call_args + assert ensure_call.args[:2] == (request, 32) + + def test_ec_connector_pending_prefetch_only_checks_future_mm_features(): """Test that future mm feature filtering only yields features beyond the computed token frontier. diff --git a/tests/v1/ec_connector/integration/README.md b/tests/v1/ec_connector/integration/README.md index a7dab5d5d9..bf2dcd76d6 100644 --- a/tests/v1/ec_connector/integration/README.md +++ b/tests/v1/ec_connector/integration/README.md @@ -1,171 +1,213 @@ -# EPD Correctness Test - -This test verifies that EPD (Encoder-Prefill-Decode) disaggregation produces identical outputs to a baseline single instance. - -## What It Tests - -- **Baseline**: Single vLLM instance serving a multimodal model -- **EPD (1E+1PD)**: 1 Encoder + 1 Prefill-Decode instance -- **Baseline (1P+1D)**: 1 Prefill + 1 Decode instance -- **EPD (1E+1P+1D)**: 1 Encoder + 1 Prefill + 1 Decode instance - -The test ensures that disaggregated encoding produces **identical** outputs to the baseline. - -Note that currently PD disaggregation set up may give slightly different results from a single instance. Therefore, we need the result from 1P+1D as the baseline for 1E+1P+1D - +# EPD Correctness Test + +This test verifies that EPD (Encoder-Prefill-Decode) disaggregation produces identical outputs to a baseline single instance. + +## What It Tests + +- **Baseline**: Single vLLM instance serving a multimodal model +- **EPD (1E+1PD)**: 1 Encoder + 1 Prefill-Decode instance +- **Baseline (1P+1D)**: 1 Prefill + 1 Decode instance +- **EPD (1E+1P+1D)**: 1 Encoder + 1 Prefill + 1 Decode instance + +The test ensures that disaggregated encoding produces **identical** outputs to the baseline. + +Note that currently PD disaggregation set up may give slightly different results from a single instance. Therefore, we need the result from 1P+1D as the baseline for 1E+1P+1D + Please refer to [Disaggregated Encoder Feature](../../../../docs/features/disagg_encoder.md) for the detailed explanation for the EPD features. - -## Files - -- `run_epd_correctness_test.sh` - Main test script (starts all instances and runs tests) -- `test_epd_correctness.py` - Python test script (compares outputs) - -## Usage - -### Multimodal Prompts (Default) - -```bash -cd vllm -./tests/v1/ec_connector/integration/run_epd_correctness_test.sh -``` - -This runs the test with actual multimodal (image) prompts. - -### Text-Only Prompts - -```bash -cd vllm -USE_MM_PROMPTS=0 ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh -``` - -This runs a quick test with text-only prompts to verify the setup works. - -### Custom Configuration - -```bash -# Use specific GPUs -GPU_E=0 GPU_PD=1 GPU_P=1 GPU_D=2 bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh - -# Use specific ports -ENDPOINT_PORT=10001 bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh - -# Use specific model -MODEL="Qwen/Qwen2.5-VL-3B-Instruct" bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh - -# Use specific storage path -EC_SHARED_STORAGE_PATH="/tmp/my_ec_cache" bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh -``` - -## How It Works - -### Step 1: Baseline - -1. Start single vLLM instance on GPU -2. Run test prompts (multimodal or text-only) -3. Save outputs to `.vllm_epd_baseline.txt` -4. Shutdown instance - -### Step 2: EPD (1E + 1PD) - -1. Clear encoder cache storage -2. Start instances and proxy -3. Run same test prompts -4. Assert outputs match baseline exactly -5. Shutdown instances - -### Step 3: EPD (1E + 1P + 1D) - -1. Clear encoder cache storage -2. Start instances and proxy -3. Run same test prompts -4. Assert outputs match baseline exactly -5. Shutdown instances - -## Test Scenarios - -### Multimodal Prompts (--use_mm_prompts) - -Tests encoder cache transfer: - -- Single image query -- Multiple images in one request -- Mixed image and text -- Image with detailed questions - -### Text-Only Prompts (default) - -Quick sanity check: - -- Simple text queries -- Text-only explanations -- Verifies proxy routing works - -## Expected Behavior - -### ✅ Test Passes When - -- All disagg outputs match baseline outputs exactly -- No errors during instance startup -- Encoder cache is properly saved and loaded -- Proxy correctly routes requests - -### ❌ Test Fails When - -- Outputs differ between baseline and disagg -- Server startup fails -- Encoder cache not found (should fall back to local execution) -- Proxy routing errors - -## Notes - -- The test uses deterministic generation (`temperature=0.0`, `seed=42`) -- Encoder cache should enable exact output reproduction -- Test cleans up all instances and cache files after completion -- Safe to run multiple times (idempotent) -- We setup the PD disagg part with NixlConnector. Please read details about EPD in `examples/disaggregated/disaggregated_encoder/README.md` - -## Requirements - -- Multiple GPUs (3 for 1E+1P+1D, 2 for 1E+1PD, 1 for baseline) - - 1E+1P+1D is runnable with 2 GPU by assign E and P on the same GPU now. -- Multimodal model (e.g., Qwen2.5-VL-3B-Instruct) -- Internet access (for accessing vllm test images) - -## Debugging - -### Check Logs - -Logs and baseline output are saved in `/tmp/` by default. -Can be customized by changing the environment variables. - -### Check Encoder Cache - -```bash -# Verify cache files are created -ls -la $EC_SHARED_STORAGE_PATH/ - -# Should see directories with mm_hash names -# Each containing encoder_cache.safetensors -``` - -### Manual Testing - -Run individual components: - -```bash -# Baseline only -python test_epd_correctness.py \ - --service_url http://localhost:8000 \ - --model_name Qwen/Qwen2.5-VL-3B-Instruct \ - --mode baseline \ - --baseline_file test_output.txt \ - --use_mm_prompts - -# Disagg only (requires baseline output file!) -python test_epd_correctness.py \ - --service_url http://localhost:8000 \ - --model_name Qwen/Qwen2.5-VL-3B-Instruct \ - --mode disagg \ - --baseline_file test_output.txt \ - --use_mm_prompts -``` + +## Files + +- `run_epd_correctness_test.sh` - Main test script (starts all instances and runs tests) +- `test_epd_correctness.py` - Python test script (compares outputs) + +## Usage + +### Multimodal Prompts (Default) + +```bash +cd vllm +./tests/v1/ec_connector/integration/run_epd_correctness_test.sh +``` + +This runs the test with actual multimodal (image) prompts. + +### Text-Only Prompts + +```bash +cd vllm +USE_MM_PROMPTS=0 ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh +``` + +This runs a quick test with text-only prompts to verify the setup works. + +### Custom Configuration + +```bash +# Use specific GPUs +GPU_E=0 GPU_PD=1 GPU_P=1 GPU_D=2 bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh + +# Use specific ports +ENDPOINT_PORT=10001 bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh + +# Use specific model +MODEL="Qwen/Qwen2.5-VL-3B-Instruct" bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh + +# Use specific storage path +EC_SHARED_STORAGE_PATH="/tmp/my_ec_cache" bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh +``` + +## How It Works + +### Mooncake EC (1E + 1PD) + +```bash +PYTHON_BIN="$PWD/.venv/bin/python" \ + bash tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh +``` + +Requires two GPUs and Mooncake TransferEngine. TCP is the default transport; +no RDMA-capable network hardware is required. +The script sets `MC_FORCE_TCP=1` in TCP mode so Mooncake cannot auto-select RDMA. +The baseline runs first on GPU 0, followed by E on GPU 0 and PD on GPU 1. +`MOONCAKE_EC_PROTOCOL=rdma` selects RDMA instead; host-specific transport +environment variables should be set by the caller. + +Three black-box cases check fixed short answers and compare with the baseline: + +- One image: read the STOP sign. +- Two different images (including a local file): identify flowers and birds. +- The same image twice: read both STOP signs. + +By default, all three requests run concurrently for two rounds, exercising +shared hashes across requests and reuse after completion. Every response is +compared, not just the final round. Set `CONCURRENCY` and `REPEAT` to override. +Prefix caching is disabled and CUDA graphs remain enabled. This is a small +correctness suite, not a performance benchmark or failure-injection suite. + +`LOG_PATH` and `BASELINE_FILE` select the log directory and reference output. +`SKIP_BASELINE=1` reuses a reference generated with the same model and test +configuration. `USE_MM_PROMPTS=0` only checks text routing, not EC transfer. + +Buildkite runs this script as `mooncake-ec-tcp-e2e-2-gpus` on two L4 GPUs. +The job is defined in `.buildkite/test_areas/disaggregated_mooncake.yaml` and +selected for changes to EC, multimodal processing, scheduler/model-runner +integration, the proxy, or these tests (subject to normal PR CI approval). +It reuses the CI image's Python packages through a system-site-packages venv +and installs the CUDA-compatible Mooncake wheel. It uses loopback networking +and TCP only, without RDMA devices or peer-memory setup. + +The job fails on startup errors, request errors, or answer mismatches. On +failure, the script prints the last 100 lines of each server/proxy log to +the CI job log; full files remain under `LOG_PATH` while the container exists. + +### Step 1: Baseline + +1. Start single vLLM instance on GPU +2. Run test prompts (multimodal or text-only) +3. Save outputs to `.vllm_epd_baseline.txt` +4. Shutdown instance + +### Step 2: EPD (1E + 1PD) + +1. Clear encoder cache storage +2. Start instances and proxy +3. Run same test prompts +4. Assert outputs match baseline exactly +5. Shutdown instances + +### Step 3: EPD (1E + 1P + 1D) + +1. Clear encoder cache storage +2. Start instances and proxy +3. Run same test prompts +4. Assert outputs match baseline exactly +5. Shutdown instances + +## Test Scenarios + +### Multimodal Prompts (--use_mm_prompts) + +Tests encoder cache transfer: + +- Single image query +- Multiple images in one request +- Mixed image and text +- Image with detailed questions + +### Text-Only Prompts (default) + +Quick sanity check: + +- Simple text queries +- Text-only explanations +- Verifies proxy routing works + +## Expected Behavior + +### ✅ Test Passes When + +- All disagg outputs match baseline outputs exactly +- No errors during instance startup +- Encoder cache is properly saved and loaded +- Proxy correctly routes requests + +### ❌ Test Fails When + +- Outputs differ between baseline and disagg +- Server startup fails +- Encoder cache not found (should fall back to local execution) +- Proxy routing errors + +## Notes + +- The test uses deterministic generation (`temperature=0.0`, `seed=42`) +- Encoder cache should enable exact output reproduction +- Test cleans up all instances and cache files after completion +- Safe to run multiple times (idempotent) +- We setup the PD disagg part with NixlConnector. Please read details about EPD in `examples/disaggregated/disaggregated_encoder/README.md` + +## Requirements + +- Multiple GPUs (3 for 1E+1P+1D, 2 for 1E+1PD, 1 for baseline) + - 1E+1P+1D is runnable with 2 GPU by assign E and P on the same GPU now. +- Multimodal model (e.g., Qwen2.5-VL-3B-Instruct) +- Internet access (for accessing vllm test images) + +## Debugging + +### Check Logs + +Logs and baseline output are saved in `/tmp/` by default. +Can be customized by changing the environment variables. + +### Check Encoder Cache + +```bash +# Verify cache files are created +ls -la $EC_SHARED_STORAGE_PATH/ + +# Should see directories with mm_hash names +# Each containing encoder_cache.safetensors +``` + +### Manual Testing + +Run individual components: + +```bash +# Baseline only +python test_epd_correctness.py \ + --service_url http://localhost:8000 \ + --model_name Qwen/Qwen2.5-VL-3B-Instruct \ + --mode baseline \ + --baseline_file test_output.txt \ + --use_mm_prompts + +# Disagg only (requires baseline output file!) +python test_epd_correctness.py \ + --service_url http://localhost:8000 \ + --model_name Qwen/Qwen2.5-VL-3B-Instruct \ + --mode disagg \ + --baseline_file test_output.txt \ + --use_mm_prompts +``` diff --git a/tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh b/tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh new file mode 100755 index 0000000000..05af1bb6fd --- /dev/null +++ b/tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh @@ -0,0 +1,289 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# +# Full-stack EPD validation with ECMooncakeConnector (Mooncake TransferEngine): +# 1) Single-GPU baseline (multimodal) -> saves baseline JSON +# 2) 1 Encoder + 1 PD with Mooncake EC -> compare outputs to baseline +# +# Usage (from repo root): +# ./tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh +# +# Env: +# MODEL HF model id (default: Qwen/Qwen2.5-VL-3B-Instruct) +# GPU_SINGLE / GPU_E / GPU_PD GPU ids (defaults 0 / 0 / 1) +# ENDPOINT_PORT, ENCODE_PORT, PREFILL_DECODE_PORT +# EC_MOONCAKE_RESERVATION_HOST consumer control host (default 127.0.0.1) +# MOONCAKE_EC_PROTOCOL tcp | rdma (default tcp) +# USE_MM_PROMPTS 1 (default) or 0 for text-only quick sanity +# TIMEOUT_SECONDS wait_for_server timeout (default 1200) +# SKIP_BASELINE set to 1 to reuse existing BASELINE_FILE +# CONCURRENCY / REPEAT concurrent requests / rounds (defaults 3 / 2) +# MAX_MODEL_LEN context length (default 16384) + +set -euo pipefail + +GIT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd) +cd "$GIT_ROOT" || exit 1 +export PYTHONPATH="${GIT_ROOT}:${PYTHONPATH:-}" +PYTHON_BIN="${PYTHON_BIN:-${GIT_ROOT}/.venv/bin/python}" + +MODEL="${MODEL:-Qwen/Qwen2.5-VL-3B-Instruct}" +USE_MM_PROMPTS="${USE_MM_PROMPTS:-1}" +TEST_ARGS=(--concurrency "${CONCURRENCY:-3}" --repeat "${REPEAT:-2}") +if [[ "$USE_MM_PROMPTS" == "1" ]]; then + TEST_ARGS+=(--use_mm_prompts --mm_smoke_test) +fi +MAX_MODEL_LEN="${MAX_MODEL_LEN:-16384}" + +GPU_SINGLE="${GPU_SINGLE:-0}" +GPU_E="${GPU_E:-0}" +GPU_PD="${GPU_PD:-1}" + +ENCODE_PORT="${ENCODE_PORT:-19534}" +PREFILL_DECODE_PORT="${PREFILL_DECODE_PORT:-19537}" +ENDPOINT_PORT="${ENDPOINT_PORT:-10002}" +BASELINE_PORT="${BASELINE_PORT:-10003}" + +EC_MOONCAKE_RESERVATION_HOST="${EC_MOONCAKE_RESERVATION_HOST:-127.0.0.1}" +EC_MOONCAKE_RESERVATION_PORT="${EC_MOONCAKE_RESERVATION_PORT:-19019}" +MOONCAKE_EC_PROTOCOL="${MOONCAKE_EC_PROTOCOL:-tcp}" +export EC_MOONCAKE_RESERVATION_HOST +export EC_MOONCAKE_RESERVATION_PORT +export MOONCAKE_EC_PROTOCOL +if [[ "$MOONCAKE_EC_PROTOCOL" == "tcp" ]]; then + # TransferEngine may otherwise auto-select RDMA on hosts with an HCA. + export MC_FORCE_TCP=1 +else + unset MC_FORCE_TCP +fi + +LOG_PATH="${LOG_PATH:-/tmp}" +BASELINE_FILE="${BASELINE_FILE:-/tmp/vllm_epd_mooncake_baseline.txt}" +TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-1200}" +PIDS=() + +mkdir -p "$LOG_PATH" + +VLLM_SERVE=("$PYTHON_BIN" -m vllm.entrypoints.cli.main serve) + +ENC_EC_JSON=$("$PYTHON_BIN" </dev/null || return 1 + if curl --max-time 2 -fsS "http://localhost:${port}/health" >/dev/null 2>&1; then + return 0 + fi + sleep 2 + done + return 1 +} + +cleanup_instances() { + if ((${#PIDS[@]} == 0)); then + return + fi + echo "Cleaning up tracked vLLM / proxy processes..." + for pid in "${PIDS[@]}"; do + kill "$pid" 2>/dev/null || true + done + for _ in {1..10}; do + local alive=0 + for pid in "${PIDS[@]}"; do + if kill -0 "$pid" 2>/dev/null; then + alive=1 + fi + done + ((alive == 0)) && break + sleep 1 + done + for pid in "${PIDS[@]}"; do + if kill -0 "$pid" 2>/dev/null; then + kill -KILL "$pid" 2>/dev/null || true + fi + wait "$pid" 2>/dev/null || true + done + PIDS=() +} + +finish() { + local status=$? + cleanup_instances + if ((status != 0)); then + for log in "${LOG_PATH}"/mooncake_epd_*.log; do + [[ -f "$log" ]] || continue + echo "=== $log (last 100 lines) ===" + tail -100 "$log" + done + fi + exit "$status" +} + +trap finish EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +run_baseline() { + echo "================================" + echo "BASELINE (single vLLM, MM if enabled)" + echo "================================" + cleanup_instances + local PORT=$BASELINE_PORT + echo "Starting baseline on GPU $GPU_SINGLE port $PORT" + CUDA_VISIBLE_DEVICES="$GPU_SINGLE" "${VLLM_SERVE[@]}" "$MODEL" \ + --port "$PORT" \ + --gpu-memory-utilization 0.75 \ + --max-num-seqs 32 \ + --max-model-len "$MAX_MODEL_LEN" \ + --no-enable-prefix-caching \ + --limit-mm-per-prompt '{"image":2,"video":0}' \ + --allowed-local-media-path "${GIT_ROOT}/tests/v1/ec_connector/integration" \ + >"${LOG_PATH}/mooncake_epd_baseline.log" 2>&1 & + local BASELINE_PID=$! + PIDS+=("$BASELINE_PID") + echo "Waiting for baseline..." + wait_for_server "$PORT" "$BASELINE_PID" || { echo "Baseline failed to start"; return 1; } + curl -s "http://127.0.0.1:${PORT}/v1/models" | head -c 200 || true + echo "" + "$PYTHON_BIN" "${GIT_ROOT}/tests/v1/ec_connector/integration/test_epd_correctness.py" \ + --service_url "http://localhost:$PORT" \ + --model_name "$MODEL" \ + --mode baseline \ + --baseline_file "$BASELINE_FILE" \ + "${TEST_ARGS[@]}" + cleanup_instances +} + +run_epd_mooncake() { + echo "================================" + echo "EPD 1E + 1PD with ECMooncakeConnector" + echo "Reservation port on consumer: $EC_MOONCAKE_RESERVATION_PORT" + echo "Mooncake protocol: $MOONCAKE_EC_PROTOCOL" + echo "================================" + cleanup_instances + + echo "Starting ENCODER on GPU $GPU_E port $ENCODE_PORT" + CUDA_VISIBLE_DEVICES="$GPU_E" "${VLLM_SERVE[@]}" "$MODEL" \ + --port "$ENCODE_PORT" \ + --gpu-memory-utilization 0.35 \ + --mm-tensor-ipc torch_shm \ + --enable-request-id-headers \ + --no-enable-prefix-caching \ + --max-num-batched-tokens "$MAX_MODEL_LEN" \ + --max-num-seqs 32 \ + --max-model-len "$MAX_MODEL_LEN" \ + --limit-mm-per-prompt '{"image":2,"video":0}' \ + --allowed-local-media-path "${GIT_ROOT}/tests/v1/ec_connector/integration" \ + --ec-transfer-config "$ENC_EC_JSON" \ + >"${LOG_PATH}/mooncake_epd_encoder.log" 2>&1 & + local ENCODER_PID=$! + PIDS+=("$ENCODER_PID") + + echo "Starting PD on GPU $GPU_PD port $PREFILL_DECODE_PORT" + CUDA_VISIBLE_DEVICES="$GPU_PD" "${VLLM_SERVE[@]}" "$MODEL" \ + --port "$PREFILL_DECODE_PORT" \ + --gpu-memory-utilization 0.75 \ + --mm-tensor-ipc torch_shm \ + --enable-mm-embeds \ + --enable-request-id-headers \ + --max-num-seqs 32 \ + --max-model-len "$MAX_MODEL_LEN" \ + --no-enable-prefix-caching \ + --limit-mm-per-prompt '{"image":2,"video":0}' \ + --allowed-local-media-path "${GIT_ROOT}/tests/v1/ec_connector/integration" \ + --ec-transfer-config "$PD_EC_JSON" \ + >"${LOG_PATH}/mooncake_epd_pd.log" 2>&1 & + local PD_PID=$! + PIDS+=("$PD_PID") + + echo "Waiting for encoder..." + wait_for_server "$ENCODE_PORT" "$ENCODER_PID" || { echo "Encoder failed to start"; return 1; } + echo "Waiting for PD..." + wait_for_server "$PREFILL_DECODE_PORT" "$PD_PID" || { echo "PD failed to start"; return 1; } + + echo "Starting EPD proxy on $ENDPOINT_PORT" + "$PYTHON_BIN" "${GIT_ROOT}/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py" \ + --host "0.0.0.0" \ + --port "$ENDPOINT_PORT" \ + --encode-servers-urls "http://localhost:$ENCODE_PORT" \ + --prefill-servers-urls "disable" \ + --decode-servers-urls "http://localhost:$PREFILL_DECODE_PORT" \ + --ec-consumer-zmq-addrs \ + "$EC_MOONCAKE_RESERVATION_ADDR" \ + >"${LOG_PATH}/mooncake_epd_proxy.log" 2>&1 & + local PROXY_PID=$! + PIDS+=("$PROXY_PID") + + echo "Waiting for proxy..." + wait_for_server "$ENDPOINT_PORT" "$PROXY_PID" || { echo "Proxy failed to start"; return 1; } + curl -s "http://127.0.0.1:${ENDPOINT_PORT}/health" || true + echo "" + + "$PYTHON_BIN" "${GIT_ROOT}/tests/v1/ec_connector/integration/test_epd_correctness.py" \ + --service_url "http://localhost:$ENDPOINT_PORT" \ + --model_name "$MODEL" \ + --mode disagg \ + --baseline_file "$BASELINE_FILE" \ + "${TEST_ARGS[@]}" + + cleanup_instances +} + +echo "================================" +echo "1E + 1PD ECMooncake end-to-end correctness" +echo "MODEL=$MODEL" +echo "================================" + +if [[ "${SKIP_BASELINE:-0}" != "1" ]]; then + run_baseline +else + echo "SKIP_BASELINE=1 -> using existing $BASELINE_FILE" + [[ -f "$BASELINE_FILE" ]] || { echo "Missing baseline file"; exit 1; } +fi + +run_epd_mooncake + +echo "================================" +echo "PASS: Mooncake EC EPD matches baseline" +echo "Logs: ${LOG_PATH}/mooncake_epd_*.log" +echo "================================" diff --git a/tests/v1/ec_connector/integration/test_epd_correctness.py b/tests/v1/ec_connector/integration/test_epd_correctness.py index ece73efa42..d3ae0f1d87 100644 --- a/tests/v1/ec_connector/integration/test_epd_correctness.py +++ b/tests/v1/ec_connector/integration/test_epd_correctness.py @@ -26,6 +26,7 @@ import argparse import json import os import time +from concurrent.futures import ThreadPoolExecutor import openai import requests @@ -138,17 +139,20 @@ def run_chat_completion( Returns: Generated text content """ - client = openai.OpenAI(api_key="EMPTY", base_url=base_url) + with openai.OpenAI( + api_key="EMPTY", base_url=base_url, timeout=120, max_retries=0 + ) as client: + completion = client.chat.completions.create( + model=model_name, + messages=messages, + max_tokens=max_tokens, + temperature=0.0, + seed=42, + ) - completion = client.chat.completions.create( - model=model_name, - messages=messages, - max_tokens=max_tokens, - temperature=0.0, - seed=42, - ) - - return completion.choices[0].message.content + content = completion.choices[0].message.content + assert content, "Expected a nonempty completion" + return content def main(): @@ -198,7 +202,19 @@ def main(): help="Skip the two-image multimodal prompt", ) + parser.add_argument( + "--mm_smoke_test", + action="store_true", + help="Use three short, fixed-answer image cases, including duplicate images", + ) + parser.add_argument("--concurrency", type=int, default=1) + parser.add_argument("--repeat", type=int, default=1) + args = parser.parse_args() + if args.concurrency < 1 or args.repeat < 1: + parser.error("--concurrency and --repeat must be positive") + if args.mm_smoke_test and (not args.use_mm_prompts or args.skip_two_image_prompt): + parser.error("--mm_smoke_test requires all multimodal prompts") print(f"Service URL: {args.service_url}") print(f"Model: {args.model_name}") @@ -235,27 +251,76 @@ def main(): test_prompts = SAMPLE_PROMPTS_TEXT print("Using text-only prompts for quick testing") + if args.mm_smoke_test: + image = SAMPLE_PROMPTS_MM[0]["messages"][0]["content"][0] + image_pair = SAMPLE_PROMPTS_MM[1]["messages"][0]["content"][:2] + cases = [ + ( + "Single image", + [image], + "What word is on the red road sign? Reply with only that word " + "in uppercase.", + "STOP", + ), + ( + "Two different images", + image_pair, + "Do these pictures contain both flowers and birds? " + "Reply with only YES or NO.", + "YES", + ), + ( + "Same image twice", + [image, image], + "Read the red road sign in each image, in image order. Reply " + "with only the two uppercase words separated by a comma and a space.", + "STOP, STOP", + ), + ] + test_prompts = [ + { + "description": description, + "expected": expected, + "messages": [ + { + "role": "user", + "content": [*images, {"type": "text", "text": question}], + } + ], + } + for description, images, question, expected in cases + ] + # Run completions service_url = f"{args.service_url}/v1" output_strs = {} - for i, prompt_data in enumerate(test_prompts): - print( - f"\nRunning prompt {i + 1}/{len(test_prompts)}: " - f"{prompt_data['description']}" - ) - - output_str = run_chat_completion( + def complete(prompt_data): + output = run_chat_completion( base_url=service_url, model_name=args.model_name, messages=prompt_data["messages"], - max_tokens=MAX_OUTPUT_LEN, + max_tokens=16 if args.mm_smoke_test else MAX_OUTPUT_LEN, ) + if args.mm_smoke_test: + output = output.strip() + assert output == prompt_data["expected"], ( + f"{prompt_data['description']}: expected {prompt_data['expected']!r}, " + f"got {output!r}" + ) + return output - # Use description as key for comparison - key = prompt_data["description"] - output_strs[key] = output_str - print(f"Output: {output_str}") + # Each round includes concurrent requests sharing image hashes; later rounds + # exercise reuse after previous requests have finished. + with ThreadPoolExecutor(max_workers=args.concurrency) as executor: + for repeat in range(args.repeat): + outputs = executor.map(complete, test_prompts) + for prompt_data, output_str in zip(test_prompts, outputs): + key = prompt_data["description"] + if args.repeat > 1: + key = f"{key} (round {repeat + 1})" + output_strs[key] = output_str + print(f"{key}: {output_str}") if args.mode in ("baseline", "baseline_pd"): # Baseline mode: Save outputs diff --git a/tests/v1/ec_connector/unit/test_epd_proxy_retry.py b/tests/v1/ec_connector/unit/test_epd_proxy_retry.py new file mode 100644 index 0000000000..b026b5537a --- /dev/null +++ b/tests/v1/ec_connector/unit/test_epd_proxy_retry.py @@ -0,0 +1,172 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Body handling across the EPD proxy's decode retries. + +Exercises the REAL helpers loaded from the ``examples/`` proxy, so a future +change to them is what these tests catch. +""" + +import asyncio +import importlib.util +from pathlib import Path + +import pytest + +PROXY_REL = "examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py" + + +@pytest.fixture(scope="module") +def proxy(): + path = Path(__file__).parents[4] / PROXY_REL + spec = importlib.util.spec_from_file_location("disagg_epd_proxy_retry", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class _Response: + def __init__(self, params): + self._params = params + + async def json(self): + return {"kv_transfer_params": self._params} + + +def test_maybe_prefill_leaves_the_caller_body_untouched(proxy, monkeypatch): + """A decode retry re-enters this function with the body it was given. + + Mutating that body in place let one attempt's `remote_block_ids` survive + into the next, so a retry whose prefill returns nothing sent decode blocks + the prefiller may already have freed. + """ + served = [{"remote_block_ids": [1, 2]}, {}] + + async def _stage(req_data, p_url, req_id): + assert "kv_transfer_params" not in req_data + return _Response(served.pop(0)) + + monkeypatch.setattr(proxy, "process_prefill_stage", _stage) + + body = {"messages": [], "stream": False} + first = asyncio.run(proxy.maybe_prefill(body, "http://prefill", "r1")) + assert first["kv_transfer_params"] == {"remote_block_ids": [1, 2]} + assert "kv_transfer_params" not in body + + second = asyncio.run(proxy.maybe_prefill(body, "http://prefill", "r1")) + assert "kv_transfer_params" not in second + + +class _EncoderResponse: + def __init__(self, params): + self.status = 200 + self._params = params + + async def json(self): + return {"ec_transfer_params": self._params} + + async def text(self): + return "" + + +class _EncoderSession: + """Serve one canned encoder reply per attempt.""" + + def __init__(self, replies): + self._replies = list(replies) + + async def post(self, url, json=None, headers=None): + return _EncoderResponse(self._replies.pop(0)) + + +def test_a_decode_retry_does_not_inherit_the_previous_handles(proxy, monkeypatch): + """The retry loop re-enters `prepare_for_decode` with the same body. + + Recording the encoder's connector handles on that body in place let + attempt 1's handle survive into attempt 2, so a second encode that + reported nothing still sent decode a handle on an embedding the encoder + no longer publishes -- the exact state the retry exists to leave behind. + """ + handle = {"metadata": {"image_grid_thw": [1, 2, 2]}, "peer_port": 1234} + monkeypatch.setattr( + proxy, + "encode_session", + _EncoderSession([{"encoder-side-hash": handle}, {}]), + ) + + async def _no_prefill(req_data, p_url, req_id): + return req_data + + monkeypatch.setattr(proxy, "maybe_prefill", _no_prefill) + + body = { + "messages": [ + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": "image"}}], + } + ], + "stream": False, + } + args = ("r1", ["http://encoder"], "http://prefill", None) + + first, _, _ = asyncio.run(proxy.prepare_for_decode(body, *args)) + reported = first["ec_transfer_params"] + assert [handle] == [value for key, value in reported.items() if key != "ec_items"] + assert "ec_transfer_params" not in body + + # Attempt 2's encode reports nothing: decode must be told nothing. + second, _, _ = asyncio.run(proxy.prepare_for_decode(body, *args)) + assert "ec_transfer_params" not in second + + +def test_raw_media_keeps_encoder_transfer_identity(proxy, monkeypatch): + handle = {"metadata": {"image_grid_thw": [1, 2, 2]}} + monkeypatch.setattr(proxy, "NO_REWRITE", True) + monkeypatch.setattr( + proxy, "encode_session", _EncoderSession([{"encoded-hash": handle}]) + ) + body = { + "messages": [ + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": "image"}}], + } + ] + } + prepared, _, _ = asyncio.run( + proxy.prepare_for_decode( + body, "request", ["http://encoder"], "", "tcp://consumer:1" + ) + ) + assert prepared["messages"] == body["messages"] + assert "ec_transfer_params" not in body + params = prepared["ec_transfer_params"] + assert params["encoded-hash"] == handle + assert params["ec_items"][0]["mm_hash"] == "encoded-hash" + assert params["ec_items"][0]["transfer_id"] + + +@pytest.mark.parametrize("server_keep_alive", ["0.1", "1", "5", "2", "30"]) +def test_pooled_connections_are_retired_before_the_server_closes_them( + proxy, monkeypatch, server_keep_alive +): + """The proxy must not hand a request a connection the server has dropped. + + vLLM closes idle keep-alive connections at `VLLM_HTTP_TIMEOUT_KEEP_ALIVE` + seconds while aiohttp pools them for 15, so a slow hop leaves a dead + connection in the pool. The next request fails with + ServerDisconnectedError and the server logs nothing, because it closed the + socket before the request arrived. + """ + monkeypatch.setenv("VLLM_HTTP_TIMEOUT_KEEP_ALIVE", server_keep_alive) + monkeypatch.setattr(proxy.app.state, "p_urls", [], raising=False) + + asyncio.run(proxy.on_startup()) + try: + # No public accessor for the pool's idle timeout. + pooled_for = proxy.encode_session.connector._keepalive_timeout + assert pooled_for < float(server_keep_alive) + assert proxy.decode_session.connector._keepalive_timeout == pooled_for + finally: + asyncio.run(proxy.on_shutdown()) diff --git a/tests/v1/ec_connector/unit/test_epd_proxy_round_robin.py b/tests/v1/ec_connector/unit/test_epd_proxy_round_robin.py index 48123235a7..c7278ad55b 100644 --- a/tests/v1/ec_connector/unit/test_epd_proxy_round_robin.py +++ b/tests/v1/ec_connector/unit/test_epd_proxy_round_robin.py @@ -32,8 +32,13 @@ def _load_proxy_module(): @pytest.fixture(scope="module") -def assign(): - return _load_proxy_module().encoder_rr_assignment +def proxy(): + return _load_proxy_module() + + +@pytest.fixture(scope="module") +def assign(proxy): + return proxy.encoder_rr_assignment def _drive(assign, e_urls, counts): @@ -80,3 +85,44 @@ def test_single_encoder_always_resolves_to_it(assign): urls, next_cursor = assign(["E0"], 0, 4) assert urls == ["E0"] * 4 assert next_cursor == 0 + + +def test_generic_e_p_d_routing_without_mooncake_is_supported(proxy): + proxy.validate_ec_consumer_routing(["http://prefill"], []) + + +def test_mooncake_e_pd_routing_is_supported(proxy): + proxy.validate_ec_consumer_routing([], ["tcp://decode:19019"]) + + +def test_mooncake_independent_prefill_routing_fails_fast(proxy): + with pytest.raises(ValueError, match=r"supports E\+PD only"): + proxy.validate_ec_consumer_routing(["http://prefill"], ["tcp://decode:19019"]) + + +def test_decode_rewrite_preserves_engine_reported_ec_hash(proxy): + request = { + "messages": [ + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": "image"}}], + } + ] + } + + rewritten = proxy.rewrite_for_decode( + request, + { + 0: { + "mm_hash": "proxy-uuid", + "ec_mm_hash": "engine-derived-hash", + "transfer_id": "transfer", + "image_grid_thw": [1, 2, 3], + } + }, + ) + + assert rewritten["messages"][0]["content"][0]["uuid"] == "proxy-uuid" + assert rewritten["ec_transfer_params"]["ec_items"] == [ + {"mm_hash": "engine-derived-hash", "transfer_id": "transfer"} + ] diff --git a/tests/v1/ec_connector/unit/test_worker_ec_connector.py b/tests/v1/ec_connector/unit/test_worker_ec_connector.py index 3dcad1e50a..01dad9d89e 100644 --- a/tests/v1/ec_connector/unit/test_worker_ec_connector.py +++ b/tests/v1/ec_connector/unit/test_worker_ec_connector.py @@ -12,6 +12,9 @@ from vllm.distributed.ec_transfer.ec_connector.base import ( ECConnectorMetadata, ) from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT +from vllm.v1.worker.ec_connector_model_runner_mixin import ( + ECConnectorModelRunnerMixin, +) from vllm.v1.worker.gpu.ec_connector import NO_OP_EC_CONNECTOR, ActiveECConnector pytestmark = pytest.mark.cpu_test @@ -52,6 +55,9 @@ def test_saves_newly_added_caches_for_every_producer(is_producer, is_consumer): saved = [call.kwargs["mm_hash"] for call in fake.save_caches.call_args_list] assert saved == (["mm_new"] if is_producer else []) + assert fake.start_save_caches.called == is_producer + if is_producer: + assert fake.start_save_caches.call_args.kwargs["encoder_cache"] is encoder_cache assert fake.start_load_caches.called == is_consumer @@ -66,6 +72,26 @@ def test_worker_meta_is_reported_on_context_exit(): assert fake.clear_connector_metadata.called +def test_v1_producer_receives_existing_encoder_cache(): + encoder_cache = {"mm_cached": None} + fake = MagicMock(spec=ECConnectorBase) + fake.is_producer = True + fake.is_consumer = False + fake.get_finished.return_value = (None, None) + + module = "vllm.v1.worker.ec_connector_model_runner_mixin" + with ( + patch(f"{module}.has_ec_transfer", return_value=True), + patch(f"{module}.get_ec_transfer", return_value=fake), + ECConnectorModelRunnerMixin.maybe_get_ec_connector_output( + _scheduler_output(), encoder_cache + ), + ): + pass + + assert fake.start_save_caches.call_args.kwargs["encoder_cache"] is encoder_cache + + def test_no_forward_reports_without_running_the_model(): connector, _ = _connector() diff --git a/vllm/config/ec_transfer.py b/vllm/config/ec_transfer.py index c64bf13ec0..3948f32059 100644 --- a/vllm/config/ec_transfer.py +++ b/vllm/config/ec_transfer.py @@ -18,6 +18,11 @@ class ECTransferConfig: ec_connector: str | None = None """The EC connector for vLLM to transmit EC caches between vLLM instances. + + Built-in options include ``ECExampleConnector`` (shared filesystem via + safetensors) and ``ECMooncakeConnector`` (Mooncake TransferEngine RDMA; + requires ``mooncake-transfer-engine`` and matching producer/consumer + ``ec_connector_extra_config``; see ``mooncake_ec_connector`` module docstring). """ engine_id: str | None = None diff --git a/vllm/distributed/ec_transfer/ec_connector/base.py b/vllm/distributed/ec_transfer/ec_connector/base.py index 9138203bf4..ea44fb6f5b 100644 --- a/vllm/distributed/ec_transfer/ec_connector/base.py +++ b/vllm/distributed/ec_transfer/ec_connector/base.py @@ -261,6 +261,25 @@ class ECConnectorBase(ABC): """ return True + def start_save_caches(self, **kwargs: Any) -> None: + """Prepare this step's outbound pushes before the model runs.""" + return + + def start_worker_services(self) -> None: + """Start Worker-side services once the model is resident.""" + return + + def take_unavailable_requests(self) -> set[str]: + """Request IDs whose encoder inputs the connector can no longer obtain. + + The Scheduler fails these; re-issuing the request re-runs the encode. + """ + return set() + + def update_state_after_free(self, request: "Request", index: int): + """Notify the connector that an encoder cache entry was released.""" + return + @abstractmethod def update_state_after_alloc(self, request: "Request", index: int): """ diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/scheduler/__init__.py b/vllm/distributed/ec_transfer/ec_connector/cpu/scheduler/__init__.py index bc52e1fb62..cd65740108 100644 --- a/vllm/distributed/ec_transfer/ec_connector/cpu/scheduler/__init__.py +++ b/vllm/distributed/ec_transfer/ec_connector/cpu/scheduler/__init__.py @@ -7,6 +7,7 @@ Owns the mmap region and the embedding cache, and handles the producer for the ECCPUConnector. """ +from collections.abc import Collection from typing import TYPE_CHECKING, Any import torch @@ -184,7 +185,10 @@ class ECCPUScheduler: return entry is not None and entry.ready def ensure_cache_available( - self, request: "Request", num_computed_tokens: int + self, + request: "Request", + num_computed_tokens: int, + local_cache_hashes: Collection[str] | None = None, ) -> bool: if not self._nixl_enabled: return True # CPU offload never blocks. diff --git a/vllm/distributed/ec_transfer/ec_connector/factory.py b/vllm/distributed/ec_transfer/ec_connector/factory.py index 598b1b22e4..1c33af5513 100644 --- a/vllm/distributed/ec_transfer/ec_connector/factory.py +++ b/vllm/distributed/ec_transfer/ec_connector/factory.py @@ -89,3 +89,9 @@ ECConnectorFactory.register_connector( "vllm.distributed.ec_transfer.ec_connector.cpu.connector", "ECCPUConnector", ) + +ECConnectorFactory.register_connector( + "ECMooncakeConnector", + "vllm.distributed.ec_transfer.ec_connector.mooncake_ec_connector", + "ECMooncakeConnector", +) diff --git a/vllm/distributed/ec_transfer/ec_connector/mooncake/__init__.py b/vllm/distributed/ec_transfer/ec_connector/mooncake/__init__.py new file mode 100644 index 0000000000..5cd38b5017 --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/mooncake/__init__.py @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Internal building blocks for the Mooncake encoder-cache connector. + +The public connector remains in :mod:`mooncake_ec_connector`. This package +separates configuration, control-plane messaging, transfer state, registered +memory, and worker/scheduler orchestration so that each component has one +owner and can be tested independently. +""" diff --git a/vllm/distributed/ec_transfer/ec_connector/mooncake/config.py b/vllm/distributed/ec_transfer/ec_connector/mooncake/config.py new file mode 100644 index 0000000000..10c9acae46 --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/mooncake/config.py @@ -0,0 +1,123 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from vllm.utils.network_utils import make_zmq_path + +if TYPE_CHECKING: + from vllm.config import VllmConfig + +# Scheduler and Worker must agree on when a remote reservation becomes stale. +_RESERVATION_TTL_SECONDS = 300 + + +def _positive_int(name: str, value: object) -> int: + message = f"ECMooncakeConnector requires {name} to be a positive integer." + if isinstance(value, bool): + raise ValueError(message) + if isinstance(value, int): + result = value + elif isinstance(value, float) and math.isfinite(value) and value.is_integer(): + result = int(value) + elif isinstance(value, str): + try: + result = int(value) + except ValueError as error: + raise ValueError(message) from error + else: + raise ValueError(message) + if result <= 0: + raise ValueError(f"ECMooncakeConnector requires {name} > 0.") + return result + + +def _positive_float(name: str, value: object) -> float: + message = f"ECMooncakeConnector requires {name} > 0." + if isinstance(value, bool): + raise ValueError(message) + try: + result = float(value) # type: ignore[arg-type] + except (TypeError, ValueError) as error: + raise ValueError(message) from error + if not math.isfinite(result) or result <= 0: + raise ValueError(message) + return result + + +@dataclass(frozen=True) +class MooncakeECConfig: + """Validated settings shared by the Scheduler and Worker roles. + + ``control_port`` is the first TP-shard port after the DP offset; + ``control_addr`` targets that shard, which advertises the full topology. + ``control_host`` is the address this instance both advertises and binds, + so reaching a Consumer means being routed to it rather than finding it on + every interface. + """ + + is_producer: bool + is_consumer: bool + protocol: str + buffer_device: str + control_host: str + control_port: int + control_addr: str + control_timeout_ms: int + push_wait_timeout_s: float + pool_size: int + + @classmethod + def from_vllm_config(cls, vllm_config: VllmConfig) -> MooncakeECConfig: + parallel_config = vllm_config.parallel_config + ec_config = vllm_config.ec_transfer_config + assert ec_config is not None + + if ec_config.is_ec_producer: + if parallel_config.tensor_parallel_size > 1: + raise ValueError( + "ECMooncakeConnector producers require tensor_parallel_size=1." + ) + if parallel_config.pipeline_parallel_size > 1: + raise ValueError( + "ECMooncakeConnector producers do not support pipeline parallelism." + ) + if parallel_config.data_parallel_size > 1: + raise ValueError( + "ECMooncakeConnector producers require data_parallel_size=1." + ) + + registered_buffer_size = _positive_int( + "ec_buffer_size", ec_config.ec_buffer_size + ) + get = ec_config.get_from_extra_config + control_port = int(ec_config.ec_port) + ( + parallel_config.data_parallel_index * parallel_config.tensor_parallel_size + ) + highest_port = control_port + parallel_config.tensor_parallel_size - 1 + if not 1 <= control_port <= highest_port <= 65535: + raise ValueError("ECMooncakeConnector ec_port must be in 1..65535.") + + return cls( + is_producer=ec_config.is_ec_producer, + is_consumer=ec_config.is_ec_consumer, + protocol=str(get("mooncake_protocol", "rdma")), + buffer_device=str(ec_config.ec_buffer_device or "cuda").lower(), + control_host=str(ec_config.ec_ip), + control_port=control_port, + control_addr=make_zmq_path("tcp", ec_config.ec_ip, control_port), + control_timeout_ms=max( + 1, + math.ceil( + _positive_float("control_timeout_s", get("control_timeout_s", 30)) + * 1000 + ), + ), + push_wait_timeout_s=_positive_float( + "push_wait_timeout_s", get("push_wait_timeout_s", 60) + ), + pool_size=registered_buffer_size, + ) diff --git a/vllm/distributed/ec_transfer/ec_connector/mooncake/control.py b/vllm/distributed/ec_transfer/ec_connector/mooncake/control.py new file mode 100644 index 0000000000..2f4a0ad605 --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/mooncake/control.py @@ -0,0 +1,460 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""ZMQ control plane for Mooncake encoder-cache reservations and events. + +Tensor bytes never travel through this module. It coordinates destination +reservations, completion/cancellation, TP-shard discovery, and readiness +notifications while :mod:`transfer` owns the Mooncake data plane. +""" + +from __future__ import annotations + +import threading +import time +from collections import deque +from collections.abc import Callable +from concurrent.futures import Executor, Future +from typing import Any + +import torch +import zmq + +from vllm.logger import init_logger +from vllm.utils.network_utils import make_zmq_path + +logger = init_logger(__name__) + +_MAX_PENDING_EVENTS = 4096 +_RESERVATION_REAP_INTERVAL_SECONDS = 1 + + +ControlRequest = dict[str, Any] +ControlResponse = dict[str, Any] + + +class ControlClient: + """Send control requests through reusable, thread-local REQ sockets.""" + + def __init__(self, timeout_ms: int) -> None: + self._context = zmq.Context() + self._timeout_ms = timeout_ms + self._local = threading.local() + self._topologies: dict[str, list[str]] = {} + self._closed = False + + def _sockets(self) -> dict[str, zmq.Socket]: + sockets = getattr(self._local, "sockets", None) + if sockets is None: + sockets = {} + self._local.sockets = sockets + return sockets + + def _discard(self, addr: str) -> None: + socket = self._sockets().pop(addr, None) + if socket is not None: + socket.close(linger=0) + + def _exchange(self, addr: str, payload: ControlRequest) -> ControlResponse: + sockets = self._sockets() + socket = sockets.get(addr) + if socket is None: + socket = self._context.socket(zmq.REQ) + socket.setsockopt(zmq.IPV6, 1) + socket.setsockopt(zmq.RCVTIMEO, self._timeout_ms) + socket.setsockopt(zmq.SNDTIMEO, self._timeout_ms) + socket.setsockopt(zmq.LINGER, 0) + socket.connect(addr) + sockets[addr] = socket + try: + socket.send_json(payload) + response = socket.recv_json() + except Exception: + self._discard(addr) + raise + assert isinstance(response, dict) + return response + + def request(self, addr: str, payload: ControlRequest) -> Any: + response = self._exchange(addr, payload) + if not response.get("ok"): + raise RuntimeError(response.get("error", "EC control request failed")) + return response.get("result") + + def discover_shards(self, base_addr: str) -> list[str] | None: + if base_addr in self._topologies: + return self._topologies[base_addr] + try: + reply = self.request(base_addr, {"op": "peers"}) + ports = reply.get("ports") if isinstance(reply, dict) else None + if not isinstance(ports, list) or not ports: + raise ValueError("invalid or empty peer list") + prefix = base_addr.rsplit(":", 1)[0] + shards = [f"{prefix}:{int(port)}" for port in ports] + except Exception: + logger.warning( + "EC Mooncake consumer at %s did not report its shards", + base_addr, + exc_info=True, + ) + return None + self._topologies[base_addr] = shards + return shards + + def close(self) -> None: + if self._closed: + return + self._closed = True + self._context.destroy(linger=0) + + +def make_cancel_request( + transfer_id: str, + reservation_id: str = "", + *, + abandon: bool = False, + refresh: bool = False, +) -> ControlRequest: + request: ControlRequest = { + "op": "cancel", + "transfer_id": transfer_id, + "reservation_id": reservation_id, + } + if abandon: + request["abandon"] = True + if refresh: + request["refresh"] = True + return request + + +class EventInbox: + """Receive Consumer readiness events without blocking the Scheduler. + + Subscribing needs one blocking control request per shard, so it runs on + `executor` when one is given: an unreachable Consumer must cost the + Scheduler nothing, not one control timeout per drain. + """ + + def __init__(self, client: ControlClient, executor: Executor | None = None) -> None: + self._client = client + self._executor = executor + self._discovery: Future[list[str] | None] | None = None + self._context: zmq.Context | None = None + self._socket: zmq.Socket | None = None + self._closed = False + self.shard_count = 1 + + def _endpoints(self, base_addr: str) -> list[str] | None: + """Ask every shard where it publishes readiness events. + + Blocking: called on `self._executor` unless there is none. + """ + shards = self._client.discover_shards(base_addr) + if shards is None: + return None + endpoints = [] + for addr in shards: + try: + event_port = self._client.request(addr, {"op": "event_port"}) + address, _ = addr.rsplit(":", 1) + endpoints.append(f"{address}:{int(event_port)}") + except Exception: + logger.warning( + "EC Mooncake could not subscribe to the event channel of " + "consumer shard %s; retrying the complete topology later.", + addr, + ) + return None + return endpoints + + def _connect(self, base_addr: str) -> None: + if self._socket is not None or self._closed: + return + if self._executor is None: + self._install(self._endpoints(base_addr)) + return + if self._discovery is None: + self._discovery = self._executor.submit(self._endpoints, base_addr) + return + if not self._discovery.done(): + return + discovery, self._discovery = self._discovery, None + try: + self._install(discovery.result()) + except Exception: + logger.warning( + "EC Mooncake event-channel discovery for %s failed; retrying.", + base_addr, + exc_info=True, + ) + + def _install(self, endpoints: list[str] | None) -> None: + """Adopt a discovered topology. + + Runs on the caller's thread so the socket is only ever touched there. + """ + if not endpoints or self._closed: + return + context = zmq.Context() + socket = context.socket(zmq.PULL) + socket.setsockopt(zmq.IPV6, 1) + for endpoint in endpoints: + socket.connect(endpoint) + self._context = context + self._socket = socket + self.shard_count = len(endpoints) + + def drain(self, base_addr: str) -> list[dict[str, Any]]: + self._connect(base_addr) + if self._socket is None: + return [] + events = [] + while True: + try: + events.append(self._socket.recv_json(flags=zmq.DONTWAIT)) + except zmq.Again: + return events + except zmq.ZMQError: + logger.warning("EC Mooncake event channel failed", exc_info=True) + return events + except (ValueError, UnicodeError): + # The event channel is a plain PULL socket: an undecodable + # frame must cost one frame, not the engine. + logger.warning( + "Discarding an undecodable EC Mooncake event.", exc_info=True + ) + + def close(self) -> None: + if self._closed: + return + self._closed = True + if self._discovery is not None: + self._discovery.cancel() + self._discovery = None + if self._socket is not None: + self._socket.close(linger=0) + self._socket = None + if self._context is not None: + self._context.term() + + +class ConsumerControlServer: + """Expose Consumer reservations and readiness events over ZMQ. + + One server runs on every receiving TP rank. The REP channel handles + reservation operations, while a PUSH channel publishes newly ready items + to the Scheduler. + """ + + def __init__( + self, + host: str, + port: int, + reserve: Callable[[dict[str, Any]], dict[str, Any]], + status: Callable[[str], dict[str, Any] | None], + complete: Callable[[str, str], tuple[bool, bool]], + cancel: Callable[[str, str, bool, bool], bool], + reap: Callable[[], int], + peer_ports: list[int] | None = None, + device: torch.device | None = None, + drain_ready: Callable[[], list[str]] = lambda: [], + ) -> None: + self.host = host + self.port = port + self.peer_ports = peer_ports or [port] + self._device = device + self.event_port: int | None = None + self._reserve = reserve + self._status = status + self._complete = complete + self._cancel = cancel + self._reap = reap + self._drain_ready = drain_ready + self._stop = threading.Event() + self._started = threading.Event() + self._thread: threading.Thread | None = None + self._startup_error: Exception | None = None + + def start(self) -> None: + def loop() -> None: + if self._device is not None and self._device.type == "cuda": + torch.accelerator.set_device_index(self._device.index or 0) + context = zmq.Context() + socket = context.socket(zmq.REP) + event_socket = context.socket(zmq.PUSH) + socket.setsockopt(zmq.IPV6, 1) + event_socket.setsockopt(zmq.IPV6, 1) + pending_events: deque[dict[str, Any]] = deque() + + def queue_event(event: dict[str, Any]) -> None: + event["shard"] = self.port + if len(pending_events) >= _MAX_PENDING_EVENTS: + dropped = pending_events.popleft() + logger.warning( + "EC Mooncake event backlog full on port %d; dropping " + "readiness for transfer_id=%s", + self.port, + dropped.get("transfer_id"), + ) + pending_events.append(event) + + def queue_ready(transfer_id: str) -> None: + status = self._status(transfer_id) + if status is not None: + queue_event({"transfer_id": transfer_id, **status}) + + last_reap_at = time.monotonic() + socket.setsockopt(zmq.RCVTIMEO, 100) + try: + socket.bind(make_zmq_path("tcp", self.host, self.port)) + event_socket.bind(make_zmq_path("tcp", self.host, 0)) + self.event_port = int( + event_socket.getsockopt(zmq.LAST_ENDPOINT) + .decode() + .rsplit(":", 1)[1] + ) + except Exception as e: + self._startup_error = e + self._started.set() + socket.close(linger=0) + event_socket.close(linger=0) + context.term() + return + self._started.set() + try: + while not self._stop.is_set(): + while pending_events: + try: + event_socket.send_json( + pending_events[0], flags=zmq.DONTWAIT + ) + except zmq.Again: + break + pending_events.popleft() + now = time.monotonic() + if now - last_reap_at >= _RESERVATION_REAP_INTERVAL_SECONDS: + self._reap() + last_reap_at = now + try: + request = socket.recv_json() + except zmq.Again: + continue + except Exception: + # The frame arrived but did not decode. REP still owes a + # reply, so answer before returning to the loop. + logger.exception( + "EC Mooncake control channel on port %d received an " + "undecodable request", + self.port, + ) + socket.send_json( + {"ok": False, "error": "malformed control request"} + ) + continue + try: + op = request.get("op") + result: Any = None + if op in ("reserve", "reserve_batch"): + items = ( + request["items"] if op == "reserve_batch" else [request] + ) + results = [] + for item in items: + try: + reserved = self._reserve(item) + results.append({"ok": True, "result": reserved}) + if reserved.get("ready"): + queue_ready(str(item["transfer_id"])) + except Exception as exc: + results.append({"ok": False, "error": str(exc)}) + if op == "reserve_batch": + result = {"items": results} + elif not results[0]["ok"]: + raise RuntimeError(results[0]["error"]) + else: + result = results[0]["result"] + elif op == "status": + result = self._status(str(request["transfer_id"])) + elif op == "event_port": + result = self.event_port + elif op == "peers": + result = {"ports": self.peer_ports} + elif op in ("complete", "complete_batch"): + items = ( + request["items"] + if op == "complete_batch" + else [request] + ) + completions = [] + for item in items: + transfer_id = str(item["transfer_id"]) + accepted, became_ready = self._complete( + transfer_id, str(item["reservation_id"]) + ) + for ready_id in self._drain_ready(): + queue_ready(ready_id) + completions.append( + { + "completed": accepted, + "became_ready": became_ready, + } + ) + if not became_ready: + continue + queue_ready(transfer_id) + result = ( + {"items": completions} + if op == "complete_batch" + else completions[0] + ) + elif op == "cancel": + result = { + "cancelled": self._cancel( + str(request["transfer_id"]), + str(request.get("reservation_id", "")), + bool(request.get("abandon", False)), + bool(request.get("refresh", False)), + ) + } + else: + raise ValueError(f"unknown control op: {op!r}") + socket.send_json({"ok": True, "result": result}) + except Exception as e: + socket.send_json({"ok": False, "error": str(e)}) + except Exception: + # `finally` closes the sockets and the thread ends, so without + # this every later reserve against this shard would surface + # only as a control timeout with nothing to attribute it to. + logger.exception( + "EC Mooncake control channel on port %d stopped serving", + self.port, + ) + raise + finally: + socket.close(linger=0) + event_socket.close(linger=0) + context.term() + + self._thread = threading.Thread( + target=loop, name="ec-mooncake-control", daemon=True + ) + self._thread.start() + if not self._started.wait(timeout=5): + raise RuntimeError("EC Mooncake control channel failed to start") + if self._startup_error is not None: + raise RuntimeError("EC Mooncake control channel failed to bind") from ( + self._startup_error + ) + logger.info( + "EC Mooncake control channel listening on tcp://%s:%d (events tcp://%s:%d)", + self.host, + self.port, + self.host, + self.event_port, + ) + + def close(self) -> None: + if self._thread is None: + return + self._stop.set() + self._thread.join() + self._thread = None diff --git a/vllm/distributed/ec_transfer/ec_connector/mooncake/memory.py b/vllm/distributed/ec_transfer/ec_connector/mooncake/memory.py new file mode 100644 index 0000000000..c6bc1c6851 --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/mooncake/memory.py @@ -0,0 +1,504 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Registered-memory allocation and residency for Mooncake transfers. + +The Producer pool stages source tensors in a registered slab. The Consumer +pool owns destination allocations from reservation through publication, +resident reuse, CUDA-safe retirement, and pressure-driven reclamation. +""" + +from __future__ import annotations + +import bisect +import threading +from collections import OrderedDict +from collections.abc import Callable +from dataclasses import dataclass +from typing import Generic, TypeVar + +import torch + +from vllm.distributed.ec_transfer.ec_connector.mooncake.transfer import ( + MooncakeTransfer, +) +from vllm.logger import init_logger + +logger = init_logger(__name__) + +_T = TypeVar("_T") + + +@dataclass +class MemoryAllocation: + """Describe a tensor view carved from the Consumer receive slab.""" + + offset: int + size: int + tensor: torch.Tensor + + +@dataclass +class _ResidentEntry(Generic[_T]): + """Store one resident value and its ownership accounting.""" + + value: _T + pinned: bool = True + leases: int = 0 + + +@dataclass +class ResidentLease(Generic[_T]): + """Represent a borrow of one resident entry.""" + + key: str + _entry: _ResidentEntry[_T] + _active: bool = True + + @property + def value(self) -> _T: + return self._entry.value + + +class ContiguousAllocator: + """Own a registered slab and its aligned regions; callers serialize access.""" + + def __init__(self, capacity: int, alignment: int = 256): + self.alignment = alignment + self._capacity = capacity + self._free = [(0, capacity)] + self.tensor: torch.Tensor | None = None + self._disabled = False + + def prepare(self, device: torch.device, transfer: MooncakeTransfer) -> None: + if self.tensor is not None or self._disabled: + return + try: + tensor = torch.empty(self._capacity, dtype=torch.uint8, device=device) + ret = transfer.register_memory(tensor) + if ret != 0: + raise RuntimeError(f"Mooncake returned {ret}") + except (RuntimeError, torch.OutOfMemoryError) as error: + self._disabled = True + logger.warning("Could not initialize the EC registered buffer: %s", error) + return + self.tensor = tensor + self._free = [(0, tensor.nbytes)] + logger.info("Registered %d-byte buffer for Mooncake EC", tensor.nbytes) + + def view( + self, offset: int, nbytes: int, shape: tuple[int, ...], dtype: torch.dtype + ) -> torch.Tensor: + assert self.tensor is not None + return self.tensor.narrow(0, offset, nbytes).view(dtype).view(shape) + + def close(self, transfer: MooncakeTransfer) -> bool: + if self.tensor is not None and not transfer.unregister_memory(self.tensor): + return False + self.tensor = None + self._free.clear() + return True + + def allocate(self, nbytes: int) -> tuple[int, int] | None: + size = (nbytes + self.alignment - 1) // self.alignment * self.alignment + for index, (offset, available) in enumerate(self._free): + if size > available: + continue + if size == available: + self._free.pop(index) + else: + self._free[index] = (offset + size, available - size) + return offset, size + return None + + def free(self, offset: int, size: int) -> None: + index = bisect.bisect_left(self._free, (offset, size)) + self._free.insert(index, (offset, size)) + if index + 1 < len(self._free): + next_offset, next_size = self._free[index + 1] + if offset + size == next_offset: + self._free[index] = (offset, size + next_size) + self._free.pop(index + 1) + if index > 0: + previous_offset, previous_size = self._free[index - 1] + current_offset, current_size = self._free[index] + if previous_offset + previous_size == current_offset: + self._free[index - 1] = ( + previous_offset, + previous_size + current_size, + ) + self._free.pop(index) + + +class ResidentPool(Generic[_T]): + """Track resident values, active leases, and LRU eviction eligibility.""" + + def __init__(self): + self._entries: dict[str, _ResidentEntry[_T]] = {} + self._evictable: OrderedDict[str, None] = OrderedDict() + + def referenced(self) -> list[str]: + return [key for key, entry in self._entries.items() if entry.pinned] + + def get(self, key: str) -> _T | None: + entry = self._entries.get(key) + return entry.value if entry is not None else None + + def insert( + self, + key: str, + value: _T, + ) -> _T | None: + """Pin an entry and return a displaced value that has no owners.""" + previous = self._entries.get(key) + entry = _ResidentEntry(value) + self._entries[key] = entry + self._evictable.pop(key, None) + if previous is not None and previous.leases == 0: + return previous.value + return None + + def pin(self, key: str) -> _T | None: + entry = self._entries.get(key) + if entry is None: + return None + self._evictable.pop(key, None) + entry.pinned = True + return entry.value + + def acquire(self, key: str) -> ResidentLease[_T] | None: + entry = self._entries.get(key) + if entry is None: + return None + entry.leases += 1 + self._evictable.pop(key, None) + return ResidentLease(key, entry) + + def release(self, lease: ResidentLease[_T]) -> _T | None: + if not lease._active: + return None + lease._active = False + entry = lease._entry + entry.leases -= 1 + current = self._entries.get(lease.key) + if current is not entry: + if entry.leases == 0: + return entry.value + return None + if not entry.pinned and entry.leases == 0: + self._evictable[lease.key] = None + return None + + def consume(self, lease: ResidentLease[_T]) -> tuple[_T, _T | None]: + current = self._entries[lease.key] + current.pinned = True + self._evictable.pop(lease.key, None) + released = self.release(lease) + return current.value, released + + def retire(self, key: str) -> None: + if key not in self._entries: + return + entry = self._entries[key] + entry.pinned = False + if entry.leases == 0: + self._evictable[key] = None + + def evict_lru(self, evict: Callable[[str, _T], bool]) -> str | None: + for key in list(self._evictable): + entry = self._entries[key] + if not evict(key, entry.value): + continue + self._evictable.pop(key, None) + del self._entries[key] + return key + return None + + def clear(self) -> None: + self._entries.clear() + self._evictable.clear() + + +@dataclass +class StagedSources: + """Own Producer tensor views and their staging-slab regions.""" + + tensors: list[torch.Tensor] + regions: list[tuple[int, int]] + + +class ProducerMemoryPool: + """Own the Producer staging slab and regions carved from it.""" + + def __init__(self, capacity: int, transfer: MooncakeTransfer) -> None: + self._transfer = transfer + self._allocator = ContiguousAllocator(capacity) + self._lock = threading.Lock() + self._local = threading.local() + + @property + def tensor(self) -> torch.Tensor | None: + return self._allocator.tensor + + def _free_regions(self, regions: list[tuple[int, int]]) -> None: + for offset, size in regions: + self._allocator.free(offset, size) + + def stage(self, tensors: list[torch.Tensor]) -> StagedSources | None: + """Copy tensors into one registered slab, or return None for fallback.""" + if not tensors: + return StagedSources([], []) + allocator = self._allocator + staged: list[torch.Tensor] = [] + regions: list[tuple[int, int]] = [] + with self._lock: + allocator.prepare(tensors[0].device, self._transfer) + pool = allocator.tensor + if pool is None: + return None + for tensor in tensors: + region = allocator.allocate(tensor.nbytes) + if region is None: + self._free_regions(regions) + return None + regions.append(region) + staged.append( + allocator.view( + region[0], tensor.nbytes, tuple(tensor.shape), tensor.dtype + ) + ) + if pool.device.type == "cuda": + stream = getattr(self._local, "stream", None) + if stream is None: + stream = torch.cuda.Stream(device=pool.device) + self._local.stream = stream + with torch.cuda.stream(stream): + for destination, source in zip(staged, tensors): + destination.copy_(source, non_blocking=True) + stream.synchronize() + else: + for destination, source in zip(staged, tensors): + destination.copy_(source) + return StagedSources(staged, regions) + + def release(self, staged: StagedSources) -> None: + if not staged.regions: + return + with self._lock: + self._free_regions(staged.regions) + + def close(self) -> None: + with self._lock: + self._allocator.close(self._transfer) + + +class ConsumerMemoryPool: + """Own the registered receive slab and resident allocation lifecycle.""" + + def __init__( + self, + capacity: int, + transfer: MooncakeTransfer, + ) -> None: + self._transfer = transfer + self._allocator = ContiguousAllocator(capacity) + self._residents: ResidentPool[MemoryAllocation] = ResidentPool() + self._retire_events: dict[str, torch.Event] = {} + self._pending_frees: list[tuple[torch.Event, MemoryAllocation]] = [] + self._reclaimed: set[str] = set() + self.lock = threading.RLock() + + @property + def tensor(self) -> torch.Tensor | None: + return self._allocator.tensor + + def prepare( + self, + device: torch.device, + ) -> None: + with self.lock: + self._allocator.prepare(device, self._transfer) + + def _free(self, allocation: MemoryAllocation) -> None: + self._allocator.free(allocation.offset, allocation.size) + + def free(self, allocation: MemoryAllocation) -> None: + with self.lock: + self._free(allocation) + + def _defer_or_free( + self, allocation: MemoryAllocation, event: torch.Event | None + ) -> None: + if event is None or event.query(): + self._free(allocation) + else: + self._pending_frees.append((event, allocation)) + + def _poll_frees_locked(self) -> None: + pending = [] + for event, allocation in self._pending_frees: + if event.query(): + self._free(allocation) + else: + pending.append((event, allocation)) + self._pending_frees = pending + + def _reclaim_locked(self, nbytes: int) -> tuple[int, int] | None: + def evict(mm_hash: str, allocation: MemoryAllocation) -> bool: + event = self._retire_events.pop(mm_hash, None) + self._defer_or_free(allocation, event) + self._reclaimed.add(mm_hash) + return True + + while self._residents.evict_lru(evict) is not None: + # Eviction defers any free whose CUDA event is still pending, so + # those bytes reach the allocator only once the event is polled. + self._poll_frees_locked() + region = self._allocator.allocate(nbytes) + if region is not None: + return region + self._poll_frees_locked() + return self._allocator.allocate(nbytes) + + def _make_allocation( + self, + region: tuple[int, int], + nbytes: int, + shape: tuple[int, ...], + dtype: torch.dtype, + ) -> MemoryAllocation: + offset, size = region + tensor = self._allocator.view(offset, nbytes, shape, dtype) + return MemoryAllocation(offset, size, tensor) + + def try_allocate( + self, nbytes: int, shape: tuple[int, ...], dtype: torch.dtype + ) -> MemoryAllocation | None: + with self.lock: + allocator = self._allocator + assert allocator.tensor is not None + self._poll_frees_locked() + region = allocator.allocate(nbytes) + if region is None: + return None + return self._make_allocation(region, nbytes, shape, dtype) + + def reclaim_and_allocate( + self, nbytes: int, shape: tuple[int, ...], dtype: torch.dtype + ) -> MemoryAllocation | None: + with self.lock: + region = self._reclaim_locked(nbytes) + if region is None: + return None + return self._make_allocation(region, nbytes, shape, dtype) + + def acquire_cached( + self, mm_hash: str, shape: tuple[int, ...], dtype: torch.dtype + ) -> ResidentLease[MemoryAllocation] | None: + with self.lock: + allocation = self._residents.get(mm_hash) + if allocation is None: + return None + if ( + tuple(allocation.tensor.shape) != shape + or allocation.tensor.dtype != dtype + ): + raise ValueError("conflicting cached tensor for mm_hash") + return self._residents.acquire(mm_hash) + + def take_resident( + self, mm_hash: str, shape: tuple[int, ...], dtype_name: str + ) -> torch.Tensor | None: + with self.lock: + allocation = self._residents.get(mm_hash) + if allocation is None: + return None + tensor = allocation.tensor + if ( + tuple(tensor.shape) != shape + or str(tensor.dtype).split(".")[-1] != dtype_name + ): + return None + self._residents.pin(mm_hash) + self._retire_events.pop(mm_hash, None) + return tensor + + def _record_release_event(self) -> torch.Event | None: + pool = self.tensor + if pool is None or pool.device.type != "cuda": + return None + event = torch.Event() + event.record(torch.accelerator.current_stream(pool.device)) + return event + + def release_cached(self, lease: ResidentLease[MemoryAllocation]) -> None: + with self.lock: + released = self._residents.release(lease) + if released is not None: + self._defer_or_free(released, self._record_release_event()) + + def publish( + self, + mm_hash: str, + allocation: MemoryAllocation, + lease: ResidentLease[MemoryAllocation] | None = None, + *, + pin: bool = True, + ) -> MemoryAllocation: + with self.lock: + if lease is not None: + canonical, released = self._residents.consume(lease) + self._retire_events.pop(mm_hash, None) + if released is not None: + self._defer_or_free(released, self._record_release_event()) + return canonical + previous = self._residents.get(mm_hash) + displaced = self._residents.insert(mm_hash, allocation) + if not pin: + self._residents.retire(mm_hash) + event = None + if previous is not None and previous is not allocation: + event = self._retire_events.pop(mm_hash, None) + if displaced is None or displaced is allocation: + return allocation + if event is None: + event = self._record_release_event() + self._defer_or_free(displaced, event) + return allocation + + def retire_stale( + self, + encoder_cache: dict[str, torch.Tensor], + reserved_hashes: set[str] | None = None, + *, + freed: list[str] | None = None, + ) -> None: + if self.tensor is None: + return + with self.lock: + for mm_hash in self._residents.referenced() if freed is None else freed: + allocation = self._residents.get(mm_hash) + if allocation is None: + continue + if encoder_cache.get(mm_hash) is allocation.tensor: + continue + if reserved_hashes and mm_hash in reserved_hashes: + continue + event = self._record_release_event() + if event is not None: + self._retire_events[mm_hash] = event + self._residents.retire(mm_hash) + self._poll_frees_locked() + + def drain_reclaimed(self) -> set[str]: + with self.lock: + reclaimed = self._reclaimed + self._reclaimed = set() + return reclaimed + + def close(self) -> None: + with self.lock: + if not self._allocator.close(self._transfer): + return + self._residents.clear() + self._retire_events.clear() + self._pending_frees.clear() diff --git a/vllm/distributed/ec_transfer/ec_connector/mooncake/metadata.py b/vllm/distributed/ec_transfer/ec_connector/mooncake/metadata.py new file mode 100644 index 0000000000..a006d1bf2f --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/mooncake/metadata.py @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Metadata exchanged by the Mooncake encoder-cache connector.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from vllm.distributed.ec_transfer.ec_connector.base import ( + ECConnectorMetadata, + ECConnectorWorkerMetadata, +) + + +@dataclass +class ECMooncakeLoadSpec: + """Describe a remote reservation or resident allocation to load.""" + + mm_hash: str + nbytes: int + shape: tuple[int, ...] + dtype: str + transfer_id: str + # The consumer pool still holds this item, so the load is a local handoff: + # no transfer, no producer. + local: bool = False + + +@dataclass +class ECMooncakePushSpec: + """Describe a destination reservation prepared before a tensor is ready.""" + + mm_hash: str + nbytes: int + shape: tuple[int, ...] + dtype: str + consumer_zmq: str + transfer_id: str + request_id: str = "" + + +@dataclass +class ECMooncakeConnectorMetadata(ECConnectorMetadata): + """Worker operations emitted for one Scheduler step.""" + + loads: list[ECMooncakeLoadSpec] = field(default_factory=list) + pushes: list[ECMooncakePushSpec] = field(default_factory=list) + freed: list[str] | None = None + + +@dataclass +class ECMooncakeWorkerMetadata(ECConnectorWorkerMetadata): + """Completion state reported from Workers to the Scheduler.""" + + loaded: set[str] = field(default_factory=set) + failed_loads: set[str] = field(default_factory=set) + # Items the receive pool dropped under pressure. The scheduler assumes an + # evicted item stays resident until told otherwise. + reclaimed: set[str] = field(default_factory=set) + pending_saves: bool = False + failed_saves: set[str] = field(default_factory=set) + + def aggregate(self, other: ECConnectorWorkerMetadata) -> ECMooncakeWorkerMetadata: + assert isinstance(other, ECMooncakeWorkerMetadata) + return ECMooncakeWorkerMetadata( + # Every tensor-parallel rank gathers the embedding from its own + # cache, so an item counts as loaded only where all of them have + # it; one rank falling short must fail the load rather than leave + # the scheduler believing it is ready. + loaded=self.loaded & other.loaded, + failed_loads=self.failed_loads | other.failed_loads, + reclaimed=self.reclaimed | other.reclaimed, + pending_saves=self.pending_saves or other.pending_saves, + failed_saves=self.failed_saves | other.failed_saves, + ) diff --git a/vllm/distributed/ec_transfer/ec_connector/mooncake/producer.py b/vllm/distributed/ec_transfer/ec_connector/mooncake/producer.py new file mode 100644 index 0000000000..48b4e4dd58 --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/mooncake/producer.py @@ -0,0 +1,418 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Producer-side push lifecycle, futures, and source-tensor ownership. + +This module records when a push may advance or release its source. Worker +orchestration performs the actual control exchanges and Mooncake writes. +""" + +from __future__ import annotations + +import threading +from collections import OrderedDict +from collections.abc import Callable +from concurrent.futures import Future, ThreadPoolExecutor +from contextlib import suppress +from dataclasses import dataclass, field +from enum import Enum, auto +from typing import Any + +import torch + +from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import ( + ECMooncakePushSpec, +) + + +def _same_destination( + existing: ECMooncakePushSpec, incoming: ECMooncakePushSpec +) -> bool: + """Whether two specs describe one push, ignoring which request asked. + + The same encoding requested twice shares a transfer id legitimately; only + a different payload or destination is a collision. + """ + return ( + existing.mm_hash == incoming.mm_hash + and existing.nbytes == incoming.nbytes + and existing.shape == incoming.shape + and existing.dtype == incoming.dtype + and existing.consumer_zmq == incoming.consumer_zmq + ) + + +class ProducerPushState(Enum): + """Lifecycle of one Producer push from reservation to terminal state.""" + + # The reservation reply and source tensor may arrive in either order. + WAITING_INPUTS = auto() + WRITING = auto() + NOTIFYING = auto() + DONE = auto() + CANCEL_PENDING = auto() + CANCELLED = auto() + FAILED = auto() + + +@dataclass +class ProducerPushRecord: + """Collect all asynchronous state for one Producer push.""" + + spec: ECMooncakePushSpec + state: ProducerPushState + reservation_future: Future[list[dict[str, Any]]] + reservations: list[dict[str, Any]] = field(default_factory=list) + shard_futures: list[Future[Any]] = field(default_factory=list) + source_tensor: torch.Tensor | None = None + source_event: torch.Event | None = None + batch_future: Future[None] | None = None + error: str | None = None + + +_ALLOWED_TRANSITIONS = { + ProducerPushState.WAITING_INPUTS: { + ProducerPushState.WRITING, + ProducerPushState.CANCEL_PENDING, + ProducerPushState.FAILED, + }, + ProducerPushState.WRITING: { + ProducerPushState.NOTIFYING, + ProducerPushState.FAILED, + }, + ProducerPushState.NOTIFYING: { + ProducerPushState.DONE, + ProducerPushState.FAILED, + }, + ProducerPushState.CANCEL_PENDING: { + ProducerPushState.CANCELLED, + ProducerPushState.FAILED, + }, + ProducerPushState.DONE: set(), + ProducerPushState.CANCELLED: set(), + ProducerPushState.FAILED: set(), +} + +_TERMINAL_STATES = { + ProducerPushState.DONE, + ProducerPushState.CANCELLED, + ProducerPushState.FAILED, +} +_TERMINAL_LIMIT = 1 << 16 + + +class ProducerPushManager: + """Own Producer push records, transitions, and source tensor leases. + + Terminal records are retained for duplicate-metadata idempotency. Separate + ordered indexes keep hot polling paths from scanning those tombstones. + """ + + def __init__(self, wake: Callable[[], None] = lambda: None) -> None: + self._records: OrderedDict[str, ProducerPushRecord] = OrderedDict() + self._active_ids: OrderedDict[str, None] = OrderedDict() + self._reapable_terminal_ids: OrderedDict[str, None] = OrderedDict() + self._unreported_ids: OrderedDict[str, None] = OrderedDict() + self._batch_ids: OrderedDict[str, None] = OrderedDict() + self._source_waiters: dict[str, OrderedDict[str, None]] = {} + self._lock = threading.RLock() + self._wake = wake + + def reserve( + self, + spec: ECMooncakePushSpec, + submit: Callable[[], Future[list[dict[str, Any]]]], + ) -> tuple[ProducerPushRecord, bool]: + with self._lock: + existing = self._records.get(spec.transfer_id) + if existing is not None: + if not _same_destination(existing.spec, spec): + raise ValueError( + f"Conflicting EC destination for transfer_id={spec.transfer_id}" + ) + return existing, False + record = ProducerPushRecord( + spec=spec, + state=ProducerPushState.WAITING_INPUTS, + reservation_future=submit(), + ) + self._records[spec.transfer_id] = record + self._active_ids[spec.transfer_id] = None + self._source_waiters.setdefault(spec.mm_hash, OrderedDict())[ + spec.transfer_id + ] = None + record.reservation_future.add_done_callback( + lambda future: self._reservation_done(record, future) + ) + return record, True + + def bind_source( + self, + mm_hash: str, + tensor: torch.Tensor, + ready_event: torch.Event | None, + ) -> None: + with self._lock: + waiters = self._source_waiters.pop(mm_hash, OrderedDict()) + for transfer_id in waiters: + record = self._records[transfer_id] + if ( + record.source_tensor is not None + or record.state is not ProducerPushState.WAITING_INPUTS + ): + continue + record.source_tensor = tensor + record.source_event = ready_event + self._wake() + + def submit_batches( + self, + executor: ThreadPoolExecutor, + run_batch: Callable[[list[ProducerPushRecord]], None], + *, + wait: bool = False, + ) -> bool: + pending_event = False + with self._lock: + grouped: dict[str, list[ProducerPushRecord]] = {} + for transfer_id in list(self._active_ids): + record = self._records[transfer_id] + if ( + record.source_tensor is not None + and record.batch_future is None + and record.state is ProducerPushState.WAITING_INPUTS + ): + if not record.reservation_future.done(): + continue + if record.source_event is not None: + if wait: + record.source_event.synchronize() + elif not record.source_event.query(): + pending_event = True + continue + grouped.setdefault(record.spec.consumer_zmq, []).append(record) + batches = list(grouped.values()) + for records in batches: + future = executor.submit(run_batch, records) + for record in records: + record.batch_future = future + self._batch_ids[record.spec.transfer_id] = None + return pending_event + + def resolve_reservations(self, record: ProducerPushRecord) -> list[dict[str, Any]]: + results = record.reservation_future.result() + with self._lock: + record.reservations = results + return list(results) + + def finish_reservations( + self, + outcomes: list[tuple[ProducerPushRecord, list[dict[str, Any]] | Exception]], + ) -> None: + with self._lock: + for record, result in outcomes: + if isinstance(result, Exception): + record.reservation_future.set_exception(result) + else: + record.reservation_future.set_result(result) + + def _reservation_done( + self, + record: ProducerPushRecord, + future: Future[list[dict[str, Any]]], + ) -> None: + try: + future.result() + except Exception as exc: + with self._lock: + self._set_error(record, exc) + if ( + record.state is ProducerPushState.WAITING_INPUTS + and record.source_tensor is None + ): + self._transition(record, ProducerPushState.FAILED) + finally: + self._wake() + + def settle_all(self, records: list[ProducerPushRecord]) -> None: + for record in records: + with suppress(Exception): + record.reservation_future.result() + + def replace_reservations( + self, + record: ProducerPushRecord, + reservations: list[dict[str, Any]], + ) -> None: + with self._lock: + record.reservations = reservations + + def track_shard_futures( + self, + records: list[ProducerPushRecord], + futures: list[Future[Any]], + ) -> None: + with self._lock: + for record in records: + record.shard_futures.extend(futures) + + def begin_writing(self, record: ProducerPushRecord) -> None: + with self._lock: + if record.source_tensor is None: + raise RuntimeError( + f"Producer push {record.spec.transfer_id!r} has no source tensor" + ) + self._transition(record, ProducerPushState.WRITING) + + def begin_notifying(self, records: list[ProducerPushRecord]) -> None: + with self._lock: + for record in records: + self._transition(record, ProducerPushState.NOTIFYING) + + def complete(self, records: list[ProducerPushRecord]) -> None: + with self._lock: + self._check_sources_releasable(records) + for record in records: + self._transition(record, ProducerPushState.DONE) + self._release_source(record) + + def fail(self, records: list[ProducerPushRecord], error: Exception) -> None: + with self._lock: + self._check_sources_releasable(records) + for record in records: + if record.state not in _TERMINAL_STATES: + self._set_error(record, error) + self._transition(record, ProducerPushState.FAILED) + self._release_source(record) + + def cancel_requests(self, request_ids: set[str] | None) -> list[ProducerPushRecord]: + """Cancel source-less waiters for requests, or all waiters at shutdown.""" + cancelled = [] + with self._lock: + for transfer_id in list(self._active_ids): + record = self._records[transfer_id] + if ( + ( + request_ids is not None + and record.spec.request_id not in request_ids + ) + or record.source_tensor is not None + or record.batch_future is not None + ): + continue + if record.state is not ProducerPushState.WAITING_INPUTS: + continue + self._transition(record, ProducerPushState.CANCEL_PENDING) + cancelled.append(record) + return cancelled + + def finish_cancel(self, record: ProducerPushRecord) -> None: + with self._lock: + if record.state is ProducerPushState.CANCEL_PENDING: + self._transition(record, ProducerPushState.CANCELLED) + + def submit_cancel( + self, + record: ProducerPushRecord, + executor: ThreadPoolExecutor, + run_cancel: Callable[[ProducerPushRecord], None], + ) -> None: + with self._lock: + future = executor.submit(run_cancel, record) + record.batch_future = future + transfer_id = record.spec.transfer_id + self._batch_ids[transfer_id] = None + if record.state in _TERMINAL_STATES: + self._reapable_terminal_ids.pop(transfer_id, None) + + def poll(self) -> list[tuple[str, str]]: + failures = [] + with self._lock: + for transfer_id in list(self._batch_ids): + record = self._records[transfer_id] + future = record.batch_future + assert future is not None + if not future.done(): + continue + try: + future.result() + except Exception as exc: + self._set_error(record, exc) + self._batch_ids.pop(transfer_id) + self._mark_reapable(record) + for transfer_id in list(self._unreported_ids): + if transfer_id in self._batch_ids: + continue + record = self._records[transfer_id] + assert record.error is not None + failures.append((record.spec.mm_hash, record.error)) + self._unreported_ids.pop(transfer_id) + self._mark_reapable(record) + self._reap_terminals() + return failures + + @property + def pending(self) -> bool: + with self._lock: + return bool(self._active_ids or self._batch_ids) + + def _release_source(self, record: ProducerPushRecord) -> None: + record.source_tensor = None + record.source_event = None + + def _set_error(self, record: ProducerPushRecord, error: BaseException) -> None: + if record.error is None: + record.error = str(error) + if record.state in _TERMINAL_STATES: + transfer_id = record.spec.transfer_id + self._unreported_ids[transfer_id] = None + self._reapable_terminal_ids.pop(transfer_id, None) + + def _reap_terminals(self) -> None: + while len(self._reapable_terminal_ids) > _TERMINAL_LIMIT: + transfer_id, _ = self._reapable_terminal_ids.popitem(last=False) + self._records.pop(transfer_id) + + def _mark_reapable(self, record: ProducerPushRecord) -> None: + transfer_id = record.spec.transfer_id + if ( + record.state in _TERMINAL_STATES + and transfer_id not in self._batch_ids + and transfer_id not in self._unreported_ids + ): + self._reapable_terminal_ids[transfer_id] = None + + def _drop_source_waiter(self, record: ProducerPushRecord) -> None: + waiters = self._source_waiters.get(record.spec.mm_hash) + if waiters is None: + return + waiters.pop(record.spec.transfer_id, None) + if not waiters: + self._source_waiters.pop(record.spec.mm_hash) + + @staticmethod + def _check_sources_releasable(records: list[ProducerPushRecord]) -> None: + for record in records: + futures = [record.reservation_future, *record.shard_futures] + if record.source_tensor is not None and not all( + future.done() for future in futures + ): + raise RuntimeError( + f"Producer push {record.spec.transfer_id!r} released its " + "source too early" + ) + + def _transition(self, record: ProducerPushRecord, state: ProducerPushState) -> None: + if state not in _ALLOWED_TRANSITIONS[record.state]: + raise RuntimeError( + f"Cannot transition producer push {record.spec.transfer_id!r} from " + f"{record.state.name} to {state.name}" + ) + record.state = state + if state is not ProducerPushState.WAITING_INPUTS: + self._drop_source_waiter(record) + if state in _TERMINAL_STATES: + transfer_id = record.spec.transfer_id + self._active_ids.pop(transfer_id, None) + if record.error is not None: + self._unreported_ids[transfer_id] = None + self._mark_reapable(record) diff --git a/vllm/distributed/ec_transfer/ec_connector/mooncake/reservation.py b/vllm/distributed/ec_transfer/ec_connector/mooncake/reservation.py new file mode 100644 index 0000000000..407b391193 --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/mooncake/reservation.py @@ -0,0 +1,487 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Consumer-side reservation lifecycle and destination-memory ownership. + +Reservations make remote writes idempotent and ensure cancellation or expiry +cannot free a destination while Mooncake may still be writing into it. +""" + +from __future__ import annotations + +import threading +import time +import uuid +from collections import OrderedDict +from dataclasses import dataclass +from enum import Enum, auto + +import torch + +from vllm.distributed.ec_transfer.ec_connector.mooncake.memory import ( + ConsumerMemoryPool, + MemoryAllocation, + ResidentLease, +) + + +class ConsumerReservationState(Enum): + """Lifecycle of one Consumer destination reservation.""" + + WRITING = auto() + READY = auto() + CANCEL_PENDING = auto() + EXPIRE_PENDING = auto() + CANCELLED = auto() + EXPIRED = auto() + + +_ALLOWED_TRANSITIONS = { + ConsumerReservationState.WRITING: { + ConsumerReservationState.READY, + ConsumerReservationState.CANCEL_PENDING, + ConsumerReservationState.EXPIRE_PENDING, + ConsumerReservationState.CANCELLED, + }, + ConsumerReservationState.READY: { + ConsumerReservationState.CANCELLED, + ConsumerReservationState.EXPIRED, + }, + ConsumerReservationState.CANCEL_PENDING: {ConsumerReservationState.CANCELLED}, + ConsumerReservationState.EXPIRE_PENDING: { + ConsumerReservationState.CANCELLED, + ConsumerReservationState.EXPIRED, + }, + ConsumerReservationState.CANCELLED: set(), + ConsumerReservationState.EXPIRED: set(), +} + +_ACTIVE_STATES = { + ConsumerReservationState.WRITING, + ConsumerReservationState.READY, + ConsumerReservationState.CANCEL_PENDING, + ConsumerReservationState.EXPIRE_PENDING, +} +_DEFERRED_STATES = { + ConsumerReservationState.CANCEL_PENDING, + ConsumerReservationState.EXPIRE_PENDING, +} +_WRITER_OWNED_STATES = { + ConsumerReservationState.WRITING, + ConsumerReservationState.CANCEL_PENDING, + ConsumerReservationState.EXPIRE_PENDING, +} + + +@dataclass +class ConsumerReservation: + """Own the identity and destination allocation of one remote push.""" + + transfer_id: str + mm_hash: str + reservation_id: str + state: ConsumerReservationState + shape: tuple[int, ...] = () + dtype: str = "" + allocation: MemoryAllocation | None = None + lease: ResidentLease[MemoryAllocation] | None = None + expires_at: float = 0 + writer_id: str = "" + + +class ConsumerReservationManager: + """Own reservation transitions and destination allocation releases. + + A remote writer owns a ``WRITING`` allocation. Cancellation and expiry are + therefore deferred until completion, unless refresh explicitly abandons + that writer before replacing its reservation. + """ + + def __init__( + self, + memory: ConsumerMemoryPool, + lease_ttl: float, + tombstone_limit: int, + ) -> None: + self._memory = memory + self._lease_ttl = lease_ttl + self._tombstone_limit = tombstone_limit + self._records: dict[str, ConsumerReservation] = {} + self._active_ids: dict[str, None] = {} + self._tombstones: OrderedDict[str, None] = OrderedDict() + self._writers: dict[str, ConsumerReservation] = {} + self._followers: dict[str, dict[str, None]] = {} + self._ready: list[str] = [] + self._condition = threading.Condition(memory.lock) + self._shutting_down = False + + def reserve( + self, + transfer_id: str, + mm_hash: str, + nbytes: int, + shape: tuple[int, ...], + dtype_name: str, + dtype: torch.dtype, + ) -> tuple[ConsumerReservation | None, bool]: + with self._condition: + if self._shutting_down: + raise RuntimeError("Consumer reservation manager is shutting down") + existing = self._records.get(transfer_id) + if ( + existing is not None + and existing.state is ConsumerReservationState.CANCELLED + ): + return existing, False + if ( + existing is not None + and existing.state is ConsumerReservationState.EXPIRED + ): + self._remove(transfer_id) + existing = None + if ( + existing is not None + and existing.state is ConsumerReservationState.EXPIRE_PENDING + ): + raise RuntimeError( + f"Transfer {transfer_id!r} still has an active writer" + ) + if existing is not None: + if ( + existing.mm_hash != mm_hash + or existing.shape != shape + or existing.dtype != dtype_name + ): + raise ValueError("conflicting reservation for transfer_id") + if existing.state not in _ACTIVE_STATES: + raise RuntimeError( + f"Cannot reserve transfer {transfer_id!r} in " + f"{existing.state.name}" + ) + if existing.state is ConsumerReservationState.WRITING: + existing.expires_at = time.monotonic() + self._lease_ttl + return existing, False + + lease = self._memory.acquire_cached(mm_hash, shape, dtype) + now = time.monotonic() + if lease is not None: + record = ConsumerReservation( + transfer_id, + mm_hash, + uuid.uuid4().hex, + ConsumerReservationState.READY, + shape, + dtype_name, + lease.value, + lease, + now + self._lease_ttl, + ) + self._insert(record) + return record, False + writer = self._writers.get(mm_hash) + if writer is not None and writer.state is ConsumerReservationState.WRITING: + if writer.shape != shape or writer.dtype != dtype_name: + raise ValueError("conflicting in-flight tensor for mm_hash") + record = ConsumerReservation( + transfer_id, + mm_hash, + uuid.uuid4().hex, + ConsumerReservationState.WRITING, + shape, + dtype_name, + writer.allocation, + expires_at=now + self._lease_ttl, + writer_id=writer.transfer_id, + ) + self._insert(record) + self._followers.setdefault(writer.transfer_id, {})[transfer_id] = None + return record, False + allocation = self._memory.try_allocate(nbytes, shape, dtype) + if allocation is None: + self._expire_locked(time.monotonic()) + allocation = self._memory.try_allocate(nbytes, shape, dtype) + if allocation is None: + allocation = self._memory.reclaim_and_allocate(nbytes, shape, dtype) + if allocation is None: + return None, False + record = ConsumerReservation( + transfer_id, + mm_hash, + uuid.uuid4().hex, + ConsumerReservationState.WRITING, + shape, + dtype_name, + allocation, + None, + now + self._lease_ttl, + ) + self._insert(record) + self._writers[mm_hash] = record + return record, True + + def status(self, transfer_id: str) -> ConsumerReservation | None: + with self._memory.lock: + record = self._records.get(transfer_id) + if record is None or record.state not in _ACTIVE_STATES: + return None + return record + + def complete(self, transfer_id: str, reservation_id: str) -> tuple[bool, bool]: + with self._condition: + try: + record = self._records.get(transfer_id) + if record is None or record.reservation_id != reservation_id: + return False, False + if record.state is ConsumerReservationState.READY: + return True, False + if record.writer_id: + return False, False + if record.state in _WRITER_OWNED_STATES: + self._publish_followers(record) + if record.state in _DEFERRED_STATES: + terminal = ( + ConsumerReservationState.CANCELLED + if record.state is ConsumerReservationState.CANCEL_PENDING + else ConsumerReservationState.EXPIRED + ) + self._terminate(record, terminal) + return True, False + if record.state is not ConsumerReservationState.WRITING: + return False, False + self._transition(record, ConsumerReservationState.READY) + record.expires_at = time.monotonic() + self._lease_ttl + return True, True + finally: + self._condition.notify_all() + + def _publish_followers(self, writer: ConsumerReservation) -> None: + if self._writers.get(writer.mm_hash) is writer: + self._writers.pop(writer.mm_hash) + followers = self._followers.pop(writer.transfer_id, {}) + if not followers: + return + assert writer.allocation is not None + allocation = self._memory.publish(writer.mm_hash, writer.allocation, pin=False) + for record in [writer, *(self._records[key] for key in followers)]: + record.allocation = allocation + record.lease = self._memory.acquire_cached( + record.mm_hash, record.shape, allocation.tensor.dtype + ) + assert record.lease is not None + if record is not writer: + record.writer_id = "" + self._transition(record, ConsumerReservationState.READY) + record.expires_at = time.monotonic() + self._lease_ttl + self._ready.append(record.transfer_id) + + def drain_ready(self) -> list[str]: + with self._memory.lock: + ready, self._ready = self._ready, [] + return ready + + def begin_shutdown(self) -> None: + """Stop new reservations and cancel everything without a remote writer.""" + with self._condition: + if self._shutting_down: + return + self._shutting_down = True + for transfer_id in list(self._active_ids): + record = self._records[transfer_id] + if record.writer_id or record.state is ConsumerReservationState.READY: + self._terminate(record, ConsumerReservationState.CANCELLED) + elif record.state is ConsumerReservationState.WRITING: + self._defer(record, ConsumerReservationState.CANCEL_PENDING) + self._condition.notify_all() + + def wait_for_writers(self, timeout: float) -> bool: + """Wait until no reservation is still owned by a remote writer.""" + + def writers_finished() -> bool: + return not any( + self._records[transfer_id].state in _WRITER_OWNED_STATES + for transfer_id in self._active_ids + ) + + with self._condition: + return self._condition.wait_for(writers_finished, timeout=max(0, timeout)) + + def cancel( + self, + transfer_id: str, + reservation_id: str, + abandon: bool = False, + refresh: bool = False, + ) -> bool: + with self._memory.lock: + record = self._records.get(transfer_id) + if ( + record is not None + and reservation_id + and record.reservation_id != reservation_id + ): + return False + if record is None: + record = ConsumerReservation( + transfer_id, + "", + "", + ConsumerReservationState.CANCELLED, + ) + self._insert(record) + self._set_tombstone_deadline(record) + self._reap_tombstones(time.monotonic()) + return True + if record.state is ConsumerReservationState.CANCELLED: + self._set_tombstone_deadline(record) + self._reap_tombstones(time.monotonic()) + return True + if refresh: + if not abandon or record.state not in { + ConsumerReservationState.WRITING, + ConsumerReservationState.EXPIRE_PENDING, + }: + return False + if record.state is ConsumerReservationState.WRITING: + self._transition(record, ConsumerReservationState.EXPIRE_PENDING) + self._terminate(record, ConsumerReservationState.EXPIRED) + self._condition.notify_all() + self._reap_tombstones(time.monotonic()) + return True + if record.writer_id: + self._terminate(record, ConsumerReservationState.CANCELLED) + return True + if record.state in _DEFERRED_STATES and not abandon: + return True + if record.state is ConsumerReservationState.WRITING and not abandon: + self._defer(record, ConsumerReservationState.CANCEL_PENDING) + return True + if record.state not in _ACTIVE_STATES: + return False + self._terminate(record, ConsumerReservationState.CANCELLED) + if self._shutting_down: + self._condition.notify_all() + self._reap_tombstones(time.monotonic()) + return True + + def take(self, transfer_id: str, mm_hash: str) -> MemoryAllocation: + with self._memory.lock: + record = self._records.get(transfer_id) + if ( + record is None + or record.state is not ConsumerReservationState.READY + or record.mm_hash != mm_hash + or record.allocation is None + ): + raise RuntimeError( + f"Pushed EC tensor is not ready for mm_hash={mm_hash}" + ) + allocation = self._memory.publish(mm_hash, record.allocation, record.lease) + record.allocation = None + record.lease = None + self._remove(transfer_id) + return allocation + + def expire(self) -> int: + with self._memory.lock: + return self._expire_locked(time.monotonic()) + + def retire_stale( + self, encoder_cache: dict[str, torch.Tensor], freed: list[str] | None = None + ) -> None: + with self._memory.lock: + self._memory.retire_stale(encoder_cache, freed=freed) + + def _expire_locked(self, now: float) -> int: + expired = 0 + for transfer_id in list(self._active_ids): + record = self._records[transfer_id] + if record.expires_at > now: + continue + if record.state is ConsumerReservationState.READY: + self._terminate(record, ConsumerReservationState.EXPIRED) + expired += 1 + elif record.state is ConsumerReservationState.WRITING: + if record.writer_id: + self._terminate(record, ConsumerReservationState.CANCELLED) + else: + self._defer(record, ConsumerReservationState.EXPIRE_PENDING) + self._reap_tombstones(now) + return expired + + def _defer( + self, record: ConsumerReservation, state: ConsumerReservationState + ) -> None: + """Keep the destination until the writer completes or abandons it.""" + self._transition(record, state) + record.expires_at = float("inf") + + def _terminate( + self, record: ConsumerReservation, state: ConsumerReservationState + ) -> None: + self._transition(record, state) + self._release(record) + self._set_tombstone_deadline(record) + + def _release(self, record: ConsumerReservation) -> None: + if record.writer_id: + followers = self._followers.get(record.writer_id, {}) + followers.pop(record.transfer_id, None) + record.allocation = None + return + if self._writers.get(record.mm_hash) is record: + self._writers.pop(record.mm_hash) + for transfer_id in list(self._followers.pop(record.transfer_id, {})): + self._terminate( + self._records[transfer_id], ConsumerReservationState.CANCELLED + ) + allocation = record.allocation + if allocation is None: + return + if record.lease is not None: + self._memory.release_cached(record.lease) + else: + self._memory.free(allocation) + record.allocation = None + record.lease = None + + def _set_tombstone_deadline(self, record: ConsumerReservation) -> None: + record.expires_at = time.monotonic() + self._lease_ttl + self._tombstones[record.transfer_id] = None + self._tombstones.move_to_end(record.transfer_id) + + def _reap_tombstones(self, now: float) -> int: + dropped = 0 + while self._tombstones: + transfer_id = next(iter(self._tombstones)) + record = self._records[transfer_id] + if ( + record.expires_at > now + and len(self._tombstones) <= self._tombstone_limit + ): + break + self._remove(transfer_id) + dropped += 1 + return dropped + + def _insert(self, record: ConsumerReservation) -> None: + self._records[record.transfer_id] = record + if record.state in _ACTIVE_STATES: + self._active_ids[record.transfer_id] = None + + def _remove(self, transfer_id: str) -> None: + self._records.pop(transfer_id, None) + self._active_ids.pop(transfer_id, None) + self._tombstones.pop(transfer_id, None) + + def _transition( + self, record: ConsumerReservation, state: ConsumerReservationState + ) -> None: + if state not in _ALLOWED_TRANSITIONS[record.state]: + raise RuntimeError( + f"Cannot transition {record.transfer_id!r} from " + f"{record.state.name} to {state.name}" + ) + record.state = state + if state in _ACTIVE_STATES: + self._active_ids[record.transfer_id] = None + else: + self._active_ids.pop(record.transfer_id, None) diff --git a/vllm/distributed/ec_transfer/ec_connector/mooncake/scheduler.py b/vllm/distributed/ec_transfer/ec_connector/mooncake/scheduler.py new file mode 100644 index 0000000000..0b52dbe9c3 --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/mooncake/scheduler.py @@ -0,0 +1,537 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Scheduler-side planning and observation for Mooncake cache transfers. + +The Scheduler prepares Producer reservations, consumes Consumer readiness +events, emits per-step Worker metadata, and converts Worker results back into +request availability without owning tensor memory or running data transfers. +""" + +from __future__ import annotations + +import math +import time +from collections import OrderedDict +from concurrent.futures import Future, ThreadPoolExecutor +from typing import TYPE_CHECKING, Any + +import torch + +from vllm.distributed.ec_transfer.ec_connector.base import ( + ECConnectorMetadata, +) +from vllm.distributed.ec_transfer.ec_connector.cpu.common import ( + _get_encoder_cache_hidden_dim, +) +from vllm.distributed.ec_transfer.ec_connector.mooncake.config import ( + _RESERVATION_TTL_SECONDS, + MooncakeECConfig, +) +from vllm.distributed.ec_transfer.ec_connector.mooncake.control import ( + ControlClient, + EventInbox, + make_cancel_request, +) +from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import ( + ECMooncakeConnectorMetadata, + ECMooncakeLoadSpec, + ECMooncakePushSpec, + ECMooncakeWorkerMetadata, +) +from vllm.distributed.ec_transfer.ec_connector.mooncake.state import ( + SchedulerTransferState, + SchedulerTransferTable, +) +from vllm.distributed.ec_transfer.ec_connector.mooncake.transfer import ( + ensure_mooncake_available, +) +from vllm.distributed.ec_transfer.ec_connector.utils import ( + PlaceholderMetadataResolver, + collect_ec_item_metadata, +) +from vllm.logger import init_logger +from vllm.v1.core.sched.output import SchedulerOutput +from vllm.v1.outputs import ECConnectorOutput + +if TYPE_CHECKING: + from vllm.config import VllmConfig + +logger = init_logger(__name__) + +# Unmatched `ec_items` entries are a protocol desync, not a per-request +# event: warn on the first few, then only once in a while. +_MAX_UNRESOLVED_TRANSFER_ID_WARNINGS = 5 + +_DRAIN_MIN_INTERVAL = 0.005 +_MAX_PENDING_EVENTS = 4096 +_MAX_TERMINAL_TRANSFER_RECORDS = 1 << 16 +_CANCEL_ATTEMPTS = 2 +_CONTROL_WORKERS = 8 + + +class ECMooncakeScheduler: + """Coordinate Mooncake transfers from the vLLM Scheduler process.""" + + def __init__(self, vllm_config: VllmConfig) -> None: + ensure_mooncake_available() + config = MooncakeECConfig.from_vllm_config(vllm_config) + + self._is_producer = config.is_producer + self._is_consumer = config.is_consumer + self._control_addr = config.control_addr + self._push_wait_timeout = config.push_wait_timeout_s + self._encoder_cache_hidden_dim = ( + _get_encoder_cache_hidden_dim(vllm_config) if config.is_producer else None + ) + self._model_config = vllm_config.model_config + self._control_client = ControlClient(config.control_timeout_ms) + self._control_executor = ThreadPoolExecutor( + max_workers=_CONTROL_WORKERS, + thread_name_prefix="ec-mooncake-control", + ) + self._event_inbox = EventInbox(self._control_client, self._control_executor) + + self._metadata_resolver = PlaceholderMetadataResolver(vllm_config.model_config) + self._unresolved_transfer_ids = 0 + self._drain_pending = True + self._drained_at = 0.0 + self._pending_cancels: dict[str, Future[Any]] = {} + self._transfers = SchedulerTransferTable( + config.pool_size, _RESERVATION_TTL_SECONDS + ) + self._scheduler_pending_work = False + self._pushes_to_prepare: dict[str, ECMooncakePushSpec] = {} + self._prepared_push_transfer_ids: set[str] = set() + self._event_ready_shards: OrderedDict[str, set[int]] = OrderedDict() + # Mirror of the engine's encoder cache, maintained from the alloc and + # free notifications the Scheduler already sends. Tracking it here + # keeps `ensure_cache_available` on the upstream two-argument shape. + self._local_cache: set[str] = set() + self._failed_saves: set[str] = set() + + def _cancel_remote(self, consumer_zmq: str, transfer_id: str) -> bool: + pending = None + for _ in range(_CANCEL_ATTEMPTS): + pending = self._control_client.discover_shards(consumer_zmq) + if pending is not None: + break + if pending is None: + raise RuntimeError( + f"Could not discover every EC consumer shard at {consumer_zmq}" + ) + + cancelled = False + error: BaseException | None = None + for _ in range(_CANCEL_ATTEMPTS): + failed = [] + for addr in pending: + try: + result = self._control_client.request( + addr, + make_cancel_request(transfer_id), + ) + except Exception as exc: + if error is None: + error = exc + failed.append(addr) + continue + cancelled |= isinstance(result, dict) and bool(result.get("cancelled")) + if not failed: + return cancelled + pending = failed + if error is not None: + raise error + return cancelled + + def _note_awaiting_push( + self, + mm_hash: str, + transfer_id: str, + request_id: str, + now: float, + ) -> None: + record = self._transfers.wait_for_event( + transfer_id, + request_id, + mm_hash, + now + self._push_wait_timeout, + ) + if record is None: + # The id names another encoding; the request is already failed. + return + if record.state is not SchedulerTransferState.WAITING_EVENT: + return + assert record.deadline is not None + if now < record.deadline: + return + elapsed = now - record.deadline + self._push_wait_timeout + self._transfers.mark_unavailable(transfer_id, now) + logger.warning( + "EC Mooncake waited %.1fs for a push of mm_hash=%s " + "(transfer_id=%s) that never arrived; " + "requests needing it fail with a retryable error.", + elapsed, + mm_hash, + transfer_id, + ) + + def take_unavailable_requests(self) -> set[str]: + failed, self._failed_saves = self._failed_saves, set() + return failed | self._transfers.take_unavailable_requests() + + def _poll_pending_cancels(self) -> None: + pending = {} + for transfer_id, future in self._pending_cancels.items(): + if not future.done(): + pending[transfer_id] = future + continue + try: + future.result() + except Exception: + logger.warning( + "EC Mooncake reservation cancellation failed", exc_info=True + ) + self._pending_cancels = pending + + def _note_shard_ready(self, data: dict[str, Any]) -> bool: + shard_count = self._event_inbox.shard_count + if shard_count <= 1: + return True + transfer_id = str(data["transfer_id"]) + record = self._transfers.get(transfer_id) + if ( + record is not None + and record.state is not SchedulerTransferState.WAITING_EVENT + ): + return False + shard = data.get("shard") + shards = self._event_ready_shards.setdefault(transfer_id, set()) + self._event_ready_shards.move_to_end(transfer_id) + shards.add(int(shard) if shard is not None else len(shards)) + if len(shards) < shard_count: + while len(self._event_ready_shards) > _MAX_PENDING_EVENTS: + self._event_ready_shards.popitem(last=False) + return False + self._event_ready_shards.pop(transfer_id, None) + return True + + def _forget_shard_readiness(self, transfer_id: str) -> None: + self._event_ready_shards.pop(transfer_id, None) + + def _store_pushed_spec(self, data: dict[str, Any]) -> bool: + transfer_id = str(data["transfer_id"]) + identifier = str(data["mm_hash"]) + _, accepted = self._transfers.observe_ready( + ECMooncakeLoadSpec( + mm_hash=identifier, + nbytes=int(data["nbytes"]), + shape=tuple(int(value) for value in data["shape"]), + dtype=str(data["dtype"]), + transfer_id=transfer_id, + ), + time.monotonic() + _RESERVATION_TTL_SECONDS, + ) + return accepted + + def _queue_cancel( + self, + transfer_id: str, + mm_hash: str = "", + request_id: str = "", + ) -> None: + if not self._transfers.cancel( + transfer_id, + time.monotonic(), + mm_hash=mm_hash, + request_id=request_id, + ): + return + self._forget_shard_readiness(transfer_id) + self._pending_cancels[transfer_id] = self._control_executor.submit( + self._cancel_remote, + self._control_addr, + transfer_id, + ) + + def _expire_transfers(self) -> None: + now = time.monotonic() + expired = self._transfers.expire(now, _MAX_TERMINAL_TRANSFER_RECORDS) + for record in expired: + self._queue_cancel(record.transfer_id) + + def _drain_push_notifications(self) -> None: + now = time.monotonic() + if not self._drain_pending and now - self._drained_at < _DRAIN_MIN_INTERVAL: + return + self._drain_pending = False + self._drained_at = now + self._poll_pending_cancels() + self._expire_transfers() + events = self._event_inbox.drain(self._control_addr) + for data in events: + try: + if not isinstance(data, dict): + raise TypeError("EC readiness event must be an object") + if not data.get("ready"): + continue + self._accept_ready_event(data) + except (KeyError, TypeError, ValueError): + # Readiness events cross a plain PULL socket, so a malformed + # one costs that event and not the engine. + logger.warning( + "Discarding a malformed EC readiness event.", exc_info=True + ) + + def _accept_ready_event(self, data: dict[str, Any]) -> None: + transfer_id = str(data["transfer_id"]) + record = self._transfers.get(transfer_id) + if record is not None and record.state in { + SchedulerTransferState.CANCELLED, + SchedulerTransferState.UNAVAILABLE, + SchedulerTransferState.EXPIRED, + SchedulerTransferState.FAILED, + }: + return + if not self._note_shard_ready(data): + return + self._store_pushed_spec(data) + + def has_cache_item(self, identifier: str) -> bool: + if not self._is_consumer: + return False + self._drain_push_notifications() + return self._transfers.has_state( + identifier, + ( + SchedulerTransferState.READY, + SchedulerTransferState.RESIDENT, + SchedulerTransferState.AVAILABLE, + ), + ) + + def _warn_unresolved_transfer_id( + self, request: Any, index: int, where: str + ) -> None: + """Report an `ec_items` entry that cannot be matched to a feature. + + A caller that cannot resolve a transfer id has to fall back to a + locally invented one or skip its bookkeeping entirely, and either way + the two sides of a transfer stop agreeing on its name. That desyncs + silently -- reservations are never released and only surface minutes + later as a full buffer pool -- so say it out loud the first time. + + `mm_hash` in `ec_items` must be a value some engine reported, never one + the caller derived itself: `mm_features[i].identifier` folds in the + engine's media_io_kwargs and mm_processor_kwargs, so a media uuid alone + no longer equals it. Omit the field to match by position instead. + """ + self._unresolved_transfer_ids += 1 + count = self._unresolved_transfer_ids + if count > _MAX_UNRESOLVED_TRANSFER_ID_WARNINGS and count % 1000: + return + params = getattr(request, "ec_transfer_params", None) + items = (params or {}).get("ec_items") or [] + logger.warning( + "EC Mooncake could not resolve a transfer id at %s for req=%s " + "index=%d: feature identifier=%s does not match any of the %d " + "ec_items %s (ec_transfer_params present=%s). Occurrence %d. " + "ec_items[].mm_hash must echo an engine-reported identifier, or " + "be omitted so the entry is matched by position.", + where, + request.request_id, + index, + request.mm_features[index].identifier[:16], + len(items), + [str(item.get("mm_hash"))[:16] for item in items[:4]], + params is not None, + count, + ) + + @staticmethod + def _request_transfer_id(request: Any, index: int) -> str | None: + params = getattr(request, "ec_transfer_params", None) or {} + items = params.get("ec_items") or [] + mm_hash = request.mm_features[index].identifier + if index < len(items): + item = items[index] + if item.get("mm_hash") in (None, mm_hash) and item.get("transfer_id"): + return str(item["transfer_id"]) + for item in items: + if item.get("mm_hash") == mm_hash and item.get("transfer_id"): + return str(item["transfer_id"]) + return None + + def ensure_cache_available(self, request: Any, num_computed_tokens: int) -> bool: + if self._is_producer: + for index, feature in enumerate(request.mm_features): + if ( + feature.mm_position.offset + feature.mm_position.length + > num_computed_tokens + ): + self._prepare_push_spec(request, index) + if not self._is_consumer: + return True + + self._drain_push_notifications() + # One timestamp for the whole decision, so the deadlines it hands out + # cannot disagree between features. + now = time.monotonic() + all_ready = True + for index, feature in enumerate(request.mm_features): + if ( + feature.mm_position.offset + feature.mm_position.length + <= num_computed_tokens + ): + continue + mm_hash = feature.identifier + transfer_id = self._request_transfer_id(request, index) + if transfer_id is not None: + self._transfers.touch_available( + transfer_id, now + _RESERVATION_TTL_SECONDS + ) + if mm_hash in self._local_cache: + continue + if self._transfers.has_state(mm_hash, (SchedulerTransferState.READY,)): + continue + if self._transfers.has_state(mm_hash, (SchedulerTransferState.LOADING,)): + all_ready = False + continue + record = self._transfers.begin_load( + mm_hash, + transfer_id, + request.request_id, + now + self._push_wait_timeout, + ) + if record is not None: + self._scheduler_pending_work = True + all_ready = False + else: + waiting_id = transfer_id or f"{request.request_id}:{index}" + self._note_awaiting_push(mm_hash, waiting_id, request.request_id, now) + all_ready = False + return all_ready + + def _prepare_push_spec(self, request: Any, index: int) -> None: + params = getattr(request, "ec_transfer_params", None) or {} + consumer_zmq = params.get("consumer_zmq") + mm_hash = request.mm_features[index].identifier + transfer_id = self._request_transfer_id(request, index) + if transfer_id is None: + # Still push: after the proxy has rewritten the item to embeds the + # consumer has no media left to fall back on, so a nameless push + # beats none. But the consumer knows this transfer by the id it + # sent, not by the one invented here, so its cancel will never + # reach the reservation this push is about to take. + if consumer_zmq: + self._warn_unresolved_transfer_id(request, index, "push prepare") + transfer_id = f"{request.request_id}:{index}" + if not consumer_zmq or transfer_id in self._prepared_push_transfer_ids: + return + num_tokens = request.get_num_encoder_embeds(index) + dtype = self._model_config.dtype + assert isinstance(dtype, torch.dtype) + assert self._encoder_cache_hidden_dim is not None + dtype_name = str(dtype).split(".")[-1] + shape = (num_tokens, self._encoder_cache_hidden_dim) + nbytes = math.prod(shape) * dtype.itemsize + self._pushes_to_prepare[transfer_id] = ECMooncakePushSpec( + mm_hash=mm_hash, + nbytes=nbytes, + shape=shape, + dtype=dtype_name, + consumer_zmq=str(consumer_zmq), + transfer_id=transfer_id, + request_id=request.request_id, + ) + self._prepared_push_transfer_ids.add(transfer_id) + + def update_state_after_alloc(self, request: Any, index: int) -> None: + self._local_cache.add(request.mm_features[index].identifier) + if self._is_producer: + self._prepare_push_spec(request, index) + + def update_state_after_free(self, request: Any, index: int) -> None: + if not self._is_consumer: + return + transfer_id = self._request_transfer_id(request, index) + if transfer_id is None: + self._warn_unresolved_transfer_id(request, index, "encoder-cache free") + return + self._queue_cancel( + transfer_id, + mm_hash=request.mm_features[index].identifier, + request_id=request.request_id, + ) + + def build_connector_meta( + self, scheduler_output: SchedulerOutput + ) -> ECConnectorMetadata: + for mm_hash in scheduler_output.free_encoder_mm_hashes: + self._local_cache.discard(mm_hash) + self._transfers.release_ready(mm_hash, time.monotonic()) + for transfer_id in self._transfers.drain_orphaned(): + self._queue_cancel(transfer_id) + meta = ECMooncakeConnectorMetadata( + freed=scheduler_output.free_encoder_mm_hashes + ) + for push_spec in self._pushes_to_prepare.values(): + meta.pushes.append(push_spec) + self._pushes_to_prepare.clear() + for record in self._transfers.take_loads_to_dispatch(): + assert record.spec is not None + meta.loads.append(record.spec) + self._poll_pending_cancels() + self._drain_pending = True + return meta + + def update_connector_output(self, connector_output: ECConnectorOutput) -> None: + meta = connector_output.ec_connector_worker_meta + if not isinstance(meta, ECMooncakeWorkerMetadata): + return + for mm_hash in meta.loaded: + self._transfers.complete_load(mm_hash) + for mm_hash in meta.failed_loads: + self._transfers.fail_load(mm_hash, time.monotonic()) + for mm_hash in meta.reclaimed: + self._transfers.reclaim(mm_hash, time.monotonic()) + self._scheduler_pending_work = meta.pending_saves + self._failed_saves.update(meta.failed_saves) + + def has_pending_push_work(self) -> bool: + return self._scheduler_pending_work + + def request_finished(self, request: Any) -> tuple[bool, dict[str, Any] | None]: + if self._is_consumer: + for index in range(len(request.mm_features)): + transfer_id = self._request_transfer_id(request, index) + if transfer_id is None: + self._warn_unresolved_transfer_id(request, index, "request finish") + continue + self._queue_cancel( + transfer_id, + mm_hash=request.mm_features[index].identifier, + request_id=request.request_id, + ) + if self._is_producer and self._prepared_push_transfer_ids: + for index in range(len(request.mm_features)): + transfer_id = self._request_transfer_id(request, index) + if transfer_id is None: + transfer_id = f"{request.request_id}:{index}" + self._prepared_push_transfer_ids.discard(transfer_id) + if not self._is_producer: + return False, None + + items = collect_ec_item_metadata(request.mm_features, self._metadata_resolver) + for index, feature in enumerate(request.mm_features): + transfer_id = self._request_transfer_id(request, index) + if transfer_id is not None: + items[feature.identifier]["transfer_id"] = transfer_id + + if not items: + return False, None + return False, items + + def close(self) -> None: + self._control_executor.shutdown(wait=True, cancel_futures=True) + self._control_client.close() + self._event_inbox.close() diff --git a/vllm/distributed/ec_transfer/ec_connector/mooncake/state.py b/vllm/distributed/ec_transfer/ec_connector/mooncake/state.py new file mode 100644 index 0000000000..a9991b20ac --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/mooncake/state.py @@ -0,0 +1,448 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Scheduler-owned lifecycle and indexes for Consumer-bound transfers.""" + +from __future__ import annotations + +from collections import OrderedDict, deque +from collections.abc import Iterable +from dataclasses import dataclass, field, replace +from enum import Enum, auto + +from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import ( + ECMooncakeLoadSpec, +) +from vllm.logger import init_logger + +logger = init_logger(__name__) + + +class SchedulerTransferState(Enum): + """Lifecycle states visible to the Scheduler. + + The states cover waiting for a Consumer event, dispatching a Worker load, + retaining a locally reusable tensor, and terminal failure conditions. + """ + + WAITING_EVENT = auto() + AVAILABLE = auto() + LOADING = auto() + READY = auto() + RESIDENT = auto() + UNAVAILABLE = auto() + EXPIRED = auto() + FAILED = auto() + CANCELLED = auto() + + +@dataclass +class SchedulerTransfer: + """Track one transfer as observed by the Scheduler.""" + + transfer_id: str + request_id: str + mm_hash: str + state: SchedulerTransferState + spec: ECMooncakeLoadSpec | None + deadline: float | None + notified_requests: set[str] = field(default_factory=set, repr=False) + + +_TERMINAL_STATES = { + SchedulerTransferState.UNAVAILABLE, + SchedulerTransferState.EXPIRED, + SchedulerTransferState.FAILED, + SchedulerTransferState.CANCELLED, +} + + +class SchedulerTransferTable: + """Own Scheduler transfer state, lookup indexes, and dispatch queues. + + The Scheduler is single-threaded, so direct transitions are sufficient; + the Producer and Consumer managers retain stricter transition matrices + because callbacks and control requests can race there. + """ + + def __init__(self, resident_capacity: int, tombstone_ttl: float) -> None: + self._resident_capacity = resident_capacity + self._tombstone_ttl = tombstone_ttl + self._records: OrderedDict[str, SchedulerTransfer] = OrderedDict() + self._active_ids: dict[str, None] = {} + self._terminal_ids: OrderedDict[str, None] = OrderedDict() + self._resident_ids: OrderedDict[str, None] = OrderedDict() + self._resident_bytes = 0 + self._hash_index: dict[str, deque[str]] = {} + self._loads_to_dispatch: OrderedDict[str, None] = OrderedDict() + self._unavailable_requests: set[str] = set() + # Records expired straight out of READY/RESIDENT: the consumer worker + # still holds their push reservation, and nothing else tells it to let + # go, so the destination buffer would sit pinned until the lease TTL. + self._orphaned: list[str] = [] + + def get(self, transfer_id: str) -> SchedulerTransfer | None: + return self._records.get(transfer_id) + + def drain_orphaned(self) -> list[str]: + """Transfer IDs expired out of READY/RESIDENT since the last drain. + + The caller must release the consumer worker's reservation for each: + `cancel()` still accepts an EXPIRED record, so the usual cancel path + applies. + """ + orphaned = self._orphaned + self._orphaned = [] + return orphaned + + def records_for_hash( + self, + mm_hash: str, + states: Iterable[SchedulerTransferState], + ) -> list[SchedulerTransfer]: + wanted = set(states) + return [ + record + for transfer_id in self._hash_index.get(mm_hash, ()) + if (record := self._records.get(transfer_id)) is not None + and record.state in wanted + ] + + def first_for_hash( + self, + mm_hash: str, + states: Iterable[SchedulerTransferState], + ) -> SchedulerTransfer | None: + states = tuple(states) + if len(states) == 1: + wanted_state = states[0] + for transfer_id in self._hash_index.get(mm_hash, ()): + record = self._records.get(transfer_id) + if record is not None and record.state is wanted_state: + return record + return None + wanted_states = set(states) + for transfer_id in self._hash_index.get(mm_hash, ()): + record = self._records.get(transfer_id) + if record is not None and record.state in wanted_states: + return record + return None + + def has_state(self, mm_hash: str, states: Iterable[SchedulerTransferState]) -> bool: + return self.first_for_hash(mm_hash, states) is not None + + def wait_for_event( + self, + transfer_id: str, + request_id: str, + mm_hash: str, + deadline: float, + ) -> SchedulerTransfer | None: + """Track a request waiting for a push. + + Returns None when `transfer_id` already names a different encoding. + The id comes from the request, so a collision has to fail that one + request; re-issuing it re-runs the encode under a fresh id. + """ + record = self._records.get(transfer_id) + if record is None: + record = SchedulerTransfer( + transfer_id=transfer_id, + request_id=request_id, + mm_hash=mm_hash, + state=SchedulerTransferState.WAITING_EVENT, + spec=None, + deadline=deadline, + ) + self._insert(record) + elif not self._identity_matches(record, mm_hash): + self._refuse_colliding_id(record, mm_hash, request_id) + return None + else: + if not record.request_id: + record.request_id = request_id + if record.state in _TERMINAL_STATES and request_id: + self._notify_unavailable(record, request_id) + return record + + def observe_ready( + self, spec: ECMooncakeLoadSpec, deadline: float + ) -> tuple[SchedulerTransfer, bool]: + transfer_id = spec.transfer_id + record = self._records.get(transfer_id) + if record is None: + record = SchedulerTransfer( + transfer_id=transfer_id, + request_id="", + mm_hash=spec.mm_hash, + state=SchedulerTransferState.WAITING_EVENT, + spec=None, + deadline=None, + ) + self._insert(record) + elif not self._identity_matches(record, spec.mm_hash): + self._refuse_colliding_id(record, spec.mm_hash, record.request_id) + return record, False + if record.state is not SchedulerTransferState.WAITING_EVENT: + return record, False + record.spec = spec + record.deadline = deadline + self._transition(record, SchedulerTransferState.AVAILABLE) + return record, True + + def touch_available(self, transfer_id: str, deadline: float) -> None: + record = self._records.get(transfer_id) + if record is not None and record.state is SchedulerTransferState.AVAILABLE: + record.deadline = deadline + + def begin_load( + self, + mm_hash: str, + transfer_id: str | None = None, + request_id: str = "", + deadline: float | None = None, + ) -> SchedulerTransfer | None: + record = self._records.get(transfer_id) if transfer_id else None + if record is not None and ( + record.mm_hash != mm_hash + or record.state is not SchedulerTransferState.AVAILABLE + ): + record = None + if record is None: + record = self.first_for_hash( + mm_hash, + ( + SchedulerTransferState.AVAILABLE, + SchedulerTransferState.RESIDENT, + ), + ) + if record is None or record.spec is None: + return None + if not record.request_id: + record.request_id = request_id + # A dispatched load that never reports back would otherwise hold this + # hash in LOADING for good, deferring every later request for it. + record.deadline = deadline + self._transition(record, SchedulerTransferState.LOADING) + self._loads_to_dispatch[record.transfer_id] = None + return record + + def take_loads_to_dispatch(self) -> list[SchedulerTransfer]: + records = [ + record + for transfer_id in self._loads_to_dispatch + if (record := self._records.get(transfer_id)) is not None + and record.state is SchedulerTransferState.LOADING + ] + self._loads_to_dispatch.clear() + return records + + def complete_load(self, mm_hash: str) -> bool: + record = self.first_for_hash(mm_hash, (SchedulerTransferState.LOADING,)) + if record is None: + return self.has_state(mm_hash, (SchedulerTransferState.READY,)) + self._transition(record, SchedulerTransferState.READY) + return True + + def fail_load(self, mm_hash: str, now: float) -> bool: + record = self.first_for_hash(mm_hash, (SchedulerTransferState.LOADING,)) + if record is None: + return self.has_state(mm_hash, (SchedulerTransferState.FAILED,)) + self._transition(record, SchedulerTransferState.FAILED, now=now) + return True + + def release_ready(self, mm_hash: str, now: float) -> None: + ready = self.records_for_hash(mm_hash, (SchedulerTransferState.READY,)) + if ready: + canonical = ready[-1] + for record in self.records_for_hash( + mm_hash, + (SchedulerTransferState.READY, SchedulerTransferState.RESIDENT), + ): + if record is not canonical: + self._transition(record, SchedulerTransferState.EXPIRED, now=now) + if canonical.spec is None: + self._transition(canonical, SchedulerTransferState.EXPIRED, now=now) + else: + canonical.spec = replace(canonical.spec, local=True) + self._transition(canonical, SchedulerTransferState.RESIDENT) + self._evict_residents(now) + + def reclaim(self, mm_hash: str, now: float) -> None: + for record in self.records_for_hash( + mm_hash, + (SchedulerTransferState.READY, SchedulerTransferState.RESIDENT), + ): + if record.state is SchedulerTransferState.READY: + record.spec = None + else: + self._transition(record, SchedulerTransferState.EXPIRED, now=now) + + def mark_unavailable(self, transfer_id: str, now: float) -> None: + record = self._records[transfer_id] + self._transition(record, SchedulerTransferState.UNAVAILABLE, now=now) + if record.request_id: + self._notify_unavailable(record, record.request_id) + + def cancel( + self, + transfer_id: str, + now: float, + mm_hash: str = "", + request_id: str = "", + ) -> bool: + record = self._records.get(transfer_id) + if ( + record is not None + and mm_hash + and not self._identity_matches(record, mm_hash) + ): + return False + if record is None: + record = SchedulerTransfer( + transfer_id=transfer_id, + request_id=request_id, + mm_hash=mm_hash, + state=SchedulerTransferState.WAITING_EVENT, + spec=None, + deadline=None, + ) + self._insert(record) + if record.state in { + SchedulerTransferState.CANCELLED, + SchedulerTransferState.READY, + SchedulerTransferState.RESIDENT, + SchedulerTransferState.FAILED, + }: + return False + self._transition(record, SchedulerTransferState.CANCELLED, now=now) + self._loads_to_dispatch.pop(transfer_id, None) + return True + + def expire(self, now: float, terminal_limit: int) -> list[SchedulerTransfer]: + if terminal_limit < 0: + raise ValueError("terminal_limit must be non-negative") + expired = [] + for transfer_id in list(self._active_ids): + record = self._records[transfer_id] + if record.deadline is None or record.deadline > now: + continue + if record.state is SchedulerTransferState.AVAILABLE: + self._transition( + record, + SchedulerTransferState.EXPIRED, + now=now, + ) + expired.append(record) + elif record.state is SchedulerTransferState.LOADING: + self._transition( + record, + SchedulerTransferState.UNAVAILABLE, + now=now, + ) + if record.request_id: + self._notify_unavailable(record, record.request_id) + expired.append(record) + while self._terminal_ids: + transfer_id = next(iter(self._terminal_ids)) + record = self._records[transfer_id] + if ( + record.deadline is not None + and record.deadline > now + and len(self._terminal_ids) <= terminal_limit + ): + break + self._remove(transfer_id) + return expired + + def take_unavailable_requests(self) -> set[str]: + unavailable = self._unavailable_requests + self._unavailable_requests = set() + return unavailable + + def _notify_unavailable(self, record: SchedulerTransfer, request_id: str) -> None: + if request_id not in record.notified_requests: + record.notified_requests.add(request_id) + self._unavailable_requests.add(request_id) + + def _insert(self, record: SchedulerTransfer) -> None: + self._records[record.transfer_id] = record + self._active_ids[record.transfer_id] = None + if record.mm_hash: + self._hash_index.setdefault(record.mm_hash, deque()).append( + record.transfer_id + ) + + @staticmethod + def _identity_matches(record: SchedulerTransfer, mm_hash: str) -> bool: + return not record.mm_hash or record.mm_hash == mm_hash + + def _refuse_colliding_id( + self, record: SchedulerTransfer, mm_hash: str, request_id: str + ) -> None: + """Report a transfer id that already names another encoding. + + Transfer ids arrive on the request, so a collision is reachable from + outside and must not reach the engine as an exception. + """ + logger.warning( + "EC Mooncake transfer_id=%s already names mm_hash=%s; refusing " + "mm_hash=%s for request %s", + record.transfer_id, + record.mm_hash[:16], + mm_hash[:16], + request_id or "", + ) + if request_id: + self._unavailable_requests.add(request_id) + + def _transition( + self, + record: SchedulerTransfer, + state: SchedulerTransferState, + now: float | None = None, + ) -> None: + if record.state is SchedulerTransferState.RESIDENT: + self._resident_ids.pop(record.transfer_id, None) + if record.spec is not None: + self._resident_bytes -= record.spec.nbytes + if state is SchedulerTransferState.RESIDENT: + self._resident_ids[record.transfer_id] = None + if record.spec is not None: + self._resident_bytes += record.spec.nbytes + if state in _TERMINAL_STATES: + if now is None: + raise ValueError("Terminal transition requires a timestamp") + # Keep a bounded tombstone so late events and repeat request IDs + # remain idempotent instead of reviving a finished transfer. + record.deadline = now + self._tombstone_ttl + self._active_ids.pop(record.transfer_id, None) + self._terminal_ids[record.transfer_id] = None + self._terminal_ids.move_to_end(record.transfer_id) + if state is SchedulerTransferState.EXPIRED and record.state in ( + SchedulerTransferState.READY, + SchedulerTransferState.RESIDENT, + ): + self._orphaned.append(record.transfer_id) + record.state = state + self._records.move_to_end(record.transfer_id) + + def _evict_residents(self, now: float) -> None: + while self._resident_bytes > self._resident_capacity and self._resident_ids: + record = self._records[next(iter(self._resident_ids))] + self._transition(record, SchedulerTransferState.EXPIRED, now=now) + + def _remove(self, transfer_id: str) -> None: + record = self._records.pop(transfer_id, None) + self._active_ids.pop(transfer_id, None) + self._terminal_ids.pop(transfer_id, None) + if transfer_id in self._resident_ids: + self._resident_ids.pop(transfer_id) + if record is not None and record.spec is not None: + self._resident_bytes -= record.spec.nbytes + self._loads_to_dispatch.pop(transfer_id, None) + if record is None or not record.mm_hash: + return + transfer_ids = self._hash_index[record.mm_hash] + transfer_ids.remove(transfer_id) + if not transfer_ids: + self._hash_index.pop(record.mm_hash) diff --git a/vllm/distributed/ec_transfer/ec_connector/mooncake/transfer.py b/vllm/distributed/ec_transfer/ec_connector/mooncake/transfer.py new file mode 100644 index 0000000000..bd1651fb8f --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/mooncake/transfer.py @@ -0,0 +1,209 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Mooncake data-plane engine and memory-registration ownership. + +The wrapper isolates optional-engine initialization, session addressing, +synchronous batched writes, and the lifetime of transient source registrations. +""" + +from __future__ import annotations + +import threading +from dataclasses import dataclass + +import torch + +from vllm.logger import init_logger + +logger = init_logger(__name__) + +_MOONCAKE_IMPORT_ERROR: ImportError | None = None +try: + from mooncake.engine import TransferEngine +except ImportError as error: + TransferEngine = None # type: ignore[misc, assignment] + _MOONCAKE_IMPORT_ERROR = error + + +def ensure_mooncake_available() -> None: + if _MOONCAKE_IMPORT_ERROR is not None: + raise ImportError( + "Install mooncake-transfer-engine to use ECMooncakeConnector." + ) from _MOONCAKE_IMPORT_ERROR + + +@dataclass +class _SourceRegistration: + """Retain one transient source range while batches reference it.""" + + tensor: torch.Tensor + nbytes: int + users: int = 1 + + +class MooncakeTransfer: + """Own a lazy Mooncake engine and transient memory registrations.""" + + def __init__(self, hostname: str, protocol: str) -> None: + self._hostname = hostname + self._protocol = protocol + self._engine: TransferEngine | None = None + self._engine_lock = threading.Lock() + self._source_registrations: dict[int, _SourceRegistration] = {} + self._pending_unregister: dict[int, torch.Tensor] = {} + self._registration_lock = threading.Lock() + self._closed = False + + def _ensure_engine(self) -> TransferEngine: + if self._engine is not None: + return self._engine + with self._engine_lock: + if self._engine is not None: + return self._engine + engine = TransferEngine() + ret = engine.initialize(self._hostname, "P2PHANDSHAKE", self._protocol, "") + if ret != 0: + raise RuntimeError("Mooncake TransferEngine initialization failed.") + self._engine = engine + logger.info( + "ECMooncakeConnector TransferEngine ready at %s:%d", + self._hostname, + engine.get_rpc_port(), + ) + return self._engine + + def ensure_ready(self) -> None: + self._ensure_engine() + + def local_session(self) -> str: + engine = self._ensure_engine() + return f"{self._hostname}:{engine.get_rpc_port()}" + + def register_memory(self, tensor: torch.Tensor) -> int: + return self._ensure_engine().batch_register_memory( + [tensor.data_ptr()], [tensor.nbytes] + ) + + def unregister_memory(self, tensor: torch.Tensor) -> bool: + engine = self._ensure_engine() + address = tensor.data_ptr() + ret = engine.unregister_memory(address) + with self._registration_lock: + if ret != 0: + logger.error( + "Mooncake EC memory unregistration failed for address %d: %d", + address, + ret, + ) + self._pending_unregister[address] = tensor + return False + self._pending_unregister.pop(address, None) + return True + + @staticmethod + def _source_range(tensor: torch.Tensor) -> tuple[int, int]: + # Encoder batches commonly split one storage into sibling tensor views. + # Registering each view's exact bytes avoids overlapping memory regions. + return tensor.data_ptr(), tensor.nbytes + + def acquire_sources(self, tensors: list[torch.Tensor]) -> list[int]: + ranges: dict[int, tuple[int, torch.Tensor]] = {} + for tensor in tensors: + address, nbytes = self._source_range(tensor) + ranges.setdefault(address, (nbytes, tensor)) + + engine = self._ensure_engine() + acquired: list[int] = [] + new_addresses: list[int] = [] + new_lengths: list[int] = [] + with self._registration_lock: + for address, (nbytes, tensor) in ranges.items(): + entry = self._source_registrations.get(address) + if entry is not None: + if entry.nbytes != nbytes: + raise RuntimeError( + "Mooncake EC source storage changed size while registered" + ) + entry.users += 1 + acquired.append(address) + continue + new_addresses.append(address) + new_lengths.append(nbytes) + self._source_registrations[address] = _SourceRegistration( + tensor=tensor, + nbytes=nbytes, + ) + acquired.append(address) + + if new_addresses: + ret = engine.batch_register_memory(new_addresses, new_lengths) + if ret != 0: + for address in acquired: + entry = self._source_registrations[address] + entry.users -= 1 + if entry.users == 0: + del self._source_registrations[address] + raise RuntimeError("Mooncake EC source registration failed") + return acquired + + def release_sources(self, addresses: list[int]) -> bool: + if not addresses: + return True + with self._registration_lock: + unused = [] + for address in addresses: + entry = self._source_registrations.get(address) + if entry is None: + continue + entry.users -= 1 + if entry.users == 0: + unused.append(address) + if not unused: + return True + ret = self._ensure_engine().batch_unregister_memory(unused) + if ret != 0: + logger.warning( + "Keeping %d EC source tensors registered after Mooncake " + "unregistration failure", + len(unused), + ) + return False + for address in unused: + del self._source_registrations[address] + self._pending_unregister.pop(address, None) + return True + + def write( + self, + session: str, + sources: list[int], + destinations: list[int], + lengths: list[int], + ) -> None: + """Write one synchronous batch, returning only at terminal status.""" + ret = self._ensure_engine().batch_transfer_sync_write( + session, sources, destinations, lengths + ) + if ret != 0: + raise RuntimeError( + f"Mooncake EC push to {session} failed with status {ret}" + ) + + def close(self) -> None: + if self._closed: + return + self._closed = True + engine = self._engine + if engine is None: + return + with self._registration_lock: + addresses = list(self._source_registrations) + addresses.extend(self._pending_unregister) + if not addresses: + return + ret = engine.batch_unregister_memory(list(dict.fromkeys(addresses))) + if ret != 0: + logger.error("Mooncake EC batch memory unregistration failed: %d", ret) + return + self._source_registrations.clear() + self._pending_unregister.clear() diff --git a/vllm/distributed/ec_transfer/ec_connector/mooncake/worker.py b/vllm/distributed/ec_transfer/ec_connector/mooncake/worker.py new file mode 100644 index 0000000000..05ce16b181 --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/mooncake/worker.py @@ -0,0 +1,940 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Worker-side orchestration of Mooncake control, memory, and data planes. + +Consumer Workers expose rank-local reservations and publish received tensors. +Producer Workers reserve every destination shard, bind computed sources, run +batched Mooncake writes, and report asynchronous completion to the Scheduler. +""" + +from __future__ import annotations + +import math +import threading +import time +from collections.abc import Callable +from concurrent.futures import Future, ThreadPoolExecutor +from functools import partial +from typing import TYPE_CHECKING, Any, TypeVar, cast + +import torch + +from vllm.distributed.ec_transfer.ec_connector.mooncake.config import ( + _RESERVATION_TTL_SECONDS, + MooncakeECConfig, +) +from vllm.distributed.ec_transfer.ec_connector.mooncake.control import ( + ConsumerControlServer, + ControlClient, + make_cancel_request, +) +from vllm.distributed.ec_transfer.ec_connector.mooncake.memory import ( + ConsumerMemoryPool, + ProducerMemoryPool, +) +from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import ( + ECMooncakeConnectorMetadata, + ECMooncakePushSpec, + ECMooncakeWorkerMetadata, +) +from vllm.distributed.ec_transfer.ec_connector.mooncake.producer import ( + ProducerPushManager, + ProducerPushRecord, +) +from vllm.distributed.ec_transfer.ec_connector.mooncake.reservation import ( + ConsumerReservationManager, + ConsumerReservationState, +) +from vllm.distributed.ec_transfer.ec_connector.mooncake.transfer import ( + MooncakeTransfer, + ensure_mooncake_available, +) +from vllm.logger import init_logger +from vllm.utils.network_utils import get_ip + +logger = init_logger(__name__) + +_T = TypeVar("_T") + +if TYPE_CHECKING: + from vllm.config import VllmConfig + +_RESERVATION_REFRESH_SECONDS = _RESERVATION_TTL_SECONDS / 2 +_MAX_CANCELLED_TRANSFER_IDS = 1 << 16 +_CANCEL_ATTEMPTS = 2 +_TRANSFER_WORKERS = 4 +_CONTROL_WORKERS = 8 + + +class _FanoutError(RuntimeError): + """Retain shard outcomes after every started task settles.""" + + def __init__(self, error: BaseException, results: list[Any | None]) -> None: + self.results = results + super().__init__(str(error)) + + +class ECMooncakeWorker: + """Orchestrate consumer reservations and producer push batches. + + Consumers may use TP, PP, and DP. Only the first PP stage receives encoder + outputs, and each TP rank exposes a consecutive control port and receives + the same source concurrently. Producers remain unsharded and unreplicated. + With DP, the caller must route both halves of a request to the same replica + and pass that replica's control address to the producer. + """ + + def __init__(self, vllm_config: VllmConfig) -> None: + ensure_mooncake_available() + config = MooncakeECConfig.from_vllm_config(vllm_config) + self.is_producer = config.is_producer + self.is_consumer = config.is_consumer + self._buffer_device = config.buffer_device + self._control_host = config.control_host + self._control_port = config.control_port + self._transfer = MooncakeTransfer(get_ip(), config.protocol) + self._consumer_memory = ConsumerMemoryPool( + config.pool_size, + self._transfer, + ) + self._reservations = ConsumerReservationManager( + self._consumer_memory, + _RESERVATION_TTL_SECONDS, + _MAX_CANCELLED_TRANSFER_IDS, + ) + self._consumer_rank_resolved = False + self._is_receiving_rank = True + self._tp_rank = 0 + self._tp_size = 1 + self._control_server: ConsumerControlServer | None = None + # Worker producer + self._producer_memory = ProducerMemoryPool( + config.pool_size, + self._transfer, + ) + self._control_client = ControlClient(config.control_timeout_ms) + self._shutdown_drain_timeout_s = config.control_timeout_ms / 1000 + self._io_executor = ThreadPoolExecutor( + max_workers=_TRANSFER_WORKERS, + thread_name_prefix="ec-mooncake-transfer", + ) + self._control_executor = ThreadPoolExecutor( + max_workers=_CONTROL_WORKERS, + thread_name_prefix="ec-mooncake-control", + ) + self._shard_pool: ThreadPoolExecutor | None = None + self._shard_pool_lock = threading.Lock() + self._push_ready = threading.Event() + self._producer_pushes = ProducerPushManager(self._push_ready.set) + self._dispatch_stop = threading.Event() + self._dispatcher: threading.Thread | None = None + self._failed_saves: set[str] = set() + self._collecting_sources = False + self._completed_loads: set[str] = set() + self._failed_loads: set[str] = set() + self._shutdown = False + if self.is_producer: + self._dispatcher = threading.Thread( + target=self._dispatch_pushes, name="ec-mooncake-ready", daemon=True + ) + self._dispatcher.start() + + def _dispatch_pushes(self) -> None: + pending_event = False + while not self._dispatch_stop.is_set(): + self._push_ready.wait(timeout=0.001 if pending_event else None) + self._push_ready.clear() + if self._dispatch_stop.is_set(): + break + if self._collecting_sources: + pending_event = False + continue + pending_event = self._producer_pushes.submit_batches( + self._io_executor, self._push_batch + ) + + def _resolve_consumer_rank(self) -> None: + """Place this worker in the consumer receive topology.""" + if self._consumer_rank_resolved: + return + self._consumer_rank_resolved = True + try: + from vllm.distributed.parallel_state import get_pp_group, get_tp_group + + tp_group = get_tp_group() + self._tp_rank = tp_group.rank_in_group + self._tp_size = tp_group.world_size + self._is_receiving_rank = get_pp_group().is_first_rank + except AssertionError: + # Groups are only absent outside a distributed run, where this + # worker is the whole consumer. + self._tp_rank = 0 + self._tp_size = 1 + self._is_receiving_rank = True + + def start_services(self) -> None: + if not self.is_consumer or self._control_server is not None: + return + self._resolve_consumer_rank() + if not self._is_receiving_rank: + # Later pipeline stages hold no encoder outputs, so they need + # neither a receive pool nor a control channel. + return + self._consumer_memory.prepare(torch.device(self._buffer_device)) + consumer_pool = self._consumer_memory.tensor + if consumer_pool is None: + raise RuntimeError( + "Mooncake push mode requires a registered consumer buffer pool." + ) + base_port = self._control_port + self._control_server = ConsumerControlServer( + # The reservation channel hands out registered-memory addresses + # and takes cancellations, so it listens where the Consumer + # advertises itself (`ec_ip`), not on every interface. + self._control_host, + base_port + self._tp_rank, + self._reserve_push_destination, + self._push_status, + self._reservations.complete, + self._reservations.cancel, + self._reservations.expire, + peer_ports=[base_port + rank for rank in range(self._tp_size)], + device=consumer_pool.device, + drain_ready=self._reservations.drain_ready, + ) + try: + self._control_server.start() + except Exception: + self._control_server.close() + self._control_server = None + raise + + def _reserve_push_destination(self, payload: dict[str, Any]) -> dict[str, Any]: + transfer_id = str(payload["transfer_id"]) + mm_hash = str(payload["mm_hash"]) + nbytes = int(payload["nbytes"]) + shape = tuple(int(value) for value in payload["shape"]) + dtype_name = str(payload["dtype"]) + dtype = getattr(torch, dtype_name, None) + if not isinstance(dtype, torch.dtype): + raise ValueError(f"Unsupported torch dtype string: {dtype_name!r}") + expected_nbytes = math.prod(shape) * dtype.itemsize + if expected_nbytes != nbytes: + raise ValueError("shape and dtype do not match nbytes") + + self._reservations.expire() + reservation, should_write = self._reservations.reserve( + transfer_id, mm_hash, nbytes, shape, dtype_name, dtype + ) + if reservation is None: + raise RuntimeError("EC consumer buffer pool is full") + if reservation.state in { + ConsumerReservationState.CANCEL_PENDING, + ConsumerReservationState.CANCELLED, + }: + return { + "reservation_id": "", + "dst_session": "", + "dst_ptr": 0, + "nbytes": nbytes, + "write": False, + "ready": False, + "cancelled": True, + } + assert reservation.allocation is not None + + return { + "reservation_id": reservation.reservation_id, + "dst_session": self._transfer.local_session(), + "dst_ptr": reservation.allocation.tensor.data_ptr(), + "nbytes": reservation.allocation.tensor.nbytes, + "write": should_write, + "ready": reservation.state is ConsumerReservationState.READY, + "cached": reservation.lease is not None, + } + + def _push_status(self, transfer_id: str) -> dict[str, Any] | None: + reservation = self._reservations.status(transfer_id) + if reservation is None: + return None + assert reservation.allocation is not None + return { + "mm_hash": reservation.mm_hash, + "ready": reservation.state is ConsumerReservationState.READY, + "reservation_id": reservation.reservation_id, + "nbytes": reservation.allocation.tensor.nbytes, + "shape": list(reservation.shape), + "dtype": reservation.dtype, + } + + def _shard_executor(self) -> ThreadPoolExecutor: + """Use a separate pool so nested shard fan-out cannot deadlock.""" + with self._shard_pool_lock: + if self._shard_pool is None: + self._shard_pool = ThreadPoolExecutor( + max_workers=32, thread_name_prefix="ec-mooncake-shard" + ) + return self._shard_pool + + def _reserve_one(self, addr: str, spec: ECMooncakePushSpec) -> dict[str, Any]: + result = self._control_client.request( + addr, + { + "op": "reserve", + "transfer_id": spec.transfer_id, + "mm_hash": spec.mm_hash, + "nbytes": spec.nbytes, + "shape": list(spec.shape), + "dtype": spec.dtype, + }, + ) + if not isinstance(result, dict): + raise RuntimeError("Invalid EC reservation response") + result["_received_at"] = time.monotonic() + result["addr"] = addr + return result + + def _run_fanout( + self, + tasks: list[Callable[[], _T]], + on_submit: Callable[[int, Future[_T]], None] | None = None, + ) -> list[_T]: + """Run every started shard task and retain partial results on failure. + + Waiting for all submitted tasks is what makes source-memory release and + partial-reservation cleanup safe after one shard fails. + """ + if not tasks: + return [] + futures: list[tuple[int, Future[_T]]] = [] + results: list[_T | None] = [None] * len(tasks) + error: BaseException | None = None + for index, task in enumerate(tasks[1:], 1): + try: + future = self._shard_executor().submit(task) + except Exception as exc: + error = exc + break + futures.append((index, future)) + if on_submit is not None: + on_submit(index, future) + if error is None: + try: + results[0] = tasks[0]() + except Exception as exc: + error = exc + for index, future in futures: + try: + results[index] = future.result() + except Exception as exc: + if error is None: + error = exc + if error is not None: + raise _FanoutError(error, results) + return cast(list[_T], results) + + def _cancel_reservations( + self, + spec: ECMooncakePushSpec, + reservations: list[dict[str, Any]], + *, + refresh: bool = False, + record: ProducerPushRecord | None = None, + ) -> None: + reservations = [ + shard for shard in reservations if not shard.get("cancelled", False) + ] + if not reservations: + return + + def cancel(shard: dict[str, Any]) -> dict[str, Any]: + addr = shard.get("addr") + if not isinstance(addr, str) or not addr: + raise RuntimeError( + "EC reservation is missing a confirmed shard address" + ) + result = self._control_client.request( + addr, + make_cancel_request( + spec.transfer_id, + str(shard.get("reservation_id", "")), + abandon=True, + refresh=refresh, + ), + ) + if not isinstance(result, dict) or not result.get("cancelled"): + raise RuntimeError( + f"Could not cancel EC reservation for mm_hash={spec.mm_hash}" + ) + return shard + + def track(_index: int, future: Future[dict[str, Any]]) -> None: + if record is not None: + self._producer_pushes.track_shard_futures([record], [future]) + + self._run_fanout([partial(cancel, shard) for shard in reservations], track) + + def _retry_cancel_reservations( + self, + spec: ECMooncakePushSpec, + reservations: list[dict[str, Any]], + *, + record: ProducerPushRecord | None = None, + ) -> None: + pending = [shard for shard in reservations if not shard.get("cancelled", False)] + error: _FanoutError | None = None + for _ in range(_CANCEL_ATTEMPTS): + try: + self._cancel_reservations(spec, pending, record=record) + except _FanoutError as exc: + error = exc + pending = [ + shard + for index, shard in enumerate(pending) + if exc.results[index] is None + ] + continue + return + assert error is not None + raise error + + def _reserve_remote(self, spec: ECMooncakePushSpec) -> list[dict[str, Any]]: + """Reserve a destination on every shard of the consumer.""" + result = self._reserve_remote_many([spec])[0] + if isinstance(result, Exception): + raise result + return result + + def _refresh_remote_reservations( + self, + spec: ECMooncakePushSpec, + reservations: list[dict[str, Any]], + record: ProducerPushRecord | None = None, + ) -> list[dict[str, Any]]: + stale = [ + shard + for shard in reservations + if not shard.get("ready", False) + and not shard.get("cached", False) + and not shard.get("cancelled", False) + ] + try: + self._cancel_reservations(spec, stale, refresh=True, record=record) + except _FanoutError as exc: + pending = [ + shard for index, shard in enumerate(stale) if exc.results[index] is None + ] + try: + self._retry_cancel_reservations(spec, pending, record=record) + except _FanoutError as cleanup_error: + raise exc from cleanup_error + raise + return self._reserve_remote(spec) + + @staticmethod + def _validate_push_source(push: ProducerPushRecord) -> None: + tensor = push.source_tensor + assert tensor is not None + spec = push.spec + if tuple(tensor.shape) != tuple(spec.shape): + raise ValueError(f"EC source shape mismatch for mm_hash={spec.mm_hash}") + if str(tensor.dtype).split(".")[-1] != spec.dtype: + raise ValueError(f"EC source dtype mismatch for mm_hash={spec.mm_hash}") + if not tensor.is_contiguous(): + raise ValueError(f"EC source must be contiguous for mm_hash={spec.mm_hash}") + if tensor.nbytes != spec.nbytes: + raise ValueError(f"EC source size mismatch for mm_hash={spec.mm_hash}") + + def start_save_caches( + self, + metadata: ECMooncakeConnectorMetadata, + encoder_cache: dict[str, torch.Tensor] | None = None, + **kwargs: Any, + ) -> None: + self._collecting_sources = True + new: dict[str, list[ProducerPushRecord]] = {} + for spec in metadata.pushes: + try: + record, created = self._producer_pushes.reserve(spec, Future) + if created: + new.setdefault(spec.consumer_zmq, []).append(record) + except ValueError: + logger.warning("Rejected conflicting EC push", exc_info=True) + self._failed_saves.add(spec.request_id) + for records in new.values(): + future = self._control_executor.submit(self._reserve_batch, records) + future.add_done_callback(partial(self._reservation_batch_done, records)) + if not isinstance(encoder_cache, dict): + return + for mm_hash in dict.fromkeys(spec.mm_hash for spec in metadata.pushes): + tensor = encoder_cache.get(mm_hash) + if tensor is not None: + self._bind_push_source(tensor, mm_hash) + + def _reserve_batch(self, records: list[ProducerPushRecord]) -> None: + """Resolve independent item futures with one reserve RPC per shard.""" + if len(records) == 1: + record = records[0] + try: + record.reservation_future.set_result(self._reserve_remote(record.spec)) + except Exception as exc: + record.reservation_future.set_exception(exc) + return + outcomes = self._reserve_remote_many([record.spec for record in records]) + self._producer_pushes.finish_reservations(list(zip(records, outcomes))) + + def _reserve_remote_many( + self, specs: list[ECMooncakePushSpec] + ) -> list[list[dict[str, Any]] | Exception]: + shards = None + for _ in range(_CANCEL_ATTEMPTS): + shards = self._control_client.discover_shards(specs[0].consumer_zmq) + if shards is not None: + break + if shards is None: + raise RuntimeError( + f"Could not discover every EC consumer shard at {specs[0].consumer_zmq}" + ) + + def reserve(addr: str) -> list[dict[str, Any]]: + if len(specs) == 1: + return [{"ok": True, "result": self._reserve_one(addr, specs[0])}] + response = self._control_client.request( + addr, + { + "op": "reserve_batch", + "items": [ + { + "transfer_id": spec.transfer_id, + "mm_hash": spec.mm_hash, + "nbytes": spec.nbytes, + "shape": list(spec.shape), + "dtype": spec.dtype, + } + for spec in specs + ], + }, + ) + items = response["items"] + if len(items) != len(specs): + raise RuntimeError("Malformed EC reservation batch response") + for item in items: + if item.get("ok"): + item["result"]["_received_at"] = time.monotonic() + return items + + fanout_error = None + results: list[Any] + try: + results = self._run_fanout([partial(reserve, addr) for addr in shards]) + except _FanoutError as exc: + results = exc.results + fanout_error = exc + outcomes: list[list[dict[str, Any]] | Exception] = [] + for index, spec in enumerate(specs): + reservations: list[dict[str, Any]] = [] + error = None + for addr, items in zip(shards, results): + item = items[index] if items is not None else {} + if item.get("ok"): + reservations.append({**item["result"], "addr": addr}) + else: + error = fanout_error or RuntimeError(item["error"]) + reservations.append({"addr": addr, "reservation_id": ""}) + if error is None: + outcomes.append(reservations) + else: + failure = _FanoutError(error, list(reservations)) + try: + self._retry_cancel_reservations(spec, reservations) + except Exception as cleanup_error: + failure.__cause__ = cleanup_error + logger.exception("Failed to release partial EC reservation batch") + outcomes.append(failure) + return outcomes + + @staticmethod + def _reservation_batch_done( + records: list[ProducerPushRecord], future: Future[None] + ) -> None: + error = future.exception() + if error is not None: + for record in records: + if not record.reservation_future.done(): + record.reservation_future.set_exception(error) + + def start_load_caches( + self, + metadata: ECMooncakeConnectorMetadata, + encoder_cache: dict[str, torch.Tensor], + **kwargs: Any, + ) -> None: + self._resolve_consumer_rank() + if not self._is_receiving_rank: + # Later pipeline stages never gather multimodal embeddings. + return + self._transfer.ensure_ready() + if self._buffer_device == "cuda" and not torch.accelerator.is_available(): + raise RuntimeError( + "ECMooncakeConnector requires CUDA for ec_buffer_device=cuda" + ) + self._reservations.retire_stale(encoder_cache, metadata.freed) + + for spec in metadata.loads: + if spec.mm_hash in encoder_cache: + if not spec.local: + # The spec's id is one shard's; cancel by transfer. + self._reservations.cancel(spec.transfer_id, "") + self._completed_loads.add(spec.mm_hash) + continue + if spec.local: + tensor = self._consumer_memory.take_resident( + spec.mm_hash, tuple(spec.shape), spec.dtype + ) + else: + try: + allocation = self._reservations.take(spec.transfer_id, spec.mm_hash) + tensor = allocation.tensor + except RuntimeError as e: + logger.warning("EC Mooncake pushed load failed: %s", e) + tensor = None + if tensor is None: + self._failed_loads.add(spec.mm_hash) + else: + encoder_cache[spec.mm_hash] = tensor + self._completed_loads.add(spec.mm_hash) + + def _push_batch(self, pushes: list[ProducerPushRecord]) -> None: + started_at = time.monotonic() + ready: list[tuple[ProducerPushRecord, dict[str, Any]]] = [] + written_pushes: dict[str, ProducerPushRecord] = {} + failure: Exception | None = None + try: + for push in pushes: + self._validate_push_source(push) + reservations = self._producer_pushes.resolve_reservations(push) + stale = [ + index + for index, shard in enumerate(reservations) + if not shard.get("ready", False) + and not shard.get("cancelled", False) + and time.monotonic() - float(shard.get("_received_at", started_at)) + >= _RESERVATION_REFRESH_SECONDS + ] + if stale: + reservations = self._refresh_remote_reservations( + push.spec, reservations, push + ) + self._producer_pushes.replace_reservations(push, reservations) + self._producer_pushes.begin_writing(push) + writable = [ + shard + for shard in reservations + if not shard.get("cached", False) + and not shard.get("cancelled", False) + and shard.get("write", True) + ] + source = push.source_tensor + assert source is not None + for shard in writable: + if int(shard["nbytes"]) != source.nbytes: + raise RuntimeError( + "Reserved EC size does not match tensor for " + f"mm_hash={push.spec.mm_hash}" + ) + ready.append((push, shard)) + written_pushes.setdefault(push.spec.transfer_id, push) + if ready: + # Stage each source once, then write it to every destination. + source_index = { + push.spec.transfer_id: index + for index, push in enumerate(written_pushes.values()) + } + tensors = [ + cast(torch.Tensor, push.source_tensor) + for push in written_pushes.values() + if push.source_tensor is not None + ] + lengths = [tensor.nbytes for tensor in tensors] + staged = self._producer_memory.stage(tensors) + registered_sources: list[int] = [] + if staged is not None: + sources = staged.tensors + else: + sources = tensors + registered_sources = self._transfer.acquire_sources(tensors) + addresses = [tensor.data_ptr() for tensor in sources] + try: + by_session: dict[str, list[tuple[int, int]]] = {} + session_records: dict[str, dict[str, ProducerPushRecord]] = {} + for push, shard in ready: + session = str(shard["dst_session"]) + by_session.setdefault(session, []).append( + (source_index[push.spec.transfer_id], int(shard["dst_ptr"])) + ) + session_records.setdefault(session, {})[ + push.spec.transfer_id + ] = push + + def write(session: str, items: list[tuple[int, int]]) -> None: + self._transfer.write( + session, + [addresses[index] for index, _ in items], + [dst for _, dst in items], + [lengths[index] for index, _ in items], + ) + + sessions = list(by_session.items()) + + # Write shards concurrently to avoid serial TP latency. + def track_write(index: int, future: Future[None]) -> None: + session = sessions[index][0] + self._producer_pushes.track_shard_futures( + list(session_records[session].values()), [future] + ) + + writes: list[Callable[[], None]] = [ + partial(write, *session) for session in sessions + ] + self._run_fanout(writes, track_write) + finally: + if staged is not None: + self._producer_memory.release(staged) + self._transfer.release_sources(registered_sources) + + self._producer_pushes.begin_notifying(pushes) + self._notify_completions(ready) + self._producer_pushes.complete(pushes) + except Exception as exc: + # Report asynchronously; raising here would fail EngineCore. + failure = exc + logger.exception( + "EC Mooncake push batch failed for mm_hashes=%s", + [push.spec.mm_hash for push in pushes], + ) + self._producer_pushes.settle_all(pushes) + self._abandon_pushes(pushes) + finally: + if failure is not None: + self._producer_pushes.fail(pushes, failure) + + def _notify_completions( + self, notifications: list[tuple[ProducerPushRecord, dict[str, Any]]] + ) -> None: + """Tell the consumer, in one message per destination, what landed.""" + if not notifications: + return + by_destination: dict[str, list[tuple[ProducerPushRecord, dict[str, Any]]]] = {} + for push, reservation in notifications: + by_destination.setdefault( + str(reservation.get("addr", push.spec.consumer_zmq)), [] + ).append((push, reservation)) + destinations = list(by_destination.items()) + + def notify( + consumer_zmq: str, + items: list[tuple[ProducerPushRecord, dict[str, Any]]], + ) -> None: + result = self._control_client.request( + consumer_zmq, + { + "op": "complete_batch", + "items": [ + { + "transfer_id": push.spec.transfer_id, + "reservation_id": reservation["reservation_id"], + } + for push, reservation in items + ], + }, + ) + completions = result.get("items", []) if isinstance(result, dict) else [] + if len(completions) != len(items): + raise RuntimeError("Malformed EC completion response") + for (push, _), completion in zip(items, completions): + if not completion.get("completed"): + raise RuntimeError( + f"Unknown EC reservation for mm_hash={push.spec.mm_hash}" + ) + + def track(index: int, future: Future[None]) -> None: + records = { + push.spec.transfer_id: push for push, _ in destinations[index][1] + } + self._producer_pushes.track_shard_futures(list(records.values()), [future]) + + self._run_fanout( + [ + partial(notify, destination, items) + for destination, items in destinations + ], + track, + ) + + @staticmethod + def _known_reservations(record: ProducerPushRecord) -> list[dict[str, Any]]: + if record.reservations: + return list(record.reservations) + try: + return list(record.reservation_future.result()) + except _FanoutError as exc: + return [result for result in exc.results if isinstance(result, dict)] + except Exception: + return [] + + def _abandon_pushes(self, pushes: list[ProducerPushRecord]) -> None: + """Release the consumer-side reservations of a batch that failed.""" + for push in pushes: + shards = self._known_reservations(push) + if not shards: + # No confirmed topology means there is no safe address to + # cancel. The original push failure remains observable via + # ProducerPushManager.fail below. + continue + try: + self._retry_cancel_reservations(push.spec, shards, record=push) + except _FanoutError: + logger.exception( + "Failed to abandon EC reservations for transfer_id=%s", + push.spec.transfer_id, + ) + + def _flush_pending_pushes(self) -> None: + self._producer_pushes.submit_batches( + self._io_executor, + self._push_batch, + ) + + def _bind_push_source(self, tensor: torch.Tensor, mm_hash: str) -> None: + ready_event = None + if tensor.device.type == "cuda": + ready_event = torch.Event() + ready_event.record(torch.accelerator.current_stream(tensor.device)) + self._producer_pushes.bind_source(mm_hash, tensor, ready_event) + + def _cancel_orphaned_reservation(self, record: ProducerPushRecord) -> None: + resolution_error: Exception | None = None + try: + reservations = self._producer_pushes.resolve_reservations(record) + except Exception as exc: + resolution_error = exc + reservations = self._known_reservations(record) + reservations = [ + shard for shard in reservations if not shard.get("cancelled", False) + ] + if not reservations: + if resolution_error is not None: + self._producer_pushes.fail([record], resolution_error) + else: + self._producer_pushes.finish_cancel(record) + return + try: + self._retry_cancel_reservations(record.spec, reservations, record=record) + except Exception as cleanup_error: + if resolution_error is not None: + combined_error = RuntimeError( + f"EC reservation resolution failed ({resolution_error}); " + f"cleanup also failed ({cleanup_error})" + ) + combined_error.__cause__ = resolution_error + cleanup_error = combined_error + self._producer_pushes.fail([record], cleanup_error) + return + if resolution_error is not None: + self._producer_pushes.fail([record], resolution_error) + else: + self._producer_pushes.finish_cancel(record) + + def get_finished( + self, finished_req_ids: set[str] + ) -> tuple[set[str] | None, set[str] | None]: + if not self.is_producer: + return None, None + + for record in self._producer_pushes.cancel_requests(finished_req_ids): + self._producer_pushes.submit_cancel( + record, + self._io_executor, + self._cancel_orphaned_reservation, + ) + return None, None + + def save_caches( + self, encoder_cache: dict[str, torch.Tensor], mm_hash: str, **kwargs: Any + ) -> None: + if not self.is_producer: + return + tensor = encoder_cache[mm_hash] + self._bind_push_source(tensor, mm_hash) + + def build_connector_worker_meta(self) -> ECMooncakeWorkerMetadata | None: + if self.is_consumer and not self._is_receiving_rank: + # `loaded` is intersected across reporting ranks, so a stage that + # never loads must not report at all rather than report nothing. + return None + + self._collecting_sources = False + self._push_ready.set() + self._flush_pending_pushes() + failures = self._producer_pushes.poll() + for mm_hash, error in failures: + logger.error( + "EC Mooncake async save failed for mm_hash=%s: %s", + mm_hash, + error, + ) + reclaimed = self._consumer_memory.drain_reclaimed() + meta = ECMooncakeWorkerMetadata( + loaded=self._completed_loads, + failed_loads=self._failed_loads, + reclaimed=reclaimed, + pending_saves=self._producer_pushes.pending, + failed_saves=self._failed_saves, + ) + self._failed_saves = set() + self._completed_loads = set() + self._failed_loads = set() + return meta + + def close(self) -> None: + if self._shutdown: + return + self._shutdown = True + dispatcher = getattr(self, "_dispatcher", None) + if dispatcher is not None: + self._dispatch_stop.set() + self._push_ready.set() + dispatcher.join() + if self._control_server is not None: + self._reservations.begin_shutdown() + self._flush_pending_pushes() + for record in self._producer_pushes.cancel_requests(None): + self._producer_pushes.submit_cancel( + record, + self._io_executor, + self._cancel_orphaned_reservation, + ) + self._control_executor.shutdown(wait=True) + self._producer_pushes.submit_batches( + self._io_executor, self._push_batch, wait=True + ) + self._io_executor.shutdown(wait=True) + if self._shard_pool is not None: + self._shard_pool.shutdown(wait=True) + # Every producer-side thread that could hold a control socket is stopped. + self._control_client.close() + drained = True + if self._control_server is not None: + drained = self._reservations.wait_for_writers( + self._shutdown_drain_timeout_s + ) + self._control_server.close() + if drained: + self._consumer_memory.close() + else: + logger.error( + "Timed out waiting for Mooncake EC writers; keeping the consumer " + "receive pool registered" + ) + self._producer_memory.close() + self._transfer.close() diff --git a/vllm/distributed/ec_transfer/ec_connector/mooncake_ec_connector.py b/vllm/distributed/ec_transfer/ec_connector/mooncake_ec_connector.py new file mode 100644 index 0000000000..2949742220 --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/mooncake_ec_connector.py @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Encoder-cache connector backed by Mooncake TransferEngine.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from vllm.distributed.ec_transfer.ec_connector.base import ( + ECConnectorBase, + ECConnectorMetadata, + ECConnectorRole, + ECConnectorWorkerMetadata, +) +from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import ( + ECMooncakeConnectorMetadata, +) +from vllm.distributed.ec_transfer.ec_connector.mooncake.scheduler import ( + ECMooncakeScheduler, +) +from vllm.distributed.ec_transfer.ec_connector.mooncake.worker import ECMooncakeWorker + +if TYPE_CHECKING: + import torch + + from vllm.config import VllmConfig + from vllm.v1.core.sched.output import SchedulerOutput + from vllm.v1.outputs import ECConnectorOutput + from vllm.v1.request import Request + + +class ECMooncakeConnector(ECConnectorBase): + """Preserve the public API while delegating to one process-role component.""" + + def __init__(self, vllm_config: VllmConfig, role: ECConnectorRole): + super().__init__(vllm_config=vllm_config, role=role) + + self._scheduler: ECMooncakeScheduler | None = None + self._worker: ECMooncakeWorker | None = None + self._closed = False + + if role == ECConnectorRole.SCHEDULER: + self._scheduler = ECMooncakeScheduler(vllm_config) + elif role == ECConnectorRole.WORKER: + self._worker = ECMooncakeWorker(vllm_config) + else: + raise ValueError(f"Unknown EC connector role: {role}") + + def start_worker_services(self) -> None: + assert self._worker is not None + self._worker.start_services() + + def start_save_caches(self, **kwargs: Any) -> None: + assert self._worker is not None + metadata = self._get_connector_metadata() + assert isinstance(metadata, ECMooncakeConnectorMetadata) + self._worker.start_save_caches(metadata, **kwargs) + + def start_load_caches( + self, encoder_cache: dict[str, torch.Tensor], **kwargs: Any + ) -> None: + assert self._worker is not None + metadata = self._get_connector_metadata() + assert isinstance(metadata, ECMooncakeConnectorMetadata) + self._worker.start_load_caches(metadata, encoder_cache, **kwargs) + + def save_caches( + self, encoder_cache: dict[str, torch.Tensor], mm_hash: str, **kwargs: Any + ) -> None: + assert self._worker is not None + self._worker.save_caches(encoder_cache, mm_hash, **kwargs) + + def get_finished( + self, finished_req_ids: set[str] + ) -> tuple[set[str] | None, set[str] | None]: + assert self._worker is not None + return self._worker.get_finished(finished_req_ids) + + def build_connector_worker_meta(self) -> ECConnectorWorkerMetadata | None: + assert self._worker is not None + return self._worker.build_connector_worker_meta() + + def take_unavailable_requests(self) -> set[str]: + assert self._scheduler is not None + return self._scheduler.take_unavailable_requests() + + def has_cache_item(self, identifier: str) -> bool: + assert self._scheduler is not None + return self._scheduler.has_cache_item(identifier) + + def ensure_cache_available( + self, request: Request, num_computed_tokens: int + ) -> bool: + assert self._scheduler is not None + return self._scheduler.ensure_cache_available(request, num_computed_tokens) + + def update_state_after_alloc(self, request: Request, index: int) -> None: + assert self._scheduler is not None + self._scheduler.update_state_after_alloc(request, index) + + def update_state_after_free(self, request: Request, index: int) -> None: + assert self._scheduler is not None + self._scheduler.update_state_after_free(request, index) + + def build_connector_meta( + self, scheduler_output: SchedulerOutput + ) -> ECConnectorMetadata: + assert self._scheduler is not None + return self._scheduler.build_connector_meta(scheduler_output) + + def update_connector_output(self, connector_output: ECConnectorOutput) -> None: + assert self._scheduler is not None + self._scheduler.update_connector_output(connector_output) + + def has_pending_push_work(self) -> bool: + assert self._scheduler is not None + return self._scheduler.has_pending_push_work() + + def request_finished(self, request: Request) -> tuple[bool, dict[str, Any] | None]: + assert self._scheduler is not None + return self._scheduler.request_finished(request) + + def shutdown(self) -> None: + if self._closed: + return + self._closed = True + if self._scheduler is not None: + self._scheduler.close() + if self._worker is not None: + self._worker.close() diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 08cd21c099..b59fe6a9ed 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -616,6 +616,17 @@ class Scheduler(SchedulerInterface): req_index += 1 continue + if ( + self.ec_connector is not None + and request.mm_features + and not self.ec_connector.ensure_cache_available( + request, + request.num_computed_tokens - request.num_output_placeholders, + ) + ): + req_index += 1 + continue + num_new_tokens = ( request.num_tokens_with_spec + request.num_output_placeholders @@ -945,13 +956,7 @@ class Scheduler(SchedulerInterface): assert num_computed_tokens <= request.num_tokens # Skip request with pending mm encoding prefetches - if ( - self.ec_connector is not None - and request.mm_features - and not self.ec_connector.ensure_cache_available( - request, num_computed_tokens - ) - ): + if self._ec_transfer_pending(request, num_computed_tokens): request_queue.pop_request() step_skipped_waiting.prepend_request(request) continue @@ -966,11 +971,18 @@ class Scheduler(SchedulerInterface): ) else: # KVTransfer: WAITING reqs have num_computed_tokens > 0 - # after async KV recvs are completed. + # after async KV recvs are completed. A streaming-input + # session resumes here too, carrying whatever media its + # latest chunk added, so this branch needs the same gate. new_computed_blocks = self.kv_cache_manager.empty_kv_cache_blocks num_new_local_computed_tokens = 0 num_computed_tokens = request.num_computed_tokens + if self._ec_transfer_pending(request, num_computed_tokens): + request_queue.pop_request() + step_skipped_waiting.prepend_request(request) + continue + encoder_inputs_to_schedule = None external_load_encoder_input = [] new_encoder_compute_budget = encoder_compute_budget @@ -2162,6 +2174,10 @@ class Scheduler(SchedulerInterface): self.grammar_compile_error_reqs.clear() if failed_kv_load_req_ids and not self.recompute_kv_load_failures: error_req_ids.update(failed_kv_load_req_ids) + if self.ec_connector is not None: + # An encoder input the connector can no longer obtain. Failing is + # retryable: re-issuing the request re-runs the encode. + error_req_ids.update(self.ec_connector.take_unavailable_requests()) if error_req_ids: error_reqs = self.finish_requests( @@ -2258,6 +2274,16 @@ class Scheduler(SchedulerInterface): return engine_core_outputs + def _ec_transfer_pending(self, request: Request, num_computed_tokens: int) -> bool: + """Whether an encoder input this request needs is still in transit.""" + return ( + self.ec_connector is not None + and bool(request.mm_features) + and not self.ec_connector.ensure_cache_available( + request, num_computed_tokens + ) + ) + @staticmethod def _is_blocked_waiting_status(status: RequestStatus) -> bool: return status in ( @@ -2344,7 +2370,7 @@ class Scheduler(SchedulerInterface): # With Whisper, as soon as we've generated a single token, # we know we're done with the encoder input. Cross Attention # KVs have been calculated and cached already. - self.encoder_cache_manager.free_encoder_input(request, input_id) + self._free_encoder_input(request, input_id) elif ( start_pos + num_tokens + spec_lookahead <= request.num_computed_tokens - request.num_output_placeholders @@ -2352,7 +2378,12 @@ class Scheduler(SchedulerInterface): # Processed, stored in the decoder KV cache, and far enough past # the placeholder range (plus the drafter's look-ahead) that no # rejection or drafter gather can reference it. - self.encoder_cache_manager.free_encoder_input(request, input_id) + self._free_encoder_input(request, input_id) + + def _free_encoder_input(self, request: Request, input_id: int) -> None: + self.encoder_cache_manager.free_encoder_input(request, input_id) + if self.ec_connector is not None: + self.ec_connector.update_state_after_free(request, input_id) def update_draft_token_ids(self, draft_token_ids: DraftTokenIds) -> None: for req_id, spec_token_ids in zip( diff --git a/vllm/v1/worker/ec_connector_model_runner_mixin.py b/vllm/v1/worker/ec_connector_model_runner_mixin.py index b3430a8d94..be676666e8 100644 --- a/vllm/v1/worker/ec_connector_model_runner_mixin.py +++ b/vllm/v1/worker/ec_connector_model_runner_mixin.py @@ -64,6 +64,12 @@ class ECConnectorModelRunnerMixin: assert scheduler_output.ec_connector_metadata is not None ec_connector.bind_connector_metadata(scheduler_output.ec_connector_metadata) + if ec_connector.is_producer: + # Pass the cache explicitly: a producer needs it to publish an item + # that is already cached from an earlier step, which this step will + # not re-encode -- `save_caches` never fires for those. + ec_connector.start_save_caches(encoder_cache=encoder_cache, **kwargs) + # Load caches for consumer or both roles if ec_connector.is_consumer: ec_connector.start_load_caches(encoder_cache, **kwargs) @@ -74,5 +80,6 @@ class ECConnectorModelRunnerMixin: output.finished_sending, output.finished_recving = ( ec_connector.get_finished(scheduler_output.finished_req_ids) ) + output.ec_connector_worker_meta = ec_connector.build_connector_worker_meta() ec_connector.clear_connector_metadata() diff --git a/vllm/v1/worker/gpu/ec_connector.py b/vllm/v1/worker/gpu/ec_connector.py index 5dc8d92359..1285d9fd14 100644 --- a/vllm/v1/worker/gpu/ec_connector.py +++ b/vllm/v1/worker/gpu/ec_connector.py @@ -62,6 +62,9 @@ class ActiveECConnector(ECConnector): assert scheduler_output.ec_connector_metadata is not None ec_connector.bind_connector_metadata(scheduler_output.ec_connector_metadata) + if ec_connector.is_producer: + ec_connector.start_save_caches(encoder_cache=self.encoder_cache) + if ec_connector.is_consumer: ec_connector.start_load_caches(self.encoder_cache) diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index e832e4acaf..53bc0550a5 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -28,6 +28,8 @@ from vllm.distributed import ( from vllm.distributed.ec_transfer import ( ensure_ec_transfer_initialized, ensure_ec_transfer_shutdown, + get_ec_transfer, + has_ec_transfer, ) from vllm.distributed.eplb.eplb_utils import override_envs_for_eplb from vllm.distributed.kv_transfer import ( @@ -501,6 +503,9 @@ class Worker(WorkerBase): ): self.model_runner.load_model(load_dummy_weights=load_dummy_weights) + if has_ec_transfer(): + get_ec_transfer().start_worker_services() + if self.vllm_config.weight_transfer_config is not None: self.weight_transfer_engine = WeightTransferEngineFactory.create_engine( self.vllm_config.weight_transfer_config,