[EPD] Add ECMooncakeConnector for encoder cache over Mooncake TransferEngine (#41567)
Signed-off-by: Teng Ma <sima.mt@alibaba-inc.com> Signed-off-by: Tianyu Guo <guoty@inferact.ai> Signed-off-by: Zhou ziheng <jiaranran2@gmail.com> Signed-off-by: jiangkuaixue123 <jiangxiaozhou111@163.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Tianyu Guo <guoty@inferact.ai> Co-authored-by: Zhou ziheng <jiaranran2@gmail.com> Co-authored-by: jiangkuaixue123 <jiangxiaozhou111@163.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Cursor
Tianyu Guo
Zhou ziheng
jiangkuaixue123
OpenAI Codex
parent
f7f060d253
commit
6fbb00b188
@@ -23,6 +23,7 @@ import argparse
|
||||
import asyncio
|
||||
import hashlib
|
||||
import io
|
||||
import itertools
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -30,6 +31,7 @@ import random
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
import pybase64 as base64
|
||||
@@ -41,9 +43,7 @@ from fastapi.responses import JSONResponse, StreamingResponse
|
||||
# FastAPI app & global state
|
||||
###############################################################################
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG, format="%(asctime)s %(levelname)s: %(message)s"
|
||||
)
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s: %(message)s")
|
||||
logger = logging.getLogger("proxy")
|
||||
|
||||
app = FastAPI()
|
||||
@@ -87,9 +87,24 @@ def encoder_rr_assignment(
|
||||
return urls, next_start
|
||||
|
||||
|
||||
def validate_ec_consumer_routing(
|
||||
prefill_urls: list[str], consumer_addrs: list[str]
|
||||
) -> None:
|
||||
"""Reject the topology whose EC destination cannot be routed safely."""
|
||||
if prefill_urls and consumer_addrs:
|
||||
raise ValueError(
|
||||
"Mooncake EC consumer routing supports E+PD only; disable independent "
|
||||
"prefill or omit --ec-consumer-zmq-addrs."
|
||||
)
|
||||
|
||||
|
||||
# Diagnostic switch: forward the original request to the decoder so the
|
||||
# only difference from the rewrite path is the rewrite itself.
|
||||
NO_REWRITE = False
|
||||
# Decode-side retries for a retryable internal error (`finish_reason="error"`,
|
||||
# e.g. an encoder embedding the connector could not deliver). Re-issuing runs
|
||||
# the encode again, which produces a fresh transfer.
|
||||
DECODE_RETRIES = 1
|
||||
|
||||
|
||||
# Grid metadata reported by the encoder instance, keyed by item index.
|
||||
@@ -136,6 +151,7 @@ def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict:
|
||||
here -- a second derivation could disagree with the encoder's.
|
||||
"""
|
||||
rewritten = 0
|
||||
transfer_items = []
|
||||
idx = 0
|
||||
new_messages = []
|
||||
for msg in req_data.get("messages", []):
|
||||
@@ -151,26 +167,42 @@ def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict:
|
||||
meta = dict(item_meta.get(idx) or {})
|
||||
idx += 1
|
||||
item_uuid = meta.pop("mm_hash", None)
|
||||
ec_mm_hash = meta.pop("ec_mm_hash", None) or item_uuid
|
||||
transfer_id = meta.pop("transfer_id", None)
|
||||
# Whatever keys the encoder reported are the metadata its model
|
||||
# declared as needed to size the placeholder range; the proxy does
|
||||
# not need to know their names.
|
||||
metadata = {k: _b64_tensor(v) for k, v in meta.items()}
|
||||
if not metadata or not item_uuid:
|
||||
# The encoder reported no metadata (e.g. the item came from its
|
||||
# processor cache); let the decoder process the media itself.
|
||||
# Nothing to size the placeholder range with. A processor cache
|
||||
# hit is not a cause on its own: with the default `lru` type the
|
||||
# engine restores the item before the scheduler reports it. It
|
||||
# goes missing when the encode request failed, or under
|
||||
# `--mm-processor-cache-type shm`, where a hit replaces the item
|
||||
# with its shared-memory address and only the worker restores
|
||||
# it. Send the media so the decoder can derive the grid itself.
|
||||
new_content.append(item)
|
||||
continue
|
||||
embeds_type = EMBEDS_TYPES[item["type"]]
|
||||
new_content.append(
|
||||
{"type": embeds_type, embeds_type: metadata, "uuid": item_uuid}
|
||||
)
|
||||
if transfer_id is not None:
|
||||
transfer_items.append(
|
||||
{"mm_hash": ec_mm_hash, "transfer_id": transfer_id}
|
||||
)
|
||||
rewritten += 1
|
||||
new_messages.append({**msg, "content": new_content})
|
||||
|
||||
if not rewritten:
|
||||
return req_data
|
||||
logger.info("Rewrote %d media item(s) as metadata references", rewritten)
|
||||
return {**req_data, "messages": new_messages}
|
||||
rewritten_request = {**req_data, "messages": new_messages}
|
||||
if transfer_items:
|
||||
ec_transfer_params = dict(req_data.get("ec_transfer_params") or {})
|
||||
ec_transfer_params["ec_items"] = transfer_items
|
||||
rewritten_request["ec_transfer_params"] = ec_transfer_params
|
||||
return rewritten_request
|
||||
|
||||
|
||||
def extract_mm_items(request_data: dict) -> list[dict]:
|
||||
@@ -196,7 +228,8 @@ async def fanout_encoder_primer(
|
||||
orig_request: dict,
|
||||
e_urls: list[str],
|
||||
req_id: str,
|
||||
) -> dict[int, dict]:
|
||||
consumer_zmq: str | None = None,
|
||||
) -> tuple[dict[int, dict], dict[str, Any]]:
|
||||
"""
|
||||
1. Build one request *per MM item* with all text removed.
|
||||
2. Send them concurrently to the encode cluster.
|
||||
@@ -206,19 +239,25 @@ async def fanout_encoder_primer(
|
||||
`ec_transfer_params`: its EC cache key and the grid its processor produced.
|
||||
The proxy still supplies the uuid so both sides key the cache the same way;
|
||||
the grid can only come from the encoder, which is the side that computed it.
|
||||
|
||||
Also returns the connector handles to put on the decode body, as a fresh
|
||||
mapping. `orig_request` is left untouched so a retry re-encodes from the
|
||||
original request instead of carrying the previous attempt's handles.
|
||||
"""
|
||||
logger.info("[%s] Processing multimodal items...", req_id)
|
||||
|
||||
mm_items = extract_mm_items(orig_request)
|
||||
if not mm_items:
|
||||
logger.info("[%s] No multimodal items, skipping encoder", req_id)
|
||||
return {} # nothing to do
|
||||
return {}, {} # nothing to do
|
||||
|
||||
logger.info("[%s] got %d multimodal items...", req_id, len(mm_items))
|
||||
|
||||
tasks = []
|
||||
item_uuids: dict[int, str] = {}
|
||||
item_transfer_ids: dict[int, str] = {}
|
||||
item_meta: dict[int, dict] = {}
|
||||
ec_params: dict[str, Any] = {}
|
||||
|
||||
# Round-robin over encode servers to distribute load a bit. The cursor
|
||||
# persists across requests so fan-out doesn't restart at e_urls[0] every
|
||||
@@ -241,6 +280,8 @@ async def fanout_encoder_primer(
|
||||
item_uuid = None if NO_REWRITE else content_uuid(item)
|
||||
if item_uuid is not None:
|
||||
item_uuids[idx] = item_uuid
|
||||
transfer_id = uuid.uuid4().hex
|
||||
item_transfer_ids[idx] = transfer_id
|
||||
|
||||
encoder_req = {
|
||||
# You *may* need to keep additional fields
|
||||
@@ -257,6 +298,19 @@ async def fanout_encoder_primer(
|
||||
# once the prompt is encoded and its embeddings are published.
|
||||
"stream": False,
|
||||
}
|
||||
if consumer_zmq is not None:
|
||||
# No mm_hash here on purpose. The encoder's own
|
||||
# `mm_features[i].identifier` is derived from the uuid *and* the
|
||||
# engine's media_io_kwargs / mm_processor_kwargs, so this proxy
|
||||
# cannot know it before the encoder runs. Sending the bare uuid
|
||||
# would fail the connector's hash match and make the producer
|
||||
# invent its own transfer id, which the consumer could then never
|
||||
# cancel. Omitting it lets the connector match by position, which
|
||||
# is exact: one encoder request carries exactly one item.
|
||||
encoder_req["ec_transfer_params"] = {
|
||||
"consumer_zmq": consumer_zmq,
|
||||
"ec_items": [{"transfer_id": transfer_id}],
|
||||
}
|
||||
tasks.append(
|
||||
encode_session.post(
|
||||
f"{target_url}/v1/chat/completions",
|
||||
@@ -304,24 +358,42 @@ async def fanout_encoder_primer(
|
||||
except Exception:
|
||||
logger.warning("[%s] Could not read encoder metadata #%d", req_id, idx)
|
||||
params = {}
|
||||
if idx in item_uuids:
|
||||
reported = params.get(item_uuids[idx])
|
||||
if params:
|
||||
# One encoder request carries exactly one item, so there is a
|
||||
# single reported entry. Do not key it by this proxy's uuid: when
|
||||
# media_io_kwargs or mm_processor_kwargs are set the engine
|
||||
# re-hashes the uuid together with them, so the encoder's own
|
||||
# `mm_features[i].identifier` is a derived value this proxy cannot
|
||||
# predict. Fall back to the sole entry, and carry the key the
|
||||
# encoder actually used through as `ec_mm_hash`.
|
||||
ec_mm_hash = item_uuids.get(idx)
|
||||
reported = params.get(ec_mm_hash)
|
||||
if reported is None and len(params) == 1:
|
||||
((ec_mm_hash, reported),) = params.items()
|
||||
if reported:
|
||||
metadata = reported.get("metadata") or {}
|
||||
if metadata:
|
||||
item_meta[idx] = {**metadata, "mm_hash": item_uuids[idx]}
|
||||
item_meta[idx] = {
|
||||
**metadata,
|
||||
"mm_hash": item_uuids.get(idx, ec_mm_hash),
|
||||
"ec_mm_hash": ec_mm_hash,
|
||||
}
|
||||
if idx in item_transfer_ids:
|
||||
item_meta[idx]["transfer_id"] = item_transfer_ids[idx]
|
||||
# Whatever the encoder reported alongside `metadata` is the
|
||||
# connector's own handle on the published embedding (for NIXL,
|
||||
# peer_host/peer_port/size_bytes). The decoder's connector
|
||||
# looks it up by mm_hash on the request, so carry it through.
|
||||
orig_request.setdefault("ec_transfer_params", {})[item_uuids[idx]] = (
|
||||
reported
|
||||
)
|
||||
ec_params[item_uuids.get(idx, ec_mm_hash)] = reported
|
||||
if NO_REWRITE and consumer_zmq is not None:
|
||||
ec_params.setdefault("ec_items", []).append(
|
||||
{"mm_hash": ec_mm_hash, "transfer_id": item_transfer_ids[idx]}
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"[%s] All %d encoder requests completed successfully", req_id, len(mm_items)
|
||||
)
|
||||
return item_meta
|
||||
return item_meta, ec_params
|
||||
|
||||
|
||||
async def maybe_prefill(
|
||||
@@ -331,8 +403,11 @@ async def maybe_prefill(
|
||||
) -> dict:
|
||||
"""
|
||||
- Do prefill-only task if p_url exist;
|
||||
- Return modified request data with kv transfer params (for nixl connector)
|
||||
- Return a new body carrying kv transfer params (for nixl connector)
|
||||
- Else, skip and return the original request data for decode
|
||||
|
||||
`req_data` is never mutated: a decode retry re-enters this function with the
|
||||
same body, and one attempt's `remote_block_ids` must not reach the next.
|
||||
"""
|
||||
if p_url:
|
||||
logger.info("[%s] Processing through prefill: %s", req_id, p_url)
|
||||
@@ -342,11 +417,9 @@ async def maybe_prefill(
|
||||
prefill_response_json = await prefill_response.json()
|
||||
kv_transfer_params = prefill_response_json.get("kv_transfer_params", {})
|
||||
if kv_transfer_params:
|
||||
req_data["kv_transfer_params"] = kv_transfer_params
|
||||
return {**req_data, "kv_transfer_params": kv_transfer_params}
|
||||
|
||||
return req_data
|
||||
else:
|
||||
return req_data
|
||||
return req_data
|
||||
|
||||
|
||||
async def process_prefill_stage(
|
||||
@@ -409,7 +482,6 @@ async def process_prefill_stage(
|
||||
###############################################################################
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def log_requests(request: Request, call_next):
|
||||
"""Middleware to log all incoming requests and responses"""
|
||||
req_id = request.headers.get("x-request-id", str(uuid.uuid4()))
|
||||
@@ -458,7 +530,20 @@ async def log_requests(request: Request, call_next):
|
||||
async def on_startup() -> None:
|
||||
global encode_session, prefill_session, decode_session
|
||||
timeout = aiohttp.ClientTimeout(total=100_000)
|
||||
connector = aiohttp.TCPConnector(limit=0, force_close=False)
|
||||
# vLLM closes an idle keep-alive connection after
|
||||
# VLLM_HTTP_TIMEOUT_KEEP_ALIVE seconds (5 by default), while aiohttp keeps
|
||||
# pooling it for 15. Reusing one it has already closed fails the request
|
||||
# with ServerDisconnectedError, and the server logs nothing at all: it
|
||||
# closed the socket before the request arrived. Retire ours first.
|
||||
server_keep_alive = float(os.getenv("VLLM_HTTP_TIMEOUT_KEEP_ALIVE", "5"))
|
||||
connector = aiohttp.TCPConnector(
|
||||
limit=0,
|
||||
**(
|
||||
{"keepalive_timeout": server_keep_alive / 2}
|
||||
if server_keep_alive > 0
|
||||
else {"force_close": True}
|
||||
),
|
||||
)
|
||||
encode_session = aiohttp.ClientSession(timeout=timeout, connector=connector)
|
||||
if app.state.p_urls:
|
||||
# only setup if prefill instance(s) exist
|
||||
@@ -482,40 +567,97 @@ async def on_shutdown() -> None:
|
||||
###############################################################################
|
||||
|
||||
|
||||
async def prepare_for_decode(
|
||||
req_data: dict,
|
||||
req_id: str,
|
||||
e_urls: list[str],
|
||||
p_url: str,
|
||||
consumer_zmq: str | None,
|
||||
) -> tuple[dict, float, float]:
|
||||
"""Encode, rewrite and prefill, returning the body to send to decode.
|
||||
|
||||
`req_data` is left untouched so a retry starts from the original media
|
||||
rather than from a body whose images are already metadata references.
|
||||
"""
|
||||
_t0 = time.perf_counter()
|
||||
item_meta, ec_params = await fanout_encoder_primer(
|
||||
req_data, e_urls, req_id, consumer_zmq
|
||||
)
|
||||
_t1 = time.perf_counter()
|
||||
prepared = req_data if NO_REWRITE else rewrite_for_decode(req_data, item_meta)
|
||||
if ec_params:
|
||||
# A fresh body every time: `rewrite_for_decode` hands back `req_data`
|
||||
# itself when it rewrote nothing, and this attempt's handles must not
|
||||
# outlive it into a retry.
|
||||
handles = dict(prepared.get("ec_transfer_params") or {})
|
||||
handles.update(ec_params)
|
||||
prepared = {**prepared, "ec_transfer_params": handles}
|
||||
_t2 = time.perf_counter()
|
||||
prepared = await maybe_prefill(prepared, p_url, req_id)
|
||||
return prepared, _t1 - _t0, _t2 - _t1
|
||||
|
||||
|
||||
async def forward_non_stream(
|
||||
req_data: dict, req_id: str, e_urls: list[str], p_url: str, d_url: str
|
||||
req_data: dict,
|
||||
req_id: str,
|
||||
e_urls: list[str],
|
||||
p_url: str,
|
||||
d_url: str,
|
||||
consumer_zmq: str | None,
|
||||
dp_rank: int | None = None,
|
||||
) -> dict:
|
||||
try:
|
||||
# Step 1: Process through Encoder instance (if has MM input)
|
||||
_t0 = time.perf_counter()
|
||||
item_meta = await fanout_encoder_primer(req_data, e_urls, req_id)
|
||||
_t1 = time.perf_counter()
|
||||
req_data = req_data if NO_REWRITE else rewrite_for_decode(req_data, item_meta)
|
||||
_t2 = time.perf_counter()
|
||||
|
||||
# Step 2: Process through Prefill instance
|
||||
req_data = await maybe_prefill(req_data, p_url, req_id)
|
||||
|
||||
# Step 3: Process through Decode instance
|
||||
logger.info("[%s] Forwarding to decode: %s", req_id, d_url)
|
||||
headers = {"x-request-id": req_id}
|
||||
|
||||
# Non-streaming response
|
||||
async with decode_session.post(
|
||||
f"{d_url}/v1/chat/completions", json=req_data, headers=headers
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
out = await resp.json()
|
||||
_t3 = time.perf_counter()
|
||||
logger.info(
|
||||
"STAGE %s encode=%.1f rewrite=%.1f decode=%.1f total=%.1f",
|
||||
"no-rewrite" if NO_REWRITE else "rewrite",
|
||||
(_t1 - _t0) * 1e3,
|
||||
(_t2 - _t1) * 1e3,
|
||||
(_t3 - _t2) * 1e3,
|
||||
(_t3 - _t0) * 1e3,
|
||||
for attempt in range(DECODE_RETRIES + 1):
|
||||
_t0 = time.perf_counter()
|
||||
prepared, encode_s, rewrite_s = await prepare_for_decode(
|
||||
req_data, req_id, e_urls, p_url, consumer_zmq
|
||||
)
|
||||
return out
|
||||
_t2 = time.perf_counter()
|
||||
|
||||
logger.info("[%s] Forwarding to decode: %s", req_id, d_url)
|
||||
headers = {"x-request-id": req_id}
|
||||
if dp_rank is not None:
|
||||
headers["X-data-parallel-rank"] = str(dp_rank)
|
||||
|
||||
async with decode_session.post(
|
||||
f"{d_url}/v1/chat/completions", json=prepared, headers=headers
|
||||
) as resp:
|
||||
if resp.status >= 400:
|
||||
detail = await resp.text()
|
||||
# 500 is the decoder's retryable internal error, which
|
||||
# includes an encoder embedding it could not obtain. Redoing
|
||||
# the encode publishes the item again.
|
||||
if resp.status == 500 and attempt < DECODE_RETRIES:
|
||||
logger.warning(
|
||||
"[%s] Decode returned 500, re-encoding and retrying "
|
||||
"(attempt %d/%d): %s",
|
||||
req_id,
|
||||
attempt + 1,
|
||||
DECODE_RETRIES,
|
||||
detail[:200],
|
||||
)
|
||||
continue
|
||||
logger.error(
|
||||
"[%s] Decode request returned status %s: %s",
|
||||
req_id,
|
||||
resp.status,
|
||||
detail,
|
||||
)
|
||||
raise HTTPException(status_code=resp.status, detail=detail)
|
||||
out = await resp.json()
|
||||
_t3 = time.perf_counter()
|
||||
logger.info(
|
||||
"STAGE %s encode=%.1f rewrite=%.1f decode=%.1f total=%.1f "
|
||||
"attempt=%d",
|
||||
"no-rewrite" if NO_REWRITE else "rewrite",
|
||||
encode_s * 1e3,
|
||||
rewrite_s * 1e3,
|
||||
(_t3 - _t2) * 1e3,
|
||||
(_t3 - _t0) * 1e3,
|
||||
attempt,
|
||||
)
|
||||
return out
|
||||
raise HTTPException(status_code=500, detail="Decode failed after re-encoding")
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
@@ -525,47 +667,66 @@ async def forward_non_stream(
|
||||
|
||||
|
||||
async def forward_stream(
|
||||
req_data: dict, req_id: str, e_urls: list[str], p_url: str, d_url: str
|
||||
req_data: dict,
|
||||
req_id: str,
|
||||
e_urls: list[str],
|
||||
p_url: str,
|
||||
d_url: str,
|
||||
consumer_zmq: str | None,
|
||||
dp_rank: int | None = None,
|
||||
) -> AsyncIterator[str]:
|
||||
try:
|
||||
# Step 1: Process through Encoder instance (if has MM input)
|
||||
_t0 = time.perf_counter()
|
||||
item_meta = await fanout_encoder_primer(req_data, e_urls, req_id)
|
||||
_t1 = time.perf_counter()
|
||||
req_data = req_data if NO_REWRITE else rewrite_for_decode(req_data, item_meta)
|
||||
_t2 = time.perf_counter()
|
||||
for attempt in range(DECODE_RETRIES + 1):
|
||||
_t0 = time.perf_counter()
|
||||
prepared, encode_s, rewrite_s = await prepare_for_decode(
|
||||
req_data, req_id, e_urls, p_url, consumer_zmq
|
||||
)
|
||||
_t2 = time.perf_counter()
|
||||
|
||||
# Step 2: Process through Prefill instance
|
||||
req_data = await maybe_prefill(req_data, p_url, req_id)
|
||||
logger.info("[%s] Starting streaming from decode: %s", req_id, d_url)
|
||||
headers = {"x-request-id": req_id}
|
||||
if dp_rank is not None:
|
||||
headers["X-data-parallel-rank"] = str(dp_rank)
|
||||
|
||||
# Step 3: Process through Decode instance
|
||||
logger.info("[%s] Starting streaming from decode: %s", req_id, d_url)
|
||||
headers = {"x-request-id": req_id}
|
||||
_first = None
|
||||
async with decode_session.post(
|
||||
f"{d_url}/v1/chat/completions",
|
||||
json=prepared,
|
||||
headers=headers,
|
||||
) as resp:
|
||||
# Retry only before the first chunk: once anything reached the
|
||||
# client the response cannot be replaced.
|
||||
if resp.status == 500 and attempt < DECODE_RETRIES:
|
||||
detail = await resp.text()
|
||||
logger.warning(
|
||||
"[%s] Decode returned 500 before streaming, re-encoding "
|
||||
"and retrying (attempt %d/%d): %s",
|
||||
req_id,
|
||||
attempt + 1,
|
||||
DECODE_RETRIES,
|
||||
detail[:200],
|
||||
)
|
||||
continue
|
||||
resp.raise_for_status()
|
||||
async for chunk in resp.content.iter_chunked(1024):
|
||||
if chunk:
|
||||
if _first is None:
|
||||
_first = time.perf_counter()
|
||||
yield chunk.decode("utf-8", errors="ignore")
|
||||
_t3 = time.perf_counter()
|
||||
|
||||
# Streaming response
|
||||
_first = None
|
||||
async with decode_session.post(
|
||||
f"{d_url}/v1/chat/completions",
|
||||
json=req_data,
|
||||
headers=headers,
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
async for chunk in resp.content.iter_chunked(1024):
|
||||
if chunk:
|
||||
if _first is None:
|
||||
_first = time.perf_counter()
|
||||
yield chunk.decode("utf-8", errors="ignore")
|
||||
_t3 = time.perf_counter()
|
||||
|
||||
logger.info(
|
||||
"STAGE %s encode=%.1f rewrite=%.2f decode_ttfb=%.1f decode_total=%.1f",
|
||||
"no-rewrite" if NO_REWRITE else "rewrite",
|
||||
(_t1 - _t0) * 1e3,
|
||||
(_t2 - _t1) * 1e3,
|
||||
((_first or _t3) - _t2) * 1e3,
|
||||
(_t3 - _t2) * 1e3,
|
||||
)
|
||||
logger.info("[%s] Streaming completed", req_id)
|
||||
logger.info(
|
||||
"STAGE %s encode=%.1f rewrite=%.2f decode_ttfb=%.1f "
|
||||
"decode_total=%.1f attempt=%d",
|
||||
"no-rewrite" if NO_REWRITE else "rewrite",
|
||||
encode_s * 1e3,
|
||||
rewrite_s * 1e3,
|
||||
((_first or _t3) - _t2) * 1e3,
|
||||
(_t3 - _t2) * 1e3,
|
||||
attempt,
|
||||
)
|
||||
logger.info("[%s] Streaming completed", req_id)
|
||||
return
|
||||
|
||||
except HTTPException:
|
||||
logger.exception("[%s] HTTPException in forward_stream", req_id)
|
||||
@@ -590,16 +751,29 @@ async def chat_completions(request: Request):
|
||||
|
||||
e_urls = app.state.e_urls # we want the full list for fan-out
|
||||
p_url = random.choice(app.state.p_urls) if app.state.p_urls else None
|
||||
d_url = random.choice(app.state.d_urls)
|
||||
decode_index = random.randrange(len(app.state.d_urls))
|
||||
d_url = app.state.d_urls[decode_index]
|
||||
dp_size = app.state.ec_consumer_dp_size
|
||||
# Round-robin the replica, then name it to both halves: the decoder
|
||||
# honours the rank header instead of its own balancer, and the encoder
|
||||
# pushes to that replica's control channel. Choosing once here means a
|
||||
# decode retry re-encodes to the same replica.
|
||||
dp_rank = next(app.state.replica_counter) % dp_size if dp_size > 1 else None
|
||||
ec_index = decode_index * dp_size + (dp_rank or 0)
|
||||
consumer_zmq = app.state.d_ec_urls[ec_index] if app.state.d_ec_urls else None
|
||||
|
||||
is_streaming = req_data.get("stream", False)
|
||||
|
||||
if is_streaming:
|
||||
return StreamingResponse(
|
||||
forward_stream(req_data, req_id, e_urls, p_url, d_url),
|
||||
forward_stream(
|
||||
req_data, req_id, e_urls, p_url, d_url, consumer_zmq, dp_rank
|
||||
),
|
||||
media_type="text/event-stream",
|
||||
)
|
||||
result = await forward_non_stream(req_data, req_id, e_urls, p_url, d_url)
|
||||
result = await forward_non_stream(
|
||||
req_data, req_id, e_urls, p_url, d_url, consumer_zmq, dp_rank
|
||||
)
|
||||
return JSONResponse(content=result)
|
||||
|
||||
except HTTPException:
|
||||
@@ -750,6 +924,14 @@ if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--host", default="0.0.0.0")
|
||||
parser.add_argument("--port", type=int, default=8000)
|
||||
parser.add_argument(
|
||||
"--log-requests",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Log every request in and out, and raise the log level to DEBUG. "
|
||||
"Off by default: the proxy is on the request path."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-rewrite",
|
||||
action="store_true",
|
||||
@@ -764,8 +946,8 @@ if __name__ == "__main__":
|
||||
"--prefill-servers-urls",
|
||||
required=True,
|
||||
help=(
|
||||
'Comma-separated prefill URLs ("http://p1:8003,http://p2:8004") ',
|
||||
'to enable E->P->D, set "disable" or "none" to enable E->PD',
|
||||
'Comma-separated prefill URLs ("http://p1:8003,http://p2:8004") '
|
||||
'to enable E->P->D, set "disable" or "none" to enable E->PD'
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
@@ -773,15 +955,64 @@ if __name__ == "__main__":
|
||||
required=True,
|
||||
help='Comma-separated decode URLs ("http://d1:8005,http://d2:8006")',
|
||||
)
|
||||
parser.add_argument(
|
||||
"--decode-retries",
|
||||
type=int,
|
||||
default=1,
|
||||
help=(
|
||||
"Re-encode and re-send when decode returns 500, which is its "
|
||||
"retryable internal error (an undeliverable encoder embedding "
|
||||
"among them). 0 disables."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ec-consumer-zmq-addrs",
|
||||
default="",
|
||||
help=(
|
||||
"Comma-separated Mooncake EC consumer control addresses, aligned "
|
||||
"with --decode-servers-urls. Required for Mooncake EC consumers and "
|
||||
"supported only in E+PD mode. With --ec-consumer-dp-size > 1, list "
|
||||
"each server's replicas consecutively: s0r0,s0r1,s1r0,s1r1."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ec-consumer-dp-size",
|
||||
type=int,
|
||||
default=1,
|
||||
help=(
|
||||
"Data-parallel replicas per EC consumer. The proxy picks a replica "
|
||||
"round-robin and names it to both halves of the request, because an "
|
||||
"encoder push has to land where the request will run."
|
||||
),
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
if args.log_requests:
|
||||
logging.getLogger().setLevel(logging.DEBUG)
|
||||
app.middleware("http")(log_requests)
|
||||
NO_REWRITE = args.no_rewrite
|
||||
DECODE_RETRIES = max(0, args.decode_retries)
|
||||
app.state.e_urls = [
|
||||
u.strip() for u in args.encode_servers_urls.split(",") if u.strip()
|
||||
]
|
||||
app.state.d_urls = [
|
||||
u.strip() for u in args.decode_servers_urls.split(",") if u.strip()
|
||||
]
|
||||
app.state.d_ec_urls = [
|
||||
u.strip() for u in args.ec_consumer_zmq_addrs.split(",") if u.strip()
|
||||
]
|
||||
if args.ec_consumer_dp_size < 1:
|
||||
parser.error("--ec-consumer-dp-size must be at least 1")
|
||||
app.state.ec_consumer_dp_size = args.ec_consumer_dp_size
|
||||
app.state.replica_counter = itertools.count()
|
||||
expected = len(app.state.d_urls) * args.ec_consumer_dp_size
|
||||
if app.state.d_ec_urls and len(app.state.d_ec_urls) != expected:
|
||||
parser.error(
|
||||
"--ec-consumer-zmq-addrs must contain one address per consumer "
|
||||
f"replica: expected {expected} "
|
||||
f"({len(app.state.d_urls)} servers x {args.ec_consumer_dp_size} replicas), "
|
||||
f"got {len(app.state.d_ec_urls)}"
|
||||
)
|
||||
# handle prefill instances
|
||||
if args.prefill_servers_urls.lower() in ("disable", "none", ""):
|
||||
app.state.p_urls = []
|
||||
@@ -793,11 +1024,21 @@ if __name__ == "__main__":
|
||||
u.strip() for u in args.prefill_servers_urls.split(",") if u.strip()
|
||||
]
|
||||
logger.info("Disaggregated prefill phase is enabled. Running E + P + D...")
|
||||
try:
|
||||
validate_ec_consumer_routing(app.state.p_urls, app.state.d_ec_urls)
|
||||
except ValueError as exc:
|
||||
parser.error(str(exc))
|
||||
|
||||
logger.info("Proxy listening on %s:%s", args.host, args.port)
|
||||
logger.info("Encode servers: %s", app.state.e_urls)
|
||||
logger.info("Prefill instances %s", app.state.p_urls)
|
||||
logger.info("Decode servers: %s", app.state.d_urls)
|
||||
if app.state.ec_consumer_dp_size > 1:
|
||||
logger.info(
|
||||
"EC consumer replicas per server: %d (control addresses: %s)",
|
||||
app.state.ec_consumer_dp_size,
|
||||
app.state.d_ec_urls,
|
||||
)
|
||||
|
||||
uvicorn.run(
|
||||
app,
|
||||
|
||||
Reference in New Issue
Block a user