[Performance][EPD] Reduce Python proxy serialization overhead (#56657)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
Tianyu Guo
2026-09-13 15:07:12 -07:00
committed by GitHub
parent b23433088b
commit 319cc5ef19
2 changed files with 160 additions and 25 deletions
@@ -33,9 +33,10 @@ from collections.abc import AsyncIterator
from typing import Any
import aiohttp
import msgspec
import uvicorn
from fastapi import FastAPI, HTTPException, Request
from fastapi.responses import JSONResponse, StreamingResponse
from fastapi.responses import JSONResponse, Response, StreamingResponse
###############################################################################
# FastAPI app & global state
@@ -264,7 +265,7 @@ async def fanout_encoder_primer(
for idx, (item, target_url) in enumerate(zip(mm_items, url_cycle)):
# Derive a *child* request id: <parent>:<index>:<random-short>
child_req_id = f"{req_id}:{idx}:{uuid.uuid4().hex[:6]}"
headers = {"x-request-id": child_req_id}
headers = {"x-request-id": child_req_id, "Content-Type": "application/json"}
# With --no-rewrite the decoder still receives the raw image and derives
# the cache key by hashing it, so the encoder must do the same -- passing
@@ -307,7 +308,7 @@ async def fanout_encoder_primer(
tasks.append(
encode_session.post(
f"{target_url}/v1/chat/completions",
json=encoder_req,
data=msgspec.json.encode(encoder_req),
headers=headers,
)
)
@@ -347,7 +348,7 @@ async def fanout_encoder_primer(
# The encoder reports each mm_hash's metadata (e.g. the grid) here,
# keyed by the same uuid this proxy assigned above.
try:
params = (await r.json()).get("ec_transfer_params") or {}
params = msgspec.json.decode(await r.read()).get("ec_transfer_params") or {}
except Exception:
logger.warning("[%s] Could not read encoder metadata #%d", req_id, idx)
params = {}
@@ -407,7 +408,7 @@ async def maybe_prefill(
prefill_response = await process_prefill_stage(req_data, p_url, req_id)
# for nixl connector to facilitate kv transfer...
prefill_response_json = await prefill_response.json()
prefill_response_json = msgspec.json.decode(await prefill_response.read())
kv_transfer_params = prefill_response_json.get("kv_transfer_params", {})
if kv_transfer_params:
return {**req_data, "kv_transfer_params": kv_transfer_params}
@@ -439,10 +440,12 @@ async def process_prefill_stage(
if "stream_options" in prefill_request:
del prefill_request["stream_options"]
headers = {"x-request-id": req_id}
headers = {"x-request-id": req_id, "Content-Type": "application/json"}
try:
prefill_response = await prefill_session.post(
f"{p_url}/v1/chat/completions", json=prefill_request, headers=headers
f"{p_url}/v1/chat/completions",
data=msgspec.json.encode(prefill_request),
headers=headers,
)
prefill_response.raise_for_status()
@@ -598,7 +601,7 @@ async def forward_non_stream(
d_url: str,
consumer_zmq: str | None,
dp_rank: int | None = None,
) -> dict:
) -> Response:
try:
for attempt in range(DECODE_RETRIES + 1):
_t0 = time.perf_counter()
@@ -608,12 +611,14 @@ async def forward_non_stream(
_t2 = time.perf_counter()
logger.info("[%s] Forwarding to decode: %s", req_id, d_url)
headers = {"x-request-id": req_id}
headers = {"x-request-id": req_id, "Content-Type": "application/json"}
if dp_rank is not None:
headers["X-data-parallel-rank"] = str(dp_rank)
async with decode_session.post(
f"{d_url}/v1/chat/completions", json=prepared, headers=headers
f"{d_url}/v1/chat/completions",
data=msgspec.json.encode(prepared),
headers=headers,
) as resp:
if resp.status >= 400:
detail = await resp.text()
@@ -637,7 +642,7 @@ async def forward_non_stream(
detail,
)
raise HTTPException(status_code=resp.status, detail=detail)
out = await resp.json()
out = await resp.read()
_t3 = time.perf_counter()
logger.info(
"STAGE %s encode=%.1f rewrite=%.1f decode=%.1f total=%.1f "
@@ -649,7 +654,15 @@ async def forward_non_stream(
(_t3 - _t0) * 1e3,
attempt,
)
return out
return Response(
content=out,
status_code=resp.status,
headers={
"Content-Type": resp.headers.get(
"Content-Type", "application/json"
)
},
)
raise HTTPException(status_code=500, detail="Decode failed after re-encoding")
except HTTPException:
@@ -667,7 +680,7 @@ async def forward_stream(
d_url: str,
consumer_zmq: str | None,
dp_rank: int | None = None,
) -> AsyncIterator[str]:
) -> AsyncIterator[bytes]:
try:
for attempt in range(DECODE_RETRIES + 1):
_t0 = time.perf_counter()
@@ -677,14 +690,14 @@ async def forward_stream(
_t2 = time.perf_counter()
logger.info("[%s] Starting streaming from decode: %s", req_id, d_url)
headers = {"x-request-id": req_id}
headers = {"x-request-id": req_id, "Content-Type": "application/json"}
if dp_rank is not None:
headers["X-data-parallel-rank"] = str(dp_rank)
_first = None
async with decode_session.post(
f"{d_url}/v1/chat/completions",
json=prepared,
data=msgspec.json.encode(prepared),
headers=headers,
) as resp:
# Retry only before the first chunk: once anything reached the
@@ -701,11 +714,11 @@ async def forward_stream(
)
continue
resp.raise_for_status()
async for chunk in resp.content.iter_chunked(1024):
async for chunk in resp.content.iter_any():
if chunk:
if _first is None:
_first = time.perf_counter()
yield chunk.decode("utf-8", errors="ignore")
yield chunk
_t3 = time.perf_counter()
logger.info(
@@ -739,7 +752,7 @@ async def forward_stream(
@app.post("/v1/chat/completions")
async def chat_completions(request: Request):
try:
req_data = await request.json()
req_data = msgspec.json.decode(await request.body())
req_id = request.headers.get("x-request-id", str(uuid.uuid4()))
e_urls = app.state.e_urls # we want the full list for fan-out
@@ -764,10 +777,9 @@ async def chat_completions(request: Request):
),
media_type="text/event-stream",
)
result = await forward_non_stream(
return await forward_non_stream(
req_data, req_id, e_urls, p_url, d_url, consumer_zmq, dp_rank
)
return JSONResponse(content=result)
except HTTPException:
raise