[Performance][EPD] Reduce Python proxy serialization overhead (#56657)
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
@@ -33,9 +33,10 @@ from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
import msgspec
|
||||
import uvicorn
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
|
||||
###############################################################################
|
||||
# FastAPI app & global state
|
||||
@@ -264,7 +265,7 @@ async def fanout_encoder_primer(
|
||||
for idx, (item, target_url) in enumerate(zip(mm_items, url_cycle)):
|
||||
# Derive a *child* request id: <parent>:<index>:<random-short>
|
||||
child_req_id = f"{req_id}:{idx}:{uuid.uuid4().hex[:6]}"
|
||||
headers = {"x-request-id": child_req_id}
|
||||
headers = {"x-request-id": child_req_id, "Content-Type": "application/json"}
|
||||
|
||||
# With --no-rewrite the decoder still receives the raw image and derives
|
||||
# the cache key by hashing it, so the encoder must do the same -- passing
|
||||
@@ -307,7 +308,7 @@ async def fanout_encoder_primer(
|
||||
tasks.append(
|
||||
encode_session.post(
|
||||
f"{target_url}/v1/chat/completions",
|
||||
json=encoder_req,
|
||||
data=msgspec.json.encode(encoder_req),
|
||||
headers=headers,
|
||||
)
|
||||
)
|
||||
@@ -347,7 +348,7 @@ async def fanout_encoder_primer(
|
||||
# The encoder reports each mm_hash's metadata (e.g. the grid) here,
|
||||
# keyed by the same uuid this proxy assigned above.
|
||||
try:
|
||||
params = (await r.json()).get("ec_transfer_params") or {}
|
||||
params = msgspec.json.decode(await r.read()).get("ec_transfer_params") or {}
|
||||
except Exception:
|
||||
logger.warning("[%s] Could not read encoder metadata #%d", req_id, idx)
|
||||
params = {}
|
||||
@@ -407,7 +408,7 @@ async def maybe_prefill(
|
||||
|
||||
prefill_response = await process_prefill_stage(req_data, p_url, req_id)
|
||||
# for nixl connector to facilitate kv transfer...
|
||||
prefill_response_json = await prefill_response.json()
|
||||
prefill_response_json = msgspec.json.decode(await prefill_response.read())
|
||||
kv_transfer_params = prefill_response_json.get("kv_transfer_params", {})
|
||||
if kv_transfer_params:
|
||||
return {**req_data, "kv_transfer_params": kv_transfer_params}
|
||||
@@ -439,10 +440,12 @@ async def process_prefill_stage(
|
||||
if "stream_options" in prefill_request:
|
||||
del prefill_request["stream_options"]
|
||||
|
||||
headers = {"x-request-id": req_id}
|
||||
headers = {"x-request-id": req_id, "Content-Type": "application/json"}
|
||||
try:
|
||||
prefill_response = await prefill_session.post(
|
||||
f"{p_url}/v1/chat/completions", json=prefill_request, headers=headers
|
||||
f"{p_url}/v1/chat/completions",
|
||||
data=msgspec.json.encode(prefill_request),
|
||||
headers=headers,
|
||||
)
|
||||
prefill_response.raise_for_status()
|
||||
|
||||
@@ -598,7 +601,7 @@ async def forward_non_stream(
|
||||
d_url: str,
|
||||
consumer_zmq: str | None,
|
||||
dp_rank: int | None = None,
|
||||
) -> dict:
|
||||
) -> Response:
|
||||
try:
|
||||
for attempt in range(DECODE_RETRIES + 1):
|
||||
_t0 = time.perf_counter()
|
||||
@@ -608,12 +611,14 @@ async def forward_non_stream(
|
||||
_t2 = time.perf_counter()
|
||||
|
||||
logger.info("[%s] Forwarding to decode: %s", req_id, d_url)
|
||||
headers = {"x-request-id": req_id}
|
||||
headers = {"x-request-id": req_id, "Content-Type": "application/json"}
|
||||
if dp_rank is not None:
|
||||
headers["X-data-parallel-rank"] = str(dp_rank)
|
||||
|
||||
async with decode_session.post(
|
||||
f"{d_url}/v1/chat/completions", json=prepared, headers=headers
|
||||
f"{d_url}/v1/chat/completions",
|
||||
data=msgspec.json.encode(prepared),
|
||||
headers=headers,
|
||||
) as resp:
|
||||
if resp.status >= 400:
|
||||
detail = await resp.text()
|
||||
@@ -637,7 +642,7 @@ async def forward_non_stream(
|
||||
detail,
|
||||
)
|
||||
raise HTTPException(status_code=resp.status, detail=detail)
|
||||
out = await resp.json()
|
||||
out = await resp.read()
|
||||
_t3 = time.perf_counter()
|
||||
logger.info(
|
||||
"STAGE %s encode=%.1f rewrite=%.1f decode=%.1f total=%.1f "
|
||||
@@ -649,7 +654,15 @@ async def forward_non_stream(
|
||||
(_t3 - _t0) * 1e3,
|
||||
attempt,
|
||||
)
|
||||
return out
|
||||
return Response(
|
||||
content=out,
|
||||
status_code=resp.status,
|
||||
headers={
|
||||
"Content-Type": resp.headers.get(
|
||||
"Content-Type", "application/json"
|
||||
)
|
||||
},
|
||||
)
|
||||
raise HTTPException(status_code=500, detail="Decode failed after re-encoding")
|
||||
|
||||
except HTTPException:
|
||||
@@ -667,7 +680,7 @@ async def forward_stream(
|
||||
d_url: str,
|
||||
consumer_zmq: str | None,
|
||||
dp_rank: int | None = None,
|
||||
) -> AsyncIterator[str]:
|
||||
) -> AsyncIterator[bytes]:
|
||||
try:
|
||||
for attempt in range(DECODE_RETRIES + 1):
|
||||
_t0 = time.perf_counter()
|
||||
@@ -677,14 +690,14 @@ async def forward_stream(
|
||||
_t2 = time.perf_counter()
|
||||
|
||||
logger.info("[%s] Starting streaming from decode: %s", req_id, d_url)
|
||||
headers = {"x-request-id": req_id}
|
||||
headers = {"x-request-id": req_id, "Content-Type": "application/json"}
|
||||
if dp_rank is not None:
|
||||
headers["X-data-parallel-rank"] = str(dp_rank)
|
||||
|
||||
_first = None
|
||||
async with decode_session.post(
|
||||
f"{d_url}/v1/chat/completions",
|
||||
json=prepared,
|
||||
data=msgspec.json.encode(prepared),
|
||||
headers=headers,
|
||||
) as resp:
|
||||
# Retry only before the first chunk: once anything reached the
|
||||
@@ -701,11 +714,11 @@ async def forward_stream(
|
||||
)
|
||||
continue
|
||||
resp.raise_for_status()
|
||||
async for chunk in resp.content.iter_chunked(1024):
|
||||
async for chunk in resp.content.iter_any():
|
||||
if chunk:
|
||||
if _first is None:
|
||||
_first = time.perf_counter()
|
||||
yield chunk.decode("utf-8", errors="ignore")
|
||||
yield chunk
|
||||
_t3 = time.perf_counter()
|
||||
|
||||
logger.info(
|
||||
@@ -739,7 +752,7 @@ async def forward_stream(
|
||||
@app.post("/v1/chat/completions")
|
||||
async def chat_completions(request: Request):
|
||||
try:
|
||||
req_data = await request.json()
|
||||
req_data = msgspec.json.decode(await request.body())
|
||||
req_id = request.headers.get("x-request-id", str(uuid.uuid4()))
|
||||
|
||||
e_urls = app.state.e_urls # we want the full list for fan-out
|
||||
@@ -764,10 +777,9 @@ async def chat_completions(request: Request):
|
||||
),
|
||||
media_type="text/event-stream",
|
||||
)
|
||||
result = await forward_non_stream(
|
||||
return await forward_non_stream(
|
||||
req_data, req_id, e_urls, p_url, d_url, consumer_zmq, dp_rank
|
||||
)
|
||||
return JSONResponse(content=result)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
|
||||
Reference in New Issue
Block a user