[Perf][EPD] Batch image requests per encoder (#57095)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
Signed-off-by: Zhou ziheng <jiaranran2@gmail.com>
Co-authored-by: Zhou ziheng <jiaranran2@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Tianyu Guo
2026-09-17 03:45:52 -07:00
committed by GitHub
co-authored by Zhou ziheng OpenAI Codex
parent 99ea5f525d
commit 0d72627565
6 changed files with 329 additions and 59 deletions
@@ -106,6 +106,22 @@ If you enable prefill instance (`--prefill-servers-urls` not disabled), you will
| `--decode-servers-urls` | Comma-separated list of decode endpoints. Non-stream and stream paths both round-robin over this list. |
| `--host`, `--port` | Bind address for the proxy itself (defaults: `0.0.0.0:8000`). |
The proxy batches images from the same user request assigned to the same encoder.
Set the proxy environment variable `ENCODER_MAX_BATCH_SIZE` to limit the number
of images per encoder subrequest. It defaults to `0` (unlimited); `1` sends each
image separately. Audio and video remain separate subrequests.
For encoders with a smaller `--limit-mm-per-prompt` image limit than P/PD, set
`ENCODER_MAX_BATCH_SIZE` no higher than the smallest encoder image limit.
For example, for encoders configured with `--limit-mm-per-prompt '{"image": 2}'`:
```bash
ENCODER_MAX_BATCH_SIZE=2 python disagg_epd_proxy.py \
--encode-servers-urls "http://e1:8001,http://e2:8002" \
--prefill-servers-urls disable \
--decode-servers-urls "http://pd1:8003"
```
Example usage:
For E + PD setup:
@@ -10,8 +10,8 @@ clusters:
For MM input we:
1. Extract *every* image/audio/video item.
2. Fire N concurrent requests to the encoder cluster
(one request per item, with **all text removed**).
2. Send concurrent encoder requests with all text removed,
grouping images assigned to the same encoder.
3. Wait for all of them to succeed.
4. Forward the *original* request to a decode server.
"""
@@ -99,6 +99,11 @@ def validate_ec_consumer_routing(
# Diagnostic switch: forward the original request to the decoder so the
# only difference from the rewrite path is the rewrite itself.
NO_REWRITE = False
# Maximum images per encoder subrequest; 0 leaves batches unlimited.
ENCODER_MAX_BATCH_SIZE = int(os.getenv("ENCODER_MAX_BATCH_SIZE", "0"))
if ENCODER_MAX_BATCH_SIZE < 0:
raise ValueError("ENCODER_MAX_BATCH_SIZE must be non-negative")
# Decode-side retries for a retryable internal error (`finish_reason="error"`,
# e.g. an encoder embedding the connector could not deliver). Re-issuing runs
# the encode again, which produces a fresh transfer.
@@ -222,7 +227,7 @@ async def fanout_encoder_primer(
req_id: str,
consumer_zmq: str | None = None,
) -> tuple[dict[int, dict], dict[str, Any]]:
"""1. Build one request *per MM item* with all text removed.
"""1. Group images by encoder, retaining per-item round-robin assignment.
2. Send them concurrently to the encode cluster.
3. Raise if any of them fails.
@@ -248,6 +253,7 @@ async def fanout_encoder_primer(
item_uuids: dict[int, str] = {}
item_transfer_ids: dict[int, str] = {}
item_meta: dict[int, dict] = {}
transfer_items: dict[int, dict[str, str]] = {}
ec_params: dict[str, Any] = {}
# Round-robin over encode servers to distribute load a bit. The cursor
@@ -259,29 +265,45 @@ async def fanout_encoder_primer(
e_urls, encoder_rr_idx, len(mm_items)
)
groups: list[tuple[str, list[int]]] = []
image_groups: dict[str, list[int]] = {}
for idx, (item, target_url) in enumerate(zip(mm_items, url_cycle)):
if item["type"] == "image_url":
indices = image_groups.get(target_url)
if indices is None or (
ENCODER_MAX_BATCH_SIZE and len(indices) >= ENCODER_MAX_BATCH_SIZE
):
indices = []
image_groups[target_url] = indices
groups.append((target_url, indices))
indices.append(idx)
else:
groups.append((target_url, [idx]))
for target_url, indices in groups:
# Derive a *child* request id: <parent>:<index>:<random-short>
child_req_id = f"{req_id}:{idx}:{uuid.uuid4().hex[:6]}"
child_req_id = f"{req_id}:{indices[0]}:{uuid.uuid4().hex[:6]}"
headers = {"x-request-id": child_req_id, "Content-Type": "application/json"}
# With --no-rewrite the decoder still receives the raw image and derives
# the cache key by hashing it, so the encoder must do the same -- passing
# a uuid here would make the two disagree and silently defeat the EC
# transfer, leaving the decoder to encode the image itself.
item_uuid = None if NO_REWRITE else content_uuid(item)
if item_uuid is not None:
item_uuids[idx] = item_uuid
transfer_id = uuid.uuid4().hex
item_transfer_ids[idx] = transfer_id
content = []
for idx in indices:
item = mm_items[idx]
item_uuid = None if NO_REWRITE else content_uuid(item)
if item_uuid is not None:
item_uuids[idx] = item_uuid
item_transfer_ids[idx] = uuid.uuid4().hex
content.append(item if item_uuid is None else {**item, "uuid": item_uuid})
encoder_req = {
"model": orig_request.get("model"),
"messages": [
{
"role": "user",
"content": [
item if item_uuid is None else {**item, "uuid": item_uuid}
],
"content": content,
},
],
# No max_tokens cap: the encoder instance never samples, it finishes
@@ -297,17 +319,13 @@ async def fanout_encoder_primer(
if key in orig_request:
encoder_req[key] = orig_request[key]
if consumer_zmq is not None:
# No mm_hash here on purpose. The encoder's own
# `mm_features[i].identifier` is derived from the uuid *and* the
# engine's media_io_kwargs / mm_processor_kwargs, so this proxy
# cannot know it before the encoder runs. Sending the bare uuid
# would fail the connector's hash match and make the producer
# invent its own transfer id, which the consumer could then never
# cancel. Omitting it lets the connector match by position, which
# is exact: one encoder request carries exactly one item.
# The engine may rehash UUIDs with processing options. Match by
# position instead; batches contain only images in input order.
encoder_req["ec_transfer_params"] = {
"consumer_zmq": consumer_zmq,
"ec_items": [{"transfer_id": transfer_id}],
"ec_items": [
{"transfer_id": item_transfer_ids[idx]} for idx in indices
],
}
tasks.append(
encode_session.post(
@@ -320,7 +338,8 @@ async def fanout_encoder_primer(
results = await asyncio.gather(*tasks, return_exceptions=True)
# Fail fast if any sub-request failed
for idx, r in enumerate(results):
for (_, indices), r in zip(groups, results):
idx = indices[0]
if isinstance(r, Exception):
logger.error(
"[%s] Encoder request #%d raised exception: %s",
@@ -357,18 +376,45 @@ async def fanout_encoder_primer(
logger.warning("[%s] Could not read encoder metadata #%d", req_id, idx)
params = {}
if params:
# One encoder request carries exactly one item, so there is a
# single reported entry. Do not key it by this proxy's uuid: when
# media_io_kwargs or mm_processor_kwargs are set the engine
# re-hashes the uuid together with them, so the encoder's own
# `mm_features[i].identifier` is a derived value this proxy cannot
# predict. Fall back to the sole entry, and carry the key the
# encoder actually used through as `ec_mm_hash`.
ec_mm_hash = item_uuids.get(idx)
reported = params.get(ec_mm_hash)
if reported is None and len(params) == 1:
((ec_mm_hash, reported),) = params.items()
if reported:
by_index = {}
for mm_hash, reported in params.items():
if len(indices) == 1:
break
for local_index in reported.get("item_indices", []):
if type(local_index) is not int or not 0 <= local_index < len(
indices
):
raise HTTPException(502, "Invalid encoder item index")
if local_index in by_index:
raise HTTPException(502, "Duplicate encoder item index")
by_index[local_index] = (mm_hash, reported)
for local_index, idx in enumerate(indices):
matched = by_index.get(local_index)
if matched is None:
# Compatibility with encoders without position metadata.
mm_hash = item_uuids.get(idx)
if mm_hash in params:
matched = (mm_hash, params[mm_hash])
elif len(indices) == 1 and len(params) == 1:
matched = next(iter(params.items()))
elif (
mm_items[idx]["type"] == "video_url"
and (orig_request.get("mm_processor_kwargs") or {}).get(
"use_audio_in_video"
)
and len(params) == 2
and consumer_zmq is None
and all(
entry.keys() <= {"metadata", "item_indices"}
for entry in params.values()
)
):
# Keep raw video for its audio/video features when no transfer
# handles need to be forwarded.
continue
else:
raise HTTPException(502, "Encoder metadata cannot be matched")
ec_mm_hash, reported = matched
metadata = reported.get("metadata") or {}
if metadata:
item_meta[idx] = {
@@ -384,12 +430,16 @@ async def fanout_encoder_primer(
# looks it up by mm_hash on the request, so carry it through.
ec_params[ec_mm_hash] = reported
if NO_REWRITE and consumer_zmq is not None:
ec_params.setdefault("ec_items", []).append(
{"mm_hash": ec_mm_hash, "transfer_id": item_transfer_ids[idx]}
)
transfer_items[idx] = {
"mm_hash": ec_mm_hash,
"transfer_id": item_transfer_ids[idx],
}
if transfer_items:
ec_params["ec_items"] = [transfer_items[idx] for idx in sorted(transfer_items)]
logger.info(
"[%s] All %d encoder requests completed successfully", req_id, len(mm_items)
"[%s] All %d encoder requests completed successfully", req_id, len(groups)
)
return item_meta, ec_params