[Perf][EPD] Batch image requests per encoder (#57095)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
Signed-off-by: Zhou ziheng <jiaranran2@gmail.com>
Co-authored-by: Zhou ziheng <jiaranran2@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Tianyu Guo
2026-09-17 03:45:52 -07:00
committed by GitHub
co-authored by Zhou ziheng OpenAI Codex
parent 99ea5f525d
commit 0d72627565
6 changed files with 329 additions and 59 deletions
@@ -106,6 +106,22 @@ If you enable prefill instance (`--prefill-servers-urls` not disabled), you will
| `--decode-servers-urls` | Comma-separated list of decode endpoints. Non-stream and stream paths both round-robin over this list. |
| `--host`, `--port` | Bind address for the proxy itself (defaults: `0.0.0.0:8000`). |
The proxy batches images from the same user request assigned to the same encoder.
Set the proxy environment variable `ENCODER_MAX_BATCH_SIZE` to limit the number
of images per encoder subrequest. It defaults to `0` (unlimited); `1` sends each
image separately. Audio and video remain separate subrequests.
For encoders with a smaller `--limit-mm-per-prompt` image limit than P/PD, set
`ENCODER_MAX_BATCH_SIZE` no higher than the smallest encoder image limit.
For example, for encoders configured with `--limit-mm-per-prompt '{"image": 2}'`:
```bash
ENCODER_MAX_BATCH_SIZE=2 python disagg_epd_proxy.py \
--encode-servers-urls "http://e1:8001,http://e2:8002" \
--prefill-servers-urls disable \
--decode-servers-urls "http://pd1:8003"
```
Example usage:
For E + PD setup:
@@ -10,8 +10,8 @@ clusters:
For MM input we:
1. Extract *every* image/audio/video item.
2. Fire N concurrent requests to the encoder cluster
(one request per item, with **all text removed**).
2. Send concurrent encoder requests with all text removed,
grouping images assigned to the same encoder.
3. Wait for all of them to succeed.
4. Forward the *original* request to a decode server.
"""
@@ -99,6 +99,11 @@ def validate_ec_consumer_routing(
# Diagnostic switch: forward the original request to the decoder so the
# only difference from the rewrite path is the rewrite itself.
NO_REWRITE = False
# Maximum images per encoder subrequest; 0 leaves batches unlimited.
ENCODER_MAX_BATCH_SIZE = int(os.getenv("ENCODER_MAX_BATCH_SIZE", "0"))
if ENCODER_MAX_BATCH_SIZE < 0:
raise ValueError("ENCODER_MAX_BATCH_SIZE must be non-negative")
# Decode-side retries for a retryable internal error (`finish_reason="error"`,
# e.g. an encoder embedding the connector could not deliver). Re-issuing runs
# the encode again, which produces a fresh transfer.
@@ -222,7 +227,7 @@ async def fanout_encoder_primer(
req_id: str,
consumer_zmq: str | None = None,
) -> tuple[dict[int, dict], dict[str, Any]]:
"""1. Build one request *per MM item* with all text removed.
"""1. Group images by encoder, retaining per-item round-robin assignment.
2. Send them concurrently to the encode cluster.
3. Raise if any of them fails.
@@ -248,6 +253,7 @@ async def fanout_encoder_primer(
item_uuids: dict[int, str] = {}
item_transfer_ids: dict[int, str] = {}
item_meta: dict[int, dict] = {}
transfer_items: dict[int, dict[str, str]] = {}
ec_params: dict[str, Any] = {}
# Round-robin over encode servers to distribute load a bit. The cursor
@@ -259,29 +265,45 @@ async def fanout_encoder_primer(
e_urls, encoder_rr_idx, len(mm_items)
)
groups: list[tuple[str, list[int]]] = []
image_groups: dict[str, list[int]] = {}
for idx, (item, target_url) in enumerate(zip(mm_items, url_cycle)):
if item["type"] == "image_url":
indices = image_groups.get(target_url)
if indices is None or (
ENCODER_MAX_BATCH_SIZE and len(indices) >= ENCODER_MAX_BATCH_SIZE
):
indices = []
image_groups[target_url] = indices
groups.append((target_url, indices))
indices.append(idx)
else:
groups.append((target_url, [idx]))
for target_url, indices in groups:
# Derive a *child* request id: <parent>:<index>:<random-short>
child_req_id = f"{req_id}:{idx}:{uuid.uuid4().hex[:6]}"
child_req_id = f"{req_id}:{indices[0]}:{uuid.uuid4().hex[:6]}"
headers = {"x-request-id": child_req_id, "Content-Type": "application/json"}
# With --no-rewrite the decoder still receives the raw image and derives
# the cache key by hashing it, so the encoder must do the same -- passing
# a uuid here would make the two disagree and silently defeat the EC
# transfer, leaving the decoder to encode the image itself.
item_uuid = None if NO_REWRITE else content_uuid(item)
if item_uuid is not None:
item_uuids[idx] = item_uuid
transfer_id = uuid.uuid4().hex
item_transfer_ids[idx] = transfer_id
content = []
for idx in indices:
item = mm_items[idx]
item_uuid = None if NO_REWRITE else content_uuid(item)
if item_uuid is not None:
item_uuids[idx] = item_uuid
item_transfer_ids[idx] = uuid.uuid4().hex
content.append(item if item_uuid is None else {**item, "uuid": item_uuid})
encoder_req = {
"model": orig_request.get("model"),
"messages": [
{
"role": "user",
"content": [
item if item_uuid is None else {**item, "uuid": item_uuid}
],
"content": content,
},
],
# No max_tokens cap: the encoder instance never samples, it finishes
@@ -297,17 +319,13 @@ async def fanout_encoder_primer(
if key in orig_request:
encoder_req[key] = orig_request[key]
if consumer_zmq is not None:
# No mm_hash here on purpose. The encoder's own
# `mm_features[i].identifier` is derived from the uuid *and* the
# engine's media_io_kwargs / mm_processor_kwargs, so this proxy
# cannot know it before the encoder runs. Sending the bare uuid
# would fail the connector's hash match and make the producer
# invent its own transfer id, which the consumer could then never
# cancel. Omitting it lets the connector match by position, which
# is exact: one encoder request carries exactly one item.
# The engine may rehash UUIDs with processing options. Match by
# position instead; batches contain only images in input order.
encoder_req["ec_transfer_params"] = {
"consumer_zmq": consumer_zmq,
"ec_items": [{"transfer_id": transfer_id}],
"ec_items": [
{"transfer_id": item_transfer_ids[idx]} for idx in indices
],
}
tasks.append(
encode_session.post(
@@ -320,7 +338,8 @@ async def fanout_encoder_primer(
results = await asyncio.gather(*tasks, return_exceptions=True)
# Fail fast if any sub-request failed
for idx, r in enumerate(results):
for (_, indices), r in zip(groups, results):
idx = indices[0]
if isinstance(r, Exception):
logger.error(
"[%s] Encoder request #%d raised exception: %s",
@@ -357,18 +376,45 @@ async def fanout_encoder_primer(
logger.warning("[%s] Could not read encoder metadata #%d", req_id, idx)
params = {}
if params:
# One encoder request carries exactly one item, so there is a
# single reported entry. Do not key it by this proxy's uuid: when
# media_io_kwargs or mm_processor_kwargs are set the engine
# re-hashes the uuid together with them, so the encoder's own
# `mm_features[i].identifier` is a derived value this proxy cannot
# predict. Fall back to the sole entry, and carry the key the
# encoder actually used through as `ec_mm_hash`.
ec_mm_hash = item_uuids.get(idx)
reported = params.get(ec_mm_hash)
if reported is None and len(params) == 1:
((ec_mm_hash, reported),) = params.items()
if reported:
by_index = {}
for mm_hash, reported in params.items():
if len(indices) == 1:
break
for local_index in reported.get("item_indices", []):
if type(local_index) is not int or not 0 <= local_index < len(
indices
):
raise HTTPException(502, "Invalid encoder item index")
if local_index in by_index:
raise HTTPException(502, "Duplicate encoder item index")
by_index[local_index] = (mm_hash, reported)
for local_index, idx in enumerate(indices):
matched = by_index.get(local_index)
if matched is None:
# Compatibility with encoders without position metadata.
mm_hash = item_uuids.get(idx)
if mm_hash in params:
matched = (mm_hash, params[mm_hash])
elif len(indices) == 1 and len(params) == 1:
matched = next(iter(params.items()))
elif (
mm_items[idx]["type"] == "video_url"
and (orig_request.get("mm_processor_kwargs") or {}).get(
"use_audio_in_video"
)
and len(params) == 2
and consumer_zmq is None
and all(
entry.keys() <= {"metadata", "item_indices"}
for entry in params.values()
)
):
# Keep raw video for its audio/video features when no transfer
# handles need to be forwarded.
continue
else:
raise HTTPException(502, "Encoder metadata cannot be matched")
ec_mm_hash, reported = matched
metadata = reported.get("metadata") or {}
if metadata:
item_meta[idx] = {
@@ -384,12 +430,16 @@ async def fanout_encoder_primer(
# looks it up by mm_hash on the request, so carry it through.
ec_params[ec_mm_hash] = reported
if NO_REWRITE and consumer_zmq is not None:
ec_params.setdefault("ec_items", []).append(
{"mm_hash": ec_mm_hash, "transfer_id": item_transfer_ids[idx]}
)
transfer_items[idx] = {
"mm_hash": ec_mm_hash,
"transfer_id": item_transfer_ids[idx],
}
if transfer_items:
ec_params["ec_items"] = [transfer_items[idx] for idx in sorted(transfer_items)]
logger.info(
"[%s] All %d encoder requests completed successfully", req_id, len(mm_items)
"[%s] All %d encoder requests completed successfully", req_id, len(groups)
)
return item_meta, ec_params
@@ -16,6 +16,7 @@ from vllm.distributed.ec_transfer.ec_connector.example_connector import (
ECExampleConnectorMetadata,
MMMeta,
)
from vllm.distributed.ec_transfer.ec_connector.utils import collect_ec_item_metadata
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
from vllm.v1.core.sched.output import SchedulerOutput
@@ -41,6 +42,20 @@ class MockRequest:
return self._token_counts[input_id]
def test_metadata_preserves_repeated_item_positions():
request = MockRequest("request", ["A", "B", "A"], [4, 4, 4])
resolver = Mock()
resolver.fields_for.return_value = {"image_grid_thw"}
for feature in request.mm_features:
feature.data = Mock()
feature.data.get_data.return_value = {"image_grid_thw": torch.tensor([1, 2, 2])}
items = collect_ec_item_metadata(request.mm_features, resolver)
assert items == {
"A": {"metadata": {"image_grid_thw": [1, 2, 2]}, "item_indices": [0, 2]},
"B": {"metadata": {"image_grid_thw": [1, 2, 2]}, "item_indices": [1]},
}
@pytest.fixture
def temp_storage(tmp_path):
"""Fixture providing temporary storage path."""
@@ -9,6 +9,8 @@ change to them is what these tests catch.
import asyncio
import importlib.util
from pathlib import Path
from types import SimpleNamespace
from typing import Any
import httpx
import msgspec
@@ -83,6 +85,128 @@ class _EncoderSession:
return _EncoderResponse(self._replies.pop(0))
@pytest.mark.parametrize("no_rewrite", [False, True])
@pytest.mark.parametrize(
"batch_size, expected_sizes", [(0, [3, 3]), (1, [1] * 6), (2, [2, 2, 1, 1])]
)
def test_image_batches_preserve_item_identity(
proxy, monkeypatch, no_rewrite, batch_size, expected_sizes
):
"""Rehashed and repeated images retain metadata and per-occurrence transfers."""
from vllm.distributed.ec_transfer.ec_connector.mooncake.scheduler import (
ECMooncakeScheduler,
)
seen = []
class Session:
async def post(self, url, data=None, headers=None):
body = msgspec.json.decode(data)
seen.append((url, body))
params: dict[str, dict[str, Any]] = {}
for index, item in enumerate(body["messages"][0]["content"]):
image = item["image_url"]["url"]
entry = params.setdefault(
image + "-processed",
{
"metadata": {"image_grid_thw": [1, 2, ord(image)]},
"item_indices": [],
"peer_port": 1234,
},
)
entry["item_indices"].append(index)
# Response ordering must not affect the image-to-metadata mapping.
return _EncoderResponse(dict(reversed(list(params.items()))))
monkeypatch.setattr(proxy, "encode_session", Session())
monkeypatch.setattr(proxy, "NO_REWRITE", no_rewrite)
monkeypatch.setattr(proxy, "ENCODER_MAX_BATCH_SIZE", batch_size)
monkeypatch.setattr(proxy, "encoder_rr_idx", 0)
images = ["A", "A", "C", "D", "A", "F"]
body = {
"messages": [
{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": image}}
for image in images
],
}
],
"mm_processor_kwargs": {"max_pixels": 262144},
}
meta, handles = asyncio.run(
proxy.fanout_encoder_primer(
body, ["http://e0", "http://e1"], "r", "tcp://consumer:1"
)
)
assert [len(b["messages"][0]["content"]) for _, b in seen] == expected_sizes
assert (
len(
{
x["transfer_id"]
for _, b in seen
for x in b["ec_transfer_params"]["ec_items"]
}
)
== 6
)
for url, batch in seen:
assert batch["mm_processor_kwargs"] == body["mm_processor_kwargs"]
for item in batch["messages"][0]["content"]:
assert item["image_url"]["url"] in (
{"A", "C"} if url.startswith("http://e0/") else {"A", "D", "F"}
)
assert [meta[i]["image_grid_thw"][-1] for i in range(6)] == list(map(ord, images))
assert [meta[i]["ec_mm_hash"] for i in range(6)] == [
x + "-processed" for x in images
]
assert handles["A-processed"]["peer_port"] == 1234
assert "ec_transfer_params" not in body
transfers_by_encoder: list[list[dict[str, str]]] = [[], []]
for url, batch in seen:
rank = 0 if url.startswith("http://e0/") else 1
transfers_by_encoder[rank].extend(batch["ec_transfer_params"]["ec_items"])
expected_transfers = [
transfers_by_encoder[i % 2][i // 2]["transfer_id"] for i in range(len(images))
]
params = (
handles
if no_rewrite
else proxy.rewrite_for_decode(body, meta)["ec_transfer_params"]
)
request = SimpleNamespace(
ec_transfer_params=params,
mm_features=[
SimpleNamespace(identifier=image + "-processed") for image in images
],
)
assert [
ECMooncakeScheduler._request_transfer_id(request, i) for i in range(len(images))
] == expected_transfers
def test_batch_rejects_ambiguous_metadata(proxy, monkeypatch):
monkeypatch.setattr(
proxy,
"encode_session",
_EncoderSession([{"unknown": {"metadata": {"image_grid_thw": [1, 2, 2]}}}]),
)
body = {
"messages": [
{
"content": [
{"type": "image_url", "image_url": {"url": image}}
for image in ["A", "B"]
]
}
]
}
with pytest.raises(proxy.HTTPException, match="cannot be matched"):
asyncio.run(proxy.fanout_encoder_primer(body, ["http://e0"], "r"))
def test_a_decode_retry_does_not_inherit_the_previous_handles(proxy, monkeypatch):
"""The retry loop re-enters `prepare_for_decode` with the same body.
@@ -187,20 +311,21 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes(
body = await request.json()
seen[stage].append(body)
if stage == "encode":
item = body["messages"][0]["content"][0]
mm_hash = item["uuid"]
if body.get("mm_processor_kwargs"):
mm_hash += "-processed"
return web.json_response(
{
"ec_transfer_params": {
mm_hash: {
"metadata": {"image_grid_thw": [[1, 2, 2]]},
"peer_port": len(seen[stage]),
}
}
}
)
params: dict[str, dict[str, Any]] = {}
for index, item in enumerate(body["messages"][0]["content"]):
mm_hash = item["uuid"]
if body.get("mm_processor_kwargs"):
mm_hash += "-processed"
entry = params.setdefault(
mm_hash,
{
"metadata": {"image_grid_thw": [[1, 2, 2]]},
"peer_port": len(seen[stage]),
"item_indices": [],
},
)
entry["item_indices"].append(index)
return web.json_response({"ec_transfer_params": params})
if stage == "prefill":
assert body["max_tokens"] == 1 and body["stream"] is False
return web.json_response(
@@ -263,7 +388,7 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes(
finally:
await proxy.on_shutdown()
assert len(seen["encode"]) == 4
assert len(seen["encode"]) == 2
assert len(seen["decode"]) == 2
for requests in seen.values():
for forwarded in requests:
@@ -296,7 +421,7 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes(
if request_options:
mm_hash += "-processed"
assert set(final["ec_transfer_params"]) - {"ec_items"} == {mm_hash}
assert final["ec_transfer_params"][mm_hash]["peer_port"] in (3, 4)
assert final["ec_transfer_params"][mm_hash]["peer_port"] == 2
if prefill:
assert final["kv_transfer_params"] == {"remote_block_ids": [2]}
@@ -324,3 +449,62 @@ def test_pooled_connections_are_retired_before_the_server_closes_them(
assert proxy.decode_session.connector._keepalive_timeout == pooled_for
finally:
asyncio.run(proxy.on_shutdown())
@pytest.mark.parametrize(
"no_rewrite, transfer",
[(False, None), (True, None), (False, "push"), (False, "handle")],
)
def test_video_audio_fallback(proxy, monkeypatch, no_rewrite, transfer):
"""Keep raw video and neighboring images, but never discard transfer handles."""
import copy
from vllm.distributed.ec_transfer.ec_connector.utils import collect_ec_item_metadata
video = {"type": "video_url", "video_url": {"url": "video-with-audio"}}
images = [{"type": "image_url", "image_url": {"url": key}} for key in ("A", "B")]
body: dict[str, Any] = {
"messages": [{"content": [images[0], video, images[1]]}],
"mm_processor_kwargs": {"use_audio_in_video": True},
}
original = copy.deepcopy(body)
# Collector indices describe two processed features from one video item.
video_metadata = collect_ec_item_metadata(
[SimpleNamespace(identifier=key, data=None) for key in ("video", "audio")], None
)
if transfer == "handle":
video_metadata["audio"]["transfer_id"] = "reservation"
image_metadata = {
key: {"metadata": {"image_grid_thw": [1, 2, size]}, "item_indices": [i]}
for i, (key, size) in enumerate((("A", 2), ("B", 4)))
}
# Images form the first group, the singleton video the second.
monkeypatch.setattr(proxy, "ENCODER_MAX_BATCH_SIZE", 0)
monkeypatch.setattr(proxy, "NO_REWRITE", no_rewrite)
replies = _EncoderSession([image_metadata, video_metadata])
monkeypatch.setattr(proxy, "encode_session", replies)
consumer = "tcp://consumer:1" if transfer == "push" else None
preparation = proxy.prepare_for_decode(body, "r", ["http://e0"], "", consumer)
if transfer:
with pytest.raises(proxy.HTTPException, match="cannot be matched"):
asyncio.run(preparation)
return
prepared, _, _ = asyncio.run(preparation)
assert body == original
assert prepared["mm_processor_kwargs"] == body["mm_processor_kwargs"]
final = prepared["messages"][0]["content"]
assert final[1] == video
handles = prepared["ec_transfer_params"]
assert not video_metadata.keys() & handles.keys()
if no_rewrite:
assert final == original["messages"][0]["content"]
assert "ec_items" not in handles
else:
for position, image, key in ((0, images[0], "A"), (2, images[1], "B")):
assert final[position] == {
"type": "image_embeds",
"image_embeds": image_metadata[key]["metadata"],
"uuid": proxy.content_uuid(image),
}
assert [item["mm_hash"] for item in handles["ec_items"]] == ["A", "B"]
@@ -67,6 +67,7 @@ def test_request_finished_producer_emits_params(monkeypatch):
assert params == {
"h1": {
"metadata": {},
"item_indices": [0],
"peer_host": "1.2.3.4",
"peer_port": 5601,
"size_bytes": 2 * 32 * 2,
@@ -91,6 +92,7 @@ def test_request_finished_announces_not_ready_entry(monkeypatch):
assert params == {
"h1": {
"metadata": {},
"item_indices": [0],
"peer_host": "1.2.3.4",
"peer_port": 5601,
"size_bytes": 2 * 32 * 2,
@@ -110,5 +112,5 @@ def test_request_finished_skips_unallocated_entry(monkeypatch):
assert delay is False
# The item's placeholder metadata is still reported even though there's
# no cache entry to transfer (empty "metadata": no fields, no transfer).
assert params == {"h1": {"metadata": {}}}
assert params == {"h1": {"metadata": {}, "item_indices": [0]}}
s.shutdown()
@@ -60,15 +60,16 @@ def collect_ec_item_metadata(
Keyed by mm_hash, each entry carries a `metadata` dict with whatever
placeholder fields `resolver` says this model needs published for its
modality, so a consumer can skip the image transform. `data` is None for
items served from the processor cache, in which case the metadata is
modality, so a consumer can skip the image transform. `item_indices`
identifies every occurrence in `mm_features`, including repeated hashes.
`data` is None for items served from the processor cache, so the metadata is
unavailable here and the consumer has to fall back to processing the
media itself. A connector that also has transfer coordinates to report
(e.g. NIXL peer_host/peer_port/size_bytes) merges those in alongside
`metadata`, not into it.
"""
items: dict[str, dict[str, Any]] = {}
for feature in mm_features:
for index, feature in enumerate(mm_features):
metadata: dict[str, Any] = {}
if feature.data is not None:
wanted = resolver.fields_for(feature.modality)
@@ -81,7 +82,9 @@ def collect_ec_item_metadata(
# Some metadata (e.g. Qwen3-VL video timestamps) is
# produced as a plain list rather than a tensor.
metadata[key] = value
items[feature.identifier] = {"metadata": metadata}
entry = items.setdefault(feature.identifier, {"item_indices": []})
entry["metadata"] = metadata
entry["item_indices"].append(index)
return items