diff --git a/examples/disaggregated/disaggregated_encoder/README.md b/examples/disaggregated/disaggregated_encoder/README.md index bd2a46abbe..71a0d0e72e 100644 --- a/examples/disaggregated/disaggregated_encoder/README.md +++ b/examples/disaggregated/disaggregated_encoder/README.md @@ -106,6 +106,22 @@ If you enable prefill instance (`--prefill-servers-urls` not disabled), you will | `--decode-servers-urls` | Comma-separated list of decode endpoints. Non-stream and stream paths both round-robin over this list. | | `--host`, `--port` | Bind address for the proxy itself (defaults: `0.0.0.0:8000`). | +The proxy batches images from the same user request assigned to the same encoder. +Set the proxy environment variable `ENCODER_MAX_BATCH_SIZE` to limit the number +of images per encoder subrequest. It defaults to `0` (unlimited); `1` sends each +image separately. Audio and video remain separate subrequests. + +For encoders with a smaller `--limit-mm-per-prompt` image limit than P/PD, set +`ENCODER_MAX_BATCH_SIZE` no higher than the smallest encoder image limit. +For example, for encoders configured with `--limit-mm-per-prompt '{"image": 2}'`: + +```bash +ENCODER_MAX_BATCH_SIZE=2 python disagg_epd_proxy.py \ + --encode-servers-urls "http://e1:8001,http://e2:8002" \ + --prefill-servers-urls disable \ + --decode-servers-urls "http://pd1:8003" +``` + Example usage: For E + PD setup: diff --git a/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py b/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py index c44c902690..3317b091fb 100644 --- a/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py +++ b/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py @@ -10,8 +10,8 @@ clusters: For MM input we: 1. Extract *every* image/audio/video item. - 2. Fire N concurrent requests to the encoder cluster - (one request per item, with **all text removed**). + 2. Send concurrent encoder requests with all text removed, + grouping images assigned to the same encoder. 3. Wait for all of them to succeed. 4. Forward the *original* request to a decode server. """ @@ -99,6 +99,11 @@ def validate_ec_consumer_routing( # Diagnostic switch: forward the original request to the decoder so the # only difference from the rewrite path is the rewrite itself. NO_REWRITE = False +# Maximum images per encoder subrequest; 0 leaves batches unlimited. +ENCODER_MAX_BATCH_SIZE = int(os.getenv("ENCODER_MAX_BATCH_SIZE", "0")) +if ENCODER_MAX_BATCH_SIZE < 0: + raise ValueError("ENCODER_MAX_BATCH_SIZE must be non-negative") + # Decode-side retries for a retryable internal error (`finish_reason="error"`, # e.g. an encoder embedding the connector could not deliver). Re-issuing runs # the encode again, which produces a fresh transfer. @@ -222,7 +227,7 @@ async def fanout_encoder_primer( req_id: str, consumer_zmq: str | None = None, ) -> tuple[dict[int, dict], dict[str, Any]]: - """1. Build one request *per MM item* with all text removed. + """1. Group images by encoder, retaining per-item round-robin assignment. 2. Send them concurrently to the encode cluster. 3. Raise if any of them fails. @@ -248,6 +253,7 @@ async def fanout_encoder_primer( item_uuids: dict[int, str] = {} item_transfer_ids: dict[int, str] = {} item_meta: dict[int, dict] = {} + transfer_items: dict[int, dict[str, str]] = {} ec_params: dict[str, Any] = {} # Round-robin over encode servers to distribute load a bit. The cursor @@ -259,29 +265,45 @@ async def fanout_encoder_primer( e_urls, encoder_rr_idx, len(mm_items) ) + groups: list[tuple[str, list[int]]] = [] + image_groups: dict[str, list[int]] = {} for idx, (item, target_url) in enumerate(zip(mm_items, url_cycle)): + if item["type"] == "image_url": + indices = image_groups.get(target_url) + if indices is None or ( + ENCODER_MAX_BATCH_SIZE and len(indices) >= ENCODER_MAX_BATCH_SIZE + ): + indices = [] + image_groups[target_url] = indices + groups.append((target_url, indices)) + indices.append(idx) + else: + groups.append((target_url, [idx])) + + for target_url, indices in groups: # Derive a *child* request id: :: - child_req_id = f"{req_id}:{idx}:{uuid.uuid4().hex[:6]}" + child_req_id = f"{req_id}:{indices[0]}:{uuid.uuid4().hex[:6]}" headers = {"x-request-id": child_req_id, "Content-Type": "application/json"} # With --no-rewrite the decoder still receives the raw image and derives # the cache key by hashing it, so the encoder must do the same -- passing # a uuid here would make the two disagree and silently defeat the EC # transfer, leaving the decoder to encode the image itself. - item_uuid = None if NO_REWRITE else content_uuid(item) - if item_uuid is not None: - item_uuids[idx] = item_uuid - transfer_id = uuid.uuid4().hex - item_transfer_ids[idx] = transfer_id + content = [] + for idx in indices: + item = mm_items[idx] + item_uuid = None if NO_REWRITE else content_uuid(item) + if item_uuid is not None: + item_uuids[idx] = item_uuid + item_transfer_ids[idx] = uuid.uuid4().hex + content.append(item if item_uuid is None else {**item, "uuid": item_uuid}) encoder_req = { "model": orig_request.get("model"), "messages": [ { "role": "user", - "content": [ - item if item_uuid is None else {**item, "uuid": item_uuid} - ], + "content": content, }, ], # No max_tokens cap: the encoder instance never samples, it finishes @@ -297,17 +319,13 @@ async def fanout_encoder_primer( if key in orig_request: encoder_req[key] = orig_request[key] if consumer_zmq is not None: - # No mm_hash here on purpose. The encoder's own - # `mm_features[i].identifier` is derived from the uuid *and* the - # engine's media_io_kwargs / mm_processor_kwargs, so this proxy - # cannot know it before the encoder runs. Sending the bare uuid - # would fail the connector's hash match and make the producer - # invent its own transfer id, which the consumer could then never - # cancel. Omitting it lets the connector match by position, which - # is exact: one encoder request carries exactly one item. + # The engine may rehash UUIDs with processing options. Match by + # position instead; batches contain only images in input order. encoder_req["ec_transfer_params"] = { "consumer_zmq": consumer_zmq, - "ec_items": [{"transfer_id": transfer_id}], + "ec_items": [ + {"transfer_id": item_transfer_ids[idx]} for idx in indices + ], } tasks.append( encode_session.post( @@ -320,7 +338,8 @@ async def fanout_encoder_primer( results = await asyncio.gather(*tasks, return_exceptions=True) # Fail fast if any sub-request failed - for idx, r in enumerate(results): + for (_, indices), r in zip(groups, results): + idx = indices[0] if isinstance(r, Exception): logger.error( "[%s] Encoder request #%d raised exception: %s", @@ -357,18 +376,45 @@ async def fanout_encoder_primer( logger.warning("[%s] Could not read encoder metadata #%d", req_id, idx) params = {} if params: - # One encoder request carries exactly one item, so there is a - # single reported entry. Do not key it by this proxy's uuid: when - # media_io_kwargs or mm_processor_kwargs are set the engine - # re-hashes the uuid together with them, so the encoder's own - # `mm_features[i].identifier` is a derived value this proxy cannot - # predict. Fall back to the sole entry, and carry the key the - # encoder actually used through as `ec_mm_hash`. - ec_mm_hash = item_uuids.get(idx) - reported = params.get(ec_mm_hash) - if reported is None and len(params) == 1: - ((ec_mm_hash, reported),) = params.items() - if reported: + by_index = {} + for mm_hash, reported in params.items(): + if len(indices) == 1: + break + for local_index in reported.get("item_indices", []): + if type(local_index) is not int or not 0 <= local_index < len( + indices + ): + raise HTTPException(502, "Invalid encoder item index") + if local_index in by_index: + raise HTTPException(502, "Duplicate encoder item index") + by_index[local_index] = (mm_hash, reported) + for local_index, idx in enumerate(indices): + matched = by_index.get(local_index) + if matched is None: + # Compatibility with encoders without position metadata. + mm_hash = item_uuids.get(idx) + if mm_hash in params: + matched = (mm_hash, params[mm_hash]) + elif len(indices) == 1 and len(params) == 1: + matched = next(iter(params.items())) + elif ( + mm_items[idx]["type"] == "video_url" + and (orig_request.get("mm_processor_kwargs") or {}).get( + "use_audio_in_video" + ) + and len(params) == 2 + and consumer_zmq is None + and all( + entry.keys() <= {"metadata", "item_indices"} + for entry in params.values() + ) + ): + # Keep raw video for its audio/video features when no transfer + # handles need to be forwarded. + continue + else: + raise HTTPException(502, "Encoder metadata cannot be matched") + ec_mm_hash, reported = matched metadata = reported.get("metadata") or {} if metadata: item_meta[idx] = { @@ -384,12 +430,16 @@ async def fanout_encoder_primer( # looks it up by mm_hash on the request, so carry it through. ec_params[ec_mm_hash] = reported if NO_REWRITE and consumer_zmq is not None: - ec_params.setdefault("ec_items", []).append( - {"mm_hash": ec_mm_hash, "transfer_id": item_transfer_ids[idx]} - ) + transfer_items[idx] = { + "mm_hash": ec_mm_hash, + "transfer_id": item_transfer_ids[idx], + } + + if transfer_items: + ec_params["ec_items"] = [transfer_items[idx] for idx in sorted(transfer_items)] logger.info( - "[%s] All %d encoder requests completed successfully", req_id, len(mm_items) + "[%s] All %d encoder requests completed successfully", req_id, len(groups) ) return item_meta, ec_params diff --git a/tests/v1/ec_connector/unit/test_ec_example_connector.py b/tests/v1/ec_connector/unit/test_ec_example_connector.py index dac809d1a4..a6645fccf4 100644 --- a/tests/v1/ec_connector/unit/test_ec_example_connector.py +++ b/tests/v1/ec_connector/unit/test_ec_example_connector.py @@ -16,6 +16,7 @@ from vllm.distributed.ec_transfer.ec_connector.example_connector import ( ECExampleConnectorMetadata, MMMeta, ) +from vllm.distributed.ec_transfer.ec_connector.utils import collect_ec_item_metadata from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange from vllm.v1.core.sched.output import SchedulerOutput @@ -41,6 +42,20 @@ class MockRequest: return self._token_counts[input_id] +def test_metadata_preserves_repeated_item_positions(): + request = MockRequest("request", ["A", "B", "A"], [4, 4, 4]) + resolver = Mock() + resolver.fields_for.return_value = {"image_grid_thw"} + for feature in request.mm_features: + feature.data = Mock() + feature.data.get_data.return_value = {"image_grid_thw": torch.tensor([1, 2, 2])} + items = collect_ec_item_metadata(request.mm_features, resolver) + assert items == { + "A": {"metadata": {"image_grid_thw": [1, 2, 2]}, "item_indices": [0, 2]}, + "B": {"metadata": {"image_grid_thw": [1, 2, 2]}, "item_indices": [1]}, + } + + @pytest.fixture def temp_storage(tmp_path): """Fixture providing temporary storage path.""" diff --git a/tests/v1/ec_connector/unit/test_epd_proxy_retry.py b/tests/v1/ec_connector/unit/test_epd_proxy_retry.py index 1a8c877c72..0c48dc84d6 100644 --- a/tests/v1/ec_connector/unit/test_epd_proxy_retry.py +++ b/tests/v1/ec_connector/unit/test_epd_proxy_retry.py @@ -9,6 +9,8 @@ change to them is what these tests catch. import asyncio import importlib.util from pathlib import Path +from types import SimpleNamespace +from typing import Any import httpx import msgspec @@ -83,6 +85,128 @@ class _EncoderSession: return _EncoderResponse(self._replies.pop(0)) +@pytest.mark.parametrize("no_rewrite", [False, True]) +@pytest.mark.parametrize( + "batch_size, expected_sizes", [(0, [3, 3]), (1, [1] * 6), (2, [2, 2, 1, 1])] +) +def test_image_batches_preserve_item_identity( + proxy, monkeypatch, no_rewrite, batch_size, expected_sizes +): + """Rehashed and repeated images retain metadata and per-occurrence transfers.""" + from vllm.distributed.ec_transfer.ec_connector.mooncake.scheduler import ( + ECMooncakeScheduler, + ) + + seen = [] + + class Session: + async def post(self, url, data=None, headers=None): + body = msgspec.json.decode(data) + seen.append((url, body)) + params: dict[str, dict[str, Any]] = {} + for index, item in enumerate(body["messages"][0]["content"]): + image = item["image_url"]["url"] + entry = params.setdefault( + image + "-processed", + { + "metadata": {"image_grid_thw": [1, 2, ord(image)]}, + "item_indices": [], + "peer_port": 1234, + }, + ) + entry["item_indices"].append(index) + # Response ordering must not affect the image-to-metadata mapping. + return _EncoderResponse(dict(reversed(list(params.items())))) + + monkeypatch.setattr(proxy, "encode_session", Session()) + monkeypatch.setattr(proxy, "NO_REWRITE", no_rewrite) + monkeypatch.setattr(proxy, "ENCODER_MAX_BATCH_SIZE", batch_size) + monkeypatch.setattr(proxy, "encoder_rr_idx", 0) + images = ["A", "A", "C", "D", "A", "F"] + body = { + "messages": [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": image}} + for image in images + ], + } + ], + "mm_processor_kwargs": {"max_pixels": 262144}, + } + meta, handles = asyncio.run( + proxy.fanout_encoder_primer( + body, ["http://e0", "http://e1"], "r", "tcp://consumer:1" + ) + ) + assert [len(b["messages"][0]["content"]) for _, b in seen] == expected_sizes + assert ( + len( + { + x["transfer_id"] + for _, b in seen + for x in b["ec_transfer_params"]["ec_items"] + } + ) + == 6 + ) + for url, batch in seen: + assert batch["mm_processor_kwargs"] == body["mm_processor_kwargs"] + for item in batch["messages"][0]["content"]: + assert item["image_url"]["url"] in ( + {"A", "C"} if url.startswith("http://e0/") else {"A", "D", "F"} + ) + assert [meta[i]["image_grid_thw"][-1] for i in range(6)] == list(map(ord, images)) + assert [meta[i]["ec_mm_hash"] for i in range(6)] == [ + x + "-processed" for x in images + ] + assert handles["A-processed"]["peer_port"] == 1234 + assert "ec_transfer_params" not in body + + transfers_by_encoder: list[list[dict[str, str]]] = [[], []] + for url, batch in seen: + rank = 0 if url.startswith("http://e0/") else 1 + transfers_by_encoder[rank].extend(batch["ec_transfer_params"]["ec_items"]) + expected_transfers = [ + transfers_by_encoder[i % 2][i // 2]["transfer_id"] for i in range(len(images)) + ] + params = ( + handles + if no_rewrite + else proxy.rewrite_for_decode(body, meta)["ec_transfer_params"] + ) + request = SimpleNamespace( + ec_transfer_params=params, + mm_features=[ + SimpleNamespace(identifier=image + "-processed") for image in images + ], + ) + assert [ + ECMooncakeScheduler._request_transfer_id(request, i) for i in range(len(images)) + ] == expected_transfers + + +def test_batch_rejects_ambiguous_metadata(proxy, monkeypatch): + monkeypatch.setattr( + proxy, + "encode_session", + _EncoderSession([{"unknown": {"metadata": {"image_grid_thw": [1, 2, 2]}}}]), + ) + body = { + "messages": [ + { + "content": [ + {"type": "image_url", "image_url": {"url": image}} + for image in ["A", "B"] + ] + } + ] + } + with pytest.raises(proxy.HTTPException, match="cannot be matched"): + asyncio.run(proxy.fanout_encoder_primer(body, ["http://e0"], "r")) + + def test_a_decode_retry_does_not_inherit_the_previous_handles(proxy, monkeypatch): """The retry loop re-enters `prepare_for_decode` with the same body. @@ -187,20 +311,21 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes( body = await request.json() seen[stage].append(body) if stage == "encode": - item = body["messages"][0]["content"][0] - mm_hash = item["uuid"] - if body.get("mm_processor_kwargs"): - mm_hash += "-processed" - return web.json_response( - { - "ec_transfer_params": { - mm_hash: { - "metadata": {"image_grid_thw": [[1, 2, 2]]}, - "peer_port": len(seen[stage]), - } - } - } - ) + params: dict[str, dict[str, Any]] = {} + for index, item in enumerate(body["messages"][0]["content"]): + mm_hash = item["uuid"] + if body.get("mm_processor_kwargs"): + mm_hash += "-processed" + entry = params.setdefault( + mm_hash, + { + "metadata": {"image_grid_thw": [[1, 2, 2]]}, + "peer_port": len(seen[stage]), + "item_indices": [], + }, + ) + entry["item_indices"].append(index) + return web.json_response({"ec_transfer_params": params}) if stage == "prefill": assert body["max_tokens"] == 1 and body["stream"] is False return web.json_response( @@ -263,7 +388,7 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes( finally: await proxy.on_shutdown() - assert len(seen["encode"]) == 4 + assert len(seen["encode"]) == 2 assert len(seen["decode"]) == 2 for requests in seen.values(): for forwarded in requests: @@ -296,7 +421,7 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes( if request_options: mm_hash += "-processed" assert set(final["ec_transfer_params"]) - {"ec_items"} == {mm_hash} - assert final["ec_transfer_params"][mm_hash]["peer_port"] in (3, 4) + assert final["ec_transfer_params"][mm_hash]["peer_port"] == 2 if prefill: assert final["kv_transfer_params"] == {"remote_block_ids": [2]} @@ -324,3 +449,62 @@ def test_pooled_connections_are_retired_before_the_server_closes_them( assert proxy.decode_session.connector._keepalive_timeout == pooled_for finally: asyncio.run(proxy.on_shutdown()) + + +@pytest.mark.parametrize( + "no_rewrite, transfer", + [(False, None), (True, None), (False, "push"), (False, "handle")], +) +def test_video_audio_fallback(proxy, monkeypatch, no_rewrite, transfer): + """Keep raw video and neighboring images, but never discard transfer handles.""" + import copy + + from vllm.distributed.ec_transfer.ec_connector.utils import collect_ec_item_metadata + + video = {"type": "video_url", "video_url": {"url": "video-with-audio"}} + images = [{"type": "image_url", "image_url": {"url": key}} for key in ("A", "B")] + body: dict[str, Any] = { + "messages": [{"content": [images[0], video, images[1]]}], + "mm_processor_kwargs": {"use_audio_in_video": True}, + } + original = copy.deepcopy(body) + # Collector indices describe two processed features from one video item. + video_metadata = collect_ec_item_metadata( + [SimpleNamespace(identifier=key, data=None) for key in ("video", "audio")], None + ) + if transfer == "handle": + video_metadata["audio"]["transfer_id"] = "reservation" + image_metadata = { + key: {"metadata": {"image_grid_thw": [1, 2, size]}, "item_indices": [i]} + for i, (key, size) in enumerate((("A", 2), ("B", 4))) + } + # Images form the first group, the singleton video the second. + monkeypatch.setattr(proxy, "ENCODER_MAX_BATCH_SIZE", 0) + monkeypatch.setattr(proxy, "NO_REWRITE", no_rewrite) + replies = _EncoderSession([image_metadata, video_metadata]) + monkeypatch.setattr(proxy, "encode_session", replies) + consumer = "tcp://consumer:1" if transfer == "push" else None + preparation = proxy.prepare_for_decode(body, "r", ["http://e0"], "", consumer) + if transfer: + with pytest.raises(proxy.HTTPException, match="cannot be matched"): + asyncio.run(preparation) + return + + prepared, _, _ = asyncio.run(preparation) + assert body == original + assert prepared["mm_processor_kwargs"] == body["mm_processor_kwargs"] + final = prepared["messages"][0]["content"] + assert final[1] == video + handles = prepared["ec_transfer_params"] + assert not video_metadata.keys() & handles.keys() + if no_rewrite: + assert final == original["messages"][0]["content"] + assert "ec_items" not in handles + else: + for position, image, key in ((0, images[0], "A"), (2, images[1], "B")): + assert final[position] == { + "type": "image_embeds", + "image_embeds": image_metadata[key]["metadata"], + "uuid": proxy.content_uuid(image), + } + assert [item["mm_hash"] for item in handles["ec_items"]] == ["A", "B"] diff --git a/tests/v1/ec_connector/unit/test_scheduler_nixl_producer.py b/tests/v1/ec_connector/unit/test_scheduler_nixl_producer.py index 156d98ae79..d7ef19b5f4 100644 --- a/tests/v1/ec_connector/unit/test_scheduler_nixl_producer.py +++ b/tests/v1/ec_connector/unit/test_scheduler_nixl_producer.py @@ -67,6 +67,7 @@ def test_request_finished_producer_emits_params(monkeypatch): assert params == { "h1": { "metadata": {}, + "item_indices": [0], "peer_host": "1.2.3.4", "peer_port": 5601, "size_bytes": 2 * 32 * 2, @@ -91,6 +92,7 @@ def test_request_finished_announces_not_ready_entry(monkeypatch): assert params == { "h1": { "metadata": {}, + "item_indices": [0], "peer_host": "1.2.3.4", "peer_port": 5601, "size_bytes": 2 * 32 * 2, @@ -110,5 +112,5 @@ def test_request_finished_skips_unallocated_entry(monkeypatch): assert delay is False # The item's placeholder metadata is still reported even though there's # no cache entry to transfer (empty "metadata": no fields, no transfer). - assert params == {"h1": {"metadata": {}}} + assert params == {"h1": {"metadata": {}, "item_indices": [0]}} s.shutdown() diff --git a/vllm/distributed/ec_transfer/ec_connector/utils.py b/vllm/distributed/ec_transfer/ec_connector/utils.py index f54d6877c1..35d26bb44b 100644 --- a/vllm/distributed/ec_transfer/ec_connector/utils.py +++ b/vllm/distributed/ec_transfer/ec_connector/utils.py @@ -60,15 +60,16 @@ def collect_ec_item_metadata( Keyed by mm_hash, each entry carries a `metadata` dict with whatever placeholder fields `resolver` says this model needs published for its - modality, so a consumer can skip the image transform. `data` is None for - items served from the processor cache, in which case the metadata is + modality, so a consumer can skip the image transform. `item_indices` + identifies every occurrence in `mm_features`, including repeated hashes. + `data` is None for items served from the processor cache, so the metadata is unavailable here and the consumer has to fall back to processing the media itself. A connector that also has transfer coordinates to report (e.g. NIXL peer_host/peer_port/size_bytes) merges those in alongside `metadata`, not into it. """ items: dict[str, dict[str, Any]] = {} - for feature in mm_features: + for index, feature in enumerate(mm_features): metadata: dict[str, Any] = {} if feature.data is not None: wanted = resolver.fields_for(feature.modality) @@ -81,7 +82,9 @@ def collect_ec_item_metadata( # Some metadata (e.g. Qwen3-VL video timestamps) is # produced as a plain list rather than a tensor. metadata[key] = value - items[feature.identifier] = {"metadata": metadata} + entry = items.setdefault(feature.identifier, {"item_indices": []}) + entry["metadata"] = metadata + entry["item_indices"].append(index) return items