From 1678b396270406c27fcab8f5b86b21fd305ac605 Mon Sep 17 00:00:00 2001 From: jiangkuaixue123 Date: Mon, 14 Sep 2026 18:05:13 +0800 Subject: [PATCH] [Bugfix][EPD] Preserve media processing options in encoder requests (#56786) Signed-off-by: jiangkuaixue123 --- .../disaggregated_encoder/disagg_epd_proxy.py | 11 ++++- .../ec_connector/unit/test_epd_proxy_retry.py | 41 ++++++++++++++++--- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py b/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py index 3912c1c67a..830da9b5d6 100644 --- a/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py +++ b/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py @@ -278,7 +278,6 @@ async def fanout_encoder_primer( item_transfer_ids[idx] = transfer_id encoder_req = { - # You *may* need to keep additional fields "model": orig_request.get("model"), "messages": [ { @@ -292,6 +291,14 @@ async def fanout_encoder_primer( # once the prompt is encoded and its embeddings are published. "stream": False, } + for key in ( + "mm_processor_kwargs", + "media_io_kwargs", + "priority", + "session_id", + ): + if key in orig_request: + encoder_req[key] = orig_request[key] if consumer_zmq is not None: # No mm_hash here on purpose. The encoder's own # `mm_features[i].identifier` is derived from the uuid *and* the @@ -378,7 +385,7 @@ async def fanout_encoder_primer( # connector's own handle on the published embedding (for NIXL, # peer_host/peer_port/size_bytes). The decoder's connector # looks it up by mm_hash on the request, so carry it through. - ec_params[item_uuids.get(idx, ec_mm_hash)] = reported + ec_params[ec_mm_hash] = reported if NO_REWRITE and consumer_zmq is not None: ec_params.setdefault("ec_items", []).append( {"mm_hash": ec_mm_hash, "transfer_id": item_transfer_ids[idx]} diff --git a/tests/v1/ec_connector/unit/test_epd_proxy_retry.py b/tests/v1/ec_connector/unit/test_epd_proxy_retry.py index 9a92b1fdbc..1a8c877c72 100644 --- a/tests/v1/ec_connector/unit/test_epd_proxy_retry.py +++ b/tests/v1/ec_connector/unit/test_epd_proxy_retry.py @@ -116,7 +116,9 @@ def test_a_decode_retry_does_not_inherit_the_previous_handles(proxy, monkeypatch first, _, _ = asyncio.run(proxy.prepare_for_decode(body, *args)) reported = first["ec_transfer_params"] - assert [handle] == [value for key, value in reported.items() if key != "ec_items"] + assert {key: value for key, value in reported.items() if key != "ec_items"} == { + "encoder-side-hash": handle + } assert "ec_transfer_params" not in body # Attempt 2's encode reports nothing: decode must be told nothing. @@ -154,10 +156,22 @@ def test_raw_media_keeps_encoder_transfer_identity(proxy, monkeypatch): @pytest.mark.asyncio @pytest.mark.parametrize("stream", [False, True]) @pytest.mark.parametrize("prefill", [False, True]) +@pytest.mark.parametrize( + "request_options", + [ + {}, + { + "mm_processor_kwargs": {"max_pixels": 262144}, + "media_io_kwargs": {"image": {"image_mode": "RGB"}}, + "priority": -3, + "session_id": "session-123", + }, + ], +) async def test_http_roundtrip_preserves_payload_and_response_bytes( - proxy, monkeypatch, stream, prefill + proxy, monkeypatch, stream, prefill, request_options ): - """Exercise real HTTP hops, rewrite and a decode retry without model servers.""" + """Preserve media and scheduling context across HTTP hops and retries.""" seen: dict[str, list[dict]] = {"encode": [], "prefill": [], "decode": []} prefix = b'data: {"content":"' # Split a multibyte character at the old 1024-byte forwarding boundary. @@ -174,10 +188,13 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes( seen[stage].append(body) if stage == "encode": item = body["messages"][0]["content"][0] + mm_hash = item["uuid"] + if body.get("mm_processor_kwargs"): + mm_hash += "-processed" return web.json_response( { "ec_transfer_params": { - item["uuid"]: { + mm_hash: { "metadata": {"image_grid_thw": [[1, 2, 2]]}, "peer_port": len(seen[stage]), } @@ -230,6 +247,7 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes( "max_tokens": 32, "seed": 42, "structured_outputs": {"choice": ["A", "B"]}, + **request_options, } await proxy.on_startup() try: @@ -247,6 +265,15 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes( assert len(seen["encode"]) == 4 assert len(seen["decode"]) == 2 + for requests in seen.values(): + for forwarded in requests: + for key in ( + "mm_processor_kwargs", + "media_io_kwargs", + "priority", + "session_id", + ): + assert forwarded.get(key) == body.get(key) final = seen["decode"][-1] for key in [ "model", @@ -265,7 +292,11 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes( "image_embeds": {"image_grid_thw": [1, 2, 2]}, "uuid": proxy.content_uuid(item), } - assert final["ec_transfer_params"][proxy.content_uuid(item)]["peer_port"] in (3, 4) + mm_hash = proxy.content_uuid(item) + if request_options: + mm_hash += "-processed" + assert set(final["ec_transfer_params"]) - {"ec_items"} == {mm_hash} + assert final["ec_transfer_params"][mm_hash]["peer_port"] in (3, 4) if prefill: assert final["kv_transfer_params"] == {"remote_block_ids": [2]}