[Perf][EPD] Batch image requests per encoder (#57095)
Signed-off-by: Tianyu Guo <guoty@inferact.ai> Signed-off-by: Zhou ziheng <jiaranran2@gmail.com> Co-authored-by: Zhou ziheng <jiaranran2@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Zhou ziheng
OpenAI Codex
parent
99ea5f525d
commit
0d72627565
@@ -106,6 +106,22 @@ If you enable prefill instance (`--prefill-servers-urls` not disabled), you will
|
||||
| `--decode-servers-urls` | Comma-separated list of decode endpoints. Non-stream and stream paths both round-robin over this list. |
|
||||
| `--host`, `--port` | Bind address for the proxy itself (defaults: `0.0.0.0:8000`). |
|
||||
|
||||
The proxy batches images from the same user request assigned to the same encoder.
|
||||
Set the proxy environment variable `ENCODER_MAX_BATCH_SIZE` to limit the number
|
||||
of images per encoder subrequest. It defaults to `0` (unlimited); `1` sends each
|
||||
image separately. Audio and video remain separate subrequests.
|
||||
|
||||
For encoders with a smaller `--limit-mm-per-prompt` image limit than P/PD, set
|
||||
`ENCODER_MAX_BATCH_SIZE` no higher than the smallest encoder image limit.
|
||||
For example, for encoders configured with `--limit-mm-per-prompt '{"image": 2}'`:
|
||||
|
||||
```bash
|
||||
ENCODER_MAX_BATCH_SIZE=2 python disagg_epd_proxy.py \
|
||||
--encode-servers-urls "http://e1:8001,http://e2:8002" \
|
||||
--prefill-servers-urls disable \
|
||||
--decode-servers-urls "http://pd1:8003"
|
||||
```
|
||||
|
||||
Example usage:
|
||||
For E + PD setup:
|
||||
|
||||
|
||||
@@ -10,8 +10,8 @@ clusters:
|
||||
|
||||
For MM input we:
|
||||
1. Extract *every* image/audio/video item.
|
||||
2. Fire N concurrent requests to the encoder cluster
|
||||
(one request per item, with **all text removed**).
|
||||
2. Send concurrent encoder requests with all text removed,
|
||||
grouping images assigned to the same encoder.
|
||||
3. Wait for all of them to succeed.
|
||||
4. Forward the *original* request to a decode server.
|
||||
"""
|
||||
@@ -99,6 +99,11 @@ def validate_ec_consumer_routing(
|
||||
# Diagnostic switch: forward the original request to the decoder so the
|
||||
# only difference from the rewrite path is the rewrite itself.
|
||||
NO_REWRITE = False
|
||||
# Maximum images per encoder subrequest; 0 leaves batches unlimited.
|
||||
ENCODER_MAX_BATCH_SIZE = int(os.getenv("ENCODER_MAX_BATCH_SIZE", "0"))
|
||||
if ENCODER_MAX_BATCH_SIZE < 0:
|
||||
raise ValueError("ENCODER_MAX_BATCH_SIZE must be non-negative")
|
||||
|
||||
# Decode-side retries for a retryable internal error (`finish_reason="error"`,
|
||||
# e.g. an encoder embedding the connector could not deliver). Re-issuing runs
|
||||
# the encode again, which produces a fresh transfer.
|
||||
@@ -222,7 +227,7 @@ async def fanout_encoder_primer(
|
||||
req_id: str,
|
||||
consumer_zmq: str | None = None,
|
||||
) -> tuple[dict[int, dict], dict[str, Any]]:
|
||||
"""1. Build one request *per MM item* with all text removed.
|
||||
"""1. Group images by encoder, retaining per-item round-robin assignment.
|
||||
2. Send them concurrently to the encode cluster.
|
||||
3. Raise if any of them fails.
|
||||
|
||||
@@ -248,6 +253,7 @@ async def fanout_encoder_primer(
|
||||
item_uuids: dict[int, str] = {}
|
||||
item_transfer_ids: dict[int, str] = {}
|
||||
item_meta: dict[int, dict] = {}
|
||||
transfer_items: dict[int, dict[str, str]] = {}
|
||||
ec_params: dict[str, Any] = {}
|
||||
|
||||
# Round-robin over encode servers to distribute load a bit. The cursor
|
||||
@@ -259,29 +265,45 @@ async def fanout_encoder_primer(
|
||||
e_urls, encoder_rr_idx, len(mm_items)
|
||||
)
|
||||
|
||||
groups: list[tuple[str, list[int]]] = []
|
||||
image_groups: dict[str, list[int]] = {}
|
||||
for idx, (item, target_url) in enumerate(zip(mm_items, url_cycle)):
|
||||
if item["type"] == "image_url":
|
||||
indices = image_groups.get(target_url)
|
||||
if indices is None or (
|
||||
ENCODER_MAX_BATCH_SIZE and len(indices) >= ENCODER_MAX_BATCH_SIZE
|
||||
):
|
||||
indices = []
|
||||
image_groups[target_url] = indices
|
||||
groups.append((target_url, indices))
|
||||
indices.append(idx)
|
||||
else:
|
||||
groups.append((target_url, [idx]))
|
||||
|
||||
for target_url, indices in groups:
|
||||
# Derive a *child* request id: <parent>:<index>:<random-short>
|
||||
child_req_id = f"{req_id}:{idx}:{uuid.uuid4().hex[:6]}"
|
||||
child_req_id = f"{req_id}:{indices[0]}:{uuid.uuid4().hex[:6]}"
|
||||
headers = {"x-request-id": child_req_id, "Content-Type": "application/json"}
|
||||
|
||||
# With --no-rewrite the decoder still receives the raw image and derives
|
||||
# the cache key by hashing it, so the encoder must do the same -- passing
|
||||
# a uuid here would make the two disagree and silently defeat the EC
|
||||
# transfer, leaving the decoder to encode the image itself.
|
||||
item_uuid = None if NO_REWRITE else content_uuid(item)
|
||||
if item_uuid is not None:
|
||||
item_uuids[idx] = item_uuid
|
||||
transfer_id = uuid.uuid4().hex
|
||||
item_transfer_ids[idx] = transfer_id
|
||||
content = []
|
||||
for idx in indices:
|
||||
item = mm_items[idx]
|
||||
item_uuid = None if NO_REWRITE else content_uuid(item)
|
||||
if item_uuid is not None:
|
||||
item_uuids[idx] = item_uuid
|
||||
item_transfer_ids[idx] = uuid.uuid4().hex
|
||||
content.append(item if item_uuid is None else {**item, "uuid": item_uuid})
|
||||
|
||||
encoder_req = {
|
||||
"model": orig_request.get("model"),
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
item if item_uuid is None else {**item, "uuid": item_uuid}
|
||||
],
|
||||
"content": content,
|
||||
},
|
||||
],
|
||||
# No max_tokens cap: the encoder instance never samples, it finishes
|
||||
@@ -297,17 +319,13 @@ async def fanout_encoder_primer(
|
||||
if key in orig_request:
|
||||
encoder_req[key] = orig_request[key]
|
||||
if consumer_zmq is not None:
|
||||
# No mm_hash here on purpose. The encoder's own
|
||||
# `mm_features[i].identifier` is derived from the uuid *and* the
|
||||
# engine's media_io_kwargs / mm_processor_kwargs, so this proxy
|
||||
# cannot know it before the encoder runs. Sending the bare uuid
|
||||
# would fail the connector's hash match and make the producer
|
||||
# invent its own transfer id, which the consumer could then never
|
||||
# cancel. Omitting it lets the connector match by position, which
|
||||
# is exact: one encoder request carries exactly one item.
|
||||
# The engine may rehash UUIDs with processing options. Match by
|
||||
# position instead; batches contain only images in input order.
|
||||
encoder_req["ec_transfer_params"] = {
|
||||
"consumer_zmq": consumer_zmq,
|
||||
"ec_items": [{"transfer_id": transfer_id}],
|
||||
"ec_items": [
|
||||
{"transfer_id": item_transfer_ids[idx]} for idx in indices
|
||||
],
|
||||
}
|
||||
tasks.append(
|
||||
encode_session.post(
|
||||
@@ -320,7 +338,8 @@ async def fanout_encoder_primer(
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
# Fail fast if any sub-request failed
|
||||
for idx, r in enumerate(results):
|
||||
for (_, indices), r in zip(groups, results):
|
||||
idx = indices[0]
|
||||
if isinstance(r, Exception):
|
||||
logger.error(
|
||||
"[%s] Encoder request #%d raised exception: %s",
|
||||
@@ -357,18 +376,45 @@ async def fanout_encoder_primer(
|
||||
logger.warning("[%s] Could not read encoder metadata #%d", req_id, idx)
|
||||
params = {}
|
||||
if params:
|
||||
# One encoder request carries exactly one item, so there is a
|
||||
# single reported entry. Do not key it by this proxy's uuid: when
|
||||
# media_io_kwargs or mm_processor_kwargs are set the engine
|
||||
# re-hashes the uuid together with them, so the encoder's own
|
||||
# `mm_features[i].identifier` is a derived value this proxy cannot
|
||||
# predict. Fall back to the sole entry, and carry the key the
|
||||
# encoder actually used through as `ec_mm_hash`.
|
||||
ec_mm_hash = item_uuids.get(idx)
|
||||
reported = params.get(ec_mm_hash)
|
||||
if reported is None and len(params) == 1:
|
||||
((ec_mm_hash, reported),) = params.items()
|
||||
if reported:
|
||||
by_index = {}
|
||||
for mm_hash, reported in params.items():
|
||||
if len(indices) == 1:
|
||||
break
|
||||
for local_index in reported.get("item_indices", []):
|
||||
if type(local_index) is not int or not 0 <= local_index < len(
|
||||
indices
|
||||
):
|
||||
raise HTTPException(502, "Invalid encoder item index")
|
||||
if local_index in by_index:
|
||||
raise HTTPException(502, "Duplicate encoder item index")
|
||||
by_index[local_index] = (mm_hash, reported)
|
||||
for local_index, idx in enumerate(indices):
|
||||
matched = by_index.get(local_index)
|
||||
if matched is None:
|
||||
# Compatibility with encoders without position metadata.
|
||||
mm_hash = item_uuids.get(idx)
|
||||
if mm_hash in params:
|
||||
matched = (mm_hash, params[mm_hash])
|
||||
elif len(indices) == 1 and len(params) == 1:
|
||||
matched = next(iter(params.items()))
|
||||
elif (
|
||||
mm_items[idx]["type"] == "video_url"
|
||||
and (orig_request.get("mm_processor_kwargs") or {}).get(
|
||||
"use_audio_in_video"
|
||||
)
|
||||
and len(params) == 2
|
||||
and consumer_zmq is None
|
||||
and all(
|
||||
entry.keys() <= {"metadata", "item_indices"}
|
||||
for entry in params.values()
|
||||
)
|
||||
):
|
||||
# Keep raw video for its audio/video features when no transfer
|
||||
# handles need to be forwarded.
|
||||
continue
|
||||
else:
|
||||
raise HTTPException(502, "Encoder metadata cannot be matched")
|
||||
ec_mm_hash, reported = matched
|
||||
metadata = reported.get("metadata") or {}
|
||||
if metadata:
|
||||
item_meta[idx] = {
|
||||
@@ -384,12 +430,16 @@ async def fanout_encoder_primer(
|
||||
# looks it up by mm_hash on the request, so carry it through.
|
||||
ec_params[ec_mm_hash] = reported
|
||||
if NO_REWRITE and consumer_zmq is not None:
|
||||
ec_params.setdefault("ec_items", []).append(
|
||||
{"mm_hash": ec_mm_hash, "transfer_id": item_transfer_ids[idx]}
|
||||
)
|
||||
transfer_items[idx] = {
|
||||
"mm_hash": ec_mm_hash,
|
||||
"transfer_id": item_transfer_ids[idx],
|
||||
}
|
||||
|
||||
if transfer_items:
|
||||
ec_params["ec_items"] = [transfer_items[idx] for idx in sorted(transfer_items)]
|
||||
|
||||
logger.info(
|
||||
"[%s] All %d encoder requests completed successfully", req_id, len(mm_items)
|
||||
"[%s] All %d encoder requests completed successfully", req_id, len(groups)
|
||||
)
|
||||
return item_meta, ec_params
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.distributed.ec_transfer.ec_connector.example_connector import (
|
||||
ECExampleConnectorMetadata,
|
||||
MMMeta,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.utils import collect_ec_item_metadata
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
|
||||
@@ -41,6 +42,20 @@ class MockRequest:
|
||||
return self._token_counts[input_id]
|
||||
|
||||
|
||||
def test_metadata_preserves_repeated_item_positions():
|
||||
request = MockRequest("request", ["A", "B", "A"], [4, 4, 4])
|
||||
resolver = Mock()
|
||||
resolver.fields_for.return_value = {"image_grid_thw"}
|
||||
for feature in request.mm_features:
|
||||
feature.data = Mock()
|
||||
feature.data.get_data.return_value = {"image_grid_thw": torch.tensor([1, 2, 2])}
|
||||
items = collect_ec_item_metadata(request.mm_features, resolver)
|
||||
assert items == {
|
||||
"A": {"metadata": {"image_grid_thw": [1, 2, 2]}, "item_indices": [0, 2]},
|
||||
"B": {"metadata": {"image_grid_thw": [1, 2, 2]}, "item_indices": [1]},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def temp_storage(tmp_path):
|
||||
"""Fixture providing temporary storage path."""
|
||||
|
||||
@@ -9,6 +9,8 @@ change to them is what these tests catch.
|
||||
import asyncio
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import msgspec
|
||||
@@ -83,6 +85,128 @@ class _EncoderSession:
|
||||
return _EncoderResponse(self._replies.pop(0))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("no_rewrite", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"batch_size, expected_sizes", [(0, [3, 3]), (1, [1] * 6), (2, [2, 2, 1, 1])]
|
||||
)
|
||||
def test_image_batches_preserve_item_identity(
|
||||
proxy, monkeypatch, no_rewrite, batch_size, expected_sizes
|
||||
):
|
||||
"""Rehashed and repeated images retain metadata and per-occurrence transfers."""
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.scheduler import (
|
||||
ECMooncakeScheduler,
|
||||
)
|
||||
|
||||
seen = []
|
||||
|
||||
class Session:
|
||||
async def post(self, url, data=None, headers=None):
|
||||
body = msgspec.json.decode(data)
|
||||
seen.append((url, body))
|
||||
params: dict[str, dict[str, Any]] = {}
|
||||
for index, item in enumerate(body["messages"][0]["content"]):
|
||||
image = item["image_url"]["url"]
|
||||
entry = params.setdefault(
|
||||
image + "-processed",
|
||||
{
|
||||
"metadata": {"image_grid_thw": [1, 2, ord(image)]},
|
||||
"item_indices": [],
|
||||
"peer_port": 1234,
|
||||
},
|
||||
)
|
||||
entry["item_indices"].append(index)
|
||||
# Response ordering must not affect the image-to-metadata mapping.
|
||||
return _EncoderResponse(dict(reversed(list(params.items()))))
|
||||
|
||||
monkeypatch.setattr(proxy, "encode_session", Session())
|
||||
monkeypatch.setattr(proxy, "NO_REWRITE", no_rewrite)
|
||||
monkeypatch.setattr(proxy, "ENCODER_MAX_BATCH_SIZE", batch_size)
|
||||
monkeypatch.setattr(proxy, "encoder_rr_idx", 0)
|
||||
images = ["A", "A", "C", "D", "A", "F"]
|
||||
body = {
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": image}}
|
||||
for image in images
|
||||
],
|
||||
}
|
||||
],
|
||||
"mm_processor_kwargs": {"max_pixels": 262144},
|
||||
}
|
||||
meta, handles = asyncio.run(
|
||||
proxy.fanout_encoder_primer(
|
||||
body, ["http://e0", "http://e1"], "r", "tcp://consumer:1"
|
||||
)
|
||||
)
|
||||
assert [len(b["messages"][0]["content"]) for _, b in seen] == expected_sizes
|
||||
assert (
|
||||
len(
|
||||
{
|
||||
x["transfer_id"]
|
||||
for _, b in seen
|
||||
for x in b["ec_transfer_params"]["ec_items"]
|
||||
}
|
||||
)
|
||||
== 6
|
||||
)
|
||||
for url, batch in seen:
|
||||
assert batch["mm_processor_kwargs"] == body["mm_processor_kwargs"]
|
||||
for item in batch["messages"][0]["content"]:
|
||||
assert item["image_url"]["url"] in (
|
||||
{"A", "C"} if url.startswith("http://e0/") else {"A", "D", "F"}
|
||||
)
|
||||
assert [meta[i]["image_grid_thw"][-1] for i in range(6)] == list(map(ord, images))
|
||||
assert [meta[i]["ec_mm_hash"] for i in range(6)] == [
|
||||
x + "-processed" for x in images
|
||||
]
|
||||
assert handles["A-processed"]["peer_port"] == 1234
|
||||
assert "ec_transfer_params" not in body
|
||||
|
||||
transfers_by_encoder: list[list[dict[str, str]]] = [[], []]
|
||||
for url, batch in seen:
|
||||
rank = 0 if url.startswith("http://e0/") else 1
|
||||
transfers_by_encoder[rank].extend(batch["ec_transfer_params"]["ec_items"])
|
||||
expected_transfers = [
|
||||
transfers_by_encoder[i % 2][i // 2]["transfer_id"] for i in range(len(images))
|
||||
]
|
||||
params = (
|
||||
handles
|
||||
if no_rewrite
|
||||
else proxy.rewrite_for_decode(body, meta)["ec_transfer_params"]
|
||||
)
|
||||
request = SimpleNamespace(
|
||||
ec_transfer_params=params,
|
||||
mm_features=[
|
||||
SimpleNamespace(identifier=image + "-processed") for image in images
|
||||
],
|
||||
)
|
||||
assert [
|
||||
ECMooncakeScheduler._request_transfer_id(request, i) for i in range(len(images))
|
||||
] == expected_transfers
|
||||
|
||||
|
||||
def test_batch_rejects_ambiguous_metadata(proxy, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
proxy,
|
||||
"encode_session",
|
||||
_EncoderSession([{"unknown": {"metadata": {"image_grid_thw": [1, 2, 2]}}}]),
|
||||
)
|
||||
body = {
|
||||
"messages": [
|
||||
{
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": image}}
|
||||
for image in ["A", "B"]
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
with pytest.raises(proxy.HTTPException, match="cannot be matched"):
|
||||
asyncio.run(proxy.fanout_encoder_primer(body, ["http://e0"], "r"))
|
||||
|
||||
|
||||
def test_a_decode_retry_does_not_inherit_the_previous_handles(proxy, monkeypatch):
|
||||
"""The retry loop re-enters `prepare_for_decode` with the same body.
|
||||
|
||||
@@ -187,20 +311,21 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes(
|
||||
body = await request.json()
|
||||
seen[stage].append(body)
|
||||
if stage == "encode":
|
||||
item = body["messages"][0]["content"][0]
|
||||
mm_hash = item["uuid"]
|
||||
if body.get("mm_processor_kwargs"):
|
||||
mm_hash += "-processed"
|
||||
return web.json_response(
|
||||
{
|
||||
"ec_transfer_params": {
|
||||
mm_hash: {
|
||||
"metadata": {"image_grid_thw": [[1, 2, 2]]},
|
||||
"peer_port": len(seen[stage]),
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
params: dict[str, dict[str, Any]] = {}
|
||||
for index, item in enumerate(body["messages"][0]["content"]):
|
||||
mm_hash = item["uuid"]
|
||||
if body.get("mm_processor_kwargs"):
|
||||
mm_hash += "-processed"
|
||||
entry = params.setdefault(
|
||||
mm_hash,
|
||||
{
|
||||
"metadata": {"image_grid_thw": [[1, 2, 2]]},
|
||||
"peer_port": len(seen[stage]),
|
||||
"item_indices": [],
|
||||
},
|
||||
)
|
||||
entry["item_indices"].append(index)
|
||||
return web.json_response({"ec_transfer_params": params})
|
||||
if stage == "prefill":
|
||||
assert body["max_tokens"] == 1 and body["stream"] is False
|
||||
return web.json_response(
|
||||
@@ -263,7 +388,7 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes(
|
||||
finally:
|
||||
await proxy.on_shutdown()
|
||||
|
||||
assert len(seen["encode"]) == 4
|
||||
assert len(seen["encode"]) == 2
|
||||
assert len(seen["decode"]) == 2
|
||||
for requests in seen.values():
|
||||
for forwarded in requests:
|
||||
@@ -296,7 +421,7 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes(
|
||||
if request_options:
|
||||
mm_hash += "-processed"
|
||||
assert set(final["ec_transfer_params"]) - {"ec_items"} == {mm_hash}
|
||||
assert final["ec_transfer_params"][mm_hash]["peer_port"] in (3, 4)
|
||||
assert final["ec_transfer_params"][mm_hash]["peer_port"] == 2
|
||||
if prefill:
|
||||
assert final["kv_transfer_params"] == {"remote_block_ids": [2]}
|
||||
|
||||
@@ -324,3 +449,62 @@ def test_pooled_connections_are_retired_before_the_server_closes_them(
|
||||
assert proxy.decode_session.connector._keepalive_timeout == pooled_for
|
||||
finally:
|
||||
asyncio.run(proxy.on_shutdown())
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"no_rewrite, transfer",
|
||||
[(False, None), (True, None), (False, "push"), (False, "handle")],
|
||||
)
|
||||
def test_video_audio_fallback(proxy, monkeypatch, no_rewrite, transfer):
|
||||
"""Keep raw video and neighboring images, but never discard transfer handles."""
|
||||
import copy
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.utils import collect_ec_item_metadata
|
||||
|
||||
video = {"type": "video_url", "video_url": {"url": "video-with-audio"}}
|
||||
images = [{"type": "image_url", "image_url": {"url": key}} for key in ("A", "B")]
|
||||
body: dict[str, Any] = {
|
||||
"messages": [{"content": [images[0], video, images[1]]}],
|
||||
"mm_processor_kwargs": {"use_audio_in_video": True},
|
||||
}
|
||||
original = copy.deepcopy(body)
|
||||
# Collector indices describe two processed features from one video item.
|
||||
video_metadata = collect_ec_item_metadata(
|
||||
[SimpleNamespace(identifier=key, data=None) for key in ("video", "audio")], None
|
||||
)
|
||||
if transfer == "handle":
|
||||
video_metadata["audio"]["transfer_id"] = "reservation"
|
||||
image_metadata = {
|
||||
key: {"metadata": {"image_grid_thw": [1, 2, size]}, "item_indices": [i]}
|
||||
for i, (key, size) in enumerate((("A", 2), ("B", 4)))
|
||||
}
|
||||
# Images form the first group, the singleton video the second.
|
||||
monkeypatch.setattr(proxy, "ENCODER_MAX_BATCH_SIZE", 0)
|
||||
monkeypatch.setattr(proxy, "NO_REWRITE", no_rewrite)
|
||||
replies = _EncoderSession([image_metadata, video_metadata])
|
||||
monkeypatch.setattr(proxy, "encode_session", replies)
|
||||
consumer = "tcp://consumer:1" if transfer == "push" else None
|
||||
preparation = proxy.prepare_for_decode(body, "r", ["http://e0"], "", consumer)
|
||||
if transfer:
|
||||
with pytest.raises(proxy.HTTPException, match="cannot be matched"):
|
||||
asyncio.run(preparation)
|
||||
return
|
||||
|
||||
prepared, _, _ = asyncio.run(preparation)
|
||||
assert body == original
|
||||
assert prepared["mm_processor_kwargs"] == body["mm_processor_kwargs"]
|
||||
final = prepared["messages"][0]["content"]
|
||||
assert final[1] == video
|
||||
handles = prepared["ec_transfer_params"]
|
||||
assert not video_metadata.keys() & handles.keys()
|
||||
if no_rewrite:
|
||||
assert final == original["messages"][0]["content"]
|
||||
assert "ec_items" not in handles
|
||||
else:
|
||||
for position, image, key in ((0, images[0], "A"), (2, images[1], "B")):
|
||||
assert final[position] == {
|
||||
"type": "image_embeds",
|
||||
"image_embeds": image_metadata[key]["metadata"],
|
||||
"uuid": proxy.content_uuid(image),
|
||||
}
|
||||
assert [item["mm_hash"] for item in handles["ec_items"]] == ["A", "B"]
|
||||
|
||||
@@ -67,6 +67,7 @@ def test_request_finished_producer_emits_params(monkeypatch):
|
||||
assert params == {
|
||||
"h1": {
|
||||
"metadata": {},
|
||||
"item_indices": [0],
|
||||
"peer_host": "1.2.3.4",
|
||||
"peer_port": 5601,
|
||||
"size_bytes": 2 * 32 * 2,
|
||||
@@ -91,6 +92,7 @@ def test_request_finished_announces_not_ready_entry(monkeypatch):
|
||||
assert params == {
|
||||
"h1": {
|
||||
"metadata": {},
|
||||
"item_indices": [0],
|
||||
"peer_host": "1.2.3.4",
|
||||
"peer_port": 5601,
|
||||
"size_bytes": 2 * 32 * 2,
|
||||
@@ -110,5 +112,5 @@ def test_request_finished_skips_unallocated_entry(monkeypatch):
|
||||
assert delay is False
|
||||
# The item's placeholder metadata is still reported even though there's
|
||||
# no cache entry to transfer (empty "metadata": no fields, no transfer).
|
||||
assert params == {"h1": {"metadata": {}}}
|
||||
assert params == {"h1": {"metadata": {}, "item_indices": [0]}}
|
||||
s.shutdown()
|
||||
|
||||
@@ -60,15 +60,16 @@ def collect_ec_item_metadata(
|
||||
|
||||
Keyed by mm_hash, each entry carries a `metadata` dict with whatever
|
||||
placeholder fields `resolver` says this model needs published for its
|
||||
modality, so a consumer can skip the image transform. `data` is None for
|
||||
items served from the processor cache, in which case the metadata is
|
||||
modality, so a consumer can skip the image transform. `item_indices`
|
||||
identifies every occurrence in `mm_features`, including repeated hashes.
|
||||
`data` is None for items served from the processor cache, so the metadata is
|
||||
unavailable here and the consumer has to fall back to processing the
|
||||
media itself. A connector that also has transfer coordinates to report
|
||||
(e.g. NIXL peer_host/peer_port/size_bytes) merges those in alongside
|
||||
`metadata`, not into it.
|
||||
"""
|
||||
items: dict[str, dict[str, Any]] = {}
|
||||
for feature in mm_features:
|
||||
for index, feature in enumerate(mm_features):
|
||||
metadata: dict[str, Any] = {}
|
||||
if feature.data is not None:
|
||||
wanted = resolver.fields_for(feature.modality)
|
||||
@@ -81,7 +82,9 @@ def collect_ec_item_metadata(
|
||||
# Some metadata (e.g. Qwen3-VL video timestamps) is
|
||||
# produced as a plain list rather than a tensor.
|
||||
metadata[key] = value
|
||||
items[feature.identifier] = {"metadata": metadata}
|
||||
entry = items.setdefault(feature.identifier, {"item_indices": []})
|
||||
entry["metadata"] = metadata
|
||||
entry["item_indices"].append(index)
|
||||
return items
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user