[EPD] Remove duplicate image preprocessing in EPD and enable preprocess on GPU (#50390)
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
@@ -2,9 +2,10 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import safetensors
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
@@ -50,6 +51,8 @@ class ECExampleConnector(ECConnectorBase):
|
||||
super().__init__(vllm_config=vllm_config, role=role)
|
||||
# req_id -> index
|
||||
self._mm_datas_need_loads: dict[str, int] = {}
|
||||
self._model_config = vllm_config.model_config
|
||||
self._metadata_fields_cache: dict[str, set[str]] = {}
|
||||
transfer_config = vllm_config.ec_transfer_config
|
||||
if transfer_config is not None:
|
||||
self._storage_path = transfer_config.get_from_extra_config(
|
||||
@@ -165,6 +168,70 @@ class ECExampleConnector(ECConnectorBase):
|
||||
self._mm_datas_need_loads.clear()
|
||||
return meta
|
||||
|
||||
def _placeholder_metadata_fields(self, modality: str) -> set[str]:
|
||||
"""Which processed keys this model needs published for `modality`.
|
||||
|
||||
Read from `MultiModalDataParser.embedding_fields`, the same declaration
|
||||
the consumer's parser requires, so the two cannot drift. An empty set
|
||||
means the modality cannot be delivered out of band, and the consumer
|
||||
will process the media itself.
|
||||
"""
|
||||
if modality in self._metadata_fields_cache:
|
||||
return self._metadata_fields_cache[modality]
|
||||
|
||||
fields: set[str] = set()
|
||||
try:
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
info = MULTIMODAL_REGISTRY.create_processor(self._model_config).info
|
||||
fields = info.data_parser.placeholder_metadata_fields(modality)
|
||||
except Exception:
|
||||
# Reporting nothing is a safe degradation: the consumer falls back to
|
||||
# processing the media itself.
|
||||
logger.warning(
|
||||
"Could not determine the placeholder metadata fields for "
|
||||
"modality %s; the consumer will preprocess the media itself.",
|
||||
modality,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
self._metadata_fields_cache[modality] = fields
|
||||
return fields
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: "Request",
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""Report each item's cache key and grid so a consumer can skip the
|
||||
image transform.
|
||||
|
||||
A consumer only needs the grid to size the prompt's placeholder range;
|
||||
the embedding itself arrives through this connector. Reporting the grid
|
||||
the producer actually computed keeps the two sides in agreement without
|
||||
the caller re-deriving it from the raw media.
|
||||
"""
|
||||
if not self.is_producer:
|
||||
return False, None
|
||||
|
||||
items = []
|
||||
for feature in request.mm_features:
|
||||
metadata = {}
|
||||
# `data` is None for items served from the processor cache, in which
|
||||
# case the metadata is unavailable here and the consumer has to fall
|
||||
# back to processing the media itself.
|
||||
if feature.data is not None:
|
||||
wanted = self._placeholder_metadata_fields(feature.modality)
|
||||
metadata = {
|
||||
key: value.tolist()
|
||||
for key, value in feature.data.get_data().items()
|
||||
if key in wanted and isinstance(value, torch.Tensor)
|
||||
}
|
||||
items.append({"mm_hash": feature.identifier, **metadata})
|
||||
|
||||
if not items:
|
||||
return False, None
|
||||
return False, {"ec_items": items}
|
||||
|
||||
# ==============================
|
||||
# Helper functions
|
||||
# ==============================
|
||||
|
||||
Reference in New Issue
Block a user