[EC Connector] P2P NIXL + CPU EC Connector (#47941)
Signed-off-by: omerpaz95 <omerpaz95@gmail.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
69cf055936
commit
8a0a7ee40a
@@ -2,8 +2,88 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""EC connector helper utilities."""
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
from vllm.v1.outputs import ECConnectorOutput, ModelRunnerOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class PlaceholderMetadataResolver:
|
||||
"""Resolves which processed keys a model needs published per modality.
|
||||
|
||||
Reads `MultiModalDataParser.embedding_fields`, the same declaration the
|
||||
consumer's parser requires, so the two cannot drift. An empty set means
|
||||
the modality cannot be delivered out of band, and the consumer will
|
||||
process the media itself.
|
||||
"""
|
||||
|
||||
def __init__(self, model_config: "ModelConfig") -> None:
|
||||
self._model_config = model_config
|
||||
self._cache: dict[str, set[str]] = {}
|
||||
|
||||
def fields_for(self, modality: str) -> set[str]:
|
||||
if modality in self._cache:
|
||||
return self._cache[modality]
|
||||
|
||||
fields: set[str] = set()
|
||||
try:
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
info = MULTIMODAL_REGISTRY.create_processor(self._model_config).info
|
||||
fields = info.data_parser.placeholder_metadata_fields(modality)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Could not determine the placeholder metadata fields for "
|
||||
"modality %s; the consumer will preprocess the media itself.",
|
||||
modality,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
self._cache[modality] = fields
|
||||
return fields
|
||||
|
||||
|
||||
def collect_ec_item_metadata(
|
||||
mm_features: "list[MultiModalFeatureSpec]",
|
||||
resolver: PlaceholderMetadataResolver,
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""Build one `ec_transfer_params` entry per feature for `request_finished()`.
|
||||
|
||||
Keyed by mm_hash, each entry carries a `metadata` dict with whatever
|
||||
placeholder fields `resolver` says this model needs published for its
|
||||
modality, so a consumer can skip the image transform. `data` is None for
|
||||
items served from the processor cache, in which case the metadata is
|
||||
unavailable here and the consumer has to fall back to processing the
|
||||
media itself. A connector that also has transfer coordinates to report
|
||||
(e.g. NIXL peer_host/peer_port/size_bytes) merges those in alongside
|
||||
`metadata`, not into it.
|
||||
"""
|
||||
items: dict[str, dict[str, Any]] = {}
|
||||
for feature in mm_features:
|
||||
metadata: dict[str, Any] = {}
|
||||
if feature.data is not None:
|
||||
wanted = resolver.fields_for(feature.modality)
|
||||
for key, value in feature.data.get_data().items():
|
||||
if key not in wanted:
|
||||
continue
|
||||
if isinstance(value, torch.Tensor):
|
||||
metadata[key] = value.tolist()
|
||||
elif is_list_of(value, (int, float)):
|
||||
# Some metadata (e.g. Qwen3-VL video timestamps) is
|
||||
# produced as a plain list rather than a tensor.
|
||||
metadata[key] = value
|
||||
items[feature.identifier] = {"metadata": metadata}
|
||||
return items
|
||||
|
||||
|
||||
class ECOutputAggregator:
|
||||
"""Merge every worker's EC connector output onto the single
|
||||
|
||||
Reference in New Issue
Block a user