[EPD] Allow KV consumers to omit MM embeddings (#52697)
Signed-off-by: zhenwei-intel <zhenwei.liu@intel.com>
This commit is contained in:
@@ -183,6 +183,7 @@ run_epd_1e_1pd() {
|
||||
--port "$PREFILL_DECODE_PORT" \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--enforce-eager \
|
||||
--enable-mm-embeds \
|
||||
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
|
||||
--enable-request-id-headers \
|
||||
--max-num-seqs "$MAX_NUM_SEQS" \
|
||||
@@ -389,6 +390,7 @@ run_epd_1e_1p_1d() {
|
||||
--port "$PREFILL_PORT" \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--enforce-eager \
|
||||
--enable-mm-embeds \
|
||||
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
|
||||
--enable-request-id-headers \
|
||||
--max-num-seqs "$MAX_NUM_SEQS" \
|
||||
@@ -415,6 +417,7 @@ run_epd_1e_1p_1d() {
|
||||
--port "$DECODE_PORT" \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--enforce-eager \
|
||||
--enable-mm-embeds \
|
||||
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
|
||||
--enable-request-id-headers \
|
||||
--max-num-seqs "$MAX_NUM_SEQS" \
|
||||
|
||||
@@ -236,17 +236,17 @@ class MultiModalConfig:
|
||||
- "direct_rpc": Use msgspec serialization via RPC
|
||||
- "torch_shm": Use torch.multiprocessing shared memory for zero-copy IPC
|
||||
Defaults to "direct_rpc". """
|
||||
mm_embeds_from_ec_connector: bool = False
|
||||
allow_missing_mm_embeddings: bool = False
|
||||
"""Whether a pre-computed-embedding input may omit the `*_embeds` tensor.
|
||||
|
||||
In an encode/prefill/decode (EPD) deployment the encoder instance publishes
|
||||
embeddings through the EC connector, so the request that reaches the
|
||||
prefill/decode instance only needs to carry the grid/size metadata that
|
||||
sizes the placeholder range — the embeddings themselves come from the
|
||||
connector, keyed by `mm_hash`.
|
||||
embeddings through the EC connector. An EC consumer loads those embeddings
|
||||
from the connector, while a KV consumer receives the resulting prompt KV
|
||||
cache. Their requests only need the grid/size metadata that sizes the
|
||||
placeholder range.
|
||||
|
||||
Derived, not user-settable: `VllmConfig.__post_init__` sets this to True
|
||||
exactly on EC consumers. Everywhere else it stays False so that a request
|
||||
on EC and KV consumers. Everywhere else it stays False so that a request
|
||||
which forgets its embeddings still fails fast in the frontend, with a clear
|
||||
error, rather than deep inside the model."""
|
||||
|
||||
|
||||
+13
-13
@@ -1577,7 +1577,7 @@ class VllmConfig:
|
||||
)
|
||||
current_platform.check_and_update_config(self)
|
||||
|
||||
self._resolve_mm_embeds_from_ec_connector()
|
||||
self._resolve_allow_missing_mm_embeddings()
|
||||
self._resolve_mm_processor_device()
|
||||
self._validate_mm_processor_device()
|
||||
|
||||
@@ -2254,14 +2254,13 @@ class VllmConfig:
|
||||
f"kernel_config={self.kernel_config!r}"
|
||||
)
|
||||
|
||||
def _resolve_mm_embeds_from_ec_connector(self) -> None:
|
||||
"""Allow `*_embeds` to be omitted only where the connector supplies them.
|
||||
def _resolve_allow_missing_mm_embeddings(self) -> None:
|
||||
"""Allow `*_embeds` tensors to be omitted on disaggregated consumers.
|
||||
|
||||
That is exactly an EC consumer: the encoder instance publishes the
|
||||
embeddings through the EC connector, so the request only has to carry
|
||||
the grid metadata that sizes the placeholder range. On every other
|
||||
deployment a missing `*_embeds` is a client error and must keep failing
|
||||
fast in the frontend.
|
||||
An EC consumer loads embeddings from its connector. A KV consumer
|
||||
receives the prompt KV produced from those embeddings, so it does not
|
||||
need the tensors either. On every other deployment a missing tensor is
|
||||
a client error and must keep failing fast in the frontend.
|
||||
"""
|
||||
model_config = self.model_config
|
||||
if model_config is None:
|
||||
@@ -2271,15 +2270,16 @@ class VllmConfig:
|
||||
return
|
||||
|
||||
ec_config = self.ec_transfer_config
|
||||
kv_config = self.kv_transfer_config
|
||||
# Derived, so overwrite unconditionally rather than honouring a value
|
||||
# that was set by hand.
|
||||
mm_config.mm_embeds_from_ec_connector = (
|
||||
mm_config.allow_missing_mm_embeddings = (
|
||||
ec_config is not None and ec_config.is_ec_consumer
|
||||
)
|
||||
if mm_config.mm_embeds_from_ec_connector:
|
||||
) or (kv_config is not None and kv_config.is_kv_consumer)
|
||||
if mm_config.allow_missing_mm_embeddings:
|
||||
logger.info_once(
|
||||
"EC consumer: pre-computed-embedding inputs may omit the "
|
||||
"embedding tensor; embeddings are loaded from the EC connector."
|
||||
"EC/KV consumer: pre-computed-embedding inputs may "
|
||||
"omit the embedding tensor."
|
||||
)
|
||||
|
||||
def _resolve_mm_processor_device(self) -> None:
|
||||
|
||||
@@ -104,7 +104,7 @@ class ColQwen3ProcessingInfo(Qwen3VLProcessingInfo):
|
||||
spatial_merge_size,
|
||||
video_needs_metadata=self._supports_video,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -101,7 +101,7 @@ class ColQwen3_5ProcessingInfo(Qwen3_5ProcessingInfo):
|
||||
spatial_merge_size,
|
||||
video_needs_metadata=self._supports_video,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -599,7 +599,7 @@ class HunYuanVLProcessingInfo(BaseProcessingInfo):
|
||||
def get_data_parser(self):
|
||||
return HunYuanVLMultiModalDataParser(
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
|
||||
@@ -936,7 +936,7 @@ class KeyeProcessingInfo(BaseProcessingInfo):
|
||||
def get_data_parser(self):
|
||||
return KeyeMultiModalDataParser(
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
def get_supported_mm_limits(
|
||||
|
||||
@@ -363,7 +363,7 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo):
|
||||
def get_data_parser(self):
|
||||
return KeyeVL1_5MultiModalDataParser(
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
def get_max_frame_per_video(self) -> int:
|
||||
|
||||
@@ -1279,7 +1279,7 @@ class LlavaOnevision2ProcessingInfo(BaseProcessingInfo):
|
||||
return LlavaOnevision2MultiModalDataParser(
|
||||
self.get_hf_config().vision_config.spatial_merge_size,
|
||||
video_needs_metadata=True,
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object):
|
||||
|
||||
@@ -653,7 +653,7 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
|
||||
def get_data_parser(self):
|
||||
return MiniCPMVMultiModalDataParser(
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
def get_model_version(self):
|
||||
|
||||
@@ -57,7 +57,7 @@ class OpenCUAProcessingInfo(Qwen2VLProcessingInfo):
|
||||
return Qwen2VLMultiModalDataParser(
|
||||
self.get_hf_config().vision_config.spatial_merge_size,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
def get_hf_config(self):
|
||||
|
||||
@@ -373,7 +373,7 @@ class Qwen2_5OmniThinkerProcessingInfo(
|
||||
target_sr=feature_extractor.sampling_rate,
|
||||
target_channels=self.get_target_channels(),
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
def get_target_channels(self) -> int:
|
||||
|
||||
@@ -857,7 +857,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
|
||||
return Qwen2VLMultiModalDataParser(
|
||||
self.get_hf_config().vision_config.spatial_merge_size,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
|
||||
@@ -910,7 +910,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
|
||||
self.get_hf_config().vision_config.spatial_merge_size,
|
||||
video_needs_metadata=True,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
def _get_vision_info(
|
||||
|
||||
@@ -554,10 +554,9 @@ class MultiModalDataParser:
|
||||
embedding inputs. If provided, validates that user-supplied
|
||||
embeddings have the correct hidden size to prevent crashes
|
||||
during model inference.
|
||||
embeds_from_ec_connector (bool): Whether pre-computed embeddings may be
|
||||
absent from the request because an encode/prefill/decode encoder
|
||||
instance publishes them through an EC connector instead. Derived by
|
||||
`BaseProcessingInfo.embeds_from_ec_connector`.
|
||||
allow_missing_mm_embeddings (bool): Whether pre-computed embedding
|
||||
tensors may be absent from the request on a disaggregated consumer.
|
||||
Derived by `BaseProcessingInfo.allow_missing_mm_embeddings`.
|
||||
"""
|
||||
|
||||
embedding_fields: Mapping[str, Mapping[str, EmbeddingFieldRole]] = {}
|
||||
@@ -594,7 +593,7 @@ class MultiModalDataParser:
|
||||
"""
|
||||
metadata = self.placeholder_metadata_fields(modality)
|
||||
values = set(self.embedding_fields.get(modality, {})) - metadata
|
||||
if self.embeds_from_ec_connector:
|
||||
if self.allow_missing_mm_embeddings:
|
||||
return metadata, values
|
||||
return metadata | values, set()
|
||||
|
||||
@@ -606,11 +605,11 @@ class MultiModalDataParser:
|
||||
audio_resample_method: Literal["pyav", "scipy", "soxr"] = "pyav",
|
||||
video_needs_metadata: bool = False,
|
||||
expected_hidden_size: int | None = None,
|
||||
embeds_from_ec_connector: bool = False,
|
||||
allow_missing_mm_embeddings: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.embeds_from_ec_connector = embeds_from_ec_connector
|
||||
self.allow_missing_mm_embeddings = allow_missing_mm_embeddings
|
||||
|
||||
self.audio_resampler = AudioResampler(
|
||||
target_sr=target_sr,
|
||||
|
||||
@@ -367,15 +367,10 @@ class BaseProcessingInfo:
|
||||
return None
|
||||
|
||||
@property
|
||||
def embeds_from_ec_connector(self) -> bool:
|
||||
"""Whether pre-computed embeddings may arrive outside the request.
|
||||
|
||||
True only on an EC consumer, where an encode/prefill/decode encoder
|
||||
instance publishes them through the connector instead, so the request
|
||||
carries only the metadata that sizes the placeholder range.
|
||||
"""
|
||||
def allow_missing_mm_embeddings(self) -> bool:
|
||||
"""Whether pre-computed embedding tensors may be omitted."""
|
||||
mm_config = self.ctx.model_config.multimodal_config
|
||||
return mm_config is not None and mm_config.mm_embeds_from_ec_connector
|
||||
return mm_config is not None and mm_config.allow_missing_mm_embeddings
|
||||
|
||||
def get_data_parser(self) -> MultiModalDataParser:
|
||||
"""
|
||||
@@ -389,7 +384,7 @@ class BaseProcessingInfo:
|
||||
"""
|
||||
return MultiModalDataParser(
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
@cached_property
|
||||
|
||||
Reference in New Issue
Block a user