[EPD] Allow KV consumers to omit MM embeddings (#52697)

Signed-off-by: zhenwei-intel <zhenwei.liu@intel.com>
This commit is contained in:
liuzhenwei
2026-08-19 02:16:08 +00:00
committed by GitHub
parent a2257f95b7
commit b1d9337e92
16 changed files with 43 additions and 46 deletions
@@ -183,6 +183,7 @@ run_epd_1e_1pd() {
--port "$PREFILL_DECODE_PORT" \
--max-model-len "$MAX_MODEL_LEN" \
--enforce-eager \
--enable-mm-embeds \
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
--enable-request-id-headers \
--max-num-seqs "$MAX_NUM_SEQS" \
@@ -389,6 +390,7 @@ run_epd_1e_1p_1d() {
--port "$PREFILL_PORT" \
--max-model-len "$MAX_MODEL_LEN" \
--enforce-eager \
--enable-mm-embeds \
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
--enable-request-id-headers \
--max-num-seqs "$MAX_NUM_SEQS" \
@@ -415,6 +417,7 @@ run_epd_1e_1p_1d() {
--port "$DECODE_PORT" \
--max-model-len "$MAX_MODEL_LEN" \
--enforce-eager \
--enable-mm-embeds \
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
--enable-request-id-headers \
--max-num-seqs "$MAX_NUM_SEQS" \
+6 -6
View File
@@ -236,17 +236,17 @@ class MultiModalConfig:
- "direct_rpc": Use msgspec serialization via RPC
- "torch_shm": Use torch.multiprocessing shared memory for zero-copy IPC
Defaults to "direct_rpc". """
mm_embeds_from_ec_connector: bool = False
allow_missing_mm_embeddings: bool = False
"""Whether a pre-computed-embedding input may omit the `*_embeds` tensor.
In an encode/prefill/decode (EPD) deployment the encoder instance publishes
embeddings through the EC connector, so the request that reaches the
prefill/decode instance only needs to carry the grid/size metadata that
sizes the placeholder range — the embeddings themselves come from the
connector, keyed by `mm_hash`.
embeddings through the EC connector. An EC consumer loads those embeddings
from the connector, while a KV consumer receives the resulting prompt KV
cache. Their requests only need the grid/size metadata that sizes the
placeholder range.
Derived, not user-settable: `VllmConfig.__post_init__` sets this to True
exactly on EC consumers. Everywhere else it stays False so that a request
on EC and KV consumers. Everywhere else it stays False so that a request
which forgets its embeddings still fails fast in the frontend, with a clear
error, rather than deep inside the model."""
+13 -13
View File
@@ -1577,7 +1577,7 @@ class VllmConfig:
)
current_platform.check_and_update_config(self)
self._resolve_mm_embeds_from_ec_connector()
self._resolve_allow_missing_mm_embeddings()
self._resolve_mm_processor_device()
self._validate_mm_processor_device()
@@ -2254,14 +2254,13 @@ class VllmConfig:
f"kernel_config={self.kernel_config!r}"
)
def _resolve_mm_embeds_from_ec_connector(self) -> None:
"""Allow `*_embeds` to be omitted only where the connector supplies them.
def _resolve_allow_missing_mm_embeddings(self) -> None:
"""Allow `*_embeds` tensors to be omitted on disaggregated consumers.
That is exactly an EC consumer: the encoder instance publishes the
embeddings through the EC connector, so the request only has to carry
the grid metadata that sizes the placeholder range. On every other
deployment a missing `*_embeds` is a client error and must keep failing
fast in the frontend.
An EC consumer loads embeddings from its connector. A KV consumer
receives the prompt KV produced from those embeddings, so it does not
need the tensors either. On every other deployment a missing tensor is
a client error and must keep failing fast in the frontend.
"""
model_config = self.model_config
if model_config is None:
@@ -2271,15 +2270,16 @@ class VllmConfig:
return
ec_config = self.ec_transfer_config
kv_config = self.kv_transfer_config
# Derived, so overwrite unconditionally rather than honouring a value
# that was set by hand.
mm_config.mm_embeds_from_ec_connector = (
mm_config.allow_missing_mm_embeddings = (
ec_config is not None and ec_config.is_ec_consumer
)
if mm_config.mm_embeds_from_ec_connector:
) or (kv_config is not None and kv_config.is_kv_consumer)
if mm_config.allow_missing_mm_embeddings:
logger.info_once(
"EC consumer: pre-computed-embedding inputs may omit the "
"embedding tensor; embeddings are loaded from the EC connector."
"EC/KV consumer: pre-computed-embedding inputs may "
"omit the embedding tensor."
)
def _resolve_mm_processor_device(self) -> None:
+1 -1
View File
@@ -104,7 +104,7 @@ class ColQwen3ProcessingInfo(Qwen3VLProcessingInfo):
spatial_merge_size,
video_needs_metadata=self._supports_video,
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
)
+1 -1
View File
@@ -101,7 +101,7 @@ class ColQwen3_5ProcessingInfo(Qwen3_5ProcessingInfo):
spatial_merge_size,
video_needs_metadata=self._supports_video,
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
)
+1 -1
View File
@@ -599,7 +599,7 @@ class HunYuanVLProcessingInfo(BaseProcessingInfo):
def get_data_parser(self):
return HunYuanVLMultiModalDataParser(
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
)
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
+1 -1
View File
@@ -936,7 +936,7 @@ class KeyeProcessingInfo(BaseProcessingInfo):
def get_data_parser(self):
return KeyeMultiModalDataParser(
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
)
def get_supported_mm_limits(
+1 -1
View File
@@ -363,7 +363,7 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo):
def get_data_parser(self):
return KeyeVL1_5MultiModalDataParser(
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
)
def get_max_frame_per_video(self) -> int:
@@ -1279,7 +1279,7 @@ class LlavaOnevision2ProcessingInfo(BaseProcessingInfo):
return LlavaOnevision2MultiModalDataParser(
self.get_hf_config().vision_config.spatial_merge_size,
video_needs_metadata=True,
embeds_from_ec_connector=self.embeds_from_ec_connector,
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
)
def get_hf_processor(self, **kwargs: object):
+1 -1
View File
@@ -653,7 +653,7 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
def get_data_parser(self):
return MiniCPMVMultiModalDataParser(
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
)
def get_model_version(self):
+1 -1
View File
@@ -57,7 +57,7 @@ class OpenCUAProcessingInfo(Qwen2VLProcessingInfo):
return Qwen2VLMultiModalDataParser(
self.get_hf_config().vision_config.spatial_merge_size,
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
)
def get_hf_config(self):
@@ -373,7 +373,7 @@ class Qwen2_5OmniThinkerProcessingInfo(
target_sr=feature_extractor.sampling_rate,
target_channels=self.get_target_channels(),
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
)
def get_target_channels(self) -> int:
+1 -1
View File
@@ -857,7 +857,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
return Qwen2VLMultiModalDataParser(
self.get_hf_config().vision_config.spatial_merge_size,
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
)
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
+1 -1
View File
@@ -910,7 +910,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
self.get_hf_config().vision_config.spatial_merge_size,
video_needs_metadata=True,
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
)
def _get_vision_info(
+6 -7
View File
@@ -554,10 +554,9 @@ class MultiModalDataParser:
embedding inputs. If provided, validates that user-supplied
embeddings have the correct hidden size to prevent crashes
during model inference.
embeds_from_ec_connector (bool): Whether pre-computed embeddings may be
absent from the request because an encode/prefill/decode encoder
instance publishes them through an EC connector instead. Derived by
`BaseProcessingInfo.embeds_from_ec_connector`.
allow_missing_mm_embeddings (bool): Whether pre-computed embedding
tensors may be absent from the request on a disaggregated consumer.
Derived by `BaseProcessingInfo.allow_missing_mm_embeddings`.
"""
embedding_fields: Mapping[str, Mapping[str, EmbeddingFieldRole]] = {}
@@ -594,7 +593,7 @@ class MultiModalDataParser:
"""
metadata = self.placeholder_metadata_fields(modality)
values = set(self.embedding_fields.get(modality, {})) - metadata
if self.embeds_from_ec_connector:
if self.allow_missing_mm_embeddings:
return metadata, values
return metadata | values, set()
@@ -606,11 +605,11 @@ class MultiModalDataParser:
audio_resample_method: Literal["pyav", "scipy", "soxr"] = "pyav",
video_needs_metadata: bool = False,
expected_hidden_size: int | None = None,
embeds_from_ec_connector: bool = False,
allow_missing_mm_embeddings: bool = False,
) -> None:
super().__init__()
self.embeds_from_ec_connector = embeds_from_ec_connector
self.allow_missing_mm_embeddings = allow_missing_mm_embeddings
self.audio_resampler = AudioResampler(
target_sr=target_sr,
+4 -9
View File
@@ -367,15 +367,10 @@ class BaseProcessingInfo:
return None
@property
def embeds_from_ec_connector(self) -> bool:
"""Whether pre-computed embeddings may arrive outside the request.
True only on an EC consumer, where an encode/prefill/decode encoder
instance publishes them through the connector instead, so the request
carries only the metadata that sizes the placeholder range.
"""
def allow_missing_mm_embeddings(self) -> bool:
"""Whether pre-computed embedding tensors may be omitted."""
mm_config = self.ctx.model_config.multimodal_config
return mm_config is not None and mm_config.mm_embeds_from_ec_connector
return mm_config is not None and mm_config.allow_missing_mm_embeddings
def get_data_parser(self) -> MultiModalDataParser:
"""
@@ -389,7 +384,7 @@ class BaseProcessingInfo:
"""
return MultiModalDataParser(
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
)
@cached_property