From b1d9337e92692da328bdf921286e59d8f8f36725 Mon Sep 17 00:00:00 2001 From: liuzhenwei Date: Wed, 19 Aug 2026 10:16:08 +0800 Subject: [PATCH] [EPD] Allow KV consumers to omit MM embeddings (#52697) Signed-off-by: zhenwei-intel --- .../integration/run_epd_correctness_test.sh | 3 +++ vllm/config/multimodal.py | 12 ++++----- vllm/config/vllm.py | 26 +++++++++---------- vllm/model_executor/models/colqwen3.py | 2 +- vllm/model_executor/models/colqwen3_5.py | 2 +- vllm/model_executor/models/hunyuan_vision.py | 2 +- vllm/model_executor/models/keye.py | 2 +- vllm/model_executor/models/keye_vl1_5.py | 2 +- .../model_executor/models/llava_onevision2.py | 2 +- vllm/model_executor/models/minicpmv.py | 2 +- vllm/model_executor/models/opencua.py | 2 +- .../models/qwen2_5_omni_thinker.py | 2 +- vllm/model_executor/models/qwen2_vl.py | 2 +- vllm/model_executor/models/qwen3_vl.py | 2 +- vllm/multimodal/parse.py | 13 +++++----- vllm/multimodal/processing/context.py | 13 +++------- 16 files changed, 43 insertions(+), 46 deletions(-) diff --git a/tests/v1/ec_connector/integration/run_epd_correctness_test.sh b/tests/v1/ec_connector/integration/run_epd_correctness_test.sh index c58df0c076..b48949f922 100644 --- a/tests/v1/ec_connector/integration/run_epd_correctness_test.sh +++ b/tests/v1/ec_connector/integration/run_epd_correctness_test.sh @@ -183,6 +183,7 @@ run_epd_1e_1pd() { --port "$PREFILL_DECODE_PORT" \ --max-model-len "$MAX_MODEL_LEN" \ --enforce-eager \ + --enable-mm-embeds \ --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \ --enable-request-id-headers \ --max-num-seqs "$MAX_NUM_SEQS" \ @@ -389,6 +390,7 @@ run_epd_1e_1p_1d() { --port "$PREFILL_PORT" \ --max-model-len "$MAX_MODEL_LEN" \ --enforce-eager \ + --enable-mm-embeds \ --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \ --enable-request-id-headers \ --max-num-seqs "$MAX_NUM_SEQS" \ @@ -415,6 +417,7 @@ run_epd_1e_1p_1d() { --port "$DECODE_PORT" \ --max-model-len "$MAX_MODEL_LEN" \ --enforce-eager \ + --enable-mm-embeds \ --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \ --enable-request-id-headers \ --max-num-seqs "$MAX_NUM_SEQS" \ diff --git a/vllm/config/multimodal.py b/vllm/config/multimodal.py index e1343bc352..04e94735a1 100644 --- a/vllm/config/multimodal.py +++ b/vllm/config/multimodal.py @@ -236,17 +236,17 @@ class MultiModalConfig: - "direct_rpc": Use msgspec serialization via RPC - "torch_shm": Use torch.multiprocessing shared memory for zero-copy IPC Defaults to "direct_rpc". """ - mm_embeds_from_ec_connector: bool = False + allow_missing_mm_embeddings: bool = False """Whether a pre-computed-embedding input may omit the `*_embeds` tensor. In an encode/prefill/decode (EPD) deployment the encoder instance publishes - embeddings through the EC connector, so the request that reaches the - prefill/decode instance only needs to carry the grid/size metadata that - sizes the placeholder range — the embeddings themselves come from the - connector, keyed by `mm_hash`. + embeddings through the EC connector. An EC consumer loads those embeddings + from the connector, while a KV consumer receives the resulting prompt KV + cache. Their requests only need the grid/size metadata that sizes the + placeholder range. Derived, not user-settable: `VllmConfig.__post_init__` sets this to True - exactly on EC consumers. Everywhere else it stays False so that a request + on EC and KV consumers. Everywhere else it stays False so that a request which forgets its embeddings still fails fast in the frontend, with a clear error, rather than deep inside the model.""" diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 5faf09fc34..ff93bed86f 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1577,7 +1577,7 @@ class VllmConfig: ) current_platform.check_and_update_config(self) - self._resolve_mm_embeds_from_ec_connector() + self._resolve_allow_missing_mm_embeddings() self._resolve_mm_processor_device() self._validate_mm_processor_device() @@ -2254,14 +2254,13 @@ class VllmConfig: f"kernel_config={self.kernel_config!r}" ) - def _resolve_mm_embeds_from_ec_connector(self) -> None: - """Allow `*_embeds` to be omitted only where the connector supplies them. + def _resolve_allow_missing_mm_embeddings(self) -> None: + """Allow `*_embeds` tensors to be omitted on disaggregated consumers. - That is exactly an EC consumer: the encoder instance publishes the - embeddings through the EC connector, so the request only has to carry - the grid metadata that sizes the placeholder range. On every other - deployment a missing `*_embeds` is a client error and must keep failing - fast in the frontend. + An EC consumer loads embeddings from its connector. A KV consumer + receives the prompt KV produced from those embeddings, so it does not + need the tensors either. On every other deployment a missing tensor is + a client error and must keep failing fast in the frontend. """ model_config = self.model_config if model_config is None: @@ -2271,15 +2270,16 @@ class VllmConfig: return ec_config = self.ec_transfer_config + kv_config = self.kv_transfer_config # Derived, so overwrite unconditionally rather than honouring a value # that was set by hand. - mm_config.mm_embeds_from_ec_connector = ( + mm_config.allow_missing_mm_embeddings = ( ec_config is not None and ec_config.is_ec_consumer - ) - if mm_config.mm_embeds_from_ec_connector: + ) or (kv_config is not None and kv_config.is_kv_consumer) + if mm_config.allow_missing_mm_embeddings: logger.info_once( - "EC consumer: pre-computed-embedding inputs may omit the " - "embedding tensor; embeddings are loaded from the EC connector." + "EC/KV consumer: pre-computed-embedding inputs may " + "omit the embedding tensor." ) def _resolve_mm_processor_device(self) -> None: diff --git a/vllm/model_executor/models/colqwen3.py b/vllm/model_executor/models/colqwen3.py index dee975ae20..a2012fffd2 100644 --- a/vllm/model_executor/models/colqwen3.py +++ b/vllm/model_executor/models/colqwen3.py @@ -104,7 +104,7 @@ class ColQwen3ProcessingInfo(Qwen3VLProcessingInfo): spatial_merge_size, video_needs_metadata=self._supports_video, expected_hidden_size=self._get_expected_hidden_size(), - embeds_from_ec_connector=self.embeds_from_ec_connector, + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings, ) diff --git a/vllm/model_executor/models/colqwen3_5.py b/vllm/model_executor/models/colqwen3_5.py index 309da6d686..760a54f485 100644 --- a/vllm/model_executor/models/colqwen3_5.py +++ b/vllm/model_executor/models/colqwen3_5.py @@ -101,7 +101,7 @@ class ColQwen3_5ProcessingInfo(Qwen3_5ProcessingInfo): spatial_merge_size, video_needs_metadata=self._supports_video, expected_hidden_size=self._get_expected_hidden_size(), - embeds_from_ec_connector=self.embeds_from_ec_connector, + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings, ) diff --git a/vllm/model_executor/models/hunyuan_vision.py b/vllm/model_executor/models/hunyuan_vision.py index d1c19f07f5..74e4c1a6ce 100644 --- a/vllm/model_executor/models/hunyuan_vision.py +++ b/vllm/model_executor/models/hunyuan_vision.py @@ -599,7 +599,7 @@ class HunYuanVLProcessingInfo(BaseProcessingInfo): def get_data_parser(self): return HunYuanVLMultiModalDataParser( expected_hidden_size=self._get_expected_hidden_size(), - embeds_from_ec_connector=self.embeds_from_ec_connector, + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings, ) def get_supported_mm_limits(self) -> Mapping[str, int | None]: diff --git a/vllm/model_executor/models/keye.py b/vllm/model_executor/models/keye.py index 28ccddd47e..2fee80b271 100644 --- a/vllm/model_executor/models/keye.py +++ b/vllm/model_executor/models/keye.py @@ -936,7 +936,7 @@ class KeyeProcessingInfo(BaseProcessingInfo): def get_data_parser(self): return KeyeMultiModalDataParser( expected_hidden_size=self._get_expected_hidden_size(), - embeds_from_ec_connector=self.embeds_from_ec_connector, + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings, ) def get_supported_mm_limits( diff --git a/vllm/model_executor/models/keye_vl1_5.py b/vllm/model_executor/models/keye_vl1_5.py index 26870ebc26..c7ecc68db1 100644 --- a/vllm/model_executor/models/keye_vl1_5.py +++ b/vllm/model_executor/models/keye_vl1_5.py @@ -363,7 +363,7 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo): def get_data_parser(self): return KeyeVL1_5MultiModalDataParser( expected_hidden_size=self._get_expected_hidden_size(), - embeds_from_ec_connector=self.embeds_from_ec_connector, + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings, ) def get_max_frame_per_video(self) -> int: diff --git a/vllm/model_executor/models/llava_onevision2.py b/vllm/model_executor/models/llava_onevision2.py index daae740c38..01f073a0d9 100644 --- a/vllm/model_executor/models/llava_onevision2.py +++ b/vllm/model_executor/models/llava_onevision2.py @@ -1279,7 +1279,7 @@ class LlavaOnevision2ProcessingInfo(BaseProcessingInfo): return LlavaOnevision2MultiModalDataParser( self.get_hf_config().vision_config.spatial_merge_size, video_needs_metadata=True, - embeds_from_ec_connector=self.embeds_from_ec_connector, + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings, ) def get_hf_processor(self, **kwargs: object): diff --git a/vllm/model_executor/models/minicpmv.py b/vllm/model_executor/models/minicpmv.py index d9358fed42..e85ac81ee5 100644 --- a/vllm/model_executor/models/minicpmv.py +++ b/vllm/model_executor/models/minicpmv.py @@ -653,7 +653,7 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo): def get_data_parser(self): return MiniCPMVMultiModalDataParser( expected_hidden_size=self._get_expected_hidden_size(), - embeds_from_ec_connector=self.embeds_from_ec_connector, + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings, ) def get_model_version(self): diff --git a/vllm/model_executor/models/opencua.py b/vllm/model_executor/models/opencua.py index 9fb0b526b3..a34475539b 100644 --- a/vllm/model_executor/models/opencua.py +++ b/vllm/model_executor/models/opencua.py @@ -57,7 +57,7 @@ class OpenCUAProcessingInfo(Qwen2VLProcessingInfo): return Qwen2VLMultiModalDataParser( self.get_hf_config().vision_config.spatial_merge_size, expected_hidden_size=self._get_expected_hidden_size(), - embeds_from_ec_connector=self.embeds_from_ec_connector, + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings, ) def get_hf_config(self): diff --git a/vllm/model_executor/models/qwen2_5_omni_thinker.py b/vllm/model_executor/models/qwen2_5_omni_thinker.py index 96a58df1dd..a512be850d 100644 --- a/vllm/model_executor/models/qwen2_5_omni_thinker.py +++ b/vllm/model_executor/models/qwen2_5_omni_thinker.py @@ -373,7 +373,7 @@ class Qwen2_5OmniThinkerProcessingInfo( target_sr=feature_extractor.sampling_rate, target_channels=self.get_target_channels(), expected_hidden_size=self._get_expected_hidden_size(), - embeds_from_ec_connector=self.embeds_from_ec_connector, + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings, ) def get_target_channels(self) -> int: diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py index e4a56eadfc..49e026534c 100644 --- a/vllm/model_executor/models/qwen2_vl.py +++ b/vllm/model_executor/models/qwen2_vl.py @@ -857,7 +857,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo): return Qwen2VLMultiModalDataParser( self.get_hf_config().vision_config.spatial_merge_size, expected_hidden_size=self._get_expected_hidden_size(), - embeds_from_ec_connector=self.embeds_from_ec_connector, + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings, ) def get_supported_mm_limits(self) -> Mapping[str, int | None]: diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index de341d302a..3f383b9bd8 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -910,7 +910,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo): self.get_hf_config().vision_config.spatial_merge_size, video_needs_metadata=True, expected_hidden_size=self._get_expected_hidden_size(), - embeds_from_ec_connector=self.embeds_from_ec_connector, + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings, ) def _get_vision_info( diff --git a/vllm/multimodal/parse.py b/vllm/multimodal/parse.py index 3053670a64..6306d6a136 100644 --- a/vllm/multimodal/parse.py +++ b/vllm/multimodal/parse.py @@ -554,10 +554,9 @@ class MultiModalDataParser: embedding inputs. If provided, validates that user-supplied embeddings have the correct hidden size to prevent crashes during model inference. - embeds_from_ec_connector (bool): Whether pre-computed embeddings may be - absent from the request because an encode/prefill/decode encoder - instance publishes them through an EC connector instead. Derived by - `BaseProcessingInfo.embeds_from_ec_connector`. + allow_missing_mm_embeddings (bool): Whether pre-computed embedding + tensors may be absent from the request on a disaggregated consumer. + Derived by `BaseProcessingInfo.allow_missing_mm_embeddings`. """ embedding_fields: Mapping[str, Mapping[str, EmbeddingFieldRole]] = {} @@ -594,7 +593,7 @@ class MultiModalDataParser: """ metadata = self.placeholder_metadata_fields(modality) values = set(self.embedding_fields.get(modality, {})) - metadata - if self.embeds_from_ec_connector: + if self.allow_missing_mm_embeddings: return metadata, values return metadata | values, set() @@ -606,11 +605,11 @@ class MultiModalDataParser: audio_resample_method: Literal["pyav", "scipy", "soxr"] = "pyav", video_needs_metadata: bool = False, expected_hidden_size: int | None = None, - embeds_from_ec_connector: bool = False, + allow_missing_mm_embeddings: bool = False, ) -> None: super().__init__() - self.embeds_from_ec_connector = embeds_from_ec_connector + self.allow_missing_mm_embeddings = allow_missing_mm_embeddings self.audio_resampler = AudioResampler( target_sr=target_sr, diff --git a/vllm/multimodal/processing/context.py b/vllm/multimodal/processing/context.py index a1d4958674..45acdf8f4d 100644 --- a/vllm/multimodal/processing/context.py +++ b/vllm/multimodal/processing/context.py @@ -367,15 +367,10 @@ class BaseProcessingInfo: return None @property - def embeds_from_ec_connector(self) -> bool: - """Whether pre-computed embeddings may arrive outside the request. - - True only on an EC consumer, where an encode/prefill/decode encoder - instance publishes them through the connector instead, so the request - carries only the metadata that sizes the placeholder range. - """ + def allow_missing_mm_embeddings(self) -> bool: + """Whether pre-computed embedding tensors may be omitted.""" mm_config = self.ctx.model_config.multimodal_config - return mm_config is not None and mm_config.mm_embeds_from_ec_connector + return mm_config is not None and mm_config.allow_missing_mm_embeddings def get_data_parser(self) -> MultiModalDataParser: """ @@ -389,7 +384,7 @@ class BaseProcessingInfo: """ return MultiModalDataParser( expected_hidden_size=self._get_expected_hidden_size(), - embeds_from_ec_connector=self.embeds_from_ec_connector, + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings, ) @cached_property