[EC] Automatically enable embedding inputs on EC/KV consumers (#56379)
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
+16
-6
@@ -1716,7 +1716,7 @@ class VllmConfig:
|
||||
)
|
||||
current_platform.check_and_update_config(self)
|
||||
|
||||
self._resolve_allow_missing_mm_embeddings()
|
||||
self._resolve_mm_embedding_inputs()
|
||||
self._resolve_mm_processor_device()
|
||||
self._validate_mm_processor_device()
|
||||
|
||||
@@ -2497,8 +2497,8 @@ class VllmConfig:
|
||||
f"kernel_config={self.kernel_config!r}"
|
||||
)
|
||||
|
||||
def _resolve_allow_missing_mm_embeddings(self) -> None:
|
||||
"""Allow `*_embeds` tensors to be omitted on disaggregated consumers.
|
||||
def _resolve_mm_embedding_inputs(self) -> None:
|
||||
"""Accept embedding inputs, tensor optional, on disaggregated consumers.
|
||||
|
||||
An EC consumer loads embeddings from its connector. A KV consumer
|
||||
receives the prompt KV produced from those embeddings, so it does not
|
||||
@@ -2519,11 +2519,21 @@ class VllmConfig:
|
||||
mm_config.allow_missing_mm_embeddings = (
|
||||
ec_config is not None and ec_config.is_ec_consumer
|
||||
) or (kv_config is not None and kv_config.is_kv_consumer)
|
||||
if mm_config.allow_missing_mm_embeddings:
|
||||
if not mm_config.allow_missing_mm_embeddings:
|
||||
return
|
||||
|
||||
if not mm_config.enable_mm_embeds:
|
||||
# Allowing missing tensors still requires enabling embedding inputs
|
||||
# for the frontend to accept metadata-only requests.
|
||||
mm_config.enable_mm_embeds = True
|
||||
logger.info_once(
|
||||
"EC/KV consumer: pre-computed-embedding inputs may "
|
||||
"omit the embedding tensor."
|
||||
"EC/KV consumer: accepting pre-computed-embedding inputs, "
|
||||
"which this role is sent by definition."
|
||||
)
|
||||
logger.info_once(
|
||||
"EC/KV consumer: pre-computed-embedding inputs may "
|
||||
"omit the embedding tensor."
|
||||
)
|
||||
|
||||
def _resolve_mm_encoder_only(self) -> None:
|
||||
"""Enable encoder-only mode for a dedicated EC producer."""
|
||||
|
||||
Reference in New Issue
Block a user