[Config][EC] Normalize producer-only encoder config (#53656)
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
@@ -329,6 +329,34 @@ def test_auto_mm_processor_device_leaves_an_explicit_request_alone():
|
||||
assert _resolve_mm_processor_device(ec_role="ec_producer", device="cpu") == "cpu"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("ec_role", "expected"),
|
||||
[
|
||||
(None, False),
|
||||
("ec_consumer", False),
|
||||
("ec_both", False),
|
||||
("ec_producer", True),
|
||||
],
|
||||
)
|
||||
def test_ec_producer_only_enables_mm_encoder_only(
|
||||
ec_role: ECRole | None, expected: bool
|
||||
):
|
||||
mm_config = MultiModalConfig()
|
||||
model_config = MagicMock(spec=ModelConfig)
|
||||
model_config.multimodal_config = mm_config
|
||||
vllm_config = MagicMock(spec=VllmConfig)
|
||||
vllm_config.model_config = model_config
|
||||
vllm_config.ec_transfer_config = (
|
||||
None
|
||||
if ec_role is None
|
||||
else ECTransferConfig(ec_connector="ECExampleConnector", ec_role=ec_role)
|
||||
)
|
||||
|
||||
VllmConfig._resolve_mm_encoder_only(vllm_config)
|
||||
|
||||
assert mm_config.mm_encoder_only is expected
|
||||
|
||||
|
||||
def test_vllm_config_runs_the_mm_processor_device_check():
|
||||
"""Startup must reach the check; the rule itself is covered above.
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ from vllm.config import (
|
||||
ECTransferConfig,
|
||||
KVTransferConfig,
|
||||
ModelConfig,
|
||||
MultiModalConfig,
|
||||
ObservabilityConfig,
|
||||
ParallelConfig,
|
||||
SchedulerConfig,
|
||||
@@ -94,6 +95,8 @@ def create_scheduler(
|
||||
seed=42,
|
||||
skip_tokenizer_init=skip_tokenizer_init,
|
||||
)
|
||||
if use_ec_connector and ec_role == "ec_producer":
|
||||
model_config.multimodal_config = MultiModalConfig()
|
||||
if max_model_len is None:
|
||||
max_model_len = max_num_batched_tokens
|
||||
scheduler_config = SchedulerConfig(
|
||||
|
||||
+17
-10
@@ -562,15 +562,6 @@ class VllmConfig:
|
||||
]
|
||||
return hash_str
|
||||
|
||||
@property
|
||||
def is_ec_producer_only(self) -> bool:
|
||||
ec_config = self.ec_transfer_config
|
||||
return (
|
||||
ec_config is not None
|
||||
and ec_config.is_ec_producer
|
||||
and not ec_config.is_ec_consumer
|
||||
)
|
||||
|
||||
@property
|
||||
def is_mm_encoder_only(self) -> bool:
|
||||
mm_config = (
|
||||
@@ -578,7 +569,7 @@ class VllmConfig:
|
||||
if self.model_config is not None
|
||||
else None
|
||||
)
|
||||
return self.is_ec_producer_only or bool(mm_config and mm_config.mm_encoder_only)
|
||||
return bool(mm_config and mm_config.mm_encoder_only)
|
||||
|
||||
@property
|
||||
def max_concurrent_batches(self) -> int:
|
||||
@@ -1121,6 +1112,8 @@ class VllmConfig:
|
||||
# To give each torch profile run a unique instance name.
|
||||
self.instance_id = f"{time.time_ns()}"
|
||||
|
||||
self._resolve_mm_encoder_only()
|
||||
|
||||
if self.performance_mode != "balanced":
|
||||
logger.info_once("Performance mode set to '%s'.", self.performance_mode)
|
||||
|
||||
@@ -2380,6 +2373,20 @@ class VllmConfig:
|
||||
"omit the embedding tensor."
|
||||
)
|
||||
|
||||
def _resolve_mm_encoder_only(self) -> None:
|
||||
"""Enable encoder-only mode for a dedicated EC producer."""
|
||||
ec_config = self.ec_transfer_config
|
||||
if ec_config is None or not ec_config.is_encode_only:
|
||||
return
|
||||
|
||||
model_config = self.model_config
|
||||
mm_config = model_config.multimodal_config if model_config is not None else None
|
||||
if mm_config is None:
|
||||
raise ValueError(
|
||||
"An EC producer-only instance requires a multimodal model."
|
||||
)
|
||||
mm_config.mm_encoder_only = True
|
||||
|
||||
def _resolve_mm_processor_device(self) -> None:
|
||||
"""Settle `--mm-processor-device=auto` now that the EC role is known.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user