[Bugfix][EC Connector] Size transfers by measured encoder output widths (#57904)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
Tianyu Guo
2026-10-12 12:32:39 +00:00
committed by GitHub
parent aa1d44df16
commit 258b97ff6b
18 changed files with 416 additions and 121 deletions
+14 -4
View File
@@ -18,12 +18,16 @@ _EXECUTOR_BACKEND = "mp"
_PCP_SIZE = 1
_EMBED_WIDTH = 32
def create_ec_vllm_config(
*,
ec_role: ECRole = "ec_both",
tensor_parallel_size: int = 1,
rank: int = 0,
dtype: torch.dtype = torch.float16,
encoder_output_widths: dict[str, int] | None = None,
) -> Mock:
"""Build a `VllmConfig` stand-in for EC connector unit tests.
@@ -31,10 +35,8 @@ def create_ec_vllm_config(
role derivation and the rank arithmetic the connector depends on behave as
they do in production.
`model_config` is a stub because nothing under test needs a real one: the
connector reads only `dtype` here, and the paths that inspect the model
itself are not exercised. Building a real `ModelConfig` would resolve an HF
model and inspect its architecture.
`model_config` supplies a dtype and embedding width without resolving an
HF model or inspecting its architecture.
Args:
ec_role: EC role this instance plays. `is_ec_producer` and
@@ -42,6 +44,8 @@ def create_ec_vllm_config(
tensor_parallel_size: TP degree.
rank: Global rank of this worker.
dtype: Encoder cache dtype.
encoder_output_widths: Per-modality encoder output width, as a
producer measures it at startup. Defaults to the embedding width.
Returns:
A `VllmConfig`-specced mock carrying the configs above.
@@ -59,9 +63,15 @@ def create_ec_vllm_config(
ec_role=ec_role,
engine_id=str(uuid.uuid4()),
)
ec_transfer_config.mm_encoder_output_widths = (
dict.fromkeys(("image", "video", "audio"), _EMBED_WIDTH)
if encoder_output_widths is None
else encoder_output_widths
)
model_config = Mock(spec=ModelConfig)
model_config.dtype = dtype
model_config.get_inputs_embeds_size.return_value = _EMBED_WIDTH
vllm_config = Mock(spec=VllmConfig)
vllm_config.ec_transfer_config = ec_transfer_config