[Bugfix][EC Connector] Size transfers by measured encoder output widths (#57904)
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
@@ -18,12 +18,16 @@ _EXECUTOR_BACKEND = "mp"
|
||||
_PCP_SIZE = 1
|
||||
|
||||
|
||||
_EMBED_WIDTH = 32
|
||||
|
||||
|
||||
def create_ec_vllm_config(
|
||||
*,
|
||||
ec_role: ECRole = "ec_both",
|
||||
tensor_parallel_size: int = 1,
|
||||
rank: int = 0,
|
||||
dtype: torch.dtype = torch.float16,
|
||||
encoder_output_widths: dict[str, int] | None = None,
|
||||
) -> Mock:
|
||||
"""Build a `VllmConfig` stand-in for EC connector unit tests.
|
||||
|
||||
@@ -31,10 +35,8 @@ def create_ec_vllm_config(
|
||||
role derivation and the rank arithmetic the connector depends on behave as
|
||||
they do in production.
|
||||
|
||||
`model_config` is a stub because nothing under test needs a real one: the
|
||||
connector reads only `dtype` here, and the paths that inspect the model
|
||||
itself are not exercised. Building a real `ModelConfig` would resolve an HF
|
||||
model and inspect its architecture.
|
||||
`model_config` supplies a dtype and embedding width without resolving an
|
||||
HF model or inspecting its architecture.
|
||||
|
||||
Args:
|
||||
ec_role: EC role this instance plays. `is_ec_producer` and
|
||||
@@ -42,6 +44,8 @@ def create_ec_vllm_config(
|
||||
tensor_parallel_size: TP degree.
|
||||
rank: Global rank of this worker.
|
||||
dtype: Encoder cache dtype.
|
||||
encoder_output_widths: Per-modality encoder output width, as a
|
||||
producer measures it at startup. Defaults to the embedding width.
|
||||
|
||||
Returns:
|
||||
A `VllmConfig`-specced mock carrying the configs above.
|
||||
@@ -59,9 +63,15 @@ def create_ec_vllm_config(
|
||||
ec_role=ec_role,
|
||||
engine_id=str(uuid.uuid4()),
|
||||
)
|
||||
ec_transfer_config.mm_encoder_output_widths = (
|
||||
dict.fromkeys(("image", "video", "audio"), _EMBED_WIDTH)
|
||||
if encoder_output_widths is None
|
||||
else encoder_output_widths
|
||||
)
|
||||
|
||||
model_config = Mock(spec=ModelConfig)
|
||||
model_config.dtype = dtype
|
||||
model_config.get_inputs_embeds_size.return_value = _EMBED_WIDTH
|
||||
|
||||
vllm_config = Mock(spec=VllmConfig)
|
||||
vllm_config.ec_transfer_config = ec_transfer_config
|
||||
|
||||
Reference in New Issue
Block a user