[Bugfix][EC Connector] Size transfers by measured encoder output widths (#57904)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
Tianyu Guo
2026-10-12 12:32:39 +00:00
committed by GitHub
parent aa1d44df16
commit 258b97ff6b
18 changed files with 416 additions and 121 deletions
@@ -20,8 +20,8 @@ import torch
from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
_get_encoder_cache_hidden_dim,
from vllm.distributed.ec_transfer.ec_connector.encoder_output_width import (
get_encoder_output_width,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.config import (
_RESERVATION_TTL_SECONDS,
@@ -85,9 +85,7 @@ class ECMooncakeScheduler:
self._is_consumer = config.is_consumer
self._control_addr = config.control_addr
self._push_wait_timeout = config.push_wait_timeout_s
self._encoder_cache_hidden_dim = (
_get_encoder_cache_hidden_dim(vllm_config) if config.is_producer else None
)
self._vllm_config = vllm_config
self._model_config = vllm_config.model_config
self._control_client = ControlClient(config.control_timeout_ms)
self._control_executor = ThreadPoolExecutor(
@@ -447,8 +445,10 @@ class ECMooncakeScheduler:
def _encoder_output_spec(self, request: Any, index: int) -> TensorSpec:
dtype = self._model_config.dtype
assert isinstance(dtype, torch.dtype)
assert self._encoder_cache_hidden_dim is not None
shape = (request.get_num_encoder_embeds(index), self._encoder_cache_hidden_dim)
hidden_dim = get_encoder_output_width(
self._vllm_config, request.mm_features[index].modality
)
shape = (request.get_num_encoder_embeds(index), hidden_dim)
return TensorSpec(shape, str(dtype), math.prod(shape) * dtype.itemsize)
def update_state_after_alloc(self, request: Any, index: int) -> None: