[Bugfix][EC Connector] Size transfers by measured encoder output widths (#57904)
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
@@ -20,8 +20,8 @@ import torch
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorMetadata,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
|
||||
_get_encoder_cache_hidden_dim,
|
||||
from vllm.distributed.ec_transfer.ec_connector.encoder_output_width import (
|
||||
get_encoder_output_width,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.config import (
|
||||
_RESERVATION_TTL_SECONDS,
|
||||
@@ -85,9 +85,7 @@ class ECMooncakeScheduler:
|
||||
self._is_consumer = config.is_consumer
|
||||
self._control_addr = config.control_addr
|
||||
self._push_wait_timeout = config.push_wait_timeout_s
|
||||
self._encoder_cache_hidden_dim = (
|
||||
_get_encoder_cache_hidden_dim(vllm_config) if config.is_producer else None
|
||||
)
|
||||
self._vllm_config = vllm_config
|
||||
self._model_config = vllm_config.model_config
|
||||
self._control_client = ControlClient(config.control_timeout_ms)
|
||||
self._control_executor = ThreadPoolExecutor(
|
||||
@@ -447,8 +445,10 @@ class ECMooncakeScheduler:
|
||||
def _encoder_output_spec(self, request: Any, index: int) -> TensorSpec:
|
||||
dtype = self._model_config.dtype
|
||||
assert isinstance(dtype, torch.dtype)
|
||||
assert self._encoder_cache_hidden_dim is not None
|
||||
shape = (request.get_num_encoder_embeds(index), self._encoder_cache_hidden_dim)
|
||||
hidden_dim = get_encoder_output_width(
|
||||
self._vllm_config, request.mm_features[index].modality
|
||||
)
|
||||
shape = (request.get_num_encoder_embeds(index), hidden_dim)
|
||||
return TensorSpec(shape, str(dtype), math.prod(shape) * dtype.itemsize)
|
||||
|
||||
def update_state_after_alloc(self, request: Any, index: int) -> None:
|
||||
|
||||
Reference in New Issue
Block a user