[Bugfix][EC Connector] Size transfers by measured encoder output widths (#57904)
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
@@ -27,6 +27,9 @@ from vllm.distributed import (
|
||||
cleanup_dist_env_and_memory,
|
||||
stateless_destroy_torch_distributed_process_group,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.encoder_output_width import (
|
||||
measure_encoder_output_widths,
|
||||
)
|
||||
from vllm.envs import enable_envs_cache
|
||||
from vllm.logger import configure_logging, init_logger
|
||||
from vllm.logging_utils.dump_input import dump_engine_exception
|
||||
@@ -148,6 +151,8 @@ class EngineCore:
|
||||
if envs.VLLM_ELASTIC_EP_SCALE_UP_LAUNCH:
|
||||
self._eep_scale_up_before_kv_init()
|
||||
|
||||
measure_encoder_output_widths(vllm_config, self.collective_rpc)
|
||||
|
||||
# Setup KV Caches and update CacheConfig after profiling.
|
||||
kv_cache_config = self._initialize_kv_caches(vllm_config)
|
||||
self.structured_output_manager = StructuredOutputManager(vllm_config)
|
||||
|
||||
Reference in New Issue
Block a user