[EC Connector] Added Build Connector Worker Meta for EC Connector (#49585)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: omerpaz95 <omerpaz95@gmail.com> Signed-off-by: omerpaz95 <73347585+omerpaz95@users.noreply.github.com> Co-authored-by: yewentao256 <zhyanwentao@126.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
yewentao256
Wentao Ye
parent
c94cdd0ae0
commit
fa9d67f782
@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Literal, TypeVar, overload
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.ec_transfer.ec_connector.utils import ECOutputAggregator
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorHandshakeMetadata,
|
||||
@@ -110,6 +111,7 @@ class Executor(ABC):
|
||||
self.is_sleeping = False
|
||||
self.sleeping_tags: set[str] = set()
|
||||
self.kv_output_aggregator: KVOutputAggregator | None = None
|
||||
self.ec_output_aggregator: ECOutputAggregator | None = None
|
||||
|
||||
@abstractmethod
|
||||
def _init_executor(self) -> None:
|
||||
@@ -283,6 +285,9 @@ class Executor(ABC):
|
||||
connector, self.parallel_config.world_size
|
||||
)
|
||||
|
||||
def init_ec_output_aggregator(self) -> None:
|
||||
self.ec_output_aggregator = ECOutputAggregator()
|
||||
|
||||
@cached_property # Avoid unnecessary RPC calls
|
||||
def supported_tasks(self) -> tuple[SupportedTask, ...]:
|
||||
output: list[tuple[SupportedTask, ...]]
|
||||
|
||||
@@ -29,6 +29,7 @@ import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import destroy_distributed_environment, destroy_model_parallel
|
||||
from vllm.distributed.device_communicators.shm_broadcast import Handle, MessageQueue
|
||||
from vllm.distributed.ec_transfer.ec_connector.utils import ECOutputAggregator
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_dcp_group,
|
||||
@@ -343,6 +344,7 @@ class MultiprocExecutor(Executor):
|
||||
non_block=non_block,
|
||||
timeout=envs.VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS,
|
||||
kv_output_aggregator=self.kv_output_aggregator,
|
||||
ec_output_aggregator=self.ec_output_aggregator,
|
||||
)
|
||||
|
||||
def sample_tokens( # type: ignore[override]
|
||||
@@ -355,6 +357,7 @@ class MultiprocExecutor(Executor):
|
||||
non_block=non_block,
|
||||
timeout=envs.VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS,
|
||||
kv_output_aggregator=self.kv_output_aggregator,
|
||||
ec_output_aggregator=self.ec_output_aggregator,
|
||||
)
|
||||
|
||||
def execute_dummy_batch(self) -> None:
|
||||
@@ -375,9 +378,10 @@ class MultiprocExecutor(Executor):
|
||||
non_block: bool = False,
|
||||
unique_reply_rank: int | None = None,
|
||||
kv_output_aggregator: KVOutputAggregator | None = None,
|
||||
ec_output_aggregator: ECOutputAggregator | None = None,
|
||||
) -> Any:
|
||||
"""Returns single result if unique_reply_rank and/or kv_output_aggregator
|
||||
is provided, otherwise list."""
|
||||
"""Returns single result if unique_reply_rank and/or an output
|
||||
aggregator is provided, otherwise list."""
|
||||
assert self.rpc_broadcast_mq is not None, (
|
||||
"collective_rpc should not be called on follower node"
|
||||
)
|
||||
@@ -387,11 +391,21 @@ class MultiprocExecutor(Executor):
|
||||
deadline = None if timeout is None else time.monotonic() + timeout
|
||||
kwargs = kwargs or {}
|
||||
|
||||
if kv_output_aggregator is not None:
|
||||
aggregators = [a for a in (kv_output_aggregator, ec_output_aggregator) if a]
|
||||
aggregate: Callable[[Any], Any]
|
||||
if aggregators:
|
||||
output_rank = None
|
||||
aggregate: Callable[[Any], Any] = partial(
|
||||
kv_output_aggregator.aggregate, output_rank=unique_reply_rank or 0
|
||||
)
|
||||
|
||||
def _aggregate(outputs: Any) -> Any:
|
||||
# Each aggregator merges its own connector's output onto
|
||||
# outputs[output_rank] in place and returns it, so chaining is safe.
|
||||
rank = unique_reply_rank or 0
|
||||
result = outputs[rank]
|
||||
for a in aggregators:
|
||||
result = a.aggregate(outputs, output_rank=rank)
|
||||
return result
|
||||
|
||||
aggregate = _aggregate
|
||||
else:
|
||||
output_rank = unique_reply_rank
|
||||
aggregate = lambda x: x
|
||||
|
||||
@@ -89,6 +89,19 @@ class RayDistributedExecutor(Executor):
|
||||
# KV connector setup
|
||||
self.has_connector = self.vllm_config.kv_transfer_config is not None
|
||||
|
||||
if (
|
||||
self.vllm_config.ec_transfer_config is not None
|
||||
and self.parallel_config.world_size > 1
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"EC connector worker metadata is not supported with the "
|
||||
"legacy Ray executor when world_size > 1: only the output "
|
||||
"of a single worker is fetched, silently dropping the "
|
||||
"other workers' EC connector state. Set "
|
||||
"VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1 to use RayExecutorV2, "
|
||||
"or use the multiprocessing executor instead."
|
||||
)
|
||||
|
||||
self.uses_sampler = self.vllm_config.model_config.runner_type != "pooling" and (
|
||||
self.vllm_config.ec_transfer_config is None
|
||||
or self.vllm_config.ec_transfer_config.is_ec_consumer
|
||||
|
||||
Reference in New Issue
Block a user