[EC Connector] Added Build Connector Worker Meta for EC Connector (#49585)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: omerpaz95 <omerpaz95@gmail.com> Signed-off-by: omerpaz95 <73347585+omerpaz95@users.noreply.github.com> Co-authored-by: yewentao256 <zhyanwentao@126.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
yewentao256
Wentao Ye
parent
c94cdd0ae0
commit
fa9d67f782
@@ -20,6 +20,8 @@ The class provides the following primitives:
|
||||
|
||||
get_finished() - called with ids of finished requests, returns
|
||||
ids of requests that have completed async sending/recving.
|
||||
build_connector_worker_meta() - builds metadata to be sent
|
||||
back to the scheduler-side connector
|
||||
"""
|
||||
|
||||
import enum
|
||||
@@ -56,6 +58,27 @@ class ECConnectorMetadata(ABC): # noqa: B024
|
||||
pass
|
||||
|
||||
|
||||
class ECConnectorWorkerMetadata(ABC):
|
||||
"""
|
||||
Abstract Metadata used to communicate back
|
||||
Worker ECConnector -> Scheduler ECConnector.
|
||||
|
||||
Each worker can output its own metadata.
|
||||
For a single engine step, all metadata objects returned by workers
|
||||
will be aggregated using the `aggregate` method below, before
|
||||
being passed to the Scheduler ECConnector.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def aggregate(
|
||||
self, other: "ECConnectorWorkerMetadata"
|
||||
) -> "ECConnectorWorkerMetadata":
|
||||
"""
|
||||
Aggregate metadata with another `ECConnectorWorkerMetadata` object.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class ECConnectorBase(ABC):
|
||||
def __init__(self, vllm_config: "VllmConfig", role: ECConnectorRole):
|
||||
self._connector_metadata: ECConnectorMetadata | None = None
|
||||
@@ -190,6 +213,16 @@ class ECConnectorBase(ABC):
|
||||
"""
|
||||
return None, None
|
||||
|
||||
def build_connector_worker_meta(self) -> ECConnectorWorkerMetadata | None:
|
||||
"""
|
||||
Build the ECConnector worker metadata for this engine step.
|
||||
|
||||
Returns:
|
||||
ECConnectorWorkerMetadata: the worker metadata.
|
||||
None if no worker metadata is available.
|
||||
"""
|
||||
return None
|
||||
|
||||
# ==============================
|
||||
# Scheduler-side methods
|
||||
# ==============================
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""EC connector helper utilities."""
|
||||
|
||||
from vllm.v1.outputs import ECConnectorOutput, ModelRunnerOutput
|
||||
|
||||
|
||||
class ECOutputAggregator:
|
||||
"""Merge every worker's EC connector output onto the single
|
||||
ModelRunnerOutput that reaches the scheduler.
|
||||
|
||||
Mirrors KVOutputAggregator: only `output_rank`'s output is returned to the
|
||||
scheduler, but the EC connector may have run on any rank.
|
||||
"""
|
||||
|
||||
def aggregate(
|
||||
self, outputs: list[ModelRunnerOutput | None], output_rank: int = 0
|
||||
) -> ModelRunnerOutput | None:
|
||||
output = outputs[output_rank]
|
||||
if not output:
|
||||
return None
|
||||
|
||||
finished_sending = set[str]()
|
||||
finished_recving = set[str]()
|
||||
worker_meta = None
|
||||
for model_runner_output in outputs:
|
||||
assert model_runner_output is not None
|
||||
ec_output = model_runner_output.ec_connector_output
|
||||
if not ec_output:
|
||||
continue
|
||||
|
||||
finished_sending |= ec_output.finished_sending or set()
|
||||
finished_recving |= ec_output.finished_recving or set()
|
||||
|
||||
if meta := ec_output.ec_connector_worker_meta:
|
||||
worker_meta = (
|
||||
meta if worker_meta is None else worker_meta.aggregate(meta)
|
||||
)
|
||||
|
||||
aggregated = ECConnectorOutput(
|
||||
finished_sending=finished_sending or None,
|
||||
finished_recving=finished_recving or None,
|
||||
ec_connector_worker_meta=worker_meta,
|
||||
)
|
||||
if aggregated.is_empty():
|
||||
output.ec_connector_output = None
|
||||
return output
|
||||
|
||||
# `output` is the shared empty output whenever `output_rank` had no work,
|
||||
# so attach through the copy-on-write helper.
|
||||
return ModelRunnerOutput.with_ec_conn_output(output, aggregated)
|
||||
Reference in New Issue
Block a user