[EC Connector] Added Build Connector Worker Meta for EC Connector (#49585)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: omerpaz95 <omerpaz95@gmail.com> Signed-off-by: omerpaz95 <73347585+omerpaz95@users.noreply.github.com> Co-authored-by: yewentao256 <zhyanwentao@126.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
yewentao256
Wentao Ye
parent
c94cdd0ae0
commit
fa9d67f782
@@ -127,6 +127,7 @@ steps:
|
|||||||
- pytest -v -s v1/test_kv_cache_spec_registry.py
|
- pytest -v -s v1/test_kv_cache_spec_registry.py
|
||||||
- pytest -v -s v1/cudagraph/test_cudagraph_manager.py
|
- pytest -v -s v1/cudagraph/test_cudagraph_manager.py
|
||||||
- pytest -v -s -m 'cpu_test' v1/kv_connector/unit
|
- pytest -v -s -m 'cpu_test' v1/kv_connector/unit
|
||||||
|
- pytest -v -s -m 'cpu_test' v1/ec_connector/unit
|
||||||
- pytest -v -s -m 'cpu_test' v1/metrics
|
- pytest -v -s -m 'cpu_test' v1/metrics
|
||||||
|
|
||||||
- label: Extract Hidden States Integration
|
- label: Extract Hidden States Integration
|
||||||
|
|||||||
@@ -0,0 +1,115 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Unit tests for ECOutputAggregator."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorWorkerMetadata
|
||||||
|
from vllm.distributed.ec_transfer.ec_connector.utils import ECOutputAggregator
|
||||||
|
from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator
|
||||||
|
from vllm.v1.outputs import (
|
||||||
|
EMPTY_MODEL_RUNNER_OUTPUT,
|
||||||
|
ECConnectorOutput,
|
||||||
|
KVConnectorOutput,
|
||||||
|
ModelRunnerOutput,
|
||||||
|
)
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.cpu_test
|
||||||
|
|
||||||
|
|
||||||
|
class FakeWorkerMeta(ECConnectorWorkerMetadata):
|
||||||
|
"""Records merge order. `aggregate` returns a new object, as the base class
|
||||||
|
declares: an aggregator discarding the return value would lose the merge.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, saves: list[str]):
|
||||||
|
self.saves = saves
|
||||||
|
|
||||||
|
def aggregate(self, other: "FakeWorkerMeta") -> "FakeWorkerMeta":
|
||||||
|
return FakeWorkerMeta(self.saves + other.saves)
|
||||||
|
|
||||||
|
|
||||||
|
def _worker_output(ec_output: ECConnectorOutput | None) -> ModelRunnerOutput:
|
||||||
|
return ModelRunnerOutput(
|
||||||
|
req_ids=[], req_id_to_index={}, ec_connector_output=ec_output
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_aggregate_folds_every_rank_onto_output_rank():
|
||||||
|
"""EC work done on any rank reaches the scheduler via output_rank's output.
|
||||||
|
|
||||||
|
The middle rank reports no worker metadata: it must neither seed nor clobber
|
||||||
|
the accumulator.
|
||||||
|
"""
|
||||||
|
outputs = [
|
||||||
|
_worker_output(
|
||||||
|
ECConnectorOutput(
|
||||||
|
finished_sending={"mm0"},
|
||||||
|
ec_connector_worker_meta=FakeWorkerMeta(["mm0"]),
|
||||||
|
)
|
||||||
|
),
|
||||||
|
_worker_output(ECConnectorOutput(finished_recving={"mm1"})),
|
||||||
|
_worker_output(
|
||||||
|
ECConnectorOutput(ec_connector_worker_meta=FakeWorkerMeta(["mm2"]))
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
result = ECOutputAggregator().aggregate(outputs, output_rank=2)
|
||||||
|
|
||||||
|
assert result is outputs[2]
|
||||||
|
assert result.ec_connector_output.finished_sending == {"mm0"}
|
||||||
|
assert result.ec_connector_output.finished_recving == {"mm1"}
|
||||||
|
assert result.ec_connector_output.ec_connector_worker_meta.saves == ["mm0", "mm2"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_aggregate_leaves_no_ec_output_when_no_worker_reported():
|
||||||
|
"""Empty per-worker reports must not reach the scheduler as an empty object."""
|
||||||
|
outputs = [_worker_output(ECConnectorOutput()), _worker_output(ECConnectorOutput())]
|
||||||
|
|
||||||
|
result = ECOutputAggregator().aggregate(outputs, output_rank=0)
|
||||||
|
|
||||||
|
assert result is outputs[0]
|
||||||
|
assert result.ec_connector_output is None
|
||||||
|
assert ECOutputAggregator().aggregate([None], output_rank=0) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_aggregate_does_not_write_through_the_shared_empty_output():
|
||||||
|
"""A rank with nothing to report yields the shared empty output singleton.
|
||||||
|
|
||||||
|
Folding another rank's metadata onto it must not write through to the
|
||||||
|
module-level object, which every later step would then carry.
|
||||||
|
"""
|
||||||
|
outputs = [
|
||||||
|
_worker_output(
|
||||||
|
ECConnectorOutput(ec_connector_worker_meta=FakeWorkerMeta(["mm0"]))
|
||||||
|
),
|
||||||
|
EMPTY_MODEL_RUNNER_OUTPUT,
|
||||||
|
]
|
||||||
|
|
||||||
|
result = ECOutputAggregator().aggregate(outputs, output_rank=1)
|
||||||
|
|
||||||
|
assert EMPTY_MODEL_RUNNER_OUTPUT.ec_connector_output is None
|
||||||
|
assert result is not EMPTY_MODEL_RUNNER_OUTPUT
|
||||||
|
assert result.ec_connector_output.ec_connector_worker_meta.saves == ["mm0"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_chaining_with_kv_aggregator_preserves_both_outputs():
|
||||||
|
"""MultiprocExecutor chains both aggregators and keeps only the last result,
|
||||||
|
so each must merge onto the same output_rank output rather than replace it.
|
||||||
|
"""
|
||||||
|
outputs = [
|
||||||
|
_worker_output(ECConnectorOutput(finished_sending={"mm0"})),
|
||||||
|
_worker_output(None),
|
||||||
|
]
|
||||||
|
outputs[1].kv_connector_output = KVConnectorOutput(invalid_block_ids={7})
|
||||||
|
|
||||||
|
result = None
|
||||||
|
for aggregator in (
|
||||||
|
KVOutputAggregator(expected_finished_count=1),
|
||||||
|
ECOutputAggregator(),
|
||||||
|
):
|
||||||
|
result = aggregator.aggregate(outputs, output_rank=1)
|
||||||
|
|
||||||
|
assert result is outputs[1]
|
||||||
|
assert result.kv_connector_output.invalid_block_ids == {7}
|
||||||
|
assert result.ec_connector_output.finished_sending == {"mm0"}
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Unit tests for the V2 GPU model runner's EC connector wrapper."""
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||||
|
ECConnectorBase,
|
||||||
|
ECConnectorMetadata,
|
||||||
|
)
|
||||||
|
from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT
|
||||||
|
from vllm.v1.worker.gpu.ec_connector import NO_OP_EC_CONNECTOR, ActiveECConnector
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.cpu_test
|
||||||
|
|
||||||
|
WORKER_META = object()
|
||||||
|
|
||||||
|
|
||||||
|
def _scheduler_output() -> SimpleNamespace:
|
||||||
|
return SimpleNamespace(
|
||||||
|
ec_connector_metadata=ECConnectorMetadata(), finished_req_ids=frozenset()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _connector(
|
||||||
|
encoder_cache: dict | None = None,
|
||||||
|
is_producer: bool = True,
|
||||||
|
is_consumer: bool = False,
|
||||||
|
) -> tuple[ActiveECConnector, MagicMock]:
|
||||||
|
fake = MagicMock(spec=ECConnectorBase)
|
||||||
|
fake.is_producer = is_producer
|
||||||
|
fake.is_consumer = is_consumer
|
||||||
|
fake.get_finished.return_value = (None, None)
|
||||||
|
fake.build_connector_worker_meta.return_value = WORKER_META
|
||||||
|
with patch("vllm.v1.worker.gpu.ec_connector.get_ec_transfer", return_value=fake):
|
||||||
|
return ActiveECConnector(SimpleNamespace(), encoder_cache or {}), fake
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("is_producer", "is_consumer"), [(True, False), (True, True), (False, True)]
|
||||||
|
)
|
||||||
|
def test_saves_newly_added_caches_for_every_producer(is_producer, is_consumer):
|
||||||
|
"""An ec_both node is also a producer: it must offload what it just computed."""
|
||||||
|
encoder_cache = {"mm_old": None}
|
||||||
|
connector, fake = _connector(encoder_cache, is_producer, is_consumer)
|
||||||
|
|
||||||
|
with connector.maybe_get_output(_scheduler_output()):
|
||||||
|
encoder_cache["mm_new"] = None
|
||||||
|
|
||||||
|
saved = [call.kwargs["mm_hash"] for call in fake.save_caches.call_args_list]
|
||||||
|
assert saved == (["mm_new"] if is_producer else [])
|
||||||
|
assert fake.start_load_caches.called == is_consumer
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_meta_is_reported_on_context_exit():
|
||||||
|
"""Reported in the finally block, so is_empty() sees it only after the exit."""
|
||||||
|
connector, fake = _connector()
|
||||||
|
|
||||||
|
with connector.maybe_get_output(_scheduler_output()) as output:
|
||||||
|
assert output.ec_connector_worker_meta is None
|
||||||
|
|
||||||
|
assert output.ec_connector_worker_meta is WORKER_META
|
||||||
|
assert fake.clear_connector_metadata.called
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_forward_reports_without_running_the_model():
|
||||||
|
connector, _ = _connector()
|
||||||
|
|
||||||
|
output = connector.no_forward(_scheduler_output())
|
||||||
|
|
||||||
|
assert output.ec_connector_output.ec_connector_worker_meta is WORKER_META
|
||||||
|
|
||||||
|
empty = NO_OP_EC_CONNECTOR.no_forward(_scheduler_output())
|
||||||
|
assert empty is EMPTY_MODEL_RUNNER_OUTPUT
|
||||||
@@ -9,6 +9,7 @@ from typing import Any
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
from vllm.distributed.ec_transfer.ec_connector.utils import ECOutputAggregator
|
||||||
from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator
|
from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator
|
||||||
from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
|
from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
|
||||||
from vllm.sampling_params import SamplingParams
|
from vllm.sampling_params import SamplingParams
|
||||||
@@ -98,6 +99,7 @@ class CustomMultiprocExecutor(MultiprocExecutor):
|
|||||||
non_block: bool = False,
|
non_block: bool = False,
|
||||||
unique_reply_rank: int | None = None,
|
unique_reply_rank: int | None = None,
|
||||||
kv_output_aggregator: KVOutputAggregator = None,
|
kv_output_aggregator: KVOutputAggregator = None,
|
||||||
|
ec_output_aggregator: ECOutputAggregator | None = None,
|
||||||
) -> Any | list[Any] | Future[Any | list[Any]]:
|
) -> Any | list[Any] | Future[Any | list[Any]]:
|
||||||
# Drop marker to show that this was run
|
# Drop marker to show that this was run
|
||||||
with open(".marker", "w"):
|
with open(".marker", "w"):
|
||||||
@@ -110,6 +112,7 @@ class CustomMultiprocExecutor(MultiprocExecutor):
|
|||||||
non_block,
|
non_block,
|
||||||
unique_reply_rank,
|
unique_reply_rank,
|
||||||
kv_output_aggregator,
|
kv_output_aggregator,
|
||||||
|
ec_output_aggregator,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -55,6 +55,7 @@ def _run_engine_core_handshake(
|
|||||||
class _FakeScheduler:
|
class _FakeScheduler:
|
||||||
def __init__(self, **kwargs: Any) -> None:
|
def __init__(self, **kwargs: Any) -> None:
|
||||||
self.connector = connector
|
self.connector = connector
|
||||||
|
self.ec_connector = None
|
||||||
|
|
||||||
def get_kv_connector(self) -> KVConnectorBase_V1:
|
def get_kv_connector(self) -> KVConnectorBase_V1:
|
||||||
return connector
|
return connector
|
||||||
|
|||||||
@@ -6,7 +6,13 @@ import numpy as np
|
|||||||
import torch
|
import torch
|
||||||
|
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
from vllm.v1.outputs import LogprobsLists, LogprobsTensors
|
from vllm.v1.outputs import (
|
||||||
|
EMPTY_MODEL_RUNNER_OUTPUT,
|
||||||
|
ECConnectorOutput,
|
||||||
|
LogprobsLists,
|
||||||
|
LogprobsTensors,
|
||||||
|
ModelRunnerOutput,
|
||||||
|
)
|
||||||
from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p
|
from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p
|
||||||
from vllm.v1.worker.gpu.sample.output import SamplingMaskTensors
|
from vllm.v1.worker.gpu.sample.output import SamplingMaskTensors
|
||||||
|
|
||||||
@@ -200,3 +206,14 @@ class TestLogprobsLists(TestCase):
|
|||||||
assert len(sliced.logprob_token_ids) == 9 # All tokens
|
assert len(sliced.logprob_token_ids) == 9 # All tokens
|
||||||
assert sliced.logprob_token_ids == self.logprobsLists.logprob_token_ids
|
assert sliced.logprob_token_ids == self.logprobsLists.logprob_token_ids
|
||||||
assert sliced.cu_num_generated_tokens is None
|
assert sliced.cu_num_generated_tokens is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_with_ec_conn_output_copies_shared_empty_output():
|
||||||
|
"""The shared empty output is copied, never written to."""
|
||||||
|
ec_output = ECConnectorOutput(finished_sending={"mm_hash"})
|
||||||
|
|
||||||
|
result = ModelRunnerOutput.with_ec_conn_output(EMPTY_MODEL_RUNNER_OUTPUT, ec_output)
|
||||||
|
|
||||||
|
assert result is not EMPTY_MODEL_RUNNER_OUTPUT
|
||||||
|
assert result.ec_connector_output is ec_output
|
||||||
|
assert EMPTY_MODEL_RUNNER_OUTPUT.ec_connector_output is None
|
||||||
|
|||||||
@@ -180,6 +180,7 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
|
|||||||
hidden_states=None,
|
hidden_states=None,
|
||||||
aux_hidden_states=None,
|
aux_hidden_states=None,
|
||||||
finished_req_ids=set(),
|
finished_req_ids=set(),
|
||||||
|
ec_connector_output=None,
|
||||||
routed_experts=None,
|
routed_experts=None,
|
||||||
num_tokens_across_dp=None,
|
num_tokens_across_dp=None,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -20,6 +20,8 @@ The class provides the following primitives:
|
|||||||
|
|
||||||
get_finished() - called with ids of finished requests, returns
|
get_finished() - called with ids of finished requests, returns
|
||||||
ids of requests that have completed async sending/recving.
|
ids of requests that have completed async sending/recving.
|
||||||
|
build_connector_worker_meta() - builds metadata to be sent
|
||||||
|
back to the scheduler-side connector
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import enum
|
import enum
|
||||||
@@ -56,6 +58,27 @@ class ECConnectorMetadata(ABC): # noqa: B024
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class ECConnectorWorkerMetadata(ABC):
|
||||||
|
"""
|
||||||
|
Abstract Metadata used to communicate back
|
||||||
|
Worker ECConnector -> Scheduler ECConnector.
|
||||||
|
|
||||||
|
Each worker can output its own metadata.
|
||||||
|
For a single engine step, all metadata objects returned by workers
|
||||||
|
will be aggregated using the `aggregate` method below, before
|
||||||
|
being passed to the Scheduler ECConnector.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def aggregate(
|
||||||
|
self, other: "ECConnectorWorkerMetadata"
|
||||||
|
) -> "ECConnectorWorkerMetadata":
|
||||||
|
"""
|
||||||
|
Aggregate metadata with another `ECConnectorWorkerMetadata` object.
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
class ECConnectorBase(ABC):
|
class ECConnectorBase(ABC):
|
||||||
def __init__(self, vllm_config: "VllmConfig", role: ECConnectorRole):
|
def __init__(self, vllm_config: "VllmConfig", role: ECConnectorRole):
|
||||||
self._connector_metadata: ECConnectorMetadata | None = None
|
self._connector_metadata: ECConnectorMetadata | None = None
|
||||||
@@ -190,6 +213,16 @@ class ECConnectorBase(ABC):
|
|||||||
"""
|
"""
|
||||||
return None, None
|
return None, None
|
||||||
|
|
||||||
|
def build_connector_worker_meta(self) -> ECConnectorWorkerMetadata | None:
|
||||||
|
"""
|
||||||
|
Build the ECConnector worker metadata for this engine step.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
ECConnectorWorkerMetadata: the worker metadata.
|
||||||
|
None if no worker metadata is available.
|
||||||
|
"""
|
||||||
|
return None
|
||||||
|
|
||||||
# ==============================
|
# ==============================
|
||||||
# Scheduler-side methods
|
# Scheduler-side methods
|
||||||
# ==============================
|
# ==============================
|
||||||
|
|||||||
@@ -0,0 +1,51 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""EC connector helper utilities."""
|
||||||
|
|
||||||
|
from vllm.v1.outputs import ECConnectorOutput, ModelRunnerOutput
|
||||||
|
|
||||||
|
|
||||||
|
class ECOutputAggregator:
|
||||||
|
"""Merge every worker's EC connector output onto the single
|
||||||
|
ModelRunnerOutput that reaches the scheduler.
|
||||||
|
|
||||||
|
Mirrors KVOutputAggregator: only `output_rank`'s output is returned to the
|
||||||
|
scheduler, but the EC connector may have run on any rank.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def aggregate(
|
||||||
|
self, outputs: list[ModelRunnerOutput | None], output_rank: int = 0
|
||||||
|
) -> ModelRunnerOutput | None:
|
||||||
|
output = outputs[output_rank]
|
||||||
|
if not output:
|
||||||
|
return None
|
||||||
|
|
||||||
|
finished_sending = set[str]()
|
||||||
|
finished_recving = set[str]()
|
||||||
|
worker_meta = None
|
||||||
|
for model_runner_output in outputs:
|
||||||
|
assert model_runner_output is not None
|
||||||
|
ec_output = model_runner_output.ec_connector_output
|
||||||
|
if not ec_output:
|
||||||
|
continue
|
||||||
|
|
||||||
|
finished_sending |= ec_output.finished_sending or set()
|
||||||
|
finished_recving |= ec_output.finished_recving or set()
|
||||||
|
|
||||||
|
if meta := ec_output.ec_connector_worker_meta:
|
||||||
|
worker_meta = (
|
||||||
|
meta if worker_meta is None else worker_meta.aggregate(meta)
|
||||||
|
)
|
||||||
|
|
||||||
|
aggregated = ECConnectorOutput(
|
||||||
|
finished_sending=finished_sending or None,
|
||||||
|
finished_recving=finished_recving or None,
|
||||||
|
ec_connector_worker_meta=worker_meta,
|
||||||
|
)
|
||||||
|
if aggregated.is_empty():
|
||||||
|
output.ec_connector_output = None
|
||||||
|
return output
|
||||||
|
|
||||||
|
# `output` is the shared empty output whenever `output_rank` had no work,
|
||||||
|
# so attach through the copy-on-write helper.
|
||||||
|
return ModelRunnerOutput.with_ec_conn_output(output, aggregated)
|
||||||
@@ -173,6 +173,8 @@ class EngineCore:
|
|||||||
)
|
)
|
||||||
if self.scheduler.connector is not None: # type: ignore
|
if self.scheduler.connector is not None: # type: ignore
|
||||||
self.model_executor.init_kv_output_aggregator(self.scheduler.connector) # type: ignore
|
self.model_executor.init_kv_output_aggregator(self.scheduler.connector) # type: ignore
|
||||||
|
if self.scheduler.ec_connector is not None: # type: ignore
|
||||||
|
self.model_executor.init_ec_output_aggregator()
|
||||||
|
|
||||||
mm_registry = MULTIMODAL_REGISTRY
|
mm_registry = MULTIMODAL_REGISTRY
|
||||||
self.mm_receiver_cache = mm_registry.engine_receiver_cache_from_config(
|
self.mm_receiver_cache = mm_registry.engine_receiver_cache_from_config(
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Literal, TypeVar, overload
|
|||||||
|
|
||||||
import vllm.envs as envs
|
import vllm.envs as envs
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
|
from vllm.distributed.ec_transfer.ec_connector.utils import ECOutputAggregator
|
||||||
from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator
|
from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator
|
||||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||||
KVConnectorHandshakeMetadata,
|
KVConnectorHandshakeMetadata,
|
||||||
@@ -110,6 +111,7 @@ class Executor(ABC):
|
|||||||
self.is_sleeping = False
|
self.is_sleeping = False
|
||||||
self.sleeping_tags: set[str] = set()
|
self.sleeping_tags: set[str] = set()
|
||||||
self.kv_output_aggregator: KVOutputAggregator | None = None
|
self.kv_output_aggregator: KVOutputAggregator | None = None
|
||||||
|
self.ec_output_aggregator: ECOutputAggregator | None = None
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def _init_executor(self) -> None:
|
def _init_executor(self) -> None:
|
||||||
@@ -283,6 +285,9 @@ class Executor(ABC):
|
|||||||
connector, self.parallel_config.world_size
|
connector, self.parallel_config.world_size
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def init_ec_output_aggregator(self) -> None:
|
||||||
|
self.ec_output_aggregator = ECOutputAggregator()
|
||||||
|
|
||||||
@cached_property # Avoid unnecessary RPC calls
|
@cached_property # Avoid unnecessary RPC calls
|
||||||
def supported_tasks(self) -> tuple[SupportedTask, ...]:
|
def supported_tasks(self) -> tuple[SupportedTask, ...]:
|
||||||
output: list[tuple[SupportedTask, ...]]
|
output: list[tuple[SupportedTask, ...]]
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ import vllm.envs as envs
|
|||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.distributed import destroy_distributed_environment, destroy_model_parallel
|
from vllm.distributed import destroy_distributed_environment, destroy_model_parallel
|
||||||
from vllm.distributed.device_communicators.shm_broadcast import Handle, MessageQueue
|
from vllm.distributed.device_communicators.shm_broadcast import Handle, MessageQueue
|
||||||
|
from vllm.distributed.ec_transfer.ec_connector.utils import ECOutputAggregator
|
||||||
from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator
|
from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator
|
||||||
from vllm.distributed.parallel_state import (
|
from vllm.distributed.parallel_state import (
|
||||||
get_dcp_group,
|
get_dcp_group,
|
||||||
@@ -343,6 +344,7 @@ class MultiprocExecutor(Executor):
|
|||||||
non_block=non_block,
|
non_block=non_block,
|
||||||
timeout=envs.VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS,
|
timeout=envs.VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS,
|
||||||
kv_output_aggregator=self.kv_output_aggregator,
|
kv_output_aggregator=self.kv_output_aggregator,
|
||||||
|
ec_output_aggregator=self.ec_output_aggregator,
|
||||||
)
|
)
|
||||||
|
|
||||||
def sample_tokens( # type: ignore[override]
|
def sample_tokens( # type: ignore[override]
|
||||||
@@ -355,6 +357,7 @@ class MultiprocExecutor(Executor):
|
|||||||
non_block=non_block,
|
non_block=non_block,
|
||||||
timeout=envs.VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS,
|
timeout=envs.VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS,
|
||||||
kv_output_aggregator=self.kv_output_aggregator,
|
kv_output_aggregator=self.kv_output_aggregator,
|
||||||
|
ec_output_aggregator=self.ec_output_aggregator,
|
||||||
)
|
)
|
||||||
|
|
||||||
def execute_dummy_batch(self) -> None:
|
def execute_dummy_batch(self) -> None:
|
||||||
@@ -375,9 +378,10 @@ class MultiprocExecutor(Executor):
|
|||||||
non_block: bool = False,
|
non_block: bool = False,
|
||||||
unique_reply_rank: int | None = None,
|
unique_reply_rank: int | None = None,
|
||||||
kv_output_aggregator: KVOutputAggregator | None = None,
|
kv_output_aggregator: KVOutputAggregator | None = None,
|
||||||
|
ec_output_aggregator: ECOutputAggregator | None = None,
|
||||||
) -> Any:
|
) -> Any:
|
||||||
"""Returns single result if unique_reply_rank and/or kv_output_aggregator
|
"""Returns single result if unique_reply_rank and/or an output
|
||||||
is provided, otherwise list."""
|
aggregator is provided, otherwise list."""
|
||||||
assert self.rpc_broadcast_mq is not None, (
|
assert self.rpc_broadcast_mq is not None, (
|
||||||
"collective_rpc should not be called on follower node"
|
"collective_rpc should not be called on follower node"
|
||||||
)
|
)
|
||||||
@@ -387,11 +391,21 @@ class MultiprocExecutor(Executor):
|
|||||||
deadline = None if timeout is None else time.monotonic() + timeout
|
deadline = None if timeout is None else time.monotonic() + timeout
|
||||||
kwargs = kwargs or {}
|
kwargs = kwargs or {}
|
||||||
|
|
||||||
if kv_output_aggregator is not None:
|
aggregators = [a for a in (kv_output_aggregator, ec_output_aggregator) if a]
|
||||||
|
aggregate: Callable[[Any], Any]
|
||||||
|
if aggregators:
|
||||||
output_rank = None
|
output_rank = None
|
||||||
aggregate: Callable[[Any], Any] = partial(
|
|
||||||
kv_output_aggregator.aggregate, output_rank=unique_reply_rank or 0
|
def _aggregate(outputs: Any) -> Any:
|
||||||
)
|
# Each aggregator merges its own connector's output onto
|
||||||
|
# outputs[output_rank] in place and returns it, so chaining is safe.
|
||||||
|
rank = unique_reply_rank or 0
|
||||||
|
result = outputs[rank]
|
||||||
|
for a in aggregators:
|
||||||
|
result = a.aggregate(outputs, output_rank=rank)
|
||||||
|
return result
|
||||||
|
|
||||||
|
aggregate = _aggregate
|
||||||
else:
|
else:
|
||||||
output_rank = unique_reply_rank
|
output_rank = unique_reply_rank
|
||||||
aggregate = lambda x: x
|
aggregate = lambda x: x
|
||||||
|
|||||||
@@ -89,6 +89,19 @@ class RayDistributedExecutor(Executor):
|
|||||||
# KV connector setup
|
# KV connector setup
|
||||||
self.has_connector = self.vllm_config.kv_transfer_config is not None
|
self.has_connector = self.vllm_config.kv_transfer_config is not None
|
||||||
|
|
||||||
|
if (
|
||||||
|
self.vllm_config.ec_transfer_config is not None
|
||||||
|
and self.parallel_config.world_size > 1
|
||||||
|
):
|
||||||
|
raise NotImplementedError(
|
||||||
|
"EC connector worker metadata is not supported with the "
|
||||||
|
"legacy Ray executor when world_size > 1: only the output "
|
||||||
|
"of a single worker is fetched, silently dropping the "
|
||||||
|
"other workers' EC connector state. Set "
|
||||||
|
"VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1 to use RayExecutorV2, "
|
||||||
|
"or use the multiprocessing executor instead."
|
||||||
|
)
|
||||||
|
|
||||||
self.uses_sampler = self.vllm_config.model_config.runner_type != "pooling" and (
|
self.uses_sampler = self.vllm_config.model_config.runner_type != "pooling" and (
|
||||||
self.vllm_config.ec_transfer_config is None
|
self.vllm_config.ec_transfer_config is None
|
||||||
or self.vllm_config.ec_transfer_config.is_ec_consumer
|
or self.vllm_config.ec_transfer_config.is_ec_consumer
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ from vllm.compilation.cuda_graph import CUDAGraphStat
|
|||||||
from vllm.v1.core.sched.output import SchedulerOutput
|
from vllm.v1.core.sched.output import SchedulerOutput
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
|
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorWorkerMetadata
|
||||||
from vllm.distributed.kv_events import KVConnectorKVEvents
|
from vllm.distributed.kv_events import KVConnectorKVEvents
|
||||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||||
KVConnectorWorkerMetadata,
|
KVConnectorWorkerMetadata,
|
||||||
@@ -23,6 +24,7 @@ else:
|
|||||||
KVConnectorStats = object
|
KVConnectorStats = object
|
||||||
KVConnectorWorkerMetadata = object
|
KVConnectorWorkerMetadata = object
|
||||||
KVConnectorKVEvents = object
|
KVConnectorKVEvents = object
|
||||||
|
ECConnectorWorkerMetadata = object
|
||||||
|
|
||||||
|
|
||||||
class LogprobsLists(NamedTuple):
|
class LogprobsLists(NamedTuple):
|
||||||
@@ -292,6 +294,14 @@ class ECConnectorOutput:
|
|||||||
# [mm_hash]
|
# [mm_hash]
|
||||||
finished_sending: set[str] | None = None
|
finished_sending: set[str] | None = None
|
||||||
finished_recving: set[str] | None = None
|
finished_recving: set[str] | None = None
|
||||||
|
ec_connector_worker_meta: ECConnectorWorkerMetadata | None = None
|
||||||
|
|
||||||
|
def is_empty(self):
|
||||||
|
return (
|
||||||
|
not self.finished_sending
|
||||||
|
and not self.finished_recving
|
||||||
|
and not self.ec_connector_worker_meta
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# ModelRunnerOutput is serialized and sent to the scheduler process.
|
# ModelRunnerOutput is serialized and sent to the scheduler process.
|
||||||
@@ -362,6 +372,32 @@ class ModelRunnerOutput:
|
|||||||
output.kv_connector_output = kv_connector_output
|
output.kv_connector_output = kv_connector_output
|
||||||
return output
|
return output
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def with_ec_conn_output_only(
|
||||||
|
ec_connector_output: ECConnectorOutput | None,
|
||||||
|
) -> "ModelRunnerOutput":
|
||||||
|
"""Return an otherwise-empty output carrying `ec_connector_output`."""
|
||||||
|
return ModelRunnerOutput.with_ec_conn_output(
|
||||||
|
EMPTY_MODEL_RUNNER_OUTPUT, ec_connector_output
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def with_ec_conn_output(
|
||||||
|
output: "ModelRunnerOutput",
|
||||||
|
ec_connector_output: ECConnectorOutput | None,
|
||||||
|
) -> "ModelRunnerOutput":
|
||||||
|
"""Return `output` carrying `ec_connector_output`.
|
||||||
|
|
||||||
|
The shared empty output is copied rather than written to, so callers
|
||||||
|
must use the return value.
|
||||||
|
"""
|
||||||
|
if ec_connector_output is None or ec_connector_output.is_empty():
|
||||||
|
return output
|
||||||
|
if output is EMPTY_MODEL_RUNNER_OUTPUT:
|
||||||
|
output = copy(EMPTY_MODEL_RUNNER_OUTPUT)
|
||||||
|
output.ec_connector_output = ec_connector_output
|
||||||
|
return output
|
||||||
|
|
||||||
|
|
||||||
# ModelRunnerOutput wrapper for async scheduling.
|
# ModelRunnerOutput wrapper for async scheduling.
|
||||||
class AsyncModelRunnerOutput(ABC):
|
class AsyncModelRunnerOutput(ABC):
|
||||||
|
|||||||
@@ -9,7 +9,11 @@ import torch
|
|||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.distributed.ec_transfer import get_ec_transfer, has_ec_transfer
|
from vllm.distributed.ec_transfer import get_ec_transfer, has_ec_transfer
|
||||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorBase
|
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorBase
|
||||||
from vllm.v1.outputs import ECConnectorOutput
|
from vllm.v1.outputs import (
|
||||||
|
EMPTY_MODEL_RUNNER_OUTPUT,
|
||||||
|
ECConnectorOutput,
|
||||||
|
ModelRunnerOutput,
|
||||||
|
)
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from vllm.v1.core.sched.output import SchedulerOutput
|
from vllm.v1.core.sched.output import SchedulerOutput
|
||||||
@@ -25,6 +29,12 @@ class ECConnector:
|
|||||||
) -> Generator[ECConnectorOutput | None, None, None]:
|
) -> Generator[ECConnectorOutput | None, None, None]:
|
||||||
yield None
|
yield None
|
||||||
|
|
||||||
|
def no_forward(
|
||||||
|
self,
|
||||||
|
scheduler_output: "SchedulerOutput",
|
||||||
|
) -> ModelRunnerOutput:
|
||||||
|
return EMPTY_MODEL_RUNNER_OUTPUT
|
||||||
|
|
||||||
|
|
||||||
class ActiveECConnector(ECConnector):
|
class ActiveECConnector(ECConnector):
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -33,9 +43,11 @@ class ActiveECConnector(ECConnector):
|
|||||||
encoder_cache: dict[str, torch.Tensor],
|
encoder_cache: dict[str, torch.Tensor],
|
||||||
) -> None:
|
) -> None:
|
||||||
self.encoder_cache = encoder_cache
|
self.encoder_cache = encoder_cache
|
||||||
self.save_new_caches = vllm_config.is_ec_producer_only
|
|
||||||
self.ec_connector = get_ec_transfer()
|
self.ec_connector = get_ec_transfer()
|
||||||
assert isinstance(self.ec_connector, ECConnectorBase)
|
assert isinstance(self.ec_connector, ECConnectorBase)
|
||||||
|
# Every producer offloads freshly computed encoder outputs, including
|
||||||
|
# an ec_both node that also reloads them.
|
||||||
|
self.save_new_caches = self.ec_connector.is_producer
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def maybe_get_output(
|
def maybe_get_output(
|
||||||
@@ -65,8 +77,19 @@ class ActiveECConnector(ECConnector):
|
|||||||
output.finished_sending, output.finished_recving = (
|
output.finished_sending, output.finished_recving = (
|
||||||
ec_connector.get_finished(scheduler_output.finished_req_ids)
|
ec_connector.get_finished(scheduler_output.finished_req_ids)
|
||||||
)
|
)
|
||||||
|
output.ec_connector_worker_meta = ec_connector.build_connector_worker_meta()
|
||||||
ec_connector.clear_connector_metadata()
|
ec_connector.clear_connector_metadata()
|
||||||
|
|
||||||
|
def no_forward(
|
||||||
|
self,
|
||||||
|
scheduler_output: "SchedulerOutput",
|
||||||
|
) -> ModelRunnerOutput:
|
||||||
|
# EC send/recv even if no work to do.
|
||||||
|
with self.maybe_get_output(scheduler_output) as ec_connector_output:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return ModelRunnerOutput.with_ec_conn_output_only(ec_connector_output)
|
||||||
|
|
||||||
|
|
||||||
NO_OP_EC_CONNECTOR = ECConnector()
|
NO_OP_EC_CONNECTOR = ECConnector()
|
||||||
|
|
||||||
|
|||||||
@@ -65,6 +65,7 @@ from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
|||||||
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
|
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
|
||||||
from vllm.v1.outputs import (
|
from vllm.v1.outputs import (
|
||||||
DraftTokenIds,
|
DraftTokenIds,
|
||||||
|
ECConnectorOutput,
|
||||||
ModelRunnerOutput,
|
ModelRunnerOutput,
|
||||||
RoutedExpertsTensors,
|
RoutedExpertsTensors,
|
||||||
make_empty_encoder_model_runner_output,
|
make_empty_encoder_model_runner_output,
|
||||||
@@ -1393,6 +1394,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
idx_mapping, num_sampled, self.req_states.num_computed_tokens.gpu
|
idx_mapping, num_sampled, self.req_states.num_computed_tokens.gpu
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _merge_ec_connector_no_forward(
|
||||||
|
self, scheduler_output: SchedulerOutput, output: ModelRunnerOutput
|
||||||
|
) -> ModelRunnerOutput:
|
||||||
|
"""Let the EC connector send/recv on a step with no work to run."""
|
||||||
|
return ModelRunnerOutput.with_ec_conn_output(
|
||||||
|
output,
|
||||||
|
self.ec_connector.no_forward(scheduler_output).ec_connector_output,
|
||||||
|
)
|
||||||
|
|
||||||
@torch.inference_mode()
|
@torch.inference_mode()
|
||||||
def execute_model(
|
def execute_model(
|
||||||
self,
|
self,
|
||||||
@@ -1414,7 +1424,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
if scheduler_output.total_num_scheduled_tokens == 0:
|
if scheduler_output.total_num_scheduled_tokens == 0:
|
||||||
# No need to run the model.
|
# No need to run the model.
|
||||||
empty_output = self.kv_connector.no_forward(scheduler_output)
|
empty_output = self.kv_connector.no_forward(scheduler_output)
|
||||||
return empty_output
|
return self._merge_ec_connector_no_forward(
|
||||||
|
scheduler_output, empty_output
|
||||||
|
)
|
||||||
|
|
||||||
# Get batch descriptor and sync across DP ranks.
|
# Get batch descriptor and sync across DP ranks.
|
||||||
num_reqs = len(scheduler_output.num_scheduled_tokens)
|
num_reqs = len(scheduler_output.num_scheduled_tokens)
|
||||||
@@ -1455,7 +1467,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
if batch_desc.num_tokens == 0:
|
if batch_desc.num_tokens == 0:
|
||||||
# All DP ranks have zero tokens to run.
|
# All DP ranks have zero tokens to run.
|
||||||
empty_output = self.kv_connector.no_forward(scheduler_output)
|
empty_output = self.kv_connector.no_forward(scheduler_output)
|
||||||
return empty_output
|
return self._merge_ec_connector_no_forward(scheduler_output, empty_output)
|
||||||
|
|
||||||
if not dummy_run:
|
if not dummy_run:
|
||||||
# Common case.
|
# Common case.
|
||||||
@@ -1572,9 +1584,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
input_ids = None
|
input_ids = None
|
||||||
|
|
||||||
if self.is_encoder_only:
|
if self.is_encoder_only:
|
||||||
output = make_empty_encoder_model_runner_output(scheduler_output)
|
return ModelRunnerOutput.with_ec_conn_output(
|
||||||
output.ec_connector_output = ec_connector_output
|
make_empty_encoder_model_runner_output(scheduler_output),
|
||||||
return output
|
ec_connector_output,
|
||||||
|
)
|
||||||
|
|
||||||
model_inputs = {
|
model_inputs = {
|
||||||
"input_ids": input_ids,
|
"input_ids": input_ids,
|
||||||
@@ -1679,6 +1692,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
hidden_states=hidden_states,
|
hidden_states=hidden_states,
|
||||||
aux_hidden_states=aux_hidden_states,
|
aux_hidden_states=aux_hidden_states,
|
||||||
finished_req_ids=finished_req_ids,
|
finished_req_ids=finished_req_ids,
|
||||||
|
ec_connector_output=ec_connector_output,
|
||||||
routed_experts=routed_experts,
|
routed_experts=routed_experts,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -1702,6 +1716,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
hidden_states = self.execute_model_state.hidden_states
|
hidden_states = self.execute_model_state.hidden_states
|
||||||
aux_hidden_states = self.execute_model_state.aux_hidden_states
|
aux_hidden_states = self.execute_model_state.aux_hidden_states
|
||||||
finished_req_ids = self.execute_model_state.finished_req_ids
|
finished_req_ids = self.execute_model_state.finished_req_ids
|
||||||
|
ec_connector_output = self.execute_model_state.ec_connector_output
|
||||||
routed_experts = self.execute_model_state.routed_experts
|
routed_experts = self.execute_model_state.routed_experts
|
||||||
self.execute_model_state = None
|
self.execute_model_state = None
|
||||||
|
|
||||||
@@ -1721,7 +1736,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
|
|
||||||
# Post-step KV connector related operations.
|
# Post-step KV connector related operations.
|
||||||
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
|
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
|
||||||
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
|
# The first PP rank holds the encoder cache, so pass its EC output on.
|
||||||
|
output = ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
|
||||||
|
return ModelRunnerOutput.with_ec_conn_output(output, ec_connector_output)
|
||||||
|
|
||||||
# Last rank: sample tokens
|
# Last rank: sample tokens
|
||||||
hidden_states, input_batch = pcp.maybe_restore_pcp_for_sampling(
|
hidden_states, input_batch = pcp.maybe_restore_pcp_for_sampling(
|
||||||
@@ -1838,6 +1855,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
# Post-step KV connector related operations.
|
# Post-step KV connector related operations.
|
||||||
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
|
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
|
||||||
model_runner_output.kv_connector_output = kv_connector_output
|
model_runner_output.kv_connector_output = kv_connector_output
|
||||||
|
model_runner_output.ec_connector_output = ec_connector_output
|
||||||
|
|
||||||
return async_output
|
return async_output
|
||||||
|
|
||||||
@@ -1854,6 +1872,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
input_batch = self.execute_model_state.input_batch
|
input_batch = self.execute_model_state.input_batch
|
||||||
hidden_states = self.execute_model_state.hidden_states
|
hidden_states = self.execute_model_state.hidden_states
|
||||||
finished_req_ids = self.execute_model_state.finished_req_ids
|
finished_req_ids = self.execute_model_state.finished_req_ids
|
||||||
|
ec_connector_output = self.execute_model_state.ec_connector_output
|
||||||
self.execute_model_state = None
|
self.execute_model_state = None
|
||||||
|
|
||||||
# Post-step KV connector related operations.
|
# Post-step KV connector related operations.
|
||||||
@@ -1861,7 +1880,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
|
|
||||||
if not self.is_last_pp_rank:
|
if not self.is_last_pp_rank:
|
||||||
self.postprocess_num_computed_tokens(input_batch)
|
self.postprocess_num_computed_tokens(input_batch)
|
||||||
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
|
output = ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
|
||||||
|
return ModelRunnerOutput.with_ec_conn_output(output, ec_connector_output)
|
||||||
|
|
||||||
assert self.pooling_runner is not None
|
assert self.pooling_runner is not None
|
||||||
pooler_output, finished_mask = self.pooling_runner.pool(
|
pooler_output, finished_mask = self.pooling_runner.pool(
|
||||||
@@ -1873,6 +1893,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
req_ids=input_batch.req_ids,
|
req_ids=input_batch.req_ids,
|
||||||
req_id_to_index={req_id: i for i, req_id in enumerate(input_batch.req_ids)},
|
req_id_to_index={req_id: i for i, req_id in enumerate(input_batch.req_ids)},
|
||||||
kv_connector_output=kv_connector_output,
|
kv_connector_output=kv_connector_output,
|
||||||
|
ec_connector_output=ec_connector_output,
|
||||||
)
|
)
|
||||||
async_output = AsyncPoolingOutput(
|
async_output = AsyncPoolingOutput(
|
||||||
model_runner_output=model_runner_output,
|
model_runner_output=model_runner_output,
|
||||||
@@ -1962,6 +1983,7 @@ class ExecuteModelState(NamedTuple):
|
|||||||
hidden_states: torch.Tensor | None
|
hidden_states: torch.Tensor | None
|
||||||
aux_hidden_states: list[torch.Tensor] | None
|
aux_hidden_states: list[torch.Tensor] | None
|
||||||
finished_req_ids: set[str]
|
finished_req_ids: set[str]
|
||||||
|
ec_connector_output: ECConnectorOutput | None
|
||||||
routed_experts: RoutedExpertsTensors | None
|
routed_experts: RoutedExpertsTensors | None
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user