diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index 8c80fead29..126bc65704 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -127,6 +127,7 @@ steps: - pytest -v -s v1/test_kv_cache_spec_registry.py - pytest -v -s v1/cudagraph/test_cudagraph_manager.py - pytest -v -s -m 'cpu_test' v1/kv_connector/unit + - pytest -v -s -m 'cpu_test' v1/ec_connector/unit - pytest -v -s -m 'cpu_test' v1/metrics - label: Extract Hidden States Integration diff --git a/tests/v1/ec_connector/unit/test_ec_output_aggregator.py b/tests/v1/ec_connector/unit/test_ec_output_aggregator.py new file mode 100644 index 0000000000..4d5b7dae10 --- /dev/null +++ b/tests/v1/ec_connector/unit/test_ec_output_aggregator.py @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for ECOutputAggregator.""" + +import pytest + +from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorWorkerMetadata +from vllm.distributed.ec_transfer.ec_connector.utils import ECOutputAggregator +from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator +from vllm.v1.outputs import ( + EMPTY_MODEL_RUNNER_OUTPUT, + ECConnectorOutput, + KVConnectorOutput, + ModelRunnerOutput, +) + +pytestmark = pytest.mark.cpu_test + + +class FakeWorkerMeta(ECConnectorWorkerMetadata): + """Records merge order. `aggregate` returns a new object, as the base class + declares: an aggregator discarding the return value would lose the merge. + """ + + def __init__(self, saves: list[str]): + self.saves = saves + + def aggregate(self, other: "FakeWorkerMeta") -> "FakeWorkerMeta": + return FakeWorkerMeta(self.saves + other.saves) + + +def _worker_output(ec_output: ECConnectorOutput | None) -> ModelRunnerOutput: + return ModelRunnerOutput( + req_ids=[], req_id_to_index={}, ec_connector_output=ec_output + ) + + +def test_aggregate_folds_every_rank_onto_output_rank(): + """EC work done on any rank reaches the scheduler via output_rank's output. + + The middle rank reports no worker metadata: it must neither seed nor clobber + the accumulator. + """ + outputs = [ + _worker_output( + ECConnectorOutput( + finished_sending={"mm0"}, + ec_connector_worker_meta=FakeWorkerMeta(["mm0"]), + ) + ), + _worker_output(ECConnectorOutput(finished_recving={"mm1"})), + _worker_output( + ECConnectorOutput(ec_connector_worker_meta=FakeWorkerMeta(["mm2"])) + ), + ] + + result = ECOutputAggregator().aggregate(outputs, output_rank=2) + + assert result is outputs[2] + assert result.ec_connector_output.finished_sending == {"mm0"} + assert result.ec_connector_output.finished_recving == {"mm1"} + assert result.ec_connector_output.ec_connector_worker_meta.saves == ["mm0", "mm2"] + + +def test_aggregate_leaves_no_ec_output_when_no_worker_reported(): + """Empty per-worker reports must not reach the scheduler as an empty object.""" + outputs = [_worker_output(ECConnectorOutput()), _worker_output(ECConnectorOutput())] + + result = ECOutputAggregator().aggregate(outputs, output_rank=0) + + assert result is outputs[0] + assert result.ec_connector_output is None + assert ECOutputAggregator().aggregate([None], output_rank=0) is None + + +def test_aggregate_does_not_write_through_the_shared_empty_output(): + """A rank with nothing to report yields the shared empty output singleton. + + Folding another rank's metadata onto it must not write through to the + module-level object, which every later step would then carry. + """ + outputs = [ + _worker_output( + ECConnectorOutput(ec_connector_worker_meta=FakeWorkerMeta(["mm0"])) + ), + EMPTY_MODEL_RUNNER_OUTPUT, + ] + + result = ECOutputAggregator().aggregate(outputs, output_rank=1) + + assert EMPTY_MODEL_RUNNER_OUTPUT.ec_connector_output is None + assert result is not EMPTY_MODEL_RUNNER_OUTPUT + assert result.ec_connector_output.ec_connector_worker_meta.saves == ["mm0"] + + +def test_chaining_with_kv_aggregator_preserves_both_outputs(): + """MultiprocExecutor chains both aggregators and keeps only the last result, + so each must merge onto the same output_rank output rather than replace it. + """ + outputs = [ + _worker_output(ECConnectorOutput(finished_sending={"mm0"})), + _worker_output(None), + ] + outputs[1].kv_connector_output = KVConnectorOutput(invalid_block_ids={7}) + + result = None + for aggregator in ( + KVOutputAggregator(expected_finished_count=1), + ECOutputAggregator(), + ): + result = aggregator.aggregate(outputs, output_rank=1) + + assert result is outputs[1] + assert result.kv_connector_output.invalid_block_ids == {7} + assert result.ec_connector_output.finished_sending == {"mm0"} diff --git a/tests/v1/ec_connector/unit/test_worker_ec_connector.py b/tests/v1/ec_connector/unit/test_worker_ec_connector.py new file mode 100644 index 0000000000..3dcad1e50a --- /dev/null +++ b/tests/v1/ec_connector/unit/test_worker_ec_connector.py @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for the V2 GPU model runner's EC connector wrapper.""" + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +from vllm.distributed.ec_transfer.ec_connector.base import ( + ECConnectorBase, + ECConnectorMetadata, +) +from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT +from vllm.v1.worker.gpu.ec_connector import NO_OP_EC_CONNECTOR, ActiveECConnector + +pytestmark = pytest.mark.cpu_test + +WORKER_META = object() + + +def _scheduler_output() -> SimpleNamespace: + return SimpleNamespace( + ec_connector_metadata=ECConnectorMetadata(), finished_req_ids=frozenset() + ) + + +def _connector( + encoder_cache: dict | None = None, + is_producer: bool = True, + is_consumer: bool = False, +) -> tuple[ActiveECConnector, MagicMock]: + fake = MagicMock(spec=ECConnectorBase) + fake.is_producer = is_producer + fake.is_consumer = is_consumer + fake.get_finished.return_value = (None, None) + fake.build_connector_worker_meta.return_value = WORKER_META + with patch("vllm.v1.worker.gpu.ec_connector.get_ec_transfer", return_value=fake): + return ActiveECConnector(SimpleNamespace(), encoder_cache or {}), fake + + +@pytest.mark.parametrize( + ("is_producer", "is_consumer"), [(True, False), (True, True), (False, True)] +) +def test_saves_newly_added_caches_for_every_producer(is_producer, is_consumer): + """An ec_both node is also a producer: it must offload what it just computed.""" + encoder_cache = {"mm_old": None} + connector, fake = _connector(encoder_cache, is_producer, is_consumer) + + with connector.maybe_get_output(_scheduler_output()): + encoder_cache["mm_new"] = None + + saved = [call.kwargs["mm_hash"] for call in fake.save_caches.call_args_list] + assert saved == (["mm_new"] if is_producer else []) + assert fake.start_load_caches.called == is_consumer + + +def test_worker_meta_is_reported_on_context_exit(): + """Reported in the finally block, so is_empty() sees it only after the exit.""" + connector, fake = _connector() + + with connector.maybe_get_output(_scheduler_output()) as output: + assert output.ec_connector_worker_meta is None + + assert output.ec_connector_worker_meta is WORKER_META + assert fake.clear_connector_metadata.called + + +def test_no_forward_reports_without_running_the_model(): + connector, _ = _connector() + + output = connector.no_forward(_scheduler_output()) + + assert output.ec_connector_output.ec_connector_worker_meta is WORKER_META + + empty = NO_OP_EC_CONNECTOR.no_forward(_scheduler_output()) + assert empty is EMPTY_MODEL_RUNNER_OUTPUT diff --git a/tests/v1/executor/test_executor.py b/tests/v1/executor/test_executor.py index c529c3204d..a21a8dafd3 100644 --- a/tests/v1/executor/test_executor.py +++ b/tests/v1/executor/test_executor.py @@ -9,6 +9,7 @@ from typing import Any import pytest +from vllm.distributed.ec_transfer.ec_connector.utils import ECOutputAggregator from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs from vllm.sampling_params import SamplingParams @@ -98,6 +99,7 @@ class CustomMultiprocExecutor(MultiprocExecutor): non_block: bool = False, unique_reply_rank: int | None = None, kv_output_aggregator: KVOutputAggregator = None, + ec_output_aggregator: ECOutputAggregator | None = None, ) -> Any | list[Any] | Future[Any | list[Any]]: # Drop marker to show that this was run with open(".marker", "w"): @@ -110,6 +112,7 @@ class CustomMultiprocExecutor(MultiprocExecutor): non_block, unique_reply_rank, kv_output_aggregator, + ec_output_aggregator, ) diff --git a/tests/v1/kv_connector/unit/test_handshake_pp_aggregation.py b/tests/v1/kv_connector/unit/test_handshake_pp_aggregation.py index 0c0f9f1f89..e5cbd977b6 100644 --- a/tests/v1/kv_connector/unit/test_handshake_pp_aggregation.py +++ b/tests/v1/kv_connector/unit/test_handshake_pp_aggregation.py @@ -55,6 +55,7 @@ def _run_engine_core_handshake( class _FakeScheduler: def __init__(self, **kwargs: Any) -> None: self.connector = connector + self.ec_connector = None def get_kv_connector(self) -> KVConnectorBase_V1: return connector diff --git a/tests/v1/test_outputs.py b/tests/v1/test_outputs.py index 06fd9b409b..d93ae96e54 100644 --- a/tests/v1/test_outputs.py +++ b/tests/v1/test_outputs.py @@ -6,7 +6,13 @@ import numpy as np import torch from vllm.platforms import current_platform -from vllm.v1.outputs import LogprobsLists, LogprobsTensors +from vllm.v1.outputs import ( + EMPTY_MODEL_RUNNER_OUTPUT, + ECConnectorOutput, + LogprobsLists, + LogprobsTensors, + ModelRunnerOutput, +) from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p from vllm.v1.worker.gpu.sample.output import SamplingMaskTensors @@ -200,3 +206,14 @@ class TestLogprobsLists(TestCase): assert len(sliced.logprob_token_ids) == 9 # All tokens assert sliced.logprob_token_ids == self.logprobsLists.logprob_token_ids assert sliced.cu_num_generated_tokens is None + + +def test_with_ec_conn_output_copies_shared_empty_output(): + """The shared empty output is copied, never written to.""" + ec_output = ECConnectorOutput(finished_sending={"mm_hash"}) + + result = ModelRunnerOutput.with_ec_conn_output(EMPTY_MODEL_RUNNER_OUTPUT, ec_output) + + assert result is not EMPTY_MODEL_RUNNER_OUTPUT + assert result.ec_connector_output is ec_output + assert EMPTY_MODEL_RUNNER_OUTPUT.ec_connector_output is None diff --git a/tests/v1/worker/test_gpu_model_runner_v2_eplb.py b/tests/v1/worker/test_gpu_model_runner_v2_eplb.py index fbeaa198d0..ebb4beb2a5 100644 --- a/tests/v1/worker/test_gpu_model_runner_v2_eplb.py +++ b/tests/v1/worker/test_gpu_model_runner_v2_eplb.py @@ -180,6 +180,7 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch): hidden_states=None, aux_hidden_states=None, finished_req_ids=set(), + ec_connector_output=None, routed_experts=None, num_tokens_across_dp=None, ) diff --git a/vllm/distributed/ec_transfer/ec_connector/base.py b/vllm/distributed/ec_transfer/ec_connector/base.py index 3c20a4a1f7..9138203bf4 100644 --- a/vllm/distributed/ec_transfer/ec_connector/base.py +++ b/vllm/distributed/ec_transfer/ec_connector/base.py @@ -20,6 +20,8 @@ The class provides the following primitives: get_finished() - called with ids of finished requests, returns ids of requests that have completed async sending/recving. + build_connector_worker_meta() - builds metadata to be sent + back to the scheduler-side connector """ import enum @@ -56,6 +58,27 @@ class ECConnectorMetadata(ABC): # noqa: B024 pass +class ECConnectorWorkerMetadata(ABC): + """ + Abstract Metadata used to communicate back + Worker ECConnector -> Scheduler ECConnector. + + Each worker can output its own metadata. + For a single engine step, all metadata objects returned by workers + will be aggregated using the `aggregate` method below, before + being passed to the Scheduler ECConnector. + """ + + @abstractmethod + def aggregate( + self, other: "ECConnectorWorkerMetadata" + ) -> "ECConnectorWorkerMetadata": + """ + Aggregate metadata with another `ECConnectorWorkerMetadata` object. + """ + pass + + class ECConnectorBase(ABC): def __init__(self, vllm_config: "VllmConfig", role: ECConnectorRole): self._connector_metadata: ECConnectorMetadata | None = None @@ -190,6 +213,16 @@ class ECConnectorBase(ABC): """ return None, None + def build_connector_worker_meta(self) -> ECConnectorWorkerMetadata | None: + """ + Build the ECConnector worker metadata for this engine step. + + Returns: + ECConnectorWorkerMetadata: the worker metadata. + None if no worker metadata is available. + """ + return None + # ============================== # Scheduler-side methods # ============================== diff --git a/vllm/distributed/ec_transfer/ec_connector/utils.py b/vllm/distributed/ec_transfer/ec_connector/utils.py new file mode 100644 index 0000000000..f5f78e6c3c --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/utils.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""EC connector helper utilities.""" + +from vllm.v1.outputs import ECConnectorOutput, ModelRunnerOutput + + +class ECOutputAggregator: + """Merge every worker's EC connector output onto the single + ModelRunnerOutput that reaches the scheduler. + + Mirrors KVOutputAggregator: only `output_rank`'s output is returned to the + scheduler, but the EC connector may have run on any rank. + """ + + def aggregate( + self, outputs: list[ModelRunnerOutput | None], output_rank: int = 0 + ) -> ModelRunnerOutput | None: + output = outputs[output_rank] + if not output: + return None + + finished_sending = set[str]() + finished_recving = set[str]() + worker_meta = None + for model_runner_output in outputs: + assert model_runner_output is not None + ec_output = model_runner_output.ec_connector_output + if not ec_output: + continue + + finished_sending |= ec_output.finished_sending or set() + finished_recving |= ec_output.finished_recving or set() + + if meta := ec_output.ec_connector_worker_meta: + worker_meta = ( + meta if worker_meta is None else worker_meta.aggregate(meta) + ) + + aggregated = ECConnectorOutput( + finished_sending=finished_sending or None, + finished_recving=finished_recving or None, + ec_connector_worker_meta=worker_meta, + ) + if aggregated.is_empty(): + output.ec_connector_output = None + return output + + # `output` is the shared empty output whenever `output_rank` had no work, + # so attach through the copy-on-write helper. + return ModelRunnerOutput.with_ec_conn_output(output, aggregated) diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index 55fe5b90c5..295c66b360 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -173,6 +173,8 @@ class EngineCore: ) if self.scheduler.connector is not None: # type: ignore self.model_executor.init_kv_output_aggregator(self.scheduler.connector) # type: ignore + if self.scheduler.ec_connector is not None: # type: ignore + self.model_executor.init_ec_output_aggregator() mm_registry = MULTIMODAL_REGISTRY self.mm_receiver_cache = mm_registry.engine_receiver_cache_from_config( diff --git a/vllm/v1/executor/abstract.py b/vllm/v1/executor/abstract.py index 864989e6ac..6fff5f032d 100644 --- a/vllm/v1/executor/abstract.py +++ b/vllm/v1/executor/abstract.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Literal, TypeVar, overload import vllm.envs as envs from vllm.config import VllmConfig +from vllm.distributed.ec_transfer.ec_connector.utils import ECOutputAggregator from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorHandshakeMetadata, @@ -110,6 +111,7 @@ class Executor(ABC): self.is_sleeping = False self.sleeping_tags: set[str] = set() self.kv_output_aggregator: KVOutputAggregator | None = None + self.ec_output_aggregator: ECOutputAggregator | None = None @abstractmethod def _init_executor(self) -> None: @@ -283,6 +285,9 @@ class Executor(ABC): connector, self.parallel_config.world_size ) + def init_ec_output_aggregator(self) -> None: + self.ec_output_aggregator = ECOutputAggregator() + @cached_property # Avoid unnecessary RPC calls def supported_tasks(self) -> tuple[SupportedTask, ...]: output: list[tuple[SupportedTask, ...]] diff --git a/vllm/v1/executor/multiproc_executor.py b/vllm/v1/executor/multiproc_executor.py index afc333723d..df20a90d51 100644 --- a/vllm/v1/executor/multiproc_executor.py +++ b/vllm/v1/executor/multiproc_executor.py @@ -29,6 +29,7 @@ import vllm.envs as envs from vllm.config import VllmConfig from vllm.distributed import destroy_distributed_environment, destroy_model_parallel from vllm.distributed.device_communicators.shm_broadcast import Handle, MessageQueue +from vllm.distributed.ec_transfer.ec_connector.utils import ECOutputAggregator from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator from vllm.distributed.parallel_state import ( get_dcp_group, @@ -343,6 +344,7 @@ class MultiprocExecutor(Executor): non_block=non_block, timeout=envs.VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS, kv_output_aggregator=self.kv_output_aggregator, + ec_output_aggregator=self.ec_output_aggregator, ) def sample_tokens( # type: ignore[override] @@ -355,6 +357,7 @@ class MultiprocExecutor(Executor): non_block=non_block, timeout=envs.VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS, kv_output_aggregator=self.kv_output_aggregator, + ec_output_aggregator=self.ec_output_aggregator, ) def execute_dummy_batch(self) -> None: @@ -375,9 +378,10 @@ class MultiprocExecutor(Executor): non_block: bool = False, unique_reply_rank: int | None = None, kv_output_aggregator: KVOutputAggregator | None = None, + ec_output_aggregator: ECOutputAggregator | None = None, ) -> Any: - """Returns single result if unique_reply_rank and/or kv_output_aggregator - is provided, otherwise list.""" + """Returns single result if unique_reply_rank and/or an output + aggregator is provided, otherwise list.""" assert self.rpc_broadcast_mq is not None, ( "collective_rpc should not be called on follower node" ) @@ -387,11 +391,21 @@ class MultiprocExecutor(Executor): deadline = None if timeout is None else time.monotonic() + timeout kwargs = kwargs or {} - if kv_output_aggregator is not None: + aggregators = [a for a in (kv_output_aggregator, ec_output_aggregator) if a] + aggregate: Callable[[Any], Any] + if aggregators: output_rank = None - aggregate: Callable[[Any], Any] = partial( - kv_output_aggregator.aggregate, output_rank=unique_reply_rank or 0 - ) + + def _aggregate(outputs: Any) -> Any: + # Each aggregator merges its own connector's output onto + # outputs[output_rank] in place and returns it, so chaining is safe. + rank = unique_reply_rank or 0 + result = outputs[rank] + for a in aggregators: + result = a.aggregate(outputs, output_rank=rank) + return result + + aggregate = _aggregate else: output_rank = unique_reply_rank aggregate = lambda x: x diff --git a/vllm/v1/executor/ray_executor.py b/vllm/v1/executor/ray_executor.py index 39749ffc25..986e4d9bbb 100644 --- a/vllm/v1/executor/ray_executor.py +++ b/vllm/v1/executor/ray_executor.py @@ -89,6 +89,19 @@ class RayDistributedExecutor(Executor): # KV connector setup self.has_connector = self.vllm_config.kv_transfer_config is not None + if ( + self.vllm_config.ec_transfer_config is not None + and self.parallel_config.world_size > 1 + ): + raise NotImplementedError( + "EC connector worker metadata is not supported with the " + "legacy Ray executor when world_size > 1: only the output " + "of a single worker is fetched, silently dropping the " + "other workers' EC connector state. Set " + "VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1 to use RayExecutorV2, " + "or use the multiprocessing executor instead." + ) + self.uses_sampler = self.vllm_config.model_config.runner_type != "pooling" and ( self.vllm_config.ec_transfer_config is None or self.vllm_config.ec_transfer_config.is_ec_consumer diff --git a/vllm/v1/outputs.py b/vllm/v1/outputs.py index 0bbee76675..80b909afbc 100644 --- a/vllm/v1/outputs.py +++ b/vllm/v1/outputs.py @@ -14,6 +14,7 @@ from vllm.compilation.cuda_graph import CUDAGraphStat from vllm.v1.core.sched.output import SchedulerOutput if TYPE_CHECKING: + from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorWorkerMetadata from vllm.distributed.kv_events import KVConnectorKVEvents from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorWorkerMetadata, @@ -23,6 +24,7 @@ else: KVConnectorStats = object KVConnectorWorkerMetadata = object KVConnectorKVEvents = object + ECConnectorWorkerMetadata = object class LogprobsLists(NamedTuple): @@ -292,6 +294,14 @@ class ECConnectorOutput: # [mm_hash] finished_sending: set[str] | None = None finished_recving: set[str] | None = None + ec_connector_worker_meta: ECConnectorWorkerMetadata | None = None + + def is_empty(self): + return ( + not self.finished_sending + and not self.finished_recving + and not self.ec_connector_worker_meta + ) # ModelRunnerOutput is serialized and sent to the scheduler process. @@ -362,6 +372,32 @@ class ModelRunnerOutput: output.kv_connector_output = kv_connector_output return output + @staticmethod + def with_ec_conn_output_only( + ec_connector_output: ECConnectorOutput | None, + ) -> "ModelRunnerOutput": + """Return an otherwise-empty output carrying `ec_connector_output`.""" + return ModelRunnerOutput.with_ec_conn_output( + EMPTY_MODEL_RUNNER_OUTPUT, ec_connector_output + ) + + @staticmethod + def with_ec_conn_output( + output: "ModelRunnerOutput", + ec_connector_output: ECConnectorOutput | None, + ) -> "ModelRunnerOutput": + """Return `output` carrying `ec_connector_output`. + + The shared empty output is copied rather than written to, so callers + must use the return value. + """ + if ec_connector_output is None or ec_connector_output.is_empty(): + return output + if output is EMPTY_MODEL_RUNNER_OUTPUT: + output = copy(EMPTY_MODEL_RUNNER_OUTPUT) + output.ec_connector_output = ec_connector_output + return output + # ModelRunnerOutput wrapper for async scheduling. class AsyncModelRunnerOutput(ABC): diff --git a/vllm/v1/worker/gpu/ec_connector.py b/vllm/v1/worker/gpu/ec_connector.py index 825763b820..5dc8d92359 100644 --- a/vllm/v1/worker/gpu/ec_connector.py +++ b/vllm/v1/worker/gpu/ec_connector.py @@ -9,7 +9,11 @@ import torch from vllm.config import VllmConfig from vllm.distributed.ec_transfer import get_ec_transfer, has_ec_transfer from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorBase -from vllm.v1.outputs import ECConnectorOutput +from vllm.v1.outputs import ( + EMPTY_MODEL_RUNNER_OUTPUT, + ECConnectorOutput, + ModelRunnerOutput, +) if TYPE_CHECKING: from vllm.v1.core.sched.output import SchedulerOutput @@ -25,6 +29,12 @@ class ECConnector: ) -> Generator[ECConnectorOutput | None, None, None]: yield None + def no_forward( + self, + scheduler_output: "SchedulerOutput", + ) -> ModelRunnerOutput: + return EMPTY_MODEL_RUNNER_OUTPUT + class ActiveECConnector(ECConnector): def __init__( @@ -33,9 +43,11 @@ class ActiveECConnector(ECConnector): encoder_cache: dict[str, torch.Tensor], ) -> None: self.encoder_cache = encoder_cache - self.save_new_caches = vllm_config.is_ec_producer_only self.ec_connector = get_ec_transfer() assert isinstance(self.ec_connector, ECConnectorBase) + # Every producer offloads freshly computed encoder outputs, including + # an ec_both node that also reloads them. + self.save_new_caches = self.ec_connector.is_producer @contextmanager def maybe_get_output( @@ -65,8 +77,19 @@ class ActiveECConnector(ECConnector): output.finished_sending, output.finished_recving = ( ec_connector.get_finished(scheduler_output.finished_req_ids) ) + output.ec_connector_worker_meta = ec_connector.build_connector_worker_meta() ec_connector.clear_connector_metadata() + def no_forward( + self, + scheduler_output: "SchedulerOutput", + ) -> ModelRunnerOutput: + # EC send/recv even if no work to do. + with self.maybe_get_output(scheduler_output) as ec_connector_output: + pass + + return ModelRunnerOutput.with_ec_conn_output_only(ec_connector_output) + NO_OP_EC_CONNECTOR = ECConnector() diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index c92ffedf59..abbe6d5d3d 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -65,6 +65,7 @@ from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec from vllm.v1.outputs import ( DraftTokenIds, + ECConnectorOutput, ModelRunnerOutput, RoutedExpertsTensors, make_empty_encoder_model_runner_output, @@ -1393,6 +1394,15 @@ class GPUModelRunner(LoRAModelRunnerMixin): idx_mapping, num_sampled, self.req_states.num_computed_tokens.gpu ) + def _merge_ec_connector_no_forward( + self, scheduler_output: SchedulerOutput, output: ModelRunnerOutput + ) -> ModelRunnerOutput: + """Let the EC connector send/recv on a step with no work to run.""" + return ModelRunnerOutput.with_ec_conn_output( + output, + self.ec_connector.no_forward(scheduler_output).ec_connector_output, + ) + @torch.inference_mode() def execute_model( self, @@ -1414,7 +1424,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): if scheduler_output.total_num_scheduled_tokens == 0: # No need to run the model. empty_output = self.kv_connector.no_forward(scheduler_output) - return empty_output + return self._merge_ec_connector_no_forward( + scheduler_output, empty_output + ) # Get batch descriptor and sync across DP ranks. num_reqs = len(scheduler_output.num_scheduled_tokens) @@ -1455,7 +1467,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): if batch_desc.num_tokens == 0: # All DP ranks have zero tokens to run. empty_output = self.kv_connector.no_forward(scheduler_output) - return empty_output + return self._merge_ec_connector_no_forward(scheduler_output, empty_output) if not dummy_run: # Common case. @@ -1572,9 +1584,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): input_ids = None if self.is_encoder_only: - output = make_empty_encoder_model_runner_output(scheduler_output) - output.ec_connector_output = ec_connector_output - return output + return ModelRunnerOutput.with_ec_conn_output( + make_empty_encoder_model_runner_output(scheduler_output), + ec_connector_output, + ) model_inputs = { "input_ids": input_ids, @@ -1679,6 +1692,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): hidden_states=hidden_states, aux_hidden_states=aux_hidden_states, finished_req_ids=finished_req_ids, + ec_connector_output=ec_connector_output, routed_experts=routed_experts, ) @@ -1702,6 +1716,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): hidden_states = self.execute_model_state.hidden_states aux_hidden_states = self.execute_model_state.aux_hidden_states finished_req_ids = self.execute_model_state.finished_req_ids + ec_connector_output = self.execute_model_state.ec_connector_output routed_experts = self.execute_model_state.routed_experts self.execute_model_state = None @@ -1721,7 +1736,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): # Post-step KV connector related operations. kv_connector_output = self.kv_connector.post_forward(finished_req_ids) - return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output) + # The first PP rank holds the encoder cache, so pass its EC output on. + output = ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output) + return ModelRunnerOutput.with_ec_conn_output(output, ec_connector_output) # Last rank: sample tokens hidden_states, input_batch = pcp.maybe_restore_pcp_for_sampling( @@ -1838,6 +1855,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): # Post-step KV connector related operations. kv_connector_output = self.kv_connector.post_forward(finished_req_ids) model_runner_output.kv_connector_output = kv_connector_output + model_runner_output.ec_connector_output = ec_connector_output return async_output @@ -1854,6 +1872,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): input_batch = self.execute_model_state.input_batch hidden_states = self.execute_model_state.hidden_states finished_req_ids = self.execute_model_state.finished_req_ids + ec_connector_output = self.execute_model_state.ec_connector_output self.execute_model_state = None # Post-step KV connector related operations. @@ -1861,7 +1880,8 @@ class GPUModelRunner(LoRAModelRunnerMixin): if not self.is_last_pp_rank: self.postprocess_num_computed_tokens(input_batch) - return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output) + output = ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output) + return ModelRunnerOutput.with_ec_conn_output(output, ec_connector_output) assert self.pooling_runner is not None pooler_output, finished_mask = self.pooling_runner.pool( @@ -1873,6 +1893,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): req_ids=input_batch.req_ids, req_id_to_index={req_id: i for i, req_id in enumerate(input_batch.req_ids)}, kv_connector_output=kv_connector_output, + ec_connector_output=ec_connector_output, ) async_output = AsyncPoolingOutput( model_runner_output=model_runner_output, @@ -1962,6 +1983,7 @@ class ExecuteModelState(NamedTuple): hidden_states: torch.Tensor | None aux_hidden_states: list[torch.Tensor] | None finished_req_ids: set[str] + ec_connector_output: ECConnectorOutput | None routed_experts: RoutedExpertsTensors | None