[EC Connector] Added Build Connector Worker Meta for EC Connector (#49585)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: omerpaz95 <omerpaz95@gmail.com> Signed-off-by: omerpaz95 <73347585+omerpaz95@users.noreply.github.com> Co-authored-by: yewentao256 <zhyanwentao@126.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
yewentao256
Wentao Ye
parent
c94cdd0ae0
commit
fa9d67f782
@@ -0,0 +1,115 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for ECOutputAggregator."""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorWorkerMetadata
|
||||
from vllm.distributed.ec_transfer.ec_connector.utils import ECOutputAggregator
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator
|
||||
from vllm.v1.outputs import (
|
||||
EMPTY_MODEL_RUNNER_OUTPUT,
|
||||
ECConnectorOutput,
|
||||
KVConnectorOutput,
|
||||
ModelRunnerOutput,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
class FakeWorkerMeta(ECConnectorWorkerMetadata):
|
||||
"""Records merge order. `aggregate` returns a new object, as the base class
|
||||
declares: an aggregator discarding the return value would lose the merge.
|
||||
"""
|
||||
|
||||
def __init__(self, saves: list[str]):
|
||||
self.saves = saves
|
||||
|
||||
def aggregate(self, other: "FakeWorkerMeta") -> "FakeWorkerMeta":
|
||||
return FakeWorkerMeta(self.saves + other.saves)
|
||||
|
||||
|
||||
def _worker_output(ec_output: ECConnectorOutput | None) -> ModelRunnerOutput:
|
||||
return ModelRunnerOutput(
|
||||
req_ids=[], req_id_to_index={}, ec_connector_output=ec_output
|
||||
)
|
||||
|
||||
|
||||
def test_aggregate_folds_every_rank_onto_output_rank():
|
||||
"""EC work done on any rank reaches the scheduler via output_rank's output.
|
||||
|
||||
The middle rank reports no worker metadata: it must neither seed nor clobber
|
||||
the accumulator.
|
||||
"""
|
||||
outputs = [
|
||||
_worker_output(
|
||||
ECConnectorOutput(
|
||||
finished_sending={"mm0"},
|
||||
ec_connector_worker_meta=FakeWorkerMeta(["mm0"]),
|
||||
)
|
||||
),
|
||||
_worker_output(ECConnectorOutput(finished_recving={"mm1"})),
|
||||
_worker_output(
|
||||
ECConnectorOutput(ec_connector_worker_meta=FakeWorkerMeta(["mm2"]))
|
||||
),
|
||||
]
|
||||
|
||||
result = ECOutputAggregator().aggregate(outputs, output_rank=2)
|
||||
|
||||
assert result is outputs[2]
|
||||
assert result.ec_connector_output.finished_sending == {"mm0"}
|
||||
assert result.ec_connector_output.finished_recving == {"mm1"}
|
||||
assert result.ec_connector_output.ec_connector_worker_meta.saves == ["mm0", "mm2"]
|
||||
|
||||
|
||||
def test_aggregate_leaves_no_ec_output_when_no_worker_reported():
|
||||
"""Empty per-worker reports must not reach the scheduler as an empty object."""
|
||||
outputs = [_worker_output(ECConnectorOutput()), _worker_output(ECConnectorOutput())]
|
||||
|
||||
result = ECOutputAggregator().aggregate(outputs, output_rank=0)
|
||||
|
||||
assert result is outputs[0]
|
||||
assert result.ec_connector_output is None
|
||||
assert ECOutputAggregator().aggregate([None], output_rank=0) is None
|
||||
|
||||
|
||||
def test_aggregate_does_not_write_through_the_shared_empty_output():
|
||||
"""A rank with nothing to report yields the shared empty output singleton.
|
||||
|
||||
Folding another rank's metadata onto it must not write through to the
|
||||
module-level object, which every later step would then carry.
|
||||
"""
|
||||
outputs = [
|
||||
_worker_output(
|
||||
ECConnectorOutput(ec_connector_worker_meta=FakeWorkerMeta(["mm0"]))
|
||||
),
|
||||
EMPTY_MODEL_RUNNER_OUTPUT,
|
||||
]
|
||||
|
||||
result = ECOutputAggregator().aggregate(outputs, output_rank=1)
|
||||
|
||||
assert EMPTY_MODEL_RUNNER_OUTPUT.ec_connector_output is None
|
||||
assert result is not EMPTY_MODEL_RUNNER_OUTPUT
|
||||
assert result.ec_connector_output.ec_connector_worker_meta.saves == ["mm0"]
|
||||
|
||||
|
||||
def test_chaining_with_kv_aggregator_preserves_both_outputs():
|
||||
"""MultiprocExecutor chains both aggregators and keeps only the last result,
|
||||
so each must merge onto the same output_rank output rather than replace it.
|
||||
"""
|
||||
outputs = [
|
||||
_worker_output(ECConnectorOutput(finished_sending={"mm0"})),
|
||||
_worker_output(None),
|
||||
]
|
||||
outputs[1].kv_connector_output = KVConnectorOutput(invalid_block_ids={7})
|
||||
|
||||
result = None
|
||||
for aggregator in (
|
||||
KVOutputAggregator(expected_finished_count=1),
|
||||
ECOutputAggregator(),
|
||||
):
|
||||
result = aggregator.aggregate(outputs, output_rank=1)
|
||||
|
||||
assert result is outputs[1]
|
||||
assert result.kv_connector_output.invalid_block_ids == {7}
|
||||
assert result.ec_connector_output.finished_sending == {"mm0"}
|
||||
@@ -0,0 +1,77 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the V2 GPU model runner's EC connector wrapper."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorBase,
|
||||
ECConnectorMetadata,
|
||||
)
|
||||
from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT
|
||||
from vllm.v1.worker.gpu.ec_connector import NO_OP_EC_CONNECTOR, ActiveECConnector
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
WORKER_META = object()
|
||||
|
||||
|
||||
def _scheduler_output() -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
ec_connector_metadata=ECConnectorMetadata(), finished_req_ids=frozenset()
|
||||
)
|
||||
|
||||
|
||||
def _connector(
|
||||
encoder_cache: dict | None = None,
|
||||
is_producer: bool = True,
|
||||
is_consumer: bool = False,
|
||||
) -> tuple[ActiveECConnector, MagicMock]:
|
||||
fake = MagicMock(spec=ECConnectorBase)
|
||||
fake.is_producer = is_producer
|
||||
fake.is_consumer = is_consumer
|
||||
fake.get_finished.return_value = (None, None)
|
||||
fake.build_connector_worker_meta.return_value = WORKER_META
|
||||
with patch("vllm.v1.worker.gpu.ec_connector.get_ec_transfer", return_value=fake):
|
||||
return ActiveECConnector(SimpleNamespace(), encoder_cache or {}), fake
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("is_producer", "is_consumer"), [(True, False), (True, True), (False, True)]
|
||||
)
|
||||
def test_saves_newly_added_caches_for_every_producer(is_producer, is_consumer):
|
||||
"""An ec_both node is also a producer: it must offload what it just computed."""
|
||||
encoder_cache = {"mm_old": None}
|
||||
connector, fake = _connector(encoder_cache, is_producer, is_consumer)
|
||||
|
||||
with connector.maybe_get_output(_scheduler_output()):
|
||||
encoder_cache["mm_new"] = None
|
||||
|
||||
saved = [call.kwargs["mm_hash"] for call in fake.save_caches.call_args_list]
|
||||
assert saved == (["mm_new"] if is_producer else [])
|
||||
assert fake.start_load_caches.called == is_consumer
|
||||
|
||||
|
||||
def test_worker_meta_is_reported_on_context_exit():
|
||||
"""Reported in the finally block, so is_empty() sees it only after the exit."""
|
||||
connector, fake = _connector()
|
||||
|
||||
with connector.maybe_get_output(_scheduler_output()) as output:
|
||||
assert output.ec_connector_worker_meta is None
|
||||
|
||||
assert output.ec_connector_worker_meta is WORKER_META
|
||||
assert fake.clear_connector_metadata.called
|
||||
|
||||
|
||||
def test_no_forward_reports_without_running_the_model():
|
||||
connector, _ = _connector()
|
||||
|
||||
output = connector.no_forward(_scheduler_output())
|
||||
|
||||
assert output.ec_connector_output.ec_connector_worker_meta is WORKER_META
|
||||
|
||||
empty = NO_OP_EC_CONNECTOR.no_forward(_scheduler_output())
|
||||
assert empty is EMPTY_MODEL_RUNNER_OUTPUT
|
||||
@@ -9,6 +9,7 @@ from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.utils import ECOutputAggregator
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
|
||||
from vllm.sampling_params import SamplingParams
|
||||
@@ -98,6 +99,7 @@ class CustomMultiprocExecutor(MultiprocExecutor):
|
||||
non_block: bool = False,
|
||||
unique_reply_rank: int | None = None,
|
||||
kv_output_aggregator: KVOutputAggregator = None,
|
||||
ec_output_aggregator: ECOutputAggregator | None = None,
|
||||
) -> Any | list[Any] | Future[Any | list[Any]]:
|
||||
# Drop marker to show that this was run
|
||||
with open(".marker", "w"):
|
||||
@@ -110,6 +112,7 @@ class CustomMultiprocExecutor(MultiprocExecutor):
|
||||
non_block,
|
||||
unique_reply_rank,
|
||||
kv_output_aggregator,
|
||||
ec_output_aggregator,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -55,6 +55,7 @@ def _run_engine_core_handshake(
|
||||
class _FakeScheduler:
|
||||
def __init__(self, **kwargs: Any) -> None:
|
||||
self.connector = connector
|
||||
self.ec_connector = None
|
||||
|
||||
def get_kv_connector(self) -> KVConnectorBase_V1:
|
||||
return connector
|
||||
|
||||
@@ -6,7 +6,13 @@ import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.outputs import LogprobsLists, LogprobsTensors
|
||||
from vllm.v1.outputs import (
|
||||
EMPTY_MODEL_RUNNER_OUTPUT,
|
||||
ECConnectorOutput,
|
||||
LogprobsLists,
|
||||
LogprobsTensors,
|
||||
ModelRunnerOutput,
|
||||
)
|
||||
from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p
|
||||
from vllm.v1.worker.gpu.sample.output import SamplingMaskTensors
|
||||
|
||||
@@ -200,3 +206,14 @@ class TestLogprobsLists(TestCase):
|
||||
assert len(sliced.logprob_token_ids) == 9 # All tokens
|
||||
assert sliced.logprob_token_ids == self.logprobsLists.logprob_token_ids
|
||||
assert sliced.cu_num_generated_tokens is None
|
||||
|
||||
|
||||
def test_with_ec_conn_output_copies_shared_empty_output():
|
||||
"""The shared empty output is copied, never written to."""
|
||||
ec_output = ECConnectorOutput(finished_sending={"mm_hash"})
|
||||
|
||||
result = ModelRunnerOutput.with_ec_conn_output(EMPTY_MODEL_RUNNER_OUTPUT, ec_output)
|
||||
|
||||
assert result is not EMPTY_MODEL_RUNNER_OUTPUT
|
||||
assert result.ec_connector_output is ec_output
|
||||
assert EMPTY_MODEL_RUNNER_OUTPUT.ec_connector_output is None
|
||||
|
||||
@@ -180,6 +180,7 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
|
||||
hidden_states=None,
|
||||
aux_hidden_states=None,
|
||||
finished_req_ids=set(),
|
||||
ec_connector_output=None,
|
||||
routed_experts=None,
|
||||
num_tokens_across_dp=None,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user