[Core] Simplify async KV output aggregation (#28327)
Signed-off-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
@@ -9,6 +9,7 @@ from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
@@ -28,12 +29,19 @@ class CustomMultiprocExecutor(MultiprocExecutor):
|
||||
kwargs: dict | None = None,
|
||||
non_block: bool = False,
|
||||
unique_reply_rank: int | None = None,
|
||||
kv_output_aggregator: KVOutputAggregator = None,
|
||||
) -> Any | list[Any] | Future[Any | list[Any]]:
|
||||
# Drop marker to show that this was run
|
||||
with open(".marker", "w"):
|
||||
...
|
||||
return super().collective_rpc(
|
||||
method, timeout, args, kwargs, non_block, unique_reply_rank
|
||||
method,
|
||||
timeout,
|
||||
args,
|
||||
kwargs,
|
||||
non_block,
|
||||
unique_reply_rank,
|
||||
kv_output_aggregator,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from concurrent.futures import Future
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -86,74 +85,6 @@ def test_aggregate_workers_output():
|
||||
assert aggregated.invalid_block_ids == {3, 4, 5}
|
||||
|
||||
|
||||
def test_async_aggregate_workers_output():
|
||||
aggregator = KVOutputAggregator(expected_finished_count=2)
|
||||
|
||||
future: Future[list[DummyModelRunnerOutput]] = Future()
|
||||
result_future = aggregator.async_aggregate(future)
|
||||
|
||||
output1 = DummyModelRunnerOutput()
|
||||
output2 = DummyModelRunnerOutput()
|
||||
future.set_result([output1, output2])
|
||||
|
||||
assert result_future.done()
|
||||
aggregated = result_future.result()
|
||||
assert aggregated is output1
|
||||
aggregated = aggregated.kv_connector_output
|
||||
assert aggregated.finished_sending is None
|
||||
assert aggregated.finished_recving is None
|
||||
assert not aggregated.invalid_block_ids
|
||||
|
||||
future = Future()
|
||||
result_future = aggregator.async_aggregate(future)
|
||||
|
||||
output1 = DummyModelRunnerOutput(
|
||||
finished_sending={"req1"}, finished_recving={"req2"}
|
||||
)
|
||||
output2 = DummyModelRunnerOutput(invalid_block_ids={1})
|
||||
future.set_result([output1, output2])
|
||||
|
||||
assert result_future.done()
|
||||
aggregated = result_future.result()
|
||||
assert aggregated is output1
|
||||
aggregated = aggregated.kv_connector_output
|
||||
assert aggregated.finished_sending is None
|
||||
assert aggregated.finished_recving is None
|
||||
assert aggregated.invalid_block_ids == {1}
|
||||
|
||||
future = Future()
|
||||
result_future = aggregator.async_aggregate(future)
|
||||
|
||||
output1 = DummyModelRunnerOutput(invalid_block_ids={2})
|
||||
output2 = DummyModelRunnerOutput(finished_sending={"req1"})
|
||||
future.set_result([output1, output2])
|
||||
|
||||
assert result_future.done()
|
||||
aggregated = result_future.result()
|
||||
assert aggregated is output1
|
||||
aggregated = aggregated.kv_connector_output
|
||||
assert aggregated.finished_sending == {"req1"}
|
||||
assert aggregated.finished_recving is None
|
||||
assert aggregated.invalid_block_ids == {2}
|
||||
|
||||
future = Future()
|
||||
result_future = aggregator.async_aggregate(future)
|
||||
|
||||
output1 = DummyModelRunnerOutput(invalid_block_ids={3, 4})
|
||||
output2 = DummyModelRunnerOutput(
|
||||
finished_recving={"req2"}, invalid_block_ids={4, 5}
|
||||
)
|
||||
future.set_result([output1, output2])
|
||||
|
||||
assert result_future.done()
|
||||
aggregated = result_future.result()
|
||||
assert aggregated is output1
|
||||
aggregated = aggregated.kv_connector_output
|
||||
assert aggregated.finished_sending is None
|
||||
assert aggregated.finished_recving == {"req2"}
|
||||
assert aggregated.invalid_block_ids == {3, 4, 5}
|
||||
|
||||
|
||||
def test_aggregate_workers_output_with_expected_finished_count():
|
||||
# We create the aggregator expecting to collect from 4 workers
|
||||
aggregator = KVOutputAggregator(expected_finished_count=4)
|
||||
|
||||
Reference in New Issue
Block a user