[PerfFix] Avoid separate thread for MP executor shm spin (take 2) (#28319)
Signed-off-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
@@ -4,6 +4,7 @@
|
||||
KV cache helper for store.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
from collections.abc import Sequence
|
||||
from concurrent.futures import CancelledError, Future
|
||||
from typing import TYPE_CHECKING, Literal
|
||||
@@ -221,38 +222,38 @@ class KVOutputAggregator:
|
||||
|
||||
def async_aggregate(
|
||||
self,
|
||||
output_futures: Sequence[Future[ModelRunnerOutput | None]],
|
||||
output_future: Future[Sequence[ModelRunnerOutput | None]],
|
||||
output_rank: int = 0,
|
||||
) -> Future[ModelRunnerOutput | None]:
|
||||
"""Takes a list of futures and returns a single future which resolves
|
||||
to the respective list of outputs."""
|
||||
"""Takes a future that resolves to a list of outputs and returns a future
|
||||
which resolves to a single aggregated output."""
|
||||
result_future: Future[ModelRunnerOutput | None] = Future()
|
||||
|
||||
outputs: list[ModelRunnerOutput | None] = [None] * len(output_futures)
|
||||
remaining = len(output_futures)
|
||||
def callback(fut):
|
||||
if result_future.done():
|
||||
return
|
||||
try:
|
||||
result_future.set_result(self.aggregate(fut.result(), output_rank))
|
||||
except CancelledError:
|
||||
result_future.cancel()
|
||||
except Exception as e:
|
||||
result_future.set_exception(e)
|
||||
|
||||
def make_callback(idx):
|
||||
def callback(fut):
|
||||
if result_future.done():
|
||||
return
|
||||
output_future.add_done_callback(callback)
|
||||
|
||||
try:
|
||||
outputs[idx] = fut.result()
|
||||
except CancelledError:
|
||||
result_future.cancel()
|
||||
except Exception as e:
|
||||
result_future.set_exception(e)
|
||||
from vllm.v1.executor.multiproc_executor import FutureWrapper
|
||||
|
||||
# this check assumes io_thread_pool uses a single thread
|
||||
nonlocal remaining
|
||||
remaining -= 1
|
||||
if not remaining:
|
||||
result_future.set_result(self.aggregate(outputs, output_rank))
|
||||
if isinstance(output_future, FutureWrapper):
|
||||
# Due to the threadless implementation of multiproc FutureWrapper,
|
||||
# we must block on the delegate future's result() method.
|
||||
delegate_result = result_future.result
|
||||
|
||||
return callback
|
||||
def result(timeout=None):
|
||||
with contextlib.suppress(Exception):
|
||||
output_future.result(timeout=timeout)
|
||||
return delegate_result()
|
||||
|
||||
for i, output_future in enumerate(output_futures):
|
||||
output_future.add_done_callback(make_callback(i))
|
||||
result_future.result = result # type: ignore[method-assign]
|
||||
|
||||
return result_future
|
||||
|
||||
|
||||
Reference in New Issue
Block a user