[PerfFix] Avoid separate thread for MP executor shm spin (take 2) (#28319)

Signed-off-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
Nick Hill
2025-11-07 22:11:03 +00:00
committed by GitHub
parent da786e339e
commit 67a2da890e
9 changed files with 156 additions and 131 deletions
@@ -4,6 +4,7 @@
KV cache helper for store.
"""
import contextlib
from collections.abc import Sequence
from concurrent.futures import CancelledError, Future
from typing import TYPE_CHECKING, Literal
@@ -221,38 +222,38 @@ class KVOutputAggregator:
def async_aggregate(
self,
output_futures: Sequence[Future[ModelRunnerOutput | None]],
output_future: Future[Sequence[ModelRunnerOutput | None]],
output_rank: int = 0,
) -> Future[ModelRunnerOutput | None]:
"""Takes a list of futures and returns a single future which resolves
to the respective list of outputs."""
"""Takes a future that resolves to a list of outputs and returns a future
which resolves to a single aggregated output."""
result_future: Future[ModelRunnerOutput | None] = Future()
outputs: list[ModelRunnerOutput | None] = [None] * len(output_futures)
remaining = len(output_futures)
def callback(fut):
if result_future.done():
return
try:
result_future.set_result(self.aggregate(fut.result(), output_rank))
except CancelledError:
result_future.cancel()
except Exception as e:
result_future.set_exception(e)
def make_callback(idx):
def callback(fut):
if result_future.done():
return
output_future.add_done_callback(callback)
try:
outputs[idx] = fut.result()
except CancelledError:
result_future.cancel()
except Exception as e:
result_future.set_exception(e)
from vllm.v1.executor.multiproc_executor import FutureWrapper
# this check assumes io_thread_pool uses a single thread
nonlocal remaining
remaining -= 1
if not remaining:
result_future.set_result(self.aggregate(outputs, output_rank))
if isinstance(output_future, FutureWrapper):
# Due to the threadless implementation of multiproc FutureWrapper,
# we must block on the delegate future's result() method.
delegate_result = result_future.result
return callback
def result(timeout=None):
with contextlib.suppress(Exception):
output_future.result(timeout=timeout)
return delegate_result()
for i, output_future in enumerate(output_futures):
output_future.add_done_callback(make_callback(i))
result_future.result = result # type: ignore[method-assign]
return result_future