[Core][Doc] Default to multiprocessing for single-node distributed case (#5230)
Co-authored-by: Antoni Baum <antoni.baum@protonmail.com>
This commit is contained in:
+17
-1
@@ -603,9 +603,25 @@ class ParallelConfig:
|
||||
f"'{self.distributed_executor_backend}'.")
|
||||
|
||||
if self.distributed_executor_backend is None and self.world_size > 1:
|
||||
# We use multiprocessing by default if world_size fits on the
|
||||
# current node and we aren't in a ray placement group.
|
||||
from torch.cuda import device_count
|
||||
|
||||
from vllm.executor import ray_utils
|
||||
backend = "mp"
|
||||
ray_found = ray_utils.ray is not None
|
||||
self.distributed_executor_backend = "ray" if ray_found else "mp"
|
||||
if device_count() < self.world_size:
|
||||
if not ray_found:
|
||||
raise ValueError("Unable to load Ray which is "
|
||||
"required for multi-node inference")
|
||||
backend = "ray"
|
||||
elif ray_found:
|
||||
from ray.util import get_current_placement_group
|
||||
if self.placement_group or get_current_placement_group():
|
||||
backend = "ray"
|
||||
self.distributed_executor_backend = backend
|
||||
logger.info("Defaulting to use %s for distributed inference",
|
||||
backend)
|
||||
|
||||
self._verify_args()
|
||||
|
||||
|
||||
@@ -19,10 +19,6 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
|
||||
"""Python multiprocessing-based multi-GPU executor"""
|
||||
|
||||
def _init_executor(self) -> None:
|
||||
assert (
|
||||
not self.speculative_config
|
||||
), "Speculative decoding not yet supported for MultiProcGPU backend."
|
||||
|
||||
# Create the parallel GPU workers.
|
||||
world_size = self.parallel_config.tensor_parallel_size
|
||||
|
||||
|
||||
@@ -65,10 +65,11 @@ def _set_future_result(future: Union[ResultFuture, asyncio.Future],
|
||||
future.set_result(result)
|
||||
return
|
||||
loop = future.get_loop()
|
||||
if result.exception is not None:
|
||||
loop.call_soon_threadsafe(future.set_exception, result.exception)
|
||||
else:
|
||||
loop.call_soon_threadsafe(future.set_result, result.value)
|
||||
if not loop.is_closed():
|
||||
if result.exception is not None:
|
||||
loop.call_soon_threadsafe(future.set_exception, result.exception)
|
||||
else:
|
||||
loop.call_soon_threadsafe(future.set_result, result.value)
|
||||
|
||||
|
||||
class ResultHandler(threading.Thread):
|
||||
|
||||
Reference in New Issue
Block a user