Use NCCL instead of ray for control-plane communication to remove serialization overhead (#2221)

This commit is contained in:
Zhuohan Li
2024-01-03 11:30:22 -08:00
committed by GitHub
parent 1066cbd152
commit fd4ea8ef5c
34 changed files with 524 additions and 262 deletions
+22 -15
View File
@@ -1,16 +1,15 @@
from typing import Optional, Tuple, TYPE_CHECKING
from typing import Optional, List, Tuple, TYPE_CHECKING
from vllm.config import ParallelConfig
from vllm.logger import init_logger
from vllm.utils import get_open_port, is_hip
from vllm.utils import is_hip, set_cuda_visible_devices, get_ip
logger = init_logger(__name__)
try:
import ray
from ray.air.util.torch_dist import TorchDistributedWorker
class RayWorkerVllm(TorchDistributedWorker):
class RayWorkerVllm:
"""Ray wrapper for vllm.worker.Worker, allowing Worker to be
lazliy initialized after Ray sets CUDA_VISIBLE_DEVICES."""
@@ -30,12 +29,22 @@ try:
executor = getattr(self, method)
return executor(*args, **kwargs)
def get_node_ip(self) -> str:
return get_ip()
def get_node_and_gpu_ids(self) -> Tuple[str, List[int]]:
node_id = ray.get_runtime_context().get_node_id()
gpu_ids = ray.get_gpu_ids()
return node_id, gpu_ids
def set_cuda_visible_devices(self, device_ids) -> None:
set_cuda_visible_devices(device_ids)
except ImportError as e:
logger.warning(f"Failed to import Ray with {e!r}. "
"For distributed inference, please install Ray with "
"`pip install ray pandas pyarrow`.")
ray = None
TorchDistributedWorker = None
RayWorkerVllm = None
if TYPE_CHECKING:
@@ -75,13 +84,11 @@ def initialize_cluster(
ray.init(address=ray_address, ignore_reinit_error=True)
if not parallel_config.worker_use_ray:
# Initialize cluster locally.
port = get_open_port()
# We need to setup the distributed init method to make sure
# the distributed megatron code (e.g., get world size) works correctly.
distributed_init_method = f"tcp://localhost:{port}"
return distributed_init_method, None
assert parallel_config.world_size == 1, (
"Ray is required if parallel_config.world_size > 1.")
return None
# Create placement group for worker processes
current_placement_group = ray.util.get_current_placement_group()
if current_placement_group:
# We are in a placement group
@@ -106,12 +113,12 @@ def initialize_cluster(
"The number of required GPUs exceeds the total number of "
"available GPUs in the cluster.")
# Create a new placement group
current_placement_group = ray.util.placement_group([{
"GPU": 1
}] * parallel_config.world_size)
placement_group_specs = ([{"GPU": 1}] * parallel_config.world_size)
current_placement_group = ray.util.placement_group(
placement_group_specs)
# Wait until PG is ready - this will block until all
# requested resources are available, and will timeout
# if they cannot be provisioned.
ray.get(current_placement_group.ready(), timeout=1800)
return None, current_placement_group
return current_placement_group