[Core][Distributed] refactor pynccl (#4591)
[Core][Distributed] refactor pynccl to hold multiple communicators (#4591)
This commit is contained in:
@@ -1,4 +1,3 @@
|
||||
import contextlib
|
||||
import time
|
||||
from enum import IntEnum
|
||||
from typing import Dict, List, NamedTuple, Optional, Set, Tuple
|
||||
@@ -12,9 +11,9 @@ from vllm.attention import (AttentionMetadata, AttentionMetadataPerStage,
|
||||
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
|
||||
ModelConfig, ParallelConfig, SchedulerConfig,
|
||||
VisionLanguageConfig)
|
||||
from vllm.distributed import broadcast_tensor_dict, with_pynccl_for_all_reduce
|
||||
from vllm.distributed.device_communicators import (custom_all_reduce,
|
||||
pynccl_utils)
|
||||
from vllm.distributed import broadcast_tensor_dict
|
||||
from vllm.distributed.communication_op import graph_capture_mode
|
||||
from vllm.distributed.device_communicators import custom_all_reduce
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.layers import LoRAMapping
|
||||
from vllm.lora.request import LoRARequest
|
||||
@@ -917,10 +916,6 @@ class ModelRunner:
|
||||
Since it is used for decoding-only, it assumes there's only 1 token
|
||||
per sequence in the batch.
|
||||
"""
|
||||
# NOTE(woosuk): This is a hack to ensure that the NCCL backend is never
|
||||
# deleted before the CUDA graphs.
|
||||
self.pynccl_backend = pynccl_utils.get_nccl_backend()
|
||||
|
||||
assert not self.model_config.enforce_eager
|
||||
logger.info("Capturing the model for CUDA graphs. This may lead to "
|
||||
"unexpected consequences if the model is not static. To "
|
||||
@@ -1046,7 +1041,7 @@ class CUDAGraphRunner:
|
||||
# Run the model once without capturing the graph.
|
||||
# This is to make sure that the captured graph does not include the
|
||||
# kernel launches for initial benchmarking (e.g., Triton autotune).
|
||||
with _maybe_pynccl():
|
||||
with graph_capture_mode():
|
||||
self.model(
|
||||
input_ids,
|
||||
positions,
|
||||
@@ -1061,7 +1056,7 @@ class CUDAGraphRunner:
|
||||
# https://stackoverflow.com/questions/31039022/python-multi-line-with-statement
|
||||
self._graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(self._graph, pool=memory_pool): # noqa: SIM117
|
||||
with _maybe_pynccl():
|
||||
with graph_capture_mode():
|
||||
hidden_states = self.model(
|
||||
input_ids,
|
||||
positions,
|
||||
@@ -1113,16 +1108,6 @@ class CUDAGraphRunner:
|
||||
return self.forward(*args, **kwargs)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _maybe_pynccl():
|
||||
if pynccl_utils.is_initialized(
|
||||
) and not custom_all_reduce.is_initialized():
|
||||
with with_pynccl_for_all_reduce():
|
||||
yield
|
||||
else:
|
||||
yield
|
||||
|
||||
|
||||
def _get_graph_batch_size(batch_size: int) -> int:
|
||||
"""Returns the padded batch size given actual batch size.
|
||||
|
||||
|
||||
@@ -11,9 +11,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
|
||||
VisionLanguageConfig)
|
||||
from vllm.distributed import (broadcast_tensor_dict,
|
||||
ensure_model_parallel_initialized,
|
||||
get_tensor_model_parallel_cpu_group,
|
||||
init_distributed_environment)
|
||||
from vllm.distributed.device_communicators import pynccl_utils
|
||||
from vllm.distributed.device_communicators.custom_all_reduce import (
|
||||
init_custom_ar)
|
||||
from vllm.lora.request import LoRARequest
|
||||
@@ -306,29 +304,10 @@ def init_worker_distributed_environment(
|
||||
ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
|
||||
parallel_config.pipeline_parallel_size)
|
||||
|
||||
if pynccl_utils.is_initialized():
|
||||
pynccl_world_size = pynccl_utils.get_world_size()
|
||||
if pynccl_world_size != parallel_config.world_size:
|
||||
raise RuntimeError(
|
||||
"pynccl is already initialized but the pynccl world "
|
||||
"size does not match parallel_config.world_size "
|
||||
f"({pynccl_world_size} vs. {parallel_config.world_size}).")
|
||||
elif parallel_config.world_size > 1:
|
||||
# NOTE(woosuk): We don't initialize pynccl process group when world size
|
||||
# is 1.
|
||||
# NOTE(kaichao): By default, pynccl is initialized for tp group.
|
||||
pynccl_utils.init_process_group(
|
||||
group=get_tensor_model_parallel_cpu_group())
|
||||
|
||||
# Initialize a custom fast all-reduce implementation.
|
||||
if not parallel_config.disable_custom_all_reduce:
|
||||
init_custom_ar()
|
||||
|
||||
# A small all_reduce for warmup.
|
||||
torch.distributed.all_reduce(torch.zeros(1).cuda())
|
||||
if pynccl_utils.is_initialized():
|
||||
pynccl_utils.all_reduce(torch.zeros(1).cuda())
|
||||
|
||||
|
||||
def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
|
||||
# Check if the GPU supports the dtype.
|
||||
|
||||
Reference in New Issue
Block a user