[Core] remove cupy dependency (#3625)
This commit is contained in:
@@ -7,7 +7,7 @@ import contextlib
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.parallel_utils import cupy_utils
|
||||
from vllm.model_executor.parallel_utils import pynccl_utils
|
||||
|
||||
# Tensor model parallel group that the current rank belongs to.
|
||||
_TENSOR_MODEL_PARALLEL_GROUP = None
|
||||
@@ -210,36 +210,36 @@ def destroy_model_parallel():
|
||||
global _PIPELINE_GLOBAL_RANKS
|
||||
_PIPELINE_GLOBAL_RANKS = None
|
||||
|
||||
# Destroy the cupy states if any.
|
||||
cupy_utils.destroy_process_group()
|
||||
# Destroy the pynccl states if any.
|
||||
pynccl_utils.destroy_process_group()
|
||||
|
||||
|
||||
# Whether to use cupy for nccl all reduce.
|
||||
# We use cupy for all reduce when using CUDA graph, because torch.distributed
|
||||
# Whether to use pynccl for nccl all reduce.
|
||||
# We use pynccl for all reduce when using CUDA graph, because torch.distributed
|
||||
# is not well supported by CUDA graph.
|
||||
_ENABLE_CUPY_FOR_ALL_REDUCE = False
|
||||
_ENABLE_PYNCCL_FOR_ALL_REDUCE = False
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def with_cupy_nccl_for_all_reduce():
|
||||
"""use CuPy nccl instead of torch.distributed for all reduce"""
|
||||
def with_pynccl_for_all_reduce():
|
||||
"""use pynccl instead of torch.distributed for all reduce"""
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
if tp_size == 1:
|
||||
# No-op.
|
||||
# NOTE(woosuk): We don't initialize CuPy when tp_size is 1.
|
||||
# NOTE(woosuk): We don't initialize pynccl when tp_size is 1.
|
||||
yield
|
||||
else:
|
||||
global _ENABLE_CUPY_FOR_ALL_REDUCE
|
||||
old = _ENABLE_CUPY_FOR_ALL_REDUCE
|
||||
_ENABLE_CUPY_FOR_ALL_REDUCE = True
|
||||
global _ENABLE_PYNCCL_FOR_ALL_REDUCE
|
||||
old = _ENABLE_PYNCCL_FOR_ALL_REDUCE
|
||||
_ENABLE_PYNCCL_FOR_ALL_REDUCE = True
|
||||
|
||||
stream = torch.cuda.current_stream()
|
||||
with cupy_utils.set_cupy_stream(stream):
|
||||
with pynccl_utils.set_pynccl_stream(stream):
|
||||
yield
|
||||
_ENABLE_CUPY_FOR_ALL_REDUCE = old
|
||||
_ENABLE_PYNCCL_FOR_ALL_REDUCE = old
|
||||
|
||||
|
||||
def is_cupy_nccl_enabled_for_all_reduce():
|
||||
"""check if CuPy nccl is enabled for all reduce"""
|
||||
global _ENABLE_CUPY_FOR_ALL_REDUCE
|
||||
return _ENABLE_CUPY_FOR_ALL_REDUCE
|
||||
def is_pynccl_enabled_for_all_reduce():
|
||||
"""check if pynccl is enabled for all reduce"""
|
||||
global _ENABLE_PYNCCL_FOR_ALL_REDUCE
|
||||
return _ENABLE_PYNCCL_FOR_ALL_REDUCE
|
||||
|
||||
Reference in New Issue
Block a user