[Core] remove cupy dependency (#3625)
This commit is contained in:
+16
-13
@@ -15,11 +15,11 @@ from vllm.lora.request import LoRARequest
|
||||
from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
|
||||
from vllm.model_executor import SamplingMetadata
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.model_executor.parallel_utils import cupy_utils, custom_all_reduce
|
||||
from vllm.model_executor.parallel_utils import custom_all_reduce, pynccl_utils
|
||||
from vllm.model_executor.parallel_utils.communication_op import (
|
||||
broadcast_tensor_dict)
|
||||
from vllm.model_executor.parallel_utils.parallel_state import (
|
||||
with_cupy_nccl_for_all_reduce)
|
||||
with_pynccl_for_all_reduce)
|
||||
from vllm.sampling_params import SamplingParams, SamplingType
|
||||
from vllm.sequence import (MultiModalData, SamplerOutput, SequenceData,
|
||||
SequenceGroupMetadata)
|
||||
@@ -764,7 +764,7 @@ class ModelRunner:
|
||||
"""
|
||||
# NOTE(woosuk): This is a hack to ensure that the NCCL backend is never
|
||||
# deleted before the CUDA graphs.
|
||||
self.cupy_nccl_backend = cupy_utils.get_nccl_backend()
|
||||
self.pynccl_backend = pynccl_utils.get_nccl_backend()
|
||||
|
||||
assert not self.model_config.enforce_eager
|
||||
logger.info("Capturing the model for CUDA graphs. This may lead to "
|
||||
@@ -794,11 +794,11 @@ class ModelRunner:
|
||||
]
|
||||
|
||||
# NOTE(woosuk): There are 3 backends for all-reduce: custom all-reduce
|
||||
# kernel, CuPy NCCL, and PyTorch NCCL. When using CUDA graph, we use
|
||||
# either custom all-reduce kernel or CuPy NCCL. When not using CUDA
|
||||
# kernel, pynccl, and PyTorch NCCL. When using CUDA graph, we use
|
||||
# either custom all-reduce kernel or pynccl. When not using CUDA
|
||||
# graph, we use either custom all-reduce kernel or PyTorch NCCL.
|
||||
# We always prioritize using custom all-reduce kernel but fall back
|
||||
# to PyTorch or CuPy NCCL if it is disabled or not supported.
|
||||
# to PyTorch or pynccl if it is disabled or not supported.
|
||||
with custom_all_reduce.capture():
|
||||
# NOTE: Capturing the largest batch size first may help reduce the
|
||||
# memory usage of CUDA graph.
|
||||
@@ -846,12 +846,14 @@ class ModelRunner:
|
||||
logger.info(f"Graph capturing finished in {elapsed_time:.0f} secs.")
|
||||
|
||||
def __del__(self) -> None:
|
||||
# Delete the CUDA graphs before deleting the CuPy NCCL communicator.
|
||||
# Delete the CUDA graphs before deleting the pynccl communicator.
|
||||
# NOTE(woosuk): This is necessary because otherwise deadlocks can
|
||||
# happen.
|
||||
# FIXME(woosuk): This is a bit hacky. Find a more robust solution.
|
||||
# TODO(youkaichao): when we get enough user feedback that pynccl is
|
||||
# more stable than cupy, we can remove this, e.g. in v0.4.1.
|
||||
self.graph_runners.clear()
|
||||
self.cupy_nccl_backend = None
|
||||
self.pynccl_backend = None
|
||||
|
||||
@property
|
||||
def vocab_size(self) -> int:
|
||||
@@ -879,7 +881,7 @@ class CUDAGraphRunner:
|
||||
# Run the model once without capturing the graph.
|
||||
# This is to make sure that the captured graph does not include the
|
||||
# kernel launches for initial benchmarking (e.g., Triton autotune).
|
||||
with _maybe_cupy_nccl():
|
||||
with _maybe_pynccl():
|
||||
self.model(
|
||||
input_ids,
|
||||
positions,
|
||||
@@ -894,7 +896,7 @@ class CUDAGraphRunner:
|
||||
# https://stackoverflow.com/questions/31039022/python-multi-line-with-statement
|
||||
self.graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(self.graph, pool=memory_pool): # noqa: SIM117
|
||||
with _maybe_cupy_nccl():
|
||||
with _maybe_pynccl():
|
||||
hidden_states = self.model(
|
||||
input_ids,
|
||||
positions,
|
||||
@@ -947,9 +949,10 @@ class CUDAGraphRunner:
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _maybe_cupy_nccl():
|
||||
if cupy_utils.is_initialized() and not custom_all_reduce.is_initialized():
|
||||
with with_cupy_nccl_for_all_reduce():
|
||||
def _maybe_pynccl():
|
||||
if pynccl_utils.is_initialized(
|
||||
) and not custom_all_reduce.is_initialized():
|
||||
with with_pynccl_for_all_reduce():
|
||||
yield
|
||||
else:
|
||||
yield
|
||||
|
||||
+14
-16
@@ -10,7 +10,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
|
||||
ParallelConfig, SchedulerConfig, VisionLanguageConfig)
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.model_executor import set_random_seed
|
||||
from vllm.model_executor.parallel_utils import cupy_utils
|
||||
from vllm.model_executor.parallel_utils import pynccl_utils
|
||||
from vllm.model_executor.parallel_utils.communication_op import (
|
||||
broadcast_tensor_dict)
|
||||
from vllm.model_executor.parallel_utils.custom_all_reduce import init_custom_ar
|
||||
@@ -75,7 +75,7 @@ class Worker:
|
||||
self.cache_engine = None
|
||||
self.gpu_cache = None
|
||||
|
||||
def init_device(self, cupy_port: Optional[int] = None) -> None:
|
||||
def init_device(self) -> None:
|
||||
if self.device_config.device.type == "cuda":
|
||||
# torch.distributed.all_reduce does not free the input tensor until
|
||||
# the synchronization point. This causes the memory usage to grow
|
||||
@@ -98,7 +98,7 @@ class Worker:
|
||||
f"Not support device type: {self.device_config.device}")
|
||||
# Initialize the distributed environment.
|
||||
init_distributed_environment(self.parallel_config, self.rank,
|
||||
cupy_port, self.distributed_init_method)
|
||||
self.distributed_init_method)
|
||||
# Set random seed.
|
||||
set_random_seed(self.model_config.seed)
|
||||
|
||||
@@ -250,7 +250,6 @@ class Worker:
|
||||
def init_distributed_environment(
|
||||
parallel_config: ParallelConfig,
|
||||
rank: int,
|
||||
cupy_port: Optional[int],
|
||||
distributed_init_method: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Initialize the distributed environment."""
|
||||
@@ -273,28 +272,27 @@ def init_distributed_environment(
|
||||
init_method=distributed_init_method,
|
||||
)
|
||||
|
||||
if cupy_utils.is_initialized():
|
||||
cupy_world_size = cupy_utils.get_world_size()
|
||||
if cupy_world_size != parallel_config.world_size:
|
||||
if pynccl_utils.is_initialized():
|
||||
pynccl_world_size = pynccl_utils.get_world_size()
|
||||
if pynccl_world_size != parallel_config.world_size:
|
||||
raise RuntimeError(
|
||||
"cupy.distributed is already initialized but the cupy world "
|
||||
"pynccl is already initialized but the pynccl world "
|
||||
"size does not match parallel_config.world_size "
|
||||
f"({cupy_world_size} vs. {parallel_config.world_size}).")
|
||||
elif (parallel_config.world_size > 1 and cupy_port is not None):
|
||||
# NOTE(woosuk): We don't initialize CuPy process group when world size
|
||||
f"({pynccl_world_size} vs. {parallel_config.world_size}).")
|
||||
elif parallel_config.world_size > 1:
|
||||
# NOTE(woosuk): We don't initialize pynccl process group when world size
|
||||
# is 1.
|
||||
# TODO(woosuk): Support multi-node connection.
|
||||
cupy_utils.init_process_group(
|
||||
pynccl_utils.init_process_group(
|
||||
world_size=parallel_config.world_size,
|
||||
rank=rank,
|
||||
host="localhost",
|
||||
port=cupy_port,
|
||||
init_method=distributed_init_method,
|
||||
)
|
||||
|
||||
# A small all_reduce for warmup.
|
||||
torch.distributed.all_reduce(torch.zeros(1).cuda())
|
||||
if cupy_utils.is_initialized():
|
||||
cupy_utils.all_reduce(torch.zeros(1).cuda())
|
||||
if pynccl_utils.is_initialized():
|
||||
pynccl_utils.all_reduce(torch.zeros(1).cuda())
|
||||
ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
|
||||
parallel_config.pipeline_parallel_size)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user