diff --git a/examples/rl/rlhf_nccl.py b/examples/rl/rlhf_nccl.py index b94d5e4db8..a9e39aaa72 100644 --- a/examples/rl/rlhf_nccl.py +++ b/examples/rl/rlhf_nccl.py @@ -29,6 +29,7 @@ causes unexpected behavior. import os import ray +import torch from ray.util.placement_group import placement_group from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy from transformers import AutoModelForCausalLM @@ -39,12 +40,24 @@ from vllm.distributed.weight_transfer.nccl_engine import ( NCCLTrainerSendWeightsArgs, NCCLWeightTransferEngine, ) +from vllm.platforms import current_platform from vllm.utils.network_utils import get_ip, get_open_port MODEL_NAME = "facebook/opt-125m" # MODEL_NAME = "inference-optimization/Qwen3-0.6B-W4A16-G128" +def get_assigned_gpu(): + """This is a temporary workaround for a runtime bug in RCCL on ROCm.""" + if not current_platform.is_rocm(): + return 0 + assigned_gpu = int(ray.get_gpu_ids()[0]) + os.environ.pop("CUDA_VISIBLE_DEVICES", None) + os.environ.pop("HIP_VISIBLE_DEVICES", None) + torch.accelerator.set_device_idx(assigned_gpu) + return assigned_gpu + + class MyLLM(LLM): """Configure the vLLM worker for Ray placement group execution.""" @@ -58,9 +71,11 @@ class TrainModel: """Ray actor that wraps the training model on a dedicated GPU.""" def __init__(self, model_name: str): + assigned_gpu = get_assigned_gpu() + self.model = AutoModelForCausalLM.from_pretrained( model_name, - ).to("cuda:0") + ).to(f"cuda:{assigned_gpu}") self.port = get_open_port() self.master_address = get_ip()