[RL] Support per-parameter source layouts for NCCL M2N (#60934)

Signed-off-by: pkousha <pkousha@nvidia.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
pkousha
2026-10-16 21:41:17 -04:00
committed by GitHub
co-authored by OpenAI Codex aoshen02
parent f8112704d5
commit 6b6a6c5c98
8 changed files with 2590 additions and 695 deletions
+25 -10
View File
@@ -17,9 +17,8 @@ Every FSDP rank builds an ``M2NTrainerWeightTransferEngine`` and calls
``send_weights()``; all ranks run every reshard, and only rank 0 drives the
inference side through its ``RayVLLMWeightSyncClient``.
Requires the ``nccl-extensions`` package (NCCL 2.30.5+) and a
``VLLM_NCCL_SO_PATH`` pointing at the same ``libnccl.so`` that
``libnccl_m2n.so`` was linked against.
Requires the ``nccl-extensions`` package (NCCL 2.30.5+). The backend creates
the data-plane communicator with ``nccl4py`` and passes it directly to M2N.
This example was written for 4xH100.
"""
@@ -28,6 +27,7 @@ from __future__ import annotations
import os
import nccl.core as nccl
import pybase64 as base64
import ray
import torch
@@ -38,7 +38,6 @@ from transformers import AutoModelForCausalLM
from vllm import LLM, SamplingParams
from vllm.config import WeightTransferConfig
from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary
from vllm.distributed.weight_transfer import (
RayVLLMWeightSyncClient,
WeightTransferTrainerFactory,
@@ -94,7 +93,15 @@ class FSDPTrainWorker:
def ready(self):
return True
def setup_engine(self, llm_handle, nccl_unique_id_b64, world_size, num_workers):
def setup_engine(
self,
llm_handle,
metadata_address,
metadata_port,
nccl_unique_id_b64,
world_size,
num_workers,
):
"""Build the trainer engine on every FSDP rank.
`DTensorModuleSource` reads each parameter's FSDP device mesh and
@@ -103,6 +110,8 @@ class FSDPTrainWorker:
"""
self.engine = WeightTransferTrainerFactory.trainer_init(
init_info=M2NTrainerInitInfo(
master_address=metadata_address,
master_port=metadata_port,
nccl_unique_id_b64=nccl_unique_id_b64,
world_size=world_size,
num_trainer_ranks=FSDP_WORLD_SIZE,
@@ -166,16 +175,22 @@ def main():
# Trainer ranks [0, FSDP_WORLD_SIZE) and inference ranks after them share
# one communicator, so the two meshes are contiguous rank intervals.
nccl = NCCLLibrary()
nccl_unique_id_b64 = base64.b64encode(
bytes(nccl.ncclGetUniqueId().internal)
).decode()
nccl_unique_id_b64 = base64.b64encode(nccl.get_unique_id().as_bytes).decode()
world_size = FSDP_WORLD_SIZE + num_workers
metadata_address = get_ip()
metadata_port = get_open_port()
print("[transfer] Initializing nccl_m2n weight transfer (all FSDP ranks)...")
ray.get(
[
w.setup_engine.remote(llm, nccl_unique_id_b64, world_size, num_workers)
w.setup_engine.remote(
llm,
metadata_address,
metadata_port,
nccl_unique_id_b64,
world_size,
num_workers,
)
for w in fsdp_workers
]
)