[rl] Stateful Trainer Send: NCCL + Sparse NCCL [3/N] (#50902)

Signed-off-by: haoaaron <ahao@anyscale.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Aaron Hao
2026-08-07 00:35:59 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent 43d691ec6b
commit 21ea5b4fa1
14 changed files with 1422 additions and 783 deletions
+18 -77
View File
@@ -41,7 +41,6 @@ causes unexpected behavior.
import asyncio
import uuid
from dataclasses import asdict
import ray
import torch
@@ -50,16 +49,12 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
import vllm
from vllm import SamplingParams
from vllm.config import WeightTransferConfig
from vllm.distributed.weight_transfer.base import (
WeightTransferInitRequest,
WeightTransferUpdateRequest,
)
from vllm.distributed.weight_transfer.nccl_engine import (
NCCLTrainerSendWeightsArgs,
NCCLWeightTransferEngine,
NCCLWeightTransferInitInfo,
NCCLWeightTransferUpdateInfo,
from vllm.distributed.weight_transfer import (
ModuleSource,
RayVLLMWeightSyncClient,
WeightTransferTrainerFactory,
)
from vllm.distributed.weight_transfer.nccl_engine import NCCLTrainerInitInfo
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_ip, get_open_port
from vllm.v1.executor import Executor
@@ -141,40 +136,23 @@ class TrainModel:
self.port = get_open_port()
self.master_address = get_ip()
def get_master_address_and_port(self):
return self.master_address, self.port
def get_weight_metadata(self):
"""Return weight names, dtypes, and shapes for weight transfer."""
names = []
dtype_names = []
shapes = []
for name, p in self.model.named_parameters():
names.append(name)
dtype_names.append(str(p.dtype).split(".")[-1])
shapes.append(list(p.shape))
return names, dtype_names, shapes
def init_weight_transfer_group(self, world_size):
"""Initialize the NCCL process group for weight transfer."""
self.model_update_group = NCCLWeightTransferEngine.trainer_init(
dict(
def init_weight_transfer(self, world_size, llm_handle):
"""Build the trainer-side weight-transfer engine and rendezvous."""
self.engine = WeightTransferTrainerFactory.trainer_init(
init_info=NCCLTrainerInitInfo(
master_address=self.master_address,
master_port=self.port,
world_size=world_size,
rank=0, # single-GPU trainer is the sole (sender) rank
packed=True,
),
client=RayVLLMWeightSyncClient(llm_handle),
source=ModuleSource(self.model),
)
def broadcast_weights(self, packed: bool = True):
"""Broadcast weights to the inference engine."""
trainer_args = NCCLTrainerSendWeightsArgs(
group=self.model_update_group,
packed=packed,
)
NCCLWeightTransferEngine.trainer_send_weights(
iterator=self.model.named_parameters(),
trainer_args=trainer_args,
)
def broadcast_weights(self):
"""Push weights to the inference engine (drives start/update/finish)."""
self.engine.send_weights()
@torch.inference_mode()
def generate(self, token_ids: list[int], max_new_tokens: int) -> list[int]:
@@ -263,32 +241,12 @@ batch_prompt_token_ids = [
# Set up the communication channel between the training process and the
# inference engine.
master_address, master_port = ray.get(train_model.get_master_address_and_port.remote())
world_size = 2 # 1 trainer + 1 inference worker
inference_handle = llm.init_weight_transfer_engine.remote(
WeightTransferInitRequest(
init_info=asdict(
NCCLWeightTransferInitInfo(
master_address=master_address,
master_port=master_port,
rank_offset=1,
world_size=world_size,
)
)
)
)
# Initialize weight transfer group on both the training actor and inference engine
train_handle = train_model.init_weight_transfer_group.remote(world_size)
ray.get([train_handle, inference_handle])
ray.get(train_model.init_weight_transfer.remote(world_size, llm))
N_NEW_TOKENS = 100
# Collect weight metadata once
names, dtype_names, shapes = ray.get(train_model.get_weight_metadata.remote())
# ── Phase 1: concurrent requests with weight sync ───────────────────
print(f"\n{'=' * 50}")
print(f"Prompts ({len(PROMPTS)}):")
@@ -306,24 +264,7 @@ gen_futures = [
ray.get(llm.pause_after_n_tokens.remote())
ray.get(llm.start_weight_update.remote())
inference_handle = llm.update_weights.remote(
WeightTransferUpdateRequest(
update_info=asdict(
NCCLWeightTransferUpdateInfo(
names=names,
dtype_names=dtype_names,
shapes=shapes,
packed=True,
)
)
)
)
train_handle = train_model.broadcast_weights.remote(packed=True)
ray.get([train_handle, inference_handle])
ray.get(llm.finish_weight_update.remote())
ray.get(train_model.broadcast_weights.remote())
ray.get(llm.resume_generation.remote())
results = ray.get(gen_futures)
+19 -113
View File
@@ -38,10 +38,12 @@ import torch
from openai import OpenAI
from transformers import AutoModelForCausalLM
from vllm.distributed.weight_transfer.nccl_engine import (
NCCLTrainerSendWeightsArgs,
NCCLWeightTransferEngine,
from vllm.distributed.weight_transfer import (
HTTPVLLMWeightSyncClient,
ModuleSource,
WeightTransferTrainerFactory,
)
from vllm.distributed.weight_transfer.nccl_engine import NCCLTrainerInitInfo
from vllm.utils.network_utils import get_ip, get_open_port
BASE_URL = "http://localhost:8000"
@@ -62,62 +64,6 @@ def generate_completions(client: OpenAI, model: str, prompts: list[str]) -> list
return results
def init_weight_transfer_engine(
base_url: str,
master_address: str,
master_port: int,
rank_offset: int,
world_size: int,
) -> None:
"""Initialize weight transfer via HTTP endpoint."""
url = f"{base_url}/init_weight_transfer_engine"
payload = {
"init_info": dict(
master_address=master_address,
master_port=master_port,
rank_offset=rank_offset,
world_size=world_size,
)
}
response = requests.post(url, json=payload, timeout=60)
response.raise_for_status()
def start_weight_update(base_url: str) -> None:
"""Start a weight update via HTTP endpoint."""
url = f"{base_url}/start_weight_update"
response = requests.post(url, json={}, timeout=60)
response.raise_for_status()
def update_weights(
base_url: str,
names: list[str],
dtype_names: list[str],
shapes: list[list[int]],
packed: bool = False,
) -> None:
"""Update weights via HTTP endpoint."""
url = f"{base_url}/update_weights"
payload = {
"update_info": dict(
names=names,
dtype_names=dtype_names,
shapes=shapes,
packed=packed,
)
}
response = requests.post(url, json=payload, timeout=300)
response.raise_for_status()
def finish_weight_update(base_url: str) -> None:
"""Finish a weight update via HTTP endpoint."""
url = f"{base_url}/finish_weight_update"
response = requests.post(url, json={}, timeout=60)
response.raise_for_status()
def pause_generation(base_url: str) -> None:
"""Pause generation via HTTP endpoint."""
url = f"{base_url}/pause"
@@ -177,75 +123,35 @@ def main():
print("-" * 50)
# Set up the communication channel between the training process and the
# vLLM server. The trainer is rank 0, vLLM worker(s) start at rank_offset.
# vLLM server. The trainer is rank 0, vLLM worker(s) start after it.
master_address = get_ip()
master_port = get_open_port()
rank_offset = 1
print(f"Initializing weight transfer: master={master_address}:{master_port}")
# Initialize weight transfer on vLLM server (this is async, server will
# wait for NCCL connection)
import threading
init_thread = threading.Thread(
target=init_weight_transfer_engine,
args=(BASE_URL, master_address, master_port, rank_offset, world_size),
)
init_thread.start()
# Initialize NCCL process group on trainer side
model_update_group = NCCLWeightTransferEngine.trainer_init(
dict(
# The trainer engine owns the full handshake. `trainer_init` kicks off the
# server's init_weight_transfer_engine (via the HTTP client) on a worker
# thread while opening the trainer's NCCL endpoint, so both ends rendezvous
# together — no manual threading needed here.
engine = WeightTransferTrainerFactory.trainer_init(
init_info=NCCLTrainerInitInfo(
master_address=master_address,
master_port=master_port,
world_size=world_size,
rank=0, # single-GPU trainer is the sole (sender) rank
packed=True,
),
client=HTTPVLLMWeightSyncClient(BASE_URL),
source=ModuleSource(train_model),
)
# Wait for init_weight_transfer_engine to complete
init_thread.join()
# Pause generation before weight sync
pause_generation(BASE_URL)
# Collect weight metadata for the update request
names = []
dtype_names = []
shapes = []
for name, p in train_model.named_parameters():
names.append(name)
dtype_names.append(str(p.dtype).split(".")[-1])
shapes.append(list(p.shape))
# Start weight update
start_weight_update(BASE_URL)
# Start the update_weights call in a separate thread since it will block
# waiting for NCCL broadcasts
# packed=True enables efficient batched tensor broadcasting
update_thread = threading.Thread(
target=update_weights,
args=(BASE_URL, names, dtype_names, shapes, True), # packed=True
)
update_thread.start()
# Broadcast all weights from trainer to vLLM workers
# One call drives start_weight_update / update_weights / finish_weight_update
# over HTTP, concurrent with the NCCL broadcast.
print("Broadcasting weights via NCCL...")
trainer_args = NCCLTrainerSendWeightsArgs(
group=model_update_group,
packed=True,
)
NCCLWeightTransferEngine.trainer_send_weights(
iterator=train_model.named_parameters(),
trainer_args=trainer_args,
)
# Wait for update_weights to complete
update_thread.join()
# Finish weight update
finish_weight_update(BASE_URL)
engine.send_weights()
# Resume generation after weight sync
resume_generation(BASE_URL)
+32 -73
View File
@@ -36,10 +36,12 @@ from transformers import AutoModelForCausalLM
from vllm import LLM, SamplingParams
from vllm.config import WeightTransferConfig
from vllm.distributed.weight_transfer.nccl_engine import (
NCCLTrainerSendWeightsArgs,
NCCLWeightTransferEngine,
from vllm.distributed.weight_transfer import (
ModuleSource,
RayVLLMWeightSyncClient,
WeightTransferTrainerFactory,
)
from vllm.distributed.weight_transfer.nccl_engine import NCCLTrainerInitInfo
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_ip, get_open_port
@@ -80,40 +82,33 @@ class TrainModel:
self.port = get_open_port()
self.master_address = get_ip()
def get_master_address_and_port(self):
return self.master_address, self.port
def init_weight_transfer(self, world_size, llm_handle):
"""Build the trainer-side weight-transfer engine.
def get_weight_metadata(self):
"""Return weight names, dtypes, and shapes for weight transfer."""
names = []
dtype_names = []
shapes = []
for name, p in self.model.named_parameters():
names.append(name)
dtype_names.append(str(p.dtype).split(".")[-1])
shapes.append(list(p.shape))
return names, dtype_names, shapes
def init_weight_transfer_group(self, world_size):
"""Initialize the NCCL process group for weight transfer."""
self.model_update_group = NCCLWeightTransferEngine.trainer_init(
dict(
`trainer_init` drives the full handshake: it kicks off the inference
side's `init_weight_transfer_engine` (via the Ray client) on a worker
thread while opening the trainer's NCCL endpoint, so both ends
rendezvous together. After this returns, `send_weights()` is callable.
"""
self.engine = WeightTransferTrainerFactory.trainer_init(
init_info=NCCLTrainerInitInfo(
master_address=self.master_address,
master_port=self.port,
world_size=world_size,
rank=0, # single-GPU trainer is the sole (sender) rank
packed=True,
),
client=RayVLLMWeightSyncClient(llm_handle),
source=ModuleSource(self.model),
)
def broadcast_weights(self, packed: bool = True):
"""Broadcast weights to the inference engine."""
trainer_args = NCCLTrainerSendWeightsArgs(
group=self.model_update_group,
packed=packed,
)
NCCLWeightTransferEngine.trainer_send_weights(
iterator=self.model.named_parameters(),
trainer_args=trainer_args,
)
def broadcast_weights(self):
"""Push the current weights to the inference engine.
Drives start/update/finish on the inference side and the NCCL
broadcast internally — one call.
"""
self.engine.send_weights()
# Initialize Ray and set the visible devices. The vLLM engine will
@@ -178,51 +173,15 @@ for output in outputs:
ray.get(llm.sleep.remote(level=0))
# Set up the communication channel between the training process and the
# inference engine.
master_address, master_port = ray.get(train_model.get_master_address_and_port.remote())
# inference engine. The trainer engine now owns the full handshake — the
# driver only has to hand it the vLLM actor handle and the world size.
world_size = ray.get(llm.get_world_size.remote()) + 1 # +1 for the trainer
inference_handle = llm.init_weight_transfer_engine.remote(
dict(
init_info=dict(
master_address=master_address,
master_port=master_port,
rank_offset=1,
world_size=world_size,
)
)
)
ray.get(train_model.init_weight_transfer.remote(world_size, llm))
# Initialize weight transfer group on both the training actor and inference engine
train_handle = train_model.init_weight_transfer_group.remote(world_size)
ray.get([train_handle, inference_handle])
# Synchronize the updated weights to the inference engine using batched API.
# Collect all weight metadata from the training actor
names, dtype_names, shapes = ray.get(train_model.get_weight_metadata.remote())
# Start weight update
ray.get(llm.start_weight_update.remote())
# Issue update_weights call with NCCL-specific update info
# packed=True enables efficient batched tensor broadcasting
inference_handle = llm.update_weights.remote(
dict(
update_info=dict(
names=names,
dtype_names=dtype_names,
shapes=shapes,
packed=True,
)
)
)
# Broadcast all weights from trainer using the weight transfer API
train_handle = train_model.broadcast_weights.remote(packed=True)
ray.get([train_handle, inference_handle])
# Finish weight update
ray.get(llm.finish_weight_update.remote())
# Synchronize the updated weights to the inference engine. The engine drives
# start_weight_update / update_weights / finish_weight_update on the inference
# side internally, concurrent with the NCCL broadcast.
ray.get(train_model.broadcast_weights.remote())
ray.get(llm.wake_up.remote(tags=["scheduling"]))
+246 -220
View File
@@ -4,69 +4,79 @@
RLHF with FSDP2 training (4 GPUs) and vLLM expert-parallel inference (4 GPUs).
8-GPU layout:
Training — 4 GPUs, PyTorch FSDP2 (fully_shard)
Inference — 4 GPUs, vLLM AsyncLLMEngine with expert parallelism +
Training — 4 GPUs, PyTorch FSDP2 (fully_shard), as Ray actors
Inference — 4 GPUs, a `vllm serve` HTTP server with expert parallelism +
data parallelism (TP=1, DP=4, enable_expert_parallel
→ EP_SIZE = TP×DP = 4)
FSDP workers are Ray actors that form a single FSDP2 process group.
Rank 0 gathers full parameters via DTensor.full_tensor() and broadcasts
them to the vLLM inference engine through the NCCL weight-transfer API.
The inference side is a standalone HTTP server (spawned by this script with
`vllm serve`), so both the weight-sync control plane (HTTP) and the NCCL data
plane run inside the rank-0 FSDP Ray actor. That lets the trainer use the
unified `TrainerWeightTransferEngine.send_weights()` with an
`HTTPVLLMWeightSyncClient` — one call drives start/update/finish on the server
concurrently with the NCCL broadcast. Every FSDP rank builds an engine and calls
`send_weights()`, so all 4 participate in the incremental `full_tensor()`
all-gather; only rank 0 holds a communicator and broadcasts (it is the only
trainer rank in the NCCL group).
The inference engine uses AsyncLLMEngine which automatically spawns
DP worker processes (no manual placement group needed). Weight sync
uses pause_generation / resume_generation.
GPU split (single node): the server takes GPUs 0-3 (CUDA_VISIBLE_DEVICES), and
Ray (training) is restricted to GPUs 4-7.
Steps:
1. Launch 4 FSDP training workers.
2. Launch AsyncLLMEngine with EP+DP (dummy weights).
3. Generate from prompts → gibberish (random weights).
4. Pause generation, transfer weights from FSDP, resume.
1. Launch the vLLM HTTP server (EP+DP, dummy weights) on GPUs 0-3.
2. Launch 4 FSDP training workers (Ray) on GPUs 4-7.
3. Generate from prompts over HTTP → gibberish (random weights).
4. Pause generation, transfer weights FSDP → server over NCCL, resume.
5. Generate from prompts → sensible output (synced weights).
Assumes a single-node cluster with 8 GPUs.
"""
import asyncio
import json
import os
import uuid
from dataclasses import asdict
import subprocess
import sys
import time
import ray
import requests
import torch
import torch.distributed as dist
from huggingface_hub import snapshot_download
from openai import OpenAI
from torch.distributed.fsdp import fully_shard
from transformers import AutoModelForCausalLM
import vllm
from vllm import SamplingParams
from vllm.config import WeightTransferConfig
from vllm.distributed.weight_transfer.base import (
WeightTransferInitRequest,
WeightTransferUpdateRequest,
)
from vllm.distributed.weight_transfer.nccl_engine import (
NCCLTrainerSendWeightsArgs,
NCCLWeightTransferEngine,
NCCLWeightTransferInitInfo,
NCCLWeightTransferUpdateInfo,
from vllm.distributed.weight_transfer import (
HTTPVLLMWeightSyncClient,
ModuleSource,
WeightTransferTrainerFactory,
)
from vllm.distributed.weight_transfer.nccl_engine import NCCLTrainerInitInfo
from vllm.utils.network_utils import get_ip, get_open_port
from vllm.v1.executor import Executor
MODEL_NAME = "Qwen/Qwen3-30B-A3B"
SERVED_MODEL_NAME = "policy"
FSDP_WORLD_SIZE = 4
INFERENCE_TP_SIZE = 1
INFERENCE_DP_SIZE = 4
# Training (FSDP) GPUs are reserved through Ray; the inference server then runs
# on the complementary GPUs (see main()). We do NOT hard-code the split via
# CUDA_VISIBLE_DEVICES before ray.init(): that only restricts Ray when ray.init()
# *starts* a local cluster, and is silently ignored when it connects to an
# existing one (e.g. a shared/managed Ray cluster), causing training and the
# server to collide on the same physical GPUs.
SERVER_PORT = 8000
BASE_URL = f"http://localhost:{SERVER_PORT}"
@ray.remote(num_gpus=1)
class FSDPTrainWorker:
"""
One FSDP2 training worker per GPU. Four of these form the FSDP group.
Rank 0 additionally handles weight transfer to the vLLM engine.
Rank 0 additionally drives weight transfer to the vLLM server.
"""
def __init__(
@@ -78,6 +88,7 @@ class FSDPTrainWorker:
fsdp_master_port: int,
):
self.rank = rank
self.engine = None
os.environ["MASTER_ADDR"] = fsdp_master_addr
os.environ["MASTER_PORT"] = str(fsdp_master_port)
@@ -89,12 +100,6 @@ class FSDPTrainWorker:
model_name, torch_dtype=torch.bfloat16
)
self.weight_names = [n for n, _ in model.named_parameters()]
self.weight_dtype_names = [
str(p.dtype).split(".")[-1] for _, p in model.named_parameters()
]
self.weight_shapes = [list(p.shape) for _, p in model.named_parameters()]
for layer in model.model.layers:
fully_shard(layer)
fully_shard(model)
@@ -103,11 +108,14 @@ class FSDPTrainWorker:
self.transfer_port = None
self.transfer_master_address = None
self.model_update_group = None
def get_rank(self):
return self.rank
def get_gpu_ids(self):
"""Physical GPU id(s) Ray assigned to this worker (for server/train split)."""
return ray.get_gpu_ids()
# ---- weight-transfer setup (rank 0 only) ----
def setup_transfer_endpoint(self):
@@ -117,95 +125,130 @@ class FSDPTrainWorker:
self.transfer_master_address = get_ip()
return self.transfer_master_address, self.transfer_port
def init_weight_transfer_group(self, transfer_world_size: int):
"""Join the weight-transfer NCCL group as rank 0 (the source)."""
assert self.rank == 0
self.model_update_group = NCCLWeightTransferEngine.trainer_init(
dict(
master_address=self.transfer_master_address,
master_port=self.transfer_port,
world_size=transfer_world_size,
),
)
def setup_engine(
self,
base_url: str,
transfer_master_address: str,
transfer_port: int,
transfer_world_size: int,
):
"""Build the trainer engine on every FSDP rank.
def get_weight_metadata(self):
"""Return weight names, dtypes, and shapes captured before FSDP wrapping."""
return self.weight_names, self.weight_dtype_names, self.weight_shapes
Called on all ranks with the shared rendezvous endpoint. Rank 0 is the
sender: `trainer_init` opens its rank-0 NCCL endpoint and, on a worker
thread, calls the server's `init_weight_transfer_engine` over HTTP so
both ends rendezvous together. The other ranks skip the rendezvous and
only join the FSDP all-gather during send_weights.
"""
self.engine = WeightTransferTrainerFactory.trainer_init(
init_info=NCCLTrainerInitInfo(
master_address=transfer_master_address,
master_port=transfer_port,
world_size=transfer_world_size,
rank=self.rank, # FSDP rank; sender is rank 0
packed=True,
),
client=HTTPVLLMWeightSyncClient(base_url),
# Yields sharded DTensors; the engine reads global shape/dtype for
# metadata (no gather) and calls full_tensor() at broadcast time.
source=ModuleSource(self.model),
)
# ---- collective ops (ALL FSDP ranks must call concurrently) ----
def gather_and_broadcast_weights(self, packed: bool = True):
def gather_and_broadcast_weights(self):
"""All-gather full parameters and broadcast them to the vLLM server.
Called on all FSDP ranks. `send_weights` gathers each param via
`full_tensor()` (a collective every rank must enter in the same order);
only rank 0 (the sender) drives the server-side update_weights
concurrently with the NCCL broadcast — the other ranks only gather.
"""
All-gather full parameters and broadcast them to vLLM.
Only rank 0 performs the actual NCCL broadcast; others just
participate in the FSDP all-gather.
full_tensor() is a collective — all FSDP ranks must call it
for each parameter in the same order. Rank 0 additionally
feeds each gathered tensor to the weight-transfer engine.
"""
if self.rank == 0:
def _full_param_iter():
for name, param in self.model.named_parameters():
yield name, param.full_tensor()
trainer_args = NCCLTrainerSendWeightsArgs(
group=self.model_update_group,
packed=packed,
)
NCCLWeightTransferEngine.trainer_send_weights(
iterator=_full_param_iter(),
trainer_args=trainer_args,
)
else:
for _, param in self.model.named_parameters():
param.full_tensor()
self.engine.send_weights()
def create_async_engine(**kwargs):
"""Create an AsyncLLMEngine directly (no subclass needed)."""
engine_args = vllm.AsyncEngineArgs(**kwargs)
vllm_config = engine_args.create_engine_config()
executor_class = Executor.get_class(vllm_config)
return vllm.AsyncLLMEngine(
vllm_config=vllm_config,
executor_class=executor_class,
log_requests=engine_args.enable_log_requests,
log_stats=not engine_args.disable_log_stats,
def start_vllm_server(server_gpus: str) -> subprocess.Popen:
"""Spawn a `vllm serve` HTTP server (EP+DP) on `server_gpus` and wait for it."""
serve_args = [
"vllm",
"serve",
MODEL_NAME,
"--served-model-name",
SERVED_MODEL_NAME,
"--tensor-parallel-size",
str(INFERENCE_TP_SIZE),
"--data-parallel-size",
str(INFERENCE_DP_SIZE),
"--enable-expert-parallel",
"--enforce-eager",
"--load-format",
"dummy",
"--gpu-memory-utilization",
"0.7",
"--port",
str(SERVER_PORT),
"--weight-transfer-config",
json.dumps({"backend": "nccl"}),
]
env = os.environ.copy()
env["CUDA_VISIBLE_DEVICES"] = server_gpus
env["VLLM_SERVER_DEV_MODE"] = "1" # exposes the weight-transfer endpoints
env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
print(f"[server] Launching: {' '.join(serve_args)} (GPUs {server_gpus})")
proc = subprocess.Popen(
serve_args,
env=env,
stdout=sys.stdout,
stderr=sys.stderr,
start_new_session=True,
)
async def generate_batch(engine, prompts, sampling_params):
"""Generate completions for a batch of prompts."""
async def gen_one(prompt):
output = None
async for request_output in engine.generate(
{"prompt": prompt},
sampling_params,
request_id=str(uuid.uuid4()),
):
output = request_output
return output
return await asyncio.gather(*[gen_one(p) for p in prompts])
# Wait for the server to come up (model load can take a while).
deadline = time.monotonic() + 1800
while True:
if proc.poll() is not None:
raise RuntimeError("vLLM server exited before becoming ready.")
try:
if requests.get(f"{BASE_URL}/health", timeout=5).status_code == 200:
break
except requests.RequestException:
pass
if time.monotonic() > deadline:
raise RuntimeError("vLLM server failed to start in time.")
time.sleep(2)
print("[server] Ready.")
return proc
async def main():
ray.init()
def generate_completions(client: OpenAI, prompts: list[str]) -> list[str]:
"""Generate completions for a batch of prompts via the OpenAI HTTP API."""
results = []
for prompt in prompts:
response = client.completions.create(
model=SERVED_MODEL_NAME,
prompt=prompt,
max_tokens=32,
temperature=0,
)
results.append(response.choices[0].text)
return results
def main():
# Download model weights to local/shared disk once.
local_model_path = snapshot_download(MODEL_NAME)
print(f"[init] Model downloaded to {local_model_path}")
# FSDP rendezvous address (single-node)
ray.init()
# FSDP rendezvous address (single-node).
fsdp_master_addr = get_ip()
fsdp_master_port = get_open_port()
# Launch 4 FSDP training workers.
# Ray allocates 1 GPU per worker; AsyncLLMEngine's internal DP
# placement groups will land on the remaining 4 GPUs.
# Launch the FSDP training workers first so Ray reserves their GPUs, then
# place the inference server on the GPUs Ray did NOT use. This keeps the two
# on disjoint physical GPUs whether ray.init() started a fresh cluster or
# connected to an existing one.
fsdp_workers = [
FSDPTrainWorker.remote(
local_model_path,
@@ -219,126 +262,109 @@ async def main():
ray.get([w.get_rank.remote() for w in fsdp_workers])
print(f"[init] {FSDP_WORLD_SIZE} FSDP training workers ready.")
# Launch vLLM with expert parallelism + data parallelism.
# AsyncLLMEngine with data_parallel_backend="ray" creates its own
# placement groups internally — no manual placement group needed.
print("[engine] Creating AsyncLLMEngine...")
engine = create_async_engine(
model=local_model_path,
enforce_eager=True,
tensor_parallel_size=INFERENCE_TP_SIZE,
data_parallel_size=INFERENCE_DP_SIZE,
enable_expert_parallel=True,
distributed_executor_backend="ray",
data_parallel_backend="ray",
weight_transfer_config=WeightTransferConfig(backend="nccl"),
load_format="dummy",
gpu_memory_utilization=0.7,
)
print("[engine] AsyncLLMEngine created.")
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
# Discover the physical GPUs Ray assigned to training; run the server on the
# complementary GPUs.
training_gpus = {
int(g)
for ids in ray.get([w.get_gpu_ids.remote() for w in fsdp_workers])
for g in ids
}
num_gpus = int(ray.cluster_resources().get("GPU", 0))
num_server_gpus = INFERENCE_TP_SIZE * INFERENCE_DP_SIZE
server_gpu_ids = [g for g in range(num_gpus) if g not in training_gpus][
:num_server_gpus
]
sampling_params = SamplingParams(temperature=0)
# Generate with dummy weights — expect gibberish.
print("[generate] Starting generation with dummy weights...")
outputs = await generate_batch(engine, prompts, sampling_params)
print("[generate] Generation complete.")
print("-" * 60)
print("BEFORE weight sync (dummy weights):")
print("-" * 60)
for output in outputs:
print(f"Prompt: {output.prompt!r}")
print(f"Generated: {output.outputs[0].text!r}")
print("-" * 60)
# --- Weight-transfer setup ---
print("[transfer] Setting up weight-transfer endpoint...")
transfer_addr, transfer_port = ray.get(
fsdp_workers[0].setup_transfer_endpoint.remote()
)
print(f"[transfer] Endpoint ready at {transfer_addr}:{transfer_port}")
transfer_world_size = INFERENCE_TP_SIZE * INFERENCE_DP_SIZE + 1
print(
f"[transfer] World size: {transfer_world_size} "
f"(1 trainer + {INFERENCE_TP_SIZE * INFERENCE_DP_SIZE} vLLM workers)"
)
print("[transfer] Initializing NCCL groups...")
train_handle = fsdp_workers[0].init_weight_transfer_group.remote(
transfer_world_size
)
await engine.init_weight_transfer_engine(
WeightTransferInitRequest(
init_info=asdict(
NCCLWeightTransferInitInfo(
master_address=transfer_addr,
master_port=transfer_port,
rank_offset=1,
world_size=transfer_world_size,
)
)
if len(server_gpu_ids) < num_server_gpus:
raise RuntimeError(
f"Need {num_server_gpus} free GPUs for the inference server but only "
f"found {server_gpu_ids} (training uses {sorted(training_gpus)} of "
f"{num_gpus} cluster GPUs)."
)
)
ray.get(train_handle)
print("[transfer] NCCL groups initialized.")
server_gpus = ",".join(str(g) for g in server_gpu_ids)
print(f"[init] Training GPUs {sorted(training_gpus)}; server GPUs [{server_gpus}].")
# --- Pause, transfer weights, resume ---
print("[sync] Pausing generation...")
await engine.pause_generation(mode="abort")
print("[sync] Generation paused.")
# Start the inference server on the complementary GPUs.
server_proc = start_vllm_server(server_gpus)
try:
client = OpenAI(base_url=f"{BASE_URL}/v1", api_key="EMPTY")
names, dtype_names, shapes = ray.get(fsdp_workers[0].get_weight_metadata.remote())
print(f"[sync] Got metadata for {len(names)} parameters.")
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
print("[sync] Starting weight update...")
await engine.start_weight_update()
print("[sync] Broadcasting weights from FSDP → vLLM...")
broadcast_handles = [
w.gather_and_broadcast_weights.remote(packed=True) for w in fsdp_workers
]
await engine.update_weights(
WeightTransferUpdateRequest(
update_info=asdict(
NCCLWeightTransferUpdateInfo(
names=names,
dtype_names=dtype_names,
shapes=shapes,
packed=True,
)
)
)
)
ray.get(broadcast_handles)
await engine.finish_weight_update()
print("[sync] Weight broadcast complete.")
print("[sync] Resuming generation...")
await engine.resume_generation()
print("[sync] Generation resumed.")
# Generate with synced weights — expect sensible output.
print("[generate] Starting generation with synced weights...")
outputs_updated = await generate_batch(engine, prompts, sampling_params)
print("[generate] Generation complete.")
print("-" * 60)
print("AFTER weight sync (real weights):")
print("-" * 60)
for output in outputs_updated:
print(f"Prompt: {output.prompt!r}")
print(f"Generated: {output.outputs[0].text!r}")
# Generate with dummy weights — expect gibberish.
print("[generate] Generating with dummy weights...")
outputs = generate_completions(client, prompts)
print("-" * 60)
print("BEFORE weight sync (dummy weights):")
print("-" * 60)
for prompt, text in zip(prompts, outputs):
print(f"Prompt: {prompt!r}")
print(f"Generated: {text!r}")
print("-" * 60)
# --- Weight-transfer setup ---
print("[transfer] Setting up weight-transfer endpoint...")
transfer_addr, transfer_port = ray.get(
fsdp_workers[0].setup_transfer_endpoint.remote()
)
print(f"[transfer] Endpoint ready at {transfer_addr}:{transfer_port}")
transfer_world_size = INFERENCE_TP_SIZE * INFERENCE_DP_SIZE + 1
print(
f"[transfer] World size: {transfer_world_size} "
f"(1 trainer + {INFERENCE_TP_SIZE * INFERENCE_DP_SIZE} vLLM workers)"
)
# Build the trainer engine on all FSDP ranks (rank 0 is the sender). The
# sender drives the server's init_weight_transfer_engine (HTTP) while
# opening the trainer NCCL endpoint, so both ends rendezvous together;
# the other ranks build a null-client engine that only gathers.
print("[transfer] Initializing NCCL groups (all FSDP ranks)...")
ray.get(
[
w.setup_engine.remote(
BASE_URL, transfer_addr, transfer_port, transfer_world_size
)
for w in fsdp_workers
]
)
print("[transfer] NCCL groups initialized.")
# --- Pause, transfer weights, resume ---
print("[sync] Pausing generation...")
requests.post(f"{BASE_URL}/pause", timeout=60).raise_for_status()
# All ranks participate in the FSDP all-gather; rank 0 additionally
# drives start/update/finish on the server and the NCCL broadcast.
print("[sync] Broadcasting weights from FSDP → vLLM...")
ray.get([w.gather_and_broadcast_weights.remote() for w in fsdp_workers])
print("[sync] Weight broadcast complete.")
print("[sync] Resuming generation...")
requests.post(f"{BASE_URL}/resume", timeout=60).raise_for_status()
# Generate with synced weights — expect sensible output.
print("[generate] Generating with synced weights...")
outputs_updated = generate_completions(client, prompts)
print("-" * 60)
print("AFTER weight sync (real weights):")
print("-" * 60)
for prompt, text in zip(prompts, outputs_updated):
print(f"Prompt: {prompt!r}")
print(f"Generated: {text!r}")
print("-" * 60)
finally:
print("[server] Shutting down...")
server_proc.terminate()
try:
server_proc.wait(timeout=30)
except subprocess.TimeoutExpired:
server_proc.kill()
if __name__ == "__main__":
asyncio.run(main())
main()
+71 -115
View File
@@ -44,12 +44,14 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
from vllm import LLM, SamplingParams
from vllm.config import WeightTransferConfig
from vllm.distributed.weight_transfer.nccl_engine import (
NCCLTrainerSendWeightsArgs,
NCCLWeightTransferEngine,
from vllm.distributed.weight_transfer import (
ModuleSource,
RayVLLMWeightSyncClient,
WeightTransferTrainerFactory,
)
from vllm.distributed.weight_transfer.nccl_engine import NCCLTrainerInitInfo
from vllm.distributed.weight_transfer.sparse_nccl_engine import (
SparseNCCLWeightTransferEngine,
SparseNCCLTrainerInitInfo,
SparseWeightPatch,
)
from vllm.utils.network_utils import get_ip, get_open_port
@@ -87,7 +89,8 @@ class TrainModel:
self.model = None
self.patched_param = None
self.pending_sparse_patches: list[SparseWeightPatch] | None = None
self.model_update_group = None
self.dense_engine = None
self.sparse_engine = None
self.master_address = get_ip()
self.port = get_open_port()
self.reset_model()
@@ -112,34 +115,32 @@ class TrainModel:
self.port = get_open_port()
return self.master_address, self.port
def init_weight_transfer_group(self, world_size: int) -> None:
self.model_update_group = NCCLWeightTransferEngine.trainer_init(
dict(
def init_dense_engine(self, world_size: int, llm_handle) -> None:
"""Build the dense NCCL trainer engine"""
self.dense_engine = WeightTransferTrainerFactory.trainer_init(
init_info=NCCLTrainerInitInfo(
master_address=self.master_address,
master_port=self.port,
world_size=world_size,
)
rank=0, # single-GPU trainer is the sole (sender) rank
packed=True,
),
client=RayVLLMWeightSyncClient(llm_handle),
source=ModuleSource(self.model),
)
def get_dense_update_info(self, packed: bool = False) -> tuple[dict, int]:
names = []
dtype_names = []
shapes = []
payload_bytes = 0
for name, param in self.model.named_parameters():
names.append(name)
dtype_names.append(str(param.dtype).split(".")[-1])
shapes.append(list(param.shape))
payload_bytes += param.numel() * param.element_size()
def shutdown_engines(self) -> None:
"""Drop the NCCL communicators before the vLLM engine they point at
goes away, so the next phase rendezvouses from a clean slate."""
for attr in ("dense_engine", "sparse_engine"):
engine = getattr(self, attr)
if engine is not None:
engine.shutdown()
setattr(self, attr, None)
return (
dict(
names=names,
dtype_names=dtype_names,
shapes=shapes,
packed=packed,
),
payload_bytes,
def dense_payload_bytes(self) -> int:
return sum(
param.numel() * param.element_size() for param in self.model.parameters()
)
@torch.inference_mode()
@@ -173,7 +174,7 @@ class TrainModel:
self,
prompts: Sequence[str],
max_patch_rows: int = MAX_PATCH_ROWS,
) -> tuple[dict[str, object], list[int], str, int]:
) -> tuple[list[int], str, int]:
selected_token_ids: list[int] = []
special_ids = set(self.tokenizer.all_special_ids)
for prompt in prompts:
@@ -220,63 +221,60 @@ class TrainModel:
flat_indices = (
row_ids.unsqueeze(1).mul(hidden_size).add(column_offsets).reshape(-1)
)
flat_values = self.patched_param[row_ids].reshape(-1).contiguous()
flat_values = self.patched_param[row_ids].detach().reshape(-1).contiguous()
self.pending_sparse_patches = [
SparseWeightPatch(
name=PATCHED_PARAM_NAME,
indices=flat_indices.to(torch.int32),
values=flat_values,
full_shape=tuple(self.patched_param.shape),
)
]
patch_digest = hashlib.sha256(
self.pending_sparse_patches[0].indices.cpu().numpy().tobytes()
+ self.pending_sparse_patches[0]
.values.detach()
.float()
.cpu()
.numpy()
.tobytes()
+ self.pending_sparse_patches[0].values.float().cpu().numpy().tobytes()
).hexdigest()
sparse_payload_bytes = (
flat_indices.numel() * torch.tensor([], dtype=torch.int32).element_size()
+ flat_values.numel() * flat_values.element_size()
)
update_info = dict(
names=[PATCHED_PARAM_NAME],
dtype_names=[str(self.patched_param.dtype).split(".")[-1]],
shapes=[list(self.patched_param.shape)],
num_updates_list=[flat_indices.numel()],
)
return update_info, selected_token_ids, patch_digest, sparse_payload_bytes
return selected_token_ids, patch_digest, sparse_payload_bytes
def broadcast_weights(self, packed: bool = False) -> float:
if self.model_update_group is None:
raise RuntimeError("Weight transfer group is not initialized")
def send_dense_weights(self) -> float:
"""One call drives start/update/finish on the inference side
concurrently with the NCCL broadcast."""
if self.dense_engine is None:
raise RuntimeError("Dense engine is not initialized")
trainer_args = NCCLTrainerSendWeightsArgs(
group=self.model_update_group,
packed=packed,
)
start = time.perf_counter()
NCCLWeightTransferEngine.trainer_send_weights(
iterator=self.model.named_parameters(),
trainer_args=trainer_args,
)
self.dense_engine.send_weights()
torch.accelerator.synchronize()
return (time.perf_counter() - start) * 1000.0
def broadcast_pending_sparse_patch(self) -> float:
if self.model_update_group is None:
raise RuntimeError("Weight transfer group is not initialized")
def init_sparse_engine(self, world_size: int, llm_handle) -> None:
"""Build the sparse trainer engine"""
self.sparse_engine = WeightTransferTrainerFactory.trainer_init(
init_info=SparseNCCLTrainerInitInfo(
master_address=self.master_address,
master_port=self.port,
world_size=world_size,
rank=0, # single-GPU trainer is the sole (sender) rank
),
client=RayVLLMWeightSyncClient(llm_handle),
)
def send_pending_sparse_patch(self) -> float:
"""One call drives start/update/finish on the inference side
concurrently with the patch broadcasts. Sparse deltas differ each round,
so the patches are passed to `send_weights` rather than fixed at init."""
if self.sparse_engine is None:
raise RuntimeError("Sparse engine is not initialized")
if self.pending_sparse_patches is None:
raise RuntimeError("Sparse patch has not been prepared")
start = time.perf_counter()
SparseNCCLWeightTransferEngine.trainer_send_weights(
iter(self.pending_sparse_patches),
NCCLTrainerSendWeightsArgs(group=self.model_update_group),
)
self.sparse_engine.send_weights(self.pending_sparse_patches)
torch.accelerator.synchronize()
self.pending_sparse_patches = None
return (time.perf_counter() - start) * 1000.0
@@ -340,39 +338,17 @@ def run_dense_phase(
dense_before = collect_vllm_generations(llm)
ray.get(llm.sleep.remote(level=0))
master_address, master_port = ray.get(train_model.create_rendezvous.remote())
ray.get(train_model.create_rendezvous.remote())
world_size = ray.get(llm.get_world_size.remote()) + 1
inference_init = llm.init_weight_transfer_engine.remote(
dict(
init_info=dict(
master_address=master_address,
master_port=master_port,
rank_offset=1,
world_size=world_size,
)
)
)
trainer_init = train_model.init_weight_transfer_group.remote(world_size)
ray.get([trainer_init, inference_init])
ray.get(llm.start_weight_update.remote())
ray.get(train_model.init_dense_engine.remote(world_size, llm))
dense_update_info, dense_payload_bytes = ray.get(
train_model.get_dense_update_info.remote()
)
_, selected_token_ids, patch_digest, _ = ray.get(
dense_payload_bytes = ray.get(train_model.dense_payload_bytes.remote())
selected_token_ids, patch_digest, _ = ray.get(
train_model.prepare_sparse_patch.remote(PROMPTS)
)
inference_update = llm.update_weights.remote(
dict(update_info=dense_update_info)
)
dense_send_ms, _ = ray.get(
[
train_model.broadcast_weights.remote(packed=False),
inference_update,
]
)
ray.get(llm.finish_weight_update.remote())
# One call drives start/update/finish + the NCCL broadcast.
dense_send_ms = ray.get(train_model.send_dense_weights.remote())
ray.get(llm.wake_up.remote(tags=["scheduling"]))
dense_after = collect_vllm_generations(llm)
@@ -386,6 +362,7 @@ def run_dense_phase(
"dense_send_ms": dense_send_ms,
}
finally:
ray.get(train_model.shutdown_engines.remote())
ray.kill(llm)
@@ -399,36 +376,14 @@ def run_sparse_phase(
sparse_before = collect_vllm_generations(llm)
ray.get(llm.sleep.remote(level=0))
master_address, master_port = ray.get(train_model.create_rendezvous.remote())
ray.get(train_model.create_rendezvous.remote())
world_size = ray.get(llm.get_world_size.remote()) + 1
inference_init = llm.init_weight_transfer_engine.remote(
dict(
init_info=dict(
master_address=master_address,
master_port=master_port,
rank_offset=1,
world_size=world_size,
)
)
)
trainer_init = train_model.init_weight_transfer_group.remote(world_size)
ray.get([trainer_init, inference_init])
ray.get(llm.start_weight_update.remote())
ray.get(train_model.init_sparse_engine.remote(world_size, llm))
sparse_update_info, selected_token_ids, patch_digest, sparse_payload_bytes = (
ray.get(train_model.prepare_sparse_patch.remote(PROMPTS))
selected_token_ids, patch_digest, sparse_payload_bytes = ray.get(
train_model.prepare_sparse_patch.remote(PROMPTS)
)
inference_update = llm.update_weights.remote(
dict(update_info=sparse_update_info)
)
sparse_send_ms, _ = ray.get(
[
train_model.broadcast_pending_sparse_patch.remote(),
inference_update,
]
)
ray.get(llm.finish_weight_update.remote())
sparse_send_ms = ray.get(train_model.send_pending_sparse_patch.remote())
ray.get(llm.wake_up.remote(tags=["scheduling"]))
sparse_after = collect_vllm_generations(llm)
@@ -442,6 +397,7 @@ def run_sparse_phase(
"sparse_send_ms": sparse_send_ms,
}
finally:
ray.get(train_model.shutdown_engines.remote())
ray.kill(llm)