[rl] Stateful Trainer Send: NCCL + Sparse NCCL [3/N] (#50902)
Signed-off-by: haoaaron <ahao@anyscale.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
43d691ec6b
commit
21ea5b4fa1
@@ -41,7 +41,6 @@ causes unexpected behavior.
|
||||
|
||||
import asyncio
|
||||
import uuid
|
||||
from dataclasses import asdict
|
||||
|
||||
import ray
|
||||
import torch
|
||||
@@ -50,16 +49,12 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
import vllm
|
||||
from vllm import SamplingParams
|
||||
from vllm.config import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
WeightTransferInitRequest,
|
||||
WeightTransferUpdateRequest,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
NCCLTrainerSendWeightsArgs,
|
||||
NCCLWeightTransferEngine,
|
||||
NCCLWeightTransferInitInfo,
|
||||
NCCLWeightTransferUpdateInfo,
|
||||
from vllm.distributed.weight_transfer import (
|
||||
ModuleSource,
|
||||
RayVLLMWeightSyncClient,
|
||||
WeightTransferTrainerFactory,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_engine import NCCLTrainerInitInfo
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_ip, get_open_port
|
||||
from vllm.v1.executor import Executor
|
||||
@@ -141,40 +136,23 @@ class TrainModel:
|
||||
self.port = get_open_port()
|
||||
self.master_address = get_ip()
|
||||
|
||||
def get_master_address_and_port(self):
|
||||
return self.master_address, self.port
|
||||
|
||||
def get_weight_metadata(self):
|
||||
"""Return weight names, dtypes, and shapes for weight transfer."""
|
||||
names = []
|
||||
dtype_names = []
|
||||
shapes = []
|
||||
for name, p in self.model.named_parameters():
|
||||
names.append(name)
|
||||
dtype_names.append(str(p.dtype).split(".")[-1])
|
||||
shapes.append(list(p.shape))
|
||||
return names, dtype_names, shapes
|
||||
|
||||
def init_weight_transfer_group(self, world_size):
|
||||
"""Initialize the NCCL process group for weight transfer."""
|
||||
self.model_update_group = NCCLWeightTransferEngine.trainer_init(
|
||||
dict(
|
||||
def init_weight_transfer(self, world_size, llm_handle):
|
||||
"""Build the trainer-side weight-transfer engine and rendezvous."""
|
||||
self.engine = WeightTransferTrainerFactory.trainer_init(
|
||||
init_info=NCCLTrainerInitInfo(
|
||||
master_address=self.master_address,
|
||||
master_port=self.port,
|
||||
world_size=world_size,
|
||||
rank=0, # single-GPU trainer is the sole (sender) rank
|
||||
packed=True,
|
||||
),
|
||||
client=RayVLLMWeightSyncClient(llm_handle),
|
||||
source=ModuleSource(self.model),
|
||||
)
|
||||
|
||||
def broadcast_weights(self, packed: bool = True):
|
||||
"""Broadcast weights to the inference engine."""
|
||||
trainer_args = NCCLTrainerSendWeightsArgs(
|
||||
group=self.model_update_group,
|
||||
packed=packed,
|
||||
)
|
||||
NCCLWeightTransferEngine.trainer_send_weights(
|
||||
iterator=self.model.named_parameters(),
|
||||
trainer_args=trainer_args,
|
||||
)
|
||||
def broadcast_weights(self):
|
||||
"""Push weights to the inference engine (drives start/update/finish)."""
|
||||
self.engine.send_weights()
|
||||
|
||||
@torch.inference_mode()
|
||||
def generate(self, token_ids: list[int], max_new_tokens: int) -> list[int]:
|
||||
@@ -263,32 +241,12 @@ batch_prompt_token_ids = [
|
||||
|
||||
# Set up the communication channel between the training process and the
|
||||
# inference engine.
|
||||
master_address, master_port = ray.get(train_model.get_master_address_and_port.remote())
|
||||
|
||||
world_size = 2 # 1 trainer + 1 inference worker
|
||||
inference_handle = llm.init_weight_transfer_engine.remote(
|
||||
WeightTransferInitRequest(
|
||||
init_info=asdict(
|
||||
NCCLWeightTransferInitInfo(
|
||||
master_address=master_address,
|
||||
master_port=master_port,
|
||||
rank_offset=1,
|
||||
world_size=world_size,
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
# Initialize weight transfer group on both the training actor and inference engine
|
||||
train_handle = train_model.init_weight_transfer_group.remote(world_size)
|
||||
ray.get([train_handle, inference_handle])
|
||||
ray.get(train_model.init_weight_transfer.remote(world_size, llm))
|
||||
|
||||
|
||||
N_NEW_TOKENS = 100
|
||||
|
||||
# Collect weight metadata once
|
||||
names, dtype_names, shapes = ray.get(train_model.get_weight_metadata.remote())
|
||||
|
||||
# ── Phase 1: concurrent requests with weight sync ───────────────────
|
||||
print(f"\n{'=' * 50}")
|
||||
print(f"Prompts ({len(PROMPTS)}):")
|
||||
@@ -306,24 +264,7 @@ gen_futures = [
|
||||
|
||||
ray.get(llm.pause_after_n_tokens.remote())
|
||||
|
||||
ray.get(llm.start_weight_update.remote())
|
||||
|
||||
inference_handle = llm.update_weights.remote(
|
||||
WeightTransferUpdateRequest(
|
||||
update_info=asdict(
|
||||
NCCLWeightTransferUpdateInfo(
|
||||
names=names,
|
||||
dtype_names=dtype_names,
|
||||
shapes=shapes,
|
||||
packed=True,
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
train_handle = train_model.broadcast_weights.remote(packed=True)
|
||||
ray.get([train_handle, inference_handle])
|
||||
|
||||
ray.get(llm.finish_weight_update.remote())
|
||||
ray.get(train_model.broadcast_weights.remote())
|
||||
|
||||
ray.get(llm.resume_generation.remote())
|
||||
results = ray.get(gen_futures)
|
||||
|
||||
+19
-113
@@ -38,10 +38,12 @@ import torch
|
||||
from openai import OpenAI
|
||||
from transformers import AutoModelForCausalLM
|
||||
|
||||
from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
NCCLTrainerSendWeightsArgs,
|
||||
NCCLWeightTransferEngine,
|
||||
from vllm.distributed.weight_transfer import (
|
||||
HTTPVLLMWeightSyncClient,
|
||||
ModuleSource,
|
||||
WeightTransferTrainerFactory,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_engine import NCCLTrainerInitInfo
|
||||
from vllm.utils.network_utils import get_ip, get_open_port
|
||||
|
||||
BASE_URL = "http://localhost:8000"
|
||||
@@ -62,62 +64,6 @@ def generate_completions(client: OpenAI, model: str, prompts: list[str]) -> list
|
||||
return results
|
||||
|
||||
|
||||
def init_weight_transfer_engine(
|
||||
base_url: str,
|
||||
master_address: str,
|
||||
master_port: int,
|
||||
rank_offset: int,
|
||||
world_size: int,
|
||||
) -> None:
|
||||
"""Initialize weight transfer via HTTP endpoint."""
|
||||
url = f"{base_url}/init_weight_transfer_engine"
|
||||
payload = {
|
||||
"init_info": dict(
|
||||
master_address=master_address,
|
||||
master_port=master_port,
|
||||
rank_offset=rank_offset,
|
||||
world_size=world_size,
|
||||
)
|
||||
}
|
||||
response = requests.post(url, json=payload, timeout=60)
|
||||
response.raise_for_status()
|
||||
|
||||
|
||||
def start_weight_update(base_url: str) -> None:
|
||||
"""Start a weight update via HTTP endpoint."""
|
||||
url = f"{base_url}/start_weight_update"
|
||||
response = requests.post(url, json={}, timeout=60)
|
||||
response.raise_for_status()
|
||||
|
||||
|
||||
def update_weights(
|
||||
base_url: str,
|
||||
names: list[str],
|
||||
dtype_names: list[str],
|
||||
shapes: list[list[int]],
|
||||
packed: bool = False,
|
||||
) -> None:
|
||||
"""Update weights via HTTP endpoint."""
|
||||
url = f"{base_url}/update_weights"
|
||||
payload = {
|
||||
"update_info": dict(
|
||||
names=names,
|
||||
dtype_names=dtype_names,
|
||||
shapes=shapes,
|
||||
packed=packed,
|
||||
)
|
||||
}
|
||||
response = requests.post(url, json=payload, timeout=300)
|
||||
response.raise_for_status()
|
||||
|
||||
|
||||
def finish_weight_update(base_url: str) -> None:
|
||||
"""Finish a weight update via HTTP endpoint."""
|
||||
url = f"{base_url}/finish_weight_update"
|
||||
response = requests.post(url, json={}, timeout=60)
|
||||
response.raise_for_status()
|
||||
|
||||
|
||||
def pause_generation(base_url: str) -> None:
|
||||
"""Pause generation via HTTP endpoint."""
|
||||
url = f"{base_url}/pause"
|
||||
@@ -177,75 +123,35 @@ def main():
|
||||
print("-" * 50)
|
||||
|
||||
# Set up the communication channel between the training process and the
|
||||
# vLLM server. The trainer is rank 0, vLLM worker(s) start at rank_offset.
|
||||
# vLLM server. The trainer is rank 0, vLLM worker(s) start after it.
|
||||
master_address = get_ip()
|
||||
master_port = get_open_port()
|
||||
rank_offset = 1
|
||||
|
||||
print(f"Initializing weight transfer: master={master_address}:{master_port}")
|
||||
|
||||
# Initialize weight transfer on vLLM server (this is async, server will
|
||||
# wait for NCCL connection)
|
||||
import threading
|
||||
|
||||
init_thread = threading.Thread(
|
||||
target=init_weight_transfer_engine,
|
||||
args=(BASE_URL, master_address, master_port, rank_offset, world_size),
|
||||
)
|
||||
init_thread.start()
|
||||
|
||||
# Initialize NCCL process group on trainer side
|
||||
model_update_group = NCCLWeightTransferEngine.trainer_init(
|
||||
dict(
|
||||
# The trainer engine owns the full handshake. `trainer_init` kicks off the
|
||||
# server's init_weight_transfer_engine (via the HTTP client) on a worker
|
||||
# thread while opening the trainer's NCCL endpoint, so both ends rendezvous
|
||||
# together — no manual threading needed here.
|
||||
engine = WeightTransferTrainerFactory.trainer_init(
|
||||
init_info=NCCLTrainerInitInfo(
|
||||
master_address=master_address,
|
||||
master_port=master_port,
|
||||
world_size=world_size,
|
||||
rank=0, # single-GPU trainer is the sole (sender) rank
|
||||
packed=True,
|
||||
),
|
||||
client=HTTPVLLMWeightSyncClient(BASE_URL),
|
||||
source=ModuleSource(train_model),
|
||||
)
|
||||
|
||||
# Wait for init_weight_transfer_engine to complete
|
||||
init_thread.join()
|
||||
|
||||
# Pause generation before weight sync
|
||||
pause_generation(BASE_URL)
|
||||
|
||||
# Collect weight metadata for the update request
|
||||
names = []
|
||||
dtype_names = []
|
||||
shapes = []
|
||||
for name, p in train_model.named_parameters():
|
||||
names.append(name)
|
||||
dtype_names.append(str(p.dtype).split(".")[-1])
|
||||
shapes.append(list(p.shape))
|
||||
|
||||
# Start weight update
|
||||
start_weight_update(BASE_URL)
|
||||
|
||||
# Start the update_weights call in a separate thread since it will block
|
||||
# waiting for NCCL broadcasts
|
||||
# packed=True enables efficient batched tensor broadcasting
|
||||
update_thread = threading.Thread(
|
||||
target=update_weights,
|
||||
args=(BASE_URL, names, dtype_names, shapes, True), # packed=True
|
||||
)
|
||||
update_thread.start()
|
||||
|
||||
# Broadcast all weights from trainer to vLLM workers
|
||||
# One call drives start_weight_update / update_weights / finish_weight_update
|
||||
# over HTTP, concurrent with the NCCL broadcast.
|
||||
print("Broadcasting weights via NCCL...")
|
||||
trainer_args = NCCLTrainerSendWeightsArgs(
|
||||
group=model_update_group,
|
||||
packed=True,
|
||||
)
|
||||
NCCLWeightTransferEngine.trainer_send_weights(
|
||||
iterator=train_model.named_parameters(),
|
||||
trainer_args=trainer_args,
|
||||
)
|
||||
|
||||
# Wait for update_weights to complete
|
||||
update_thread.join()
|
||||
|
||||
# Finish weight update
|
||||
finish_weight_update(BASE_URL)
|
||||
engine.send_weights()
|
||||
|
||||
# Resume generation after weight sync
|
||||
resume_generation(BASE_URL)
|
||||
|
||||
+32
-73
@@ -36,10 +36,12 @@ from transformers import AutoModelForCausalLM
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.config import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
NCCLTrainerSendWeightsArgs,
|
||||
NCCLWeightTransferEngine,
|
||||
from vllm.distributed.weight_transfer import (
|
||||
ModuleSource,
|
||||
RayVLLMWeightSyncClient,
|
||||
WeightTransferTrainerFactory,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_engine import NCCLTrainerInitInfo
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_ip, get_open_port
|
||||
|
||||
@@ -80,40 +82,33 @@ class TrainModel:
|
||||
self.port = get_open_port()
|
||||
self.master_address = get_ip()
|
||||
|
||||
def get_master_address_and_port(self):
|
||||
return self.master_address, self.port
|
||||
def init_weight_transfer(self, world_size, llm_handle):
|
||||
"""Build the trainer-side weight-transfer engine.
|
||||
|
||||
def get_weight_metadata(self):
|
||||
"""Return weight names, dtypes, and shapes for weight transfer."""
|
||||
names = []
|
||||
dtype_names = []
|
||||
shapes = []
|
||||
for name, p in self.model.named_parameters():
|
||||
names.append(name)
|
||||
dtype_names.append(str(p.dtype).split(".")[-1])
|
||||
shapes.append(list(p.shape))
|
||||
return names, dtype_names, shapes
|
||||
|
||||
def init_weight_transfer_group(self, world_size):
|
||||
"""Initialize the NCCL process group for weight transfer."""
|
||||
self.model_update_group = NCCLWeightTransferEngine.trainer_init(
|
||||
dict(
|
||||
`trainer_init` drives the full handshake: it kicks off the inference
|
||||
side's `init_weight_transfer_engine` (via the Ray client) on a worker
|
||||
thread while opening the trainer's NCCL endpoint, so both ends
|
||||
rendezvous together. After this returns, `send_weights()` is callable.
|
||||
"""
|
||||
self.engine = WeightTransferTrainerFactory.trainer_init(
|
||||
init_info=NCCLTrainerInitInfo(
|
||||
master_address=self.master_address,
|
||||
master_port=self.port,
|
||||
world_size=world_size,
|
||||
rank=0, # single-GPU trainer is the sole (sender) rank
|
||||
packed=True,
|
||||
),
|
||||
client=RayVLLMWeightSyncClient(llm_handle),
|
||||
source=ModuleSource(self.model),
|
||||
)
|
||||
|
||||
def broadcast_weights(self, packed: bool = True):
|
||||
"""Broadcast weights to the inference engine."""
|
||||
trainer_args = NCCLTrainerSendWeightsArgs(
|
||||
group=self.model_update_group,
|
||||
packed=packed,
|
||||
)
|
||||
NCCLWeightTransferEngine.trainer_send_weights(
|
||||
iterator=self.model.named_parameters(),
|
||||
trainer_args=trainer_args,
|
||||
)
|
||||
def broadcast_weights(self):
|
||||
"""Push the current weights to the inference engine.
|
||||
|
||||
Drives start/update/finish on the inference side and the NCCL
|
||||
broadcast internally — one call.
|
||||
"""
|
||||
self.engine.send_weights()
|
||||
|
||||
|
||||
# Initialize Ray and set the visible devices. The vLLM engine will
|
||||
@@ -178,51 +173,15 @@ for output in outputs:
|
||||
ray.get(llm.sleep.remote(level=0))
|
||||
|
||||
# Set up the communication channel between the training process and the
|
||||
# inference engine.
|
||||
master_address, master_port = ray.get(train_model.get_master_address_and_port.remote())
|
||||
|
||||
# inference engine. The trainer engine now owns the full handshake — the
|
||||
# driver only has to hand it the vLLM actor handle and the world size.
|
||||
world_size = ray.get(llm.get_world_size.remote()) + 1 # +1 for the trainer
|
||||
inference_handle = llm.init_weight_transfer_engine.remote(
|
||||
dict(
|
||||
init_info=dict(
|
||||
master_address=master_address,
|
||||
master_port=master_port,
|
||||
rank_offset=1,
|
||||
world_size=world_size,
|
||||
)
|
||||
)
|
||||
)
|
||||
ray.get(train_model.init_weight_transfer.remote(world_size, llm))
|
||||
|
||||
# Initialize weight transfer group on both the training actor and inference engine
|
||||
train_handle = train_model.init_weight_transfer_group.remote(world_size)
|
||||
ray.get([train_handle, inference_handle])
|
||||
|
||||
# Synchronize the updated weights to the inference engine using batched API.
|
||||
# Collect all weight metadata from the training actor
|
||||
names, dtype_names, shapes = ray.get(train_model.get_weight_metadata.remote())
|
||||
|
||||
# Start weight update
|
||||
ray.get(llm.start_weight_update.remote())
|
||||
|
||||
# Issue update_weights call with NCCL-specific update info
|
||||
# packed=True enables efficient batched tensor broadcasting
|
||||
inference_handle = llm.update_weights.remote(
|
||||
dict(
|
||||
update_info=dict(
|
||||
names=names,
|
||||
dtype_names=dtype_names,
|
||||
shapes=shapes,
|
||||
packed=True,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
# Broadcast all weights from trainer using the weight transfer API
|
||||
train_handle = train_model.broadcast_weights.remote(packed=True)
|
||||
ray.get([train_handle, inference_handle])
|
||||
|
||||
# Finish weight update
|
||||
ray.get(llm.finish_weight_update.remote())
|
||||
# Synchronize the updated weights to the inference engine. The engine drives
|
||||
# start_weight_update / update_weights / finish_weight_update on the inference
|
||||
# side internally, concurrent with the NCCL broadcast.
|
||||
ray.get(train_model.broadcast_weights.remote())
|
||||
|
||||
ray.get(llm.wake_up.remote(tags=["scheduling"]))
|
||||
|
||||
|
||||
+246
-220
@@ -4,69 +4,79 @@
|
||||
RLHF with FSDP2 training (4 GPUs) and vLLM expert-parallel inference (4 GPUs).
|
||||
|
||||
8-GPU layout:
|
||||
Training — 4 GPUs, PyTorch FSDP2 (fully_shard)
|
||||
Inference — 4 GPUs, vLLM AsyncLLMEngine with expert parallelism +
|
||||
Training — 4 GPUs, PyTorch FSDP2 (fully_shard), as Ray actors
|
||||
Inference — 4 GPUs, a `vllm serve` HTTP server with expert parallelism +
|
||||
data parallelism (TP=1, DP=4, enable_expert_parallel
|
||||
→ EP_SIZE = TP×DP = 4)
|
||||
|
||||
FSDP workers are Ray actors that form a single FSDP2 process group.
|
||||
Rank 0 gathers full parameters via DTensor.full_tensor() and broadcasts
|
||||
them to the vLLM inference engine through the NCCL weight-transfer API.
|
||||
The inference side is a standalone HTTP server (spawned by this script with
|
||||
`vllm serve`), so both the weight-sync control plane (HTTP) and the NCCL data
|
||||
plane run inside the rank-0 FSDP Ray actor. That lets the trainer use the
|
||||
unified `TrainerWeightTransferEngine.send_weights()` with an
|
||||
`HTTPVLLMWeightSyncClient` — one call drives start/update/finish on the server
|
||||
concurrently with the NCCL broadcast. Every FSDP rank builds an engine and calls
|
||||
`send_weights()`, so all 4 participate in the incremental `full_tensor()`
|
||||
all-gather; only rank 0 holds a communicator and broadcasts (it is the only
|
||||
trainer rank in the NCCL group).
|
||||
|
||||
The inference engine uses AsyncLLMEngine which automatically spawns
|
||||
DP worker processes (no manual placement group needed). Weight sync
|
||||
uses pause_generation / resume_generation.
|
||||
GPU split (single node): the server takes GPUs 0-3 (CUDA_VISIBLE_DEVICES), and
|
||||
Ray (training) is restricted to GPUs 4-7.
|
||||
|
||||
Steps:
|
||||
1. Launch 4 FSDP training workers.
|
||||
2. Launch AsyncLLMEngine with EP+DP (dummy weights).
|
||||
3. Generate from prompts → gibberish (random weights).
|
||||
4. Pause generation, transfer weights from FSDP, resume.
|
||||
1. Launch the vLLM HTTP server (EP+DP, dummy weights) on GPUs 0-3.
|
||||
2. Launch 4 FSDP training workers (Ray) on GPUs 4-7.
|
||||
3. Generate from prompts over HTTP → gibberish (random weights).
|
||||
4. Pause generation, transfer weights FSDP → server over NCCL, resume.
|
||||
5. Generate from prompts → sensible output (synced weights).
|
||||
|
||||
Assumes a single-node cluster with 8 GPUs.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from dataclasses import asdict
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
import ray
|
||||
import requests
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from huggingface_hub import snapshot_download
|
||||
from openai import OpenAI
|
||||
from torch.distributed.fsdp import fully_shard
|
||||
from transformers import AutoModelForCausalLM
|
||||
|
||||
import vllm
|
||||
from vllm import SamplingParams
|
||||
from vllm.config import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
WeightTransferInitRequest,
|
||||
WeightTransferUpdateRequest,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
NCCLTrainerSendWeightsArgs,
|
||||
NCCLWeightTransferEngine,
|
||||
NCCLWeightTransferInitInfo,
|
||||
NCCLWeightTransferUpdateInfo,
|
||||
from vllm.distributed.weight_transfer import (
|
||||
HTTPVLLMWeightSyncClient,
|
||||
ModuleSource,
|
||||
WeightTransferTrainerFactory,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_engine import NCCLTrainerInitInfo
|
||||
from vllm.utils.network_utils import get_ip, get_open_port
|
||||
from vllm.v1.executor import Executor
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-30B-A3B"
|
||||
SERVED_MODEL_NAME = "policy"
|
||||
|
||||
FSDP_WORLD_SIZE = 4
|
||||
INFERENCE_TP_SIZE = 1
|
||||
INFERENCE_DP_SIZE = 4
|
||||
|
||||
# Training (FSDP) GPUs are reserved through Ray; the inference server then runs
|
||||
# on the complementary GPUs (see main()). We do NOT hard-code the split via
|
||||
# CUDA_VISIBLE_DEVICES before ray.init(): that only restricts Ray when ray.init()
|
||||
# *starts* a local cluster, and is silently ignored when it connects to an
|
||||
# existing one (e.g. a shared/managed Ray cluster), causing training and the
|
||||
# server to collide on the same physical GPUs.
|
||||
SERVER_PORT = 8000
|
||||
BASE_URL = f"http://localhost:{SERVER_PORT}"
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
class FSDPTrainWorker:
|
||||
"""
|
||||
One FSDP2 training worker per GPU. Four of these form the FSDP group.
|
||||
Rank 0 additionally handles weight transfer to the vLLM engine.
|
||||
Rank 0 additionally drives weight transfer to the vLLM server.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -78,6 +88,7 @@ class FSDPTrainWorker:
|
||||
fsdp_master_port: int,
|
||||
):
|
||||
self.rank = rank
|
||||
self.engine = None
|
||||
|
||||
os.environ["MASTER_ADDR"] = fsdp_master_addr
|
||||
os.environ["MASTER_PORT"] = str(fsdp_master_port)
|
||||
@@ -89,12 +100,6 @@ class FSDPTrainWorker:
|
||||
model_name, torch_dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
self.weight_names = [n for n, _ in model.named_parameters()]
|
||||
self.weight_dtype_names = [
|
||||
str(p.dtype).split(".")[-1] for _, p in model.named_parameters()
|
||||
]
|
||||
self.weight_shapes = [list(p.shape) for _, p in model.named_parameters()]
|
||||
|
||||
for layer in model.model.layers:
|
||||
fully_shard(layer)
|
||||
fully_shard(model)
|
||||
@@ -103,11 +108,14 @@ class FSDPTrainWorker:
|
||||
|
||||
self.transfer_port = None
|
||||
self.transfer_master_address = None
|
||||
self.model_update_group = None
|
||||
|
||||
def get_rank(self):
|
||||
return self.rank
|
||||
|
||||
def get_gpu_ids(self):
|
||||
"""Physical GPU id(s) Ray assigned to this worker (for server/train split)."""
|
||||
return ray.get_gpu_ids()
|
||||
|
||||
# ---- weight-transfer setup (rank 0 only) ----
|
||||
|
||||
def setup_transfer_endpoint(self):
|
||||
@@ -117,95 +125,130 @@ class FSDPTrainWorker:
|
||||
self.transfer_master_address = get_ip()
|
||||
return self.transfer_master_address, self.transfer_port
|
||||
|
||||
def init_weight_transfer_group(self, transfer_world_size: int):
|
||||
"""Join the weight-transfer NCCL group as rank 0 (the source)."""
|
||||
assert self.rank == 0
|
||||
self.model_update_group = NCCLWeightTransferEngine.trainer_init(
|
||||
dict(
|
||||
master_address=self.transfer_master_address,
|
||||
master_port=self.transfer_port,
|
||||
world_size=transfer_world_size,
|
||||
),
|
||||
)
|
||||
def setup_engine(
|
||||
self,
|
||||
base_url: str,
|
||||
transfer_master_address: str,
|
||||
transfer_port: int,
|
||||
transfer_world_size: int,
|
||||
):
|
||||
"""Build the trainer engine on every FSDP rank.
|
||||
|
||||
def get_weight_metadata(self):
|
||||
"""Return weight names, dtypes, and shapes captured before FSDP wrapping."""
|
||||
return self.weight_names, self.weight_dtype_names, self.weight_shapes
|
||||
Called on all ranks with the shared rendezvous endpoint. Rank 0 is the
|
||||
sender: `trainer_init` opens its rank-0 NCCL endpoint and, on a worker
|
||||
thread, calls the server's `init_weight_transfer_engine` over HTTP so
|
||||
both ends rendezvous together. The other ranks skip the rendezvous and
|
||||
only join the FSDP all-gather during send_weights.
|
||||
"""
|
||||
self.engine = WeightTransferTrainerFactory.trainer_init(
|
||||
init_info=NCCLTrainerInitInfo(
|
||||
master_address=transfer_master_address,
|
||||
master_port=transfer_port,
|
||||
world_size=transfer_world_size,
|
||||
rank=self.rank, # FSDP rank; sender is rank 0
|
||||
packed=True,
|
||||
),
|
||||
client=HTTPVLLMWeightSyncClient(base_url),
|
||||
# Yields sharded DTensors; the engine reads global shape/dtype for
|
||||
# metadata (no gather) and calls full_tensor() at broadcast time.
|
||||
source=ModuleSource(self.model),
|
||||
)
|
||||
|
||||
# ---- collective ops (ALL FSDP ranks must call concurrently) ----
|
||||
|
||||
def gather_and_broadcast_weights(self, packed: bool = True):
|
||||
def gather_and_broadcast_weights(self):
|
||||
"""All-gather full parameters and broadcast them to the vLLM server.
|
||||
|
||||
Called on all FSDP ranks. `send_weights` gathers each param via
|
||||
`full_tensor()` (a collective every rank must enter in the same order);
|
||||
only rank 0 (the sender) drives the server-side update_weights
|
||||
concurrently with the NCCL broadcast — the other ranks only gather.
|
||||
"""
|
||||
All-gather full parameters and broadcast them to vLLM.
|
||||
Only rank 0 performs the actual NCCL broadcast; others just
|
||||
participate in the FSDP all-gather.
|
||||
|
||||
full_tensor() is a collective — all FSDP ranks must call it
|
||||
for each parameter in the same order. Rank 0 additionally
|
||||
feeds each gathered tensor to the weight-transfer engine.
|
||||
"""
|
||||
if self.rank == 0:
|
||||
|
||||
def _full_param_iter():
|
||||
for name, param in self.model.named_parameters():
|
||||
yield name, param.full_tensor()
|
||||
|
||||
trainer_args = NCCLTrainerSendWeightsArgs(
|
||||
group=self.model_update_group,
|
||||
packed=packed,
|
||||
)
|
||||
NCCLWeightTransferEngine.trainer_send_weights(
|
||||
iterator=_full_param_iter(),
|
||||
trainer_args=trainer_args,
|
||||
)
|
||||
else:
|
||||
for _, param in self.model.named_parameters():
|
||||
param.full_tensor()
|
||||
self.engine.send_weights()
|
||||
|
||||
|
||||
def create_async_engine(**kwargs):
|
||||
"""Create an AsyncLLMEngine directly (no subclass needed)."""
|
||||
engine_args = vllm.AsyncEngineArgs(**kwargs)
|
||||
vllm_config = engine_args.create_engine_config()
|
||||
executor_class = Executor.get_class(vllm_config)
|
||||
return vllm.AsyncLLMEngine(
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
log_requests=engine_args.enable_log_requests,
|
||||
log_stats=not engine_args.disable_log_stats,
|
||||
def start_vllm_server(server_gpus: str) -> subprocess.Popen:
|
||||
"""Spawn a `vllm serve` HTTP server (EP+DP) on `server_gpus` and wait for it."""
|
||||
serve_args = [
|
||||
"vllm",
|
||||
"serve",
|
||||
MODEL_NAME,
|
||||
"--served-model-name",
|
||||
SERVED_MODEL_NAME,
|
||||
"--tensor-parallel-size",
|
||||
str(INFERENCE_TP_SIZE),
|
||||
"--data-parallel-size",
|
||||
str(INFERENCE_DP_SIZE),
|
||||
"--enable-expert-parallel",
|
||||
"--enforce-eager",
|
||||
"--load-format",
|
||||
"dummy",
|
||||
"--gpu-memory-utilization",
|
||||
"0.7",
|
||||
"--port",
|
||||
str(SERVER_PORT),
|
||||
"--weight-transfer-config",
|
||||
json.dumps({"backend": "nccl"}),
|
||||
]
|
||||
env = os.environ.copy()
|
||||
env["CUDA_VISIBLE_DEVICES"] = server_gpus
|
||||
env["VLLM_SERVER_DEV_MODE"] = "1" # exposes the weight-transfer endpoints
|
||||
env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
print(f"[server] Launching: {' '.join(serve_args)} (GPUs {server_gpus})")
|
||||
proc = subprocess.Popen(
|
||||
serve_args,
|
||||
env=env,
|
||||
stdout=sys.stdout,
|
||||
stderr=sys.stderr,
|
||||
start_new_session=True,
|
||||
)
|
||||
|
||||
|
||||
async def generate_batch(engine, prompts, sampling_params):
|
||||
"""Generate completions for a batch of prompts."""
|
||||
|
||||
async def gen_one(prompt):
|
||||
output = None
|
||||
async for request_output in engine.generate(
|
||||
{"prompt": prompt},
|
||||
sampling_params,
|
||||
request_id=str(uuid.uuid4()),
|
||||
):
|
||||
output = request_output
|
||||
return output
|
||||
|
||||
return await asyncio.gather(*[gen_one(p) for p in prompts])
|
||||
# Wait for the server to come up (model load can take a while).
|
||||
deadline = time.monotonic() + 1800
|
||||
while True:
|
||||
if proc.poll() is not None:
|
||||
raise RuntimeError("vLLM server exited before becoming ready.")
|
||||
try:
|
||||
if requests.get(f"{BASE_URL}/health", timeout=5).status_code == 200:
|
||||
break
|
||||
except requests.RequestException:
|
||||
pass
|
||||
if time.monotonic() > deadline:
|
||||
raise RuntimeError("vLLM server failed to start in time.")
|
||||
time.sleep(2)
|
||||
print("[server] Ready.")
|
||||
return proc
|
||||
|
||||
|
||||
async def main():
|
||||
ray.init()
|
||||
def generate_completions(client: OpenAI, prompts: list[str]) -> list[str]:
|
||||
"""Generate completions for a batch of prompts via the OpenAI HTTP API."""
|
||||
results = []
|
||||
for prompt in prompts:
|
||||
response = client.completions.create(
|
||||
model=SERVED_MODEL_NAME,
|
||||
prompt=prompt,
|
||||
max_tokens=32,
|
||||
temperature=0,
|
||||
)
|
||||
results.append(response.choices[0].text)
|
||||
return results
|
||||
|
||||
|
||||
def main():
|
||||
# Download model weights to local/shared disk once.
|
||||
local_model_path = snapshot_download(MODEL_NAME)
|
||||
print(f"[init] Model downloaded to {local_model_path}")
|
||||
|
||||
# FSDP rendezvous address (single-node)
|
||||
ray.init()
|
||||
|
||||
# FSDP rendezvous address (single-node).
|
||||
fsdp_master_addr = get_ip()
|
||||
fsdp_master_port = get_open_port()
|
||||
|
||||
# Launch 4 FSDP training workers.
|
||||
# Ray allocates 1 GPU per worker; AsyncLLMEngine's internal DP
|
||||
# placement groups will land on the remaining 4 GPUs.
|
||||
# Launch the FSDP training workers first so Ray reserves their GPUs, then
|
||||
# place the inference server on the GPUs Ray did NOT use. This keeps the two
|
||||
# on disjoint physical GPUs whether ray.init() started a fresh cluster or
|
||||
# connected to an existing one.
|
||||
fsdp_workers = [
|
||||
FSDPTrainWorker.remote(
|
||||
local_model_path,
|
||||
@@ -219,126 +262,109 @@ async def main():
|
||||
ray.get([w.get_rank.remote() for w in fsdp_workers])
|
||||
print(f"[init] {FSDP_WORLD_SIZE} FSDP training workers ready.")
|
||||
|
||||
# Launch vLLM with expert parallelism + data parallelism.
|
||||
# AsyncLLMEngine with data_parallel_backend="ray" creates its own
|
||||
# placement groups internally — no manual placement group needed.
|
||||
print("[engine] Creating AsyncLLMEngine...")
|
||||
engine = create_async_engine(
|
||||
model=local_model_path,
|
||||
enforce_eager=True,
|
||||
tensor_parallel_size=INFERENCE_TP_SIZE,
|
||||
data_parallel_size=INFERENCE_DP_SIZE,
|
||||
enable_expert_parallel=True,
|
||||
distributed_executor_backend="ray",
|
||||
data_parallel_backend="ray",
|
||||
weight_transfer_config=WeightTransferConfig(backend="nccl"),
|
||||
load_format="dummy",
|
||||
gpu_memory_utilization=0.7,
|
||||
)
|
||||
print("[engine] AsyncLLMEngine created.")
|
||||
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
# Discover the physical GPUs Ray assigned to training; run the server on the
|
||||
# complementary GPUs.
|
||||
training_gpus = {
|
||||
int(g)
|
||||
for ids in ray.get([w.get_gpu_ids.remote() for w in fsdp_workers])
|
||||
for g in ids
|
||||
}
|
||||
num_gpus = int(ray.cluster_resources().get("GPU", 0))
|
||||
num_server_gpus = INFERENCE_TP_SIZE * INFERENCE_DP_SIZE
|
||||
server_gpu_ids = [g for g in range(num_gpus) if g not in training_gpus][
|
||||
:num_server_gpus
|
||||
]
|
||||
sampling_params = SamplingParams(temperature=0)
|
||||
|
||||
# Generate with dummy weights — expect gibberish.
|
||||
print("[generate] Starting generation with dummy weights...")
|
||||
outputs = await generate_batch(engine, prompts, sampling_params)
|
||||
print("[generate] Generation complete.")
|
||||
|
||||
print("-" * 60)
|
||||
print("BEFORE weight sync (dummy weights):")
|
||||
print("-" * 60)
|
||||
for output in outputs:
|
||||
print(f"Prompt: {output.prompt!r}")
|
||||
print(f"Generated: {output.outputs[0].text!r}")
|
||||
print("-" * 60)
|
||||
|
||||
# --- Weight-transfer setup ---
|
||||
print("[transfer] Setting up weight-transfer endpoint...")
|
||||
transfer_addr, transfer_port = ray.get(
|
||||
fsdp_workers[0].setup_transfer_endpoint.remote()
|
||||
)
|
||||
print(f"[transfer] Endpoint ready at {transfer_addr}:{transfer_port}")
|
||||
|
||||
transfer_world_size = INFERENCE_TP_SIZE * INFERENCE_DP_SIZE + 1
|
||||
print(
|
||||
f"[transfer] World size: {transfer_world_size} "
|
||||
f"(1 trainer + {INFERENCE_TP_SIZE * INFERENCE_DP_SIZE} vLLM workers)"
|
||||
)
|
||||
|
||||
print("[transfer] Initializing NCCL groups...")
|
||||
train_handle = fsdp_workers[0].init_weight_transfer_group.remote(
|
||||
transfer_world_size
|
||||
)
|
||||
await engine.init_weight_transfer_engine(
|
||||
WeightTransferInitRequest(
|
||||
init_info=asdict(
|
||||
NCCLWeightTransferInitInfo(
|
||||
master_address=transfer_addr,
|
||||
master_port=transfer_port,
|
||||
rank_offset=1,
|
||||
world_size=transfer_world_size,
|
||||
)
|
||||
)
|
||||
if len(server_gpu_ids) < num_server_gpus:
|
||||
raise RuntimeError(
|
||||
f"Need {num_server_gpus} free GPUs for the inference server but only "
|
||||
f"found {server_gpu_ids} (training uses {sorted(training_gpus)} of "
|
||||
f"{num_gpus} cluster GPUs)."
|
||||
)
|
||||
)
|
||||
ray.get(train_handle)
|
||||
print("[transfer] NCCL groups initialized.")
|
||||
server_gpus = ",".join(str(g) for g in server_gpu_ids)
|
||||
print(f"[init] Training GPUs {sorted(training_gpus)}; server GPUs [{server_gpus}].")
|
||||
|
||||
# --- Pause, transfer weights, resume ---
|
||||
print("[sync] Pausing generation...")
|
||||
await engine.pause_generation(mode="abort")
|
||||
print("[sync] Generation paused.")
|
||||
# Start the inference server on the complementary GPUs.
|
||||
server_proc = start_vllm_server(server_gpus)
|
||||
try:
|
||||
client = OpenAI(base_url=f"{BASE_URL}/v1", api_key="EMPTY")
|
||||
|
||||
names, dtype_names, shapes = ray.get(fsdp_workers[0].get_weight_metadata.remote())
|
||||
print(f"[sync] Got metadata for {len(names)} parameters.")
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
|
||||
print("[sync] Starting weight update...")
|
||||
await engine.start_weight_update()
|
||||
|
||||
print("[sync] Broadcasting weights from FSDP → vLLM...")
|
||||
broadcast_handles = [
|
||||
w.gather_and_broadcast_weights.remote(packed=True) for w in fsdp_workers
|
||||
]
|
||||
await engine.update_weights(
|
||||
WeightTransferUpdateRequest(
|
||||
update_info=asdict(
|
||||
NCCLWeightTransferUpdateInfo(
|
||||
names=names,
|
||||
dtype_names=dtype_names,
|
||||
shapes=shapes,
|
||||
packed=True,
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
ray.get(broadcast_handles)
|
||||
|
||||
await engine.finish_weight_update()
|
||||
print("[sync] Weight broadcast complete.")
|
||||
|
||||
print("[sync] Resuming generation...")
|
||||
await engine.resume_generation()
|
||||
print("[sync] Generation resumed.")
|
||||
|
||||
# Generate with synced weights — expect sensible output.
|
||||
print("[generate] Starting generation with synced weights...")
|
||||
outputs_updated = await generate_batch(engine, prompts, sampling_params)
|
||||
print("[generate] Generation complete.")
|
||||
|
||||
print("-" * 60)
|
||||
print("AFTER weight sync (real weights):")
|
||||
print("-" * 60)
|
||||
for output in outputs_updated:
|
||||
print(f"Prompt: {output.prompt!r}")
|
||||
print(f"Generated: {output.outputs[0].text!r}")
|
||||
# Generate with dummy weights — expect gibberish.
|
||||
print("[generate] Generating with dummy weights...")
|
||||
outputs = generate_completions(client, prompts)
|
||||
print("-" * 60)
|
||||
print("BEFORE weight sync (dummy weights):")
|
||||
print("-" * 60)
|
||||
for prompt, text in zip(prompts, outputs):
|
||||
print(f"Prompt: {prompt!r}")
|
||||
print(f"Generated: {text!r}")
|
||||
print("-" * 60)
|
||||
|
||||
# --- Weight-transfer setup ---
|
||||
print("[transfer] Setting up weight-transfer endpoint...")
|
||||
transfer_addr, transfer_port = ray.get(
|
||||
fsdp_workers[0].setup_transfer_endpoint.remote()
|
||||
)
|
||||
print(f"[transfer] Endpoint ready at {transfer_addr}:{transfer_port}")
|
||||
|
||||
transfer_world_size = INFERENCE_TP_SIZE * INFERENCE_DP_SIZE + 1
|
||||
print(
|
||||
f"[transfer] World size: {transfer_world_size} "
|
||||
f"(1 trainer + {INFERENCE_TP_SIZE * INFERENCE_DP_SIZE} vLLM workers)"
|
||||
)
|
||||
|
||||
# Build the trainer engine on all FSDP ranks (rank 0 is the sender). The
|
||||
# sender drives the server's init_weight_transfer_engine (HTTP) while
|
||||
# opening the trainer NCCL endpoint, so both ends rendezvous together;
|
||||
# the other ranks build a null-client engine that only gathers.
|
||||
print("[transfer] Initializing NCCL groups (all FSDP ranks)...")
|
||||
ray.get(
|
||||
[
|
||||
w.setup_engine.remote(
|
||||
BASE_URL, transfer_addr, transfer_port, transfer_world_size
|
||||
)
|
||||
for w in fsdp_workers
|
||||
]
|
||||
)
|
||||
print("[transfer] NCCL groups initialized.")
|
||||
|
||||
# --- Pause, transfer weights, resume ---
|
||||
print("[sync] Pausing generation...")
|
||||
requests.post(f"{BASE_URL}/pause", timeout=60).raise_for_status()
|
||||
|
||||
# All ranks participate in the FSDP all-gather; rank 0 additionally
|
||||
# drives start/update/finish on the server and the NCCL broadcast.
|
||||
print("[sync] Broadcasting weights from FSDP → vLLM...")
|
||||
ray.get([w.gather_and_broadcast_weights.remote() for w in fsdp_workers])
|
||||
print("[sync] Weight broadcast complete.")
|
||||
|
||||
print("[sync] Resuming generation...")
|
||||
requests.post(f"{BASE_URL}/resume", timeout=60).raise_for_status()
|
||||
|
||||
# Generate with synced weights — expect sensible output.
|
||||
print("[generate] Generating with synced weights...")
|
||||
outputs_updated = generate_completions(client, prompts)
|
||||
print("-" * 60)
|
||||
print("AFTER weight sync (real weights):")
|
||||
print("-" * 60)
|
||||
for prompt, text in zip(prompts, outputs_updated):
|
||||
print(f"Prompt: {prompt!r}")
|
||||
print(f"Generated: {text!r}")
|
||||
print("-" * 60)
|
||||
finally:
|
||||
print("[server] Shutting down...")
|
||||
server_proc.terminate()
|
||||
try:
|
||||
server_proc.wait(timeout=30)
|
||||
except subprocess.TimeoutExpired:
|
||||
server_proc.kill()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
main()
|
||||
|
||||
+71
-115
@@ -44,12 +44,14 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.config import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
NCCLTrainerSendWeightsArgs,
|
||||
NCCLWeightTransferEngine,
|
||||
from vllm.distributed.weight_transfer import (
|
||||
ModuleSource,
|
||||
RayVLLMWeightSyncClient,
|
||||
WeightTransferTrainerFactory,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_engine import NCCLTrainerInitInfo
|
||||
from vllm.distributed.weight_transfer.sparse_nccl_engine import (
|
||||
SparseNCCLWeightTransferEngine,
|
||||
SparseNCCLTrainerInitInfo,
|
||||
SparseWeightPatch,
|
||||
)
|
||||
from vllm.utils.network_utils import get_ip, get_open_port
|
||||
@@ -87,7 +89,8 @@ class TrainModel:
|
||||
self.model = None
|
||||
self.patched_param = None
|
||||
self.pending_sparse_patches: list[SparseWeightPatch] | None = None
|
||||
self.model_update_group = None
|
||||
self.dense_engine = None
|
||||
self.sparse_engine = None
|
||||
self.master_address = get_ip()
|
||||
self.port = get_open_port()
|
||||
self.reset_model()
|
||||
@@ -112,34 +115,32 @@ class TrainModel:
|
||||
self.port = get_open_port()
|
||||
return self.master_address, self.port
|
||||
|
||||
def init_weight_transfer_group(self, world_size: int) -> None:
|
||||
self.model_update_group = NCCLWeightTransferEngine.trainer_init(
|
||||
dict(
|
||||
def init_dense_engine(self, world_size: int, llm_handle) -> None:
|
||||
"""Build the dense NCCL trainer engine"""
|
||||
self.dense_engine = WeightTransferTrainerFactory.trainer_init(
|
||||
init_info=NCCLTrainerInitInfo(
|
||||
master_address=self.master_address,
|
||||
master_port=self.port,
|
||||
world_size=world_size,
|
||||
)
|
||||
rank=0, # single-GPU trainer is the sole (sender) rank
|
||||
packed=True,
|
||||
),
|
||||
client=RayVLLMWeightSyncClient(llm_handle),
|
||||
source=ModuleSource(self.model),
|
||||
)
|
||||
|
||||
def get_dense_update_info(self, packed: bool = False) -> tuple[dict, int]:
|
||||
names = []
|
||||
dtype_names = []
|
||||
shapes = []
|
||||
payload_bytes = 0
|
||||
for name, param in self.model.named_parameters():
|
||||
names.append(name)
|
||||
dtype_names.append(str(param.dtype).split(".")[-1])
|
||||
shapes.append(list(param.shape))
|
||||
payload_bytes += param.numel() * param.element_size()
|
||||
def shutdown_engines(self) -> None:
|
||||
"""Drop the NCCL communicators before the vLLM engine they point at
|
||||
goes away, so the next phase rendezvouses from a clean slate."""
|
||||
for attr in ("dense_engine", "sparse_engine"):
|
||||
engine = getattr(self, attr)
|
||||
if engine is not None:
|
||||
engine.shutdown()
|
||||
setattr(self, attr, None)
|
||||
|
||||
return (
|
||||
dict(
|
||||
names=names,
|
||||
dtype_names=dtype_names,
|
||||
shapes=shapes,
|
||||
packed=packed,
|
||||
),
|
||||
payload_bytes,
|
||||
def dense_payload_bytes(self) -> int:
|
||||
return sum(
|
||||
param.numel() * param.element_size() for param in self.model.parameters()
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
@@ -173,7 +174,7 @@ class TrainModel:
|
||||
self,
|
||||
prompts: Sequence[str],
|
||||
max_patch_rows: int = MAX_PATCH_ROWS,
|
||||
) -> tuple[dict[str, object], list[int], str, int]:
|
||||
) -> tuple[list[int], str, int]:
|
||||
selected_token_ids: list[int] = []
|
||||
special_ids = set(self.tokenizer.all_special_ids)
|
||||
for prompt in prompts:
|
||||
@@ -220,63 +221,60 @@ class TrainModel:
|
||||
flat_indices = (
|
||||
row_ids.unsqueeze(1).mul(hidden_size).add(column_offsets).reshape(-1)
|
||||
)
|
||||
flat_values = self.patched_param[row_ids].reshape(-1).contiguous()
|
||||
flat_values = self.patched_param[row_ids].detach().reshape(-1).contiguous()
|
||||
self.pending_sparse_patches = [
|
||||
SparseWeightPatch(
|
||||
name=PATCHED_PARAM_NAME,
|
||||
indices=flat_indices.to(torch.int32),
|
||||
values=flat_values,
|
||||
full_shape=tuple(self.patched_param.shape),
|
||||
)
|
||||
]
|
||||
patch_digest = hashlib.sha256(
|
||||
self.pending_sparse_patches[0].indices.cpu().numpy().tobytes()
|
||||
+ self.pending_sparse_patches[0]
|
||||
.values.detach()
|
||||
.float()
|
||||
.cpu()
|
||||
.numpy()
|
||||
.tobytes()
|
||||
+ self.pending_sparse_patches[0].values.float().cpu().numpy().tobytes()
|
||||
).hexdigest()
|
||||
|
||||
sparse_payload_bytes = (
|
||||
flat_indices.numel() * torch.tensor([], dtype=torch.int32).element_size()
|
||||
+ flat_values.numel() * flat_values.element_size()
|
||||
)
|
||||
update_info = dict(
|
||||
names=[PATCHED_PARAM_NAME],
|
||||
dtype_names=[str(self.patched_param.dtype).split(".")[-1]],
|
||||
shapes=[list(self.patched_param.shape)],
|
||||
num_updates_list=[flat_indices.numel()],
|
||||
)
|
||||
return update_info, selected_token_ids, patch_digest, sparse_payload_bytes
|
||||
return selected_token_ids, patch_digest, sparse_payload_bytes
|
||||
|
||||
def broadcast_weights(self, packed: bool = False) -> float:
|
||||
if self.model_update_group is None:
|
||||
raise RuntimeError("Weight transfer group is not initialized")
|
||||
def send_dense_weights(self) -> float:
|
||||
"""One call drives start/update/finish on the inference side
|
||||
concurrently with the NCCL broadcast."""
|
||||
if self.dense_engine is None:
|
||||
raise RuntimeError("Dense engine is not initialized")
|
||||
|
||||
trainer_args = NCCLTrainerSendWeightsArgs(
|
||||
group=self.model_update_group,
|
||||
packed=packed,
|
||||
)
|
||||
start = time.perf_counter()
|
||||
NCCLWeightTransferEngine.trainer_send_weights(
|
||||
iterator=self.model.named_parameters(),
|
||||
trainer_args=trainer_args,
|
||||
)
|
||||
self.dense_engine.send_weights()
|
||||
torch.accelerator.synchronize()
|
||||
return (time.perf_counter() - start) * 1000.0
|
||||
|
||||
def broadcast_pending_sparse_patch(self) -> float:
|
||||
if self.model_update_group is None:
|
||||
raise RuntimeError("Weight transfer group is not initialized")
|
||||
def init_sparse_engine(self, world_size: int, llm_handle) -> None:
|
||||
"""Build the sparse trainer engine"""
|
||||
self.sparse_engine = WeightTransferTrainerFactory.trainer_init(
|
||||
init_info=SparseNCCLTrainerInitInfo(
|
||||
master_address=self.master_address,
|
||||
master_port=self.port,
|
||||
world_size=world_size,
|
||||
rank=0, # single-GPU trainer is the sole (sender) rank
|
||||
),
|
||||
client=RayVLLMWeightSyncClient(llm_handle),
|
||||
)
|
||||
|
||||
def send_pending_sparse_patch(self) -> float:
|
||||
"""One call drives start/update/finish on the inference side
|
||||
concurrently with the patch broadcasts. Sparse deltas differ each round,
|
||||
so the patches are passed to `send_weights` rather than fixed at init."""
|
||||
if self.sparse_engine is None:
|
||||
raise RuntimeError("Sparse engine is not initialized")
|
||||
if self.pending_sparse_patches is None:
|
||||
raise RuntimeError("Sparse patch has not been prepared")
|
||||
|
||||
start = time.perf_counter()
|
||||
SparseNCCLWeightTransferEngine.trainer_send_weights(
|
||||
iter(self.pending_sparse_patches),
|
||||
NCCLTrainerSendWeightsArgs(group=self.model_update_group),
|
||||
)
|
||||
self.sparse_engine.send_weights(self.pending_sparse_patches)
|
||||
torch.accelerator.synchronize()
|
||||
self.pending_sparse_patches = None
|
||||
return (time.perf_counter() - start) * 1000.0
|
||||
@@ -340,39 +338,17 @@ def run_dense_phase(
|
||||
dense_before = collect_vllm_generations(llm)
|
||||
|
||||
ray.get(llm.sleep.remote(level=0))
|
||||
master_address, master_port = ray.get(train_model.create_rendezvous.remote())
|
||||
ray.get(train_model.create_rendezvous.remote())
|
||||
world_size = ray.get(llm.get_world_size.remote()) + 1
|
||||
inference_init = llm.init_weight_transfer_engine.remote(
|
||||
dict(
|
||||
init_info=dict(
|
||||
master_address=master_address,
|
||||
master_port=master_port,
|
||||
rank_offset=1,
|
||||
world_size=world_size,
|
||||
)
|
||||
)
|
||||
)
|
||||
trainer_init = train_model.init_weight_transfer_group.remote(world_size)
|
||||
ray.get([trainer_init, inference_init])
|
||||
ray.get(llm.start_weight_update.remote())
|
||||
ray.get(train_model.init_dense_engine.remote(world_size, llm))
|
||||
|
||||
dense_update_info, dense_payload_bytes = ray.get(
|
||||
train_model.get_dense_update_info.remote()
|
||||
)
|
||||
_, selected_token_ids, patch_digest, _ = ray.get(
|
||||
dense_payload_bytes = ray.get(train_model.dense_payload_bytes.remote())
|
||||
selected_token_ids, patch_digest, _ = ray.get(
|
||||
train_model.prepare_sparse_patch.remote(PROMPTS)
|
||||
)
|
||||
|
||||
inference_update = llm.update_weights.remote(
|
||||
dict(update_info=dense_update_info)
|
||||
)
|
||||
dense_send_ms, _ = ray.get(
|
||||
[
|
||||
train_model.broadcast_weights.remote(packed=False),
|
||||
inference_update,
|
||||
]
|
||||
)
|
||||
ray.get(llm.finish_weight_update.remote())
|
||||
# One call drives start/update/finish + the NCCL broadcast.
|
||||
dense_send_ms = ray.get(train_model.send_dense_weights.remote())
|
||||
ray.get(llm.wake_up.remote(tags=["scheduling"]))
|
||||
|
||||
dense_after = collect_vllm_generations(llm)
|
||||
@@ -386,6 +362,7 @@ def run_dense_phase(
|
||||
"dense_send_ms": dense_send_ms,
|
||||
}
|
||||
finally:
|
||||
ray.get(train_model.shutdown_engines.remote())
|
||||
ray.kill(llm)
|
||||
|
||||
|
||||
@@ -399,36 +376,14 @@ def run_sparse_phase(
|
||||
sparse_before = collect_vllm_generations(llm)
|
||||
|
||||
ray.get(llm.sleep.remote(level=0))
|
||||
master_address, master_port = ray.get(train_model.create_rendezvous.remote())
|
||||
ray.get(train_model.create_rendezvous.remote())
|
||||
world_size = ray.get(llm.get_world_size.remote()) + 1
|
||||
inference_init = llm.init_weight_transfer_engine.remote(
|
||||
dict(
|
||||
init_info=dict(
|
||||
master_address=master_address,
|
||||
master_port=master_port,
|
||||
rank_offset=1,
|
||||
world_size=world_size,
|
||||
)
|
||||
)
|
||||
)
|
||||
trainer_init = train_model.init_weight_transfer_group.remote(world_size)
|
||||
ray.get([trainer_init, inference_init])
|
||||
ray.get(llm.start_weight_update.remote())
|
||||
ray.get(train_model.init_sparse_engine.remote(world_size, llm))
|
||||
|
||||
sparse_update_info, selected_token_ids, patch_digest, sparse_payload_bytes = (
|
||||
ray.get(train_model.prepare_sparse_patch.remote(PROMPTS))
|
||||
selected_token_ids, patch_digest, sparse_payload_bytes = ray.get(
|
||||
train_model.prepare_sparse_patch.remote(PROMPTS)
|
||||
)
|
||||
|
||||
inference_update = llm.update_weights.remote(
|
||||
dict(update_info=sparse_update_info)
|
||||
)
|
||||
sparse_send_ms, _ = ray.get(
|
||||
[
|
||||
train_model.broadcast_pending_sparse_patch.remote(),
|
||||
inference_update,
|
||||
]
|
||||
)
|
||||
ray.get(llm.finish_weight_update.remote())
|
||||
sparse_send_ms = ray.get(train_model.send_pending_sparse_patch.remote())
|
||||
ray.get(llm.wake_up.remote(tags=["scheduling"]))
|
||||
|
||||
sparse_after = collect_vllm_generations(llm)
|
||||
@@ -442,6 +397,7 @@ def run_sparse_phase(
|
||||
"sparse_send_ms": sparse_send_ms,
|
||||
}
|
||||
finally:
|
||||
ray.get(train_model.shutdown_engines.remote())
|
||||
ray.kill(llm)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user