[rl] Stateful Trainer Send: IPC [2/N] (#48981)

Signed-off-by: haoaaron <ahao@anyscale.com>
This commit is contained in:
Aaron Hao
2026-07-30 21:59:08 +08:00
committed by GitHub
parent 59e831c09a
commit 30b4e7f479
8 changed files with 391 additions and 421 deletions
+14 -37
View File
@@ -46,10 +46,12 @@ import torch
from openai import OpenAI
from transformers import AutoModelForCausalLM
from vllm.distributed.weight_transfer.ipc_engine import (
IPCTrainerSendWeightsArgs,
IPCWeightTransferEngine,
from vllm.distributed.weight_transfer import (
HTTPVLLMWeightSyncClient,
ModuleSource,
WeightTransferTrainerFactory,
)
from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo
BASE_URL = "http://localhost:8000"
MODEL_NAME = "facebook/opt-125m"
@@ -72,28 +74,6 @@ def generate_completions(client: OpenAI, model: str, prompts: list[str]) -> list
return results
def init_weight_transfer_engine(base_url: str) -> None:
"""Initialize weight transfer via HTTP endpoint (no-op for IPC)."""
url = f"{base_url}/init_weight_transfer_engine"
payload = {"init_info": dict()}
response = requests.post(url, json=payload, timeout=60)
response.raise_for_status()
def start_weight_update(base_url: str) -> None:
"""Start a weight update via HTTP endpoint."""
url = f"{base_url}/start_weight_update"
response = requests.post(url, json={}, timeout=60)
response.raise_for_status()
def finish_weight_update(base_url: str) -> None:
"""Finish a weight update via HTTP endpoint."""
url = f"{base_url}/finish_weight_update"
response = requests.post(url, json={}, timeout=60)
response.raise_for_status()
def pause_generation(base_url: str) -> None:
"""Pause generation via HTTP endpoint."""
url = f"{base_url}/pause"
@@ -159,23 +139,20 @@ def main():
print("Initializing weight transfer (IPC backend)...")
# Initialize weight transfer on vLLM server (no-op for IPC, but still required)
init_weight_transfer_engine(BASE_URL)
# The trainer engine drives the inference side over HTTP. init for IPC is a
# no-op rendezvous; the same client carries start/update/finish.
engine = WeightTransferTrainerFactory.trainer_init(
init_info=IPCTrainerInitInfo(rank=0, packed=False), # rank 0 = sender
client=HTTPVLLMWeightSyncClient(BASE_URL),
source=ModuleSource(train_model),
)
# Pause generation before weight sync
pause_generation(BASE_URL)
# Start weight update, broadcast via IPC, then finish
start_weight_update(BASE_URL)
print("Broadcasting weights via CUDA IPC (HTTP)...")
trainer_args = IPCTrainerSendWeightsArgs(send_mode="http", url=BASE_URL)
IPCWeightTransferEngine.trainer_send_weights(
iterator=train_model.named_parameters(),
trainer_args=trainer_args,
)
finish_weight_update(BASE_URL)
# One call drives start_weight_update / update_weights / finish_weight_update.
engine.send_weights()
# Resume generation after weight sync
resume_generation(BASE_URL)
+18 -22
View File
@@ -29,10 +29,12 @@ from transformers import AutoModelForCausalLM
from vllm import LLM, SamplingParams
from vllm.config import WeightTransferConfig
from vllm.distributed.weight_transfer.ipc_engine import (
IPCTrainerSendWeightsArgs,
IPCWeightTransferEngine,
from vllm.distributed.weight_transfer import (
ModuleSource,
RayVLLMWeightSyncClient,
WeightTransferTrainerFactory,
)
from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo
class MyLLM(LLM):
@@ -65,23 +67,19 @@ class TrainModel:
self.llm_handle = llm_handle
def init_weight_transfer(self):
# IPC backend doesn't need initialization info
ray.get(
self.llm_handle.init_weight_transfer_engine.remote(dict(init_info=dict()))
"""Build the trainer-side IPC engine (no rendezvous needed for IPC)."""
self.engine = WeightTransferTrainerFactory.trainer_init(
init_info=IPCTrainerInitInfo(rank=0, packed=False), # rank 0 = sender
client=RayVLLMWeightSyncClient(self.llm_handle),
source=ModuleSource(self.train_model),
)
def broadcast_weights(
self, llm_handle: ray.actor.ActorHandle, packed: bool = False
):
"""Broadcast weights to the inference engine using IPC."""
self.llm_handle = llm_handle
trainer_args = IPCTrainerSendWeightsArgs(
send_mode="ray", llm_handle=llm_handle, packed=packed
)
IPCWeightTransferEngine.trainer_send_weights(
iterator=self.train_model.named_parameters(),
trainer_args=trainer_args,
)
def broadcast_weights(self):
"""Broadcast weights to the inference engine using IPC.
Drives start/update/finish on the inference side internally.
"""
self.engine.send_weights()
ray.init()
@@ -138,10 +136,8 @@ for output in outputs:
ray.get(llm.sleep.remote(level=0))
ray.get(train_model.init_weight_transfer.remote())
# Start weight update, sync weights, then finish
ray.get(llm.start_weight_update.remote())
ray.get(train_model.broadcast_weights.remote(llm))
ray.get(llm.finish_weight_update.remote())
# One call drives start_weight_update / update_weights / finish_weight_update.
ray.get(train_model.broadcast_weights.remote())
ray.get(llm.wake_up.remote(tags=["scheduling"]))
+49 -100
View File
@@ -14,8 +14,10 @@ Layout (4 GPUs, TP=1, DP=4, EP):
* A ``DataParallelInferenceEngine`` actor spawns all 4 LLM actors,
waits for initialization, and orchestrates generation / weight-sync.
Uses the built-in ``ray`` send_mode: each FSDP worker calls
``trainer_send_weights`` targeting its colocated LLM actor.
Every FSDP rank builds an ``IPCTrainerWeightTransferEngine`` (via ``trainer_init``)
and calls ``send_weights()``; all ranks join the IPC handle all-gather, and only
rank 0 (the sender) ships the merged handles and drives the DP LLM actors through
its ``RayVLLMWeightSyncClient``.
This example was run on 4xH100.
"""
@@ -23,7 +25,6 @@ This example was run on 4xH100.
from __future__ import annotations
import os
from dataclasses import asdict
import ray
import torch
@@ -31,19 +32,26 @@ import torch.distributed as dist
from huggingface_hub import snapshot_download
from ray.util.placement_group import placement_group
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
from torch.distributed._tensor import DTensor
from torch.distributed.fsdp import fully_shard
from transformers import AutoModelForCausalLM
from vllm import LLM, SamplingParams
from vllm.config import WeightTransferConfig
from vllm.distributed.weight_transfer.ipc_engine import (
IPCTrainerSendWeightsArgs,
IPCWeightTransferEngine,
IPCWeightTransferInitInfo,
from vllm.distributed.weight_transfer import (
ModuleSource,
RayVLLMWeightSyncClient,
WeightTransferTrainerFactory,
)
from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo
from vllm.utils.network_utils import get_ip, get_open_port
# The inference side only needs the backend; the packed wire params are
# trainer-side and get propagated to the workers at the init handshake.
WEIGHT_TRANSFER_CONFIG = WeightTransferConfig(backend="ipc")
# Packed IPC transfer with a 1 GB buffer (matches the per-chunk buffer size).
PACKED = True
PACKED_BUFFER_SIZE_BYTES = 1024 * 1024 * 1024
TRAIN_GPU_FRACTION = float(os.environ.get("RLHF_IPC_TRAIN_GPU_FRACTION", "0.42"))
VLLM_GPU_FRACTION = float(os.environ.get("RLHF_IPC_VLLM_GPU_FRACTION", "0.42"))
@@ -125,72 +133,34 @@ class FSDPTrainWorker:
def get_weight_metadata(self):
return self.weight_names, self.weight_dtype_names, self.weight_shapes
def gather_and_broadcast_weights_ipc(self, llm_handle, packed: bool = True):
"""All-gather full params; all ranks create IPC handles, rank 0 sends.
def setup_engine(self, llm_handles):
"""Build the trainer IPC engine on every FSDP rank.
All ranks must call trainer_send_weights so they participate in the
all_gather_object collective inside _all_gather_and_merge_handles.
Only rank 0 actually sends the payload to vLLM (gated by _is_rank_zero).
Called on all ranks: rank 0 becomes the sender (drives the inference
side and performs the no-op IPC init handshake against all DP LLM
actors); the other ranks hold a null-client engine and only join the
IPC handle all-gather during send_weights.
"""
def _full_param_iter():
# HF's Qwen3MoeExperts (and other recent HF MoE impls) packs
# all experts into two fused 3-D tensors per layer:
# experts.gate_up_proj shape (E, 2*I, H)
# experts.down_proj shape (E, H, I)
# vLLM's Qwen3MoE load_weights still expects the older
# per-expert HF layout (experts.<i>.gate_proj.weight,
# experts.<i>.up_proj.weight, experts.<i>.down_proj.weight),
# so we un-fuse on the fly. Split order matches HF's forward:
# gate, up = linear(x, gate_up_proj[i]).chunk(2, dim=-1)
# → rows [:I] of gate_up_proj[i] are gate, rows [I:] are up.
params = self.model.state_dict()
for name in list(params.keys()):
param = params.pop(name)
if isinstance(param, DTensor):
tensor = param.full_tensor().detach().contiguous()
else:
tensor = param.detach().contiguous()
del param
if name.endswith(".experts.gate_up_proj") and tensor.dim() == 3:
prefix = name[: -len(".gate_up_proj")]
num_experts, two_inter, _ = tensor.shape
inter = two_inter // 2
for i in range(num_experts):
expert = tensor[i]
yield (
f"{prefix}.{i}.gate_proj.weight",
expert[:inter].contiguous(),
)
yield (
f"{prefix}.{i}.up_proj.weight",
expert[inter:].contiguous(),
)
del tensor
elif name.endswith(".experts.down_proj") and tensor.dim() == 3:
prefix = name[: -len(".down_proj")]
num_experts = tensor.shape[0]
for i in range(num_experts):
yield (
f"{prefix}.{i}.down_proj.weight",
tensor[i].contiguous(),
)
del tensor
else:
yield name, tensor
trainer_args = IPCTrainerSendWeightsArgs(
send_mode="ray",
llm_handle=llm_handle,
packed=packed,
packed_buffer_size_bytes=1024 * 1024 * 1024, # 1 GB
)
IPCWeightTransferEngine.trainer_send_weights(
iterator=_full_param_iter(),
trainer_args=trainer_args,
self.engine = WeightTransferTrainerFactory.trainer_init(
init_info=IPCTrainerInitInfo(
rank=self.rank, # FSDP rank; sender is 0
packed=PACKED,
packed_buffer_size_bytes=PACKED_BUFFER_SIZE_BYTES,
),
client=RayVLLMWeightSyncClient(llm_handles),
source=ModuleSource(self.model),
)
def gather_and_broadcast_weights_ipc(self):
"""All-gather full params across FSDP ranks; rank 0 sends to vLLM.
Called on all ranks concurrently. `send_weights` gathers each param and
contributes to the IPC handle all-gather on every rank; only rank 0 (the
sender) drives start/update/finish on the inference side (the other
ranks' client RPCs no-op).
"""
self.engine.send_weights()
@ray.remote(num_cpus=1)
class DataParallelInferenceEngine:
@@ -224,7 +194,7 @@ class DataParallelInferenceEngine:
distributed_executor_backend="ray",
enable_expert_parallel=True,
gpu_memory_utilization=0.35,
weight_transfer_config=WeightTransferConfig(backend="ipc"),
weight_transfer_config=WEIGHT_TRANSFER_CONFIG,
enable_sleep_mode=True,
load_format="dummy",
dp_rank=r,
@@ -267,22 +237,6 @@ class DataParallelInferenceEngine:
rank_idx += 1
return ordered
def init_weight_transfer(self):
ray.get(
[
actor.init_weight_transfer_engine.remote(
dict(init_info=asdict(IPCWeightTransferInitInfo()))
)
for actor in self.llm_actors
]
)
def start_weight_update(self):
ray.get([actor.start_weight_update.remote() for actor in self.llm_actors])
def finish_weight_update(self):
ray.get([actor.finish_weight_update.remote() for actor in self.llm_actors])
def sleep(self, level: int = 0):
ray.get([actor.sleep.remote(level=level) for actor in self.llm_actors])
@@ -370,8 +324,10 @@ def main():
print("-" * 60)
# --- Weight transfer ---
print("[transfer] Initializing IPC weight transfer...")
ray.get(inference_engine.init_weight_transfer.remote())
# The rank-0 FSDP worker owns the trainer engine and drives the inference
# side (init/start/update/finish) for all DP actors via the Ray client.
print("[transfer] Initializing IPC weight transfer (all FSDP ranks)...")
ray.get([w.setup_engine.remote(llm_actors) for w in fsdp_workers])
# Two-phase sleep/wake pattern:
# 1. sleep(level=1) — offload weights to CPU, discard KV cache
@@ -384,18 +340,11 @@ def main():
print("[sync] Waking weights (KV cache stays free)...")
ray.get(inference_engine.wake_up.remote(tags=["weights"]))
print("[sync] Starting weight update...")
ray.get(inference_engine.start_weight_update.remote())
# All FSDP ranks participate in the IPC handle all-gather; rank 0's engine
# additionally drives start_weight_update / update_weights /
# finish_weight_update on the inference side.
print("[sync] Packed IPC transfer FSDP → vLLM...")
ray.get(
[
w.gather_and_broadcast_weights_ipc.remote(llm_actors, packed=True)
for w in fsdp_workers
]
)
ray.get(inference_engine.finish_weight_update.remote())
ray.get([w.gather_and_broadcast_weights_ipc.remote() for w in fsdp_workers])
print("[sync] Weight transfer complete.")
print("[sync] Waking KV cache + scheduling...")