[Docs][RL] Rewrite weight-transfer docs; standardize examples (#51729)

Signed-off-by: haoaaron <ahao@anyscale.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Aaron Hao
2026-08-11 18:53:15 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent 9b3382685e
commit 793ca6998a
16 changed files with 1211 additions and 1142 deletions
+132 -108
View File
@@ -1,45 +1,35 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Demonstrates reinforcement learning from human feedback (RLHF) using vLLM
via HTTP API, with IPC-based weight syncing APIs.
RLHF weight syncing against a `vllm serve` HTTP server, using CUDA IPC for the
data plane.
Unlike rlhf_nccl.py which uses NCCL and can use separate GPUs, this script
uses CUDA IPC which requires the training model and vLLM server to be on the
same GPU. Memory must be carefully managed to fit both models.
* OpenAI-compatible API for inference requests
* HTTP endpoints for the weight-transfer control plane
* CUDA IPC handles for the weight data plane
Unlike rlhf.py which creates a vLLM instance programmatically, this script
assumes you have already started a vLLM server using `vllm serve`. It uses:
- OpenAI-compatible API for inference requests
- HTTP endpoints for weight transfer control plane
- CUDA IPC for actual weight data transfer
1-GPU layout (single node): IPC shares GPU memory directly, so the server (TP=1)
and the training model both live on GPU 0. The server is started with
`--gpu-memory-utilization 0.5` to leave room for the training model.
Prerequisites:
Start a vLLM server with weight transfer enabled and reduced GPU memory
utilization to leave room for the training model:
The script starts the server itself, then:
$ VLLM_SERVER_DEV_MODE=1 VLLM_ALLOW_INSECURE_SERIALIZATION=1 \
vllm serve facebook/opt-125m --enforce-eager \
--weight-transfer-config '{"backend": "ipc"}' \
--load-format dummy \
--gpu-memory-utilization 0.5
1. Generate over HTTP → gibberish (server started with dummy weights).
2. Pause generation, sync real weights trainer → server over IPC, resume.
3. Generate again → sensible output.
Then run this script:
IPC handles are pickled for HTTP transport, so both sides need
`VLLM_ALLOW_INSECURE_SERIALIZATION=1`; this script sets it for itself and for
the server it spawns.
$ python rlhf_http_ipc.py
The example performs the following steps:
* Load the training model on GPU 0 (same GPU as the vLLM server).
* Generate text using the vLLM server via OpenAI-compatible API. The output
is expected to be nonsense because the server is initialized with dummy weights.
* Initialize weight transfer via HTTP endpoint (no-op for IPC).
* Broadcast the real weights from the training model to the vLLM server
using CUDA IPC handles.
* Generate text again to show normal output after the weight update.
Run:
$ python examples/rl/rlhf_http_ipc.py
"""
import os
import subprocess
import sys
import time
import requests
import torch
@@ -53,12 +43,76 @@ from vllm.distributed.weight_transfer import (
)
from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo
BASE_URL = "http://localhost:8000"
MODEL_NAME = "facebook/opt-125m"
# Enable insecure serialization for IPC handle serialization
SERVER_PORT = 8000
BASE_URL = f"http://localhost:{SERVER_PORT}"
# IPC requires colocation: the server and the training model share this GPU.
SERVER_DEVICE_IDS = "0"
TRAINER_DEVICE = "cuda:0"
# Leave room on the shared GPU for the training model.
SERVER_GPU_MEMORY_UTILIZATION = 0.5
# Needed to (de)serialize IPC handles across the HTTP boundary.
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
PROMPTS = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
def start_vllm_server() -> subprocess.Popen:
"""Spawn `vllm serve` and block until it is healthy."""
serve_args = [
"vllm",
"serve",
MODEL_NAME,
"--tensor-parallel-size",
"1",
"--device-ids",
SERVER_DEVICE_IDS,
"--enforce-eager",
"--load-format",
"dummy",
"--gpu-memory-utilization",
str(SERVER_GPU_MEMORY_UTILIZATION),
"--port",
str(SERVER_PORT),
"--weight-transfer-config",
'{"backend": "ipc"}',
]
env = os.environ.copy()
# Exposes the weight-transfer and pause/resume endpoints.
env["VLLM_SERVER_DEV_MODE"] = "1"
env["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
print(f"[server] Launching: {' '.join(serve_args)}")
proc = subprocess.Popen(
serve_args,
env=env,
stdout=sys.stdout,
stderr=sys.stderr,
start_new_session=True,
)
deadline = time.monotonic() + 900
while True:
if proc.poll() is not None:
raise RuntimeError("vLLM server exited before becoming ready.")
try:
if requests.get(f"{BASE_URL}/health", timeout=5).status_code == 200:
break
except requests.RequestException:
pass
if time.monotonic() > deadline:
raise RuntimeError("vLLM server failed to start in time.")
time.sleep(2)
print("[server] Ready.")
return proc
def generate_completions(client: OpenAI, model: str, prompts: list[str]) -> list[str]:
"""Generate completions using the OpenAI-compatible API."""
@@ -76,100 +130,70 @@ def generate_completions(client: OpenAI, model: str, prompts: list[str]) -> list
def pause_generation(base_url: str) -> None:
"""Pause generation via HTTP endpoint."""
url = f"{base_url}/pause"
response = requests.post(url, timeout=60)
response.raise_for_status()
requests.post(f"{base_url}/pause", timeout=60).raise_for_status()
def resume_generation(base_url: str) -> None:
"""Resume generation via HTTP endpoint."""
url = f"{base_url}/resume"
response = requests.post(url, timeout=60)
response.raise_for_status()
requests.post(f"{base_url}/resume", timeout=60).raise_for_status()
def get_world_size(base_url: str) -> int:
"""Get world size from the vLLM server."""
url = f"{base_url}/get_world_size"
response = requests.get(url, timeout=10)
response.raise_for_status()
return response.json()["world_size"]
def main():
# IPC requires the training model to be on the same GPU as the vLLM server
# The server should be started on GPU 0 with reduced memory utilization
device = "cuda:0"
torch.accelerator.set_device_index(device)
# Load the training model on the same GPU as the server
# Use bfloat16 to reduce memory footprint
print(f"Loading training model: {MODEL_NAME} on {device}")
print(
"Note: Ensure the vLLM server was started with --gpu-memory-utilization 0.5 "
"or lower to leave room for the training model."
)
train_model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, dtype=torch.bfloat16)
train_model.to(device)
train_model.eval() # Set to eval mode to save memory
# Create OpenAI client pointing to the vLLM server
client = OpenAI(
base_url=f"{BASE_URL}/v1",
api_key="EMPTY", # vLLM doesn't require an API key by default
)
# Test prompts
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
# Generate text before weight update. The output is expected to be nonsense
# because the server is initialized with dummy weights.
def print_generations(label: str, prompts: list[str], outputs: list[str]) -> None:
print("-" * 50)
print("Generating text BEFORE weight update (expect nonsense):")
print(label)
print("-" * 50)
outputs = generate_completions(client, MODEL_NAME, prompts)
for prompt, generated_text in zip(prompts, outputs):
print(f"Prompt: {prompt!r}\nGenerated text: {generated_text!r}")
print("-" * 50)
print("Initializing weight transfer (IPC backend)...")
# The trainer engine drives the inference side over HTTP. init for IPC is a
# no-op rendezvous; the same client carries start/update/finish.
engine = WeightTransferTrainerFactory.trainer_init(
init_info=IPCTrainerInitInfo(rank=0, packed=False), # rank 0 = sender
client=HTTPVLLMWeightSyncClient(BASE_URL),
source=ModuleSource(train_model),
)
def main():
server_proc = start_vllm_server()
try:
# The training model must sit on the same physical GPU as the server.
torch.accelerator.set_device_index(TRAINER_DEVICE)
# Pause generation before weight sync
pause_generation(BASE_URL)
print(f"[trainer] Loading training model: {MODEL_NAME} on {TRAINER_DEVICE}")
train_model = AutoModelForCausalLM.from_pretrained(
MODEL_NAME, dtype=torch.bfloat16
)
train_model.to(TRAINER_DEVICE)
train_model.eval() # eval mode to save memory on the shared GPU
print("Broadcasting weights via CUDA IPC (HTTP)...")
# One call drives start_weight_update / update_weights / finish_weight_update.
engine.send_weights()
client = OpenAI(base_url=f"{BASE_URL}/v1", api_key="EMPTY")
# Resume generation after weight sync
resume_generation(BASE_URL)
# Generate with dummy weights — expect nonsense.
outputs = generate_completions(client, MODEL_NAME, PROMPTS)
print_generations("BEFORE weight sync (dummy weights):", PROMPTS, outputs)
# Generate text after weight update. The output is expected to be normal
# because the real weights are now loaded.
print("-" * 50)
print("Generating text AFTER weight update:")
print("-" * 50)
outputs_updated = generate_completions(client, MODEL_NAME, prompts)
for prompt, generated_text in zip(prompts, outputs_updated):
print(f"Prompt: {prompt!r}\nGenerated text: {generated_text!r}")
print("-" * 50)
# IPC needs no data-plane rendezvous; `trainer_init` only ships the
# `packed` flag, which the server must decode with.
print("[transfer] Initializing IPC weight transfer...")
engine = WeightTransferTrainerFactory.trainer_init(
init_info=IPCTrainerInitInfo(rank=0, packed=False), # rank 0 = sender
client=HTTPVLLMWeightSyncClient(BASE_URL),
source=ModuleSource(train_model),
)
# Note: The training model and IPC handles remain in memory.
# In a real RLHF training loop, you would update the training model
# and create new IPC handles for each weight update.
pause_generation(BASE_URL)
# Drives start_weight_update / update_weights / finish_weight_update.
print("[sync] Sharing weights via CUDA IPC...")
engine.send_weights()
print("[sync] Weight transfer complete.")
resume_generation(BASE_URL)
# Generate with the synced weights — expect sensible output.
outputs_updated = generate_completions(client, MODEL_NAME, PROMPTS)
print_generations("AFTER weight sync (real weights):", PROMPTS, outputs_updated)
finally:
print("[server] Shutting down...")
server_proc.terminate()
try:
server_proc.wait(timeout=30)
except subprocess.TimeoutExpired:
server_proc.kill()
if __name__ == "__main__":
+146 -102
View File
@@ -1,38 +1,33 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Demonstrates reinforcement learning from human feedback (RLHF) using vLLM
via HTTP API, with native weight syncing APIs.
RLHF weight syncing against a `vllm serve` HTTP server, using NCCL for the
data plane.
Unlike rlhf.py which creates a vLLM instance programmatically, this script
assumes you have already started a vLLM server using `vllm serve`. It uses:
- OpenAI-compatible API for inference requests
- HTTP endpoints for weight transfer control plane
- NCCL for actual weight data transfer
* OpenAI-compatible API for inference requests
* HTTP endpoints for the weight-transfer control plane
* NCCL for the weight data plane
Prerequisites:
Start a vLLM server with weight transfer enabled:
3-GPU layout (single node):
Inference — GPUs 0-1, `vllm serve` with TP=2 and fp8 quantization
Training — GPU 2, a bf16 Hugging Face model in this process
(the server quantizes to fp8 as it loads)
$ VLLM_SERVER_DEV_MODE=1 vllm serve facebook/opt-125m \
--enforce-eager \
--weight-transfer-config '{"backend": "nccl"}' \
--load-format dummy
The script starts the server itself, then:
Then run this script:
1. Generate over HTTP → gibberish (server started with dummy weights).
2. Pause generation, sync real weights trainer → server over NCCL, resume.
3. Generate again → sensible output.
$ python rlhf_http.py
The example performs the following steps:
* Load the training model on GPU 0.
* Generate text using the vLLM server via OpenAI-compatible API. The output
is expected to be nonsense because the server is initialized with dummy weights.
* Initialize weight transfer via HTTP endpoint.
* Broadcast the real weights from the training model to the vLLM server
using NCCL.
* Generate text again to show normal output after the weight update.
Run:
$ python examples/rl/rlhf_http_nccl.py
"""
import os
import subprocess
import sys
import time
import requests
import torch
from openai import OpenAI
@@ -46,9 +41,71 @@ from vllm.distributed.weight_transfer import (
from vllm.distributed.weight_transfer.nccl_engine import NCCLTrainerInitInfo
from vllm.utils.network_utils import get_ip, get_open_port
BASE_URL = "http://localhost:8000"
MODEL_NAME = "facebook/opt-125m"
SERVER_PORT = 8000
BASE_URL = f"http://localhost:{SERVER_PORT}"
INFERENCE_TP_SIZE = 2
# Physical GPUs for the server; the trainer takes the next one.
SERVER_DEVICE_IDS = "0,1"
TRAINER_DEVICE = "cuda:2"
PROMPTS = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
def start_vllm_server() -> subprocess.Popen:
"""Spawn `vllm serve` and block until it is healthy."""
serve_args = [
"vllm",
"serve",
MODEL_NAME,
"--tensor-parallel-size",
str(INFERENCE_TP_SIZE),
"--device-ids",
SERVER_DEVICE_IDS,
"--quantization",
"fp8",
"--enforce-eager",
"--load-format",
"dummy",
"--port",
str(SERVER_PORT),
"--weight-transfer-config",
'{"backend": "nccl"}',
]
env = os.environ.copy()
# Exposes the weight-transfer and pause/resume endpoints.
env["VLLM_SERVER_DEV_MODE"] = "1"
print(f"[server] Launching: {' '.join(serve_args)}")
proc = subprocess.Popen(
serve_args,
env=env,
stdout=sys.stdout,
stderr=sys.stderr,
start_new_session=True,
)
deadline = time.monotonic() + 900
while True:
if proc.poll() is not None:
raise RuntimeError("vLLM server exited before becoming ready.")
try:
if requests.get(f"{BASE_URL}/health", timeout=5).status_code == 200:
break
except requests.RequestException:
pass
if time.monotonic() > deadline:
raise RuntimeError("vLLM server failed to start in time.")
time.sleep(2)
print("[server] Ready.")
return proc
def generate_completions(client: OpenAI, model: str, prompts: list[str]) -> list[str]:
"""Generate completions using the OpenAI-compatible API."""
@@ -66,105 +123,92 @@ def generate_completions(client: OpenAI, model: str, prompts: list[str]) -> list
def pause_generation(base_url: str) -> None:
"""Pause generation via HTTP endpoint."""
url = f"{base_url}/pause"
response = requests.post(url, timeout=60)
response.raise_for_status()
requests.post(f"{base_url}/pause", timeout=60).raise_for_status()
def resume_generation(base_url: str) -> None:
"""Resume generation via HTTP endpoint."""
url = f"{base_url}/resume"
response = requests.post(url, timeout=60)
response.raise_for_status()
requests.post(f"{base_url}/resume", timeout=60).raise_for_status()
def get_world_size(base_url: str) -> int:
"""Get world size from the vLLM server."""
url = f"{base_url}/get_world_size"
response = requests.get(url, timeout=10)
"""Get the number of inference workers from the vLLM server."""
response = requests.get(f"{base_url}/get_world_size", timeout=10)
response.raise_for_status()
return response.json()["world_size"]
def main():
# Get the inference world size from the vLLM server
inference_world_size = get_world_size(BASE_URL)
world_size = inference_world_size + 1 # +1 for the trainer
device = f"cuda:{inference_world_size}"
torch.accelerator.set_device_index(device)
# Load the training model
print(f"Loading training model: {MODEL_NAME}")
train_model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, dtype=torch.bfloat16)
train_model.to(device)
# Create OpenAI client pointing to the vLLM server
client = OpenAI(
base_url=f"{BASE_URL}/v1",
api_key="EMPTY", # vLLM doesn't require an API key by default
)
# Test prompts
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
# Generate text before weight update. The output is expected to be nonsense
# because the server is initialized with dummy weights.
def print_generations(label: str, prompts: list[str], outputs: list[str]) -> None:
print("-" * 50)
print("Generating text BEFORE weight update (expect nonsense):")
print(label)
print("-" * 50)
outputs = generate_completions(client, MODEL_NAME, prompts)
for prompt, generated_text in zip(prompts, outputs):
print(f"Prompt: {prompt!r}\nGenerated text: {generated_text!r}")
print("-" * 50)
# Set up the communication channel between the training process and the
# vLLM server. The trainer is rank 0, vLLM worker(s) start after it.
master_address = get_ip()
master_port = get_open_port()
print(f"Initializing weight transfer: master={master_address}:{master_port}")
def main():
server_proc = start_vllm_server()
try:
# The trainer sits on the GPU after the server's, and is NCCL rank 0.
torch.accelerator.set_device_index(TRAINER_DEVICE)
# The trainer engine owns the full handshake. `trainer_init` kicks off the
# server's init_weight_transfer_engine (via the HTTP client) on a worker
# thread while opening the trainer's NCCL endpoint, so both ends rendezvous
# together — no manual threading needed here.
engine = WeightTransferTrainerFactory.trainer_init(
init_info=NCCLTrainerInitInfo(
master_address=master_address,
master_port=master_port,
world_size=world_size,
rank=0, # single-GPU trainer is the sole (sender) rank
packed=True,
),
client=HTTPVLLMWeightSyncClient(BASE_URL),
source=ModuleSource(train_model),
)
print(f"[trainer] Loading training model: {MODEL_NAME} on {TRAINER_DEVICE}")
train_model = AutoModelForCausalLM.from_pretrained(
MODEL_NAME, dtype=torch.bfloat16
)
train_model.to(TRAINER_DEVICE)
# Pause generation before weight sync
pause_generation(BASE_URL)
client = OpenAI(base_url=f"{BASE_URL}/v1", api_key="EMPTY")
# One call drives start_weight_update / update_weights / finish_weight_update
# over HTTP, concurrent with the NCCL broadcast.
print("Broadcasting weights via NCCL...")
engine.send_weights()
# Generate with dummy weights — expect nonsense.
outputs = generate_completions(client, MODEL_NAME, PROMPTS)
print_generations("BEFORE weight sync (dummy weights):", PROMPTS, outputs)
# Resume generation after weight sync
resume_generation(BASE_URL)
# The transfer NCCL group is the trainer plus every inference worker.
world_size = get_world_size(BASE_URL) + 1
master_address = get_ip()
master_port = get_open_port()
print(
f"[transfer] Rendezvous at {master_address}:{master_port}, "
f"world_size={world_size} (1 trainer + {world_size - 1} vLLM workers)"
)
# Generate text after weight update. The output is expected to be normal
# because the real weights are now loaded.
print("-" * 50)
print("Generating text AFTER weight update:")
print("-" * 50)
outputs_updated = generate_completions(client, MODEL_NAME, prompts)
for prompt, generated_text in zip(prompts, outputs_updated):
print(f"Prompt: {prompt!r}\nGenerated text: {generated_text!r}")
print("-" * 50)
# `trainer_init` drives the handshake: it initializes the server's
# transfer engine while opening the trainer's own NCCL endpoint, so both
# ends rendezvous together.
engine = WeightTransferTrainerFactory.trainer_init(
init_info=NCCLTrainerInitInfo(
master_address=master_address,
master_port=master_port,
world_size=world_size,
rank=0, # single-GPU trainer is the sole (sender) rank
packed=True,
),
client=HTTPVLLMWeightSyncClient(BASE_URL),
source=ModuleSource(train_model),
)
pause_generation(BASE_URL)
# Drives start_weight_update / update_weights / finish_weight_update,
# concurrent with the NCCL broadcast.
print("[sync] Broadcasting weights via NCCL...")
engine.send_weights()
print("[sync] Weight broadcast complete.")
resume_generation(BASE_URL)
# Generate with the synced weights — expect sensible output.
outputs_updated = generate_completions(client, MODEL_NAME, PROMPTS)
print_generations("AFTER weight sync (real weights):", PROMPTS, outputs_updated)
finally:
print("[server] Shutting down...")
server_proc.terminate()
try:
server_proc.wait(timeout=30)
except subprocess.TimeoutExpired:
server_proc.kill()
if __name__ == "__main__":
-151
View File
@@ -1,151 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Demonstrates reinforcement learning from human feedback (RLHF) using vLLM and Ray,
with IPC-based weight syncing APIs
The script colocates the training and inference workloads onto the same GPU using Ray.
The example performs the following steps:
* Request a placement group of 1 GPU.
* Place the inference model on the above GPU using the placement group.
* Place and load the training model on the same GPU using the placement group.
* Generate text from a list of prompts using the inference engine.
* Update the weights of the training model and broadcast the updated weights
to the inference engine by using CUDA IPC handles. Note that
for demonstration purposes we simply zero out the weights.
This example assumes a single-node cluster with a single GPU,
but can be extended to multiple GPUs.
"""
import os
import ray
from ray.util.placement_group import placement_group
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
from transformers import AutoModelForCausalLM
from vllm import LLM, SamplingParams
from vllm.config import WeightTransferConfig
from vllm.distributed.weight_transfer import (
ModuleSource,
RayVLLMWeightSyncClient,
WeightTransferTrainerFactory,
)
from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo
class MyLLM(LLM):
"""Configure the vLLM worker for Ray placement group execution."""
def __init__(self, *args, **kwargs):
# Remove the top-level CUDA_VISIBLE_DEVICES variable set by Ray
# so that vLLM can manage its own device placement within the worker.
os.environ.pop("CUDA_VISIBLE_DEVICES", None)
# Each worker uses 0.4 GPU so that two instances fit on the same GPU.
os.environ["VLLM_RAY_PER_WORKER_GPUS"] = "0.4"
os.environ["VLLM_RAY_BUNDLE_INDICES"] = "0"
# needed for ipc handle serialization
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
super().__init__(*args, **kwargs)
# Load the OPT-125M model onto GPU 0 for the training workload.
MODEL_NAME = "facebook/opt-125m"
@ray.remote
class TrainModel:
def __init__(self, llm_handle: ray.actor.ActorHandle):
self.train_model = AutoModelForCausalLM.from_pretrained(
MODEL_NAME,
)
self.train_model.to("cuda:0")
self.llm_handle = llm_handle
def init_weight_transfer(self):
"""Build the trainer-side IPC engine (no rendezvous needed for IPC)."""
self.engine = WeightTransferTrainerFactory.trainer_init(
init_info=IPCTrainerInitInfo(rank=0, packed=False), # rank 0 = sender
client=RayVLLMWeightSyncClient(self.llm_handle),
source=ModuleSource(self.train_model),
)
def broadcast_weights(self):
"""Broadcast weights to the inference engine using IPC.
Drives start/update/finish on the inference side internally.
"""
self.engine.send_weights()
ray.init()
pg_colocate = placement_group([{"GPU": 1, "CPU": 0}])
ray.get(pg_colocate.ready())
llm = ray.remote(
num_cpus=0,
num_gpus=0,
scheduling_strategy=PlacementGroupSchedulingStrategy(
placement_group=pg_colocate,
placement_group_capture_child_tasks=True,
),
)(MyLLM).remote(
model=MODEL_NAME,
enforce_eager=True,
tensor_parallel_size=1,
distributed_executor_backend="ray",
gpu_memory_utilization=0.7,
weight_transfer_config=WeightTransferConfig(backend="ipc"),
load_format="dummy",
)
train_model = TrainModel.options(
num_gpus=0.1,
num_cpus=0,
scheduling_strategy=PlacementGroupSchedulingStrategy(
placement_group=pg_colocate, placement_group_capture_child_tasks=True
),
).remote(llm)
# Generate text from the prompts.
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
sampling_params = SamplingParams(temperature=0)
outputs = ray.get(llm.generate.remote(prompts, sampling_params))
print("-" * 50)
for output in outputs:
prompt = output.prompt
generated_text = output.outputs[0].text
print(f"Prompt: {prompt!r}\nGenerated text: {generated_text!r}")
print("-" * 50)
ray.get(llm.sleep.remote(level=0))
ray.get(train_model.init_weight_transfer.remote())
# One call drives start_weight_update / update_weights / finish_weight_update.
ray.get(train_model.broadcast_weights.remote())
ray.get(llm.wake_up.remote(tags=["scheduling"]))
outputs_packed = ray.get(llm.generate.remote(prompts, sampling_params))
print("-" * 50)
print("Results after packed/chunked IPC weight sync:")
for output in outputs_packed:
prompt = output.prompt
generated_text = output.outputs[0].text
print(f"Prompt: {prompt!r}\nGenerated text: {generated_text!r}")
print("-" * 50)
+181 -232
View File
@@ -4,96 +4,88 @@
RLHF with FSDP2 training and vLLM expert-parallel inference using **CUDA IPC**
weight transfer and **packed** tensors.
Layout (4 GPUs, TP=1, DP=4, EP):
* One Ray placement group per GPU.
* Each PG holds one FSDP training worker and one vLLM ``LLM`` instance
(sync API) using fractional GPUs so both fit on the same device.
* The 4 ``LLM`` instances form a DP group via env-var-based SPMD
coordination (``VLLM_DP_RANK``, ``VLLM_DP_SIZE``, etc.), the same
mechanism used by ``examples/offline_inference/data_parallel.py``.
* A ``DataParallelInferenceEngine`` actor spawns all 4 LLM actors,
waits for initialization, and orchestrates generation / weight-sync.
Multi-rank version of `rlhf_http_ipc.py`: the trainer is 4 FSDP2 Ray actors
colocated with a data-parallel `vllm serve` on the same 4 physical GPUs.
4-GPU layout (single node), all colocated:
Training — 4 GPUs, PyTorch FSDP2 (fully_shard), as Ray actors
Inference — the same 4 GPUs, `vllm serve --data-parallel-size 4 -tp 1
--enable-expert-parallel` (EP_SIZE = TP x DP = 4)
IPC requires the trainer and the server to sit on the same GPUs, so the script
reserves the training GPUs through Ray first, asks Ray which ones it got, and
pins the server to exactly those with `--device-ids`.
Both sides share each GPU, so the server is capped with
`--gpu-memory-utilization` and its weights are moved aside for the transfer:
1. `/sleep?level=1` — offload server weights to CPU, drop KV cache
2. `/wake_up?tags=weights` — weights back on GPU, KV cache still free
3. packed IPC transfer — overwrite weights with room to spare
4. `/wake_up?tags=kv_cache&tags=scheduling` — re-allocate KV cache, resume
Every FSDP rank builds an ``IPCTrainerWeightTransferEngine`` (via ``trainer_init``)
and calls ``send_weights()``; all ranks join the IPC handle all-gather, and only
rank 0 (the sender) ships the merged handles and drives the DP LLM actors through
its ``RayVLLMWeightSyncClient``.
rank 0 (the sender) ships the merged handles and drives the server.
This example was run on 4xH100.
Run:
$ python examples/rl/rlhf_ipc_fsdp_ep.py
"""
from __future__ import annotations
import os
import subprocess
import sys
import time
import ray
import requests
import torch
import torch.distributed as dist
from huggingface_hub import snapshot_download
from ray.util.placement_group import placement_group
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
from openai import OpenAI
from torch.distributed.fsdp import fully_shard
from transformers import AutoModelForCausalLM
from vllm import LLM, SamplingParams
from vllm.config import WeightTransferConfig
from vllm.distributed.weight_transfer import (
HTTPVLLMWeightSyncClient,
ModuleSource,
RayVLLMWeightSyncClient,
WeightTransferTrainerFactory,
)
from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo
from vllm.utils.network_utils import get_ip, get_open_port
# The inference side only needs the backend; the packed wire params are
# trainer-side and get propagated to the workers at the init handshake.
WEIGHT_TRANSFER_CONFIG = WeightTransferConfig(backend="ipc")
# Packed IPC transfer with a 1 GB buffer (matches the per-chunk buffer size).
PACKED = True
PACKED_BUFFER_SIZE_BYTES = 1024 * 1024 * 1024
TRAIN_GPU_FRACTION = float(os.environ.get("RLHF_IPC_TRAIN_GPU_FRACTION", "0.42"))
VLLM_GPU_FRACTION = float(os.environ.get("RLHF_IPC_VLLM_GPU_FRACTION", "0.42"))
MODEL_NAME = "Qwen/Qwen3-30B-A3B"
SERVED_MODEL_NAME = "policy"
FSDP_WORLD_SIZE = 4
INFERENCE_TP_SIZE = 1
INFERENCE_DP_SIZE = 4
# Packed IPC transfer with a 1 GB chunk buffer.
PACKED = True
PACKED_BUFFER_SIZE_BYTES = 1024 * 1024 * 1024
class MyLLM(LLM):
"""LLM subclass that configures DP env vars for SPMD coordination."""
# The server shares each GPU with a training rank, so cap what it reserves.
SERVER_GPU_MEMORY_UTILIZATION = 0.35
def __init__(
self,
*args,
dp_rank: int = 0,
dp_size: int = 1,
dp_master_ip: str = "127.0.0.1",
dp_master_port: int = 0,
**kwargs,
):
os.environ.pop("CUDA_VISIBLE_DEVICES", None)
os.environ["VLLM_RAY_PER_WORKER_GPUS"] = str(VLLM_GPU_FRACTION)
os.environ["VLLM_RAY_BUNDLE_INDICES"] = "0"
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
SERVER_PORT = 8000
BASE_URL = f"http://localhost:{SERVER_PORT}"
os.environ["VLLM_DP_RANK"] = str(dp_rank)
os.environ["VLLM_DP_RANK_LOCAL"] = str(dp_rank)
os.environ["VLLM_DP_SIZE"] = str(dp_size)
os.environ["VLLM_DP_MASTER_IP"] = dp_master_ip
os.environ["VLLM_DP_MASTER_PORT"] = str(dp_master_port)
super().__init__(*args, **kwargs)
def ready(self):
return True
PROMPTS = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
@ray.remote(num_cpus=0, num_gpus=TRAIN_GPU_FRACTION)
@ray.remote(num_gpus=1)
class FSDPTrainWorker:
"""One FSDP2 worker per GPU; colocated with vLLM DP rank via placement group."""
"""One FSDP2 worker per GPU; colocated with one vLLM DP rank."""
def __init__(
self,
@@ -115,12 +107,6 @@ class FSDPTrainWorker:
model_name, torch_dtype=torch.bfloat16
)
self.weight_names = [n for n, _ in model.named_parameters()]
self.weight_dtype_names = [
str(p.dtype).split(".")[-1] for _, p in model.named_parameters()
]
self.weight_shapes = [list(p.shape) for _, p in model.named_parameters()]
for layer in model.model.layers:
fully_shard(layer)
fully_shard(model)
@@ -130,237 +116,200 @@ class FSDPTrainWorker:
def get_rank(self):
return self.rank
def get_weight_metadata(self):
return self.weight_names, self.weight_dtype_names, self.weight_shapes
def get_gpu_ids(self):
"""Physical GPU id(s) Ray assigned to this worker."""
return ray.get_gpu_ids()
def setup_engine(self, llm_handles):
"""Build the trainer IPC engine on every FSDP rank.
Called on all ranks: rank 0 becomes the sender (drives the inference
side and performs the no-op IPC init handshake against all DP LLM
actors); the other ranks hold a null-client engine and only join the
IPC handle all-gather during send_weights.
"""
def setup_engine(self, base_url: str):
"""Build the trainer IPC engine. Called on every FSDP rank."""
self.engine = WeightTransferTrainerFactory.trainer_init(
init_info=IPCTrainerInitInfo(
rank=self.rank, # FSDP rank; sender is 0
packed=PACKED,
packed_buffer_size_bytes=PACKED_BUFFER_SIZE_BYTES,
),
client=RayVLLMWeightSyncClient(llm_handles),
client=HTTPVLLMWeightSyncClient(base_url),
source=ModuleSource(self.model),
)
def gather_and_broadcast_weights_ipc(self):
"""All-gather full params across FSDP ranks; rank 0 sends to vLLM.
Called on all ranks concurrently. `send_weights` gathers each param and
contributes to the IPC handle all-gather on every rank; only rank 0 (the
sender) drives start/update/finish on the inference side (the other
ranks' client RPCs no-op).
"""
"""Send the current weights to vLLM. Called on every FSDP rank."""
self.engine.send_weights()
@ray.remote(num_cpus=1)
class DataParallelInferenceEngine:
"""Manages a pool of DP-sharded vLLM LLM actors.
def start_vllm_server(model_path: str, device_ids: str) -> subprocess.Popen:
"""Spawn a `vllm serve` HTTP server (DP+EP) pinned to `device_ids`."""
serve_args = [
"vllm",
"serve",
model_path,
"--served-model-name",
SERVED_MODEL_NAME,
"--tensor-parallel-size",
str(INFERENCE_TP_SIZE),
"--data-parallel-size",
str(INFERENCE_DP_SIZE),
"--enable-expert-parallel",
# Pins the server to the same physical GPUs as the training ranks.
"--device-ids",
device_ids,
"--enable-sleep-mode",
"--enforce-eager",
"--load-format",
"dummy",
"--gpu-memory-utilization",
str(SERVER_GPU_MEMORY_UTILIZATION),
"--port",
str(SERVER_PORT),
"--weight-transfer-config",
'{"backend": "ipc"}',
]
env = os.environ.copy()
env["VLLM_SERVER_DEV_MODE"] = "1" # exposes weight-transfer + sleep endpoints
env["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1" # IPC handles over HTTP
env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
print(f"[server] Launching: {' '.join(serve_args)} (GPUs {device_ids})")
proc = subprocess.Popen(
serve_args,
env=env,
stdout=sys.stdout,
stderr=sys.stderr,
start_new_session=True,
)
Spawns one MyLLM actor per placement group, waits for all engines to
finish initializing, and exposes generation / weight-sync helpers.
"""
# Wait for the server to come up (model load can take a while).
deadline = time.monotonic() + 1800
while True:
if proc.poll() is not None:
raise RuntimeError("vLLM server exited before becoming ready.")
try:
if requests.get(f"{BASE_URL}/health", timeout=5).status_code == 200:
break
except requests.RequestException:
pass
if time.monotonic() > deadline:
raise RuntimeError("vLLM server failed to start in time.")
time.sleep(2)
print("[server] Ready.")
return proc
def __init__(
self,
model: str,
pgs: list,
dp_master_ip: str,
dp_master_port: int,
):
dp_size = len(pgs)
self.llm_actors = []
for r in range(dp_size):
sched = PlacementGroupSchedulingStrategy(
placement_group=pgs[r],
placement_group_capture_child_tasks=True,
)
actor = (
ray.remote(num_cpus=0, num_gpus=0)(MyLLM)
.options(scheduling_strategy=sched)
.remote(
model=model,
enforce_eager=True,
tensor_parallel_size=INFERENCE_TP_SIZE,
distributed_executor_backend="ray",
enable_expert_parallel=True,
gpu_memory_utilization=0.35,
weight_transfer_config=WEIGHT_TRANSFER_CONFIG,
enable_sleep_mode=True,
load_format="dummy",
dp_rank=r,
dp_size=dp_size,
dp_master_ip=dp_master_ip,
dp_master_port=dp_master_port,
)
)
self.llm_actors.append(actor)
ray.get([actor.ready.remote() for actor in self.llm_actors])
def generate_completions(client: OpenAI, prompts: list[str]) -> list[str]:
"""Generate completions via the OpenAI HTTP API."""
results = []
for prompt in prompts:
response = client.completions.create(
model=SERVED_MODEL_NAME,
prompt=prompt,
max_tokens=32,
temperature=0,
)
results.append(response.choices[0].text)
return results
def get_llm_actors(self):
return self.llm_actors
def generate(self, prompts: list[str], sampling_params):
"""Distribute prompts round-robin across DP ranks and collect results."""
dp_size = len(self.llm_actors)
per_rank: list[list[str]] = [[] for _ in range(dp_size)]
indices: list[list[int]] = [[] for _ in range(dp_size)]
def sleep_engine(base_url: str, level: int) -> None:
"""Put the engine to sleep (level 1 offloads weights, drops KV cache)."""
response = requests.post(f"{base_url}/sleep", params={"level": level}, timeout=600)
response.raise_for_status()
for i, prompt in enumerate(prompts):
rank = i % dp_size
per_rank[rank].append(prompt)
indices[rank].append(i)
refs = [
actor.generate.remote(per_rank[r], sampling_params)
for r, actor in enumerate(self.llm_actors)
if per_rank[r]
]
all_outputs = ray.get(refs)
def wake_up_engine(base_url: str, tags: list[str] | None = None) -> None:
"""Wake the engine, optionally only for specific memory tags."""
params = [("tags", tag) for tag in tags] if tags else None
response = requests.post(f"{base_url}/wake_up", params=params, timeout=600)
response.raise_for_status()
ordered = [None] * len(prompts)
rank_idx = 0
for r in range(dp_size):
if per_rank[r]:
for local_i, orig_i in enumerate(indices[r]):
ordered[orig_i] = all_outputs[rank_idx][local_i]
rank_idx += 1
return ordered
def sleep(self, level: int = 0):
ray.get([actor.sleep.remote(level=level) for actor in self.llm_actors])
def wake_up(self, tags: list[str] | None = None):
ray.get([actor.wake_up.remote(tags=tags) for actor in self.llm_actors])
def print_generations(label: str, prompts: list[str], outputs: list[str]) -> None:
print("-" * 60)
print(label)
print("-" * 60)
for prompt, text in zip(prompts, outputs):
print(f"Prompt: {prompt!r}")
print(f"Generated: {text!r}")
print("-" * 60)
def main():
ray.init(
runtime_env={
"env_vars": {
# The trainer pickles IPC handles for the HTTP client.
"VLLM_ALLOW_INSECURE_SERIALIZATION": "1",
}
}
)
assert TRAIN_GPU_FRACTION + VLLM_GPU_FRACTION <= 1.0, (
"Train + vLLM GPU fractions must sum to at most 1.0 per bundle."
)
local_model_path = snapshot_download(MODEL_NAME)
print(f"[init] Model downloaded to {local_model_path}")
fsdp_master_addr = get_ip()
fsdp_master_port = get_open_port()
dp_master_port = get_open_port()
dp_master_ip = get_ip()
# Create one placement group per DP rank (one GPU each).
pgs = []
for _ in range(INFERENCE_DP_SIZE):
pg = placement_group([{"GPU": 1, "CPU": 1}])
pgs.append(pg)
ray.get([pg.ready() for pg in pgs])
print(f"[init] {len(pgs)} placement groups ready.")
# Launch FSDP training workers, one per PG.
scheduling = [
PlacementGroupSchedulingStrategy(
placement_group=pgs[r],
placement_group_capture_child_tasks=True,
)
for r in range(FSDP_WORLD_SIZE)
]
# Launch the training workers first so Ray reserves their GPUs; the server
# is then pinned to those same physical GPUs.
fsdp_workers = [
FSDPTrainWorker.options(scheduling_strategy=scheduling[r]).remote(
FSDPTrainWorker.remote(
local_model_path,
r,
rank,
FSDP_WORLD_SIZE,
fsdp_master_addr,
fsdp_master_port,
)
for r in range(FSDP_WORLD_SIZE)
for rank in range(FSDP_WORLD_SIZE)
]
ray.get([w.get_rank.remote() for w in fsdp_workers])
print(f"[init] {FSDP_WORLD_SIZE} FSDP workers ready.")
print(f"[init] {FSDP_WORLD_SIZE} FSDP training workers ready.")
# Launch DP inference engine (spawns and initializes all LLM actors).
inference_engine = DataParallelInferenceEngine.remote(
model=local_model_path,
pgs=pgs,
dp_master_ip=dp_master_ip,
dp_master_port=dp_master_port,
training_gpus = sorted(
int(g)
for ids in ray.get([w.get_gpu_ids.remote() for w in fsdp_workers])
for g in ids
)
llm_actors = ray.get(inference_engine.get_llm_actors.remote())
print(f"[init] {INFERENCE_DP_SIZE} LLM actors ready.")
if len(training_gpus) != INFERENCE_TP_SIZE * INFERENCE_DP_SIZE:
raise RuntimeError(
f"Need {INFERENCE_TP_SIZE * INFERENCE_DP_SIZE} colocated GPUs but "
f"Ray assigned training to {training_gpus}."
)
device_ids = ",".join(str(g) for g in training_gpus)
print(f"[init] Colocating training and inference on GPUs [{device_ids}].")
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
sampling_params = SamplingParams(temperature=0)
server_proc = start_vllm_server(local_model_path, device_ids)
try:
client = OpenAI(base_url=f"{BASE_URL}/v1", api_key="EMPTY")
print("[generate] Generating with dummy weights...")
outputs = ray.get(inference_engine.generate.remote(prompts, sampling_params))
print("-" * 60)
print("BEFORE weight sync (dummy weights):")
print("-" * 60)
for output in outputs:
print(f"Prompt: {output.prompt!r}")
print(f"Generated: {output.outputs[0].text!r}")
print("-" * 60)
print("[generate] Generating with dummy weights...")
outputs = generate_completions(client, PROMPTS)
print_generations("BEFORE weight sync (dummy weights):", PROMPTS, outputs)
# --- Weight transfer ---
# The rank-0 FSDP worker owns the trainer engine and drives the inference
# side (init/start/update/finish) for all DP actors via the Ray client.
print("[transfer] Initializing IPC weight transfer (all FSDP ranks)...")
ray.get([w.setup_engine.remote(llm_actors) for w in fsdp_workers])
# --- Weight transfer ---
print("[transfer] Initializing IPC weight transfer (all FSDP ranks)...")
ray.get([w.setup_engine.remote(BASE_URL) for w in fsdp_workers])
# Two-phase sleep/wake pattern:
# 1. sleep(level=1) — offload weights to CPU, discard KV cache
# 2. wake_up(tags=["weights"]) — bring weights back to GPU (KV cache still free)
# 3. IPC weight transfer — overwrite weights, plenty of room without KV cache
# 4. wake_up(tags=["kv_cache"]) — re-allocate KV cache for inference
print("[sync] Sleeping engines (offload weights + free KV cache)...")
ray.get(inference_engine.sleep.remote(level=1))
print("[sync] Sleeping engine (offload weights + free KV cache)...")
sleep_engine(BASE_URL, level=1)
print("[sync] Waking weights (KV cache stays free)...")
ray.get(inference_engine.wake_up.remote(tags=["weights"]))
print("[sync] Waking weights (KV cache stays free)...")
wake_up_engine(BASE_URL, tags=["weights"])
# All FSDP ranks participate in the IPC handle all-gather; rank 0's engine
# additionally drives start_weight_update / update_weights /
# finish_weight_update on the inference side.
print("[sync] Packed IPC transfer FSDP → vLLM...")
ray.get([w.gather_and_broadcast_weights_ipc.remote() for w in fsdp_workers])
print("[sync] Weight transfer complete.")
print("[sync] Packed IPC transfer FSDP -> vLLM...")
ray.get([w.gather_and_broadcast_weights_ipc.remote() for w in fsdp_workers])
print("[sync] Weight transfer complete.")
print("[sync] Waking KV cache + scheduling...")
ray.get(inference_engine.wake_up.remote(tags=["kv_cache", "scheduling"]))
print("[sync] Waking KV cache + scheduling...")
wake_up_engine(BASE_URL, tags=["kv_cache", "scheduling"])
print("[generate] Generating with synced weights...")
outputs_updated = ray.get(
inference_engine.generate.remote(prompts, sampling_params)
)
print("-" * 60)
print("AFTER weight sync (real weights):")
print("-" * 60)
for output in outputs_updated:
print(f"Prompt: {output.prompt!r}")
print(f"Generated: {output.outputs[0].text!r}")
print("-" * 60)
print("[generate] Generating with synced weights...")
outputs_updated = generate_completions(client, PROMPTS)
print_generations("AFTER weight sync (real weights):", PROMPTS, outputs_updated)
finally:
print("[server] Shutting down...")
server_proc.terminate()
try:
server_proc.wait(timeout=30)
except subprocess.TimeoutExpired:
server_proc.kill()
if __name__ == "__main__":
-196
View File
@@ -1,196 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Demonstrates reinforcement learning using vLLM and Ray,
with native weight syncing APIs at engine instance.
The script separates training and inference workloads onto distinct GPUs
so that Ray can manage process placement and inter-process communication.
A Hugging Face Transformer model occupies one GPU for training, whereas a
2x tensor-parallel vLLM inference engine occupies two GPUs.
The example performs the following steps:
* Load the training model on one gpu (scheduled via ray)
* Initialize the inference model with dummy weights across
two gpus using vLLM's tensor parallelism and Ray placement groups.
* Generate gibberish from a list of prompts using the randomly initialized
inference engine.
* Update the weights of the training model and broadcast the updated weights
to the inference engine by using a Ray collective RPC group.
* Generating from the list of prompts after weight sync should result
in sensible outputs.
This example assumes a single-node cluster with three GPUs, but Ray
supports multi-node clusters. vLLM expects the GPUs are only used for vLLM
workloads. Residual GPU activity interferes with vLLM memory profiling and
causes unexpected behavior.
"""
import os
import ray
import torch
from ray.util.placement_group import placement_group
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
from transformers import AutoModelForCausalLM
from vllm import LLM, SamplingParams
from vllm.config import WeightTransferConfig
from vllm.distributed.weight_transfer import (
ModuleSource,
RayVLLMWeightSyncClient,
WeightTransferTrainerFactory,
)
from vllm.distributed.weight_transfer.nccl_engine import NCCLTrainerInitInfo
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_ip, get_open_port
MODEL_NAME = "facebook/opt-125m"
# MODEL_NAME = "inference-optimization/Qwen3-0.6B-W4A16-G128"
def get_assigned_gpu():
"""This is a temporary workaround for a runtime bug in RCCL on ROCm."""
if not current_platform.is_rocm():
return 0
assigned_gpu = int(ray.get_gpu_ids()[0])
os.environ.pop("CUDA_VISIBLE_DEVICES", None)
os.environ.pop("HIP_VISIBLE_DEVICES", None)
torch.accelerator.set_device_idx(assigned_gpu)
return assigned_gpu
class MyLLM(LLM):
"""Configure the vLLM worker for Ray placement group execution."""
def __init__(self, *args, **kwargs):
os.environ["VLLM_RAY_BUNDLE_INDICES"] = "0,1"
super().__init__(*args, **kwargs)
@ray.remote(num_gpus=1)
class TrainModel:
"""Ray actor that wraps the training model on a dedicated GPU."""
def __init__(self, model_name: str):
assigned_gpu = get_assigned_gpu()
self.model = AutoModelForCausalLM.from_pretrained(
model_name,
).to(f"cuda:{assigned_gpu}")
self.port = get_open_port()
self.master_address = get_ip()
def init_weight_transfer(self, world_size, llm_handle):
"""Build the trainer-side weight-transfer engine.
`trainer_init` drives the full handshake: it kicks off the inference
side's `init_weight_transfer_engine` (via the Ray client) on a worker
thread while opening the trainer's NCCL endpoint, so both ends
rendezvous together. After this returns, `send_weights()` is callable.
"""
self.engine = WeightTransferTrainerFactory.trainer_init(
init_info=NCCLTrainerInitInfo(
master_address=self.master_address,
master_port=self.port,
world_size=world_size,
rank=0, # single-GPU trainer is the sole (sender) rank
packed=True,
),
client=RayVLLMWeightSyncClient(llm_handle),
source=ModuleSource(self.model),
)
def broadcast_weights(self):
"""Push the current weights to the inference engine.
Drives start/update/finish on the inference side and the NCCL
broadcast internally — one call.
"""
self.engine.send_weights()
# Initialize Ray and set the visible devices. The vLLM engine will
# be placed on GPUs 1 and 2.
ray.init()
# Create a placement group that reserves GPU 1–2 for the vLLM inference engine.
# Learn more about Ray placement groups:
# https://docs.ray.io/en/latest/placement-groups.html
# Launch the training model actor. Ray's resource scheduler will allocate
# 1 GPU (via num_gpus=1 in the decorator), ensuring pg_inference gets different GPUs.
train_model = TrainModel.remote(MODEL_NAME)
pg_inference = placement_group([{"GPU": 1, "CPU": 0}] * 2)
ray.get(pg_inference.ready())
scheduling_inference = PlacementGroupSchedulingStrategy(
placement_group=pg_inference,
placement_group_capture_child_tasks=True,
placement_group_bundle_index=0,
)
# Launch the vLLM inference engine. The `enforce_eager` flag reduces
# start-up latency.
# Note: Weight transfer APIs (init_weight_transfer_engine, update_weights)
# are now native to vLLM workers.
llm = ray.remote(
num_cpus=0,
num_gpus=0,
scheduling_strategy=scheduling_inference,
)(MyLLM).remote(
model=MODEL_NAME,
enforce_eager=True,
tensor_parallel_size=2,
data_parallel_size=1,
distributed_executor_backend="ray",
weight_transfer_config=WeightTransferConfig(backend="nccl"),
load_format="dummy",
quantization="fp8",
)
# Generate text from the prompts.
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
sampling_params = SamplingParams(temperature=0)
outputs = ray.get(llm.generate.remote(prompts, sampling_params))
# Generate text with the initial model. The output is expected to be nonsense
# because the weights are randomly initialized.
print("-" * 50)
for output in outputs:
prompt = output.prompt
generated_text = output.outputs[0].text
print(f"Prompt: {prompt!r}\nGenerated text: {generated_text!r}")
print("-" * 50)
ray.get(llm.sleep.remote(level=0))
# Set up the communication channel between the training process and the
# inference engine. The trainer engine now owns the full handshake — the
# driver only has to hand it the vLLM actor handle and the world size.
world_size = ray.get(llm.get_world_size.remote()) + 1 # +1 for the trainer
ray.get(train_model.init_weight_transfer.remote(world_size, llm))
# Synchronize the updated weights to the inference engine. The engine drives
# start_weight_update / update_weights / finish_weight_update on the inference
# side internally, concurrent with the NCCL broadcast.
ray.get(train_model.broadcast_weights.remote())
ray.get(llm.wake_up.remote(tags=["scheduling"]))
# Generate text with the updated model. The output is expected to be normal
# because the weights are updated.
outputs_updated = ray.get(llm.generate.remote(prompts, sampling_params))
print("-" * 50)
for output in outputs_updated:
prompt = output.prompt
generated_text = output.outputs[0].text
print(f"Prompt: {prompt!r}\nGenerated text: {generated_text!r}")
print("-" * 50)