[Docs][RL] Rewrite weight-transfer docs; standardize examples (#51729)
Signed-off-by: haoaaron <ahao@anyscale.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
9b3382685e
commit
793ca6998a
+132
-108
@@ -1,45 +1,35 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Demonstrates reinforcement learning from human feedback (RLHF) using vLLM
|
||||
via HTTP API, with IPC-based weight syncing APIs.
|
||||
RLHF weight syncing against a `vllm serve` HTTP server, using CUDA IPC for the
|
||||
data plane.
|
||||
|
||||
Unlike rlhf_nccl.py which uses NCCL and can use separate GPUs, this script
|
||||
uses CUDA IPC which requires the training model and vLLM server to be on the
|
||||
same GPU. Memory must be carefully managed to fit both models.
|
||||
* OpenAI-compatible API for inference requests
|
||||
* HTTP endpoints for the weight-transfer control plane
|
||||
* CUDA IPC handles for the weight data plane
|
||||
|
||||
Unlike rlhf.py which creates a vLLM instance programmatically, this script
|
||||
assumes you have already started a vLLM server using `vllm serve`. It uses:
|
||||
- OpenAI-compatible API for inference requests
|
||||
- HTTP endpoints for weight transfer control plane
|
||||
- CUDA IPC for actual weight data transfer
|
||||
1-GPU layout (single node): IPC shares GPU memory directly, so the server (TP=1)
|
||||
and the training model both live on GPU 0. The server is started with
|
||||
`--gpu-memory-utilization 0.5` to leave room for the training model.
|
||||
|
||||
Prerequisites:
|
||||
Start a vLLM server with weight transfer enabled and reduced GPU memory
|
||||
utilization to leave room for the training model:
|
||||
The script starts the server itself, then:
|
||||
|
||||
$ VLLM_SERVER_DEV_MODE=1 VLLM_ALLOW_INSECURE_SERIALIZATION=1 \
|
||||
vllm serve facebook/opt-125m --enforce-eager \
|
||||
--weight-transfer-config '{"backend": "ipc"}' \
|
||||
--load-format dummy \
|
||||
--gpu-memory-utilization 0.5
|
||||
1. Generate over HTTP → gibberish (server started with dummy weights).
|
||||
2. Pause generation, sync real weights trainer → server over IPC, resume.
|
||||
3. Generate again → sensible output.
|
||||
|
||||
Then run this script:
|
||||
IPC handles are pickled for HTTP transport, so both sides need
|
||||
`VLLM_ALLOW_INSECURE_SERIALIZATION=1`; this script sets it for itself and for
|
||||
the server it spawns.
|
||||
|
||||
$ python rlhf_http_ipc.py
|
||||
|
||||
The example performs the following steps:
|
||||
|
||||
* Load the training model on GPU 0 (same GPU as the vLLM server).
|
||||
* Generate text using the vLLM server via OpenAI-compatible API. The output
|
||||
is expected to be nonsense because the server is initialized with dummy weights.
|
||||
* Initialize weight transfer via HTTP endpoint (no-op for IPC).
|
||||
* Broadcast the real weights from the training model to the vLLM server
|
||||
using CUDA IPC handles.
|
||||
* Generate text again to show normal output after the weight update.
|
||||
Run:
|
||||
$ python examples/rl/rlhf_http_ipc.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
import requests
|
||||
import torch
|
||||
@@ -53,12 +43,76 @@ from vllm.distributed.weight_transfer import (
|
||||
)
|
||||
from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo
|
||||
|
||||
BASE_URL = "http://localhost:8000"
|
||||
MODEL_NAME = "facebook/opt-125m"
|
||||
|
||||
# Enable insecure serialization for IPC handle serialization
|
||||
SERVER_PORT = 8000
|
||||
BASE_URL = f"http://localhost:{SERVER_PORT}"
|
||||
|
||||
# IPC requires colocation: the server and the training model share this GPU.
|
||||
SERVER_DEVICE_IDS = "0"
|
||||
TRAINER_DEVICE = "cuda:0"
|
||||
# Leave room on the shared GPU for the training model.
|
||||
SERVER_GPU_MEMORY_UTILIZATION = 0.5
|
||||
|
||||
# Needed to (de)serialize IPC handles across the HTTP boundary.
|
||||
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
|
||||
|
||||
PROMPTS = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
|
||||
|
||||
def start_vllm_server() -> subprocess.Popen:
|
||||
"""Spawn `vllm serve` and block until it is healthy."""
|
||||
serve_args = [
|
||||
"vllm",
|
||||
"serve",
|
||||
MODEL_NAME,
|
||||
"--tensor-parallel-size",
|
||||
"1",
|
||||
"--device-ids",
|
||||
SERVER_DEVICE_IDS,
|
||||
"--enforce-eager",
|
||||
"--load-format",
|
||||
"dummy",
|
||||
"--gpu-memory-utilization",
|
||||
str(SERVER_GPU_MEMORY_UTILIZATION),
|
||||
"--port",
|
||||
str(SERVER_PORT),
|
||||
"--weight-transfer-config",
|
||||
'{"backend": "ipc"}',
|
||||
]
|
||||
env = os.environ.copy()
|
||||
# Exposes the weight-transfer and pause/resume endpoints.
|
||||
env["VLLM_SERVER_DEV_MODE"] = "1"
|
||||
env["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
|
||||
print(f"[server] Launching: {' '.join(serve_args)}")
|
||||
proc = subprocess.Popen(
|
||||
serve_args,
|
||||
env=env,
|
||||
stdout=sys.stdout,
|
||||
stderr=sys.stderr,
|
||||
start_new_session=True,
|
||||
)
|
||||
|
||||
deadline = time.monotonic() + 900
|
||||
while True:
|
||||
if proc.poll() is not None:
|
||||
raise RuntimeError("vLLM server exited before becoming ready.")
|
||||
try:
|
||||
if requests.get(f"{BASE_URL}/health", timeout=5).status_code == 200:
|
||||
break
|
||||
except requests.RequestException:
|
||||
pass
|
||||
if time.monotonic() > deadline:
|
||||
raise RuntimeError("vLLM server failed to start in time.")
|
||||
time.sleep(2)
|
||||
print("[server] Ready.")
|
||||
return proc
|
||||
|
||||
|
||||
def generate_completions(client: OpenAI, model: str, prompts: list[str]) -> list[str]:
|
||||
"""Generate completions using the OpenAI-compatible API."""
|
||||
@@ -76,100 +130,70 @@ def generate_completions(client: OpenAI, model: str, prompts: list[str]) -> list
|
||||
|
||||
def pause_generation(base_url: str) -> None:
|
||||
"""Pause generation via HTTP endpoint."""
|
||||
url = f"{base_url}/pause"
|
||||
response = requests.post(url, timeout=60)
|
||||
response.raise_for_status()
|
||||
requests.post(f"{base_url}/pause", timeout=60).raise_for_status()
|
||||
|
||||
|
||||
def resume_generation(base_url: str) -> None:
|
||||
"""Resume generation via HTTP endpoint."""
|
||||
url = f"{base_url}/resume"
|
||||
response = requests.post(url, timeout=60)
|
||||
response.raise_for_status()
|
||||
requests.post(f"{base_url}/resume", timeout=60).raise_for_status()
|
||||
|
||||
|
||||
def get_world_size(base_url: str) -> int:
|
||||
"""Get world size from the vLLM server."""
|
||||
url = f"{base_url}/get_world_size"
|
||||
response = requests.get(url, timeout=10)
|
||||
response.raise_for_status()
|
||||
return response.json()["world_size"]
|
||||
|
||||
|
||||
def main():
|
||||
# IPC requires the training model to be on the same GPU as the vLLM server
|
||||
# The server should be started on GPU 0 with reduced memory utilization
|
||||
device = "cuda:0"
|
||||
torch.accelerator.set_device_index(device)
|
||||
|
||||
# Load the training model on the same GPU as the server
|
||||
# Use bfloat16 to reduce memory footprint
|
||||
print(f"Loading training model: {MODEL_NAME} on {device}")
|
||||
print(
|
||||
"Note: Ensure the vLLM server was started with --gpu-memory-utilization 0.5 "
|
||||
"or lower to leave room for the training model."
|
||||
)
|
||||
train_model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, dtype=torch.bfloat16)
|
||||
train_model.to(device)
|
||||
train_model.eval() # Set to eval mode to save memory
|
||||
|
||||
# Create OpenAI client pointing to the vLLM server
|
||||
client = OpenAI(
|
||||
base_url=f"{BASE_URL}/v1",
|
||||
api_key="EMPTY", # vLLM doesn't require an API key by default
|
||||
)
|
||||
|
||||
# Test prompts
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
|
||||
# Generate text before weight update. The output is expected to be nonsense
|
||||
# because the server is initialized with dummy weights.
|
||||
def print_generations(label: str, prompts: list[str], outputs: list[str]) -> None:
|
||||
print("-" * 50)
|
||||
print("Generating text BEFORE weight update (expect nonsense):")
|
||||
print(label)
|
||||
print("-" * 50)
|
||||
outputs = generate_completions(client, MODEL_NAME, prompts)
|
||||
for prompt, generated_text in zip(prompts, outputs):
|
||||
print(f"Prompt: {prompt!r}\nGenerated text: {generated_text!r}")
|
||||
print("-" * 50)
|
||||
|
||||
print("Initializing weight transfer (IPC backend)...")
|
||||
|
||||
# The trainer engine drives the inference side over HTTP. init for IPC is a
|
||||
# no-op rendezvous; the same client carries start/update/finish.
|
||||
engine = WeightTransferTrainerFactory.trainer_init(
|
||||
init_info=IPCTrainerInitInfo(rank=0, packed=False), # rank 0 = sender
|
||||
client=HTTPVLLMWeightSyncClient(BASE_URL),
|
||||
source=ModuleSource(train_model),
|
||||
)
|
||||
def main():
|
||||
server_proc = start_vllm_server()
|
||||
try:
|
||||
# The training model must sit on the same physical GPU as the server.
|
||||
torch.accelerator.set_device_index(TRAINER_DEVICE)
|
||||
|
||||
# Pause generation before weight sync
|
||||
pause_generation(BASE_URL)
|
||||
print(f"[trainer] Loading training model: {MODEL_NAME} on {TRAINER_DEVICE}")
|
||||
train_model = AutoModelForCausalLM.from_pretrained(
|
||||
MODEL_NAME, dtype=torch.bfloat16
|
||||
)
|
||||
train_model.to(TRAINER_DEVICE)
|
||||
train_model.eval() # eval mode to save memory on the shared GPU
|
||||
|
||||
print("Broadcasting weights via CUDA IPC (HTTP)...")
|
||||
# One call drives start_weight_update / update_weights / finish_weight_update.
|
||||
engine.send_weights()
|
||||
client = OpenAI(base_url=f"{BASE_URL}/v1", api_key="EMPTY")
|
||||
|
||||
# Resume generation after weight sync
|
||||
resume_generation(BASE_URL)
|
||||
# Generate with dummy weights — expect nonsense.
|
||||
outputs = generate_completions(client, MODEL_NAME, PROMPTS)
|
||||
print_generations("BEFORE weight sync (dummy weights):", PROMPTS, outputs)
|
||||
|
||||
# Generate text after weight update. The output is expected to be normal
|
||||
# because the real weights are now loaded.
|
||||
print("-" * 50)
|
||||
print("Generating text AFTER weight update:")
|
||||
print("-" * 50)
|
||||
outputs_updated = generate_completions(client, MODEL_NAME, prompts)
|
||||
for prompt, generated_text in zip(prompts, outputs_updated):
|
||||
print(f"Prompt: {prompt!r}\nGenerated text: {generated_text!r}")
|
||||
print("-" * 50)
|
||||
# IPC needs no data-plane rendezvous; `trainer_init` only ships the
|
||||
# `packed` flag, which the server must decode with.
|
||||
print("[transfer] Initializing IPC weight transfer...")
|
||||
engine = WeightTransferTrainerFactory.trainer_init(
|
||||
init_info=IPCTrainerInitInfo(rank=0, packed=False), # rank 0 = sender
|
||||
client=HTTPVLLMWeightSyncClient(BASE_URL),
|
||||
source=ModuleSource(train_model),
|
||||
)
|
||||
|
||||
# Note: The training model and IPC handles remain in memory.
|
||||
# In a real RLHF training loop, you would update the training model
|
||||
# and create new IPC handles for each weight update.
|
||||
pause_generation(BASE_URL)
|
||||
|
||||
# Drives start_weight_update / update_weights / finish_weight_update.
|
||||
print("[sync] Sharing weights via CUDA IPC...")
|
||||
engine.send_weights()
|
||||
print("[sync] Weight transfer complete.")
|
||||
|
||||
resume_generation(BASE_URL)
|
||||
|
||||
# Generate with the synced weights — expect sensible output.
|
||||
outputs_updated = generate_completions(client, MODEL_NAME, PROMPTS)
|
||||
print_generations("AFTER weight sync (real weights):", PROMPTS, outputs_updated)
|
||||
finally:
|
||||
print("[server] Shutting down...")
|
||||
server_proc.terminate()
|
||||
try:
|
||||
server_proc.wait(timeout=30)
|
||||
except subprocess.TimeoutExpired:
|
||||
server_proc.kill()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+146
-102
@@ -1,38 +1,33 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Demonstrates reinforcement learning from human feedback (RLHF) using vLLM
|
||||
via HTTP API, with native weight syncing APIs.
|
||||
RLHF weight syncing against a `vllm serve` HTTP server, using NCCL for the
|
||||
data plane.
|
||||
|
||||
Unlike rlhf.py which creates a vLLM instance programmatically, this script
|
||||
assumes you have already started a vLLM server using `vllm serve`. It uses:
|
||||
- OpenAI-compatible API for inference requests
|
||||
- HTTP endpoints for weight transfer control plane
|
||||
- NCCL for actual weight data transfer
|
||||
* OpenAI-compatible API for inference requests
|
||||
* HTTP endpoints for the weight-transfer control plane
|
||||
* NCCL for the weight data plane
|
||||
|
||||
Prerequisites:
|
||||
Start a vLLM server with weight transfer enabled:
|
||||
3-GPU layout (single node):
|
||||
Inference — GPUs 0-1, `vllm serve` with TP=2 and fp8 quantization
|
||||
Training — GPU 2, a bf16 Hugging Face model in this process
|
||||
(the server quantizes to fp8 as it loads)
|
||||
|
||||
$ VLLM_SERVER_DEV_MODE=1 vllm serve facebook/opt-125m \
|
||||
--enforce-eager \
|
||||
--weight-transfer-config '{"backend": "nccl"}' \
|
||||
--load-format dummy
|
||||
The script starts the server itself, then:
|
||||
|
||||
Then run this script:
|
||||
1. Generate over HTTP → gibberish (server started with dummy weights).
|
||||
2. Pause generation, sync real weights trainer → server over NCCL, resume.
|
||||
3. Generate again → sensible output.
|
||||
|
||||
$ python rlhf_http.py
|
||||
|
||||
The example performs the following steps:
|
||||
|
||||
* Load the training model on GPU 0.
|
||||
* Generate text using the vLLM server via OpenAI-compatible API. The output
|
||||
is expected to be nonsense because the server is initialized with dummy weights.
|
||||
* Initialize weight transfer via HTTP endpoint.
|
||||
* Broadcast the real weights from the training model to the vLLM server
|
||||
using NCCL.
|
||||
* Generate text again to show normal output after the weight update.
|
||||
Run:
|
||||
$ python examples/rl/rlhf_http_nccl.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
import requests
|
||||
import torch
|
||||
from openai import OpenAI
|
||||
@@ -46,9 +41,71 @@ from vllm.distributed.weight_transfer import (
|
||||
from vllm.distributed.weight_transfer.nccl_engine import NCCLTrainerInitInfo
|
||||
from vllm.utils.network_utils import get_ip, get_open_port
|
||||
|
||||
BASE_URL = "http://localhost:8000"
|
||||
MODEL_NAME = "facebook/opt-125m"
|
||||
|
||||
SERVER_PORT = 8000
|
||||
BASE_URL = f"http://localhost:{SERVER_PORT}"
|
||||
|
||||
INFERENCE_TP_SIZE = 2
|
||||
# Physical GPUs for the server; the trainer takes the next one.
|
||||
SERVER_DEVICE_IDS = "0,1"
|
||||
TRAINER_DEVICE = "cuda:2"
|
||||
|
||||
PROMPTS = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
|
||||
|
||||
def start_vllm_server() -> subprocess.Popen:
|
||||
"""Spawn `vllm serve` and block until it is healthy."""
|
||||
serve_args = [
|
||||
"vllm",
|
||||
"serve",
|
||||
MODEL_NAME,
|
||||
"--tensor-parallel-size",
|
||||
str(INFERENCE_TP_SIZE),
|
||||
"--device-ids",
|
||||
SERVER_DEVICE_IDS,
|
||||
"--quantization",
|
||||
"fp8",
|
||||
"--enforce-eager",
|
||||
"--load-format",
|
||||
"dummy",
|
||||
"--port",
|
||||
str(SERVER_PORT),
|
||||
"--weight-transfer-config",
|
||||
'{"backend": "nccl"}',
|
||||
]
|
||||
env = os.environ.copy()
|
||||
# Exposes the weight-transfer and pause/resume endpoints.
|
||||
env["VLLM_SERVER_DEV_MODE"] = "1"
|
||||
print(f"[server] Launching: {' '.join(serve_args)}")
|
||||
proc = subprocess.Popen(
|
||||
serve_args,
|
||||
env=env,
|
||||
stdout=sys.stdout,
|
||||
stderr=sys.stderr,
|
||||
start_new_session=True,
|
||||
)
|
||||
|
||||
deadline = time.monotonic() + 900
|
||||
while True:
|
||||
if proc.poll() is not None:
|
||||
raise RuntimeError("vLLM server exited before becoming ready.")
|
||||
try:
|
||||
if requests.get(f"{BASE_URL}/health", timeout=5).status_code == 200:
|
||||
break
|
||||
except requests.RequestException:
|
||||
pass
|
||||
if time.monotonic() > deadline:
|
||||
raise RuntimeError("vLLM server failed to start in time.")
|
||||
time.sleep(2)
|
||||
print("[server] Ready.")
|
||||
return proc
|
||||
|
||||
|
||||
def generate_completions(client: OpenAI, model: str, prompts: list[str]) -> list[str]:
|
||||
"""Generate completions using the OpenAI-compatible API."""
|
||||
@@ -66,105 +123,92 @@ def generate_completions(client: OpenAI, model: str, prompts: list[str]) -> list
|
||||
|
||||
def pause_generation(base_url: str) -> None:
|
||||
"""Pause generation via HTTP endpoint."""
|
||||
url = f"{base_url}/pause"
|
||||
response = requests.post(url, timeout=60)
|
||||
response.raise_for_status()
|
||||
requests.post(f"{base_url}/pause", timeout=60).raise_for_status()
|
||||
|
||||
|
||||
def resume_generation(base_url: str) -> None:
|
||||
"""Resume generation via HTTP endpoint."""
|
||||
url = f"{base_url}/resume"
|
||||
response = requests.post(url, timeout=60)
|
||||
response.raise_for_status()
|
||||
requests.post(f"{base_url}/resume", timeout=60).raise_for_status()
|
||||
|
||||
|
||||
def get_world_size(base_url: str) -> int:
|
||||
"""Get world size from the vLLM server."""
|
||||
url = f"{base_url}/get_world_size"
|
||||
response = requests.get(url, timeout=10)
|
||||
"""Get the number of inference workers from the vLLM server."""
|
||||
response = requests.get(f"{base_url}/get_world_size", timeout=10)
|
||||
response.raise_for_status()
|
||||
return response.json()["world_size"]
|
||||
|
||||
|
||||
def main():
|
||||
# Get the inference world size from the vLLM server
|
||||
inference_world_size = get_world_size(BASE_URL)
|
||||
world_size = inference_world_size + 1 # +1 for the trainer
|
||||
device = f"cuda:{inference_world_size}"
|
||||
torch.accelerator.set_device_index(device)
|
||||
|
||||
# Load the training model
|
||||
print(f"Loading training model: {MODEL_NAME}")
|
||||
train_model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, dtype=torch.bfloat16)
|
||||
train_model.to(device)
|
||||
|
||||
# Create OpenAI client pointing to the vLLM server
|
||||
client = OpenAI(
|
||||
base_url=f"{BASE_URL}/v1",
|
||||
api_key="EMPTY", # vLLM doesn't require an API key by default
|
||||
)
|
||||
|
||||
# Test prompts
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
|
||||
# Generate text before weight update. The output is expected to be nonsense
|
||||
# because the server is initialized with dummy weights.
|
||||
def print_generations(label: str, prompts: list[str], outputs: list[str]) -> None:
|
||||
print("-" * 50)
|
||||
print("Generating text BEFORE weight update (expect nonsense):")
|
||||
print(label)
|
||||
print("-" * 50)
|
||||
outputs = generate_completions(client, MODEL_NAME, prompts)
|
||||
for prompt, generated_text in zip(prompts, outputs):
|
||||
print(f"Prompt: {prompt!r}\nGenerated text: {generated_text!r}")
|
||||
print("-" * 50)
|
||||
|
||||
# Set up the communication channel between the training process and the
|
||||
# vLLM server. The trainer is rank 0, vLLM worker(s) start after it.
|
||||
master_address = get_ip()
|
||||
master_port = get_open_port()
|
||||
|
||||
print(f"Initializing weight transfer: master={master_address}:{master_port}")
|
||||
def main():
|
||||
server_proc = start_vllm_server()
|
||||
try:
|
||||
# The trainer sits on the GPU after the server's, and is NCCL rank 0.
|
||||
torch.accelerator.set_device_index(TRAINER_DEVICE)
|
||||
|
||||
# The trainer engine owns the full handshake. `trainer_init` kicks off the
|
||||
# server's init_weight_transfer_engine (via the HTTP client) on a worker
|
||||
# thread while opening the trainer's NCCL endpoint, so both ends rendezvous
|
||||
# together — no manual threading needed here.
|
||||
engine = WeightTransferTrainerFactory.trainer_init(
|
||||
init_info=NCCLTrainerInitInfo(
|
||||
master_address=master_address,
|
||||
master_port=master_port,
|
||||
world_size=world_size,
|
||||
rank=0, # single-GPU trainer is the sole (sender) rank
|
||||
packed=True,
|
||||
),
|
||||
client=HTTPVLLMWeightSyncClient(BASE_URL),
|
||||
source=ModuleSource(train_model),
|
||||
)
|
||||
print(f"[trainer] Loading training model: {MODEL_NAME} on {TRAINER_DEVICE}")
|
||||
train_model = AutoModelForCausalLM.from_pretrained(
|
||||
MODEL_NAME, dtype=torch.bfloat16
|
||||
)
|
||||
train_model.to(TRAINER_DEVICE)
|
||||
|
||||
# Pause generation before weight sync
|
||||
pause_generation(BASE_URL)
|
||||
client = OpenAI(base_url=f"{BASE_URL}/v1", api_key="EMPTY")
|
||||
|
||||
# One call drives start_weight_update / update_weights / finish_weight_update
|
||||
# over HTTP, concurrent with the NCCL broadcast.
|
||||
print("Broadcasting weights via NCCL...")
|
||||
engine.send_weights()
|
||||
# Generate with dummy weights — expect nonsense.
|
||||
outputs = generate_completions(client, MODEL_NAME, PROMPTS)
|
||||
print_generations("BEFORE weight sync (dummy weights):", PROMPTS, outputs)
|
||||
|
||||
# Resume generation after weight sync
|
||||
resume_generation(BASE_URL)
|
||||
# The transfer NCCL group is the trainer plus every inference worker.
|
||||
world_size = get_world_size(BASE_URL) + 1
|
||||
master_address = get_ip()
|
||||
master_port = get_open_port()
|
||||
print(
|
||||
f"[transfer] Rendezvous at {master_address}:{master_port}, "
|
||||
f"world_size={world_size} (1 trainer + {world_size - 1} vLLM workers)"
|
||||
)
|
||||
|
||||
# Generate text after weight update. The output is expected to be normal
|
||||
# because the real weights are now loaded.
|
||||
print("-" * 50)
|
||||
print("Generating text AFTER weight update:")
|
||||
print("-" * 50)
|
||||
outputs_updated = generate_completions(client, MODEL_NAME, prompts)
|
||||
for prompt, generated_text in zip(prompts, outputs_updated):
|
||||
print(f"Prompt: {prompt!r}\nGenerated text: {generated_text!r}")
|
||||
print("-" * 50)
|
||||
# `trainer_init` drives the handshake: it initializes the server's
|
||||
# transfer engine while opening the trainer's own NCCL endpoint, so both
|
||||
# ends rendezvous together.
|
||||
engine = WeightTransferTrainerFactory.trainer_init(
|
||||
init_info=NCCLTrainerInitInfo(
|
||||
master_address=master_address,
|
||||
master_port=master_port,
|
||||
world_size=world_size,
|
||||
rank=0, # single-GPU trainer is the sole (sender) rank
|
||||
packed=True,
|
||||
),
|
||||
client=HTTPVLLMWeightSyncClient(BASE_URL),
|
||||
source=ModuleSource(train_model),
|
||||
)
|
||||
|
||||
pause_generation(BASE_URL)
|
||||
|
||||
# Drives start_weight_update / update_weights / finish_weight_update,
|
||||
# concurrent with the NCCL broadcast.
|
||||
print("[sync] Broadcasting weights via NCCL...")
|
||||
engine.send_weights()
|
||||
print("[sync] Weight broadcast complete.")
|
||||
|
||||
resume_generation(BASE_URL)
|
||||
|
||||
# Generate with the synced weights — expect sensible output.
|
||||
outputs_updated = generate_completions(client, MODEL_NAME, PROMPTS)
|
||||
print_generations("AFTER weight sync (real weights):", PROMPTS, outputs_updated)
|
||||
finally:
|
||||
print("[server] Shutting down...")
|
||||
server_proc.terminate()
|
||||
try:
|
||||
server_proc.wait(timeout=30)
|
||||
except subprocess.TimeoutExpired:
|
||||
server_proc.kill()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,151 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Demonstrates reinforcement learning from human feedback (RLHF) using vLLM and Ray,
|
||||
with IPC-based weight syncing APIs
|
||||
|
||||
The script colocates the training and inference workloads onto the same GPU using Ray.
|
||||
|
||||
The example performs the following steps:
|
||||
|
||||
* Request a placement group of 1 GPU.
|
||||
* Place the inference model on the above GPU using the placement group.
|
||||
* Place and load the training model on the same GPU using the placement group.
|
||||
* Generate text from a list of prompts using the inference engine.
|
||||
* Update the weights of the training model and broadcast the updated weights
|
||||
to the inference engine by using CUDA IPC handles. Note that
|
||||
for demonstration purposes we simply zero out the weights.
|
||||
|
||||
This example assumes a single-node cluster with a single GPU,
|
||||
but can be extended to multiple GPUs.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import ray
|
||||
from ray.util.placement_group import placement_group
|
||||
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
|
||||
from transformers import AutoModelForCausalLM
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.config import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer import (
|
||||
ModuleSource,
|
||||
RayVLLMWeightSyncClient,
|
||||
WeightTransferTrainerFactory,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo
|
||||
|
||||
|
||||
class MyLLM(LLM):
|
||||
"""Configure the vLLM worker for Ray placement group execution."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
# Remove the top-level CUDA_VISIBLE_DEVICES variable set by Ray
|
||||
# so that vLLM can manage its own device placement within the worker.
|
||||
os.environ.pop("CUDA_VISIBLE_DEVICES", None)
|
||||
# Each worker uses 0.4 GPU so that two instances fit on the same GPU.
|
||||
os.environ["VLLM_RAY_PER_WORKER_GPUS"] = "0.4"
|
||||
os.environ["VLLM_RAY_BUNDLE_INDICES"] = "0"
|
||||
# needed for ipc handle serialization
|
||||
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
|
||||
# Load the OPT-125M model onto GPU 0 for the training workload.
|
||||
|
||||
MODEL_NAME = "facebook/opt-125m"
|
||||
|
||||
|
||||
@ray.remote
|
||||
class TrainModel:
|
||||
def __init__(self, llm_handle: ray.actor.ActorHandle):
|
||||
self.train_model = AutoModelForCausalLM.from_pretrained(
|
||||
MODEL_NAME,
|
||||
)
|
||||
self.train_model.to("cuda:0")
|
||||
self.llm_handle = llm_handle
|
||||
|
||||
def init_weight_transfer(self):
|
||||
"""Build the trainer-side IPC engine (no rendezvous needed for IPC)."""
|
||||
self.engine = WeightTransferTrainerFactory.trainer_init(
|
||||
init_info=IPCTrainerInitInfo(rank=0, packed=False), # rank 0 = sender
|
||||
client=RayVLLMWeightSyncClient(self.llm_handle),
|
||||
source=ModuleSource(self.train_model),
|
||||
)
|
||||
|
||||
def broadcast_weights(self):
|
||||
"""Broadcast weights to the inference engine using IPC.
|
||||
|
||||
Drives start/update/finish on the inference side internally.
|
||||
"""
|
||||
self.engine.send_weights()
|
||||
|
||||
|
||||
ray.init()
|
||||
|
||||
pg_colocate = placement_group([{"GPU": 1, "CPU": 0}])
|
||||
ray.get(pg_colocate.ready())
|
||||
|
||||
|
||||
llm = ray.remote(
|
||||
num_cpus=0,
|
||||
num_gpus=0,
|
||||
scheduling_strategy=PlacementGroupSchedulingStrategy(
|
||||
placement_group=pg_colocate,
|
||||
placement_group_capture_child_tasks=True,
|
||||
),
|
||||
)(MyLLM).remote(
|
||||
model=MODEL_NAME,
|
||||
enforce_eager=True,
|
||||
tensor_parallel_size=1,
|
||||
distributed_executor_backend="ray",
|
||||
gpu_memory_utilization=0.7,
|
||||
weight_transfer_config=WeightTransferConfig(backend="ipc"),
|
||||
load_format="dummy",
|
||||
)
|
||||
|
||||
train_model = TrainModel.options(
|
||||
num_gpus=0.1,
|
||||
num_cpus=0,
|
||||
scheduling_strategy=PlacementGroupSchedulingStrategy(
|
||||
placement_group=pg_colocate, placement_group_capture_child_tasks=True
|
||||
),
|
||||
).remote(llm)
|
||||
|
||||
|
||||
# Generate text from the prompts.
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
|
||||
sampling_params = SamplingParams(temperature=0)
|
||||
|
||||
outputs = ray.get(llm.generate.remote(prompts, sampling_params))
|
||||
|
||||
print("-" * 50)
|
||||
for output in outputs:
|
||||
prompt = output.prompt
|
||||
generated_text = output.outputs[0].text
|
||||
print(f"Prompt: {prompt!r}\nGenerated text: {generated_text!r}")
|
||||
print("-" * 50)
|
||||
|
||||
ray.get(llm.sleep.remote(level=0))
|
||||
|
||||
ray.get(train_model.init_weight_transfer.remote())
|
||||
# One call drives start_weight_update / update_weights / finish_weight_update.
|
||||
ray.get(train_model.broadcast_weights.remote())
|
||||
|
||||
ray.get(llm.wake_up.remote(tags=["scheduling"]))
|
||||
|
||||
outputs_packed = ray.get(llm.generate.remote(prompts, sampling_params))
|
||||
print("-" * 50)
|
||||
print("Results after packed/chunked IPC weight sync:")
|
||||
for output in outputs_packed:
|
||||
prompt = output.prompt
|
||||
generated_text = output.outputs[0].text
|
||||
print(f"Prompt: {prompt!r}\nGenerated text: {generated_text!r}")
|
||||
print("-" * 50)
|
||||
+181
-232
@@ -4,96 +4,88 @@
|
||||
RLHF with FSDP2 training and vLLM expert-parallel inference using **CUDA IPC**
|
||||
weight transfer and **packed** tensors.
|
||||
|
||||
Layout (4 GPUs, TP=1, DP=4, EP):
|
||||
* One Ray placement group per GPU.
|
||||
* Each PG holds one FSDP training worker and one vLLM ``LLM`` instance
|
||||
(sync API) using fractional GPUs so both fit on the same device.
|
||||
* The 4 ``LLM`` instances form a DP group via env-var-based SPMD
|
||||
coordination (``VLLM_DP_RANK``, ``VLLM_DP_SIZE``, etc.), the same
|
||||
mechanism used by ``examples/offline_inference/data_parallel.py``.
|
||||
* A ``DataParallelInferenceEngine`` actor spawns all 4 LLM actors,
|
||||
waits for initialization, and orchestrates generation / weight-sync.
|
||||
Multi-rank version of `rlhf_http_ipc.py`: the trainer is 4 FSDP2 Ray actors
|
||||
colocated with a data-parallel `vllm serve` on the same 4 physical GPUs.
|
||||
|
||||
4-GPU layout (single node), all colocated:
|
||||
Training — 4 GPUs, PyTorch FSDP2 (fully_shard), as Ray actors
|
||||
Inference — the same 4 GPUs, `vllm serve --data-parallel-size 4 -tp 1
|
||||
--enable-expert-parallel` (EP_SIZE = TP x DP = 4)
|
||||
|
||||
IPC requires the trainer and the server to sit on the same GPUs, so the script
|
||||
reserves the training GPUs through Ray first, asks Ray which ones it got, and
|
||||
pins the server to exactly those with `--device-ids`.
|
||||
|
||||
Both sides share each GPU, so the server is capped with
|
||||
`--gpu-memory-utilization` and its weights are moved aside for the transfer:
|
||||
|
||||
1. `/sleep?level=1` — offload server weights to CPU, drop KV cache
|
||||
2. `/wake_up?tags=weights` — weights back on GPU, KV cache still free
|
||||
3. packed IPC transfer — overwrite weights with room to spare
|
||||
4. `/wake_up?tags=kv_cache&tags=scheduling` — re-allocate KV cache, resume
|
||||
|
||||
Every FSDP rank builds an ``IPCTrainerWeightTransferEngine`` (via ``trainer_init``)
|
||||
and calls ``send_weights()``; all ranks join the IPC handle all-gather, and only
|
||||
rank 0 (the sender) ships the merged handles and drives the DP LLM actors through
|
||||
its ``RayVLLMWeightSyncClient``.
|
||||
rank 0 (the sender) ships the merged handles and drives the server.
|
||||
|
||||
This example was run on 4xH100.
|
||||
|
||||
Run:
|
||||
$ python examples/rl/rlhf_ipc_fsdp_ep.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
import ray
|
||||
import requests
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from huggingface_hub import snapshot_download
|
||||
from ray.util.placement_group import placement_group
|
||||
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
|
||||
from openai import OpenAI
|
||||
from torch.distributed.fsdp import fully_shard
|
||||
from transformers import AutoModelForCausalLM
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.config import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer import (
|
||||
HTTPVLLMWeightSyncClient,
|
||||
ModuleSource,
|
||||
RayVLLMWeightSyncClient,
|
||||
WeightTransferTrainerFactory,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo
|
||||
from vllm.utils.network_utils import get_ip, get_open_port
|
||||
|
||||
# The inference side only needs the backend; the packed wire params are
|
||||
# trainer-side and get propagated to the workers at the init handshake.
|
||||
WEIGHT_TRANSFER_CONFIG = WeightTransferConfig(backend="ipc")
|
||||
# Packed IPC transfer with a 1 GB buffer (matches the per-chunk buffer size).
|
||||
PACKED = True
|
||||
PACKED_BUFFER_SIZE_BYTES = 1024 * 1024 * 1024
|
||||
|
||||
TRAIN_GPU_FRACTION = float(os.environ.get("RLHF_IPC_TRAIN_GPU_FRACTION", "0.42"))
|
||||
VLLM_GPU_FRACTION = float(os.environ.get("RLHF_IPC_VLLM_GPU_FRACTION", "0.42"))
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-30B-A3B"
|
||||
SERVED_MODEL_NAME = "policy"
|
||||
|
||||
FSDP_WORLD_SIZE = 4
|
||||
INFERENCE_TP_SIZE = 1
|
||||
INFERENCE_DP_SIZE = 4
|
||||
|
||||
# Packed IPC transfer with a 1 GB chunk buffer.
|
||||
PACKED = True
|
||||
PACKED_BUFFER_SIZE_BYTES = 1024 * 1024 * 1024
|
||||
|
||||
class MyLLM(LLM):
|
||||
"""LLM subclass that configures DP env vars for SPMD coordination."""
|
||||
# The server shares each GPU with a training rank, so cap what it reserves.
|
||||
SERVER_GPU_MEMORY_UTILIZATION = 0.35
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*args,
|
||||
dp_rank: int = 0,
|
||||
dp_size: int = 1,
|
||||
dp_master_ip: str = "127.0.0.1",
|
||||
dp_master_port: int = 0,
|
||||
**kwargs,
|
||||
):
|
||||
os.environ.pop("CUDA_VISIBLE_DEVICES", None)
|
||||
os.environ["VLLM_RAY_PER_WORKER_GPUS"] = str(VLLM_GPU_FRACTION)
|
||||
os.environ["VLLM_RAY_BUNDLE_INDICES"] = "0"
|
||||
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
|
||||
SERVER_PORT = 8000
|
||||
BASE_URL = f"http://localhost:{SERVER_PORT}"
|
||||
|
||||
os.environ["VLLM_DP_RANK"] = str(dp_rank)
|
||||
os.environ["VLLM_DP_RANK_LOCAL"] = str(dp_rank)
|
||||
os.environ["VLLM_DP_SIZE"] = str(dp_size)
|
||||
os.environ["VLLM_DP_MASTER_IP"] = dp_master_ip
|
||||
os.environ["VLLM_DP_MASTER_PORT"] = str(dp_master_port)
|
||||
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def ready(self):
|
||||
return True
|
||||
PROMPTS = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
|
||||
|
||||
@ray.remote(num_cpus=0, num_gpus=TRAIN_GPU_FRACTION)
|
||||
@ray.remote(num_gpus=1)
|
||||
class FSDPTrainWorker:
|
||||
"""One FSDP2 worker per GPU; colocated with vLLM DP rank via placement group."""
|
||||
"""One FSDP2 worker per GPU; colocated with one vLLM DP rank."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -115,12 +107,6 @@ class FSDPTrainWorker:
|
||||
model_name, torch_dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
self.weight_names = [n for n, _ in model.named_parameters()]
|
||||
self.weight_dtype_names = [
|
||||
str(p.dtype).split(".")[-1] for _, p in model.named_parameters()
|
||||
]
|
||||
self.weight_shapes = [list(p.shape) for _, p in model.named_parameters()]
|
||||
|
||||
for layer in model.model.layers:
|
||||
fully_shard(layer)
|
||||
fully_shard(model)
|
||||
@@ -130,237 +116,200 @@ class FSDPTrainWorker:
|
||||
def get_rank(self):
|
||||
return self.rank
|
||||
|
||||
def get_weight_metadata(self):
|
||||
return self.weight_names, self.weight_dtype_names, self.weight_shapes
|
||||
def get_gpu_ids(self):
|
||||
"""Physical GPU id(s) Ray assigned to this worker."""
|
||||
return ray.get_gpu_ids()
|
||||
|
||||
def setup_engine(self, llm_handles):
|
||||
"""Build the trainer IPC engine on every FSDP rank.
|
||||
|
||||
Called on all ranks: rank 0 becomes the sender (drives the inference
|
||||
side and performs the no-op IPC init handshake against all DP LLM
|
||||
actors); the other ranks hold a null-client engine and only join the
|
||||
IPC handle all-gather during send_weights.
|
||||
"""
|
||||
def setup_engine(self, base_url: str):
|
||||
"""Build the trainer IPC engine. Called on every FSDP rank."""
|
||||
self.engine = WeightTransferTrainerFactory.trainer_init(
|
||||
init_info=IPCTrainerInitInfo(
|
||||
rank=self.rank, # FSDP rank; sender is 0
|
||||
packed=PACKED,
|
||||
packed_buffer_size_bytes=PACKED_BUFFER_SIZE_BYTES,
|
||||
),
|
||||
client=RayVLLMWeightSyncClient(llm_handles),
|
||||
client=HTTPVLLMWeightSyncClient(base_url),
|
||||
source=ModuleSource(self.model),
|
||||
)
|
||||
|
||||
def gather_and_broadcast_weights_ipc(self):
|
||||
"""All-gather full params across FSDP ranks; rank 0 sends to vLLM.
|
||||
|
||||
Called on all ranks concurrently. `send_weights` gathers each param and
|
||||
contributes to the IPC handle all-gather on every rank; only rank 0 (the
|
||||
sender) drives start/update/finish on the inference side (the other
|
||||
ranks' client RPCs no-op).
|
||||
"""
|
||||
"""Send the current weights to vLLM. Called on every FSDP rank."""
|
||||
self.engine.send_weights()
|
||||
|
||||
|
||||
@ray.remote(num_cpus=1)
|
||||
class DataParallelInferenceEngine:
|
||||
"""Manages a pool of DP-sharded vLLM LLM actors.
|
||||
def start_vllm_server(model_path: str, device_ids: str) -> subprocess.Popen:
|
||||
"""Spawn a `vllm serve` HTTP server (DP+EP) pinned to `device_ids`."""
|
||||
serve_args = [
|
||||
"vllm",
|
||||
"serve",
|
||||
model_path,
|
||||
"--served-model-name",
|
||||
SERVED_MODEL_NAME,
|
||||
"--tensor-parallel-size",
|
||||
str(INFERENCE_TP_SIZE),
|
||||
"--data-parallel-size",
|
||||
str(INFERENCE_DP_SIZE),
|
||||
"--enable-expert-parallel",
|
||||
# Pins the server to the same physical GPUs as the training ranks.
|
||||
"--device-ids",
|
||||
device_ids,
|
||||
"--enable-sleep-mode",
|
||||
"--enforce-eager",
|
||||
"--load-format",
|
||||
"dummy",
|
||||
"--gpu-memory-utilization",
|
||||
str(SERVER_GPU_MEMORY_UTILIZATION),
|
||||
"--port",
|
||||
str(SERVER_PORT),
|
||||
"--weight-transfer-config",
|
||||
'{"backend": "ipc"}',
|
||||
]
|
||||
env = os.environ.copy()
|
||||
env["VLLM_SERVER_DEV_MODE"] = "1" # exposes weight-transfer + sleep endpoints
|
||||
env["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1" # IPC handles over HTTP
|
||||
env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
print(f"[server] Launching: {' '.join(serve_args)} (GPUs {device_ids})")
|
||||
proc = subprocess.Popen(
|
||||
serve_args,
|
||||
env=env,
|
||||
stdout=sys.stdout,
|
||||
stderr=sys.stderr,
|
||||
start_new_session=True,
|
||||
)
|
||||
|
||||
Spawns one MyLLM actor per placement group, waits for all engines to
|
||||
finish initializing, and exposes generation / weight-sync helpers.
|
||||
"""
|
||||
# Wait for the server to come up (model load can take a while).
|
||||
deadline = time.monotonic() + 1800
|
||||
while True:
|
||||
if proc.poll() is not None:
|
||||
raise RuntimeError("vLLM server exited before becoming ready.")
|
||||
try:
|
||||
if requests.get(f"{BASE_URL}/health", timeout=5).status_code == 200:
|
||||
break
|
||||
except requests.RequestException:
|
||||
pass
|
||||
if time.monotonic() > deadline:
|
||||
raise RuntimeError("vLLM server failed to start in time.")
|
||||
time.sleep(2)
|
||||
print("[server] Ready.")
|
||||
return proc
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: str,
|
||||
pgs: list,
|
||||
dp_master_ip: str,
|
||||
dp_master_port: int,
|
||||
):
|
||||
dp_size = len(pgs)
|
||||
self.llm_actors = []
|
||||
for r in range(dp_size):
|
||||
sched = PlacementGroupSchedulingStrategy(
|
||||
placement_group=pgs[r],
|
||||
placement_group_capture_child_tasks=True,
|
||||
)
|
||||
actor = (
|
||||
ray.remote(num_cpus=0, num_gpus=0)(MyLLM)
|
||||
.options(scheduling_strategy=sched)
|
||||
.remote(
|
||||
model=model,
|
||||
enforce_eager=True,
|
||||
tensor_parallel_size=INFERENCE_TP_SIZE,
|
||||
distributed_executor_backend="ray",
|
||||
enable_expert_parallel=True,
|
||||
gpu_memory_utilization=0.35,
|
||||
weight_transfer_config=WEIGHT_TRANSFER_CONFIG,
|
||||
enable_sleep_mode=True,
|
||||
load_format="dummy",
|
||||
dp_rank=r,
|
||||
dp_size=dp_size,
|
||||
dp_master_ip=dp_master_ip,
|
||||
dp_master_port=dp_master_port,
|
||||
)
|
||||
)
|
||||
self.llm_actors.append(actor)
|
||||
|
||||
ray.get([actor.ready.remote() for actor in self.llm_actors])
|
||||
def generate_completions(client: OpenAI, prompts: list[str]) -> list[str]:
|
||||
"""Generate completions via the OpenAI HTTP API."""
|
||||
results = []
|
||||
for prompt in prompts:
|
||||
response = client.completions.create(
|
||||
model=SERVED_MODEL_NAME,
|
||||
prompt=prompt,
|
||||
max_tokens=32,
|
||||
temperature=0,
|
||||
)
|
||||
results.append(response.choices[0].text)
|
||||
return results
|
||||
|
||||
def get_llm_actors(self):
|
||||
return self.llm_actors
|
||||
|
||||
def generate(self, prompts: list[str], sampling_params):
|
||||
"""Distribute prompts round-robin across DP ranks and collect results."""
|
||||
dp_size = len(self.llm_actors)
|
||||
per_rank: list[list[str]] = [[] for _ in range(dp_size)]
|
||||
indices: list[list[int]] = [[] for _ in range(dp_size)]
|
||||
def sleep_engine(base_url: str, level: int) -> None:
|
||||
"""Put the engine to sleep (level 1 offloads weights, drops KV cache)."""
|
||||
response = requests.post(f"{base_url}/sleep", params={"level": level}, timeout=600)
|
||||
response.raise_for_status()
|
||||
|
||||
for i, prompt in enumerate(prompts):
|
||||
rank = i % dp_size
|
||||
per_rank[rank].append(prompt)
|
||||
indices[rank].append(i)
|
||||
|
||||
refs = [
|
||||
actor.generate.remote(per_rank[r], sampling_params)
|
||||
for r, actor in enumerate(self.llm_actors)
|
||||
if per_rank[r]
|
||||
]
|
||||
all_outputs = ray.get(refs)
|
||||
def wake_up_engine(base_url: str, tags: list[str] | None = None) -> None:
|
||||
"""Wake the engine, optionally only for specific memory tags."""
|
||||
params = [("tags", tag) for tag in tags] if tags else None
|
||||
response = requests.post(f"{base_url}/wake_up", params=params, timeout=600)
|
||||
response.raise_for_status()
|
||||
|
||||
ordered = [None] * len(prompts)
|
||||
rank_idx = 0
|
||||
for r in range(dp_size):
|
||||
if per_rank[r]:
|
||||
for local_i, orig_i in enumerate(indices[r]):
|
||||
ordered[orig_i] = all_outputs[rank_idx][local_i]
|
||||
rank_idx += 1
|
||||
return ordered
|
||||
|
||||
def sleep(self, level: int = 0):
|
||||
ray.get([actor.sleep.remote(level=level) for actor in self.llm_actors])
|
||||
|
||||
def wake_up(self, tags: list[str] | None = None):
|
||||
ray.get([actor.wake_up.remote(tags=tags) for actor in self.llm_actors])
|
||||
def print_generations(label: str, prompts: list[str], outputs: list[str]) -> None:
|
||||
print("-" * 60)
|
||||
print(label)
|
||||
print("-" * 60)
|
||||
for prompt, text in zip(prompts, outputs):
|
||||
print(f"Prompt: {prompt!r}")
|
||||
print(f"Generated: {text!r}")
|
||||
print("-" * 60)
|
||||
|
||||
|
||||
def main():
|
||||
ray.init(
|
||||
runtime_env={
|
||||
"env_vars": {
|
||||
# The trainer pickles IPC handles for the HTTP client.
|
||||
"VLLM_ALLOW_INSECURE_SERIALIZATION": "1",
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
assert TRAIN_GPU_FRACTION + VLLM_GPU_FRACTION <= 1.0, (
|
||||
"Train + vLLM GPU fractions must sum to at most 1.0 per bundle."
|
||||
)
|
||||
|
||||
local_model_path = snapshot_download(MODEL_NAME)
|
||||
print(f"[init] Model downloaded to {local_model_path}")
|
||||
|
||||
fsdp_master_addr = get_ip()
|
||||
fsdp_master_port = get_open_port()
|
||||
dp_master_port = get_open_port()
|
||||
dp_master_ip = get_ip()
|
||||
|
||||
# Create one placement group per DP rank (one GPU each).
|
||||
pgs = []
|
||||
for _ in range(INFERENCE_DP_SIZE):
|
||||
pg = placement_group([{"GPU": 1, "CPU": 1}])
|
||||
pgs.append(pg)
|
||||
ray.get([pg.ready() for pg in pgs])
|
||||
print(f"[init] {len(pgs)} placement groups ready.")
|
||||
|
||||
# Launch FSDP training workers, one per PG.
|
||||
scheduling = [
|
||||
PlacementGroupSchedulingStrategy(
|
||||
placement_group=pgs[r],
|
||||
placement_group_capture_child_tasks=True,
|
||||
)
|
||||
for r in range(FSDP_WORLD_SIZE)
|
||||
]
|
||||
|
||||
# Launch the training workers first so Ray reserves their GPUs; the server
|
||||
# is then pinned to those same physical GPUs.
|
||||
fsdp_workers = [
|
||||
FSDPTrainWorker.options(scheduling_strategy=scheduling[r]).remote(
|
||||
FSDPTrainWorker.remote(
|
||||
local_model_path,
|
||||
r,
|
||||
rank,
|
||||
FSDP_WORLD_SIZE,
|
||||
fsdp_master_addr,
|
||||
fsdp_master_port,
|
||||
)
|
||||
for r in range(FSDP_WORLD_SIZE)
|
||||
for rank in range(FSDP_WORLD_SIZE)
|
||||
]
|
||||
ray.get([w.get_rank.remote() for w in fsdp_workers])
|
||||
print(f"[init] {FSDP_WORLD_SIZE} FSDP workers ready.")
|
||||
print(f"[init] {FSDP_WORLD_SIZE} FSDP training workers ready.")
|
||||
|
||||
# Launch DP inference engine (spawns and initializes all LLM actors).
|
||||
inference_engine = DataParallelInferenceEngine.remote(
|
||||
model=local_model_path,
|
||||
pgs=pgs,
|
||||
dp_master_ip=dp_master_ip,
|
||||
dp_master_port=dp_master_port,
|
||||
training_gpus = sorted(
|
||||
int(g)
|
||||
for ids in ray.get([w.get_gpu_ids.remote() for w in fsdp_workers])
|
||||
for g in ids
|
||||
)
|
||||
llm_actors = ray.get(inference_engine.get_llm_actors.remote())
|
||||
print(f"[init] {INFERENCE_DP_SIZE} LLM actors ready.")
|
||||
if len(training_gpus) != INFERENCE_TP_SIZE * INFERENCE_DP_SIZE:
|
||||
raise RuntimeError(
|
||||
f"Need {INFERENCE_TP_SIZE * INFERENCE_DP_SIZE} colocated GPUs but "
|
||||
f"Ray assigned training to {training_gpus}."
|
||||
)
|
||||
device_ids = ",".join(str(g) for g in training_gpus)
|
||||
print(f"[init] Colocating training and inference on GPUs [{device_ids}].")
|
||||
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
sampling_params = SamplingParams(temperature=0)
|
||||
server_proc = start_vllm_server(local_model_path, device_ids)
|
||||
try:
|
||||
client = OpenAI(base_url=f"{BASE_URL}/v1", api_key="EMPTY")
|
||||
|
||||
print("[generate] Generating with dummy weights...")
|
||||
outputs = ray.get(inference_engine.generate.remote(prompts, sampling_params))
|
||||
print("-" * 60)
|
||||
print("BEFORE weight sync (dummy weights):")
|
||||
print("-" * 60)
|
||||
for output in outputs:
|
||||
print(f"Prompt: {output.prompt!r}")
|
||||
print(f"Generated: {output.outputs[0].text!r}")
|
||||
print("-" * 60)
|
||||
print("[generate] Generating with dummy weights...")
|
||||
outputs = generate_completions(client, PROMPTS)
|
||||
print_generations("BEFORE weight sync (dummy weights):", PROMPTS, outputs)
|
||||
|
||||
# --- Weight transfer ---
|
||||
# The rank-0 FSDP worker owns the trainer engine and drives the inference
|
||||
# side (init/start/update/finish) for all DP actors via the Ray client.
|
||||
print("[transfer] Initializing IPC weight transfer (all FSDP ranks)...")
|
||||
ray.get([w.setup_engine.remote(llm_actors) for w in fsdp_workers])
|
||||
# --- Weight transfer ---
|
||||
print("[transfer] Initializing IPC weight transfer (all FSDP ranks)...")
|
||||
ray.get([w.setup_engine.remote(BASE_URL) for w in fsdp_workers])
|
||||
|
||||
# Two-phase sleep/wake pattern:
|
||||
# 1. sleep(level=1) — offload weights to CPU, discard KV cache
|
||||
# 2. wake_up(tags=["weights"]) — bring weights back to GPU (KV cache still free)
|
||||
# 3. IPC weight transfer — overwrite weights, plenty of room without KV cache
|
||||
# 4. wake_up(tags=["kv_cache"]) — re-allocate KV cache for inference
|
||||
print("[sync] Sleeping engines (offload weights + free KV cache)...")
|
||||
ray.get(inference_engine.sleep.remote(level=1))
|
||||
print("[sync] Sleeping engine (offload weights + free KV cache)...")
|
||||
sleep_engine(BASE_URL, level=1)
|
||||
|
||||
print("[sync] Waking weights (KV cache stays free)...")
|
||||
ray.get(inference_engine.wake_up.remote(tags=["weights"]))
|
||||
print("[sync] Waking weights (KV cache stays free)...")
|
||||
wake_up_engine(BASE_URL, tags=["weights"])
|
||||
|
||||
# All FSDP ranks participate in the IPC handle all-gather; rank 0's engine
|
||||
# additionally drives start_weight_update / update_weights /
|
||||
# finish_weight_update on the inference side.
|
||||
print("[sync] Packed IPC transfer FSDP → vLLM...")
|
||||
ray.get([w.gather_and_broadcast_weights_ipc.remote() for w in fsdp_workers])
|
||||
print("[sync] Weight transfer complete.")
|
||||
print("[sync] Packed IPC transfer FSDP -> vLLM...")
|
||||
ray.get([w.gather_and_broadcast_weights_ipc.remote() for w in fsdp_workers])
|
||||
print("[sync] Weight transfer complete.")
|
||||
|
||||
print("[sync] Waking KV cache + scheduling...")
|
||||
ray.get(inference_engine.wake_up.remote(tags=["kv_cache", "scheduling"]))
|
||||
print("[sync] Waking KV cache + scheduling...")
|
||||
wake_up_engine(BASE_URL, tags=["kv_cache", "scheduling"])
|
||||
|
||||
print("[generate] Generating with synced weights...")
|
||||
outputs_updated = ray.get(
|
||||
inference_engine.generate.remote(prompts, sampling_params)
|
||||
)
|
||||
print("-" * 60)
|
||||
print("AFTER weight sync (real weights):")
|
||||
print("-" * 60)
|
||||
for output in outputs_updated:
|
||||
print(f"Prompt: {output.prompt!r}")
|
||||
print(f"Generated: {output.outputs[0].text!r}")
|
||||
print("-" * 60)
|
||||
print("[generate] Generating with synced weights...")
|
||||
outputs_updated = generate_completions(client, PROMPTS)
|
||||
print_generations("AFTER weight sync (real weights):", PROMPTS, outputs_updated)
|
||||
finally:
|
||||
print("[server] Shutting down...")
|
||||
server_proc.terminate()
|
||||
try:
|
||||
server_proc.wait(timeout=30)
|
||||
except subprocess.TimeoutExpired:
|
||||
server_proc.kill()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,196 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Demonstrates reinforcement learning using vLLM and Ray,
|
||||
with native weight syncing APIs at engine instance.
|
||||
|
||||
The script separates training and inference workloads onto distinct GPUs
|
||||
so that Ray can manage process placement and inter-process communication.
|
||||
A Hugging Face Transformer model occupies one GPU for training, whereas a
|
||||
2x tensor-parallel vLLM inference engine occupies two GPUs.
|
||||
|
||||
The example performs the following steps:
|
||||
* Load the training model on one gpu (scheduled via ray)
|
||||
* Initialize the inference model with dummy weights across
|
||||
two gpus using vLLM's tensor parallelism and Ray placement groups.
|
||||
* Generate gibberish from a list of prompts using the randomly initialized
|
||||
inference engine.
|
||||
* Update the weights of the training model and broadcast the updated weights
|
||||
to the inference engine by using a Ray collective RPC group.
|
||||
* Generating from the list of prompts after weight sync should result
|
||||
in sensible outputs.
|
||||
|
||||
This example assumes a single-node cluster with three GPUs, but Ray
|
||||
supports multi-node clusters. vLLM expects the GPUs are only used for vLLM
|
||||
workloads. Residual GPU activity interferes with vLLM memory profiling and
|
||||
causes unexpected behavior.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import ray
|
||||
import torch
|
||||
from ray.util.placement_group import placement_group
|
||||
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
|
||||
from transformers import AutoModelForCausalLM
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.config import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer import (
|
||||
ModuleSource,
|
||||
RayVLLMWeightSyncClient,
|
||||
WeightTransferTrainerFactory,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_engine import NCCLTrainerInitInfo
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_ip, get_open_port
|
||||
|
||||
MODEL_NAME = "facebook/opt-125m"
|
||||
# MODEL_NAME = "inference-optimization/Qwen3-0.6B-W4A16-G128"
|
||||
|
||||
|
||||
def get_assigned_gpu():
|
||||
"""This is a temporary workaround for a runtime bug in RCCL on ROCm."""
|
||||
if not current_platform.is_rocm():
|
||||
return 0
|
||||
assigned_gpu = int(ray.get_gpu_ids()[0])
|
||||
os.environ.pop("CUDA_VISIBLE_DEVICES", None)
|
||||
os.environ.pop("HIP_VISIBLE_DEVICES", None)
|
||||
torch.accelerator.set_device_idx(assigned_gpu)
|
||||
return assigned_gpu
|
||||
|
||||
|
||||
class MyLLM(LLM):
|
||||
"""Configure the vLLM worker for Ray placement group execution."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
os.environ["VLLM_RAY_BUNDLE_INDICES"] = "0,1"
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
class TrainModel:
|
||||
"""Ray actor that wraps the training model on a dedicated GPU."""
|
||||
|
||||
def __init__(self, model_name: str):
|
||||
assigned_gpu = get_assigned_gpu()
|
||||
|
||||
self.model = AutoModelForCausalLM.from_pretrained(
|
||||
model_name,
|
||||
).to(f"cuda:{assigned_gpu}")
|
||||
|
||||
self.port = get_open_port()
|
||||
self.master_address = get_ip()
|
||||
|
||||
def init_weight_transfer(self, world_size, llm_handle):
|
||||
"""Build the trainer-side weight-transfer engine.
|
||||
|
||||
`trainer_init` drives the full handshake: it kicks off the inference
|
||||
side's `init_weight_transfer_engine` (via the Ray client) on a worker
|
||||
thread while opening the trainer's NCCL endpoint, so both ends
|
||||
rendezvous together. After this returns, `send_weights()` is callable.
|
||||
"""
|
||||
self.engine = WeightTransferTrainerFactory.trainer_init(
|
||||
init_info=NCCLTrainerInitInfo(
|
||||
master_address=self.master_address,
|
||||
master_port=self.port,
|
||||
world_size=world_size,
|
||||
rank=0, # single-GPU trainer is the sole (sender) rank
|
||||
packed=True,
|
||||
),
|
||||
client=RayVLLMWeightSyncClient(llm_handle),
|
||||
source=ModuleSource(self.model),
|
||||
)
|
||||
|
||||
def broadcast_weights(self):
|
||||
"""Push the current weights to the inference engine.
|
||||
|
||||
Drives start/update/finish on the inference side and the NCCL
|
||||
broadcast internally — one call.
|
||||
"""
|
||||
self.engine.send_weights()
|
||||
|
||||
|
||||
# Initialize Ray and set the visible devices. The vLLM engine will
|
||||
# be placed on GPUs 1 and 2.
|
||||
ray.init()
|
||||
|
||||
# Create a placement group that reserves GPU 1–2 for the vLLM inference engine.
|
||||
# Learn more about Ray placement groups:
|
||||
# https://docs.ray.io/en/latest/placement-groups.html
|
||||
# Launch the training model actor. Ray's resource scheduler will allocate
|
||||
# 1 GPU (via num_gpus=1 in the decorator), ensuring pg_inference gets different GPUs.
|
||||
train_model = TrainModel.remote(MODEL_NAME)
|
||||
|
||||
pg_inference = placement_group([{"GPU": 1, "CPU": 0}] * 2)
|
||||
ray.get(pg_inference.ready())
|
||||
scheduling_inference = PlacementGroupSchedulingStrategy(
|
||||
placement_group=pg_inference,
|
||||
placement_group_capture_child_tasks=True,
|
||||
placement_group_bundle_index=0,
|
||||
)
|
||||
|
||||
# Launch the vLLM inference engine. The `enforce_eager` flag reduces
|
||||
# start-up latency.
|
||||
# Note: Weight transfer APIs (init_weight_transfer_engine, update_weights)
|
||||
# are now native to vLLM workers.
|
||||
llm = ray.remote(
|
||||
num_cpus=0,
|
||||
num_gpus=0,
|
||||
scheduling_strategy=scheduling_inference,
|
||||
)(MyLLM).remote(
|
||||
model=MODEL_NAME,
|
||||
enforce_eager=True,
|
||||
tensor_parallel_size=2,
|
||||
data_parallel_size=1,
|
||||
distributed_executor_backend="ray",
|
||||
weight_transfer_config=WeightTransferConfig(backend="nccl"),
|
||||
load_format="dummy",
|
||||
quantization="fp8",
|
||||
)
|
||||
|
||||
# Generate text from the prompts.
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
|
||||
sampling_params = SamplingParams(temperature=0)
|
||||
|
||||
outputs = ray.get(llm.generate.remote(prompts, sampling_params))
|
||||
|
||||
# Generate text with the initial model. The output is expected to be nonsense
|
||||
# because the weights are randomly initialized.
|
||||
print("-" * 50)
|
||||
for output in outputs:
|
||||
prompt = output.prompt
|
||||
generated_text = output.outputs[0].text
|
||||
print(f"Prompt: {prompt!r}\nGenerated text: {generated_text!r}")
|
||||
print("-" * 50)
|
||||
|
||||
ray.get(llm.sleep.remote(level=0))
|
||||
|
||||
# Set up the communication channel between the training process and the
|
||||
# inference engine. The trainer engine now owns the full handshake — the
|
||||
# driver only has to hand it the vLLM actor handle and the world size.
|
||||
world_size = ray.get(llm.get_world_size.remote()) + 1 # +1 for the trainer
|
||||
ray.get(train_model.init_weight_transfer.remote(world_size, llm))
|
||||
|
||||
# Synchronize the updated weights to the inference engine. The engine drives
|
||||
# start_weight_update / update_weights / finish_weight_update on the inference
|
||||
# side internally, concurrent with the NCCL broadcast.
|
||||
ray.get(train_model.broadcast_weights.remote())
|
||||
|
||||
ray.get(llm.wake_up.remote(tags=["scheduling"]))
|
||||
|
||||
# Generate text with the updated model. The output is expected to be normal
|
||||
# because the weights are updated.
|
||||
outputs_updated = ray.get(llm.generate.remote(prompts, sampling_params))
|
||||
print("-" * 50)
|
||||
for output in outputs_updated:
|
||||
prompt = output.prompt
|
||||
generated_text = output.outputs[0].text
|
||||
print(f"Prompt: {prompt!r}\nGenerated text: {generated_text!r}")
|
||||
print("-" * 50)
|
||||
Reference in New Issue
Block a user