Weight sync refactor + move sparse nccl engine (#44353)

Signed-off-by: hao-aaron <ahao@anyscale.com>
Signed-off-by: haoaaron <ahao@anyscale.com>
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Aaron Hao
2026-07-01 01:25:19 -07:00
committed by GitHub
co-authored by Claude Sonnet 4.6
parent f651a8a9a4
commit 77a9c5ae28
32 changed files with 951 additions and 1044 deletions
+1 -1
View File
@@ -58,7 +58,7 @@ class Fp8PerTensorOnlineLinearMethod(LinearMethodBase):
### High Level Weight Transfer API
The layerwise reloading system is integrated with the post-training weight transfer system. To use layerwise reloading in conjunction to the weight transfer system, follow the examples found [here](../../examples/rl/). Layerwise reloading is controlled by the `WeightTransferUpdateInfo.is_checkpoint_format` flag and is set to `True` by default.
The layerwise reloading system is integrated with the post-training weight transfer system. To use layerwise reloading in conjunction to the weight transfer system, follow the examples found [here](../../examples/rl/). Checkpoint-format weight transfer engines (e.g. the NCCL and IPC backends) run layerwise reloading automatically inside their `start_weight_update`/`finish_weight_update` lifecycle.
### Mid Level `reload_weights` API
+3 -2
View File
@@ -17,6 +17,7 @@ The weight transfer system follows a **four-phase protocol** with a pluggable ba
| ------- | --------- | -------- |
| [NCCL](nccl.md) | NCCL broadcast | Separate GPUs for training and inference |
| [IPC](ipc.md) | CUDA IPC handles | Colocated training and inference on same GPU |
| [sparse_nccl](nccl.md#sparse-nccl) | NCCL broadcast | Sparse flat-index weight patches (TP=1/PP=1) |
## Configuration
@@ -41,7 +42,7 @@ vllm serve my-model \
--weight-transfer-config '{"backend": "nccl"}'
```
The `backend` field accepts `"nccl"` (default) or `"ipc"`.
The `backend` field accepts `"nccl"` (default), `"ipc"`, or `"sparse_nccl"`.
## API Endpoints
@@ -69,7 +70,7 @@ Both backends provide static methods that the trainer calls to send weights. The
EngineClass.trainer_init(init_info)
# 2. Start weight update on inference side
llm.start_weight_update(is_checkpoint_format=True)
llm.start_weight_update()
# 3. Send weights to inference workers
EngineClass.trainer_send_weights(
+28 -15
View File
@@ -11,15 +11,23 @@ The `WeightTransferEngine` is a generic abstract class parameterized by two data
### Abstract Methods
Subclasses must implement these four methods:
Subclasses must implement these methods:
| Method | Side | Description |
| ------ | ---- | ----------- |
| `init_transfer_engine(init_info)` | Inference | Initialize the communication channel on each inference worker |
| `receive_weights(update_info, load_weights)` | Inference | Receive weights and call `load_weights` incrementally |
| `start_weight_update()` | Inference | Prepare for an update (e.g. begin layerwise reload); no-op for in-place engines |
| `finish_weight_update()` | Inference | Finalize the update (e.g. finalize layerwise reload); no-op for in-place engines |
| `receive_weights(update_info)` | Inference | Receive weights and load them into `self.model` |
| `shutdown()` | Inference | Clean up resources |
| `trainer_send_weights(iterator, trainer_args)` | Trainer | Static method to send weights from the trainer process |
The base class provides two methods:
1. `__init__` : Engines receive `config` (`WeightTransferConfig`), `vllm_config` (`VllmConfig`), `device` (`torch.device`) and `model` (`nn.Module`)
2. `update_weights(update_info_dict)`: Thin wrapper for `receive_weights`: parses
the dict into user-specified data type, calls `receive_weights`, and synchronizes the device. Subclasses implement `receive_weights`.
### Request Classes
The API-level request classes provide backend-agnostic serialization using plain dictionaries. The engine's `parse_init_info` and `parse_update_info` methods convert these dictionaries into typed dataclasses.
@@ -81,7 +89,7 @@ class MyUpdateInfo(WeightTransferUpdateInfo):
### 2. Implement the Engine
```python
from collections.abc import Callable, Iterator
from collections.abc import Iterator
from typing import Any
import torch
@@ -93,18 +101,25 @@ class MyWeightTransferEngine(WeightTransferEngine[MyInitInfo, MyUpdateInfo]):
# Set up connection to trainer using init_info.endpoint, etc.
...
def receive_weights(
self,
update_info: MyUpdateInfo,
load_weights: Callable[[list[tuple[str, torch.Tensor]]], None],
) -> None:
# Receive each weight and call load_weights incrementally
def start_weight_update(self) -> None:
# Checkpoint-format engines: run initialize_layerwise_reload(self.model).
# In-place engines: no-op
...
def finish_weight_update(self) -> None:
# Checkpoint-format engines: run finalize_layerwise_reload(...).
# In-place engines: no-op
...
def receive_weights(self, update_info: MyUpdateInfo) -> None:
weights = []
for name, dtype_name, shape in zip(
update_info.names, update_info.dtype_names, update_info.shapes
):
dtype = getattr(torch, dtype_name)
weight = self._fetch_weight(name, shape, dtype)
load_weights([(name, weight)])
weights.append((name, weight))
self.model.load_weights(weights)
def shutdown(self) -> None:
# Clean up resources
@@ -121,9 +136,6 @@ class MyWeightTransferEngine(WeightTransferEngine[MyInitInfo, MyUpdateInfo]):
...
```
!!! important
The `load_weights` callable passed to `receive_weights` should be called **incrementally** (one or a few weights at a time) rather than accumulating all weights first. This avoids GPU out-of-memory errors with large models.
### 3. Register with the Factory
```python
@@ -147,7 +159,7 @@ Once registered, users can select your backend via `WeightTransferConfig(backend
## WeightTransferEngineFactory
The factory uses a registry pattern with lazy loading. Built-in engines (`nccl` and `ipc`) are registered at import time but their modules are only loaded when the backend is actually requested. This avoids importing heavy dependencies (like NCCL communicators) when they aren't needed.
The factory uses a registry pattern with lazy loading. Built-in engines (`nccl`, `ipc`, and `sparse_nccl`) are registered at import time but their modules are only loaded when the backend is actually requested. This avoids importing heavy dependencies (like NCCL communicators) when they aren't needed.
```python
from vllm.distributed.weight_transfer.factory import WeightTransferEngineFactory
@@ -155,7 +167,8 @@ from vllm.distributed.weight_transfer.factory import WeightTransferEngineFactory
# Create an engine from config
engine = WeightTransferEngineFactory.create_engine(
config=weight_transfer_config,
parallel_config=parallel_config,
vllm_config=vllm_config,
device=device,
model=model,
)
```
+2 -2
View File
@@ -55,7 +55,7 @@ trainer_args = IPCTrainerSendWeightsArgs(
llm_handle=llm_actor_handle,
)
# start
ray.get(llm_actor_handle.start_weight_update.remote(is_checkpoint_format=True))
ray.get(llm_actor_handle.start_weight_update.remote())
# send weights
IPCWeightTransferEngine.trainer_send_weights(
iterator=model.named_parameters(),
@@ -80,7 +80,7 @@ trainer_args = IPCTrainerSendWeightsArgs(
# start
base_url = "http://localhost:8000"
url = f"{base_url}/start_weight_update"
response = requests.post(url, json={"is_checkpoint_format": True}, timeout=60)
response = requests.post(url, json={}, timeout=60)
response.raise_for_status()
# send weights
IPCWeightTransferEngine.trainer_send_weights(
+14 -11
View File
@@ -11,7 +11,7 @@ The NCCL weight transfer engine uses [NCCL](https://developer.nvidia.com/nccl) b
## How It Works
1. The trainer and all inference workers join a shared NCCL process group using `StatelessProcessGroup` (vLLM's torch.distributed-independent group abstraction).
2. The trainer broadcasts weights to all workers simultaneously. Each worker receives and loads weights incrementally.
2. The trainer broadcasts weights to all workers simultaneously. Each worker receives and loads the weights.
3. Optionally, **packed tensor broadcasting** batches multiple small tensors into larger buffers with double/triple buffering and CUDA stream overlap for higher throughput. This implementation is based on [NeMo-RL's packed tensor](https://github.com/NVIDIA-NeMo/RL/blob/main/nemo_rl/utils/packed_tensor.py).
## Initialization
@@ -93,7 +93,7 @@ remaining three steps are:
from vllm.distributed.weight_transfer.base import WeightTransferUpdateRequest
# 1. Start the weight update
llm.start_weight_update(is_checkpoint_format=True)
llm.start_weight_update()
# 2. Receive weights (can be called multiple times for chunked transfers)
llm.update_weights(
@@ -116,19 +116,22 @@ must match the order in which the trainer iterates over its parameters.
`start_weight_update` must be called before `update_weights`, and
`finish_weight_update` must be called after all weight chunks have been
transferred. The `is_checkpoint_format` flag controls whether layerwise reload
processing is applied (`True` for checkpoint-format weights, `False` for
pre-processed kernel-format weights).
transferred. The NCCL engine receives checkpoint-format weights and applies
layerwise reload processing automatically inside `start_weight_update` /
`finish_weight_update`.
Sparse NCCL patches still use `update_kind="sparse_flat"` inside
`update_info`, but they should be wrapped in
`start_weight_update(is_checkpoint_format=False)` because sparse patches apply
directly to runtime/kernel-format parameters. The current sparse MVP requires
`TP=1` and `PP=1`.
## Sparse NCCL
Sparse, flat-index weight patches use a separate backend,
`WeightTransferConfig(backend="sparse_nccl")`, implemented by
`SparseNCCLWeightTransferEngine`. It shares only NCCL process-group
initialization with the dense engine; patches are applied directly in place to
existing parameters (no layerwise reload). The current sparse MVP requires
`TP=1` and `PP=1`. See the example below.
## Examples
- [RLHF with NCCL weight syncing (offline, Ray)](../../../examples/rl/rlhf_nccl.py) - Trainer on one GPU, 2x tensor-parallel vLLM engine on two others, with packed NCCL weight broadcast
- [RLHF with sparse NCCL weight syncing (offline, Ray)](../../../examples/rl/rlhf_sparse_nccl.py) - Dense-vs-sparse equivalence demo with a real model on a 2-GPU trainer/inference setup; sparse patches use `start_weight_update(is_checkpoint_format=False)` and currently require `TP=1` and `PP=1`
- [RLHF with sparse NCCL weight syncing (offline, Ray)](../../../examples/rl/rlhf_sparse_nccl.py) - Dense-vs-sparse equivalence demo with a real model on a 2-GPU trainer/inference setup; sparse patches use `backend="sparse_nccl"` and currently require `TP=1` and `PP=1`
- [RLHF with async weight syncing (offline, Ray)](../../../examples/rl/rlhf_async_new_apis.py) - Async generation with mid-flight pause, weight sync, resume, and validation against a fresh model
- [RLHF with NCCL weight syncing (online serving, HTTP)](../../../examples/rl/rlhf_http_nccl.py) - Weight transfer with a running vLLM HTTP server using HTTP control plane and NCCL data plane
+1 -1
View File
@@ -306,7 +306,7 @@ gen_futures = [
ray.get(llm.pause_after_n_tokens.remote())
ray.get(llm.start_weight_update.remote(is_checkpoint_format=True))
ray.get(llm.start_weight_update.remote())
inference_handle = llm.update_weights.remote(
WeightTransferUpdateRequest(
+3 -7
View File
@@ -80,14 +80,10 @@ def init_weight_transfer_engine(base_url: str) -> None:
response.raise_for_status()
def start_weight_update(
base_url: str,
is_checkpoint_format: bool = True,
) -> None:
def start_weight_update(base_url: str) -> None:
"""Start a weight update via HTTP endpoint."""
url = f"{base_url}/start_weight_update"
payload = {"is_checkpoint_format": is_checkpoint_format}
response = requests.post(url, json=payload, timeout=60)
response = requests.post(url, json={}, timeout=60)
response.raise_for_status()
@@ -170,7 +166,7 @@ def main():
pause_generation(BASE_URL)
# Start weight update, broadcast via IPC, then finish
start_weight_update(BASE_URL, is_checkpoint_format=False)
start_weight_update(BASE_URL)
print("Broadcasting weights via CUDA IPC (HTTP)...")
trainer_args = IPCTrainerSendWeightsArgs(send_mode="http", url=BASE_URL)
+3 -7
View File
@@ -83,14 +83,10 @@ def init_weight_transfer_engine(
response.raise_for_status()
def start_weight_update(
base_url: str,
is_checkpoint_format: bool = True,
) -> None:
def start_weight_update(base_url: str) -> None:
"""Start a weight update via HTTP endpoint."""
url = f"{base_url}/start_weight_update"
payload = {"is_checkpoint_format": is_checkpoint_format}
response = requests.post(url, json=payload, timeout=60)
response = requests.post(url, json={}, timeout=60)
response.raise_for_status()
@@ -223,7 +219,7 @@ def main():
shapes.append(list(p.shape))
# Start weight update
start_weight_update(BASE_URL, is_checkpoint_format=True)
start_weight_update(BASE_URL)
# Start the update_weights call in a separate thread since it will block
# waiting for NCCL broadcasts
+1 -1
View File
@@ -139,7 +139,7 @@ ray.get(llm.sleep.remote(level=0))
ray.get(train_model.init_weight_transfer.remote())
# Start weight update, sync weights, then finish
ray.get(llm.start_weight_update.remote(is_checkpoint_format=True))
ray.get(llm.start_weight_update.remote())
ray.get(train_model.broadcast_weights.remote(llm))
ray.get(llm.finish_weight_update.remote())
+3 -10
View File
@@ -277,15 +277,8 @@ class DataParallelInferenceEngine:
]
)
def start_weight_update(self, is_checkpoint_format: bool = True):
ray.get(
[
actor.start_weight_update.remote(
is_checkpoint_format=is_checkpoint_format
)
for actor in self.llm_actors
]
)
def start_weight_update(self):
ray.get([actor.start_weight_update.remote() for actor in self.llm_actors])
def finish_weight_update(self):
ray.get([actor.finish_weight_update.remote() for actor in self.llm_actors])
@@ -392,7 +385,7 @@ def main():
ray.get(inference_engine.wake_up.remote(tags=["weights"]))
print("[sync] Starting weight update...")
ray.get(inference_engine.start_weight_update.remote(is_checkpoint_format=True))
ray.get(inference_engine.start_weight_update.remote())
print("[sync] Packed IPC transfer FSDP → vLLM...")
ray.get(
+1 -1
View File
@@ -202,7 +202,7 @@ ray.get([train_handle, inference_handle])
names, dtype_names, shapes = ray.get(train_model.get_weight_metadata.remote())
# Start weight update
ray.get(llm.start_weight_update.remote(is_checkpoint_format=True))
ray.get(llm.start_weight_update.remote())
# Issue update_weights call with NCCL-specific update info
# packed=True enables efficient batched tensor broadcasting
+1 -1
View File
@@ -299,7 +299,7 @@ async def main():
print(f"[sync] Got metadata for {len(names)} parameters.")
print("[sync] Starting weight update...")
await engine.start_weight_update(is_checkpoint_format=True)
await engine.start_weight_update()
print("[sync] Broadcasting weights from FSDP → vLLM...")
broadcast_handles = [
+11 -8
View File
@@ -44,11 +44,14 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
from vllm import LLM, SamplingParams
from vllm.config import WeightTransferConfig
from vllm.distributed.weight_transfer.base import SparseWeightPatch
from vllm.distributed.weight_transfer.nccl_engine import (
NCCLTrainerSendWeightsArgs,
NCCLWeightTransferEngine,
)
from vllm.distributed.weight_transfer.sparse_nccl_engine import (
SparseNCCLWeightTransferEngine,
SparseWeightPatch,
)
from vllm.utils.network_utils import get_ip, get_open_port
MODEL_NAME = "Qwen/Qwen2.5-0.5B-Instruct"
@@ -244,7 +247,6 @@ class TrainModel:
dtype_names=[str(self.patched_param.dtype).split(".")[-1]],
shapes=[list(self.patched_param.shape)],
num_updates_list=[flat_indices.numel()],
update_kind="sparse_flat",
)
return update_info, selected_token_ids, patch_digest, sparse_payload_bytes
@@ -271,7 +273,7 @@ class TrainModel:
raise RuntimeError("Sparse patch has not been prepared")
start = time.perf_counter()
NCCLWeightTransferEngine.trainer_send_sparse_weights(
SparseNCCLWeightTransferEngine.trainer_send_weights(
iter(self.pending_sparse_patches),
NCCLTrainerSendWeightsArgs(group=self.model_update_group),
)
@@ -282,6 +284,7 @@ class TrainModel:
def launch_llm(
scheduling_inference: PlacementGroupSchedulingStrategy,
backend: str = "nccl",
):
return ray.remote(
num_cpus=0,
@@ -293,7 +296,7 @@ def launch_llm(
tensor_parallel_size=1,
distributed_executor_backend="ray",
gpu_memory_utilization=0.7,
weight_transfer_config=WeightTransferConfig(backend="nccl"),
weight_transfer_config=WeightTransferConfig(backend=backend),
)
@@ -332,7 +335,7 @@ def run_dense_phase(
scheduling_inference: PlacementGroupSchedulingStrategy,
) -> dict[str, object]:
ray.get(train_model.reset_model.remote())
llm = launch_llm(scheduling_inference)
llm = launch_llm(scheduling_inference, backend="nccl")
try:
dense_before = collect_vllm_generations(llm)
@@ -351,7 +354,7 @@ def run_dense_phase(
)
trainer_init = train_model.init_weight_transfer_group.remote(world_size)
ray.get([trainer_init, inference_init])
ray.get(llm.start_weight_update.remote(is_checkpoint_format=True))
ray.get(llm.start_weight_update.remote())
dense_update_info, dense_payload_bytes = ray.get(
train_model.get_dense_update_info.remote()
@@ -391,7 +394,7 @@ def run_sparse_phase(
scheduling_inference: PlacementGroupSchedulingStrategy,
) -> dict[str, object]:
ray.get(train_model.reset_model.remote())
llm = launch_llm(scheduling_inference)
llm = launch_llm(scheduling_inference, backend="sparse_nccl")
try:
sparse_before = collect_vllm_generations(llm)
@@ -410,7 +413,7 @@ def run_sparse_phase(
)
trainer_init = train_model.init_weight_transfer_group.remote(world_size)
ray.get([trainer_init, inference_init])
ray.get(llm.start_weight_update.remote(is_checkpoint_format=False))
ray.get(llm.start_weight_update.remote())
sparse_update_info, selected_token_ids, patch_digest, sparse_payload_bytes = (
ray.get(train_model.prepare_sparse_patch.remote(PROMPTS))
+266 -253
View File
@@ -18,7 +18,6 @@ from torch.multiprocessing.reductions import reduce_tensor
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer import WeightTransferEngineFactory
from vllm.distributed.weight_transfer.base import SparseWeightPatch
from vllm.distributed.weight_transfer.ipc_engine import (
IPCWeightTransferEngine,
IPCWeightTransferInitInfo,
@@ -29,6 +28,11 @@ from vllm.distributed.weight_transfer.nccl_engine import (
NCCLWeightTransferInitInfo,
NCCLWeightTransferUpdateInfo,
)
from vllm.distributed.weight_transfer.sparse_nccl_engine import (
SparseNCCLWeightTransferEngine,
SparseNCCLWeightTransferUpdateInfo,
SparseWeightPatch,
)
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
@@ -80,6 +84,18 @@ def create_mock_parallel_config(
return config
def create_mock_vllm_config(
rank: int = 0,
world_size: int = 1,
dp_rank: int = 0,
) -> MagicMock:
"""Create a mock VllmConfig exposing parallel_config and model_config."""
vllm_config = MagicMock()
vllm_config.parallel_config = create_mock_parallel_config(rank, world_size, dp_rank)
vllm_config.model_config = MagicMock()
return vllm_config
# --- Unit Tests: NCCLWeightTransferUpdateInfo Validation ---
@@ -87,7 +103,6 @@ class TestNCCLWeightTransferUpdateInfoValidation:
"""Test NCCLWeightTransferUpdateInfo dataclass validation."""
def test_valid_update_info(self):
"""Test creating valid NCCLWeightTransferUpdateInfo."""
info = NCCLWeightTransferUpdateInfo(
names=["layer.weight", "layer.bias"],
dtype_names=["float32", "float32"],
@@ -98,7 +113,6 @@ class TestNCCLWeightTransferUpdateInfoValidation:
assert info.shapes == [[10, 10], [10]]
def test_mismatched_dtype_names_raises(self):
"""Test that mismatched dtype_names length raises ValueError."""
with pytest.raises(ValueError, match="dtype_names"):
NCCLWeightTransferUpdateInfo(
names=["layer.weight", "layer.bias"],
@@ -107,7 +121,6 @@ class TestNCCLWeightTransferUpdateInfoValidation:
)
def test_mismatched_shapes_raises(self):
"""Test that mismatched shapes length raises ValueError."""
with pytest.raises(ValueError, match="shapes"):
NCCLWeightTransferUpdateInfo(
names=["layer.weight", "layer.bias"],
@@ -116,73 +129,59 @@ class TestNCCLWeightTransferUpdateInfoValidation:
)
def test_empty_lists_valid(self):
"""Test that empty lists are valid."""
info = NCCLWeightTransferUpdateInfo(
names=[],
dtype_names=[],
shapes=[],
)
info = NCCLWeightTransferUpdateInfo(names=[], dtype_names=[], shapes=[])
assert len(info.names) == 0
# --- Unit Tests: SparseNCCLWeightTransferUpdateInfo Validation ---
class TestSparseNCCLWeightTransferUpdateInfoValidation:
"""Test SparseNCCLWeightTransferUpdateInfo dataclass validation."""
def test_valid_sparse_update_info(self):
"""Test creating valid sparse NCCL update info."""
info = NCCLWeightTransferUpdateInfo(
info = SparseNCCLWeightTransferUpdateInfo(
names=["layer.weight", "layer.bias"],
dtype_names=["float32", "bfloat16"],
shapes=[[10, 10], [10]],
num_updates_list=[4, 2],
update_kind="sparse_flat",
)
assert info.update_kind == "sparse_flat"
assert info.num_updates_list == [4, 2]
def test_sparse_update_requires_num_updates_list(self):
with pytest.raises(ValueError, match="`num_updates_list` is required"):
NCCLWeightTransferUpdateInfo(
names=["layer.weight"],
def test_mismatched_dtype_names_raises(self):
with pytest.raises(ValueError, match="dtype_names"):
SparseNCCLWeightTransferUpdateInfo(
names=["layer.weight", "layer.bias"],
dtype_names=["float32"],
shapes=[[10, 10]],
update_kind="sparse_flat",
shapes=[[10, 10], [10]],
num_updates_list=[4, 2],
)
def test_sparse_update_rejects_empty_num_updates_list(self):
def test_rejects_empty_num_updates_list(self):
with pytest.raises(ValueError, match="cannot be empty"):
NCCLWeightTransferUpdateInfo(
SparseNCCLWeightTransferUpdateInfo(
names=[],
dtype_names=[],
shapes=[],
num_updates_list=[],
update_kind="sparse_flat",
)
def test_sparse_update_rejects_packed(self):
with pytest.raises(ValueError, match="cannot be combined with `packed=True`"):
NCCLWeightTransferUpdateInfo(
names=["layer.weight"],
dtype_names=["float32"],
shapes=[[10, 10]],
num_updates_list=[3],
update_kind="sparse_flat",
packed=True,
)
def test_sparse_update_rejects_mismatched_num_updates(self):
def test_rejects_mismatched_num_updates(self):
with pytest.raises(ValueError, match="`num_updates_list`"):
NCCLWeightTransferUpdateInfo(
SparseNCCLWeightTransferUpdateInfo(
names=["layer.weight", "layer.bias"],
dtype_names=["float32", "float32"],
shapes=[[10, 10], [10]],
num_updates_list=[3],
update_kind="sparse_flat",
)
def test_dense_update_rejects_sparse_metadata(self):
with pytest.raises(ValueError, match="Sparse metadata"):
NCCLWeightTransferUpdateInfo(
def test_rejects_negative_num_updates(self):
with pytest.raises(ValueError, match="non-negative"):
SparseNCCLWeightTransferUpdateInfo(
names=["layer.weight"],
dtype_names=["float32"],
shapes=[[10, 10]],
num_updates_list=[3],
num_updates_list=[-1],
)
@@ -192,14 +191,17 @@ class TestNCCLWeightTransferUpdateInfoValidation:
class TestNCCLEngineParsing:
"""Test NCCLWeightTransferEngine parsing methods."""
def test_parse_init_info_valid(self):
"""Test parsing valid init info dict."""
def _make_engine(self):
config = WeightTransferConfig(backend="nccl")
parallel_config = create_mock_parallel_config()
engine = NCCLWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
return NCCLWeightTransferEngine(
config,
create_mock_vllm_config(),
"cuda",
MagicMock(spec=torch.nn.Module),
)
def test_parse_init_info_valid(self):
engine = self._make_engine()
init_info = engine.parse_init_info(
{
"master_address": "127.0.0.1",
@@ -208,7 +210,6 @@ class TestNCCLEngineParsing:
"world_size": 3,
}
)
assert isinstance(init_info, NCCLWeightTransferInitInfo)
assert init_info.master_address == "127.0.0.1"
assert init_info.master_port == 12345
@@ -216,29 +217,12 @@ class TestNCCLEngineParsing:
assert init_info.world_size == 3
def test_parse_init_info_missing_field_raises(self):
"""Test parsing init info with missing required field."""
config = WeightTransferConfig(backend="nccl")
parallel_config = create_mock_parallel_config()
engine = NCCLWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
engine = self._make_engine()
with pytest.raises(ValueError, match="Invalid init_info"):
engine.parse_init_info(
{
"master_address": "127.0.0.1",
# Missing master_port, rank_offset, world_size
}
)
engine.parse_init_info({"master_address": "127.0.0.1"})
def test_parse_update_info_valid(self):
"""Test parsing valid update info dict."""
config = WeightTransferConfig(backend="nccl")
parallel_config = create_mock_parallel_config()
engine = NCCLWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
engine = self._make_engine()
update_info = engine.parse_update_info(
{
"names": ["w1", "w2"],
@@ -246,7 +230,6 @@ class TestNCCLEngineParsing:
"shapes": [[100, 100], [50]],
}
)
assert isinstance(update_info, NCCLWeightTransferUpdateInfo)
assert update_info.names == ["w1", "w2"]
assert update_info.dtype_names == ["float32", "bfloat16"]
@@ -260,40 +243,140 @@ class TestEngineRegistry:
"""Test weight transfer engine registry."""
def test_create_engine_nccl(self):
"""Test factory creates NCCL engine."""
config = WeightTransferConfig(backend="nccl")
parallel_config = create_mock_parallel_config()
engine = WeightTransferEngineFactory.create_engine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module)
)
assert isinstance(engine, NCCLWeightTransferEngine)
def test_create_engine_ipc(self):
"""Test factory creates IPC engine."""
config = WeightTransferConfig(backend="ipc")
parallel_config = create_mock_parallel_config()
engine = WeightTransferEngineFactory.create_engine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module)
)
assert isinstance(engine, IPCWeightTransferEngine)
def test_create_engine_sparse_nccl(self):
config = WeightTransferConfig(backend="sparse_nccl")
engine = WeightTransferEngineFactory.create_engine(
config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module)
)
assert isinstance(engine, SparseNCCLWeightTransferEngine)
def test_create_engine_invalid_backend(self):
"""Test factory raises for invalid backend."""
config = WeightTransferConfig(backend="invalid")
parallel_config = create_mock_parallel_config()
with pytest.raises(ValueError, match="Invalid weight transfer backend"):
WeightTransferEngineFactory.create_engine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
config,
create_mock_vllm_config(),
"cuda",
MagicMock(spec=torch.nn.Module),
)
def test_register_duplicate_raises(self):
"""Test registering duplicate engine name raises."""
with pytest.raises(ValueError, match="already registered"):
WeightTransferEngineFactory.register_engine(
"nccl", NCCLWeightTransferEngine
)
# --- Unit Tests: Sparse patch application (CPU) ---
class TestSparseNCCLPatchApplication:
"""Test SparseNCCLWeightTransferEngine._apply_patch on a real param."""
def _make_engine(self, model):
config = WeightTransferConfig(backend="sparse_nccl")
return SparseNCCLWeightTransferEngine(
config, create_mock_vllm_config(), "cpu", model
)
def _make_model(self, numel: int = 8):
model = torch.nn.Module()
model.register_parameter(
"w", torch.nn.Parameter(torch.zeros(numel), requires_grad=False)
)
def get_parameter(name):
assert name == "w"
return model.w
model.get_parameter = get_parameter
return model
def test_apply_patch_updates_only_selected_entries(self):
model = self._make_model(8)
engine = self._make_engine(model)
engine._apply_patch(
SparseWeightPatch(
name="w",
indices=torch.tensor([1, 3], dtype=torch.int32),
values=torch.tensor([5.0, 7.0], dtype=torch.float32),
)
)
expected = torch.zeros(8)
expected[1] = 5.0
expected[3] = 7.0
assert torch.equal(model.w.data, expected)
def test_apply_patch_rejects_mismatched_lengths(self):
model = self._make_model(8)
engine = self._make_engine(model)
with pytest.raises(ValueError, match="matching lengths"):
engine._apply_patch(
SparseWeightPatch(
name="w",
indices=torch.tensor([1, 3], dtype=torch.int32),
values=torch.tensor([5.0], dtype=torch.float32),
)
)
def test_apply_patch_rejects_non_int32_indices(self):
model = self._make_model(8)
engine = self._make_engine(model)
with pytest.raises(ValueError, match="int32 indices"):
engine._apply_patch(
SparseWeightPatch(
name="w",
indices=torch.tensor([1], dtype=torch.int64),
values=torch.tensor([5.0], dtype=torch.float32),
)
)
def test_apply_patch_rejects_dtype_mismatch(self):
model = self._make_model(8)
engine = self._make_engine(model)
with pytest.raises(ValueError, match="does not match"):
engine._apply_patch(
SparseWeightPatch(
name="w",
indices=torch.tensor([1], dtype=torch.int32),
values=torch.tensor([5.0], dtype=torch.bfloat16),
)
)
def test_apply_patch_rejects_non_contiguous_param(self):
model = torch.nn.Module()
model.register_parameter(
"w",
torch.nn.Parameter(
torch.arange(12, dtype=torch.float32).view(3, 4).t(),
requires_grad=False,
),
)
model.get_parameter = lambda name: model.w
engine = self._make_engine(model)
with pytest.raises(NotImplementedError, match="contiguous params"):
engine._apply_patch(
SparseWeightPatch(
name="w",
indices=torch.tensor([1], dtype=torch.int32),
values=torch.tensor([1.0], dtype=torch.float32),
)
)
# --- Test receive_weights without init raises ---
@@ -303,42 +386,37 @@ def test_nccl_receive_weights_without_init_raises():
pytest.skip("Need at least 1 GPU for this test")
config = WeightTransferConfig(backend="nccl")
parallel_config = create_mock_parallel_config()
engine = NCCLWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module)
)
update_info = NCCLWeightTransferUpdateInfo(
names=["w"],
dtype_names=["float32"],
shapes=[[10]],
names=["w"], dtype_names=["float32"], shapes=[[10]]
)
with pytest.raises(RuntimeError, match="not initialized"):
engine.receive_weights(update_info, lambda x: None)
engine.receive_weights(update_info)
def test_nccl_receive_sparse_weights_without_init_raises():
def test_sparse_nccl_receive_weights_without_init_raises():
"""Test that sparse receive raises if init_transfer_engine wasn't called."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
config = WeightTransferConfig(backend="nccl")
parallel_config = create_mock_parallel_config()
engine = NCCLWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
config = WeightTransferConfig(backend="sparse_nccl")
engine = SparseNCCLWeightTransferEngine(
config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module)
)
update_info = NCCLWeightTransferUpdateInfo(
update_info = SparseNCCLWeightTransferUpdateInfo(
names=["w"],
dtype_names=["float32"],
shapes=[[10]],
num_updates_list=[2],
update_kind="sparse_flat",
)
with pytest.raises(RuntimeError, match="not initialized"):
engine.receive_sparse_weights(update_info, lambda x: None)
engine.receive_weights(update_info)
# --- Integration Test: NCCL Weight Transfer Between Ray Tasks ---
@@ -387,6 +465,7 @@ def inference_receive_tensor(
tensor_dtype: str,
) -> dict:
"""Inference task that receives tensor via NCCLWeightTransferEngine."""
import contextlib
from unittest.mock import MagicMock
import torch
@@ -401,17 +480,32 @@ def inference_receive_tensor(
NCCLWeightTransferUpdateInfo,
)
# Create engine with mock parallel config
class Recorder(torch.nn.Module):
def __init__(self):
super().__init__()
self.received = []
def load_weights(self, weights):
for name, tensor in weights:
self.received.append((name, tensor.clone()))
config = WeightTransferConfig(backend="nccl")
vllm_config = MagicMock()
parallel_config = MagicMock(spec=ParallelConfig)
parallel_config.rank = 0
parallel_config.world_size = 1
parallel_config.data_parallel_rank = 0
parallel_config.data_parallel_index = 0
vllm_config.parallel_config = parallel_config
vllm_config.model_config = MagicMock()
engine = NCCLWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
recorder = Recorder()
engine = NCCLWeightTransferEngine(config, vllm_config, "cuda", recorder)
# Transport-only test: bypass the set_current_vllm_config context that
# receive_weights enters, since vllm_config here is a mock.
import vllm.config as _vllm_config_mod
_vllm_config_mod.set_current_vllm_config = lambda cfg: contextlib.nullcontext()
# Initialize the engine (joins as rank 1)
init_info = NCCLWeightTransferInitInfo(
@@ -422,20 +516,12 @@ def inference_receive_tensor(
)
engine.init_transfer_engine(init_info)
# Receive weights with a no-op load_weights that captures the tensor
received_tensors = []
def noop_load_weights(weights: list[tuple[str, torch.Tensor]]):
for name, tensor in weights:
# Clone tensor to keep it after engine cleans up
received_tensors.append((name, tensor.clone()))
update_info = NCCLWeightTransferUpdateInfo(
names=["test.weight"],
dtype_names=[tensor_dtype],
shapes=[tensor_shape],
)
engine.receive_weights(update_info, noop_load_weights)
engine.receive_weights(update_info)
torch.accelerator.synchronize()
# Verify we received the tensor
@@ -443,11 +529,10 @@ def inference_receive_tensor(
received_shape = None
received_sum = None
if len(received_tensors) == 1:
name, tensor = received_tensors[0]
if len(recorder.received) == 1:
name, tensor = recorder.received[0]
received_shape = list(tensor.shape)
received_sum = tensor.sum().item()
# Check shape matches and values are all 1s (trainer sends ones)
if received_shape == tensor_shape:
expected_sum = 1.0 * torch.tensor(tensor_shape).prod().item()
if abs(received_sum - expected_sum) < 0.01:
@@ -478,11 +563,9 @@ def test_nccl_weight_transfer_between_processes():
master_port = get_open_port()
world_size = 2 # 1 trainer + 1 inference worker
# Tensor to transfer: 100x100 ones
tensor_shape = [100, 100]
tensor_dtype = "float32"
# Start both tasks concurrently - Ray assigns GPUs automatically
inference_future = inference_receive_tensor.remote(
master_address, master_port, world_size, tensor_shape, tensor_dtype
)
@@ -490,7 +573,6 @@ def test_nccl_weight_transfer_between_processes():
master_address, master_port, world_size, tensor_shape, tensor_dtype
)
# Wait for both to complete
trainer_result, result = ray.get([trainer_future, inference_future])
assert trainer_result, "Trainer should complete successfully"
@@ -514,10 +596,12 @@ def trainer_broadcast_sparse_tensor(
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.utils import StatelessProcessGroup
from vllm.distributed.weight_transfer.base import SparseWeightPatch
from vllm.distributed.weight_transfer.nccl_engine import (
NCCLTrainerSendWeightsArgs,
NCCLWeightTransferEngine,
)
from vllm.distributed.weight_transfer.sparse_nccl_engine import (
SparseNCCLWeightTransferEngine,
SparseWeightPatch,
)
pg = StatelessProcessGroup.create(
@@ -533,7 +617,7 @@ def trainer_broadcast_sparse_tensor(
indices=torch.tensor([1, 7, 25], dtype=torch.int32, device=device),
values=torch.tensor([10.0, 20.0, 30.0], dtype=torch.float32, device=device),
)
NCCLWeightTransferEngine.trainer_send_sparse_weights(
SparseNCCLWeightTransferEngine.trainer_send_weights(
iter([patch]),
NCCLTrainerSendWeightsArgs(group=comm),
)
@@ -547,7 +631,7 @@ def inference_receive_sparse_tensor(
master_port: int,
world_size: int,
) -> dict:
"""Inference task that receives sparse patches via NCCLWeightTransferEngine."""
"""Inference task that receives sparse patches via the sparse engine."""
from unittest.mock import MagicMock
import torch
@@ -556,22 +640,40 @@ def inference_receive_sparse_tensor(
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.nccl_engine import (
NCCLWeightTransferEngine,
NCCLWeightTransferInitInfo,
NCCLWeightTransferUpdateInfo,
from vllm.distributed.weight_transfer.sparse_nccl_engine import (
SparseNCCLWeightTransferEngine,
SparseNCCLWeightTransferUpdateInfo,
)
config = WeightTransferConfig(backend="nccl")
config = WeightTransferConfig(backend="sparse_nccl")
vllm_config = MagicMock()
parallel_config = MagicMock(spec=ParallelConfig)
parallel_config.rank = 0
parallel_config.world_size = 1
parallel_config.data_parallel_rank = 0
parallel_config.data_parallel_index = 0
vllm_config.parallel_config = parallel_config
vllm_config.model_config = MagicMock()
engine = NCCLWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
# Real module holding the target parameter the patch will modify.
model = torch.nn.Module()
model.register_parameter(
"w", torch.nn.Parameter(torch.zeros(30, device="cuda"), requires_grad=False)
)
model.get_parameter = lambda name: model.w
update_info = SparseNCCLWeightTransferUpdateInfo(
names=["w"],
dtype_names=["float32"],
shapes=[[30]],
num_updates_list=[3],
)
engine = SparseNCCLWeightTransferEngine(config, vllm_config, "cuda", model)
from vllm.distributed.weight_transfer.nccl_common import (
NCCLWeightTransferInitInfo,
)
engine.init_transfer_engine(
NCCLWeightTransferInitInfo(
master_address=master_address,
@@ -580,32 +682,18 @@ def inference_receive_sparse_tensor(
world_size=world_size,
)
)
target = torch.zeros(30, dtype=torch.float32, device=device)
def apply_sparse_patches(patches: list[SparseWeightPatch]):
for patch in patches:
target.index_copy_(0, patch.indices.to(torch.long), patch.values)
update_info = NCCLWeightTransferUpdateInfo(
names=["test.weight"],
dtype_names=["float32"],
shapes=[[30]],
num_updates_list=[3],
update_kind="sparse_flat",
)
engine.receive_sparse_weights(update_info, apply_sparse_patches)
engine.receive_weights(update_info)
torch.accelerator.synchronize()
expected = torch.zeros(30, dtype=torch.float32, device=device)
expected[[1, 7, 25]] = torch.tensor(
[10.0, 20.0, 30.0], dtype=torch.float32, device=device
)
success = torch.equal(target, expected)
success = torch.equal(model.w.data, expected)
engine.shutdown()
return {
"success": success,
"selected_values": target[[1, 7, 25]].cpu().tolist(),
"selected_values": model.w.data[[1, 7, 25]].cpu().tolist(),
}
@@ -644,11 +732,9 @@ class TestIPCWeightTransferUpdateInfoValidation:
"""Test IPCWeightTransferUpdateInfo dataclass validation."""
def test_valid_update_info(self):
"""Test creating valid IPCWeightTransferUpdateInfo."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
# Create a dummy tensor and IPC handle
dummy_tensor = torch.ones(10, 10, device="cuda:0")
_, ipc_handle = reduce_tensor(dummy_tensor)
gpu_uuid = str(torch.cuda.get_device_properties(0).uuid)
@@ -666,7 +752,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
assert len(info.ipc_handles) == 1
def test_mismatched_dtype_names_raises(self):
"""Test that mismatched dtype_names length raises ValueError."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
@@ -684,7 +769,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
)
def test_mismatched_shapes_raises(self):
"""Test that mismatched shapes length raises ValueError."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
@@ -702,7 +786,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
)
def test_mismatched_ipc_handles_raises(self):
"""Test that mismatched ipc_handles length raises ValueError."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
@@ -719,44 +802,7 @@ class TestIPCWeightTransferUpdateInfoValidation:
ipc_handles=ipc_handles,
)
def test_sparse_update_kind_rejected(self):
"""Test that IPC backend rejects sparse update metadata."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
dummy_tensor = torch.ones(10, 10, device="cuda:0")
ipc_handle = reduce_tensor(dummy_tensor)
gpu_uuid = str(torch.cuda.get_device_properties(0).uuid)
ipc_handles = [{gpu_uuid: ipc_handle}]
with pytest.raises(NotImplementedError, match="dense updates"):
IPCWeightTransferUpdateInfo(
names=["layer.weight"],
dtype_names=["float32"],
shapes=[[10, 10]],
num_updates_list=[1],
ipc_handles=ipc_handles,
update_kind="sparse_flat",
)
def test_sparse_methods_not_supported(self):
"""Test that IPC engine inherits sparse rejection from the base class."""
config = WeightTransferConfig(backend="ipc")
parallel_config = create_mock_parallel_config()
engine = IPCWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
with pytest.raises(NotImplementedError, match="(?i)sparse weight updates"):
engine.receive_sparse_weights(MagicMock(), lambda _: None)
with pytest.raises(NotImplementedError, match="(?i)sparse weight updates"):
engine.trainer_send_sparse_weights(
iter([]),
{"mode": "http", "url": "http://localhost:8000"},
)
def test_valid_update_info_from_pickled(self, monkeypatch):
"""Test creating IPCWeightTransferUpdateInfo from pickled handles."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
@@ -779,7 +825,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
assert info.ipc_handles_pickled is None
def test_pickled_requires_insecure_serialization_flag(self, monkeypatch):
"""Test that pickled handles are rejected unless env flag is enabled."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "0")
with pytest.raises(ValueError, match="VLLM_ALLOW_INSECURE_SERIALIZATION=1"):
@@ -791,7 +836,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
)
def test_both_handles_and_pickled_raises(self):
"""Test that providing both ipc_handles and ipc_handles_pickled raises."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
@@ -812,7 +856,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
)
def test_neither_handles_nor_pickled_raises(self):
"""Test that providing neither ipc_handles nor ipc_handles_pickled raises."""
with pytest.raises(ValueError, match="must be provided"):
IPCWeightTransferUpdateInfo(
names=["layer.weight"],
@@ -821,7 +864,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
)
def test_empty_lists_valid(self):
"""Test that empty lists are valid."""
info = IPCWeightTransferUpdateInfo(
names=[],
dtype_names=[],
@@ -837,18 +879,21 @@ class TestIPCWeightTransferUpdateInfoValidation:
class TestIPCEngineParsing:
"""Test IPCWeightTransferEngine parsing methods."""
def _make_engine(self):
config = WeightTransferConfig(backend="ipc")
return IPCWeightTransferEngine(
config,
create_mock_vllm_config(),
"cuda",
MagicMock(spec=torch.nn.Module),
)
def test_parse_update_info_valid(self):
"""Test parsing valid update info dict."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
config = WeightTransferConfig(backend="ipc")
parallel_config = create_mock_parallel_config()
engine = IPCWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
engine = self._make_engine()
# Create dummy IPC handles
dummy_tensor1 = torch.ones(100, 100, device="cuda:0")
dummy_tensor2 = torch.ones(50, device="cuda:0")
_, ipc_args1 = reduce_tensor(dummy_tensor1)
@@ -872,17 +917,12 @@ class TestIPCEngineParsing:
assert len(update_info.ipc_handles) == 2
def test_parse_update_info_pickled(self, monkeypatch):
"""Test parsing update info with pickled IPC handles (HTTP path)."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
config = WeightTransferConfig(backend="ipc")
parallel_config = create_mock_parallel_config()
engine = IPCWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
engine = self._make_engine()
dummy_tensor1 = torch.ones(100, 100, device="cuda:0")
dummy_tensor2 = torch.ones(50, device="cuda:0")
@@ -909,12 +949,7 @@ class TestIPCEngineParsing:
assert gpu_uuid in update_info.ipc_handles[1]
def test_parse_update_info_ignores_none_pickled_handles(self):
"""Test Ray/asdict payloads with a null pickled field use ipc_handles."""
config = WeightTransferConfig(backend="ipc")
parallel_config = create_mock_parallel_config()
engine = IPCWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
engine = self._make_engine()
ipc_handles = [{"gpu-uuid": ("ipc-args",)}]
update_info = engine.parse_update_info(
@@ -931,15 +966,10 @@ class TestIPCEngineParsing:
assert update_info.ipc_handles == ipc_handles
def test_parse_update_info_both_handles_and_pickled_raises(self):
"""Test that providing both ipc_handles and ipc_handles_pickled raises."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
config = WeightTransferConfig(backend="ipc")
parallel_config = create_mock_parallel_config()
engine = IPCWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
engine = self._make_engine()
dummy_tensor = torch.ones(10, 10, device="cuda:0")
_, ipc_handle = reduce_tensor(dummy_tensor)
@@ -981,9 +1011,6 @@ class TrainerActor:
self.tensor = torch.ones(tensor_shape, dtype=dtype, device=device)
self.tensor.fill_(42.0) # Fill with 42 to verify correct transfer
# Create IPC handle (tensor must stay alive for IPC to work)
# reduce_tensor returns (rebuild_func, args); we only send args
# since the receiver imports rebuild_cuda_tensor directly.
_, ipc_args = reduce_tensor(self.tensor)
gpu_uuid = get_physical_gpu_id(device.index)
@@ -1007,6 +1034,7 @@ def inference_receive_ipc_tensor(
mode: str = "ray",
) -> dict:
"""Inference task that receives tensor via IPCWeightTransferEngine."""
import contextlib
import os
# Worker-side: ipc_handles_pickled is deserialized via pickle.
@@ -1025,30 +1053,36 @@ def inference_receive_ipc_tensor(
IPCWeightTransferEngine,
)
# Create engine with mock parallel config
class Recorder(torch.nn.Module):
def __init__(self):
super().__init__()
self.received = []
def load_weights(self, weights):
for name, tensor in weights:
self.received.append((name, tensor.clone()))
config = WeightTransferConfig(backend="ipc")
vllm_config = MagicMock()
parallel_config = MagicMock(spec=ParallelConfig)
parallel_config.rank = 0
parallel_config.world_size = 1
parallel_config.data_parallel_rank = 0
parallel_config.data_parallel_index = 0
vllm_config.parallel_config = parallel_config
vllm_config.model_config = MagicMock()
engine = IPCWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
recorder = Recorder()
engine = IPCWeightTransferEngine(config, vllm_config, "cuda", recorder)
# Transport-only test: bypass the set_current_vllm_config context that
# receive_weights enters, since vllm_config here is a mock.
import vllm.config as _vllm_config_mod
_vllm_config_mod.set_current_vllm_config = lambda cfg: contextlib.nullcontext()
# Initialize the engine (no-op for IPC)
init_info = IPCWeightTransferInitInfo()
engine.init_transfer_engine(init_info)
# Receive weights with a no-op load_weights that captures the tensor
received_tensors = []
def noop_load_weights(weights: list[tuple[str, torch.Tensor]]):
for name, tensor in weights:
# Clone tensor to keep it after engine cleans up
received_tensors.append((name, tensor.clone()))
ipc_handles = [{ipc_handle_dict["gpu_uuid"]: ipc_handle_dict["ipc_handle"]}]
if mode == "ray":
@@ -1059,7 +1093,6 @@ def inference_receive_ipc_tensor(
"ipc_handles": ipc_handles,
}
elif mode == "http":
# Simulate HTTP transport: pickle + base64 encode handles
pickled = base64.b64encode(pickle.dumps(ipc_handles)).decode("utf-8")
update_dict = {
"names": ["test.weight"],
@@ -1071,19 +1104,17 @@ def inference_receive_ipc_tensor(
raise ValueError(f"Unknown mode: {mode}")
update_info = engine.parse_update_info(update_dict)
engine.receive_weights(update_info, noop_load_weights)
engine.receive_weights(update_info)
torch.accelerator.synchronize()
# Verify we received the tensor
success = False
received_shape = None
received_sum = None
if len(received_tensors) == 1:
name, tensor = received_tensors[0]
if len(recorder.received) == 1:
name, tensor = recorder.received[0]
received_shape = list(tensor.shape)
received_sum = tensor.sum().item()
# Check shape matches and values are all 42s (trainer sends 42s)
if received_shape == ipc_handle_dict["shape"]:
expected_sum = 42.0 * torch.tensor(ipc_handle_dict["shape"]).prod().item()
if abs(received_sum - expected_sum) < 0.01:
@@ -1104,23 +1135,12 @@ def inference_receive_ipc_tensor(
)
@pytest.mark.parametrize("mode", ["ray", "http"])
def test_ipc_weight_transfer_between_processes(mode: str):
"""Test IPC weight transfer from trainer to inference process using Ray.
Parametrized over transport modes:
- 'ray': ipc_handles passed directly.
- 'http': ipc_handles pickled + base64-encoded, deserialized in
parse_update_info before constructing the dataclass.
IPC requires same-GPU access, so we use a placement group to co-locate
the trainer actor and inference task on the same GPU.
"""
"""Test IPC weight transfer from trainer to inference process using Ray."""
from ray.util.placement_group import placement_group
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
_init_ray_for_weight_transfer()
# Create a placement group to ensure both processes are on the same GPU
# Use fractional GPUs so both tasks can share the same GPU bundle
pg = placement_group([{"GPU": 1, "CPU": 2}])
ray.get(pg.ready())
@@ -1129,20 +1149,15 @@ def test_ipc_weight_transfer_between_processes(mode: str):
placement_group_capture_child_tasks=True,
)
# Tensor to transfer: 100x100 filled with 42s
tensor_shape = [100, 100]
tensor_dtype = "float32"
# Create trainer actor that holds the tensor and IPC handle (stays alive)
trainer_actor = TrainerActor.options( # type: ignore[attr-defined]
scheduling_strategy=scheduling_strategy
).remote(tensor_shape, tensor_dtype)
# Get IPC handle dict (tensor stays alive in trainer actor)
ipc_handle_dict = ray.get(trainer_actor.get_ipc_handle_dict.remote())
# Receive tensor in inference process using IPC handles (on same GPU)
# Trainer actor stays alive during this operation
inference_result = ray.get(
inference_receive_ipc_tensor.options(
scheduling_strategy=scheduling_strategy
@@ -1162,12 +1177,10 @@ def test_ipc_receive_weights_missing_gpu_uuid_raises():
pytest.skip("Need at least 1 GPU for this test")
config = WeightTransferConfig(backend="ipc")
parallel_config = create_mock_parallel_config()
engine = IPCWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module)
)
# Create IPC handle with wrong GPU UUID
dummy_tensor = torch.ones(10, 10, device="cuda:0")
_, ipc_handle = reduce_tensor(dummy_tensor)
wrong_uuid = "wrong-uuid-12345"
@@ -1181,4 +1194,4 @@ def test_ipc_receive_weights_missing_gpu_uuid_raises():
)
with pytest.raises(ValueError, match="IPC handle not found"):
engine.receive_weights(update_info, lambda x: None)
engine.receive_weights(update_info)
@@ -8,7 +8,6 @@ actual NCCL communication.
"""
import os
from collections.abc import Callable
from dataclasses import dataclass
from unittest.mock import patch
@@ -48,7 +47,6 @@ class MockUpdateInfo(WeightTransferUpdateInfo):
names: list[str] | None = None
dtype_names: list[str] | None = None
shapes: list[list[int]] | None = None
num_updates_list: list[int] | None = None
class MockWeightTransferEngine(WeightTransferEngine[MockInitInfo, MockUpdateInfo]):
@@ -59,16 +57,20 @@ class MockWeightTransferEngine(WeightTransferEngine[MockInitInfo, MockUpdateInfo
# Class-level tracking for verification across processes
init_transfer_engine_called: bool = False
start_called: bool = False
receive_weights_called: bool = False
finish_called: bool = False
shutdown_called: bool = False
last_init_info: MockInitInfo | None = None
last_update_info: MockUpdateInfo | None = None
def __init__(self, config, parallel_config, model):
super().__init__(config, parallel_config, model)
def __init__(self, config, vllm_config, device, model):
super().__init__(config, vllm_config, device, model)
# Reset tracking on init
MockWeightTransferEngine.init_transfer_engine_called = False
MockWeightTransferEngine.start_called = False
MockWeightTransferEngine.receive_weights_called = False
MockWeightTransferEngine.finish_called = False
MockWeightTransferEngine.shutdown_called = False
MockWeightTransferEngine.last_init_info = None
MockWeightTransferEngine.last_update_info = None
@@ -77,37 +79,28 @@ class MockWeightTransferEngine(WeightTransferEngine[MockInitInfo, MockUpdateInfo
MockWeightTransferEngine.init_transfer_engine_called = True
MockWeightTransferEngine.last_init_info = init_info
def receive_weights(
self,
update_info: MockUpdateInfo,
load_weights: Callable[[list[tuple[str, torch.Tensor]]], None],
) -> None:
MockWeightTransferEngine.receive_weights_called = True
MockWeightTransferEngine.last_update_info = update_info
# Simulate loading weights by calling load_weights with empty list
# (In real implementation, this would receive and load actual weights)
load_weights([])
def start_weight_update(self) -> None:
MockWeightTransferEngine.start_called = True
def receive_sparse_weights(
self,
update_info: MockUpdateInfo,
apply_patches: Callable[[list], None],
) -> None:
def finish_weight_update(self) -> None:
MockWeightTransferEngine.finish_called = True
def receive_weights(self, update_info: MockUpdateInfo) -> None:
MockWeightTransferEngine.receive_weights_called = True
MockWeightTransferEngine.last_update_info = update_info
apply_patches([])
def shutdown(self) -> None:
MockWeightTransferEngine.shutdown_called = True
def trainer_send_weights(self, *args, **kwargs):
@staticmethod
def trainer_send_weights(*args, **kwargs):
"""Mock method to simulate trainer sending weights."""
pass
def mock_create_engine(config, parallel_config, model):
def mock_create_engine(config, vllm_config, device, model):
"""Mock factory function that returns our mock engine."""
return MockWeightTransferEngine(config, parallel_config, model)
return MockWeightTransferEngine(config, vllm_config, device, model)
# --- Tests ---
@@ -208,7 +201,7 @@ def test_update_weights_calls_engine():
llm.init_weight_transfer_engine(
WeightTransferInitRequest(init_info={"test_param": "init"})
)
llm.start_weight_update(is_checkpoint_format=True)
llm.start_weight_update()
# Call update_weights
test_names = ["layer.weight", "layer.bias"]
@@ -243,61 +236,6 @@ def test_update_weights_calls_engine():
llm.finish_weight_update()
@create_new_process_for_each_test()
def test_update_weights_passes_sparse_metadata():
"""Test sparse update metadata is forwarded unchanged to the engine."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0"
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
with patch(
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
mock_create_engine,
):
llm = LLM(
model=MODEL_NAME,
enforce_eager=True,
load_format="dummy",
tensor_parallel_size=1,
weight_transfer_config=WeightTransferConfig(backend="nccl"),
)
llm.init_weight_transfer_engine(
WeightTransferInitRequest(init_info={"test_param": "init"})
)
llm.start_weight_update(is_checkpoint_format=False)
llm.update_weights(
WeightTransferUpdateRequest(
update_info={
"names": ["layer.weight"],
"dtype_names": ["bfloat16"],
"shapes": [[100]],
"num_updates_list": [3],
"update_kind": "sparse_flat",
}
)
)
def check_sparse_update_called(self):
engine = self.weight_transfer_engine
if not engine.receive_weights_called:
return None
info = engine.last_update_info
return (
info.update_kind,
info.num_updates_list,
)
results = llm.collective_rpc(check_sparse_update_called)
for result in results:
assert result == ("sparse_flat", [3])
llm.finish_weight_update()
@create_new_process_for_each_test()
def test_full_weight_transfer_flow():
"""Test the complete weight transfer flow: init -> start -> update -> finish."""
@@ -327,7 +265,7 @@ def test_full_weight_transfer_flow():
)
# Step 2: Start weight update
llm.start_weight_update(is_checkpoint_format=True)
llm.start_weight_update()
# Step 3: Update weights
llm.update_weights(
-69
View File
@@ -7,7 +7,6 @@ from unittest.mock import Mock
import numpy as np
import pytest
import torch
import torch.nn as nn
import vllm.v1.worker.gpu_model_runner as gpu_model_runner_module
from vllm.config import (
@@ -23,7 +22,6 @@ from vllm.distributed.parallel_state import (
init_distributed_environment,
initialize_model_parallel,
)
from vllm.distributed.weight_transfer.base import SparseWeightPatch
from vllm.lora.layers import LoRAMappingType
from vllm.lora.request import LoRARequest
from vllm.model_executor.layers.attention import Attention
@@ -864,73 +862,6 @@ def test_sample_passes_reordered_draft_probs_to_rejection_sampler():
assert torch.equal(passed_draft_probs, expected_draft_probs)
def test_apply_sparse_weight_patches_updates_only_selected_entries():
class DummyModel(nn.Module):
def __init__(self):
super().__init__()
self.weight = nn.Parameter(torch.zeros(6, dtype=torch.float32))
runner = object.__new__(GPUModelRunner)
runner.model = DummyModel()
runner.apply_sparse_weight_patches(
[
SparseWeightPatch(
name="weight",
indices=torch.tensor([1, 4], dtype=torch.int32),
values=torch.tensor([3.5, -2.0], dtype=torch.float32),
)
]
)
expected = torch.tensor([0.0, 3.5, 0.0, 0.0, -2.0, 0.0], dtype=torch.float32)
assert torch.equal(runner.get_model().weight.data, expected)
def test_apply_sparse_weight_patches_rejects_mismatched_lengths():
class DummyModel(nn.Module):
def __init__(self):
super().__init__()
self.weight = nn.Parameter(torch.zeros(4, dtype=torch.float32))
runner = object.__new__(GPUModelRunner)
runner.model = DummyModel()
with pytest.raises(ValueError, match="matching lengths"):
runner.apply_sparse_weight_patches(
[
SparseWeightPatch(
name="weight",
indices=torch.tensor([1, 2], dtype=torch.int32),
values=torch.tensor([1.0], dtype=torch.float32),
)
]
)
def test_apply_sparse_weight_patches_rejects_non_contiguous_param():
class DummyModel(nn.Module):
def __init__(self):
super().__init__()
self.weight = nn.Parameter(
torch.arange(12, dtype=torch.float32).view(3, 4).t()
)
runner = object.__new__(GPUModelRunner)
runner.model = DummyModel()
with pytest.raises(NotImplementedError, match="contiguous params"):
runner.apply_sparse_weight_patches(
[
SparseWeightPatch(
name="weight",
indices=torch.tensor([1], dtype=torch.int32),
values=torch.tensor([1.0], dtype=torch.float32),
)
]
)
def test_init_kv_cache_with_kv_sharing_invalid_target_layer_order(default_vllm_config):
torch.set_default_dtype(torch.float16)
layer_0 = "model.layers.0.self_attn.attn"
@@ -1,155 +1,94 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for GPUWorker weight-transfer pass-through behavior.
from types import SimpleNamespace
from unittest.mock import MagicMock
The worker no longer contains transport, layerwise, or sparse logic: it only
delegates to the configured weight transfer engine and tracks whether an update
session is active. These tests verify that delegation and the session guard.
"""
import pytest
import torch
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.base import SparseWeightPatch
from vllm.distributed.weight_transfer.nccl_engine import NCCLWeightTransferEngine
from vllm.v1.worker.gpu_worker import Worker
def _make_nccl_engine() -> NCCLWeightTransferEngine:
parallel_config = MagicMock(spec=ParallelConfig)
parallel_config.rank = 0
parallel_config.world_size = 1
parallel_config.data_parallel_rank = 0
parallel_config.data_parallel_index = 0
return NCCLWeightTransferEngine(
WeightTransferConfig(backend="nccl"),
parallel_config,
MagicMock(spec=torch.nn.Module),
)
class _RecordingEngine:
"""Minimal stand-in for a weight transfer engine."""
def __init__(self, raise_on_update: bool = False):
self.raise_on_update = raise_on_update
self.started = False
self.finished = False
self.update_calls: list[dict] = []
def start_weight_update(self) -> None:
self.started = True
def update_weights(self, update_info: dict) -> None:
self.update_calls.append(update_info)
if self.raise_on_update:
raise ValueError("boom")
def finish_weight_update(self) -> None:
self.finished = True
def test_update_weights_sparse_dispatches_to_sparse_receive(monkeypatch):
monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None)
def _make_worker(engine: _RecordingEngine | None) -> Worker:
worker = object.__new__(Worker)
worker.device = "cpu"
worker.parallel_config = SimpleNamespace(world_size=1)
worker.weight_transfer_engine = _make_nccl_engine()
worker._weight_update_active = True
worker._is_checkpoint_format = False
applied_patches = []
def apply_sparse_weight_patches(patches):
applied_patches.extend(patches)
worker.model_runner = SimpleNamespace(
apply_sparse_weight_patches=apply_sparse_weight_patches,
)
received_kinds = []
def receive_sparse_weights(update_info, apply_patches):
received_kinds.append(update_info.update_kind)
apply_patches(
[
SparseWeightPatch(
name="layer.weight",
indices=torch.tensor([1], dtype=torch.int32),
values=torch.tensor([2.0], dtype=torch.float32),
)
]
)
worker.weight_transfer_engine.receive_sparse_weights = receive_sparse_weights
Worker.update_weights(
worker,
{
"names": ["layer.weight"],
"dtype_names": ["float32"],
"shapes": [[4]],
"num_updates_list": [1],
"update_kind": "sparse_flat",
},
)
assert received_kinds == ["sparse_flat"]
assert len(applied_patches) == 1
assert torch.equal(applied_patches[0].indices, torch.tensor([1], dtype=torch.int32))
worker.weight_transfer_engine = engine
worker._weight_update_active = False
return worker
def test_update_weights_sparse_rejects_tp_or_pp(monkeypatch):
monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None)
def test_start_update_finish_delegates_to_engine():
engine = _RecordingEngine()
worker = _make_worker(engine)
worker = object.__new__(Worker)
worker.device = "cpu"
worker.parallel_config = SimpleNamespace(world_size=2)
worker.weight_transfer_engine = _make_nccl_engine()
worker._weight_update_active = True
worker._is_checkpoint_format = False
worker.model_runner = SimpleNamespace(apply_sparse_weight_patches=lambda _: None)
Worker.start_weight_update(worker)
assert engine.started is True
assert worker._weight_update_active is True
with pytest.raises(NotImplementedError, match="TP=1 and PP=1"):
Worker.update_weights(
worker,
{
"names": ["layer.weight"],
"dtype_names": ["float32"],
"shapes": [[4]],
"num_updates_list": [1],
"update_kind": "sparse_flat",
},
)
Worker.update_weights(worker, {"names": ["w"]})
assert engine.update_calls == [{"names": ["w"]}]
assert worker._weight_update_active is True
Worker.finish_weight_update(worker)
assert engine.finished is True
assert worker._weight_update_active is False
assert worker._is_checkpoint_format is True
def test_update_weights_sparse_rejects_checkpoint_format(monkeypatch):
monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None)
def test_double_start_raises():
worker = _make_worker(_RecordingEngine())
Worker.start_weight_update(worker)
with pytest.raises(RuntimeError, match="already"):
Worker.start_weight_update(worker)
worker = object.__new__(Worker)
worker.device = "cpu"
worker.parallel_config = SimpleNamespace(world_size=1)
worker.weight_transfer_engine = _make_nccl_engine()
worker._weight_update_active = True
worker._is_checkpoint_format = True
worker.model_runner = SimpleNamespace(model=MagicMock())
with pytest.raises(ValueError, match="start_weight_update"):
Worker.update_weights(
worker,
{
"names": ["layer.weight"],
"dtype_names": ["float32"],
"shapes": [[4]],
"num_updates_list": [1],
"update_kind": "sparse_flat",
},
)
def test_update_without_start_raises():
worker = _make_worker(_RecordingEngine())
with pytest.raises(RuntimeError, match="start_weight_update must be called"):
Worker.update_weights(worker, {"names": ["w"]})
def test_finish_without_start_raises():
worker = _make_worker(_RecordingEngine())
with pytest.raises(RuntimeError, match="without a matching"):
Worker.finish_weight_update(worker)
def test_update_resets_active_on_error():
engine = _RecordingEngine(raise_on_update=True)
worker = _make_worker(engine)
Worker.start_weight_update(worker)
with pytest.raises(ValueError, match="boom"):
Worker.update_weights(worker, {"names": ["w"]})
# A failed update ends the session so the next start is clean.
assert worker._weight_update_active is False
assert worker._is_checkpoint_format is True
def test_update_weights_resets_state_when_update_info_is_invalid(monkeypatch):
monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None)
worker = object.__new__(Worker)
worker.device = "cpu"
worker.parallel_config = SimpleNamespace(world_size=1)
worker.weight_transfer_engine = _make_nccl_engine()
worker._weight_update_active = True
worker._is_checkpoint_format = False
with pytest.raises(ValueError, match="cannot be empty"):
Worker.update_weights(
worker,
{
"names": [],
"dtype_names": [],
"shapes": [],
"num_updates_list": [],
"update_kind": "sparse_flat",
},
)
assert worker._weight_update_active is False
assert worker._is_checkpoint_format is True
def test_missing_engine_raises():
worker = _make_worker(None)
with pytest.raises(RuntimeError, match="Weight transfer not configured"):
Worker.start_weight_update(worker)
+1 -1
View File
@@ -9,7 +9,7 @@ from vllm.config.utils import config
class WeightTransferConfig:
"""Configuration for weight transfer during RL training."""
backend: Literal["nccl", "ipc"] | str = "nccl"
backend: Literal["nccl", "ipc", "sparse_nccl"] | str = "nccl"
"""The backend to use for weight transfer. Validated against the
`WeightTransferEngineFactory` registry at engine creation time.
"""
+60 -74
View File
@@ -3,12 +3,15 @@
"""Base class for weight transfer engines."""
from abc import ABC, abstractmethod
from collections.abc import Callable, Iterator
from dataclasses import KW_ONLY, dataclass, field
from typing import Any, Generic, Literal, TypeVar
from collections.abc import Iterator
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Generic, TypeVar
import torch
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
@@ -28,44 +31,7 @@ class WeightTransferInitInfo(ABC): # noqa: B024
class WeightTransferUpdateInfo(ABC): # noqa: B024
"""Base class for backend-specific weight update info."""
_: KW_ONLY
update_kind: Literal["dense", "sparse_flat"] = "dense"
"""Weight update format."""
num_updates_list: list[int] | None = None
"""Number of sparse entries to receive for each parameter in ``names``."""
def __post_init__(self) -> None:
if self.update_kind not in ("dense", "sparse_flat"):
raise ValueError(f"Unsupported update_kind: {self.update_kind}")
if self.update_kind == "dense":
if self.num_updates_list is not None:
raise ValueError(
"Sparse metadata is only supported for `update_kind='sparse_flat'`"
)
return
if self.num_updates_list is None:
raise ValueError("`num_updates_list` is required for sparse updates")
if len(self.num_updates_list) == 0:
raise ValueError("`num_updates_list` cannot be empty for sparse updates")
if any(num_updates < 0 for num_updates in self.num_updates_list):
raise ValueError("Sparse `num_updates_list` entries must be non-negative")
names = getattr(self, "names", None)
if names is not None and len(self.num_updates_list) != len(names):
raise ValueError(
f"`num_updates_list` should be of the same size as `names`: "
f"got {len(self.num_updates_list)} and {len(names)}"
)
@dataclass
class SparseWeightPatch:
"""A sparse in-place patch for one existing parameter."""
name: str
indices: torch.Tensor
values: torch.Tensor
pass
# API-level request classes (accept dicts for backend-agnostic serialization)
@@ -89,9 +55,15 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
from a trainer to inference workers.
This abstraction separates weight transfer transport logic from the worker
implementation, allowing different backends (NCCL, CUDA IPC[TODO], RDMA[TODO]) to be
implementation, allowing different backends (NCCL, CUDA IPC, RDMA[TODO]) to be
plugged in.
Each engine owns its full weight-update lifecycle: `start_weight_update`,
`update_weights`, and `finish_weight_update`. Layerwise reloading (used by
checkpoint-format engines) is opted into per engine by running it inside
`start_weight_update`/`finish_weight_update`. Engines that apply weights in
place (e.g. sparse patches) leave those methods as no-ops.
Subclasses should define:
init_info_cls: Type of backend-specific initialization info
update_info_cls: Type of backend-specific update info
@@ -104,7 +76,8 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
def __init__(
self,
config: WeightTransferConfig,
parallel_config: ParallelConfig,
vllm_config: "VllmConfig",
device: torch.device,
model: torch.nn.Module,
) -> None:
"""
@@ -112,11 +85,15 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
Args:
config: The configuration for the weight transfer engine
parallel_config: The configuration for the parallel setup
vllm_config: The full vLLM config (provides parallel/model config)
device: The device this worker's model lives on
model: The local model instance which will receive the weights
"""
self.config = config
self.parallel_config = parallel_config
self.vllm_config = vllm_config
self.parallel_config: ParallelConfig = vllm_config.parallel_config
self.model_config = vllm_config.model_config
self.device = device
self.model = model
def parse_init_info(self, init_dict: dict[str, Any]) -> TInitInfo:
@@ -171,32 +148,50 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
raise NotImplementedError
@abstractmethod
def receive_weights(
self,
update_info: TUpdateInfo,
load_weights: Callable[[list[tuple[str, torch.Tensor]]], None],
) -> None:
def start_weight_update(self) -> None:
"""
Receive weights from the trainer and load them incrementally.
Prepare the engine for a new weight update.
Engines that receive weights in checkpoint format initialize layerwise reloading
here, else this is typically a no-op.
See: https://docs.vllm.ai/en/latest/training/layerwise/ for more details.
"""
raise NotImplementedError
@abstractmethod
def finish_weight_update(self) -> None:
"""
Finalize the current weight update.
Checkpoint-format engines finalize layerwise reloading here; engines
that apply weights in place leave this as a no-op.
"""
raise NotImplementedError
def update_weights(self, update_info: dict[str, Any]) -> None:
"""
Receive one weight update chunk and load it into the model.
Args:
update_info: Dictionary containing backend-specific update info
"""
typed_update_info = self.parse_update_info(update_info)
self.receive_weights(typed_update_info)
# NCCL broadcast / IPC paths may be asynchronous. Synchronize here so the
# next step uses the new weights.
torch.accelerator.synchronize()
@abstractmethod
def receive_weights(self, update_info: TUpdateInfo) -> None:
"""
Receive weights from the trainer and load them into the model.
Args:
update_info: Backend-specific update info containing parameter metadata
and any backend-specific data
load_weights: Callable that loads weights into the model. Called
incrementally for each weight to avoid OOM.
"""
raise NotImplementedError
def receive_sparse_weights(
self,
update_info: TUpdateInfo,
apply_patches: Callable[[list[SparseWeightPatch]], None],
) -> None:
"""Receive sparse weight patches from the trainer."""
raise NotImplementedError(
f"{self.__class__.__name__} does not support sparse weight updates"
)
@abstractmethod
def shutdown(self) -> None:
"""
@@ -208,7 +203,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
@staticmethod
@abstractmethod
def trainer_send_weights(
iterator: Iterator[tuple[str, torch.Tensor]],
iterator: Iterator[Any],
trainer_args: dict[str, Any] | Any,
) -> None:
"""
@@ -218,8 +213,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
to send weights to all inference workers.
Args:
iterator: Iterator of model parameters. Returns (name, tensor) tuples.
The tensors should be on the appropriate device for the backend.
iterator: Iterator of backend-specific items to send.
trainer_args: Dictionary containing backend-specific arguments needed
to send weights. The structure depends on the backend:
- NCCL: Contains 'group', 'src', 'packed', etc.
@@ -231,11 +225,3 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
>>> engine.trainer_send_weights(param_iter, trainer_args)
"""
raise NotImplementedError
@staticmethod
def trainer_send_sparse_weights(
_iterator: Iterator[SparseWeightPatch],
_trainer_args: dict[str, Any] | Any,
) -> None:
"""Send sparse weight patches from trainer to inference workers."""
raise NotImplementedError("Sparse weight updates are not supported")
+12 -4
View File
@@ -12,7 +12,7 @@ from vllm.logger import init_logger
if TYPE_CHECKING:
import torch
from vllm.config.parallel import ParallelConfig
from vllm.config import VllmConfig
from vllm.config.weight_transfer import WeightTransferConfig
logger = init_logger(__name__)
@@ -76,14 +76,16 @@ class WeightTransferEngineFactory:
def create_engine(
cls,
config: "WeightTransferConfig",
parallel_config: "ParallelConfig",
vllm_config: "VllmConfig",
device: "torch.device",
model: "torch.nn.Module",
) -> WeightTransferEngine:
"""Create a weight transfer engine instance.
Args:
config: Weight transfer configuration containing the backend name
parallel_config: Parallel configuration for the engine
vllm_config: The full vLLM config (provides parallel/model config)
device: The device this worker's model lives on
model: The local model instance which will receive the weights
Returns:
@@ -106,7 +108,7 @@ class WeightTransferEngineFactory:
engine_cls.__name__,
)
return engine_cls(config, parallel_config, model)
return engine_cls(config, vllm_config, device, model)
# Register built-in weight transfer engines here.
@@ -124,3 +126,9 @@ WeightTransferEngineFactory.register_engine(
"vllm.distributed.weight_transfer.ipc_engine",
"IPCWeightTransferEngine",
)
WeightTransferEngineFactory.register_engine(
"sparse_nccl",
"vllm.distributed.weight_transfer.sparse_nccl_engine",
"SparseNCCLWeightTransferEngine",
)
+34 -20
View File
@@ -5,7 +5,7 @@
import pickle
from collections.abc import Callable, Iterator
from dataclasses import asdict, dataclass
from typing import Any
from typing import TYPE_CHECKING, Any
import pybase64 as base64
import ray
@@ -14,13 +14,15 @@ import torch
from torch.multiprocessing.reductions import rebuild_cuda_tensor, reduce_tensor
from vllm import envs
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.base import (
WeightTransferEngine,
WeightTransferInitInfo,
WeightTransferUpdateInfo,
)
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.distributed.weight_transfer.packed_tensor import (
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
packed_ipc_consumer,
@@ -87,10 +89,6 @@ class IPCWeightTransferUpdateInfo(WeightTransferUpdateInfo):
"""Whether this update uses packed tensor format."""
def __post_init__(self):
super().__post_init__()
if self.update_kind != "dense":
raise NotImplementedError("IPC weight transfer only supports dense updates")
if self.ipc_handles_pickled is not None:
if self.ipc_handles is not None:
raise ValueError(
@@ -152,7 +150,8 @@ class IPCWeightTransferEngine(
def __init__(
self,
config: WeightTransferConfig,
parallel_config: ParallelConfig,
vllm_config: "VllmConfig",
device: torch.device,
model: torch.nn.Module,
) -> None:
"""
@@ -160,10 +159,11 @@ class IPCWeightTransferEngine(
Args:
config: The configuration for the weight transfer engine
parallel_config: The configuration for the parallel setup
vllm_config: The full vLLM config
device: The device this worker's model lives on
model: The local model instance which will receive the weights
"""
super().__init__(config, parallel_config, model)
super().__init__(config, vllm_config, device, model)
def parse_update_info(
self, update_dict: dict[str, Any]
@@ -206,22 +206,37 @@ class IPCWeightTransferEngine(
"""
pass
def receive_weights(
self,
update_info: IPCWeightTransferUpdateInfo,
load_weights: Callable[[list[tuple[str, torch.Tensor]]], None],
) -> None:
def start_weight_update(self) -> None:
"""Initialize layerwise reloading for the incoming checkpoint weights."""
from vllm.model_executor.model_loader.reload import (
initialize_layerwise_reload,
)
initialize_layerwise_reload(self.model)
def finish_weight_update(self) -> None:
"""Finalize layerwise reloading after all weights have been received."""
from vllm.model_executor.model_loader.reload import (
finalize_layerwise_reload,
)
finalize_layerwise_reload(self.model, self.model_config)
def receive_weights(self, update_info: IPCWeightTransferUpdateInfo) -> None:
"""
Receive weights from the trainer via CUDA IPC handles.
Receive weights from the trainer via CUDA IPC handles and load them.
Args:
update_info: IPC update info containing parameter names, dtypes, shapes,
and IPC handles. Each IPC handle is a mapping between physical
GPU UUID and the rebuild_cuda_tensor args tuple.
load_weights: Callable that loads weights into the model. Called
incrementally for each weight to avoid OOM.
"""
device_index = torch.accelerator.current_device_index()
# Use the worker's assigned device rather than the ambient current
# device: the receive path is no longer wrapped in
# `with torch.device(self.device)` by the caller, so the current device
# is not guaranteed to match self.device. The IPC tensors must be
# rebuilt on the device the model lives on.
device_index = self.device.index
if update_info.packed:
assert update_info.tensor_sizes is not None
@@ -234,7 +249,6 @@ class IPCWeightTransferEngine(
tensor_sizes=update_info.tensor_sizes,
device_index=device_index,
)
load_weights(weights)
else:
assert isinstance(update_info.ipc_handles, list)
weights = []
@@ -260,7 +274,7 @@ class IPCWeightTransferEngine(
weight = rebuild_cuda_tensor(*list_args)
weights.append((name, weight))
load_weights(weights)
self.model.load_weights(weights)
def shutdown(self) -> None:
pass
@@ -0,0 +1,120 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared NCCL initialization helpers for weight transfer engines.
The dense (`NCCLWeightTransferEngine`) and sparse
(`SparseNCCLWeightTransferEngine`) backends are independent engines that share
*only* their process-group initialization. That common logic lives here so the
sparse engine does not have to subclass the dense one.
"""
from dataclasses import dataclass
from typing import TYPE_CHECKING
import torch
if TYPE_CHECKING:
from vllm.config.parallel import ParallelConfig
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.weight_transfer.base import WeightTransferInitInfo
@dataclass
class NCCLWeightTransferInitInfo(WeightTransferInitInfo):
"""Initialization info for NCCL-based weight transfer backends."""
master_address: str
master_port: int
rank_offset: int
world_size: int
def stateless_init_process_group(
master_address: str,
master_port: int,
rank: int,
world_size: int,
device,
) -> "PyNcclCommunicator":
"""
vLLM provides `StatelessProcessGroup` to create a process group
without considering the global process group in torch.distributed.
It is recommended to create `StatelessProcessGroup`, and then initialize
the data-plane communication (NCCL) between external (train processes)
and vLLM workers.
"""
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.utils import StatelessProcessGroup
pg = StatelessProcessGroup.create(
host=master_address, port=master_port, rank=rank, world_size=world_size
)
return PyNcclCommunicator(pg, device=device)
def worker_init_process_group(
init_info: NCCLWeightTransferInitInfo,
parallel_config: "ParallelConfig",
) -> "PyNcclCommunicator":
"""Create the trainer<->worker NCCL group on an inference worker.
Computes a unique rank for this worker across all data-parallel groups and
joins the stateless process group with the trainer.
"""
# Calculate the global rank in the trainer-worker process group.
# Must account for data parallel to get unique ranks across all workers.
dp_rank = parallel_config.data_parallel_index
world_size_per_dp = parallel_config.world_size # TP * PP
rank_within_dp = parallel_config.rank
# Unique rank across all DP groups
worker_rank = dp_rank * world_size_per_dp + rank_within_dp
rank = worker_rank + init_info.rank_offset
device = torch.accelerator.current_device_index()
return stateless_init_process_group(
init_info.master_address,
init_info.master_port,
rank,
init_info.world_size,
device=device,
)
def trainer_init(
init_info: NCCLWeightTransferInitInfo | dict,
) -> "PyNcclCommunicator":
"""
Initialize NCCL process group for trainer-side weight transfer.
The trainer is always rank 0 in the process group. Uses the current
CUDA device (torch.accelerator.current_device_index()).
Args:
init_info: Either an NCCLWeightTransferInitInfo object or a dict with keys:
- master_address: str
- master_port: int
- world_size: int
Returns:
PyNcclCommunicator for weight transfer.
"""
if isinstance(init_info, dict):
master_address = init_info["master_address"]
master_port = init_info["master_port"]
world_size = init_info["world_size"]
else:
master_address = init_info.master_address
master_port = init_info.master_port
world_size = init_info.world_size
# Trainer is always rank 0
device = torch.accelerator.current_device_index()
return stateless_init_process_group(
master_address,
master_port,
0,
world_size,
device,
)
+49 -197
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""NCCL-based weight transfer engine."""
"""NCCL-based (dense) weight transfer engine."""
from collections.abc import Callable, Iterator
from dataclasses import dataclass
@@ -9,31 +9,32 @@ from typing import TYPE_CHECKING, Any
import torch
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.base import (
SparseWeightPatch,
WeightTransferEngine,
WeightTransferInitInfo,
WeightTransferUpdateInfo,
)
from vllm.distributed.weight_transfer.nccl_common import (
NCCLWeightTransferInitInfo,
trainer_init,
worker_init_process_group,
)
from vllm.distributed.weight_transfer.packed_tensor import (
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
DEFAULT_PACKED_NUM_BUFFERS,
packed_nccl_broadcast_consumer,
)
@dataclass
class NCCLWeightTransferInitInfo(WeightTransferInitInfo):
"""Initialization info for NCCL weight transfer backend."""
master_address: str
master_port: int
rank_offset: int
world_size: int
# Re-exported for backward compatibility; canonical home is nccl_common.
__all__ = [
"NCCLWeightTransferInitInfo",
"NCCLTrainerSendWeightsArgs",
"NCCLWeightTransferUpdateInfo",
"NCCLWeightTransferEngine",
]
@dataclass
@@ -82,7 +83,6 @@ class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
def __post_init__(self):
"""Validate that all lists have the same length."""
super().__post_init__()
num_params = len(self.names)
if len(self.dtype_names) != num_params:
raise ValueError(
@@ -94,13 +94,6 @@ class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
f"`shapes` should be of the same size as `names`: "
f"got {len(self.shapes)} and {len(self.names)}"
)
if self.update_kind == "dense":
return
if self.packed:
raise ValueError(
"`update_kind='sparse_flat'` cannot be combined with `packed=True`"
)
class NCCLWeightTransferEngine(
@@ -109,8 +102,10 @@ class NCCLWeightTransferEngine(
"""
Weight transfer engine using NCCL for communication between trainer and workers.
This implementation uses NCCL broadcast operations to transfer weights from
the trainer (rank 0) to all inference workers in a process group.
This implementation uses NCCL broadcast operations to transfer dense
checkpoint-format weights from the trainer (rank 0) to all inference workers
in a process group. Received weights are loaded via the model's
`load_weights` using the layerwise reload lifecycle.
"""
# Define backend-specific dataclass types
@@ -120,18 +115,11 @@ class NCCLWeightTransferEngine(
def __init__(
self,
config: WeightTransferConfig,
parallel_config: ParallelConfig,
vllm_config: "VllmConfig",
device: torch.device,
model: torch.nn.Module,
) -> None:
"""
Initialize the NCCL weight transfer engine.
Args:
config: The configuration for the weight transfer engine
parallel_config: The configuration for the parallel setup
model: The local model instance which will receive the weights
"""
super().__init__(config, parallel_config, model)
super().__init__(config, vllm_config, device, model)
self.model_update_group: PyNcclCommunicator | None = None
def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None:
@@ -142,35 +130,29 @@ class NCCLWeightTransferEngine(
init_info: NCCL initialization info containing master address, port,
rank offset, and world size
"""
# Calculate the global rank in the trainer-worker process group
# Must account for data parallel to get unique ranks across all workers
dp_rank = self.parallel_config.data_parallel_index
world_size_per_dp = self.parallel_config.world_size # TP * PP
rank_within_dp = self.parallel_config.rank
# Unique rank across all DP groups
worker_rank = dp_rank * world_size_per_dp + rank_within_dp
rank = worker_rank + init_info.rank_offset
# Create stateless process group
device = torch.accelerator.current_device_index()
self.model_update_group = (
NCCLWeightTransferEngine._stateless_init_process_group(
init_info.master_address,
init_info.master_port,
rank,
init_info.world_size,
device=device,
)
self.model_update_group = worker_init_process_group(
init_info, self.parallel_config
)
def receive_weights(
self,
update_info: NCCLWeightTransferUpdateInfo,
load_weights: Callable[[list[tuple[str, torch.Tensor]]], None],
) -> None:
def start_weight_update(self) -> None:
"""Initialize layerwise reloading for the incoming checkpoint weights."""
from vllm.model_executor.model_loader.reload import (
initialize_layerwise_reload,
)
initialize_layerwise_reload(self.model)
def finish_weight_update(self) -> None:
"""Finalize layerwise reloading after all weights have been received."""
from vllm.model_executor.model_loader.reload import (
finalize_layerwise_reload,
)
finalize_layerwise_reload(self.model, self.model_config)
def receive_weights(self, update_info: NCCLWeightTransferUpdateInfo) -> None:
"""
Receive weights from trainer via NCCL broadcast and load them incrementally.
Receive weights from trainer via NCCL broadcast.
If update_info.packed is True, uses packed tensor broadcasting for
efficient transfer of multiple weights in batches. Otherwise, uses simple
@@ -179,19 +161,12 @@ class NCCLWeightTransferEngine(
Args:
update_info: NCCL update info containing parameter names, dtypes, shapes,
and packed flag
load_weights: Callable that loads weights into the model. Called
incrementally for each batch of weights to avoid OOM.
"""
if self.model_update_group is None:
raise RuntimeError(
"NCCL weight transfer not initialized. "
"Call init_transfer_engine() first."
)
if update_info.update_kind != "dense":
raise ValueError(
"Sparse updates must use `receive_sparse_weights`, not "
"`receive_weights`"
)
if update_info.packed:
# Build iterator of (name, (shape, dtype)) from update_info
@@ -206,9 +181,10 @@ class NCCLWeightTransferEngine(
iterator=state_dict_info_iterator(),
group=self.model_update_group,
src=0,
post_unpack_func=load_weights,
post_unpack_func=self.model.load_weights,
buffer_size_bytes=update_info.packed_buffer_size_bytes,
num_buffers=update_info.packed_num_buffers,
device=self.device,
)
else:
# Use simple one-by-one broadcasting
@@ -216,49 +192,13 @@ class NCCLWeightTransferEngine(
update_info.names, update_info.dtype_names, update_info.shapes
):
dtype = getattr(torch, dtype_name)
weight = torch.empty(shape, dtype=dtype, device="cuda")
weight = torch.empty(shape, dtype=dtype, device=self.device)
self.model_update_group.broadcast(
weight, src=0, stream=torch.cuda.current_stream()
)
load_weights([(name, weight)])
self.model.load_weights([(name, weight)])
del weight
def receive_sparse_weights(
self,
update_info: NCCLWeightTransferUpdateInfo,
apply_patches: Callable[[list[SparseWeightPatch]], None],
) -> None:
"""Receive sparse flat-index patches from trainer via NCCL."""
if self.model_update_group is None:
raise RuntimeError(
"NCCL weight transfer not initialized. "
"Call init_transfer_engine() first."
)
if update_info.update_kind != "sparse_flat":
raise ValueError("Sparse receive path requires `update_kind='sparse_flat'`")
assert update_info.num_updates_list is not None
for name, dtype_name, num_updates in zip(
update_info.names,
update_info.dtype_names,
update_info.num_updates_list,
):
dtype = getattr(torch, dtype_name)
device = torch.accelerator.current_device_index()
indices = torch.empty(num_updates, dtype=torch.int32, device=device)
values = torch.empty(num_updates, dtype=dtype, device=device)
self.model_update_group.broadcast(
indices, src=0, stream=torch.cuda.current_stream()
)
self.model_update_group.broadcast(
values, src=0, stream=torch.cuda.current_stream()
)
apply_patches(
[SparseWeightPatch(name=name, indices=indices, values=values)]
)
del indices
del values
def shutdown(self) -> None:
if self.model_update_group is not None:
# Clean up the communicator by removing the reference
@@ -269,7 +209,7 @@ class NCCLWeightTransferEngine(
iterator: Iterator[tuple[str, torch.Tensor]],
trainer_args: dict[str, Any] | NCCLTrainerSendWeightsArgs,
) -> None:
"""Broadcast weights from trainer to vLLM workers.
"""Broadcast dense weights from trainer to vLLM workers.
Args:
iterator: Iterator of model parameters. Returns (name, tensor) tuples
@@ -322,94 +262,6 @@ class NCCLWeightTransferEngine(
stream=args.stream or torch.cuda.current_stream(),
)
@staticmethod
def trainer_send_sparse_weights(
iterator: Iterator[SparseWeightPatch],
trainer_args: dict[str, Any] | NCCLTrainerSendWeightsArgs,
) -> None:
"""Broadcast sparse flat-index patches from trainer to vLLM workers."""
if isinstance(trainer_args, dict):
args = NCCLTrainerSendWeightsArgs(**trainer_args)
else:
args = trainer_args
if args.packed:
raise ValueError(
"Sparse NCCL updates cannot be combined with `packed=True`"
)
stream = args.stream or torch.cuda.current_stream()
for patch in iterator:
args.group.broadcast(patch.indices, src=args.src, stream=stream)
args.group.broadcast(patch.values, src=args.src, stream=stream)
@staticmethod
def trainer_init(
init_info: NCCLWeightTransferInitInfo | dict,
) -> "PyNcclCommunicator":
"""
Initialize NCCL process group for trainer-side weight transfer.
The trainer is always rank 0 in the process group. Uses the current
CUDA device (torch.accelerator.current_device_index()).
Args:
init_info: Either an NCCLWeightTransferInitInfo object or a dict with keys:
- master_address: str
- master_port: int
- world_size: int
Returns:
PyNcclCommunicator for weight transfer.
Example:
>>> from vllm.distributed.weight_transfer.nccl_engine import (
... NCCLWeightTransferEngine,
... )
>>> group = NCCLWeightTransferEngine.trainer_init(
... dict(
... master_address=master_address,
... master_port=master_port,
... world_size=world_size,
... ),
... )
"""
if isinstance(init_info, dict):
master_address = init_info["master_address"]
master_port = init_info["master_port"]
world_size = init_info["world_size"]
else:
# NCCLWeightTransferInitInfo object
master_address = init_info.master_address
master_port = init_info.master_port
world_size = init_info.world_size
# Trainer is always rank 0
device = torch.accelerator.current_device_index()
return NCCLWeightTransferEngine._stateless_init_process_group(
master_address,
master_port,
0,
world_size,
device,
)
@staticmethod
def _stateless_init_process_group(
master_address, master_port, rank, world_size, device
):
"""
vLLM provides `StatelessProcessGroup` to create a process group
without considering the global process group in torch.distributed.
It is recommended to create `StatelessProcessGroup`, and then initialize
the data-plane communication (NCCL) between external (train processes)
and vLLM workers.
"""
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.utils import StatelessProcessGroup
pg = StatelessProcessGroup.create(
host=master_address, port=master_port, rank=rank, world_size=world_size
)
pynccl = PyNcclCommunicator(pg, device=device)
return pynccl
# Trainer-side process-group setup. Delegates to the shared helper so the
# sparse engine can reuse the exact same rendezvous without subclassing.
trainer_init = staticmethod(trainer_init)
@@ -186,6 +186,7 @@ def packed_nccl_broadcast_consumer(
post_unpack_func: Callable[[list[tuple[str, torch.Tensor]]], None],
buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS,
device: torch.device | str = "cuda",
) -> None:
"""Consume packed tensors and unpack them into a list of tensors.
@@ -199,6 +200,8 @@ def packed_nccl_broadcast_consumer(
Both producer and consumer must use the same value.
num_buffers: Number of buffers for double/triple buffering.
Both producer and consumer must use the same value.
device: Device for the receive buffers. Must match the device the NCCL
communicator was created on (the worker's assigned device).
"""
target_packed_tensor_size = buffer_size_bytes
@@ -211,7 +214,7 @@ def packed_nccl_broadcast_consumer(
]
packing_tensor_sizes: list[int] = [0 for _ in range(num_buffers)]
packed_tensors: list[torch.Tensor] = [
torch.empty(0, dtype=torch.uint8, device="cuda") for _ in range(num_buffers)
torch.empty(0, dtype=torch.uint8, device=device) for _ in range(num_buffers)
]
while True:
@@ -234,7 +237,7 @@ def packed_nccl_broadcast_consumer(
break
# Create a packed tensor and broadcast it
packed_tensors[buffer_idx] = torch.empty(
packing_tensor_sizes[buffer_idx], dtype=torch.uint8, device="cuda"
packing_tensor_sizes[buffer_idx], dtype=torch.uint8, device=device
)
group.broadcast(packed_tensors[buffer_idx], src=src)
# Load the packed tensor into the model
@@ -259,7 +262,7 @@ def packed_nccl_broadcast_consumer(
packed_tensors[buffer_idx] = torch.empty(
packing_tensor_sizes[buffer_idx],
dtype=torch.uint8,
device="cuda",
device=device,
)
group.broadcast(packed_tensors[buffer_idx], src=src)
# Load the packed tensor into the model
@@ -0,0 +1,223 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Sparse NCCL weight transfer engine.
A standalone engine (not a subclass of `NCCLWeightTransferEngine`) for applying
sparse, flat-index weight patches in place. It shares only NCCL process-group
initialization with the dense engine (via `nccl_common`); the update path
applies index/value patches directly to existing model parameters and never runs
layerwise reload.
MVP limitations:
* TP=1 and PP=1 only
* uses runtime/kernel-format parameter names
* not composable with checkpoint-format or packed updates
"""
from collections.abc import Iterator
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
import torch
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.base import (
WeightTransferEngine,
WeightTransferUpdateInfo,
)
from vllm.distributed.weight_transfer.nccl_common import (
NCCLWeightTransferInitInfo,
trainer_init,
worker_init_process_group,
)
from vllm.distributed.weight_transfer.nccl_engine import NCCLTrainerSendWeightsArgs
__all__ = [
"SparseWeightPatch",
"SparseNCCLWeightTransferUpdateInfo",
"SparseNCCLWeightTransferEngine",
]
@dataclass
class SparseWeightPatch:
"""A sparse in-place patch for one existing parameter."""
name: str
indices: torch.Tensor
values: torch.Tensor
@dataclass
class SparseNCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
"""Update info for the sparse NCCL weight transfer backend."""
names: list[str]
dtype_names: list[str]
shapes: list[list[int]]
num_updates_list: list[int]
"""Number of sparse entries to receive for each parameter in ``names``."""
def __post_init__(self) -> None:
num_params = len(self.names)
if len(self.dtype_names) != num_params:
raise ValueError(
f"`dtype_names` should be of the same size as `names`: "
f"got {len(self.dtype_names)} and {len(self.names)}"
)
if len(self.shapes) != num_params:
raise ValueError(
f"`shapes` should be of the same size as `names`: "
f"got {len(self.shapes)} and {len(self.names)}"
)
if len(self.num_updates_list) == 0:
raise ValueError("`num_updates_list` cannot be empty for sparse updates")
if len(self.num_updates_list) != num_params:
raise ValueError(
f"`num_updates_list` should be of the same size as `names`: "
f"got {len(self.num_updates_list)} and {len(self.names)}"
)
if any(num_updates < 0 for num_updates in self.num_updates_list):
raise ValueError("Sparse `num_updates_list` entries must be non-negative")
class SparseNCCLWeightTransferEngine(
WeightTransferEngine[NCCLWeightTransferInitInfo, SparseNCCLWeightTransferUpdateInfo]
):
"""
Sparse weight transfer engine using NCCL.
Receives flat-index (indices, values) patches broadcast from the trainer
(rank 0) and applies them in place to existing model parameters. Weights are
applied directly without layerwise reload, so `start_weight_update` and
`finish_weight_update` are no-ops.
"""
init_info_cls = NCCLWeightTransferInitInfo
update_info_cls = SparseNCCLWeightTransferUpdateInfo
def __init__(
self,
config: WeightTransferConfig,
vllm_config: "VllmConfig",
device: torch.device,
model: torch.nn.Module,
) -> None:
super().__init__(config, vllm_config, device, model)
self.model_update_group: PyNcclCommunicator | None = None
def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None:
"""Initialize the NCCL process group with the trainer."""
self.model_update_group = worker_init_process_group(
init_info, self.parallel_config
)
def start_weight_update(self) -> None:
"""No-op: sparse patches are applied in place, no layerwise reload."""
if self.parallel_config.world_size != 1:
raise NotImplementedError(
"Sparse weight updates currently require TP=1 and PP=1"
)
def finish_weight_update(self) -> None:
"""No-op: sparse patches are applied in place, no layerwise reload."""
pass
def receive_weights(self, update_info: SparseNCCLWeightTransferUpdateInfo) -> None:
"""Receive sparse flat-index patches from the trainer and apply them."""
if self.model_update_group is None:
raise RuntimeError(
"NCCL weight transfer not initialized. "
"Call init_transfer_engine() first."
)
# Use the worker's assigned device rather than the ambient current
# device: the receive path is no longer wrapped in
# `with torch.device(self.device)` by the caller, so the current device
# is not guaranteed to match self.device. The recv buffers must live on
# the same device as the NCCL communicator (created on self.device).
device = self.device
for name, dtype_name, num_updates in zip(
update_info.names,
update_info.dtype_names,
update_info.num_updates_list,
):
dtype = getattr(torch, dtype_name)
indices = torch.empty(num_updates, dtype=torch.int32, device=device)
values = torch.empty(num_updates, dtype=dtype, device=device)
self.model_update_group.broadcast(
indices, src=0, stream=torch.cuda.current_stream()
)
self.model_update_group.broadcast(
values, src=0, stream=torch.cuda.current_stream()
)
self._apply_patch(
SparseWeightPatch(name=name, indices=indices, values=values)
)
del indices
del values
def _apply_patch(self, patch: SparseWeightPatch) -> None:
"""Apply a single sparse flat-index patch to an existing model param."""
param = self.model.get_parameter(patch.name)
if not param.data.is_contiguous():
raise NotImplementedError(
"Sparse weight updates currently require contiguous params: "
f"{patch.name}"
)
if patch.indices.dtype != torch.int32:
raise ValueError(
f"Sparse weight updates currently require int32 indices: {patch.name}"
)
if patch.indices.ndim != 1 or patch.values.ndim != 1:
raise ValueError(
f"Sparse weight patches must be 1D flattened updates: {patch.name}"
)
if patch.indices.numel() != patch.values.numel():
raise ValueError(
f"`indices` and `values` must have matching lengths for {patch.name}"
)
if patch.values.dtype != param.dtype:
raise ValueError(
f"Sparse values dtype {patch.values.dtype} does not match "
f"parameter dtype {param.dtype} for {patch.name}"
)
flat_param = param.data.view(-1)
flat_param.index_copy_(
0,
patch.indices.to(device=flat_param.device, dtype=torch.long),
patch.values.to(device=flat_param.device),
)
def shutdown(self) -> None:
if self.model_update_group is not None:
self.model_update_group = None
@staticmethod
def trainer_send_weights(
iterator: Iterator[SparseWeightPatch],
trainer_args: dict[str, Any] | NCCLTrainerSendWeightsArgs,
) -> None:
"""Broadcast sparse flat-index patches from trainer to vLLM workers."""
if isinstance(trainer_args, dict):
args = NCCLTrainerSendWeightsArgs(**trainer_args)
else:
args = trainer_args
if args.packed:
raise ValueError(
"Sparse NCCL updates cannot be combined with `packed=True`"
)
stream = args.stream or torch.cuda.current_stream()
for patch in iterator:
args.group.broadcast(patch.indices, src=args.src, stream=stream)
args.group.broadcast(patch.values, src=args.src, stream=stream)
# Trainer-side process-group setup (shared with the dense engine).
trainer_init = staticmethod(trainer_init)
+1 -1
View File
@@ -244,7 +244,7 @@ class EngineClient(ABC):
"""Initialize weight transfer for RL training."""
raise NotImplementedError
async def start_weight_update(self, is_checkpoint_format: bool = True) -> None:
async def start_weight_update(self) -> None:
"""Start a new weight update."""
raise NotImplementedError
+2 -5
View File
@@ -873,12 +873,9 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
"init_weight_transfer_engine", kwargs={"init_info": init_info_dict}
)
def start_weight_update(self, is_checkpoint_format: bool = True) -> None:
def start_weight_update(self) -> None:
"""Start a new weight update."""
self.llm_engine.collective_rpc(
"start_weight_update",
kwargs={"is_checkpoint_format": is_checkpoint_format},
)
self.llm_engine.collective_rpc("start_weight_update")
def update_weights(self, request: WeightTransferUpdateRequest | dict) -> None:
"""
@@ -129,14 +129,7 @@ async def init_weight_transfer_engine(raw_request: Request):
@router.post("/start_weight_update")
async def start_weight_update(raw_request: Request):
try:
body = await raw_request.json()
except json.JSONDecodeError as e:
raise HTTPException(status_code=400, detail="Invalid JSON format") from e # noqa: B904
is_checkpoint_format = body.get("is_checkpoint_format", True)
await engine_client(raw_request).start_weight_update(
is_checkpoint_format=is_checkpoint_format
)
await engine_client(raw_request).start_weight_update()
return JSONResponse(content={"message": "Weight update started"})
+2 -5
View File
@@ -1080,12 +1080,9 @@ class AsyncLLM(EngineClient):
"init_weight_transfer_engine", kwargs={"init_info": init_info_dict}
)
async def start_weight_update(self, is_checkpoint_format: bool = True) -> None:
async def start_weight_update(self) -> None:
"""Start a new weight update."""
await self.collective_rpc(
"start_weight_update",
kwargs={"is_checkpoint_format": is_checkpoint_format},
)
await self.collective_rpc("start_weight_update")
async def update_weights(self, request: WeightTransferUpdateRequest) -> None:
"""
-6
View File
@@ -377,12 +377,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
GPUModelRunnerV1.reload_weights(self, *args, **kwargs) # type: ignore[arg-type]
def apply_sparse_weight_patches(self, *args, **kwargs) -> None:
# TODO: Use full version instead of import when fully migrated to v2
from vllm.v1.worker.gpu_model_runner import GPUModelRunner as GPUModelRunnerV1
GPUModelRunnerV1.apply_sparse_weight_patches(self, *args, **kwargs) # type: ignore[arg-type]
def update_config(self, *args, **kwargs) -> None:
# TODO(Wentao): Use full version instead of import when fully migrated to v2
from vllm.v1.worker.gpu_model_runner import GPUModelRunner as GPUModelRunnerV1
-39
View File
@@ -49,7 +49,6 @@ from vllm.distributed.parallel_state import (
is_global_first_rank,
prepare_communication_buffer_for_model,
)
from vllm.distributed.weight_transfer.base import SparseWeightPatch
from vllm.forward_context import (
BatchDescriptor,
set_forward_context,
@@ -3244,44 +3243,6 @@ class GPUModelRunner(
return self.model.unwrap()
return self.model
def apply_sparse_weight_patches(self, patches: Iterable[SparseWeightPatch]) -> None:
"""Apply sparse flat-index patches directly to existing model params."""
model = self.get_model()
for patch in patches:
param = model.get_parameter(patch.name)
if not param.data.is_contiguous():
raise NotImplementedError(
"Sparse weight updates currently require contiguous params: "
f"{patch.name}"
)
if patch.indices.dtype != torch.int32:
raise ValueError(
"Sparse weight updates currently require int32 indices: "
f"{patch.name}"
)
if patch.indices.ndim != 1 or patch.values.ndim != 1:
raise ValueError(
f"Sparse weight patches must be 1D flattened updates: {patch.name}"
)
if patch.indices.numel() != patch.values.numel():
raise ValueError(
"`indices` and `values` must have matching lengths for "
f"{patch.name}"
)
if patch.values.dtype != param.dtype:
raise ValueError(
f"Sparse values dtype {patch.values.dtype} does not match "
f"parameter dtype {param.dtype} for {patch.name}"
)
flat_param = param.data.view(-1)
flat_param.index_copy_(
0,
patch.indices.to(device=flat_param.device, dtype=torch.long),
patch.values.to(device=flat_param.device),
)
def get_supported_generation_tasks(self) -> list[GenerationTask]:
model = self.get_model()
supported_tasks = list[GenerationTask]()
+14 -81
View File
@@ -155,7 +155,6 @@ class Worker(WorkerBase):
# is available, since the engine needs a reference to the model.
self.weight_transfer_engine: WeightTransferEngine | None = None
self._weight_update_active = False
self._is_checkpoint_format = True
# Torch/CUDA profiler. Enabled and configured through profiler_config.
# Profiler wrapper is created lazily in profile() when start is called,
@@ -416,7 +415,8 @@ class Worker(WorkerBase):
if self.vllm_config.weight_transfer_config is not None:
self.weight_transfer_engine = WeightTransferEngineFactory.create_engine(
self.vllm_config.weight_transfer_config,
self.vllm_config.parallel_config,
self.vllm_config,
self.device,
self.model_runner.get_model(),
)
@@ -1161,16 +1161,16 @@ class Worker(WorkerBase):
typed_init_info = self.weight_transfer_engine.parse_init_info(init_info)
self.weight_transfer_engine.init_transfer_engine(typed_init_info)
def start_weight_update(self, is_checkpoint_format: bool = True) -> None:
def start_weight_update(self) -> None:
"""
Start a new weight update session.
Args:
is_checkpoint_format: Whether incoming weights are in checkpoint
format (need layerwise processing) or kernel format (direct
copy / sparse patch application).
Delegates engine-specific preparation (e.g. layerwise reload setup) to
the configured weight transfer engine. The worker only tracks that a
session is active.
"""
self._check_weight_transfer_engine()
assert self.weight_transfer_engine is not None
if self._weight_update_active:
raise RuntimeError(
@@ -1178,16 +1178,7 @@ class Worker(WorkerBase):
"active. Call finish_weight_update first."
)
if is_checkpoint_format:
from vllm.model_executor.model_loader.reload import (
initialize_layerwise_reload,
)
model = self.model_runner.model
with torch.device(self.device):
initialize_layerwise_reload(model)
self._is_checkpoint_format = is_checkpoint_format
self.weight_transfer_engine.start_weight_update()
self._weight_update_active = True
def update_weights(self, update_info: dict) -> None:
@@ -1208,82 +1199,24 @@ class Worker(WorkerBase):
"start_weight_update must be called before update_weights."
)
update_succeeded = False
try:
# Parse dict into backend-specific typed dataclass
typed_update_info = self.weight_transfer_engine.parse_update_info(
update_info
)
with torch.device(self.device):
if self._is_checkpoint_format:
if typed_update_info.update_kind != "dense":
raise ValueError(
"Sparse weight updates require "
"`start_weight_update(is_checkpoint_format=False)`."
)
model = self.model_runner.model
# Use layerwise reload pattern for checkpoint format weights
self.weight_transfer_engine.receive_weights(
typed_update_info,
load_weights=model.load_weights,
)
elif typed_update_info.update_kind == "sparse_flat":
if self.parallel_config.world_size != 1:
raise NotImplementedError(
"Sparse weight updates currently require TP=1 and PP=1"
)
self.weight_transfer_engine.receive_sparse_weights(
typed_update_info,
apply_patches=self.model_runner.apply_sparse_weight_patches,
)
else:
model = self.model_runner.model
# Weights are already in kernel format, copy directly.
def load_weights_direct(
weights: list[tuple[str, torch.Tensor]],
) -> None:
for name, weight in weights:
param = model.get_parameter(name)
param.copy_(weight)
self.weight_transfer_engine.receive_weights(
typed_update_info,
load_weights=load_weights_direct,
)
# NCCL broadcast/packed path are asynchronous.
# Sync here so the next step uses the new weights.
torch.accelerator.synchronize()
update_succeeded = True
finally:
if not update_succeeded:
self._weight_update_active = False
self._is_checkpoint_format = True
self.weight_transfer_engine.update_weights(update_info)
except BaseException:
self._weight_update_active = False
raise
def finish_weight_update(self) -> None:
"""Finish the current weight update session."""
self._check_weight_transfer_engine()
assert self.weight_transfer_engine is not None
if not self._weight_update_active:
raise RuntimeError(
"finish_weight_update called without a matching start_weight_update."
)
if self._is_checkpoint_format:
from vllm.model_executor.model_loader.reload import (
finalize_layerwise_reload,
)
model = self.model_runner.model
with torch.device(self.device):
finalize_layerwise_reload(model, self.model_config)
self.weight_transfer_engine.finish_weight_update()
self._weight_update_active = False
self._is_checkpoint_format = True
def shutdown(self) -> None:
gc.unfreeze()