Weight sync refactor + move sparse nccl engine (#44353)
Signed-off-by: hao-aaron <ahao@anyscale.com> Signed-off-by: haoaaron <ahao@anyscale.com> Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
f651a8a9a4
commit
77a9c5ae28
@@ -58,7 +58,7 @@ class Fp8PerTensorOnlineLinearMethod(LinearMethodBase):
|
||||
|
||||
### High Level Weight Transfer API
|
||||
|
||||
The layerwise reloading system is integrated with the post-training weight transfer system. To use layerwise reloading in conjunction to the weight transfer system, follow the examples found [here](../../examples/rl/). Layerwise reloading is controlled by the `WeightTransferUpdateInfo.is_checkpoint_format` flag and is set to `True` by default.
|
||||
The layerwise reloading system is integrated with the post-training weight transfer system. To use layerwise reloading in conjunction to the weight transfer system, follow the examples found [here](../../examples/rl/). Checkpoint-format weight transfer engines (e.g. the NCCL and IPC backends) run layerwise reloading automatically inside their `start_weight_update`/`finish_weight_update` lifecycle.
|
||||
|
||||
### Mid Level `reload_weights` API
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ The weight transfer system follows a **four-phase protocol** with a pluggable ba
|
||||
| ------- | --------- | -------- |
|
||||
| [NCCL](nccl.md) | NCCL broadcast | Separate GPUs for training and inference |
|
||||
| [IPC](ipc.md) | CUDA IPC handles | Colocated training and inference on same GPU |
|
||||
| [sparse_nccl](nccl.md#sparse-nccl) | NCCL broadcast | Sparse flat-index weight patches (TP=1/PP=1) |
|
||||
|
||||
## Configuration
|
||||
|
||||
@@ -41,7 +42,7 @@ vllm serve my-model \
|
||||
--weight-transfer-config '{"backend": "nccl"}'
|
||||
```
|
||||
|
||||
The `backend` field accepts `"nccl"` (default) or `"ipc"`.
|
||||
The `backend` field accepts `"nccl"` (default), `"ipc"`, or `"sparse_nccl"`.
|
||||
|
||||
## API Endpoints
|
||||
|
||||
@@ -69,7 +70,7 @@ Both backends provide static methods that the trainer calls to send weights. The
|
||||
EngineClass.trainer_init(init_info)
|
||||
|
||||
# 2. Start weight update on inference side
|
||||
llm.start_weight_update(is_checkpoint_format=True)
|
||||
llm.start_weight_update()
|
||||
|
||||
# 3. Send weights to inference workers
|
||||
EngineClass.trainer_send_weights(
|
||||
|
||||
@@ -11,15 +11,23 @@ The `WeightTransferEngine` is a generic abstract class parameterized by two data
|
||||
|
||||
### Abstract Methods
|
||||
|
||||
Subclasses must implement these four methods:
|
||||
Subclasses must implement these methods:
|
||||
|
||||
| Method | Side | Description |
|
||||
| ------ | ---- | ----------- |
|
||||
| `init_transfer_engine(init_info)` | Inference | Initialize the communication channel on each inference worker |
|
||||
| `receive_weights(update_info, load_weights)` | Inference | Receive weights and call `load_weights` incrementally |
|
||||
| `start_weight_update()` | Inference | Prepare for an update (e.g. begin layerwise reload); no-op for in-place engines |
|
||||
| `finish_weight_update()` | Inference | Finalize the update (e.g. finalize layerwise reload); no-op for in-place engines |
|
||||
| `receive_weights(update_info)` | Inference | Receive weights and load them into `self.model` |
|
||||
| `shutdown()` | Inference | Clean up resources |
|
||||
| `trainer_send_weights(iterator, trainer_args)` | Trainer | Static method to send weights from the trainer process |
|
||||
|
||||
The base class provides two methods:
|
||||
|
||||
1. `__init__` : Engines receive `config` (`WeightTransferConfig`), `vllm_config` (`VllmConfig`), `device` (`torch.device`) and `model` (`nn.Module`)
|
||||
2. `update_weights(update_info_dict)`: Thin wrapper for `receive_weights`: parses
|
||||
the dict into user-specified data type, calls `receive_weights`, and synchronizes the device. Subclasses implement `receive_weights`.
|
||||
|
||||
### Request Classes
|
||||
|
||||
The API-level request classes provide backend-agnostic serialization using plain dictionaries. The engine's `parse_init_info` and `parse_update_info` methods convert these dictionaries into typed dataclasses.
|
||||
@@ -81,7 +89,7 @@ class MyUpdateInfo(WeightTransferUpdateInfo):
|
||||
### 2. Implement the Engine
|
||||
|
||||
```python
|
||||
from collections.abc import Callable, Iterator
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
import torch
|
||||
|
||||
@@ -93,18 +101,25 @@ class MyWeightTransferEngine(WeightTransferEngine[MyInitInfo, MyUpdateInfo]):
|
||||
# Set up connection to trainer using init_info.endpoint, etc.
|
||||
...
|
||||
|
||||
def receive_weights(
|
||||
self,
|
||||
update_info: MyUpdateInfo,
|
||||
load_weights: Callable[[list[tuple[str, torch.Tensor]]], None],
|
||||
) -> None:
|
||||
# Receive each weight and call load_weights incrementally
|
||||
def start_weight_update(self) -> None:
|
||||
# Checkpoint-format engines: run initialize_layerwise_reload(self.model).
|
||||
# In-place engines: no-op
|
||||
...
|
||||
|
||||
def finish_weight_update(self) -> None:
|
||||
# Checkpoint-format engines: run finalize_layerwise_reload(...).
|
||||
# In-place engines: no-op
|
||||
...
|
||||
|
||||
def receive_weights(self, update_info: MyUpdateInfo) -> None:
|
||||
weights = []
|
||||
for name, dtype_name, shape in zip(
|
||||
update_info.names, update_info.dtype_names, update_info.shapes
|
||||
):
|
||||
dtype = getattr(torch, dtype_name)
|
||||
weight = self._fetch_weight(name, shape, dtype)
|
||||
load_weights([(name, weight)])
|
||||
weights.append((name, weight))
|
||||
self.model.load_weights(weights)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
# Clean up resources
|
||||
@@ -121,9 +136,6 @@ class MyWeightTransferEngine(WeightTransferEngine[MyInitInfo, MyUpdateInfo]):
|
||||
...
|
||||
```
|
||||
|
||||
!!! important
|
||||
The `load_weights` callable passed to `receive_weights` should be called **incrementally** (one or a few weights at a time) rather than accumulating all weights first. This avoids GPU out-of-memory errors with large models.
|
||||
|
||||
### 3. Register with the Factory
|
||||
|
||||
```python
|
||||
@@ -147,7 +159,7 @@ Once registered, users can select your backend via `WeightTransferConfig(backend
|
||||
|
||||
## WeightTransferEngineFactory
|
||||
|
||||
The factory uses a registry pattern with lazy loading. Built-in engines (`nccl` and `ipc`) are registered at import time but their modules are only loaded when the backend is actually requested. This avoids importing heavy dependencies (like NCCL communicators) when they aren't needed.
|
||||
The factory uses a registry pattern with lazy loading. Built-in engines (`nccl`, `ipc`, and `sparse_nccl`) are registered at import time but their modules are only loaded when the backend is actually requested. This avoids importing heavy dependencies (like NCCL communicators) when they aren't needed.
|
||||
|
||||
```python
|
||||
from vllm.distributed.weight_transfer.factory import WeightTransferEngineFactory
|
||||
@@ -155,7 +167,8 @@ from vllm.distributed.weight_transfer.factory import WeightTransferEngineFactory
|
||||
# Create an engine from config
|
||||
engine = WeightTransferEngineFactory.create_engine(
|
||||
config=weight_transfer_config,
|
||||
parallel_config=parallel_config,
|
||||
vllm_config=vllm_config,
|
||||
device=device,
|
||||
model=model,
|
||||
)
|
||||
```
|
||||
|
||||
@@ -55,7 +55,7 @@ trainer_args = IPCTrainerSendWeightsArgs(
|
||||
llm_handle=llm_actor_handle,
|
||||
)
|
||||
# start
|
||||
ray.get(llm_actor_handle.start_weight_update.remote(is_checkpoint_format=True))
|
||||
ray.get(llm_actor_handle.start_weight_update.remote())
|
||||
# send weights
|
||||
IPCWeightTransferEngine.trainer_send_weights(
|
||||
iterator=model.named_parameters(),
|
||||
@@ -80,7 +80,7 @@ trainer_args = IPCTrainerSendWeightsArgs(
|
||||
# start
|
||||
base_url = "http://localhost:8000"
|
||||
url = f"{base_url}/start_weight_update"
|
||||
response = requests.post(url, json={"is_checkpoint_format": True}, timeout=60)
|
||||
response = requests.post(url, json={}, timeout=60)
|
||||
response.raise_for_status()
|
||||
# send weights
|
||||
IPCWeightTransferEngine.trainer_send_weights(
|
||||
|
||||
@@ -11,7 +11,7 @@ The NCCL weight transfer engine uses [NCCL](https://developer.nvidia.com/nccl) b
|
||||
## How It Works
|
||||
|
||||
1. The trainer and all inference workers join a shared NCCL process group using `StatelessProcessGroup` (vLLM's torch.distributed-independent group abstraction).
|
||||
2. The trainer broadcasts weights to all workers simultaneously. Each worker receives and loads weights incrementally.
|
||||
2. The trainer broadcasts weights to all workers simultaneously. Each worker receives and loads the weights.
|
||||
3. Optionally, **packed tensor broadcasting** batches multiple small tensors into larger buffers with double/triple buffering and CUDA stream overlap for higher throughput. This implementation is based on [NeMo-RL's packed tensor](https://github.com/NVIDIA-NeMo/RL/blob/main/nemo_rl/utils/packed_tensor.py).
|
||||
|
||||
## Initialization
|
||||
@@ -93,7 +93,7 @@ remaining three steps are:
|
||||
from vllm.distributed.weight_transfer.base import WeightTransferUpdateRequest
|
||||
|
||||
# 1. Start the weight update
|
||||
llm.start_weight_update(is_checkpoint_format=True)
|
||||
llm.start_weight_update()
|
||||
|
||||
# 2. Receive weights (can be called multiple times for chunked transfers)
|
||||
llm.update_weights(
|
||||
@@ -116,19 +116,22 @@ must match the order in which the trainer iterates over its parameters.
|
||||
|
||||
`start_weight_update` must be called before `update_weights`, and
|
||||
`finish_weight_update` must be called after all weight chunks have been
|
||||
transferred. The `is_checkpoint_format` flag controls whether layerwise reload
|
||||
processing is applied (`True` for checkpoint-format weights, `False` for
|
||||
pre-processed kernel-format weights).
|
||||
transferred. The NCCL engine receives checkpoint-format weights and applies
|
||||
layerwise reload processing automatically inside `start_weight_update` /
|
||||
`finish_weight_update`.
|
||||
|
||||
Sparse NCCL patches still use `update_kind="sparse_flat"` inside
|
||||
`update_info`, but they should be wrapped in
|
||||
`start_weight_update(is_checkpoint_format=False)` because sparse patches apply
|
||||
directly to runtime/kernel-format parameters. The current sparse MVP requires
|
||||
`TP=1` and `PP=1`.
|
||||
## Sparse NCCL
|
||||
|
||||
Sparse, flat-index weight patches use a separate backend,
|
||||
`WeightTransferConfig(backend="sparse_nccl")`, implemented by
|
||||
`SparseNCCLWeightTransferEngine`. It shares only NCCL process-group
|
||||
initialization with the dense engine; patches are applied directly in place to
|
||||
existing parameters (no layerwise reload). The current sparse MVP requires
|
||||
`TP=1` and `PP=1`. See the example below.
|
||||
|
||||
## Examples
|
||||
|
||||
- [RLHF with NCCL weight syncing (offline, Ray)](../../../examples/rl/rlhf_nccl.py) - Trainer on one GPU, 2x tensor-parallel vLLM engine on two others, with packed NCCL weight broadcast
|
||||
- [RLHF with sparse NCCL weight syncing (offline, Ray)](../../../examples/rl/rlhf_sparse_nccl.py) - Dense-vs-sparse equivalence demo with a real model on a 2-GPU trainer/inference setup; sparse patches use `start_weight_update(is_checkpoint_format=False)` and currently require `TP=1` and `PP=1`
|
||||
- [RLHF with sparse NCCL weight syncing (offline, Ray)](../../../examples/rl/rlhf_sparse_nccl.py) - Dense-vs-sparse equivalence demo with a real model on a 2-GPU trainer/inference setup; sparse patches use `backend="sparse_nccl"` and currently require `TP=1` and `PP=1`
|
||||
- [RLHF with async weight syncing (offline, Ray)](../../../examples/rl/rlhf_async_new_apis.py) - Async generation with mid-flight pause, weight sync, resume, and validation against a fresh model
|
||||
- [RLHF with NCCL weight syncing (online serving, HTTP)](../../../examples/rl/rlhf_http_nccl.py) - Weight transfer with a running vLLM HTTP server using HTTP control plane and NCCL data plane
|
||||
|
||||
@@ -306,7 +306,7 @@ gen_futures = [
|
||||
|
||||
ray.get(llm.pause_after_n_tokens.remote())
|
||||
|
||||
ray.get(llm.start_weight_update.remote(is_checkpoint_format=True))
|
||||
ray.get(llm.start_weight_update.remote())
|
||||
|
||||
inference_handle = llm.update_weights.remote(
|
||||
WeightTransferUpdateRequest(
|
||||
|
||||
@@ -80,14 +80,10 @@ def init_weight_transfer_engine(base_url: str) -> None:
|
||||
response.raise_for_status()
|
||||
|
||||
|
||||
def start_weight_update(
|
||||
base_url: str,
|
||||
is_checkpoint_format: bool = True,
|
||||
) -> None:
|
||||
def start_weight_update(base_url: str) -> None:
|
||||
"""Start a weight update via HTTP endpoint."""
|
||||
url = f"{base_url}/start_weight_update"
|
||||
payload = {"is_checkpoint_format": is_checkpoint_format}
|
||||
response = requests.post(url, json=payload, timeout=60)
|
||||
response = requests.post(url, json={}, timeout=60)
|
||||
response.raise_for_status()
|
||||
|
||||
|
||||
@@ -170,7 +166,7 @@ def main():
|
||||
pause_generation(BASE_URL)
|
||||
|
||||
# Start weight update, broadcast via IPC, then finish
|
||||
start_weight_update(BASE_URL, is_checkpoint_format=False)
|
||||
start_weight_update(BASE_URL)
|
||||
|
||||
print("Broadcasting weights via CUDA IPC (HTTP)...")
|
||||
trainer_args = IPCTrainerSendWeightsArgs(send_mode="http", url=BASE_URL)
|
||||
|
||||
@@ -83,14 +83,10 @@ def init_weight_transfer_engine(
|
||||
response.raise_for_status()
|
||||
|
||||
|
||||
def start_weight_update(
|
||||
base_url: str,
|
||||
is_checkpoint_format: bool = True,
|
||||
) -> None:
|
||||
def start_weight_update(base_url: str) -> None:
|
||||
"""Start a weight update via HTTP endpoint."""
|
||||
url = f"{base_url}/start_weight_update"
|
||||
payload = {"is_checkpoint_format": is_checkpoint_format}
|
||||
response = requests.post(url, json=payload, timeout=60)
|
||||
response = requests.post(url, json={}, timeout=60)
|
||||
response.raise_for_status()
|
||||
|
||||
|
||||
@@ -223,7 +219,7 @@ def main():
|
||||
shapes.append(list(p.shape))
|
||||
|
||||
# Start weight update
|
||||
start_weight_update(BASE_URL, is_checkpoint_format=True)
|
||||
start_weight_update(BASE_URL)
|
||||
|
||||
# Start the update_weights call in a separate thread since it will block
|
||||
# waiting for NCCL broadcasts
|
||||
|
||||
@@ -139,7 +139,7 @@ ray.get(llm.sleep.remote(level=0))
|
||||
|
||||
ray.get(train_model.init_weight_transfer.remote())
|
||||
# Start weight update, sync weights, then finish
|
||||
ray.get(llm.start_weight_update.remote(is_checkpoint_format=True))
|
||||
ray.get(llm.start_weight_update.remote())
|
||||
ray.get(train_model.broadcast_weights.remote(llm))
|
||||
ray.get(llm.finish_weight_update.remote())
|
||||
|
||||
|
||||
@@ -277,15 +277,8 @@ class DataParallelInferenceEngine:
|
||||
]
|
||||
)
|
||||
|
||||
def start_weight_update(self, is_checkpoint_format: bool = True):
|
||||
ray.get(
|
||||
[
|
||||
actor.start_weight_update.remote(
|
||||
is_checkpoint_format=is_checkpoint_format
|
||||
)
|
||||
for actor in self.llm_actors
|
||||
]
|
||||
)
|
||||
def start_weight_update(self):
|
||||
ray.get([actor.start_weight_update.remote() for actor in self.llm_actors])
|
||||
|
||||
def finish_weight_update(self):
|
||||
ray.get([actor.finish_weight_update.remote() for actor in self.llm_actors])
|
||||
@@ -392,7 +385,7 @@ def main():
|
||||
ray.get(inference_engine.wake_up.remote(tags=["weights"]))
|
||||
|
||||
print("[sync] Starting weight update...")
|
||||
ray.get(inference_engine.start_weight_update.remote(is_checkpoint_format=True))
|
||||
ray.get(inference_engine.start_weight_update.remote())
|
||||
|
||||
print("[sync] Packed IPC transfer FSDP → vLLM...")
|
||||
ray.get(
|
||||
|
||||
@@ -202,7 +202,7 @@ ray.get([train_handle, inference_handle])
|
||||
names, dtype_names, shapes = ray.get(train_model.get_weight_metadata.remote())
|
||||
|
||||
# Start weight update
|
||||
ray.get(llm.start_weight_update.remote(is_checkpoint_format=True))
|
||||
ray.get(llm.start_weight_update.remote())
|
||||
|
||||
# Issue update_weights call with NCCL-specific update info
|
||||
# packed=True enables efficient batched tensor broadcasting
|
||||
|
||||
@@ -299,7 +299,7 @@ async def main():
|
||||
print(f"[sync] Got metadata for {len(names)} parameters.")
|
||||
|
||||
print("[sync] Starting weight update...")
|
||||
await engine.start_weight_update(is_checkpoint_format=True)
|
||||
await engine.start_weight_update()
|
||||
|
||||
print("[sync] Broadcasting weights from FSDP → vLLM...")
|
||||
broadcast_handles = [
|
||||
|
||||
@@ -44,11 +44,14 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.config import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import SparseWeightPatch
|
||||
from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
NCCLTrainerSendWeightsArgs,
|
||||
NCCLWeightTransferEngine,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.sparse_nccl_engine import (
|
||||
SparseNCCLWeightTransferEngine,
|
||||
SparseWeightPatch,
|
||||
)
|
||||
from vllm.utils.network_utils import get_ip, get_open_port
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen2.5-0.5B-Instruct"
|
||||
@@ -244,7 +247,6 @@ class TrainModel:
|
||||
dtype_names=[str(self.patched_param.dtype).split(".")[-1]],
|
||||
shapes=[list(self.patched_param.shape)],
|
||||
num_updates_list=[flat_indices.numel()],
|
||||
update_kind="sparse_flat",
|
||||
)
|
||||
return update_info, selected_token_ids, patch_digest, sparse_payload_bytes
|
||||
|
||||
@@ -271,7 +273,7 @@ class TrainModel:
|
||||
raise RuntimeError("Sparse patch has not been prepared")
|
||||
|
||||
start = time.perf_counter()
|
||||
NCCLWeightTransferEngine.trainer_send_sparse_weights(
|
||||
SparseNCCLWeightTransferEngine.trainer_send_weights(
|
||||
iter(self.pending_sparse_patches),
|
||||
NCCLTrainerSendWeightsArgs(group=self.model_update_group),
|
||||
)
|
||||
@@ -282,6 +284,7 @@ class TrainModel:
|
||||
|
||||
def launch_llm(
|
||||
scheduling_inference: PlacementGroupSchedulingStrategy,
|
||||
backend: str = "nccl",
|
||||
):
|
||||
return ray.remote(
|
||||
num_cpus=0,
|
||||
@@ -293,7 +296,7 @@ def launch_llm(
|
||||
tensor_parallel_size=1,
|
||||
distributed_executor_backend="ray",
|
||||
gpu_memory_utilization=0.7,
|
||||
weight_transfer_config=WeightTransferConfig(backend="nccl"),
|
||||
weight_transfer_config=WeightTransferConfig(backend=backend),
|
||||
)
|
||||
|
||||
|
||||
@@ -332,7 +335,7 @@ def run_dense_phase(
|
||||
scheduling_inference: PlacementGroupSchedulingStrategy,
|
||||
) -> dict[str, object]:
|
||||
ray.get(train_model.reset_model.remote())
|
||||
llm = launch_llm(scheduling_inference)
|
||||
llm = launch_llm(scheduling_inference, backend="nccl")
|
||||
try:
|
||||
dense_before = collect_vllm_generations(llm)
|
||||
|
||||
@@ -351,7 +354,7 @@ def run_dense_phase(
|
||||
)
|
||||
trainer_init = train_model.init_weight_transfer_group.remote(world_size)
|
||||
ray.get([trainer_init, inference_init])
|
||||
ray.get(llm.start_weight_update.remote(is_checkpoint_format=True))
|
||||
ray.get(llm.start_weight_update.remote())
|
||||
|
||||
dense_update_info, dense_payload_bytes = ray.get(
|
||||
train_model.get_dense_update_info.remote()
|
||||
@@ -391,7 +394,7 @@ def run_sparse_phase(
|
||||
scheduling_inference: PlacementGroupSchedulingStrategy,
|
||||
) -> dict[str, object]:
|
||||
ray.get(train_model.reset_model.remote())
|
||||
llm = launch_llm(scheduling_inference)
|
||||
llm = launch_llm(scheduling_inference, backend="sparse_nccl")
|
||||
try:
|
||||
sparse_before = collect_vllm_generations(llm)
|
||||
|
||||
@@ -410,7 +413,7 @@ def run_sparse_phase(
|
||||
)
|
||||
trainer_init = train_model.init_weight_transfer_group.remote(world_size)
|
||||
ray.get([trainer_init, inference_init])
|
||||
ray.get(llm.start_weight_update.remote(is_checkpoint_format=False))
|
||||
ray.get(llm.start_weight_update.remote())
|
||||
|
||||
sparse_update_info, selected_token_ids, patch_digest, sparse_payload_bytes = (
|
||||
ray.get(train_model.prepare_sparse_patch.remote(PROMPTS))
|
||||
|
||||
@@ -18,7 +18,6 @@ from torch.multiprocessing.reductions import reduce_tensor
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer import WeightTransferEngineFactory
|
||||
from vllm.distributed.weight_transfer.base import SparseWeightPatch
|
||||
from vllm.distributed.weight_transfer.ipc_engine import (
|
||||
IPCWeightTransferEngine,
|
||||
IPCWeightTransferInitInfo,
|
||||
@@ -29,6 +28,11 @@ from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
NCCLWeightTransferInitInfo,
|
||||
NCCLWeightTransferUpdateInfo,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.sparse_nccl_engine import (
|
||||
SparseNCCLWeightTransferEngine,
|
||||
SparseNCCLWeightTransferUpdateInfo,
|
||||
SparseWeightPatch,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
@@ -80,6 +84,18 @@ def create_mock_parallel_config(
|
||||
return config
|
||||
|
||||
|
||||
def create_mock_vllm_config(
|
||||
rank: int = 0,
|
||||
world_size: int = 1,
|
||||
dp_rank: int = 0,
|
||||
) -> MagicMock:
|
||||
"""Create a mock VllmConfig exposing parallel_config and model_config."""
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.parallel_config = create_mock_parallel_config(rank, world_size, dp_rank)
|
||||
vllm_config.model_config = MagicMock()
|
||||
return vllm_config
|
||||
|
||||
|
||||
# --- Unit Tests: NCCLWeightTransferUpdateInfo Validation ---
|
||||
|
||||
|
||||
@@ -87,7 +103,6 @@ class TestNCCLWeightTransferUpdateInfoValidation:
|
||||
"""Test NCCLWeightTransferUpdateInfo dataclass validation."""
|
||||
|
||||
def test_valid_update_info(self):
|
||||
"""Test creating valid NCCLWeightTransferUpdateInfo."""
|
||||
info = NCCLWeightTransferUpdateInfo(
|
||||
names=["layer.weight", "layer.bias"],
|
||||
dtype_names=["float32", "float32"],
|
||||
@@ -98,7 +113,6 @@ class TestNCCLWeightTransferUpdateInfoValidation:
|
||||
assert info.shapes == [[10, 10], [10]]
|
||||
|
||||
def test_mismatched_dtype_names_raises(self):
|
||||
"""Test that mismatched dtype_names length raises ValueError."""
|
||||
with pytest.raises(ValueError, match="dtype_names"):
|
||||
NCCLWeightTransferUpdateInfo(
|
||||
names=["layer.weight", "layer.bias"],
|
||||
@@ -107,7 +121,6 @@ class TestNCCLWeightTransferUpdateInfoValidation:
|
||||
)
|
||||
|
||||
def test_mismatched_shapes_raises(self):
|
||||
"""Test that mismatched shapes length raises ValueError."""
|
||||
with pytest.raises(ValueError, match="shapes"):
|
||||
NCCLWeightTransferUpdateInfo(
|
||||
names=["layer.weight", "layer.bias"],
|
||||
@@ -116,73 +129,59 @@ class TestNCCLWeightTransferUpdateInfoValidation:
|
||||
)
|
||||
|
||||
def test_empty_lists_valid(self):
|
||||
"""Test that empty lists are valid."""
|
||||
info = NCCLWeightTransferUpdateInfo(
|
||||
names=[],
|
||||
dtype_names=[],
|
||||
shapes=[],
|
||||
)
|
||||
info = NCCLWeightTransferUpdateInfo(names=[], dtype_names=[], shapes=[])
|
||||
assert len(info.names) == 0
|
||||
|
||||
|
||||
# --- Unit Tests: SparseNCCLWeightTransferUpdateInfo Validation ---
|
||||
|
||||
|
||||
class TestSparseNCCLWeightTransferUpdateInfoValidation:
|
||||
"""Test SparseNCCLWeightTransferUpdateInfo dataclass validation."""
|
||||
|
||||
def test_valid_sparse_update_info(self):
|
||||
"""Test creating valid sparse NCCL update info."""
|
||||
info = NCCLWeightTransferUpdateInfo(
|
||||
info = SparseNCCLWeightTransferUpdateInfo(
|
||||
names=["layer.weight", "layer.bias"],
|
||||
dtype_names=["float32", "bfloat16"],
|
||||
shapes=[[10, 10], [10]],
|
||||
num_updates_list=[4, 2],
|
||||
update_kind="sparse_flat",
|
||||
)
|
||||
assert info.update_kind == "sparse_flat"
|
||||
assert info.num_updates_list == [4, 2]
|
||||
|
||||
def test_sparse_update_requires_num_updates_list(self):
|
||||
with pytest.raises(ValueError, match="`num_updates_list` is required"):
|
||||
NCCLWeightTransferUpdateInfo(
|
||||
names=["layer.weight"],
|
||||
def test_mismatched_dtype_names_raises(self):
|
||||
with pytest.raises(ValueError, match="dtype_names"):
|
||||
SparseNCCLWeightTransferUpdateInfo(
|
||||
names=["layer.weight", "layer.bias"],
|
||||
dtype_names=["float32"],
|
||||
shapes=[[10, 10]],
|
||||
update_kind="sparse_flat",
|
||||
shapes=[[10, 10], [10]],
|
||||
num_updates_list=[4, 2],
|
||||
)
|
||||
|
||||
def test_sparse_update_rejects_empty_num_updates_list(self):
|
||||
def test_rejects_empty_num_updates_list(self):
|
||||
with pytest.raises(ValueError, match="cannot be empty"):
|
||||
NCCLWeightTransferUpdateInfo(
|
||||
SparseNCCLWeightTransferUpdateInfo(
|
||||
names=[],
|
||||
dtype_names=[],
|
||||
shapes=[],
|
||||
num_updates_list=[],
|
||||
update_kind="sparse_flat",
|
||||
)
|
||||
|
||||
def test_sparse_update_rejects_packed(self):
|
||||
with pytest.raises(ValueError, match="cannot be combined with `packed=True`"):
|
||||
NCCLWeightTransferUpdateInfo(
|
||||
names=["layer.weight"],
|
||||
dtype_names=["float32"],
|
||||
shapes=[[10, 10]],
|
||||
num_updates_list=[3],
|
||||
update_kind="sparse_flat",
|
||||
packed=True,
|
||||
)
|
||||
|
||||
def test_sparse_update_rejects_mismatched_num_updates(self):
|
||||
def test_rejects_mismatched_num_updates(self):
|
||||
with pytest.raises(ValueError, match="`num_updates_list`"):
|
||||
NCCLWeightTransferUpdateInfo(
|
||||
SparseNCCLWeightTransferUpdateInfo(
|
||||
names=["layer.weight", "layer.bias"],
|
||||
dtype_names=["float32", "float32"],
|
||||
shapes=[[10, 10], [10]],
|
||||
num_updates_list=[3],
|
||||
update_kind="sparse_flat",
|
||||
)
|
||||
|
||||
def test_dense_update_rejects_sparse_metadata(self):
|
||||
with pytest.raises(ValueError, match="Sparse metadata"):
|
||||
NCCLWeightTransferUpdateInfo(
|
||||
def test_rejects_negative_num_updates(self):
|
||||
with pytest.raises(ValueError, match="non-negative"):
|
||||
SparseNCCLWeightTransferUpdateInfo(
|
||||
names=["layer.weight"],
|
||||
dtype_names=["float32"],
|
||||
shapes=[[10, 10]],
|
||||
num_updates_list=[3],
|
||||
num_updates_list=[-1],
|
||||
)
|
||||
|
||||
|
||||
@@ -192,14 +191,17 @@ class TestNCCLWeightTransferUpdateInfoValidation:
|
||||
class TestNCCLEngineParsing:
|
||||
"""Test NCCLWeightTransferEngine parsing methods."""
|
||||
|
||||
def test_parse_init_info_valid(self):
|
||||
"""Test parsing valid init info dict."""
|
||||
def _make_engine(self):
|
||||
config = WeightTransferConfig(backend="nccl")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = NCCLWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
return NCCLWeightTransferEngine(
|
||||
config,
|
||||
create_mock_vllm_config(),
|
||||
"cuda",
|
||||
MagicMock(spec=torch.nn.Module),
|
||||
)
|
||||
|
||||
def test_parse_init_info_valid(self):
|
||||
engine = self._make_engine()
|
||||
init_info = engine.parse_init_info(
|
||||
{
|
||||
"master_address": "127.0.0.1",
|
||||
@@ -208,7 +210,6 @@ class TestNCCLEngineParsing:
|
||||
"world_size": 3,
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(init_info, NCCLWeightTransferInitInfo)
|
||||
assert init_info.master_address == "127.0.0.1"
|
||||
assert init_info.master_port == 12345
|
||||
@@ -216,29 +217,12 @@ class TestNCCLEngineParsing:
|
||||
assert init_info.world_size == 3
|
||||
|
||||
def test_parse_init_info_missing_field_raises(self):
|
||||
"""Test parsing init info with missing required field."""
|
||||
config = WeightTransferConfig(backend="nccl")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = NCCLWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
|
||||
engine = self._make_engine()
|
||||
with pytest.raises(ValueError, match="Invalid init_info"):
|
||||
engine.parse_init_info(
|
||||
{
|
||||
"master_address": "127.0.0.1",
|
||||
# Missing master_port, rank_offset, world_size
|
||||
}
|
||||
)
|
||||
engine.parse_init_info({"master_address": "127.0.0.1"})
|
||||
|
||||
def test_parse_update_info_valid(self):
|
||||
"""Test parsing valid update info dict."""
|
||||
config = WeightTransferConfig(backend="nccl")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = NCCLWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
|
||||
engine = self._make_engine()
|
||||
update_info = engine.parse_update_info(
|
||||
{
|
||||
"names": ["w1", "w2"],
|
||||
@@ -246,7 +230,6 @@ class TestNCCLEngineParsing:
|
||||
"shapes": [[100, 100], [50]],
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(update_info, NCCLWeightTransferUpdateInfo)
|
||||
assert update_info.names == ["w1", "w2"]
|
||||
assert update_info.dtype_names == ["float32", "bfloat16"]
|
||||
@@ -260,40 +243,140 @@ class TestEngineRegistry:
|
||||
"""Test weight transfer engine registry."""
|
||||
|
||||
def test_create_engine_nccl(self):
|
||||
"""Test factory creates NCCL engine."""
|
||||
config = WeightTransferConfig(backend="nccl")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = WeightTransferEngineFactory.create_engine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
assert isinstance(engine, NCCLWeightTransferEngine)
|
||||
|
||||
def test_create_engine_ipc(self):
|
||||
"""Test factory creates IPC engine."""
|
||||
config = WeightTransferConfig(backend="ipc")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = WeightTransferEngineFactory.create_engine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
assert isinstance(engine, IPCWeightTransferEngine)
|
||||
|
||||
def test_create_engine_sparse_nccl(self):
|
||||
config = WeightTransferConfig(backend="sparse_nccl")
|
||||
engine = WeightTransferEngineFactory.create_engine(
|
||||
config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
assert isinstance(engine, SparseNCCLWeightTransferEngine)
|
||||
|
||||
def test_create_engine_invalid_backend(self):
|
||||
"""Test factory raises for invalid backend."""
|
||||
config = WeightTransferConfig(backend="invalid")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
with pytest.raises(ValueError, match="Invalid weight transfer backend"):
|
||||
WeightTransferEngineFactory.create_engine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
config,
|
||||
create_mock_vllm_config(),
|
||||
"cuda",
|
||||
MagicMock(spec=torch.nn.Module),
|
||||
)
|
||||
|
||||
def test_register_duplicate_raises(self):
|
||||
"""Test registering duplicate engine name raises."""
|
||||
with pytest.raises(ValueError, match="already registered"):
|
||||
WeightTransferEngineFactory.register_engine(
|
||||
"nccl", NCCLWeightTransferEngine
|
||||
)
|
||||
|
||||
|
||||
# --- Unit Tests: Sparse patch application (CPU) ---
|
||||
|
||||
|
||||
class TestSparseNCCLPatchApplication:
|
||||
"""Test SparseNCCLWeightTransferEngine._apply_patch on a real param."""
|
||||
|
||||
def _make_engine(self, model):
|
||||
config = WeightTransferConfig(backend="sparse_nccl")
|
||||
return SparseNCCLWeightTransferEngine(
|
||||
config, create_mock_vllm_config(), "cpu", model
|
||||
)
|
||||
|
||||
def _make_model(self, numel: int = 8):
|
||||
model = torch.nn.Module()
|
||||
model.register_parameter(
|
||||
"w", torch.nn.Parameter(torch.zeros(numel), requires_grad=False)
|
||||
)
|
||||
|
||||
def get_parameter(name):
|
||||
assert name == "w"
|
||||
return model.w
|
||||
|
||||
model.get_parameter = get_parameter
|
||||
return model
|
||||
|
||||
def test_apply_patch_updates_only_selected_entries(self):
|
||||
model = self._make_model(8)
|
||||
engine = self._make_engine(model)
|
||||
engine._apply_patch(
|
||||
SparseWeightPatch(
|
||||
name="w",
|
||||
indices=torch.tensor([1, 3], dtype=torch.int32),
|
||||
values=torch.tensor([5.0, 7.0], dtype=torch.float32),
|
||||
)
|
||||
)
|
||||
expected = torch.zeros(8)
|
||||
expected[1] = 5.0
|
||||
expected[3] = 7.0
|
||||
assert torch.equal(model.w.data, expected)
|
||||
|
||||
def test_apply_patch_rejects_mismatched_lengths(self):
|
||||
model = self._make_model(8)
|
||||
engine = self._make_engine(model)
|
||||
with pytest.raises(ValueError, match="matching lengths"):
|
||||
engine._apply_patch(
|
||||
SparseWeightPatch(
|
||||
name="w",
|
||||
indices=torch.tensor([1, 3], dtype=torch.int32),
|
||||
values=torch.tensor([5.0], dtype=torch.float32),
|
||||
)
|
||||
)
|
||||
|
||||
def test_apply_patch_rejects_non_int32_indices(self):
|
||||
model = self._make_model(8)
|
||||
engine = self._make_engine(model)
|
||||
with pytest.raises(ValueError, match="int32 indices"):
|
||||
engine._apply_patch(
|
||||
SparseWeightPatch(
|
||||
name="w",
|
||||
indices=torch.tensor([1], dtype=torch.int64),
|
||||
values=torch.tensor([5.0], dtype=torch.float32),
|
||||
)
|
||||
)
|
||||
|
||||
def test_apply_patch_rejects_dtype_mismatch(self):
|
||||
model = self._make_model(8)
|
||||
engine = self._make_engine(model)
|
||||
with pytest.raises(ValueError, match="does not match"):
|
||||
engine._apply_patch(
|
||||
SparseWeightPatch(
|
||||
name="w",
|
||||
indices=torch.tensor([1], dtype=torch.int32),
|
||||
values=torch.tensor([5.0], dtype=torch.bfloat16),
|
||||
)
|
||||
)
|
||||
|
||||
def test_apply_patch_rejects_non_contiguous_param(self):
|
||||
model = torch.nn.Module()
|
||||
model.register_parameter(
|
||||
"w",
|
||||
torch.nn.Parameter(
|
||||
torch.arange(12, dtype=torch.float32).view(3, 4).t(),
|
||||
requires_grad=False,
|
||||
),
|
||||
)
|
||||
model.get_parameter = lambda name: model.w
|
||||
engine = self._make_engine(model)
|
||||
with pytest.raises(NotImplementedError, match="contiguous params"):
|
||||
engine._apply_patch(
|
||||
SparseWeightPatch(
|
||||
name="w",
|
||||
indices=torch.tensor([1], dtype=torch.int32),
|
||||
values=torch.tensor([1.0], dtype=torch.float32),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# --- Test receive_weights without init raises ---
|
||||
|
||||
|
||||
@@ -303,42 +386,37 @@ def test_nccl_receive_weights_without_init_raises():
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
config = WeightTransferConfig(backend="nccl")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = NCCLWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
|
||||
update_info = NCCLWeightTransferUpdateInfo(
|
||||
names=["w"],
|
||||
dtype_names=["float32"],
|
||||
shapes=[[10]],
|
||||
names=["w"], dtype_names=["float32"], shapes=[[10]]
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="not initialized"):
|
||||
engine.receive_weights(update_info, lambda x: None)
|
||||
engine.receive_weights(update_info)
|
||||
|
||||
|
||||
def test_nccl_receive_sparse_weights_without_init_raises():
|
||||
def test_sparse_nccl_receive_weights_without_init_raises():
|
||||
"""Test that sparse receive raises if init_transfer_engine wasn't called."""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
config = WeightTransferConfig(backend="nccl")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = NCCLWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
config = WeightTransferConfig(backend="sparse_nccl")
|
||||
engine = SparseNCCLWeightTransferEngine(
|
||||
config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
|
||||
update_info = NCCLWeightTransferUpdateInfo(
|
||||
update_info = SparseNCCLWeightTransferUpdateInfo(
|
||||
names=["w"],
|
||||
dtype_names=["float32"],
|
||||
shapes=[[10]],
|
||||
num_updates_list=[2],
|
||||
update_kind="sparse_flat",
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="not initialized"):
|
||||
engine.receive_sparse_weights(update_info, lambda x: None)
|
||||
engine.receive_weights(update_info)
|
||||
|
||||
|
||||
# --- Integration Test: NCCL Weight Transfer Between Ray Tasks ---
|
||||
@@ -387,6 +465,7 @@ def inference_receive_tensor(
|
||||
tensor_dtype: str,
|
||||
) -> dict:
|
||||
"""Inference task that receives tensor via NCCLWeightTransferEngine."""
|
||||
import contextlib
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import torch
|
||||
@@ -401,17 +480,32 @@ def inference_receive_tensor(
|
||||
NCCLWeightTransferUpdateInfo,
|
||||
)
|
||||
|
||||
# Create engine with mock parallel config
|
||||
class Recorder(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.received = []
|
||||
|
||||
def load_weights(self, weights):
|
||||
for name, tensor in weights:
|
||||
self.received.append((name, tensor.clone()))
|
||||
|
||||
config = WeightTransferConfig(backend="nccl")
|
||||
vllm_config = MagicMock()
|
||||
parallel_config = MagicMock(spec=ParallelConfig)
|
||||
parallel_config.rank = 0
|
||||
parallel_config.world_size = 1
|
||||
parallel_config.data_parallel_rank = 0
|
||||
parallel_config.data_parallel_index = 0
|
||||
vllm_config.parallel_config = parallel_config
|
||||
vllm_config.model_config = MagicMock()
|
||||
|
||||
engine = NCCLWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
recorder = Recorder()
|
||||
engine = NCCLWeightTransferEngine(config, vllm_config, "cuda", recorder)
|
||||
# Transport-only test: bypass the set_current_vllm_config context that
|
||||
# receive_weights enters, since vllm_config here is a mock.
|
||||
import vllm.config as _vllm_config_mod
|
||||
|
||||
_vllm_config_mod.set_current_vllm_config = lambda cfg: contextlib.nullcontext()
|
||||
|
||||
# Initialize the engine (joins as rank 1)
|
||||
init_info = NCCLWeightTransferInitInfo(
|
||||
@@ -422,20 +516,12 @@ def inference_receive_tensor(
|
||||
)
|
||||
engine.init_transfer_engine(init_info)
|
||||
|
||||
# Receive weights with a no-op load_weights that captures the tensor
|
||||
received_tensors = []
|
||||
|
||||
def noop_load_weights(weights: list[tuple[str, torch.Tensor]]):
|
||||
for name, tensor in weights:
|
||||
# Clone tensor to keep it after engine cleans up
|
||||
received_tensors.append((name, tensor.clone()))
|
||||
|
||||
update_info = NCCLWeightTransferUpdateInfo(
|
||||
names=["test.weight"],
|
||||
dtype_names=[tensor_dtype],
|
||||
shapes=[tensor_shape],
|
||||
)
|
||||
engine.receive_weights(update_info, noop_load_weights)
|
||||
engine.receive_weights(update_info)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
# Verify we received the tensor
|
||||
@@ -443,11 +529,10 @@ def inference_receive_tensor(
|
||||
received_shape = None
|
||||
received_sum = None
|
||||
|
||||
if len(received_tensors) == 1:
|
||||
name, tensor = received_tensors[0]
|
||||
if len(recorder.received) == 1:
|
||||
name, tensor = recorder.received[0]
|
||||
received_shape = list(tensor.shape)
|
||||
received_sum = tensor.sum().item()
|
||||
# Check shape matches and values are all 1s (trainer sends ones)
|
||||
if received_shape == tensor_shape:
|
||||
expected_sum = 1.0 * torch.tensor(tensor_shape).prod().item()
|
||||
if abs(received_sum - expected_sum) < 0.01:
|
||||
@@ -478,11 +563,9 @@ def test_nccl_weight_transfer_between_processes():
|
||||
master_port = get_open_port()
|
||||
world_size = 2 # 1 trainer + 1 inference worker
|
||||
|
||||
# Tensor to transfer: 100x100 ones
|
||||
tensor_shape = [100, 100]
|
||||
tensor_dtype = "float32"
|
||||
|
||||
# Start both tasks concurrently - Ray assigns GPUs automatically
|
||||
inference_future = inference_receive_tensor.remote(
|
||||
master_address, master_port, world_size, tensor_shape, tensor_dtype
|
||||
)
|
||||
@@ -490,7 +573,6 @@ def test_nccl_weight_transfer_between_processes():
|
||||
master_address, master_port, world_size, tensor_shape, tensor_dtype
|
||||
)
|
||||
|
||||
# Wait for both to complete
|
||||
trainer_result, result = ray.get([trainer_future, inference_future])
|
||||
|
||||
assert trainer_result, "Trainer should complete successfully"
|
||||
@@ -514,10 +596,12 @@ def trainer_broadcast_sparse_tensor(
|
||||
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
from vllm.distributed.weight_transfer.base import SparseWeightPatch
|
||||
from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
NCCLTrainerSendWeightsArgs,
|
||||
NCCLWeightTransferEngine,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.sparse_nccl_engine import (
|
||||
SparseNCCLWeightTransferEngine,
|
||||
SparseWeightPatch,
|
||||
)
|
||||
|
||||
pg = StatelessProcessGroup.create(
|
||||
@@ -533,7 +617,7 @@ def trainer_broadcast_sparse_tensor(
|
||||
indices=torch.tensor([1, 7, 25], dtype=torch.int32, device=device),
|
||||
values=torch.tensor([10.0, 20.0, 30.0], dtype=torch.float32, device=device),
|
||||
)
|
||||
NCCLWeightTransferEngine.trainer_send_sparse_weights(
|
||||
SparseNCCLWeightTransferEngine.trainer_send_weights(
|
||||
iter([patch]),
|
||||
NCCLTrainerSendWeightsArgs(group=comm),
|
||||
)
|
||||
@@ -547,7 +631,7 @@ def inference_receive_sparse_tensor(
|
||||
master_port: int,
|
||||
world_size: int,
|
||||
) -> dict:
|
||||
"""Inference task that receives sparse patches via NCCLWeightTransferEngine."""
|
||||
"""Inference task that receives sparse patches via the sparse engine."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import torch
|
||||
@@ -556,22 +640,40 @@ def inference_receive_sparse_tensor(
|
||||
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
NCCLWeightTransferEngine,
|
||||
NCCLWeightTransferInitInfo,
|
||||
NCCLWeightTransferUpdateInfo,
|
||||
from vllm.distributed.weight_transfer.sparse_nccl_engine import (
|
||||
SparseNCCLWeightTransferEngine,
|
||||
SparseNCCLWeightTransferUpdateInfo,
|
||||
)
|
||||
|
||||
config = WeightTransferConfig(backend="nccl")
|
||||
config = WeightTransferConfig(backend="sparse_nccl")
|
||||
vllm_config = MagicMock()
|
||||
parallel_config = MagicMock(spec=ParallelConfig)
|
||||
parallel_config.rank = 0
|
||||
parallel_config.world_size = 1
|
||||
parallel_config.data_parallel_rank = 0
|
||||
parallel_config.data_parallel_index = 0
|
||||
vllm_config.parallel_config = parallel_config
|
||||
vllm_config.model_config = MagicMock()
|
||||
|
||||
engine = NCCLWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
# Real module holding the target parameter the patch will modify.
|
||||
model = torch.nn.Module()
|
||||
model.register_parameter(
|
||||
"w", torch.nn.Parameter(torch.zeros(30, device="cuda"), requires_grad=False)
|
||||
)
|
||||
model.get_parameter = lambda name: model.w
|
||||
|
||||
update_info = SparseNCCLWeightTransferUpdateInfo(
|
||||
names=["w"],
|
||||
dtype_names=["float32"],
|
||||
shapes=[[30]],
|
||||
num_updates_list=[3],
|
||||
)
|
||||
|
||||
engine = SparseNCCLWeightTransferEngine(config, vllm_config, "cuda", model)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
NCCLWeightTransferInitInfo,
|
||||
)
|
||||
|
||||
engine.init_transfer_engine(
|
||||
NCCLWeightTransferInitInfo(
|
||||
master_address=master_address,
|
||||
@@ -580,32 +682,18 @@ def inference_receive_sparse_tensor(
|
||||
world_size=world_size,
|
||||
)
|
||||
)
|
||||
|
||||
target = torch.zeros(30, dtype=torch.float32, device=device)
|
||||
|
||||
def apply_sparse_patches(patches: list[SparseWeightPatch]):
|
||||
for patch in patches:
|
||||
target.index_copy_(0, patch.indices.to(torch.long), patch.values)
|
||||
|
||||
update_info = NCCLWeightTransferUpdateInfo(
|
||||
names=["test.weight"],
|
||||
dtype_names=["float32"],
|
||||
shapes=[[30]],
|
||||
num_updates_list=[3],
|
||||
update_kind="sparse_flat",
|
||||
)
|
||||
engine.receive_sparse_weights(update_info, apply_sparse_patches)
|
||||
engine.receive_weights(update_info)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
expected = torch.zeros(30, dtype=torch.float32, device=device)
|
||||
expected[[1, 7, 25]] = torch.tensor(
|
||||
[10.0, 20.0, 30.0], dtype=torch.float32, device=device
|
||||
)
|
||||
success = torch.equal(target, expected)
|
||||
success = torch.equal(model.w.data, expected)
|
||||
engine.shutdown()
|
||||
return {
|
||||
"success": success,
|
||||
"selected_values": target[[1, 7, 25]].cpu().tolist(),
|
||||
"selected_values": model.w.data[[1, 7, 25]].cpu().tolist(),
|
||||
}
|
||||
|
||||
|
||||
@@ -644,11 +732,9 @@ class TestIPCWeightTransferUpdateInfoValidation:
|
||||
"""Test IPCWeightTransferUpdateInfo dataclass validation."""
|
||||
|
||||
def test_valid_update_info(self):
|
||||
"""Test creating valid IPCWeightTransferUpdateInfo."""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
# Create a dummy tensor and IPC handle
|
||||
dummy_tensor = torch.ones(10, 10, device="cuda:0")
|
||||
_, ipc_handle = reduce_tensor(dummy_tensor)
|
||||
gpu_uuid = str(torch.cuda.get_device_properties(0).uuid)
|
||||
@@ -666,7 +752,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
|
||||
assert len(info.ipc_handles) == 1
|
||||
|
||||
def test_mismatched_dtype_names_raises(self):
|
||||
"""Test that mismatched dtype_names length raises ValueError."""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
@@ -684,7 +769,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
|
||||
)
|
||||
|
||||
def test_mismatched_shapes_raises(self):
|
||||
"""Test that mismatched shapes length raises ValueError."""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
@@ -702,7 +786,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
|
||||
)
|
||||
|
||||
def test_mismatched_ipc_handles_raises(self):
|
||||
"""Test that mismatched ipc_handles length raises ValueError."""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
@@ -719,44 +802,7 @@ class TestIPCWeightTransferUpdateInfoValidation:
|
||||
ipc_handles=ipc_handles,
|
||||
)
|
||||
|
||||
def test_sparse_update_kind_rejected(self):
|
||||
"""Test that IPC backend rejects sparse update metadata."""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
dummy_tensor = torch.ones(10, 10, device="cuda:0")
|
||||
ipc_handle = reduce_tensor(dummy_tensor)
|
||||
gpu_uuid = str(torch.cuda.get_device_properties(0).uuid)
|
||||
ipc_handles = [{gpu_uuid: ipc_handle}]
|
||||
|
||||
with pytest.raises(NotImplementedError, match="dense updates"):
|
||||
IPCWeightTransferUpdateInfo(
|
||||
names=["layer.weight"],
|
||||
dtype_names=["float32"],
|
||||
shapes=[[10, 10]],
|
||||
num_updates_list=[1],
|
||||
ipc_handles=ipc_handles,
|
||||
update_kind="sparse_flat",
|
||||
)
|
||||
|
||||
def test_sparse_methods_not_supported(self):
|
||||
"""Test that IPC engine inherits sparse rejection from the base class."""
|
||||
config = WeightTransferConfig(backend="ipc")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = IPCWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
|
||||
with pytest.raises(NotImplementedError, match="(?i)sparse weight updates"):
|
||||
engine.receive_sparse_weights(MagicMock(), lambda _: None)
|
||||
with pytest.raises(NotImplementedError, match="(?i)sparse weight updates"):
|
||||
engine.trainer_send_sparse_weights(
|
||||
iter([]),
|
||||
{"mode": "http", "url": "http://localhost:8000"},
|
||||
)
|
||||
|
||||
def test_valid_update_info_from_pickled(self, monkeypatch):
|
||||
"""Test creating IPCWeightTransferUpdateInfo from pickled handles."""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
@@ -779,7 +825,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
|
||||
assert info.ipc_handles_pickled is None
|
||||
|
||||
def test_pickled_requires_insecure_serialization_flag(self, monkeypatch):
|
||||
"""Test that pickled handles are rejected unless env flag is enabled."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "0")
|
||||
|
||||
with pytest.raises(ValueError, match="VLLM_ALLOW_INSECURE_SERIALIZATION=1"):
|
||||
@@ -791,7 +836,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
|
||||
)
|
||||
|
||||
def test_both_handles_and_pickled_raises(self):
|
||||
"""Test that providing both ipc_handles and ipc_handles_pickled raises."""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
@@ -812,7 +856,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
|
||||
)
|
||||
|
||||
def test_neither_handles_nor_pickled_raises(self):
|
||||
"""Test that providing neither ipc_handles nor ipc_handles_pickled raises."""
|
||||
with pytest.raises(ValueError, match="must be provided"):
|
||||
IPCWeightTransferUpdateInfo(
|
||||
names=["layer.weight"],
|
||||
@@ -821,7 +864,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
|
||||
)
|
||||
|
||||
def test_empty_lists_valid(self):
|
||||
"""Test that empty lists are valid."""
|
||||
info = IPCWeightTransferUpdateInfo(
|
||||
names=[],
|
||||
dtype_names=[],
|
||||
@@ -837,18 +879,21 @@ class TestIPCWeightTransferUpdateInfoValidation:
|
||||
class TestIPCEngineParsing:
|
||||
"""Test IPCWeightTransferEngine parsing methods."""
|
||||
|
||||
def _make_engine(self):
|
||||
config = WeightTransferConfig(backend="ipc")
|
||||
return IPCWeightTransferEngine(
|
||||
config,
|
||||
create_mock_vllm_config(),
|
||||
"cuda",
|
||||
MagicMock(spec=torch.nn.Module),
|
||||
)
|
||||
|
||||
def test_parse_update_info_valid(self):
|
||||
"""Test parsing valid update info dict."""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
config = WeightTransferConfig(backend="ipc")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = IPCWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
engine = self._make_engine()
|
||||
|
||||
# Create dummy IPC handles
|
||||
dummy_tensor1 = torch.ones(100, 100, device="cuda:0")
|
||||
dummy_tensor2 = torch.ones(50, device="cuda:0")
|
||||
_, ipc_args1 = reduce_tensor(dummy_tensor1)
|
||||
@@ -872,17 +917,12 @@ class TestIPCEngineParsing:
|
||||
assert len(update_info.ipc_handles) == 2
|
||||
|
||||
def test_parse_update_info_pickled(self, monkeypatch):
|
||||
"""Test parsing update info with pickled IPC handles (HTTP path)."""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
config = WeightTransferConfig(backend="ipc")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = IPCWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
engine = self._make_engine()
|
||||
|
||||
dummy_tensor1 = torch.ones(100, 100, device="cuda:0")
|
||||
dummy_tensor2 = torch.ones(50, device="cuda:0")
|
||||
@@ -909,12 +949,7 @@ class TestIPCEngineParsing:
|
||||
assert gpu_uuid in update_info.ipc_handles[1]
|
||||
|
||||
def test_parse_update_info_ignores_none_pickled_handles(self):
|
||||
"""Test Ray/asdict payloads with a null pickled field use ipc_handles."""
|
||||
config = WeightTransferConfig(backend="ipc")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = IPCWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
engine = self._make_engine()
|
||||
ipc_handles = [{"gpu-uuid": ("ipc-args",)}]
|
||||
|
||||
update_info = engine.parse_update_info(
|
||||
@@ -931,15 +966,10 @@ class TestIPCEngineParsing:
|
||||
assert update_info.ipc_handles == ipc_handles
|
||||
|
||||
def test_parse_update_info_both_handles_and_pickled_raises(self):
|
||||
"""Test that providing both ipc_handles and ipc_handles_pickled raises."""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
config = WeightTransferConfig(backend="ipc")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = IPCWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
engine = self._make_engine()
|
||||
|
||||
dummy_tensor = torch.ones(10, 10, device="cuda:0")
|
||||
_, ipc_handle = reduce_tensor(dummy_tensor)
|
||||
@@ -981,9 +1011,6 @@ class TrainerActor:
|
||||
self.tensor = torch.ones(tensor_shape, dtype=dtype, device=device)
|
||||
self.tensor.fill_(42.0) # Fill with 42 to verify correct transfer
|
||||
|
||||
# Create IPC handle (tensor must stay alive for IPC to work)
|
||||
# reduce_tensor returns (rebuild_func, args); we only send args
|
||||
# since the receiver imports rebuild_cuda_tensor directly.
|
||||
_, ipc_args = reduce_tensor(self.tensor)
|
||||
gpu_uuid = get_physical_gpu_id(device.index)
|
||||
|
||||
@@ -1007,6 +1034,7 @@ def inference_receive_ipc_tensor(
|
||||
mode: str = "ray",
|
||||
) -> dict:
|
||||
"""Inference task that receives tensor via IPCWeightTransferEngine."""
|
||||
import contextlib
|
||||
import os
|
||||
|
||||
# Worker-side: ipc_handles_pickled is deserialized via pickle.
|
||||
@@ -1025,30 +1053,36 @@ def inference_receive_ipc_tensor(
|
||||
IPCWeightTransferEngine,
|
||||
)
|
||||
|
||||
# Create engine with mock parallel config
|
||||
class Recorder(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.received = []
|
||||
|
||||
def load_weights(self, weights):
|
||||
for name, tensor in weights:
|
||||
self.received.append((name, tensor.clone()))
|
||||
|
||||
config = WeightTransferConfig(backend="ipc")
|
||||
vllm_config = MagicMock()
|
||||
parallel_config = MagicMock(spec=ParallelConfig)
|
||||
parallel_config.rank = 0
|
||||
parallel_config.world_size = 1
|
||||
parallel_config.data_parallel_rank = 0
|
||||
parallel_config.data_parallel_index = 0
|
||||
vllm_config.parallel_config = parallel_config
|
||||
vllm_config.model_config = MagicMock()
|
||||
|
||||
engine = IPCWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
recorder = Recorder()
|
||||
engine = IPCWeightTransferEngine(config, vllm_config, "cuda", recorder)
|
||||
# Transport-only test: bypass the set_current_vllm_config context that
|
||||
# receive_weights enters, since vllm_config here is a mock.
|
||||
import vllm.config as _vllm_config_mod
|
||||
|
||||
_vllm_config_mod.set_current_vllm_config = lambda cfg: contextlib.nullcontext()
|
||||
|
||||
# Initialize the engine (no-op for IPC)
|
||||
init_info = IPCWeightTransferInitInfo()
|
||||
engine.init_transfer_engine(init_info)
|
||||
|
||||
# Receive weights with a no-op load_weights that captures the tensor
|
||||
received_tensors = []
|
||||
|
||||
def noop_load_weights(weights: list[tuple[str, torch.Tensor]]):
|
||||
for name, tensor in weights:
|
||||
# Clone tensor to keep it after engine cleans up
|
||||
received_tensors.append((name, tensor.clone()))
|
||||
|
||||
ipc_handles = [{ipc_handle_dict["gpu_uuid"]: ipc_handle_dict["ipc_handle"]}]
|
||||
|
||||
if mode == "ray":
|
||||
@@ -1059,7 +1093,6 @@ def inference_receive_ipc_tensor(
|
||||
"ipc_handles": ipc_handles,
|
||||
}
|
||||
elif mode == "http":
|
||||
# Simulate HTTP transport: pickle + base64 encode handles
|
||||
pickled = base64.b64encode(pickle.dumps(ipc_handles)).decode("utf-8")
|
||||
update_dict = {
|
||||
"names": ["test.weight"],
|
||||
@@ -1071,19 +1104,17 @@ def inference_receive_ipc_tensor(
|
||||
raise ValueError(f"Unknown mode: {mode}")
|
||||
|
||||
update_info = engine.parse_update_info(update_dict)
|
||||
engine.receive_weights(update_info, noop_load_weights)
|
||||
engine.receive_weights(update_info)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
# Verify we received the tensor
|
||||
success = False
|
||||
received_shape = None
|
||||
received_sum = None
|
||||
|
||||
if len(received_tensors) == 1:
|
||||
name, tensor = received_tensors[0]
|
||||
if len(recorder.received) == 1:
|
||||
name, tensor = recorder.received[0]
|
||||
received_shape = list(tensor.shape)
|
||||
received_sum = tensor.sum().item()
|
||||
# Check shape matches and values are all 42s (trainer sends 42s)
|
||||
if received_shape == ipc_handle_dict["shape"]:
|
||||
expected_sum = 42.0 * torch.tensor(ipc_handle_dict["shape"]).prod().item()
|
||||
if abs(received_sum - expected_sum) < 0.01:
|
||||
@@ -1104,23 +1135,12 @@ def inference_receive_ipc_tensor(
|
||||
)
|
||||
@pytest.mark.parametrize("mode", ["ray", "http"])
|
||||
def test_ipc_weight_transfer_between_processes(mode: str):
|
||||
"""Test IPC weight transfer from trainer to inference process using Ray.
|
||||
|
||||
Parametrized over transport modes:
|
||||
- 'ray': ipc_handles passed directly.
|
||||
- 'http': ipc_handles pickled + base64-encoded, deserialized in
|
||||
parse_update_info before constructing the dataclass.
|
||||
|
||||
IPC requires same-GPU access, so we use a placement group to co-locate
|
||||
the trainer actor and inference task on the same GPU.
|
||||
"""
|
||||
"""Test IPC weight transfer from trainer to inference process using Ray."""
|
||||
from ray.util.placement_group import placement_group
|
||||
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
|
||||
|
||||
_init_ray_for_weight_transfer()
|
||||
|
||||
# Create a placement group to ensure both processes are on the same GPU
|
||||
# Use fractional GPUs so both tasks can share the same GPU bundle
|
||||
pg = placement_group([{"GPU": 1, "CPU": 2}])
|
||||
ray.get(pg.ready())
|
||||
|
||||
@@ -1129,20 +1149,15 @@ def test_ipc_weight_transfer_between_processes(mode: str):
|
||||
placement_group_capture_child_tasks=True,
|
||||
)
|
||||
|
||||
# Tensor to transfer: 100x100 filled with 42s
|
||||
tensor_shape = [100, 100]
|
||||
tensor_dtype = "float32"
|
||||
|
||||
# Create trainer actor that holds the tensor and IPC handle (stays alive)
|
||||
trainer_actor = TrainerActor.options( # type: ignore[attr-defined]
|
||||
scheduling_strategy=scheduling_strategy
|
||||
).remote(tensor_shape, tensor_dtype)
|
||||
|
||||
# Get IPC handle dict (tensor stays alive in trainer actor)
|
||||
ipc_handle_dict = ray.get(trainer_actor.get_ipc_handle_dict.remote())
|
||||
|
||||
# Receive tensor in inference process using IPC handles (on same GPU)
|
||||
# Trainer actor stays alive during this operation
|
||||
inference_result = ray.get(
|
||||
inference_receive_ipc_tensor.options(
|
||||
scheduling_strategy=scheduling_strategy
|
||||
@@ -1162,12 +1177,10 @@ def test_ipc_receive_weights_missing_gpu_uuid_raises():
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
config = WeightTransferConfig(backend="ipc")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = IPCWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
|
||||
# Create IPC handle with wrong GPU UUID
|
||||
dummy_tensor = torch.ones(10, 10, device="cuda:0")
|
||||
_, ipc_handle = reduce_tensor(dummy_tensor)
|
||||
wrong_uuid = "wrong-uuid-12345"
|
||||
@@ -1181,4 +1194,4 @@ def test_ipc_receive_weights_missing_gpu_uuid_raises():
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="IPC handle not found"):
|
||||
engine.receive_weights(update_info, lambda x: None)
|
||||
engine.receive_weights(update_info)
|
||||
|
||||
@@ -8,7 +8,6 @@ actual NCCL communication.
|
||||
"""
|
||||
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -48,7 +47,6 @@ class MockUpdateInfo(WeightTransferUpdateInfo):
|
||||
names: list[str] | None = None
|
||||
dtype_names: list[str] | None = None
|
||||
shapes: list[list[int]] | None = None
|
||||
num_updates_list: list[int] | None = None
|
||||
|
||||
|
||||
class MockWeightTransferEngine(WeightTransferEngine[MockInitInfo, MockUpdateInfo]):
|
||||
@@ -59,16 +57,20 @@ class MockWeightTransferEngine(WeightTransferEngine[MockInitInfo, MockUpdateInfo
|
||||
|
||||
# Class-level tracking for verification across processes
|
||||
init_transfer_engine_called: bool = False
|
||||
start_called: bool = False
|
||||
receive_weights_called: bool = False
|
||||
finish_called: bool = False
|
||||
shutdown_called: bool = False
|
||||
last_init_info: MockInitInfo | None = None
|
||||
last_update_info: MockUpdateInfo | None = None
|
||||
|
||||
def __init__(self, config, parallel_config, model):
|
||||
super().__init__(config, parallel_config, model)
|
||||
def __init__(self, config, vllm_config, device, model):
|
||||
super().__init__(config, vllm_config, device, model)
|
||||
# Reset tracking on init
|
||||
MockWeightTransferEngine.init_transfer_engine_called = False
|
||||
MockWeightTransferEngine.start_called = False
|
||||
MockWeightTransferEngine.receive_weights_called = False
|
||||
MockWeightTransferEngine.finish_called = False
|
||||
MockWeightTransferEngine.shutdown_called = False
|
||||
MockWeightTransferEngine.last_init_info = None
|
||||
MockWeightTransferEngine.last_update_info = None
|
||||
@@ -77,37 +79,28 @@ class MockWeightTransferEngine(WeightTransferEngine[MockInitInfo, MockUpdateInfo
|
||||
MockWeightTransferEngine.init_transfer_engine_called = True
|
||||
MockWeightTransferEngine.last_init_info = init_info
|
||||
|
||||
def receive_weights(
|
||||
self,
|
||||
update_info: MockUpdateInfo,
|
||||
load_weights: Callable[[list[tuple[str, torch.Tensor]]], None],
|
||||
) -> None:
|
||||
MockWeightTransferEngine.receive_weights_called = True
|
||||
MockWeightTransferEngine.last_update_info = update_info
|
||||
# Simulate loading weights by calling load_weights with empty list
|
||||
# (In real implementation, this would receive and load actual weights)
|
||||
load_weights([])
|
||||
def start_weight_update(self) -> None:
|
||||
MockWeightTransferEngine.start_called = True
|
||||
|
||||
def receive_sparse_weights(
|
||||
self,
|
||||
update_info: MockUpdateInfo,
|
||||
apply_patches: Callable[[list], None],
|
||||
) -> None:
|
||||
def finish_weight_update(self) -> None:
|
||||
MockWeightTransferEngine.finish_called = True
|
||||
|
||||
def receive_weights(self, update_info: MockUpdateInfo) -> None:
|
||||
MockWeightTransferEngine.receive_weights_called = True
|
||||
MockWeightTransferEngine.last_update_info = update_info
|
||||
apply_patches([])
|
||||
|
||||
def shutdown(self) -> None:
|
||||
MockWeightTransferEngine.shutdown_called = True
|
||||
|
||||
def trainer_send_weights(self, *args, **kwargs):
|
||||
@staticmethod
|
||||
def trainer_send_weights(*args, **kwargs):
|
||||
"""Mock method to simulate trainer sending weights."""
|
||||
pass
|
||||
|
||||
|
||||
def mock_create_engine(config, parallel_config, model):
|
||||
def mock_create_engine(config, vllm_config, device, model):
|
||||
"""Mock factory function that returns our mock engine."""
|
||||
return MockWeightTransferEngine(config, parallel_config, model)
|
||||
return MockWeightTransferEngine(config, vllm_config, device, model)
|
||||
|
||||
|
||||
# --- Tests ---
|
||||
@@ -208,7 +201,7 @@ def test_update_weights_calls_engine():
|
||||
llm.init_weight_transfer_engine(
|
||||
WeightTransferInitRequest(init_info={"test_param": "init"})
|
||||
)
|
||||
llm.start_weight_update(is_checkpoint_format=True)
|
||||
llm.start_weight_update()
|
||||
|
||||
# Call update_weights
|
||||
test_names = ["layer.weight", "layer.bias"]
|
||||
@@ -243,61 +236,6 @@ def test_update_weights_calls_engine():
|
||||
llm.finish_weight_update()
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_update_weights_passes_sparse_metadata():
|
||||
"""Test sparse update metadata is forwarded unchanged to the engine."""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0"
|
||||
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
|
||||
|
||||
with patch(
|
||||
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
|
||||
mock_create_engine,
|
||||
):
|
||||
llm = LLM(
|
||||
model=MODEL_NAME,
|
||||
enforce_eager=True,
|
||||
load_format="dummy",
|
||||
tensor_parallel_size=1,
|
||||
weight_transfer_config=WeightTransferConfig(backend="nccl"),
|
||||
)
|
||||
|
||||
llm.init_weight_transfer_engine(
|
||||
WeightTransferInitRequest(init_info={"test_param": "init"})
|
||||
)
|
||||
llm.start_weight_update(is_checkpoint_format=False)
|
||||
|
||||
llm.update_weights(
|
||||
WeightTransferUpdateRequest(
|
||||
update_info={
|
||||
"names": ["layer.weight"],
|
||||
"dtype_names": ["bfloat16"],
|
||||
"shapes": [[100]],
|
||||
"num_updates_list": [3],
|
||||
"update_kind": "sparse_flat",
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
def check_sparse_update_called(self):
|
||||
engine = self.weight_transfer_engine
|
||||
if not engine.receive_weights_called:
|
||||
return None
|
||||
info = engine.last_update_info
|
||||
return (
|
||||
info.update_kind,
|
||||
info.num_updates_list,
|
||||
)
|
||||
|
||||
results = llm.collective_rpc(check_sparse_update_called)
|
||||
for result in results:
|
||||
assert result == ("sparse_flat", [3])
|
||||
|
||||
llm.finish_weight_update()
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_full_weight_transfer_flow():
|
||||
"""Test the complete weight transfer flow: init -> start -> update -> finish."""
|
||||
@@ -327,7 +265,7 @@ def test_full_weight_transfer_flow():
|
||||
)
|
||||
|
||||
# Step 2: Start weight update
|
||||
llm.start_weight_update(is_checkpoint_format=True)
|
||||
llm.start_weight_update()
|
||||
|
||||
# Step 3: Update weights
|
||||
llm.update_weights(
|
||||
|
||||
@@ -7,7 +7,6 @@ from unittest.mock import Mock
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import vllm.v1.worker.gpu_model_runner as gpu_model_runner_module
|
||||
from vllm.config import (
|
||||
@@ -23,7 +22,6 @@ from vllm.distributed.parallel_state import (
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.base import SparseWeightPatch
|
||||
from vllm.lora.layers import LoRAMappingType
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
@@ -864,73 +862,6 @@ def test_sample_passes_reordered_draft_probs_to_rejection_sampler():
|
||||
assert torch.equal(passed_draft_probs, expected_draft_probs)
|
||||
|
||||
|
||||
def test_apply_sparse_weight_patches_updates_only_selected_entries():
|
||||
class DummyModel(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.zeros(6, dtype=torch.float32))
|
||||
|
||||
runner = object.__new__(GPUModelRunner)
|
||||
runner.model = DummyModel()
|
||||
|
||||
runner.apply_sparse_weight_patches(
|
||||
[
|
||||
SparseWeightPatch(
|
||||
name="weight",
|
||||
indices=torch.tensor([1, 4], dtype=torch.int32),
|
||||
values=torch.tensor([3.5, -2.0], dtype=torch.float32),
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
expected = torch.tensor([0.0, 3.5, 0.0, 0.0, -2.0, 0.0], dtype=torch.float32)
|
||||
assert torch.equal(runner.get_model().weight.data, expected)
|
||||
|
||||
|
||||
def test_apply_sparse_weight_patches_rejects_mismatched_lengths():
|
||||
class DummyModel(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.zeros(4, dtype=torch.float32))
|
||||
|
||||
runner = object.__new__(GPUModelRunner)
|
||||
runner.model = DummyModel()
|
||||
|
||||
with pytest.raises(ValueError, match="matching lengths"):
|
||||
runner.apply_sparse_weight_patches(
|
||||
[
|
||||
SparseWeightPatch(
|
||||
name="weight",
|
||||
indices=torch.tensor([1, 2], dtype=torch.int32),
|
||||
values=torch.tensor([1.0], dtype=torch.float32),
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_apply_sparse_weight_patches_rejects_non_contiguous_param():
|
||||
class DummyModel(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(
|
||||
torch.arange(12, dtype=torch.float32).view(3, 4).t()
|
||||
)
|
||||
|
||||
runner = object.__new__(GPUModelRunner)
|
||||
runner.model = DummyModel()
|
||||
|
||||
with pytest.raises(NotImplementedError, match="contiguous params"):
|
||||
runner.apply_sparse_weight_patches(
|
||||
[
|
||||
SparseWeightPatch(
|
||||
name="weight",
|
||||
indices=torch.tensor([1], dtype=torch.int32),
|
||||
values=torch.tensor([1.0], dtype=torch.float32),
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_init_kv_cache_with_kv_sharing_invalid_target_layer_order(default_vllm_config):
|
||||
torch.set_default_dtype(torch.float16)
|
||||
layer_0 = "model.layers.0.self_attn.attn"
|
||||
|
||||
@@ -1,155 +1,94 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for GPUWorker weight-transfer pass-through behavior.
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
The worker no longer contains transport, layerwise, or sparse logic: it only
|
||||
delegates to the configured weight transfer engine and tracks whether an update
|
||||
session is active. These tests verify that delegation and the session guard.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import SparseWeightPatch
|
||||
from vllm.distributed.weight_transfer.nccl_engine import NCCLWeightTransferEngine
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
|
||||
def _make_nccl_engine() -> NCCLWeightTransferEngine:
|
||||
parallel_config = MagicMock(spec=ParallelConfig)
|
||||
parallel_config.rank = 0
|
||||
parallel_config.world_size = 1
|
||||
parallel_config.data_parallel_rank = 0
|
||||
parallel_config.data_parallel_index = 0
|
||||
return NCCLWeightTransferEngine(
|
||||
WeightTransferConfig(backend="nccl"),
|
||||
parallel_config,
|
||||
MagicMock(spec=torch.nn.Module),
|
||||
)
|
||||
class _RecordingEngine:
|
||||
"""Minimal stand-in for a weight transfer engine."""
|
||||
|
||||
def __init__(self, raise_on_update: bool = False):
|
||||
self.raise_on_update = raise_on_update
|
||||
self.started = False
|
||||
self.finished = False
|
||||
self.update_calls: list[dict] = []
|
||||
|
||||
def start_weight_update(self) -> None:
|
||||
self.started = True
|
||||
|
||||
def update_weights(self, update_info: dict) -> None:
|
||||
self.update_calls.append(update_info)
|
||||
if self.raise_on_update:
|
||||
raise ValueError("boom")
|
||||
|
||||
def finish_weight_update(self) -> None:
|
||||
self.finished = True
|
||||
|
||||
|
||||
def test_update_weights_sparse_dispatches_to_sparse_receive(monkeypatch):
|
||||
monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None)
|
||||
|
||||
def _make_worker(engine: _RecordingEngine | None) -> Worker:
|
||||
worker = object.__new__(Worker)
|
||||
worker.device = "cpu"
|
||||
worker.parallel_config = SimpleNamespace(world_size=1)
|
||||
worker.weight_transfer_engine = _make_nccl_engine()
|
||||
worker._weight_update_active = True
|
||||
worker._is_checkpoint_format = False
|
||||
|
||||
applied_patches = []
|
||||
|
||||
def apply_sparse_weight_patches(patches):
|
||||
applied_patches.extend(patches)
|
||||
|
||||
worker.model_runner = SimpleNamespace(
|
||||
apply_sparse_weight_patches=apply_sparse_weight_patches,
|
||||
)
|
||||
|
||||
received_kinds = []
|
||||
|
||||
def receive_sparse_weights(update_info, apply_patches):
|
||||
received_kinds.append(update_info.update_kind)
|
||||
apply_patches(
|
||||
[
|
||||
SparseWeightPatch(
|
||||
name="layer.weight",
|
||||
indices=torch.tensor([1], dtype=torch.int32),
|
||||
values=torch.tensor([2.0], dtype=torch.float32),
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
worker.weight_transfer_engine.receive_sparse_weights = receive_sparse_weights
|
||||
|
||||
Worker.update_weights(
|
||||
worker,
|
||||
{
|
||||
"names": ["layer.weight"],
|
||||
"dtype_names": ["float32"],
|
||||
"shapes": [[4]],
|
||||
"num_updates_list": [1],
|
||||
"update_kind": "sparse_flat",
|
||||
},
|
||||
)
|
||||
|
||||
assert received_kinds == ["sparse_flat"]
|
||||
assert len(applied_patches) == 1
|
||||
assert torch.equal(applied_patches[0].indices, torch.tensor([1], dtype=torch.int32))
|
||||
worker.weight_transfer_engine = engine
|
||||
worker._weight_update_active = False
|
||||
return worker
|
||||
|
||||
|
||||
def test_update_weights_sparse_rejects_tp_or_pp(monkeypatch):
|
||||
monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None)
|
||||
def test_start_update_finish_delegates_to_engine():
|
||||
engine = _RecordingEngine()
|
||||
worker = _make_worker(engine)
|
||||
|
||||
worker = object.__new__(Worker)
|
||||
worker.device = "cpu"
|
||||
worker.parallel_config = SimpleNamespace(world_size=2)
|
||||
worker.weight_transfer_engine = _make_nccl_engine()
|
||||
worker._weight_update_active = True
|
||||
worker._is_checkpoint_format = False
|
||||
worker.model_runner = SimpleNamespace(apply_sparse_weight_patches=lambda _: None)
|
||||
Worker.start_weight_update(worker)
|
||||
assert engine.started is True
|
||||
assert worker._weight_update_active is True
|
||||
|
||||
with pytest.raises(NotImplementedError, match="TP=1 and PP=1"):
|
||||
Worker.update_weights(
|
||||
worker,
|
||||
{
|
||||
"names": ["layer.weight"],
|
||||
"dtype_names": ["float32"],
|
||||
"shapes": [[4]],
|
||||
"num_updates_list": [1],
|
||||
"update_kind": "sparse_flat",
|
||||
},
|
||||
)
|
||||
Worker.update_weights(worker, {"names": ["w"]})
|
||||
assert engine.update_calls == [{"names": ["w"]}]
|
||||
assert worker._weight_update_active is True
|
||||
|
||||
Worker.finish_weight_update(worker)
|
||||
assert engine.finished is True
|
||||
assert worker._weight_update_active is False
|
||||
assert worker._is_checkpoint_format is True
|
||||
|
||||
|
||||
def test_update_weights_sparse_rejects_checkpoint_format(monkeypatch):
|
||||
monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None)
|
||||
def test_double_start_raises():
|
||||
worker = _make_worker(_RecordingEngine())
|
||||
Worker.start_weight_update(worker)
|
||||
with pytest.raises(RuntimeError, match="already"):
|
||||
Worker.start_weight_update(worker)
|
||||
|
||||
worker = object.__new__(Worker)
|
||||
worker.device = "cpu"
|
||||
worker.parallel_config = SimpleNamespace(world_size=1)
|
||||
worker.weight_transfer_engine = _make_nccl_engine()
|
||||
worker._weight_update_active = True
|
||||
worker._is_checkpoint_format = True
|
||||
worker.model_runner = SimpleNamespace(model=MagicMock())
|
||||
|
||||
with pytest.raises(ValueError, match="start_weight_update"):
|
||||
Worker.update_weights(
|
||||
worker,
|
||||
{
|
||||
"names": ["layer.weight"],
|
||||
"dtype_names": ["float32"],
|
||||
"shapes": [[4]],
|
||||
"num_updates_list": [1],
|
||||
"update_kind": "sparse_flat",
|
||||
},
|
||||
)
|
||||
def test_update_without_start_raises():
|
||||
worker = _make_worker(_RecordingEngine())
|
||||
with pytest.raises(RuntimeError, match="start_weight_update must be called"):
|
||||
Worker.update_weights(worker, {"names": ["w"]})
|
||||
|
||||
|
||||
def test_finish_without_start_raises():
|
||||
worker = _make_worker(_RecordingEngine())
|
||||
with pytest.raises(RuntimeError, match="without a matching"):
|
||||
Worker.finish_weight_update(worker)
|
||||
|
||||
|
||||
def test_update_resets_active_on_error():
|
||||
engine = _RecordingEngine(raise_on_update=True)
|
||||
worker = _make_worker(engine)
|
||||
Worker.start_weight_update(worker)
|
||||
|
||||
with pytest.raises(ValueError, match="boom"):
|
||||
Worker.update_weights(worker, {"names": ["w"]})
|
||||
|
||||
# A failed update ends the session so the next start is clean.
|
||||
assert worker._weight_update_active is False
|
||||
assert worker._is_checkpoint_format is True
|
||||
|
||||
|
||||
def test_update_weights_resets_state_when_update_info_is_invalid(monkeypatch):
|
||||
monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None)
|
||||
|
||||
worker = object.__new__(Worker)
|
||||
worker.device = "cpu"
|
||||
worker.parallel_config = SimpleNamespace(world_size=1)
|
||||
worker.weight_transfer_engine = _make_nccl_engine()
|
||||
worker._weight_update_active = True
|
||||
worker._is_checkpoint_format = False
|
||||
|
||||
with pytest.raises(ValueError, match="cannot be empty"):
|
||||
Worker.update_weights(
|
||||
worker,
|
||||
{
|
||||
"names": [],
|
||||
"dtype_names": [],
|
||||
"shapes": [],
|
||||
"num_updates_list": [],
|
||||
"update_kind": "sparse_flat",
|
||||
},
|
||||
)
|
||||
assert worker._weight_update_active is False
|
||||
assert worker._is_checkpoint_format is True
|
||||
def test_missing_engine_raises():
|
||||
worker = _make_worker(None)
|
||||
with pytest.raises(RuntimeError, match="Weight transfer not configured"):
|
||||
Worker.start_weight_update(worker)
|
||||
|
||||
@@ -9,7 +9,7 @@ from vllm.config.utils import config
|
||||
class WeightTransferConfig:
|
||||
"""Configuration for weight transfer during RL training."""
|
||||
|
||||
backend: Literal["nccl", "ipc"] | str = "nccl"
|
||||
backend: Literal["nccl", "ipc", "sparse_nccl"] | str = "nccl"
|
||||
"""The backend to use for weight transfer. Validated against the
|
||||
`WeightTransferEngineFactory` registry at engine creation time.
|
||||
"""
|
||||
|
||||
@@ -3,12 +3,15 @@
|
||||
"""Base class for weight transfer engines."""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable, Iterator
|
||||
from dataclasses import KW_ONLY, dataclass, field
|
||||
from typing import Any, Generic, Literal, TypeVar
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, Generic, TypeVar
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
|
||||
@@ -28,44 +31,7 @@ class WeightTransferInitInfo(ABC): # noqa: B024
|
||||
class WeightTransferUpdateInfo(ABC): # noqa: B024
|
||||
"""Base class for backend-specific weight update info."""
|
||||
|
||||
_: KW_ONLY
|
||||
update_kind: Literal["dense", "sparse_flat"] = "dense"
|
||||
"""Weight update format."""
|
||||
num_updates_list: list[int] | None = None
|
||||
"""Number of sparse entries to receive for each parameter in ``names``."""
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.update_kind not in ("dense", "sparse_flat"):
|
||||
raise ValueError(f"Unsupported update_kind: {self.update_kind}")
|
||||
if self.update_kind == "dense":
|
||||
if self.num_updates_list is not None:
|
||||
raise ValueError(
|
||||
"Sparse metadata is only supported for `update_kind='sparse_flat'`"
|
||||
)
|
||||
return
|
||||
|
||||
if self.num_updates_list is None:
|
||||
raise ValueError("`num_updates_list` is required for sparse updates")
|
||||
if len(self.num_updates_list) == 0:
|
||||
raise ValueError("`num_updates_list` cannot be empty for sparse updates")
|
||||
if any(num_updates < 0 for num_updates in self.num_updates_list):
|
||||
raise ValueError("Sparse `num_updates_list` entries must be non-negative")
|
||||
|
||||
names = getattr(self, "names", None)
|
||||
if names is not None and len(self.num_updates_list) != len(names):
|
||||
raise ValueError(
|
||||
f"`num_updates_list` should be of the same size as `names`: "
|
||||
f"got {len(self.num_updates_list)} and {len(names)}"
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SparseWeightPatch:
|
||||
"""A sparse in-place patch for one existing parameter."""
|
||||
|
||||
name: str
|
||||
indices: torch.Tensor
|
||||
values: torch.Tensor
|
||||
pass
|
||||
|
||||
|
||||
# API-level request classes (accept dicts for backend-agnostic serialization)
|
||||
@@ -89,9 +55,15 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
from a trainer to inference workers.
|
||||
|
||||
This abstraction separates weight transfer transport logic from the worker
|
||||
implementation, allowing different backends (NCCL, CUDA IPC[TODO], RDMA[TODO]) to be
|
||||
implementation, allowing different backends (NCCL, CUDA IPC, RDMA[TODO]) to be
|
||||
plugged in.
|
||||
|
||||
Each engine owns its full weight-update lifecycle: `start_weight_update`,
|
||||
`update_weights`, and `finish_weight_update`. Layerwise reloading (used by
|
||||
checkpoint-format engines) is opted into per engine by running it inside
|
||||
`start_weight_update`/`finish_weight_update`. Engines that apply weights in
|
||||
place (e.g. sparse patches) leave those methods as no-ops.
|
||||
|
||||
Subclasses should define:
|
||||
init_info_cls: Type of backend-specific initialization info
|
||||
update_info_cls: Type of backend-specific update info
|
||||
@@ -104,7 +76,8 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
def __init__(
|
||||
self,
|
||||
config: WeightTransferConfig,
|
||||
parallel_config: ParallelConfig,
|
||||
vllm_config: "VllmConfig",
|
||||
device: torch.device,
|
||||
model: torch.nn.Module,
|
||||
) -> None:
|
||||
"""
|
||||
@@ -112,11 +85,15 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
|
||||
Args:
|
||||
config: The configuration for the weight transfer engine
|
||||
parallel_config: The configuration for the parallel setup
|
||||
vllm_config: The full vLLM config (provides parallel/model config)
|
||||
device: The device this worker's model lives on
|
||||
model: The local model instance which will receive the weights
|
||||
"""
|
||||
self.config = config
|
||||
self.parallel_config = parallel_config
|
||||
self.vllm_config = vllm_config
|
||||
self.parallel_config: ParallelConfig = vllm_config.parallel_config
|
||||
self.model_config = vllm_config.model_config
|
||||
self.device = device
|
||||
self.model = model
|
||||
|
||||
def parse_init_info(self, init_dict: dict[str, Any]) -> TInitInfo:
|
||||
@@ -171,32 +148,50 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def receive_weights(
|
||||
self,
|
||||
update_info: TUpdateInfo,
|
||||
load_weights: Callable[[list[tuple[str, torch.Tensor]]], None],
|
||||
) -> None:
|
||||
def start_weight_update(self) -> None:
|
||||
"""
|
||||
Receive weights from the trainer and load them incrementally.
|
||||
Prepare the engine for a new weight update.
|
||||
|
||||
Engines that receive weights in checkpoint format initialize layerwise reloading
|
||||
here, else this is typically a no-op.
|
||||
See: https://docs.vllm.ai/en/latest/training/layerwise/ for more details.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def finish_weight_update(self) -> None:
|
||||
"""
|
||||
Finalize the current weight update.
|
||||
|
||||
Checkpoint-format engines finalize layerwise reloading here; engines
|
||||
that apply weights in place leave this as a no-op.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def update_weights(self, update_info: dict[str, Any]) -> None:
|
||||
"""
|
||||
Receive one weight update chunk and load it into the model.
|
||||
|
||||
Args:
|
||||
update_info: Dictionary containing backend-specific update info
|
||||
"""
|
||||
typed_update_info = self.parse_update_info(update_info)
|
||||
self.receive_weights(typed_update_info)
|
||||
# NCCL broadcast / IPC paths may be asynchronous. Synchronize here so the
|
||||
# next step uses the new weights.
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@abstractmethod
|
||||
def receive_weights(self, update_info: TUpdateInfo) -> None:
|
||||
"""
|
||||
Receive weights from the trainer and load them into the model.
|
||||
|
||||
Args:
|
||||
update_info: Backend-specific update info containing parameter metadata
|
||||
and any backend-specific data
|
||||
load_weights: Callable that loads weights into the model. Called
|
||||
incrementally for each weight to avoid OOM.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def receive_sparse_weights(
|
||||
self,
|
||||
update_info: TUpdateInfo,
|
||||
apply_patches: Callable[[list[SparseWeightPatch]], None],
|
||||
) -> None:
|
||||
"""Receive sparse weight patches from the trainer."""
|
||||
raise NotImplementedError(
|
||||
f"{self.__class__.__name__} does not support sparse weight updates"
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def shutdown(self) -> None:
|
||||
"""
|
||||
@@ -208,7 +203,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def trainer_send_weights(
|
||||
iterator: Iterator[tuple[str, torch.Tensor]],
|
||||
iterator: Iterator[Any],
|
||||
trainer_args: dict[str, Any] | Any,
|
||||
) -> None:
|
||||
"""
|
||||
@@ -218,8 +213,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
to send weights to all inference workers.
|
||||
|
||||
Args:
|
||||
iterator: Iterator of model parameters. Returns (name, tensor) tuples.
|
||||
The tensors should be on the appropriate device for the backend.
|
||||
iterator: Iterator of backend-specific items to send.
|
||||
trainer_args: Dictionary containing backend-specific arguments needed
|
||||
to send weights. The structure depends on the backend:
|
||||
- NCCL: Contains 'group', 'src', 'packed', etc.
|
||||
@@ -231,11 +225,3 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
>>> engine.trainer_send_weights(param_iter, trainer_args)
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
def trainer_send_sparse_weights(
|
||||
_iterator: Iterator[SparseWeightPatch],
|
||||
_trainer_args: dict[str, Any] | Any,
|
||||
) -> None:
|
||||
"""Send sparse weight patches from trainer to inference workers."""
|
||||
raise NotImplementedError("Sparse weight updates are not supported")
|
||||
|
||||
@@ -12,7 +12,7 @@ from vllm.logger import init_logger
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -76,14 +76,16 @@ class WeightTransferEngineFactory:
|
||||
def create_engine(
|
||||
cls,
|
||||
config: "WeightTransferConfig",
|
||||
parallel_config: "ParallelConfig",
|
||||
vllm_config: "VllmConfig",
|
||||
device: "torch.device",
|
||||
model: "torch.nn.Module",
|
||||
) -> WeightTransferEngine:
|
||||
"""Create a weight transfer engine instance.
|
||||
|
||||
Args:
|
||||
config: Weight transfer configuration containing the backend name
|
||||
parallel_config: Parallel configuration for the engine
|
||||
vllm_config: The full vLLM config (provides parallel/model config)
|
||||
device: The device this worker's model lives on
|
||||
model: The local model instance which will receive the weights
|
||||
|
||||
Returns:
|
||||
@@ -106,7 +108,7 @@ class WeightTransferEngineFactory:
|
||||
engine_cls.__name__,
|
||||
)
|
||||
|
||||
return engine_cls(config, parallel_config, model)
|
||||
return engine_cls(config, vllm_config, device, model)
|
||||
|
||||
|
||||
# Register built-in weight transfer engines here.
|
||||
@@ -124,3 +126,9 @@ WeightTransferEngineFactory.register_engine(
|
||||
"vllm.distributed.weight_transfer.ipc_engine",
|
||||
"IPCWeightTransferEngine",
|
||||
)
|
||||
|
||||
WeightTransferEngineFactory.register_engine(
|
||||
"sparse_nccl",
|
||||
"vllm.distributed.weight_transfer.sparse_nccl_engine",
|
||||
"SparseNCCLWeightTransferEngine",
|
||||
)
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
import pickle
|
||||
from collections.abc import Callable, Iterator
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pybase64 as base64
|
||||
import ray
|
||||
@@ -14,13 +14,15 @@ import torch
|
||||
from torch.multiprocessing.reductions import rebuild_cuda_tensor, reduce_tensor
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
WeightTransferEngine,
|
||||
WeightTransferInitInfo,
|
||||
WeightTransferUpdateInfo,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.weight_transfer.packed_tensor import (
|
||||
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
packed_ipc_consumer,
|
||||
@@ -87,10 +89,6 @@ class IPCWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
"""Whether this update uses packed tensor format."""
|
||||
|
||||
def __post_init__(self):
|
||||
super().__post_init__()
|
||||
if self.update_kind != "dense":
|
||||
raise NotImplementedError("IPC weight transfer only supports dense updates")
|
||||
|
||||
if self.ipc_handles_pickled is not None:
|
||||
if self.ipc_handles is not None:
|
||||
raise ValueError(
|
||||
@@ -152,7 +150,8 @@ class IPCWeightTransferEngine(
|
||||
def __init__(
|
||||
self,
|
||||
config: WeightTransferConfig,
|
||||
parallel_config: ParallelConfig,
|
||||
vllm_config: "VllmConfig",
|
||||
device: torch.device,
|
||||
model: torch.nn.Module,
|
||||
) -> None:
|
||||
"""
|
||||
@@ -160,10 +159,11 @@ class IPCWeightTransferEngine(
|
||||
|
||||
Args:
|
||||
config: The configuration for the weight transfer engine
|
||||
parallel_config: The configuration for the parallel setup
|
||||
vllm_config: The full vLLM config
|
||||
device: The device this worker's model lives on
|
||||
model: The local model instance which will receive the weights
|
||||
"""
|
||||
super().__init__(config, parallel_config, model)
|
||||
super().__init__(config, vllm_config, device, model)
|
||||
|
||||
def parse_update_info(
|
||||
self, update_dict: dict[str, Any]
|
||||
@@ -206,22 +206,37 @@ class IPCWeightTransferEngine(
|
||||
"""
|
||||
pass
|
||||
|
||||
def receive_weights(
|
||||
self,
|
||||
update_info: IPCWeightTransferUpdateInfo,
|
||||
load_weights: Callable[[list[tuple[str, torch.Tensor]]], None],
|
||||
) -> None:
|
||||
def start_weight_update(self) -> None:
|
||||
"""Initialize layerwise reloading for the incoming checkpoint weights."""
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
initialize_layerwise_reload,
|
||||
)
|
||||
|
||||
initialize_layerwise_reload(self.model)
|
||||
|
||||
def finish_weight_update(self) -> None:
|
||||
"""Finalize layerwise reloading after all weights have been received."""
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
finalize_layerwise_reload,
|
||||
)
|
||||
|
||||
finalize_layerwise_reload(self.model, self.model_config)
|
||||
|
||||
def receive_weights(self, update_info: IPCWeightTransferUpdateInfo) -> None:
|
||||
"""
|
||||
Receive weights from the trainer via CUDA IPC handles.
|
||||
Receive weights from the trainer via CUDA IPC handles and load them.
|
||||
|
||||
Args:
|
||||
update_info: IPC update info containing parameter names, dtypes, shapes,
|
||||
and IPC handles. Each IPC handle is a mapping between physical
|
||||
GPU UUID and the rebuild_cuda_tensor args tuple.
|
||||
load_weights: Callable that loads weights into the model. Called
|
||||
incrementally for each weight to avoid OOM.
|
||||
"""
|
||||
device_index = torch.accelerator.current_device_index()
|
||||
# Use the worker's assigned device rather than the ambient current
|
||||
# device: the receive path is no longer wrapped in
|
||||
# `with torch.device(self.device)` by the caller, so the current device
|
||||
# is not guaranteed to match self.device. The IPC tensors must be
|
||||
# rebuilt on the device the model lives on.
|
||||
device_index = self.device.index
|
||||
|
||||
if update_info.packed:
|
||||
assert update_info.tensor_sizes is not None
|
||||
@@ -234,7 +249,6 @@ class IPCWeightTransferEngine(
|
||||
tensor_sizes=update_info.tensor_sizes,
|
||||
device_index=device_index,
|
||||
)
|
||||
load_weights(weights)
|
||||
else:
|
||||
assert isinstance(update_info.ipc_handles, list)
|
||||
weights = []
|
||||
@@ -260,7 +274,7 @@ class IPCWeightTransferEngine(
|
||||
weight = rebuild_cuda_tensor(*list_args)
|
||||
weights.append((name, weight))
|
||||
|
||||
load_weights(weights)
|
||||
self.model.load_weights(weights)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
pass
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared NCCL initialization helpers for weight transfer engines.
|
||||
|
||||
The dense (`NCCLWeightTransferEngine`) and sparse
|
||||
(`SparseNCCLWeightTransferEngine`) backends are independent engines that share
|
||||
*only* their process-group initialization. That common logic lives here so the
|
||||
sparse engine does not have to subclass the dense one.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
|
||||
from vllm.distributed.weight_transfer.base import WeightTransferInitInfo
|
||||
|
||||
|
||||
@dataclass
|
||||
class NCCLWeightTransferInitInfo(WeightTransferInitInfo):
|
||||
"""Initialization info for NCCL-based weight transfer backends."""
|
||||
|
||||
master_address: str
|
||||
master_port: int
|
||||
rank_offset: int
|
||||
world_size: int
|
||||
|
||||
|
||||
def stateless_init_process_group(
|
||||
master_address: str,
|
||||
master_port: int,
|
||||
rank: int,
|
||||
world_size: int,
|
||||
device,
|
||||
) -> "PyNcclCommunicator":
|
||||
"""
|
||||
vLLM provides `StatelessProcessGroup` to create a process group
|
||||
without considering the global process group in torch.distributed.
|
||||
It is recommended to create `StatelessProcessGroup`, and then initialize
|
||||
the data-plane communication (NCCL) between external (train processes)
|
||||
and vLLM workers.
|
||||
"""
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
|
||||
pg = StatelessProcessGroup.create(
|
||||
host=master_address, port=master_port, rank=rank, world_size=world_size
|
||||
)
|
||||
return PyNcclCommunicator(pg, device=device)
|
||||
|
||||
|
||||
def worker_init_process_group(
|
||||
init_info: NCCLWeightTransferInitInfo,
|
||||
parallel_config: "ParallelConfig",
|
||||
) -> "PyNcclCommunicator":
|
||||
"""Create the trainer<->worker NCCL group on an inference worker.
|
||||
|
||||
Computes a unique rank for this worker across all data-parallel groups and
|
||||
joins the stateless process group with the trainer.
|
||||
"""
|
||||
# Calculate the global rank in the trainer-worker process group.
|
||||
# Must account for data parallel to get unique ranks across all workers.
|
||||
dp_rank = parallel_config.data_parallel_index
|
||||
world_size_per_dp = parallel_config.world_size # TP * PP
|
||||
rank_within_dp = parallel_config.rank
|
||||
|
||||
# Unique rank across all DP groups
|
||||
worker_rank = dp_rank * world_size_per_dp + rank_within_dp
|
||||
rank = worker_rank + init_info.rank_offset
|
||||
|
||||
device = torch.accelerator.current_device_index()
|
||||
return stateless_init_process_group(
|
||||
init_info.master_address,
|
||||
init_info.master_port,
|
||||
rank,
|
||||
init_info.world_size,
|
||||
device=device,
|
||||
)
|
||||
|
||||
|
||||
def trainer_init(
|
||||
init_info: NCCLWeightTransferInitInfo | dict,
|
||||
) -> "PyNcclCommunicator":
|
||||
"""
|
||||
Initialize NCCL process group for trainer-side weight transfer.
|
||||
|
||||
The trainer is always rank 0 in the process group. Uses the current
|
||||
CUDA device (torch.accelerator.current_device_index()).
|
||||
|
||||
Args:
|
||||
init_info: Either an NCCLWeightTransferInitInfo object or a dict with keys:
|
||||
- master_address: str
|
||||
- master_port: int
|
||||
- world_size: int
|
||||
|
||||
Returns:
|
||||
PyNcclCommunicator for weight transfer.
|
||||
"""
|
||||
if isinstance(init_info, dict):
|
||||
master_address = init_info["master_address"]
|
||||
master_port = init_info["master_port"]
|
||||
world_size = init_info["world_size"]
|
||||
else:
|
||||
master_address = init_info.master_address
|
||||
master_port = init_info.master_port
|
||||
world_size = init_info.world_size
|
||||
|
||||
# Trainer is always rank 0
|
||||
device = torch.accelerator.current_device_index()
|
||||
return stateless_init_process_group(
|
||||
master_address,
|
||||
master_port,
|
||||
0,
|
||||
world_size,
|
||||
device,
|
||||
)
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""NCCL-based weight transfer engine."""
|
||||
"""NCCL-based (dense) weight transfer engine."""
|
||||
|
||||
from collections.abc import Callable, Iterator
|
||||
from dataclasses import dataclass
|
||||
@@ -9,31 +9,32 @@ from typing import TYPE_CHECKING, Any
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
SparseWeightPatch,
|
||||
WeightTransferEngine,
|
||||
WeightTransferInitInfo,
|
||||
WeightTransferUpdateInfo,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
NCCLWeightTransferInitInfo,
|
||||
trainer_init,
|
||||
worker_init_process_group,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.packed_tensor import (
|
||||
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
DEFAULT_PACKED_NUM_BUFFERS,
|
||||
packed_nccl_broadcast_consumer,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class NCCLWeightTransferInitInfo(WeightTransferInitInfo):
|
||||
"""Initialization info for NCCL weight transfer backend."""
|
||||
|
||||
master_address: str
|
||||
master_port: int
|
||||
rank_offset: int
|
||||
world_size: int
|
||||
# Re-exported for backward compatibility; canonical home is nccl_common.
|
||||
__all__ = [
|
||||
"NCCLWeightTransferInitInfo",
|
||||
"NCCLTrainerSendWeightsArgs",
|
||||
"NCCLWeightTransferUpdateInfo",
|
||||
"NCCLWeightTransferEngine",
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -82,7 +83,6 @@ class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
|
||||
def __post_init__(self):
|
||||
"""Validate that all lists have the same length."""
|
||||
super().__post_init__()
|
||||
num_params = len(self.names)
|
||||
if len(self.dtype_names) != num_params:
|
||||
raise ValueError(
|
||||
@@ -94,13 +94,6 @@ class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
f"`shapes` should be of the same size as `names`: "
|
||||
f"got {len(self.shapes)} and {len(self.names)}"
|
||||
)
|
||||
if self.update_kind == "dense":
|
||||
return
|
||||
|
||||
if self.packed:
|
||||
raise ValueError(
|
||||
"`update_kind='sparse_flat'` cannot be combined with `packed=True`"
|
||||
)
|
||||
|
||||
|
||||
class NCCLWeightTransferEngine(
|
||||
@@ -109,8 +102,10 @@ class NCCLWeightTransferEngine(
|
||||
"""
|
||||
Weight transfer engine using NCCL for communication between trainer and workers.
|
||||
|
||||
This implementation uses NCCL broadcast operations to transfer weights from
|
||||
the trainer (rank 0) to all inference workers in a process group.
|
||||
This implementation uses NCCL broadcast operations to transfer dense
|
||||
checkpoint-format weights from the trainer (rank 0) to all inference workers
|
||||
in a process group. Received weights are loaded via the model's
|
||||
`load_weights` using the layerwise reload lifecycle.
|
||||
"""
|
||||
|
||||
# Define backend-specific dataclass types
|
||||
@@ -120,18 +115,11 @@ class NCCLWeightTransferEngine(
|
||||
def __init__(
|
||||
self,
|
||||
config: WeightTransferConfig,
|
||||
parallel_config: ParallelConfig,
|
||||
vllm_config: "VllmConfig",
|
||||
device: torch.device,
|
||||
model: torch.nn.Module,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize the NCCL weight transfer engine.
|
||||
|
||||
Args:
|
||||
config: The configuration for the weight transfer engine
|
||||
parallel_config: The configuration for the parallel setup
|
||||
model: The local model instance which will receive the weights
|
||||
"""
|
||||
super().__init__(config, parallel_config, model)
|
||||
super().__init__(config, vllm_config, device, model)
|
||||
self.model_update_group: PyNcclCommunicator | None = None
|
||||
|
||||
def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None:
|
||||
@@ -142,35 +130,29 @@ class NCCLWeightTransferEngine(
|
||||
init_info: NCCL initialization info containing master address, port,
|
||||
rank offset, and world size
|
||||
"""
|
||||
|
||||
# Calculate the global rank in the trainer-worker process group
|
||||
# Must account for data parallel to get unique ranks across all workers
|
||||
dp_rank = self.parallel_config.data_parallel_index
|
||||
world_size_per_dp = self.parallel_config.world_size # TP * PP
|
||||
rank_within_dp = self.parallel_config.rank
|
||||
|
||||
# Unique rank across all DP groups
|
||||
worker_rank = dp_rank * world_size_per_dp + rank_within_dp
|
||||
rank = worker_rank + init_info.rank_offset
|
||||
# Create stateless process group
|
||||
device = torch.accelerator.current_device_index()
|
||||
self.model_update_group = (
|
||||
NCCLWeightTransferEngine._stateless_init_process_group(
|
||||
init_info.master_address,
|
||||
init_info.master_port,
|
||||
rank,
|
||||
init_info.world_size,
|
||||
device=device,
|
||||
)
|
||||
self.model_update_group = worker_init_process_group(
|
||||
init_info, self.parallel_config
|
||||
)
|
||||
|
||||
def receive_weights(
|
||||
self,
|
||||
update_info: NCCLWeightTransferUpdateInfo,
|
||||
load_weights: Callable[[list[tuple[str, torch.Tensor]]], None],
|
||||
) -> None:
|
||||
def start_weight_update(self) -> None:
|
||||
"""Initialize layerwise reloading for the incoming checkpoint weights."""
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
initialize_layerwise_reload,
|
||||
)
|
||||
|
||||
initialize_layerwise_reload(self.model)
|
||||
|
||||
def finish_weight_update(self) -> None:
|
||||
"""Finalize layerwise reloading after all weights have been received."""
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
finalize_layerwise_reload,
|
||||
)
|
||||
|
||||
finalize_layerwise_reload(self.model, self.model_config)
|
||||
|
||||
def receive_weights(self, update_info: NCCLWeightTransferUpdateInfo) -> None:
|
||||
"""
|
||||
Receive weights from trainer via NCCL broadcast and load them incrementally.
|
||||
Receive weights from trainer via NCCL broadcast.
|
||||
|
||||
If update_info.packed is True, uses packed tensor broadcasting for
|
||||
efficient transfer of multiple weights in batches. Otherwise, uses simple
|
||||
@@ -179,19 +161,12 @@ class NCCLWeightTransferEngine(
|
||||
Args:
|
||||
update_info: NCCL update info containing parameter names, dtypes, shapes,
|
||||
and packed flag
|
||||
load_weights: Callable that loads weights into the model. Called
|
||||
incrementally for each batch of weights to avoid OOM.
|
||||
"""
|
||||
if self.model_update_group is None:
|
||||
raise RuntimeError(
|
||||
"NCCL weight transfer not initialized. "
|
||||
"Call init_transfer_engine() first."
|
||||
)
|
||||
if update_info.update_kind != "dense":
|
||||
raise ValueError(
|
||||
"Sparse updates must use `receive_sparse_weights`, not "
|
||||
"`receive_weights`"
|
||||
)
|
||||
|
||||
if update_info.packed:
|
||||
# Build iterator of (name, (shape, dtype)) from update_info
|
||||
@@ -206,9 +181,10 @@ class NCCLWeightTransferEngine(
|
||||
iterator=state_dict_info_iterator(),
|
||||
group=self.model_update_group,
|
||||
src=0,
|
||||
post_unpack_func=load_weights,
|
||||
post_unpack_func=self.model.load_weights,
|
||||
buffer_size_bytes=update_info.packed_buffer_size_bytes,
|
||||
num_buffers=update_info.packed_num_buffers,
|
||||
device=self.device,
|
||||
)
|
||||
else:
|
||||
# Use simple one-by-one broadcasting
|
||||
@@ -216,49 +192,13 @@ class NCCLWeightTransferEngine(
|
||||
update_info.names, update_info.dtype_names, update_info.shapes
|
||||
):
|
||||
dtype = getattr(torch, dtype_name)
|
||||
weight = torch.empty(shape, dtype=dtype, device="cuda")
|
||||
weight = torch.empty(shape, dtype=dtype, device=self.device)
|
||||
self.model_update_group.broadcast(
|
||||
weight, src=0, stream=torch.cuda.current_stream()
|
||||
)
|
||||
load_weights([(name, weight)])
|
||||
self.model.load_weights([(name, weight)])
|
||||
del weight
|
||||
|
||||
def receive_sparse_weights(
|
||||
self,
|
||||
update_info: NCCLWeightTransferUpdateInfo,
|
||||
apply_patches: Callable[[list[SparseWeightPatch]], None],
|
||||
) -> None:
|
||||
"""Receive sparse flat-index patches from trainer via NCCL."""
|
||||
if self.model_update_group is None:
|
||||
raise RuntimeError(
|
||||
"NCCL weight transfer not initialized. "
|
||||
"Call init_transfer_engine() first."
|
||||
)
|
||||
if update_info.update_kind != "sparse_flat":
|
||||
raise ValueError("Sparse receive path requires `update_kind='sparse_flat'`")
|
||||
assert update_info.num_updates_list is not None
|
||||
|
||||
for name, dtype_name, num_updates in zip(
|
||||
update_info.names,
|
||||
update_info.dtype_names,
|
||||
update_info.num_updates_list,
|
||||
):
|
||||
dtype = getattr(torch, dtype_name)
|
||||
device = torch.accelerator.current_device_index()
|
||||
indices = torch.empty(num_updates, dtype=torch.int32, device=device)
|
||||
values = torch.empty(num_updates, dtype=dtype, device=device)
|
||||
self.model_update_group.broadcast(
|
||||
indices, src=0, stream=torch.cuda.current_stream()
|
||||
)
|
||||
self.model_update_group.broadcast(
|
||||
values, src=0, stream=torch.cuda.current_stream()
|
||||
)
|
||||
apply_patches(
|
||||
[SparseWeightPatch(name=name, indices=indices, values=values)]
|
||||
)
|
||||
del indices
|
||||
del values
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.model_update_group is not None:
|
||||
# Clean up the communicator by removing the reference
|
||||
@@ -269,7 +209,7 @@ class NCCLWeightTransferEngine(
|
||||
iterator: Iterator[tuple[str, torch.Tensor]],
|
||||
trainer_args: dict[str, Any] | NCCLTrainerSendWeightsArgs,
|
||||
) -> None:
|
||||
"""Broadcast weights from trainer to vLLM workers.
|
||||
"""Broadcast dense weights from trainer to vLLM workers.
|
||||
|
||||
Args:
|
||||
iterator: Iterator of model parameters. Returns (name, tensor) tuples
|
||||
@@ -322,94 +262,6 @@ class NCCLWeightTransferEngine(
|
||||
stream=args.stream or torch.cuda.current_stream(),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def trainer_send_sparse_weights(
|
||||
iterator: Iterator[SparseWeightPatch],
|
||||
trainer_args: dict[str, Any] | NCCLTrainerSendWeightsArgs,
|
||||
) -> None:
|
||||
"""Broadcast sparse flat-index patches from trainer to vLLM workers."""
|
||||
if isinstance(trainer_args, dict):
|
||||
args = NCCLTrainerSendWeightsArgs(**trainer_args)
|
||||
else:
|
||||
args = trainer_args
|
||||
|
||||
if args.packed:
|
||||
raise ValueError(
|
||||
"Sparse NCCL updates cannot be combined with `packed=True`"
|
||||
)
|
||||
|
||||
stream = args.stream or torch.cuda.current_stream()
|
||||
for patch in iterator:
|
||||
args.group.broadcast(patch.indices, src=args.src, stream=stream)
|
||||
args.group.broadcast(patch.values, src=args.src, stream=stream)
|
||||
|
||||
@staticmethod
|
||||
def trainer_init(
|
||||
init_info: NCCLWeightTransferInitInfo | dict,
|
||||
) -> "PyNcclCommunicator":
|
||||
"""
|
||||
Initialize NCCL process group for trainer-side weight transfer.
|
||||
|
||||
The trainer is always rank 0 in the process group. Uses the current
|
||||
CUDA device (torch.accelerator.current_device_index()).
|
||||
|
||||
Args:
|
||||
init_info: Either an NCCLWeightTransferInitInfo object or a dict with keys:
|
||||
- master_address: str
|
||||
- master_port: int
|
||||
- world_size: int
|
||||
|
||||
Returns:
|
||||
PyNcclCommunicator for weight transfer.
|
||||
|
||||
Example:
|
||||
>>> from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
... NCCLWeightTransferEngine,
|
||||
... )
|
||||
>>> group = NCCLWeightTransferEngine.trainer_init(
|
||||
... dict(
|
||||
... master_address=master_address,
|
||||
... master_port=master_port,
|
||||
... world_size=world_size,
|
||||
... ),
|
||||
... )
|
||||
"""
|
||||
if isinstance(init_info, dict):
|
||||
master_address = init_info["master_address"]
|
||||
master_port = init_info["master_port"]
|
||||
world_size = init_info["world_size"]
|
||||
else:
|
||||
# NCCLWeightTransferInitInfo object
|
||||
master_address = init_info.master_address
|
||||
master_port = init_info.master_port
|
||||
world_size = init_info.world_size
|
||||
|
||||
# Trainer is always rank 0
|
||||
device = torch.accelerator.current_device_index()
|
||||
return NCCLWeightTransferEngine._stateless_init_process_group(
|
||||
master_address,
|
||||
master_port,
|
||||
0,
|
||||
world_size,
|
||||
device,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _stateless_init_process_group(
|
||||
master_address, master_port, rank, world_size, device
|
||||
):
|
||||
"""
|
||||
vLLM provides `StatelessProcessGroup` to create a process group
|
||||
without considering the global process group in torch.distributed.
|
||||
It is recommended to create `StatelessProcessGroup`, and then initialize
|
||||
the data-plane communication (NCCL) between external (train processes)
|
||||
and vLLM workers.
|
||||
"""
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
|
||||
pg = StatelessProcessGroup.create(
|
||||
host=master_address, port=master_port, rank=rank, world_size=world_size
|
||||
)
|
||||
pynccl = PyNcclCommunicator(pg, device=device)
|
||||
return pynccl
|
||||
# Trainer-side process-group setup. Delegates to the shared helper so the
|
||||
# sparse engine can reuse the exact same rendezvous without subclassing.
|
||||
trainer_init = staticmethod(trainer_init)
|
||||
|
||||
@@ -186,6 +186,7 @@ def packed_nccl_broadcast_consumer(
|
||||
post_unpack_func: Callable[[list[tuple[str, torch.Tensor]]], None],
|
||||
buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS,
|
||||
device: torch.device | str = "cuda",
|
||||
) -> None:
|
||||
"""Consume packed tensors and unpack them into a list of tensors.
|
||||
|
||||
@@ -199,6 +200,8 @@ def packed_nccl_broadcast_consumer(
|
||||
Both producer and consumer must use the same value.
|
||||
num_buffers: Number of buffers for double/triple buffering.
|
||||
Both producer and consumer must use the same value.
|
||||
device: Device for the receive buffers. Must match the device the NCCL
|
||||
communicator was created on (the worker's assigned device).
|
||||
|
||||
"""
|
||||
target_packed_tensor_size = buffer_size_bytes
|
||||
@@ -211,7 +214,7 @@ def packed_nccl_broadcast_consumer(
|
||||
]
|
||||
packing_tensor_sizes: list[int] = [0 for _ in range(num_buffers)]
|
||||
packed_tensors: list[torch.Tensor] = [
|
||||
torch.empty(0, dtype=torch.uint8, device="cuda") for _ in range(num_buffers)
|
||||
torch.empty(0, dtype=torch.uint8, device=device) for _ in range(num_buffers)
|
||||
]
|
||||
|
||||
while True:
|
||||
@@ -234,7 +237,7 @@ def packed_nccl_broadcast_consumer(
|
||||
break
|
||||
# Create a packed tensor and broadcast it
|
||||
packed_tensors[buffer_idx] = torch.empty(
|
||||
packing_tensor_sizes[buffer_idx], dtype=torch.uint8, device="cuda"
|
||||
packing_tensor_sizes[buffer_idx], dtype=torch.uint8, device=device
|
||||
)
|
||||
group.broadcast(packed_tensors[buffer_idx], src=src)
|
||||
# Load the packed tensor into the model
|
||||
@@ -259,7 +262,7 @@ def packed_nccl_broadcast_consumer(
|
||||
packed_tensors[buffer_idx] = torch.empty(
|
||||
packing_tensor_sizes[buffer_idx],
|
||||
dtype=torch.uint8,
|
||||
device="cuda",
|
||||
device=device,
|
||||
)
|
||||
group.broadcast(packed_tensors[buffer_idx], src=src)
|
||||
# Load the packed tensor into the model
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Sparse NCCL weight transfer engine.
|
||||
|
||||
A standalone engine (not a subclass of `NCCLWeightTransferEngine`) for applying
|
||||
sparse, flat-index weight patches in place. It shares only NCCL process-group
|
||||
initialization with the dense engine (via `nccl_common`); the update path
|
||||
applies index/value patches directly to existing model parameters and never runs
|
||||
layerwise reload.
|
||||
|
||||
MVP limitations:
|
||||
* TP=1 and PP=1 only
|
||||
* uses runtime/kernel-format parameter names
|
||||
* not composable with checkpoint-format or packed updates
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
WeightTransferEngine,
|
||||
WeightTransferUpdateInfo,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
NCCLWeightTransferInitInfo,
|
||||
trainer_init,
|
||||
worker_init_process_group,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_engine import NCCLTrainerSendWeightsArgs
|
||||
|
||||
__all__ = [
|
||||
"SparseWeightPatch",
|
||||
"SparseNCCLWeightTransferUpdateInfo",
|
||||
"SparseNCCLWeightTransferEngine",
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class SparseWeightPatch:
|
||||
"""A sparse in-place patch for one existing parameter."""
|
||||
|
||||
name: str
|
||||
indices: torch.Tensor
|
||||
values: torch.Tensor
|
||||
|
||||
|
||||
@dataclass
|
||||
class SparseNCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
"""Update info for the sparse NCCL weight transfer backend."""
|
||||
|
||||
names: list[str]
|
||||
dtype_names: list[str]
|
||||
shapes: list[list[int]]
|
||||
num_updates_list: list[int]
|
||||
"""Number of sparse entries to receive for each parameter in ``names``."""
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
num_params = len(self.names)
|
||||
if len(self.dtype_names) != num_params:
|
||||
raise ValueError(
|
||||
f"`dtype_names` should be of the same size as `names`: "
|
||||
f"got {len(self.dtype_names)} and {len(self.names)}"
|
||||
)
|
||||
if len(self.shapes) != num_params:
|
||||
raise ValueError(
|
||||
f"`shapes` should be of the same size as `names`: "
|
||||
f"got {len(self.shapes)} and {len(self.names)}"
|
||||
)
|
||||
if len(self.num_updates_list) == 0:
|
||||
raise ValueError("`num_updates_list` cannot be empty for sparse updates")
|
||||
if len(self.num_updates_list) != num_params:
|
||||
raise ValueError(
|
||||
f"`num_updates_list` should be of the same size as `names`: "
|
||||
f"got {len(self.num_updates_list)} and {len(self.names)}"
|
||||
)
|
||||
if any(num_updates < 0 for num_updates in self.num_updates_list):
|
||||
raise ValueError("Sparse `num_updates_list` entries must be non-negative")
|
||||
|
||||
|
||||
class SparseNCCLWeightTransferEngine(
|
||||
WeightTransferEngine[NCCLWeightTransferInitInfo, SparseNCCLWeightTransferUpdateInfo]
|
||||
):
|
||||
"""
|
||||
Sparse weight transfer engine using NCCL.
|
||||
|
||||
Receives flat-index (indices, values) patches broadcast from the trainer
|
||||
(rank 0) and applies them in place to existing model parameters. Weights are
|
||||
applied directly without layerwise reload, so `start_weight_update` and
|
||||
`finish_weight_update` are no-ops.
|
||||
"""
|
||||
|
||||
init_info_cls = NCCLWeightTransferInitInfo
|
||||
update_info_cls = SparseNCCLWeightTransferUpdateInfo
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: WeightTransferConfig,
|
||||
vllm_config: "VllmConfig",
|
||||
device: torch.device,
|
||||
model: torch.nn.Module,
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, device, model)
|
||||
self.model_update_group: PyNcclCommunicator | None = None
|
||||
|
||||
def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None:
|
||||
"""Initialize the NCCL process group with the trainer."""
|
||||
self.model_update_group = worker_init_process_group(
|
||||
init_info, self.parallel_config
|
||||
)
|
||||
|
||||
def start_weight_update(self) -> None:
|
||||
"""No-op: sparse patches are applied in place, no layerwise reload."""
|
||||
if self.parallel_config.world_size != 1:
|
||||
raise NotImplementedError(
|
||||
"Sparse weight updates currently require TP=1 and PP=1"
|
||||
)
|
||||
|
||||
def finish_weight_update(self) -> None:
|
||||
"""No-op: sparse patches are applied in place, no layerwise reload."""
|
||||
pass
|
||||
|
||||
def receive_weights(self, update_info: SparseNCCLWeightTransferUpdateInfo) -> None:
|
||||
"""Receive sparse flat-index patches from the trainer and apply them."""
|
||||
if self.model_update_group is None:
|
||||
raise RuntimeError(
|
||||
"NCCL weight transfer not initialized. "
|
||||
"Call init_transfer_engine() first."
|
||||
)
|
||||
|
||||
# Use the worker's assigned device rather than the ambient current
|
||||
# device: the receive path is no longer wrapped in
|
||||
# `with torch.device(self.device)` by the caller, so the current device
|
||||
# is not guaranteed to match self.device. The recv buffers must live on
|
||||
# the same device as the NCCL communicator (created on self.device).
|
||||
device = self.device
|
||||
for name, dtype_name, num_updates in zip(
|
||||
update_info.names,
|
||||
update_info.dtype_names,
|
||||
update_info.num_updates_list,
|
||||
):
|
||||
dtype = getattr(torch, dtype_name)
|
||||
indices = torch.empty(num_updates, dtype=torch.int32, device=device)
|
||||
values = torch.empty(num_updates, dtype=dtype, device=device)
|
||||
self.model_update_group.broadcast(
|
||||
indices, src=0, stream=torch.cuda.current_stream()
|
||||
)
|
||||
self.model_update_group.broadcast(
|
||||
values, src=0, stream=torch.cuda.current_stream()
|
||||
)
|
||||
self._apply_patch(
|
||||
SparseWeightPatch(name=name, indices=indices, values=values)
|
||||
)
|
||||
del indices
|
||||
del values
|
||||
|
||||
def _apply_patch(self, patch: SparseWeightPatch) -> None:
|
||||
"""Apply a single sparse flat-index patch to an existing model param."""
|
||||
param = self.model.get_parameter(patch.name)
|
||||
if not param.data.is_contiguous():
|
||||
raise NotImplementedError(
|
||||
"Sparse weight updates currently require contiguous params: "
|
||||
f"{patch.name}"
|
||||
)
|
||||
if patch.indices.dtype != torch.int32:
|
||||
raise ValueError(
|
||||
f"Sparse weight updates currently require int32 indices: {patch.name}"
|
||||
)
|
||||
if patch.indices.ndim != 1 or patch.values.ndim != 1:
|
||||
raise ValueError(
|
||||
f"Sparse weight patches must be 1D flattened updates: {patch.name}"
|
||||
)
|
||||
if patch.indices.numel() != patch.values.numel():
|
||||
raise ValueError(
|
||||
f"`indices` and `values` must have matching lengths for {patch.name}"
|
||||
)
|
||||
if patch.values.dtype != param.dtype:
|
||||
raise ValueError(
|
||||
f"Sparse values dtype {patch.values.dtype} does not match "
|
||||
f"parameter dtype {param.dtype} for {patch.name}"
|
||||
)
|
||||
|
||||
flat_param = param.data.view(-1)
|
||||
flat_param.index_copy_(
|
||||
0,
|
||||
patch.indices.to(device=flat_param.device, dtype=torch.long),
|
||||
patch.values.to(device=flat_param.device),
|
||||
)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.model_update_group is not None:
|
||||
self.model_update_group = None
|
||||
|
||||
@staticmethod
|
||||
def trainer_send_weights(
|
||||
iterator: Iterator[SparseWeightPatch],
|
||||
trainer_args: dict[str, Any] | NCCLTrainerSendWeightsArgs,
|
||||
) -> None:
|
||||
"""Broadcast sparse flat-index patches from trainer to vLLM workers."""
|
||||
if isinstance(trainer_args, dict):
|
||||
args = NCCLTrainerSendWeightsArgs(**trainer_args)
|
||||
else:
|
||||
args = trainer_args
|
||||
|
||||
if args.packed:
|
||||
raise ValueError(
|
||||
"Sparse NCCL updates cannot be combined with `packed=True`"
|
||||
)
|
||||
|
||||
stream = args.stream or torch.cuda.current_stream()
|
||||
for patch in iterator:
|
||||
args.group.broadcast(patch.indices, src=args.src, stream=stream)
|
||||
args.group.broadcast(patch.values, src=args.src, stream=stream)
|
||||
|
||||
# Trainer-side process-group setup (shared with the dense engine).
|
||||
trainer_init = staticmethod(trainer_init)
|
||||
@@ -244,7 +244,7 @@ class EngineClient(ABC):
|
||||
"""Initialize weight transfer for RL training."""
|
||||
raise NotImplementedError
|
||||
|
||||
async def start_weight_update(self, is_checkpoint_format: bool = True) -> None:
|
||||
async def start_weight_update(self) -> None:
|
||||
"""Start a new weight update."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@@ -873,12 +873,9 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
"init_weight_transfer_engine", kwargs={"init_info": init_info_dict}
|
||||
)
|
||||
|
||||
def start_weight_update(self, is_checkpoint_format: bool = True) -> None:
|
||||
def start_weight_update(self) -> None:
|
||||
"""Start a new weight update."""
|
||||
self.llm_engine.collective_rpc(
|
||||
"start_weight_update",
|
||||
kwargs={"is_checkpoint_format": is_checkpoint_format},
|
||||
)
|
||||
self.llm_engine.collective_rpc("start_weight_update")
|
||||
|
||||
def update_weights(self, request: WeightTransferUpdateRequest | dict) -> None:
|
||||
"""
|
||||
|
||||
@@ -129,14 +129,7 @@ async def init_weight_transfer_engine(raw_request: Request):
|
||||
|
||||
@router.post("/start_weight_update")
|
||||
async def start_weight_update(raw_request: Request):
|
||||
try:
|
||||
body = await raw_request.json()
|
||||
except json.JSONDecodeError as e:
|
||||
raise HTTPException(status_code=400, detail="Invalid JSON format") from e # noqa: B904
|
||||
is_checkpoint_format = body.get("is_checkpoint_format", True)
|
||||
await engine_client(raw_request).start_weight_update(
|
||||
is_checkpoint_format=is_checkpoint_format
|
||||
)
|
||||
await engine_client(raw_request).start_weight_update()
|
||||
return JSONResponse(content={"message": "Weight update started"})
|
||||
|
||||
|
||||
|
||||
@@ -1080,12 +1080,9 @@ class AsyncLLM(EngineClient):
|
||||
"init_weight_transfer_engine", kwargs={"init_info": init_info_dict}
|
||||
)
|
||||
|
||||
async def start_weight_update(self, is_checkpoint_format: bool = True) -> None:
|
||||
async def start_weight_update(self) -> None:
|
||||
"""Start a new weight update."""
|
||||
await self.collective_rpc(
|
||||
"start_weight_update",
|
||||
kwargs={"is_checkpoint_format": is_checkpoint_format},
|
||||
)
|
||||
await self.collective_rpc("start_weight_update")
|
||||
|
||||
async def update_weights(self, request: WeightTransferUpdateRequest) -> None:
|
||||
"""
|
||||
|
||||
@@ -377,12 +377,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
GPUModelRunnerV1.reload_weights(self, *args, **kwargs) # type: ignore[arg-type]
|
||||
|
||||
def apply_sparse_weight_patches(self, *args, **kwargs) -> None:
|
||||
# TODO: Use full version instead of import when fully migrated to v2
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner as GPUModelRunnerV1
|
||||
|
||||
GPUModelRunnerV1.apply_sparse_weight_patches(self, *args, **kwargs) # type: ignore[arg-type]
|
||||
|
||||
def update_config(self, *args, **kwargs) -> None:
|
||||
# TODO(Wentao): Use full version instead of import when fully migrated to v2
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner as GPUModelRunnerV1
|
||||
|
||||
@@ -49,7 +49,6 @@ from vllm.distributed.parallel_state import (
|
||||
is_global_first_rank,
|
||||
prepare_communication_buffer_for_model,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.base import SparseWeightPatch
|
||||
from vllm.forward_context import (
|
||||
BatchDescriptor,
|
||||
set_forward_context,
|
||||
@@ -3244,44 +3243,6 @@ class GPUModelRunner(
|
||||
return self.model.unwrap()
|
||||
return self.model
|
||||
|
||||
def apply_sparse_weight_patches(self, patches: Iterable[SparseWeightPatch]) -> None:
|
||||
"""Apply sparse flat-index patches directly to existing model params."""
|
||||
model = self.get_model()
|
||||
for patch in patches:
|
||||
param = model.get_parameter(patch.name)
|
||||
if not param.data.is_contiguous():
|
||||
raise NotImplementedError(
|
||||
"Sparse weight updates currently require contiguous params: "
|
||||
f"{patch.name}"
|
||||
)
|
||||
|
||||
if patch.indices.dtype != torch.int32:
|
||||
raise ValueError(
|
||||
"Sparse weight updates currently require int32 indices: "
|
||||
f"{patch.name}"
|
||||
)
|
||||
if patch.indices.ndim != 1 or patch.values.ndim != 1:
|
||||
raise ValueError(
|
||||
f"Sparse weight patches must be 1D flattened updates: {patch.name}"
|
||||
)
|
||||
if patch.indices.numel() != patch.values.numel():
|
||||
raise ValueError(
|
||||
"`indices` and `values` must have matching lengths for "
|
||||
f"{patch.name}"
|
||||
)
|
||||
if patch.values.dtype != param.dtype:
|
||||
raise ValueError(
|
||||
f"Sparse values dtype {patch.values.dtype} does not match "
|
||||
f"parameter dtype {param.dtype} for {patch.name}"
|
||||
)
|
||||
|
||||
flat_param = param.data.view(-1)
|
||||
flat_param.index_copy_(
|
||||
0,
|
||||
patch.indices.to(device=flat_param.device, dtype=torch.long),
|
||||
patch.values.to(device=flat_param.device),
|
||||
)
|
||||
|
||||
def get_supported_generation_tasks(self) -> list[GenerationTask]:
|
||||
model = self.get_model()
|
||||
supported_tasks = list[GenerationTask]()
|
||||
|
||||
@@ -155,7 +155,6 @@ class Worker(WorkerBase):
|
||||
# is available, since the engine needs a reference to the model.
|
||||
self.weight_transfer_engine: WeightTransferEngine | None = None
|
||||
self._weight_update_active = False
|
||||
self._is_checkpoint_format = True
|
||||
|
||||
# Torch/CUDA profiler. Enabled and configured through profiler_config.
|
||||
# Profiler wrapper is created lazily in profile() when start is called,
|
||||
@@ -416,7 +415,8 @@ class Worker(WorkerBase):
|
||||
if self.vllm_config.weight_transfer_config is not None:
|
||||
self.weight_transfer_engine = WeightTransferEngineFactory.create_engine(
|
||||
self.vllm_config.weight_transfer_config,
|
||||
self.vllm_config.parallel_config,
|
||||
self.vllm_config,
|
||||
self.device,
|
||||
self.model_runner.get_model(),
|
||||
)
|
||||
|
||||
@@ -1161,16 +1161,16 @@ class Worker(WorkerBase):
|
||||
typed_init_info = self.weight_transfer_engine.parse_init_info(init_info)
|
||||
self.weight_transfer_engine.init_transfer_engine(typed_init_info)
|
||||
|
||||
def start_weight_update(self, is_checkpoint_format: bool = True) -> None:
|
||||
def start_weight_update(self) -> None:
|
||||
"""
|
||||
Start a new weight update session.
|
||||
|
||||
Args:
|
||||
is_checkpoint_format: Whether incoming weights are in checkpoint
|
||||
format (need layerwise processing) or kernel format (direct
|
||||
copy / sparse patch application).
|
||||
Delegates engine-specific preparation (e.g. layerwise reload setup) to
|
||||
the configured weight transfer engine. The worker only tracks that a
|
||||
session is active.
|
||||
"""
|
||||
self._check_weight_transfer_engine()
|
||||
assert self.weight_transfer_engine is not None
|
||||
|
||||
if self._weight_update_active:
|
||||
raise RuntimeError(
|
||||
@@ -1178,16 +1178,7 @@ class Worker(WorkerBase):
|
||||
"active. Call finish_weight_update first."
|
||||
)
|
||||
|
||||
if is_checkpoint_format:
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
initialize_layerwise_reload,
|
||||
)
|
||||
|
||||
model = self.model_runner.model
|
||||
with torch.device(self.device):
|
||||
initialize_layerwise_reload(model)
|
||||
|
||||
self._is_checkpoint_format = is_checkpoint_format
|
||||
self.weight_transfer_engine.start_weight_update()
|
||||
self._weight_update_active = True
|
||||
|
||||
def update_weights(self, update_info: dict) -> None:
|
||||
@@ -1208,82 +1199,24 @@ class Worker(WorkerBase):
|
||||
"start_weight_update must be called before update_weights."
|
||||
)
|
||||
|
||||
update_succeeded = False
|
||||
try:
|
||||
# Parse dict into backend-specific typed dataclass
|
||||
typed_update_info = self.weight_transfer_engine.parse_update_info(
|
||||
update_info
|
||||
)
|
||||
|
||||
with torch.device(self.device):
|
||||
if self._is_checkpoint_format:
|
||||
if typed_update_info.update_kind != "dense":
|
||||
raise ValueError(
|
||||
"Sparse weight updates require "
|
||||
"`start_weight_update(is_checkpoint_format=False)`."
|
||||
)
|
||||
|
||||
model = self.model_runner.model
|
||||
|
||||
# Use layerwise reload pattern for checkpoint format weights
|
||||
self.weight_transfer_engine.receive_weights(
|
||||
typed_update_info,
|
||||
load_weights=model.load_weights,
|
||||
)
|
||||
elif typed_update_info.update_kind == "sparse_flat":
|
||||
if self.parallel_config.world_size != 1:
|
||||
raise NotImplementedError(
|
||||
"Sparse weight updates currently require TP=1 and PP=1"
|
||||
)
|
||||
self.weight_transfer_engine.receive_sparse_weights(
|
||||
typed_update_info,
|
||||
apply_patches=self.model_runner.apply_sparse_weight_patches,
|
||||
)
|
||||
else:
|
||||
model = self.model_runner.model
|
||||
|
||||
# Weights are already in kernel format, copy directly.
|
||||
def load_weights_direct(
|
||||
weights: list[tuple[str, torch.Tensor]],
|
||||
) -> None:
|
||||
for name, weight in weights:
|
||||
param = model.get_parameter(name)
|
||||
param.copy_(weight)
|
||||
|
||||
self.weight_transfer_engine.receive_weights(
|
||||
typed_update_info,
|
||||
load_weights=load_weights_direct,
|
||||
)
|
||||
|
||||
# NCCL broadcast/packed path are asynchronous.
|
||||
# Sync here so the next step uses the new weights.
|
||||
torch.accelerator.synchronize()
|
||||
update_succeeded = True
|
||||
finally:
|
||||
if not update_succeeded:
|
||||
self._weight_update_active = False
|
||||
self._is_checkpoint_format = True
|
||||
self.weight_transfer_engine.update_weights(update_info)
|
||||
except BaseException:
|
||||
self._weight_update_active = False
|
||||
raise
|
||||
|
||||
def finish_weight_update(self) -> None:
|
||||
"""Finish the current weight update session."""
|
||||
self._check_weight_transfer_engine()
|
||||
assert self.weight_transfer_engine is not None
|
||||
|
||||
if not self._weight_update_active:
|
||||
raise RuntimeError(
|
||||
"finish_weight_update called without a matching start_weight_update."
|
||||
)
|
||||
|
||||
if self._is_checkpoint_format:
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
finalize_layerwise_reload,
|
||||
)
|
||||
|
||||
model = self.model_runner.model
|
||||
with torch.device(self.device):
|
||||
finalize_layerwise_reload(model, self.model_config)
|
||||
|
||||
self.weight_transfer_engine.finish_weight_update()
|
||||
self._weight_update_active = False
|
||||
self._is_checkpoint_format = True
|
||||
|
||||
def shutdown(self) -> None:
|
||||
gc.unfreeze()
|
||||
|
||||
Reference in New Issue
Block a user