diff --git a/docs/training/layerwise.md b/docs/training/layerwise.md index 9e7d187710..5072fdb6df 100644 --- a/docs/training/layerwise.md +++ b/docs/training/layerwise.md @@ -58,7 +58,7 @@ class Fp8PerTensorOnlineLinearMethod(LinearMethodBase): ### High Level Weight Transfer API -The layerwise reloading system is integrated with the post-training weight transfer system. To use layerwise reloading in conjunction to the weight transfer system, follow the examples found [here](../../examples/rl/). Layerwise reloading is controlled by the `WeightTransferUpdateInfo.is_checkpoint_format` flag and is set to `True` by default. +The layerwise reloading system is integrated with the post-training weight transfer system. To use layerwise reloading in conjunction to the weight transfer system, follow the examples found [here](../../examples/rl/). Checkpoint-format weight transfer engines (e.g. the NCCL and IPC backends) run layerwise reloading automatically inside their `start_weight_update`/`finish_weight_update` lifecycle. ### Mid Level `reload_weights` API diff --git a/docs/training/weight_transfer/README.md b/docs/training/weight_transfer/README.md index 4368483e8b..7579e5fd4d 100644 --- a/docs/training/weight_transfer/README.md +++ b/docs/training/weight_transfer/README.md @@ -17,6 +17,7 @@ The weight transfer system follows a **four-phase protocol** with a pluggable ba | ------- | --------- | -------- | | [NCCL](nccl.md) | NCCL broadcast | Separate GPUs for training and inference | | [IPC](ipc.md) | CUDA IPC handles | Colocated training and inference on same GPU | +| [sparse_nccl](nccl.md#sparse-nccl) | NCCL broadcast | Sparse flat-index weight patches (TP=1/PP=1) | ## Configuration @@ -41,7 +42,7 @@ vllm serve my-model \ --weight-transfer-config '{"backend": "nccl"}' ``` -The `backend` field accepts `"nccl"` (default) or `"ipc"`. +The `backend` field accepts `"nccl"` (default), `"ipc"`, or `"sparse_nccl"`. ## API Endpoints @@ -69,7 +70,7 @@ Both backends provide static methods that the trainer calls to send weights. The EngineClass.trainer_init(init_info) # 2. Start weight update on inference side -llm.start_weight_update(is_checkpoint_format=True) +llm.start_weight_update() # 3. Send weights to inference workers EngineClass.trainer_send_weights( diff --git a/docs/training/weight_transfer/base.md b/docs/training/weight_transfer/base.md index ace228b009..0208264966 100644 --- a/docs/training/weight_transfer/base.md +++ b/docs/training/weight_transfer/base.md @@ -11,15 +11,23 @@ The `WeightTransferEngine` is a generic abstract class parameterized by two data ### Abstract Methods -Subclasses must implement these four methods: +Subclasses must implement these methods: | Method | Side | Description | | ------ | ---- | ----------- | | `init_transfer_engine(init_info)` | Inference | Initialize the communication channel on each inference worker | -| `receive_weights(update_info, load_weights)` | Inference | Receive weights and call `load_weights` incrementally | +| `start_weight_update()` | Inference | Prepare for an update (e.g. begin layerwise reload); no-op for in-place engines | +| `finish_weight_update()` | Inference | Finalize the update (e.g. finalize layerwise reload); no-op for in-place engines | +| `receive_weights(update_info)` | Inference | Receive weights and load them into `self.model` | | `shutdown()` | Inference | Clean up resources | | `trainer_send_weights(iterator, trainer_args)` | Trainer | Static method to send weights from the trainer process | +The base class provides two methods: + +1. `__init__` : Engines receive `config` (`WeightTransferConfig`), `vllm_config` (`VllmConfig`), `device` (`torch.device`) and `model` (`nn.Module`) +2. `update_weights(update_info_dict)`: Thin wrapper for `receive_weights`: parses +the dict into user-specified data type, calls `receive_weights`, and synchronizes the device. Subclasses implement `receive_weights`. + ### Request Classes The API-level request classes provide backend-agnostic serialization using plain dictionaries. The engine's `parse_init_info` and `parse_update_info` methods convert these dictionaries into typed dataclasses. @@ -81,7 +89,7 @@ class MyUpdateInfo(WeightTransferUpdateInfo): ### 2. Implement the Engine ```python -from collections.abc import Callable, Iterator +from collections.abc import Iterator from typing import Any import torch @@ -93,18 +101,25 @@ class MyWeightTransferEngine(WeightTransferEngine[MyInitInfo, MyUpdateInfo]): # Set up connection to trainer using init_info.endpoint, etc. ... - def receive_weights( - self, - update_info: MyUpdateInfo, - load_weights: Callable[[list[tuple[str, torch.Tensor]]], None], - ) -> None: - # Receive each weight and call load_weights incrementally + def start_weight_update(self) -> None: + # Checkpoint-format engines: run initialize_layerwise_reload(self.model). + # In-place engines: no-op + ... + + def finish_weight_update(self) -> None: + # Checkpoint-format engines: run finalize_layerwise_reload(...). + # In-place engines: no-op + ... + + def receive_weights(self, update_info: MyUpdateInfo) -> None: + weights = [] for name, dtype_name, shape in zip( update_info.names, update_info.dtype_names, update_info.shapes ): dtype = getattr(torch, dtype_name) weight = self._fetch_weight(name, shape, dtype) - load_weights([(name, weight)]) + weights.append((name, weight)) + self.model.load_weights(weights) def shutdown(self) -> None: # Clean up resources @@ -121,9 +136,6 @@ class MyWeightTransferEngine(WeightTransferEngine[MyInitInfo, MyUpdateInfo]): ... ``` -!!! important - The `load_weights` callable passed to `receive_weights` should be called **incrementally** (one or a few weights at a time) rather than accumulating all weights first. This avoids GPU out-of-memory errors with large models. - ### 3. Register with the Factory ```python @@ -147,7 +159,7 @@ Once registered, users can select your backend via `WeightTransferConfig(backend ## WeightTransferEngineFactory -The factory uses a registry pattern with lazy loading. Built-in engines (`nccl` and `ipc`) are registered at import time but their modules are only loaded when the backend is actually requested. This avoids importing heavy dependencies (like NCCL communicators) when they aren't needed. +The factory uses a registry pattern with lazy loading. Built-in engines (`nccl`, `ipc`, and `sparse_nccl`) are registered at import time but their modules are only loaded when the backend is actually requested. This avoids importing heavy dependencies (like NCCL communicators) when they aren't needed. ```python from vllm.distributed.weight_transfer.factory import WeightTransferEngineFactory @@ -155,7 +167,8 @@ from vllm.distributed.weight_transfer.factory import WeightTransferEngineFactory # Create an engine from config engine = WeightTransferEngineFactory.create_engine( config=weight_transfer_config, - parallel_config=parallel_config, + vllm_config=vllm_config, + device=device, model=model, ) ``` diff --git a/docs/training/weight_transfer/ipc.md b/docs/training/weight_transfer/ipc.md index 21fc8ad70d..f76272d2cb 100644 --- a/docs/training/weight_transfer/ipc.md +++ b/docs/training/weight_transfer/ipc.md @@ -55,7 +55,7 @@ trainer_args = IPCTrainerSendWeightsArgs( llm_handle=llm_actor_handle, ) # start -ray.get(llm_actor_handle.start_weight_update.remote(is_checkpoint_format=True)) +ray.get(llm_actor_handle.start_weight_update.remote()) # send weights IPCWeightTransferEngine.trainer_send_weights( iterator=model.named_parameters(), @@ -80,7 +80,7 @@ trainer_args = IPCTrainerSendWeightsArgs( # start base_url = "http://localhost:8000" url = f"{base_url}/start_weight_update" -response = requests.post(url, json={"is_checkpoint_format": True}, timeout=60) +response = requests.post(url, json={}, timeout=60) response.raise_for_status() # send weights IPCWeightTransferEngine.trainer_send_weights( diff --git a/docs/training/weight_transfer/nccl.md b/docs/training/weight_transfer/nccl.md index 7b53121856..481b7c5f28 100644 --- a/docs/training/weight_transfer/nccl.md +++ b/docs/training/weight_transfer/nccl.md @@ -11,7 +11,7 @@ The NCCL weight transfer engine uses [NCCL](https://developer.nvidia.com/nccl) b ## How It Works 1. The trainer and all inference workers join a shared NCCL process group using `StatelessProcessGroup` (vLLM's torch.distributed-independent group abstraction). -2. The trainer broadcasts weights to all workers simultaneously. Each worker receives and loads weights incrementally. +2. The trainer broadcasts weights to all workers simultaneously. Each worker receives and loads the weights. 3. Optionally, **packed tensor broadcasting** batches multiple small tensors into larger buffers with double/triple buffering and CUDA stream overlap for higher throughput. This implementation is based on [NeMo-RL's packed tensor](https://github.com/NVIDIA-NeMo/RL/blob/main/nemo_rl/utils/packed_tensor.py). ## Initialization @@ -93,7 +93,7 @@ remaining three steps are: from vllm.distributed.weight_transfer.base import WeightTransferUpdateRequest # 1. Start the weight update -llm.start_weight_update(is_checkpoint_format=True) +llm.start_weight_update() # 2. Receive weights (can be called multiple times for chunked transfers) llm.update_weights( @@ -116,19 +116,22 @@ must match the order in which the trainer iterates over its parameters. `start_weight_update` must be called before `update_weights`, and `finish_weight_update` must be called after all weight chunks have been -transferred. The `is_checkpoint_format` flag controls whether layerwise reload -processing is applied (`True` for checkpoint-format weights, `False` for -pre-processed kernel-format weights). +transferred. The NCCL engine receives checkpoint-format weights and applies +layerwise reload processing automatically inside `start_weight_update` / +`finish_weight_update`. -Sparse NCCL patches still use `update_kind="sparse_flat"` inside -`update_info`, but they should be wrapped in -`start_weight_update(is_checkpoint_format=False)` because sparse patches apply -directly to runtime/kernel-format parameters. The current sparse MVP requires -`TP=1` and `PP=1`. +## Sparse NCCL + +Sparse, flat-index weight patches use a separate backend, +`WeightTransferConfig(backend="sparse_nccl")`, implemented by +`SparseNCCLWeightTransferEngine`. It shares only NCCL process-group +initialization with the dense engine; patches are applied directly in place to +existing parameters (no layerwise reload). The current sparse MVP requires +`TP=1` and `PP=1`. See the example below. ## Examples - [RLHF with NCCL weight syncing (offline, Ray)](../../../examples/rl/rlhf_nccl.py) - Trainer on one GPU, 2x tensor-parallel vLLM engine on two others, with packed NCCL weight broadcast -- [RLHF with sparse NCCL weight syncing (offline, Ray)](../../../examples/rl/rlhf_sparse_nccl.py) - Dense-vs-sparse equivalence demo with a real model on a 2-GPU trainer/inference setup; sparse patches use `start_weight_update(is_checkpoint_format=False)` and currently require `TP=1` and `PP=1` +- [RLHF with sparse NCCL weight syncing (offline, Ray)](../../../examples/rl/rlhf_sparse_nccl.py) - Dense-vs-sparse equivalence demo with a real model on a 2-GPU trainer/inference setup; sparse patches use `backend="sparse_nccl"` and currently require `TP=1` and `PP=1` - [RLHF with async weight syncing (offline, Ray)](../../../examples/rl/rlhf_async_new_apis.py) - Async generation with mid-flight pause, weight sync, resume, and validation against a fresh model - [RLHF with NCCL weight syncing (online serving, HTTP)](../../../examples/rl/rlhf_http_nccl.py) - Weight transfer with a running vLLM HTTP server using HTTP control plane and NCCL data plane diff --git a/examples/rl/rlhf_async_new_apis.py b/examples/rl/rlhf_async_new_apis.py index 9c3f4700d9..7043182ab1 100644 --- a/examples/rl/rlhf_async_new_apis.py +++ b/examples/rl/rlhf_async_new_apis.py @@ -306,7 +306,7 @@ gen_futures = [ ray.get(llm.pause_after_n_tokens.remote()) -ray.get(llm.start_weight_update.remote(is_checkpoint_format=True)) +ray.get(llm.start_weight_update.remote()) inference_handle = llm.update_weights.remote( WeightTransferUpdateRequest( diff --git a/examples/rl/rlhf_http_ipc.py b/examples/rl/rlhf_http_ipc.py index 16c5be8dd6..0a0efcbee3 100644 --- a/examples/rl/rlhf_http_ipc.py +++ b/examples/rl/rlhf_http_ipc.py @@ -80,14 +80,10 @@ def init_weight_transfer_engine(base_url: str) -> None: response.raise_for_status() -def start_weight_update( - base_url: str, - is_checkpoint_format: bool = True, -) -> None: +def start_weight_update(base_url: str) -> None: """Start a weight update via HTTP endpoint.""" url = f"{base_url}/start_weight_update" - payload = {"is_checkpoint_format": is_checkpoint_format} - response = requests.post(url, json=payload, timeout=60) + response = requests.post(url, json={}, timeout=60) response.raise_for_status() @@ -170,7 +166,7 @@ def main(): pause_generation(BASE_URL) # Start weight update, broadcast via IPC, then finish - start_weight_update(BASE_URL, is_checkpoint_format=False) + start_weight_update(BASE_URL) print("Broadcasting weights via CUDA IPC (HTTP)...") trainer_args = IPCTrainerSendWeightsArgs(send_mode="http", url=BASE_URL) diff --git a/examples/rl/rlhf_http_nccl.py b/examples/rl/rlhf_http_nccl.py index 01aafe43f1..b40b8de32f 100644 --- a/examples/rl/rlhf_http_nccl.py +++ b/examples/rl/rlhf_http_nccl.py @@ -83,14 +83,10 @@ def init_weight_transfer_engine( response.raise_for_status() -def start_weight_update( - base_url: str, - is_checkpoint_format: bool = True, -) -> None: +def start_weight_update(base_url: str) -> None: """Start a weight update via HTTP endpoint.""" url = f"{base_url}/start_weight_update" - payload = {"is_checkpoint_format": is_checkpoint_format} - response = requests.post(url, json=payload, timeout=60) + response = requests.post(url, json={}, timeout=60) response.raise_for_status() @@ -223,7 +219,7 @@ def main(): shapes.append(list(p.shape)) # Start weight update - start_weight_update(BASE_URL, is_checkpoint_format=True) + start_weight_update(BASE_URL) # Start the update_weights call in a separate thread since it will block # waiting for NCCL broadcasts diff --git a/examples/rl/rlhf_ipc.py b/examples/rl/rlhf_ipc.py index afebbd240a..cb85428987 100644 --- a/examples/rl/rlhf_ipc.py +++ b/examples/rl/rlhf_ipc.py @@ -139,7 +139,7 @@ ray.get(llm.sleep.remote(level=0)) ray.get(train_model.init_weight_transfer.remote()) # Start weight update, sync weights, then finish -ray.get(llm.start_weight_update.remote(is_checkpoint_format=True)) +ray.get(llm.start_weight_update.remote()) ray.get(train_model.broadcast_weights.remote(llm)) ray.get(llm.finish_weight_update.remote()) diff --git a/examples/rl/rlhf_ipc_fsdp_ep.py b/examples/rl/rlhf_ipc_fsdp_ep.py index 0fb0a93ca8..77ac6b4cfc 100644 --- a/examples/rl/rlhf_ipc_fsdp_ep.py +++ b/examples/rl/rlhf_ipc_fsdp_ep.py @@ -277,15 +277,8 @@ class DataParallelInferenceEngine: ] ) - def start_weight_update(self, is_checkpoint_format: bool = True): - ray.get( - [ - actor.start_weight_update.remote( - is_checkpoint_format=is_checkpoint_format - ) - for actor in self.llm_actors - ] - ) + def start_weight_update(self): + ray.get([actor.start_weight_update.remote() for actor in self.llm_actors]) def finish_weight_update(self): ray.get([actor.finish_weight_update.remote() for actor in self.llm_actors]) @@ -392,7 +385,7 @@ def main(): ray.get(inference_engine.wake_up.remote(tags=["weights"])) print("[sync] Starting weight update...") - ray.get(inference_engine.start_weight_update.remote(is_checkpoint_format=True)) + ray.get(inference_engine.start_weight_update.remote()) print("[sync] Packed IPC transfer FSDP → vLLM...") ray.get( diff --git a/examples/rl/rlhf_nccl.py b/examples/rl/rlhf_nccl.py index a9e39aaa72..bebd6bc70d 100644 --- a/examples/rl/rlhf_nccl.py +++ b/examples/rl/rlhf_nccl.py @@ -202,7 +202,7 @@ ray.get([train_handle, inference_handle]) names, dtype_names, shapes = ray.get(train_model.get_weight_metadata.remote()) # Start weight update -ray.get(llm.start_weight_update.remote(is_checkpoint_format=True)) +ray.get(llm.start_weight_update.remote()) # Issue update_weights call with NCCL-specific update info # packed=True enables efficient batched tensor broadcasting diff --git a/examples/rl/rlhf_nccl_fsdp_ep.py b/examples/rl/rlhf_nccl_fsdp_ep.py index 4337e6fea5..860db058ca 100644 --- a/examples/rl/rlhf_nccl_fsdp_ep.py +++ b/examples/rl/rlhf_nccl_fsdp_ep.py @@ -299,7 +299,7 @@ async def main(): print(f"[sync] Got metadata for {len(names)} parameters.") print("[sync] Starting weight update...") - await engine.start_weight_update(is_checkpoint_format=True) + await engine.start_weight_update() print("[sync] Broadcasting weights from FSDP → vLLM...") broadcast_handles = [ diff --git a/examples/rl/rlhf_sparse_nccl.py b/examples/rl/rlhf_sparse_nccl.py index bddd28b648..09cf5bfbaa 100644 --- a/examples/rl/rlhf_sparse_nccl.py +++ b/examples/rl/rlhf_sparse_nccl.py @@ -44,11 +44,14 @@ from transformers import AutoModelForCausalLM, AutoTokenizer from vllm import LLM, SamplingParams from vllm.config import WeightTransferConfig -from vllm.distributed.weight_transfer.base import SparseWeightPatch from vllm.distributed.weight_transfer.nccl_engine import ( NCCLTrainerSendWeightsArgs, NCCLWeightTransferEngine, ) +from vllm.distributed.weight_transfer.sparse_nccl_engine import ( + SparseNCCLWeightTransferEngine, + SparseWeightPatch, +) from vllm.utils.network_utils import get_ip, get_open_port MODEL_NAME = "Qwen/Qwen2.5-0.5B-Instruct" @@ -244,7 +247,6 @@ class TrainModel: dtype_names=[str(self.patched_param.dtype).split(".")[-1]], shapes=[list(self.patched_param.shape)], num_updates_list=[flat_indices.numel()], - update_kind="sparse_flat", ) return update_info, selected_token_ids, patch_digest, sparse_payload_bytes @@ -271,7 +273,7 @@ class TrainModel: raise RuntimeError("Sparse patch has not been prepared") start = time.perf_counter() - NCCLWeightTransferEngine.trainer_send_sparse_weights( + SparseNCCLWeightTransferEngine.trainer_send_weights( iter(self.pending_sparse_patches), NCCLTrainerSendWeightsArgs(group=self.model_update_group), ) @@ -282,6 +284,7 @@ class TrainModel: def launch_llm( scheduling_inference: PlacementGroupSchedulingStrategy, + backend: str = "nccl", ): return ray.remote( num_cpus=0, @@ -293,7 +296,7 @@ def launch_llm( tensor_parallel_size=1, distributed_executor_backend="ray", gpu_memory_utilization=0.7, - weight_transfer_config=WeightTransferConfig(backend="nccl"), + weight_transfer_config=WeightTransferConfig(backend=backend), ) @@ -332,7 +335,7 @@ def run_dense_phase( scheduling_inference: PlacementGroupSchedulingStrategy, ) -> dict[str, object]: ray.get(train_model.reset_model.remote()) - llm = launch_llm(scheduling_inference) + llm = launch_llm(scheduling_inference, backend="nccl") try: dense_before = collect_vllm_generations(llm) @@ -351,7 +354,7 @@ def run_dense_phase( ) trainer_init = train_model.init_weight_transfer_group.remote(world_size) ray.get([trainer_init, inference_init]) - ray.get(llm.start_weight_update.remote(is_checkpoint_format=True)) + ray.get(llm.start_weight_update.remote()) dense_update_info, dense_payload_bytes = ray.get( train_model.get_dense_update_info.remote() @@ -391,7 +394,7 @@ def run_sparse_phase( scheduling_inference: PlacementGroupSchedulingStrategy, ) -> dict[str, object]: ray.get(train_model.reset_model.remote()) - llm = launch_llm(scheduling_inference) + llm = launch_llm(scheduling_inference, backend="sparse_nccl") try: sparse_before = collect_vllm_generations(llm) @@ -410,7 +413,7 @@ def run_sparse_phase( ) trainer_init = train_model.init_weight_transfer_group.remote(world_size) ray.get([trainer_init, inference_init]) - ray.get(llm.start_weight_update.remote(is_checkpoint_format=False)) + ray.get(llm.start_weight_update.remote()) sparse_update_info, selected_token_ids, patch_digest, sparse_payload_bytes = ( ray.get(train_model.prepare_sparse_patch.remote(PROMPTS)) diff --git a/tests/distributed/test_weight_transfer.py b/tests/distributed/test_weight_transfer.py index ec56d24a40..715a8069d7 100644 --- a/tests/distributed/test_weight_transfer.py +++ b/tests/distributed/test_weight_transfer.py @@ -18,7 +18,6 @@ from torch.multiprocessing.reductions import reduce_tensor from vllm.config.parallel import ParallelConfig from vllm.config.weight_transfer import WeightTransferConfig from vllm.distributed.weight_transfer import WeightTransferEngineFactory -from vllm.distributed.weight_transfer.base import SparseWeightPatch from vllm.distributed.weight_transfer.ipc_engine import ( IPCWeightTransferEngine, IPCWeightTransferInitInfo, @@ -29,6 +28,11 @@ from vllm.distributed.weight_transfer.nccl_engine import ( NCCLWeightTransferInitInfo, NCCLWeightTransferUpdateInfo, ) +from vllm.distributed.weight_transfer.sparse_nccl_engine import ( + SparseNCCLWeightTransferEngine, + SparseNCCLWeightTransferUpdateInfo, + SparseWeightPatch, +) from vllm.platforms import current_platform from vllm.utils.network_utils import get_open_port @@ -80,6 +84,18 @@ def create_mock_parallel_config( return config +def create_mock_vllm_config( + rank: int = 0, + world_size: int = 1, + dp_rank: int = 0, +) -> MagicMock: + """Create a mock VllmConfig exposing parallel_config and model_config.""" + vllm_config = MagicMock() + vllm_config.parallel_config = create_mock_parallel_config(rank, world_size, dp_rank) + vllm_config.model_config = MagicMock() + return vllm_config + + # --- Unit Tests: NCCLWeightTransferUpdateInfo Validation --- @@ -87,7 +103,6 @@ class TestNCCLWeightTransferUpdateInfoValidation: """Test NCCLWeightTransferUpdateInfo dataclass validation.""" def test_valid_update_info(self): - """Test creating valid NCCLWeightTransferUpdateInfo.""" info = NCCLWeightTransferUpdateInfo( names=["layer.weight", "layer.bias"], dtype_names=["float32", "float32"], @@ -98,7 +113,6 @@ class TestNCCLWeightTransferUpdateInfoValidation: assert info.shapes == [[10, 10], [10]] def test_mismatched_dtype_names_raises(self): - """Test that mismatched dtype_names length raises ValueError.""" with pytest.raises(ValueError, match="dtype_names"): NCCLWeightTransferUpdateInfo( names=["layer.weight", "layer.bias"], @@ -107,7 +121,6 @@ class TestNCCLWeightTransferUpdateInfoValidation: ) def test_mismatched_shapes_raises(self): - """Test that mismatched shapes length raises ValueError.""" with pytest.raises(ValueError, match="shapes"): NCCLWeightTransferUpdateInfo( names=["layer.weight", "layer.bias"], @@ -116,73 +129,59 @@ class TestNCCLWeightTransferUpdateInfoValidation: ) def test_empty_lists_valid(self): - """Test that empty lists are valid.""" - info = NCCLWeightTransferUpdateInfo( - names=[], - dtype_names=[], - shapes=[], - ) + info = NCCLWeightTransferUpdateInfo(names=[], dtype_names=[], shapes=[]) assert len(info.names) == 0 + +# --- Unit Tests: SparseNCCLWeightTransferUpdateInfo Validation --- + + +class TestSparseNCCLWeightTransferUpdateInfoValidation: + """Test SparseNCCLWeightTransferUpdateInfo dataclass validation.""" + def test_valid_sparse_update_info(self): - """Test creating valid sparse NCCL update info.""" - info = NCCLWeightTransferUpdateInfo( + info = SparseNCCLWeightTransferUpdateInfo( names=["layer.weight", "layer.bias"], dtype_names=["float32", "bfloat16"], shapes=[[10, 10], [10]], num_updates_list=[4, 2], - update_kind="sparse_flat", ) - assert info.update_kind == "sparse_flat" assert info.num_updates_list == [4, 2] - def test_sparse_update_requires_num_updates_list(self): - with pytest.raises(ValueError, match="`num_updates_list` is required"): - NCCLWeightTransferUpdateInfo( - names=["layer.weight"], + def test_mismatched_dtype_names_raises(self): + with pytest.raises(ValueError, match="dtype_names"): + SparseNCCLWeightTransferUpdateInfo( + names=["layer.weight", "layer.bias"], dtype_names=["float32"], - shapes=[[10, 10]], - update_kind="sparse_flat", + shapes=[[10, 10], [10]], + num_updates_list=[4, 2], ) - def test_sparse_update_rejects_empty_num_updates_list(self): + def test_rejects_empty_num_updates_list(self): with pytest.raises(ValueError, match="cannot be empty"): - NCCLWeightTransferUpdateInfo( + SparseNCCLWeightTransferUpdateInfo( names=[], dtype_names=[], shapes=[], num_updates_list=[], - update_kind="sparse_flat", ) - def test_sparse_update_rejects_packed(self): - with pytest.raises(ValueError, match="cannot be combined with `packed=True`"): - NCCLWeightTransferUpdateInfo( - names=["layer.weight"], - dtype_names=["float32"], - shapes=[[10, 10]], - num_updates_list=[3], - update_kind="sparse_flat", - packed=True, - ) - - def test_sparse_update_rejects_mismatched_num_updates(self): + def test_rejects_mismatched_num_updates(self): with pytest.raises(ValueError, match="`num_updates_list`"): - NCCLWeightTransferUpdateInfo( + SparseNCCLWeightTransferUpdateInfo( names=["layer.weight", "layer.bias"], dtype_names=["float32", "float32"], shapes=[[10, 10], [10]], num_updates_list=[3], - update_kind="sparse_flat", ) - def test_dense_update_rejects_sparse_metadata(self): - with pytest.raises(ValueError, match="Sparse metadata"): - NCCLWeightTransferUpdateInfo( + def test_rejects_negative_num_updates(self): + with pytest.raises(ValueError, match="non-negative"): + SparseNCCLWeightTransferUpdateInfo( names=["layer.weight"], dtype_names=["float32"], shapes=[[10, 10]], - num_updates_list=[3], + num_updates_list=[-1], ) @@ -192,14 +191,17 @@ class TestNCCLWeightTransferUpdateInfoValidation: class TestNCCLEngineParsing: """Test NCCLWeightTransferEngine parsing methods.""" - def test_parse_init_info_valid(self): - """Test parsing valid init info dict.""" + def _make_engine(self): config = WeightTransferConfig(backend="nccl") - parallel_config = create_mock_parallel_config() - engine = NCCLWeightTransferEngine( - config, parallel_config, MagicMock(spec=torch.nn.Module) + return NCCLWeightTransferEngine( + config, + create_mock_vllm_config(), + "cuda", + MagicMock(spec=torch.nn.Module), ) + def test_parse_init_info_valid(self): + engine = self._make_engine() init_info = engine.parse_init_info( { "master_address": "127.0.0.1", @@ -208,7 +210,6 @@ class TestNCCLEngineParsing: "world_size": 3, } ) - assert isinstance(init_info, NCCLWeightTransferInitInfo) assert init_info.master_address == "127.0.0.1" assert init_info.master_port == 12345 @@ -216,29 +217,12 @@ class TestNCCLEngineParsing: assert init_info.world_size == 3 def test_parse_init_info_missing_field_raises(self): - """Test parsing init info with missing required field.""" - config = WeightTransferConfig(backend="nccl") - parallel_config = create_mock_parallel_config() - engine = NCCLWeightTransferEngine( - config, parallel_config, MagicMock(spec=torch.nn.Module) - ) - + engine = self._make_engine() with pytest.raises(ValueError, match="Invalid init_info"): - engine.parse_init_info( - { - "master_address": "127.0.0.1", - # Missing master_port, rank_offset, world_size - } - ) + engine.parse_init_info({"master_address": "127.0.0.1"}) def test_parse_update_info_valid(self): - """Test parsing valid update info dict.""" - config = WeightTransferConfig(backend="nccl") - parallel_config = create_mock_parallel_config() - engine = NCCLWeightTransferEngine( - config, parallel_config, MagicMock(spec=torch.nn.Module) - ) - + engine = self._make_engine() update_info = engine.parse_update_info( { "names": ["w1", "w2"], @@ -246,7 +230,6 @@ class TestNCCLEngineParsing: "shapes": [[100, 100], [50]], } ) - assert isinstance(update_info, NCCLWeightTransferUpdateInfo) assert update_info.names == ["w1", "w2"] assert update_info.dtype_names == ["float32", "bfloat16"] @@ -260,40 +243,140 @@ class TestEngineRegistry: """Test weight transfer engine registry.""" def test_create_engine_nccl(self): - """Test factory creates NCCL engine.""" config = WeightTransferConfig(backend="nccl") - parallel_config = create_mock_parallel_config() engine = WeightTransferEngineFactory.create_engine( - config, parallel_config, MagicMock(spec=torch.nn.Module) + config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module) ) assert isinstance(engine, NCCLWeightTransferEngine) def test_create_engine_ipc(self): - """Test factory creates IPC engine.""" config = WeightTransferConfig(backend="ipc") - parallel_config = create_mock_parallel_config() engine = WeightTransferEngineFactory.create_engine( - config, parallel_config, MagicMock(spec=torch.nn.Module) + config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module) ) assert isinstance(engine, IPCWeightTransferEngine) + def test_create_engine_sparse_nccl(self): + config = WeightTransferConfig(backend="sparse_nccl") + engine = WeightTransferEngineFactory.create_engine( + config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module) + ) + assert isinstance(engine, SparseNCCLWeightTransferEngine) + def test_create_engine_invalid_backend(self): - """Test factory raises for invalid backend.""" config = WeightTransferConfig(backend="invalid") - parallel_config = create_mock_parallel_config() with pytest.raises(ValueError, match="Invalid weight transfer backend"): WeightTransferEngineFactory.create_engine( - config, parallel_config, MagicMock(spec=torch.nn.Module) + config, + create_mock_vllm_config(), + "cuda", + MagicMock(spec=torch.nn.Module), ) def test_register_duplicate_raises(self): - """Test registering duplicate engine name raises.""" with pytest.raises(ValueError, match="already registered"): WeightTransferEngineFactory.register_engine( "nccl", NCCLWeightTransferEngine ) +# --- Unit Tests: Sparse patch application (CPU) --- + + +class TestSparseNCCLPatchApplication: + """Test SparseNCCLWeightTransferEngine._apply_patch on a real param.""" + + def _make_engine(self, model): + config = WeightTransferConfig(backend="sparse_nccl") + return SparseNCCLWeightTransferEngine( + config, create_mock_vllm_config(), "cpu", model + ) + + def _make_model(self, numel: int = 8): + model = torch.nn.Module() + model.register_parameter( + "w", torch.nn.Parameter(torch.zeros(numel), requires_grad=False) + ) + + def get_parameter(name): + assert name == "w" + return model.w + + model.get_parameter = get_parameter + return model + + def test_apply_patch_updates_only_selected_entries(self): + model = self._make_model(8) + engine = self._make_engine(model) + engine._apply_patch( + SparseWeightPatch( + name="w", + indices=torch.tensor([1, 3], dtype=torch.int32), + values=torch.tensor([5.0, 7.0], dtype=torch.float32), + ) + ) + expected = torch.zeros(8) + expected[1] = 5.0 + expected[3] = 7.0 + assert torch.equal(model.w.data, expected) + + def test_apply_patch_rejects_mismatched_lengths(self): + model = self._make_model(8) + engine = self._make_engine(model) + with pytest.raises(ValueError, match="matching lengths"): + engine._apply_patch( + SparseWeightPatch( + name="w", + indices=torch.tensor([1, 3], dtype=torch.int32), + values=torch.tensor([5.0], dtype=torch.float32), + ) + ) + + def test_apply_patch_rejects_non_int32_indices(self): + model = self._make_model(8) + engine = self._make_engine(model) + with pytest.raises(ValueError, match="int32 indices"): + engine._apply_patch( + SparseWeightPatch( + name="w", + indices=torch.tensor([1], dtype=torch.int64), + values=torch.tensor([5.0], dtype=torch.float32), + ) + ) + + def test_apply_patch_rejects_dtype_mismatch(self): + model = self._make_model(8) + engine = self._make_engine(model) + with pytest.raises(ValueError, match="does not match"): + engine._apply_patch( + SparseWeightPatch( + name="w", + indices=torch.tensor([1], dtype=torch.int32), + values=torch.tensor([5.0], dtype=torch.bfloat16), + ) + ) + + def test_apply_patch_rejects_non_contiguous_param(self): + model = torch.nn.Module() + model.register_parameter( + "w", + torch.nn.Parameter( + torch.arange(12, dtype=torch.float32).view(3, 4).t(), + requires_grad=False, + ), + ) + model.get_parameter = lambda name: model.w + engine = self._make_engine(model) + with pytest.raises(NotImplementedError, match="contiguous params"): + engine._apply_patch( + SparseWeightPatch( + name="w", + indices=torch.tensor([1], dtype=torch.int32), + values=torch.tensor([1.0], dtype=torch.float32), + ) + ) + + # --- Test receive_weights without init raises --- @@ -303,42 +386,37 @@ def test_nccl_receive_weights_without_init_raises(): pytest.skip("Need at least 1 GPU for this test") config = WeightTransferConfig(backend="nccl") - parallel_config = create_mock_parallel_config() engine = NCCLWeightTransferEngine( - config, parallel_config, MagicMock(spec=torch.nn.Module) + config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module) ) update_info = NCCLWeightTransferUpdateInfo( - names=["w"], - dtype_names=["float32"], - shapes=[[10]], + names=["w"], dtype_names=["float32"], shapes=[[10]] ) with pytest.raises(RuntimeError, match="not initialized"): - engine.receive_weights(update_info, lambda x: None) + engine.receive_weights(update_info) -def test_nccl_receive_sparse_weights_without_init_raises(): +def test_sparse_nccl_receive_weights_without_init_raises(): """Test that sparse receive raises if init_transfer_engine wasn't called.""" if torch.accelerator.device_count() < 1: pytest.skip("Need at least 1 GPU for this test") - config = WeightTransferConfig(backend="nccl") - parallel_config = create_mock_parallel_config() - engine = NCCLWeightTransferEngine( - config, parallel_config, MagicMock(spec=torch.nn.Module) + config = WeightTransferConfig(backend="sparse_nccl") + engine = SparseNCCLWeightTransferEngine( + config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module) ) - update_info = NCCLWeightTransferUpdateInfo( + update_info = SparseNCCLWeightTransferUpdateInfo( names=["w"], dtype_names=["float32"], shapes=[[10]], num_updates_list=[2], - update_kind="sparse_flat", ) with pytest.raises(RuntimeError, match="not initialized"): - engine.receive_sparse_weights(update_info, lambda x: None) + engine.receive_weights(update_info) # --- Integration Test: NCCL Weight Transfer Between Ray Tasks --- @@ -387,6 +465,7 @@ def inference_receive_tensor( tensor_dtype: str, ) -> dict: """Inference task that receives tensor via NCCLWeightTransferEngine.""" + import contextlib from unittest.mock import MagicMock import torch @@ -401,17 +480,32 @@ def inference_receive_tensor( NCCLWeightTransferUpdateInfo, ) - # Create engine with mock parallel config + class Recorder(torch.nn.Module): + def __init__(self): + super().__init__() + self.received = [] + + def load_weights(self, weights): + for name, tensor in weights: + self.received.append((name, tensor.clone())) + config = WeightTransferConfig(backend="nccl") + vllm_config = MagicMock() parallel_config = MagicMock(spec=ParallelConfig) parallel_config.rank = 0 parallel_config.world_size = 1 parallel_config.data_parallel_rank = 0 parallel_config.data_parallel_index = 0 + vllm_config.parallel_config = parallel_config + vllm_config.model_config = MagicMock() - engine = NCCLWeightTransferEngine( - config, parallel_config, MagicMock(spec=torch.nn.Module) - ) + recorder = Recorder() + engine = NCCLWeightTransferEngine(config, vllm_config, "cuda", recorder) + # Transport-only test: bypass the set_current_vllm_config context that + # receive_weights enters, since vllm_config here is a mock. + import vllm.config as _vllm_config_mod + + _vllm_config_mod.set_current_vllm_config = lambda cfg: contextlib.nullcontext() # Initialize the engine (joins as rank 1) init_info = NCCLWeightTransferInitInfo( @@ -422,20 +516,12 @@ def inference_receive_tensor( ) engine.init_transfer_engine(init_info) - # Receive weights with a no-op load_weights that captures the tensor - received_tensors = [] - - def noop_load_weights(weights: list[tuple[str, torch.Tensor]]): - for name, tensor in weights: - # Clone tensor to keep it after engine cleans up - received_tensors.append((name, tensor.clone())) - update_info = NCCLWeightTransferUpdateInfo( names=["test.weight"], dtype_names=[tensor_dtype], shapes=[tensor_shape], ) - engine.receive_weights(update_info, noop_load_weights) + engine.receive_weights(update_info) torch.accelerator.synchronize() # Verify we received the tensor @@ -443,11 +529,10 @@ def inference_receive_tensor( received_shape = None received_sum = None - if len(received_tensors) == 1: - name, tensor = received_tensors[0] + if len(recorder.received) == 1: + name, tensor = recorder.received[0] received_shape = list(tensor.shape) received_sum = tensor.sum().item() - # Check shape matches and values are all 1s (trainer sends ones) if received_shape == tensor_shape: expected_sum = 1.0 * torch.tensor(tensor_shape).prod().item() if abs(received_sum - expected_sum) < 0.01: @@ -478,11 +563,9 @@ def test_nccl_weight_transfer_between_processes(): master_port = get_open_port() world_size = 2 # 1 trainer + 1 inference worker - # Tensor to transfer: 100x100 ones tensor_shape = [100, 100] tensor_dtype = "float32" - # Start both tasks concurrently - Ray assigns GPUs automatically inference_future = inference_receive_tensor.remote( master_address, master_port, world_size, tensor_shape, tensor_dtype ) @@ -490,7 +573,6 @@ def test_nccl_weight_transfer_between_processes(): master_address, master_port, world_size, tensor_shape, tensor_dtype ) - # Wait for both to complete trainer_result, result = ray.get([trainer_future, inference_future]) assert trainer_result, "Trainer should complete successfully" @@ -514,10 +596,12 @@ def trainer_broadcast_sparse_tensor( from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator from vllm.distributed.utils import StatelessProcessGroup - from vllm.distributed.weight_transfer.base import SparseWeightPatch from vllm.distributed.weight_transfer.nccl_engine import ( NCCLTrainerSendWeightsArgs, - NCCLWeightTransferEngine, + ) + from vllm.distributed.weight_transfer.sparse_nccl_engine import ( + SparseNCCLWeightTransferEngine, + SparseWeightPatch, ) pg = StatelessProcessGroup.create( @@ -533,7 +617,7 @@ def trainer_broadcast_sparse_tensor( indices=torch.tensor([1, 7, 25], dtype=torch.int32, device=device), values=torch.tensor([10.0, 20.0, 30.0], dtype=torch.float32, device=device), ) - NCCLWeightTransferEngine.trainer_send_sparse_weights( + SparseNCCLWeightTransferEngine.trainer_send_weights( iter([patch]), NCCLTrainerSendWeightsArgs(group=comm), ) @@ -547,7 +631,7 @@ def inference_receive_sparse_tensor( master_port: int, world_size: int, ) -> dict: - """Inference task that receives sparse patches via NCCLWeightTransferEngine.""" + """Inference task that receives sparse patches via the sparse engine.""" from unittest.mock import MagicMock import torch @@ -556,22 +640,40 @@ def inference_receive_sparse_tensor( from vllm.config.parallel import ParallelConfig from vllm.config.weight_transfer import WeightTransferConfig - from vllm.distributed.weight_transfer.nccl_engine import ( - NCCLWeightTransferEngine, - NCCLWeightTransferInitInfo, - NCCLWeightTransferUpdateInfo, + from vllm.distributed.weight_transfer.sparse_nccl_engine import ( + SparseNCCLWeightTransferEngine, + SparseNCCLWeightTransferUpdateInfo, ) - config = WeightTransferConfig(backend="nccl") + config = WeightTransferConfig(backend="sparse_nccl") + vllm_config = MagicMock() parallel_config = MagicMock(spec=ParallelConfig) parallel_config.rank = 0 parallel_config.world_size = 1 parallel_config.data_parallel_rank = 0 parallel_config.data_parallel_index = 0 + vllm_config.parallel_config = parallel_config + vllm_config.model_config = MagicMock() - engine = NCCLWeightTransferEngine( - config, parallel_config, MagicMock(spec=torch.nn.Module) + # Real module holding the target parameter the patch will modify. + model = torch.nn.Module() + model.register_parameter( + "w", torch.nn.Parameter(torch.zeros(30, device="cuda"), requires_grad=False) ) + model.get_parameter = lambda name: model.w + + update_info = SparseNCCLWeightTransferUpdateInfo( + names=["w"], + dtype_names=["float32"], + shapes=[[30]], + num_updates_list=[3], + ) + + engine = SparseNCCLWeightTransferEngine(config, vllm_config, "cuda", model) + from vllm.distributed.weight_transfer.nccl_common import ( + NCCLWeightTransferInitInfo, + ) + engine.init_transfer_engine( NCCLWeightTransferInitInfo( master_address=master_address, @@ -580,32 +682,18 @@ def inference_receive_sparse_tensor( world_size=world_size, ) ) - - target = torch.zeros(30, dtype=torch.float32, device=device) - - def apply_sparse_patches(patches: list[SparseWeightPatch]): - for patch in patches: - target.index_copy_(0, patch.indices.to(torch.long), patch.values) - - update_info = NCCLWeightTransferUpdateInfo( - names=["test.weight"], - dtype_names=["float32"], - shapes=[[30]], - num_updates_list=[3], - update_kind="sparse_flat", - ) - engine.receive_sparse_weights(update_info, apply_sparse_patches) + engine.receive_weights(update_info) torch.accelerator.synchronize() expected = torch.zeros(30, dtype=torch.float32, device=device) expected[[1, 7, 25]] = torch.tensor( [10.0, 20.0, 30.0], dtype=torch.float32, device=device ) - success = torch.equal(target, expected) + success = torch.equal(model.w.data, expected) engine.shutdown() return { "success": success, - "selected_values": target[[1, 7, 25]].cpu().tolist(), + "selected_values": model.w.data[[1, 7, 25]].cpu().tolist(), } @@ -644,11 +732,9 @@ class TestIPCWeightTransferUpdateInfoValidation: """Test IPCWeightTransferUpdateInfo dataclass validation.""" def test_valid_update_info(self): - """Test creating valid IPCWeightTransferUpdateInfo.""" if torch.accelerator.device_count() < 1: pytest.skip("Need at least 1 GPU for this test") - # Create a dummy tensor and IPC handle dummy_tensor = torch.ones(10, 10, device="cuda:0") _, ipc_handle = reduce_tensor(dummy_tensor) gpu_uuid = str(torch.cuda.get_device_properties(0).uuid) @@ -666,7 +752,6 @@ class TestIPCWeightTransferUpdateInfoValidation: assert len(info.ipc_handles) == 1 def test_mismatched_dtype_names_raises(self): - """Test that mismatched dtype_names length raises ValueError.""" if torch.accelerator.device_count() < 1: pytest.skip("Need at least 1 GPU for this test") @@ -684,7 +769,6 @@ class TestIPCWeightTransferUpdateInfoValidation: ) def test_mismatched_shapes_raises(self): - """Test that mismatched shapes length raises ValueError.""" if torch.accelerator.device_count() < 1: pytest.skip("Need at least 1 GPU for this test") @@ -702,7 +786,6 @@ class TestIPCWeightTransferUpdateInfoValidation: ) def test_mismatched_ipc_handles_raises(self): - """Test that mismatched ipc_handles length raises ValueError.""" if torch.accelerator.device_count() < 1: pytest.skip("Need at least 1 GPU for this test") @@ -719,44 +802,7 @@ class TestIPCWeightTransferUpdateInfoValidation: ipc_handles=ipc_handles, ) - def test_sparse_update_kind_rejected(self): - """Test that IPC backend rejects sparse update metadata.""" - if torch.accelerator.device_count() < 1: - pytest.skip("Need at least 1 GPU for this test") - - dummy_tensor = torch.ones(10, 10, device="cuda:0") - ipc_handle = reduce_tensor(dummy_tensor) - gpu_uuid = str(torch.cuda.get_device_properties(0).uuid) - ipc_handles = [{gpu_uuid: ipc_handle}] - - with pytest.raises(NotImplementedError, match="dense updates"): - IPCWeightTransferUpdateInfo( - names=["layer.weight"], - dtype_names=["float32"], - shapes=[[10, 10]], - num_updates_list=[1], - ipc_handles=ipc_handles, - update_kind="sparse_flat", - ) - - def test_sparse_methods_not_supported(self): - """Test that IPC engine inherits sparse rejection from the base class.""" - config = WeightTransferConfig(backend="ipc") - parallel_config = create_mock_parallel_config() - engine = IPCWeightTransferEngine( - config, parallel_config, MagicMock(spec=torch.nn.Module) - ) - - with pytest.raises(NotImplementedError, match="(?i)sparse weight updates"): - engine.receive_sparse_weights(MagicMock(), lambda _: None) - with pytest.raises(NotImplementedError, match="(?i)sparse weight updates"): - engine.trainer_send_sparse_weights( - iter([]), - {"mode": "http", "url": "http://localhost:8000"}, - ) - def test_valid_update_info_from_pickled(self, monkeypatch): - """Test creating IPCWeightTransferUpdateInfo from pickled handles.""" if torch.accelerator.device_count() < 1: pytest.skip("Need at least 1 GPU for this test") @@ -779,7 +825,6 @@ class TestIPCWeightTransferUpdateInfoValidation: assert info.ipc_handles_pickled is None def test_pickled_requires_insecure_serialization_flag(self, monkeypatch): - """Test that pickled handles are rejected unless env flag is enabled.""" monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "0") with pytest.raises(ValueError, match="VLLM_ALLOW_INSECURE_SERIALIZATION=1"): @@ -791,7 +836,6 @@ class TestIPCWeightTransferUpdateInfoValidation: ) def test_both_handles_and_pickled_raises(self): - """Test that providing both ipc_handles and ipc_handles_pickled raises.""" if torch.accelerator.device_count() < 1: pytest.skip("Need at least 1 GPU for this test") @@ -812,7 +856,6 @@ class TestIPCWeightTransferUpdateInfoValidation: ) def test_neither_handles_nor_pickled_raises(self): - """Test that providing neither ipc_handles nor ipc_handles_pickled raises.""" with pytest.raises(ValueError, match="must be provided"): IPCWeightTransferUpdateInfo( names=["layer.weight"], @@ -821,7 +864,6 @@ class TestIPCWeightTransferUpdateInfoValidation: ) def test_empty_lists_valid(self): - """Test that empty lists are valid.""" info = IPCWeightTransferUpdateInfo( names=[], dtype_names=[], @@ -837,18 +879,21 @@ class TestIPCWeightTransferUpdateInfoValidation: class TestIPCEngineParsing: """Test IPCWeightTransferEngine parsing methods.""" + def _make_engine(self): + config = WeightTransferConfig(backend="ipc") + return IPCWeightTransferEngine( + config, + create_mock_vllm_config(), + "cuda", + MagicMock(spec=torch.nn.Module), + ) + def test_parse_update_info_valid(self): - """Test parsing valid update info dict.""" if torch.accelerator.device_count() < 1: pytest.skip("Need at least 1 GPU for this test") - config = WeightTransferConfig(backend="ipc") - parallel_config = create_mock_parallel_config() - engine = IPCWeightTransferEngine( - config, parallel_config, MagicMock(spec=torch.nn.Module) - ) + engine = self._make_engine() - # Create dummy IPC handles dummy_tensor1 = torch.ones(100, 100, device="cuda:0") dummy_tensor2 = torch.ones(50, device="cuda:0") _, ipc_args1 = reduce_tensor(dummy_tensor1) @@ -872,17 +917,12 @@ class TestIPCEngineParsing: assert len(update_info.ipc_handles) == 2 def test_parse_update_info_pickled(self, monkeypatch): - """Test parsing update info with pickled IPC handles (HTTP path).""" if torch.accelerator.device_count() < 1: pytest.skip("Need at least 1 GPU for this test") monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") - config = WeightTransferConfig(backend="ipc") - parallel_config = create_mock_parallel_config() - engine = IPCWeightTransferEngine( - config, parallel_config, MagicMock(spec=torch.nn.Module) - ) + engine = self._make_engine() dummy_tensor1 = torch.ones(100, 100, device="cuda:0") dummy_tensor2 = torch.ones(50, device="cuda:0") @@ -909,12 +949,7 @@ class TestIPCEngineParsing: assert gpu_uuid in update_info.ipc_handles[1] def test_parse_update_info_ignores_none_pickled_handles(self): - """Test Ray/asdict payloads with a null pickled field use ipc_handles.""" - config = WeightTransferConfig(backend="ipc") - parallel_config = create_mock_parallel_config() - engine = IPCWeightTransferEngine( - config, parallel_config, MagicMock(spec=torch.nn.Module) - ) + engine = self._make_engine() ipc_handles = [{"gpu-uuid": ("ipc-args",)}] update_info = engine.parse_update_info( @@ -931,15 +966,10 @@ class TestIPCEngineParsing: assert update_info.ipc_handles == ipc_handles def test_parse_update_info_both_handles_and_pickled_raises(self): - """Test that providing both ipc_handles and ipc_handles_pickled raises.""" if torch.accelerator.device_count() < 1: pytest.skip("Need at least 1 GPU for this test") - config = WeightTransferConfig(backend="ipc") - parallel_config = create_mock_parallel_config() - engine = IPCWeightTransferEngine( - config, parallel_config, MagicMock(spec=torch.nn.Module) - ) + engine = self._make_engine() dummy_tensor = torch.ones(10, 10, device="cuda:0") _, ipc_handle = reduce_tensor(dummy_tensor) @@ -981,9 +1011,6 @@ class TrainerActor: self.tensor = torch.ones(tensor_shape, dtype=dtype, device=device) self.tensor.fill_(42.0) # Fill with 42 to verify correct transfer - # Create IPC handle (tensor must stay alive for IPC to work) - # reduce_tensor returns (rebuild_func, args); we only send args - # since the receiver imports rebuild_cuda_tensor directly. _, ipc_args = reduce_tensor(self.tensor) gpu_uuid = get_physical_gpu_id(device.index) @@ -1007,6 +1034,7 @@ def inference_receive_ipc_tensor( mode: str = "ray", ) -> dict: """Inference task that receives tensor via IPCWeightTransferEngine.""" + import contextlib import os # Worker-side: ipc_handles_pickled is deserialized via pickle. @@ -1025,30 +1053,36 @@ def inference_receive_ipc_tensor( IPCWeightTransferEngine, ) - # Create engine with mock parallel config + class Recorder(torch.nn.Module): + def __init__(self): + super().__init__() + self.received = [] + + def load_weights(self, weights): + for name, tensor in weights: + self.received.append((name, tensor.clone())) + config = WeightTransferConfig(backend="ipc") + vllm_config = MagicMock() parallel_config = MagicMock(spec=ParallelConfig) parallel_config.rank = 0 parallel_config.world_size = 1 parallel_config.data_parallel_rank = 0 parallel_config.data_parallel_index = 0 + vllm_config.parallel_config = parallel_config + vllm_config.model_config = MagicMock() - engine = IPCWeightTransferEngine( - config, parallel_config, MagicMock(spec=torch.nn.Module) - ) + recorder = Recorder() + engine = IPCWeightTransferEngine(config, vllm_config, "cuda", recorder) + # Transport-only test: bypass the set_current_vllm_config context that + # receive_weights enters, since vllm_config here is a mock. + import vllm.config as _vllm_config_mod + + _vllm_config_mod.set_current_vllm_config = lambda cfg: contextlib.nullcontext() - # Initialize the engine (no-op for IPC) init_info = IPCWeightTransferInitInfo() engine.init_transfer_engine(init_info) - # Receive weights with a no-op load_weights that captures the tensor - received_tensors = [] - - def noop_load_weights(weights: list[tuple[str, torch.Tensor]]): - for name, tensor in weights: - # Clone tensor to keep it after engine cleans up - received_tensors.append((name, tensor.clone())) - ipc_handles = [{ipc_handle_dict["gpu_uuid"]: ipc_handle_dict["ipc_handle"]}] if mode == "ray": @@ -1059,7 +1093,6 @@ def inference_receive_ipc_tensor( "ipc_handles": ipc_handles, } elif mode == "http": - # Simulate HTTP transport: pickle + base64 encode handles pickled = base64.b64encode(pickle.dumps(ipc_handles)).decode("utf-8") update_dict = { "names": ["test.weight"], @@ -1071,19 +1104,17 @@ def inference_receive_ipc_tensor( raise ValueError(f"Unknown mode: {mode}") update_info = engine.parse_update_info(update_dict) - engine.receive_weights(update_info, noop_load_weights) + engine.receive_weights(update_info) torch.accelerator.synchronize() - # Verify we received the tensor success = False received_shape = None received_sum = None - if len(received_tensors) == 1: - name, tensor = received_tensors[0] + if len(recorder.received) == 1: + name, tensor = recorder.received[0] received_shape = list(tensor.shape) received_sum = tensor.sum().item() - # Check shape matches and values are all 42s (trainer sends 42s) if received_shape == ipc_handle_dict["shape"]: expected_sum = 42.0 * torch.tensor(ipc_handle_dict["shape"]).prod().item() if abs(received_sum - expected_sum) < 0.01: @@ -1104,23 +1135,12 @@ def inference_receive_ipc_tensor( ) @pytest.mark.parametrize("mode", ["ray", "http"]) def test_ipc_weight_transfer_between_processes(mode: str): - """Test IPC weight transfer from trainer to inference process using Ray. - - Parametrized over transport modes: - - 'ray': ipc_handles passed directly. - - 'http': ipc_handles pickled + base64-encoded, deserialized in - parse_update_info before constructing the dataclass. - - IPC requires same-GPU access, so we use a placement group to co-locate - the trainer actor and inference task on the same GPU. - """ + """Test IPC weight transfer from trainer to inference process using Ray.""" from ray.util.placement_group import placement_group from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy _init_ray_for_weight_transfer() - # Create a placement group to ensure both processes are on the same GPU - # Use fractional GPUs so both tasks can share the same GPU bundle pg = placement_group([{"GPU": 1, "CPU": 2}]) ray.get(pg.ready()) @@ -1129,20 +1149,15 @@ def test_ipc_weight_transfer_between_processes(mode: str): placement_group_capture_child_tasks=True, ) - # Tensor to transfer: 100x100 filled with 42s tensor_shape = [100, 100] tensor_dtype = "float32" - # Create trainer actor that holds the tensor and IPC handle (stays alive) trainer_actor = TrainerActor.options( # type: ignore[attr-defined] scheduling_strategy=scheduling_strategy ).remote(tensor_shape, tensor_dtype) - # Get IPC handle dict (tensor stays alive in trainer actor) ipc_handle_dict = ray.get(trainer_actor.get_ipc_handle_dict.remote()) - # Receive tensor in inference process using IPC handles (on same GPU) - # Trainer actor stays alive during this operation inference_result = ray.get( inference_receive_ipc_tensor.options( scheduling_strategy=scheduling_strategy @@ -1162,12 +1177,10 @@ def test_ipc_receive_weights_missing_gpu_uuid_raises(): pytest.skip("Need at least 1 GPU for this test") config = WeightTransferConfig(backend="ipc") - parallel_config = create_mock_parallel_config() engine = IPCWeightTransferEngine( - config, parallel_config, MagicMock(spec=torch.nn.Module) + config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module) ) - # Create IPC handle with wrong GPU UUID dummy_tensor = torch.ones(10, 10, device="cuda:0") _, ipc_handle = reduce_tensor(dummy_tensor) wrong_uuid = "wrong-uuid-12345" @@ -1181,4 +1194,4 @@ def test_ipc_receive_weights_missing_gpu_uuid_raises(): ) with pytest.raises(ValueError, match="IPC handle not found"): - engine.receive_weights(update_info, lambda x: None) + engine.receive_weights(update_info) diff --git a/tests/entrypoints/weight_transfer/test_weight_transfer_llm.py b/tests/entrypoints/weight_transfer/test_weight_transfer_llm.py index 1dd89afcf8..9088b3c5e8 100644 --- a/tests/entrypoints/weight_transfer/test_weight_transfer_llm.py +++ b/tests/entrypoints/weight_transfer/test_weight_transfer_llm.py @@ -8,7 +8,6 @@ actual NCCL communication. """ import os -from collections.abc import Callable from dataclasses import dataclass from unittest.mock import patch @@ -48,7 +47,6 @@ class MockUpdateInfo(WeightTransferUpdateInfo): names: list[str] | None = None dtype_names: list[str] | None = None shapes: list[list[int]] | None = None - num_updates_list: list[int] | None = None class MockWeightTransferEngine(WeightTransferEngine[MockInitInfo, MockUpdateInfo]): @@ -59,16 +57,20 @@ class MockWeightTransferEngine(WeightTransferEngine[MockInitInfo, MockUpdateInfo # Class-level tracking for verification across processes init_transfer_engine_called: bool = False + start_called: bool = False receive_weights_called: bool = False + finish_called: bool = False shutdown_called: bool = False last_init_info: MockInitInfo | None = None last_update_info: MockUpdateInfo | None = None - def __init__(self, config, parallel_config, model): - super().__init__(config, parallel_config, model) + def __init__(self, config, vllm_config, device, model): + super().__init__(config, vllm_config, device, model) # Reset tracking on init MockWeightTransferEngine.init_transfer_engine_called = False + MockWeightTransferEngine.start_called = False MockWeightTransferEngine.receive_weights_called = False + MockWeightTransferEngine.finish_called = False MockWeightTransferEngine.shutdown_called = False MockWeightTransferEngine.last_init_info = None MockWeightTransferEngine.last_update_info = None @@ -77,37 +79,28 @@ class MockWeightTransferEngine(WeightTransferEngine[MockInitInfo, MockUpdateInfo MockWeightTransferEngine.init_transfer_engine_called = True MockWeightTransferEngine.last_init_info = init_info - def receive_weights( - self, - update_info: MockUpdateInfo, - load_weights: Callable[[list[tuple[str, torch.Tensor]]], None], - ) -> None: - MockWeightTransferEngine.receive_weights_called = True - MockWeightTransferEngine.last_update_info = update_info - # Simulate loading weights by calling load_weights with empty list - # (In real implementation, this would receive and load actual weights) - load_weights([]) + def start_weight_update(self) -> None: + MockWeightTransferEngine.start_called = True - def receive_sparse_weights( - self, - update_info: MockUpdateInfo, - apply_patches: Callable[[list], None], - ) -> None: + def finish_weight_update(self) -> None: + MockWeightTransferEngine.finish_called = True + + def receive_weights(self, update_info: MockUpdateInfo) -> None: MockWeightTransferEngine.receive_weights_called = True MockWeightTransferEngine.last_update_info = update_info - apply_patches([]) def shutdown(self) -> None: MockWeightTransferEngine.shutdown_called = True - def trainer_send_weights(self, *args, **kwargs): + @staticmethod + def trainer_send_weights(*args, **kwargs): """Mock method to simulate trainer sending weights.""" pass -def mock_create_engine(config, parallel_config, model): +def mock_create_engine(config, vllm_config, device, model): """Mock factory function that returns our mock engine.""" - return MockWeightTransferEngine(config, parallel_config, model) + return MockWeightTransferEngine(config, vllm_config, device, model) # --- Tests --- @@ -208,7 +201,7 @@ def test_update_weights_calls_engine(): llm.init_weight_transfer_engine( WeightTransferInitRequest(init_info={"test_param": "init"}) ) - llm.start_weight_update(is_checkpoint_format=True) + llm.start_weight_update() # Call update_weights test_names = ["layer.weight", "layer.bias"] @@ -243,61 +236,6 @@ def test_update_weights_calls_engine(): llm.finish_weight_update() -@create_new_process_for_each_test() -def test_update_weights_passes_sparse_metadata(): - """Test sparse update metadata is forwarded unchanged to the engine.""" - if torch.accelerator.device_count() < 1: - pytest.skip("Need at least 1 GPU for this test") - - os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0" - os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1" - - with patch( - "vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine", - mock_create_engine, - ): - llm = LLM( - model=MODEL_NAME, - enforce_eager=True, - load_format="dummy", - tensor_parallel_size=1, - weight_transfer_config=WeightTransferConfig(backend="nccl"), - ) - - llm.init_weight_transfer_engine( - WeightTransferInitRequest(init_info={"test_param": "init"}) - ) - llm.start_weight_update(is_checkpoint_format=False) - - llm.update_weights( - WeightTransferUpdateRequest( - update_info={ - "names": ["layer.weight"], - "dtype_names": ["bfloat16"], - "shapes": [[100]], - "num_updates_list": [3], - "update_kind": "sparse_flat", - } - ) - ) - - def check_sparse_update_called(self): - engine = self.weight_transfer_engine - if not engine.receive_weights_called: - return None - info = engine.last_update_info - return ( - info.update_kind, - info.num_updates_list, - ) - - results = llm.collective_rpc(check_sparse_update_called) - for result in results: - assert result == ("sparse_flat", [3]) - - llm.finish_weight_update() - - @create_new_process_for_each_test() def test_full_weight_transfer_flow(): """Test the complete weight transfer flow: init -> start -> update -> finish.""" @@ -327,7 +265,7 @@ def test_full_weight_transfer_flow(): ) # Step 2: Start weight update - llm.start_weight_update(is_checkpoint_format=True) + llm.start_weight_update() # Step 3: Update weights llm.update_weights( diff --git a/tests/v1/worker/test_gpu_model_runner.py b/tests/v1/worker/test_gpu_model_runner.py index 75d8c9c746..6d538bc69d 100644 --- a/tests/v1/worker/test_gpu_model_runner.py +++ b/tests/v1/worker/test_gpu_model_runner.py @@ -7,7 +7,6 @@ from unittest.mock import Mock import numpy as np import pytest import torch -import torch.nn as nn import vllm.v1.worker.gpu_model_runner as gpu_model_runner_module from vllm.config import ( @@ -23,7 +22,6 @@ from vllm.distributed.parallel_state import ( init_distributed_environment, initialize_model_parallel, ) -from vllm.distributed.weight_transfer.base import SparseWeightPatch from vllm.lora.layers import LoRAMappingType from vllm.lora.request import LoRARequest from vllm.model_executor.layers.attention import Attention @@ -864,73 +862,6 @@ def test_sample_passes_reordered_draft_probs_to_rejection_sampler(): assert torch.equal(passed_draft_probs, expected_draft_probs) -def test_apply_sparse_weight_patches_updates_only_selected_entries(): - class DummyModel(nn.Module): - def __init__(self): - super().__init__() - self.weight = nn.Parameter(torch.zeros(6, dtype=torch.float32)) - - runner = object.__new__(GPUModelRunner) - runner.model = DummyModel() - - runner.apply_sparse_weight_patches( - [ - SparseWeightPatch( - name="weight", - indices=torch.tensor([1, 4], dtype=torch.int32), - values=torch.tensor([3.5, -2.0], dtype=torch.float32), - ) - ] - ) - - expected = torch.tensor([0.0, 3.5, 0.0, 0.0, -2.0, 0.0], dtype=torch.float32) - assert torch.equal(runner.get_model().weight.data, expected) - - -def test_apply_sparse_weight_patches_rejects_mismatched_lengths(): - class DummyModel(nn.Module): - def __init__(self): - super().__init__() - self.weight = nn.Parameter(torch.zeros(4, dtype=torch.float32)) - - runner = object.__new__(GPUModelRunner) - runner.model = DummyModel() - - with pytest.raises(ValueError, match="matching lengths"): - runner.apply_sparse_weight_patches( - [ - SparseWeightPatch( - name="weight", - indices=torch.tensor([1, 2], dtype=torch.int32), - values=torch.tensor([1.0], dtype=torch.float32), - ) - ] - ) - - -def test_apply_sparse_weight_patches_rejects_non_contiguous_param(): - class DummyModel(nn.Module): - def __init__(self): - super().__init__() - self.weight = nn.Parameter( - torch.arange(12, dtype=torch.float32).view(3, 4).t() - ) - - runner = object.__new__(GPUModelRunner) - runner.model = DummyModel() - - with pytest.raises(NotImplementedError, match="contiguous params"): - runner.apply_sparse_weight_patches( - [ - SparseWeightPatch( - name="weight", - indices=torch.tensor([1], dtype=torch.int32), - values=torch.tensor([1.0], dtype=torch.float32), - ) - ] - ) - - def test_init_kv_cache_with_kv_sharing_invalid_target_layer_order(default_vllm_config): torch.set_default_dtype(torch.float16) layer_0 = "model.layers.0.self_attn.attn" diff --git a/tests/v1/worker/test_gpu_worker_weight_transfer.py b/tests/v1/worker/test_gpu_worker_weight_transfer.py index dba0f65854..93c2e916b1 100644 --- a/tests/v1/worker/test_gpu_worker_weight_transfer.py +++ b/tests/v1/worker/test_gpu_worker_weight_transfer.py @@ -1,155 +1,94 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for GPUWorker weight-transfer pass-through behavior. -from types import SimpleNamespace -from unittest.mock import MagicMock +The worker no longer contains transport, layerwise, or sparse logic: it only +delegates to the configured weight transfer engine and tracks whether an update +session is active. These tests verify that delegation and the session guard. +""" import pytest -import torch -from vllm.config.parallel import ParallelConfig -from vllm.config.weight_transfer import WeightTransferConfig -from vllm.distributed.weight_transfer.base import SparseWeightPatch -from vllm.distributed.weight_transfer.nccl_engine import NCCLWeightTransferEngine from vllm.v1.worker.gpu_worker import Worker -def _make_nccl_engine() -> NCCLWeightTransferEngine: - parallel_config = MagicMock(spec=ParallelConfig) - parallel_config.rank = 0 - parallel_config.world_size = 1 - parallel_config.data_parallel_rank = 0 - parallel_config.data_parallel_index = 0 - return NCCLWeightTransferEngine( - WeightTransferConfig(backend="nccl"), - parallel_config, - MagicMock(spec=torch.nn.Module), - ) +class _RecordingEngine: + """Minimal stand-in for a weight transfer engine.""" + + def __init__(self, raise_on_update: bool = False): + self.raise_on_update = raise_on_update + self.started = False + self.finished = False + self.update_calls: list[dict] = [] + + def start_weight_update(self) -> None: + self.started = True + + def update_weights(self, update_info: dict) -> None: + self.update_calls.append(update_info) + if self.raise_on_update: + raise ValueError("boom") + + def finish_weight_update(self) -> None: + self.finished = True -def test_update_weights_sparse_dispatches_to_sparse_receive(monkeypatch): - monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None) - +def _make_worker(engine: _RecordingEngine | None) -> Worker: worker = object.__new__(Worker) - worker.device = "cpu" - worker.parallel_config = SimpleNamespace(world_size=1) - worker.weight_transfer_engine = _make_nccl_engine() - worker._weight_update_active = True - worker._is_checkpoint_format = False - - applied_patches = [] - - def apply_sparse_weight_patches(patches): - applied_patches.extend(patches) - - worker.model_runner = SimpleNamespace( - apply_sparse_weight_patches=apply_sparse_weight_patches, - ) - - received_kinds = [] - - def receive_sparse_weights(update_info, apply_patches): - received_kinds.append(update_info.update_kind) - apply_patches( - [ - SparseWeightPatch( - name="layer.weight", - indices=torch.tensor([1], dtype=torch.int32), - values=torch.tensor([2.0], dtype=torch.float32), - ) - ] - ) - - worker.weight_transfer_engine.receive_sparse_weights = receive_sparse_weights - - Worker.update_weights( - worker, - { - "names": ["layer.weight"], - "dtype_names": ["float32"], - "shapes": [[4]], - "num_updates_list": [1], - "update_kind": "sparse_flat", - }, - ) - - assert received_kinds == ["sparse_flat"] - assert len(applied_patches) == 1 - assert torch.equal(applied_patches[0].indices, torch.tensor([1], dtype=torch.int32)) + worker.weight_transfer_engine = engine + worker._weight_update_active = False + return worker -def test_update_weights_sparse_rejects_tp_or_pp(monkeypatch): - monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None) +def test_start_update_finish_delegates_to_engine(): + engine = _RecordingEngine() + worker = _make_worker(engine) - worker = object.__new__(Worker) - worker.device = "cpu" - worker.parallel_config = SimpleNamespace(world_size=2) - worker.weight_transfer_engine = _make_nccl_engine() - worker._weight_update_active = True - worker._is_checkpoint_format = False - worker.model_runner = SimpleNamespace(apply_sparse_weight_patches=lambda _: None) + Worker.start_weight_update(worker) + assert engine.started is True + assert worker._weight_update_active is True - with pytest.raises(NotImplementedError, match="TP=1 and PP=1"): - Worker.update_weights( - worker, - { - "names": ["layer.weight"], - "dtype_names": ["float32"], - "shapes": [[4]], - "num_updates_list": [1], - "update_kind": "sparse_flat", - }, - ) + Worker.update_weights(worker, {"names": ["w"]}) + assert engine.update_calls == [{"names": ["w"]}] + assert worker._weight_update_active is True + + Worker.finish_weight_update(worker) + assert engine.finished is True assert worker._weight_update_active is False - assert worker._is_checkpoint_format is True -def test_update_weights_sparse_rejects_checkpoint_format(monkeypatch): - monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None) +def test_double_start_raises(): + worker = _make_worker(_RecordingEngine()) + Worker.start_weight_update(worker) + with pytest.raises(RuntimeError, match="already"): + Worker.start_weight_update(worker) - worker = object.__new__(Worker) - worker.device = "cpu" - worker.parallel_config = SimpleNamespace(world_size=1) - worker.weight_transfer_engine = _make_nccl_engine() - worker._weight_update_active = True - worker._is_checkpoint_format = True - worker.model_runner = SimpleNamespace(model=MagicMock()) - with pytest.raises(ValueError, match="start_weight_update"): - Worker.update_weights( - worker, - { - "names": ["layer.weight"], - "dtype_names": ["float32"], - "shapes": [[4]], - "num_updates_list": [1], - "update_kind": "sparse_flat", - }, - ) +def test_update_without_start_raises(): + worker = _make_worker(_RecordingEngine()) + with pytest.raises(RuntimeError, match="start_weight_update must be called"): + Worker.update_weights(worker, {"names": ["w"]}) + + +def test_finish_without_start_raises(): + worker = _make_worker(_RecordingEngine()) + with pytest.raises(RuntimeError, match="without a matching"): + Worker.finish_weight_update(worker) + + +def test_update_resets_active_on_error(): + engine = _RecordingEngine(raise_on_update=True) + worker = _make_worker(engine) + Worker.start_weight_update(worker) + + with pytest.raises(ValueError, match="boom"): + Worker.update_weights(worker, {"names": ["w"]}) + + # A failed update ends the session so the next start is clean. assert worker._weight_update_active is False - assert worker._is_checkpoint_format is True -def test_update_weights_resets_state_when_update_info_is_invalid(monkeypatch): - monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None) - - worker = object.__new__(Worker) - worker.device = "cpu" - worker.parallel_config = SimpleNamespace(world_size=1) - worker.weight_transfer_engine = _make_nccl_engine() - worker._weight_update_active = True - worker._is_checkpoint_format = False - - with pytest.raises(ValueError, match="cannot be empty"): - Worker.update_weights( - worker, - { - "names": [], - "dtype_names": [], - "shapes": [], - "num_updates_list": [], - "update_kind": "sparse_flat", - }, - ) - assert worker._weight_update_active is False - assert worker._is_checkpoint_format is True +def test_missing_engine_raises(): + worker = _make_worker(None) + with pytest.raises(RuntimeError, match="Weight transfer not configured"): + Worker.start_weight_update(worker) diff --git a/vllm/config/weight_transfer.py b/vllm/config/weight_transfer.py index 86e75c7965..c1add61411 100644 --- a/vllm/config/weight_transfer.py +++ b/vllm/config/weight_transfer.py @@ -9,7 +9,7 @@ from vllm.config.utils import config class WeightTransferConfig: """Configuration for weight transfer during RL training.""" - backend: Literal["nccl", "ipc"] | str = "nccl" + backend: Literal["nccl", "ipc", "sparse_nccl"] | str = "nccl" """The backend to use for weight transfer. Validated against the `WeightTransferEngineFactory` registry at engine creation time. """ diff --git a/vllm/distributed/weight_transfer/base.py b/vllm/distributed/weight_transfer/base.py index eda209c3f6..5ed99ec546 100644 --- a/vllm/distributed/weight_transfer/base.py +++ b/vllm/distributed/weight_transfer/base.py @@ -3,12 +3,15 @@ """Base class for weight transfer engines.""" from abc import ABC, abstractmethod -from collections.abc import Callable, Iterator -from dataclasses import KW_ONLY, dataclass, field -from typing import Any, Generic, Literal, TypeVar +from collections.abc import Iterator +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Generic, TypeVar import torch +if TYPE_CHECKING: + from vllm.config import VllmConfig + from vllm.config.parallel import ParallelConfig from vllm.config.weight_transfer import WeightTransferConfig @@ -28,44 +31,7 @@ class WeightTransferInitInfo(ABC): # noqa: B024 class WeightTransferUpdateInfo(ABC): # noqa: B024 """Base class for backend-specific weight update info.""" - _: KW_ONLY - update_kind: Literal["dense", "sparse_flat"] = "dense" - """Weight update format.""" - num_updates_list: list[int] | None = None - """Number of sparse entries to receive for each parameter in ``names``.""" - - def __post_init__(self) -> None: - if self.update_kind not in ("dense", "sparse_flat"): - raise ValueError(f"Unsupported update_kind: {self.update_kind}") - if self.update_kind == "dense": - if self.num_updates_list is not None: - raise ValueError( - "Sparse metadata is only supported for `update_kind='sparse_flat'`" - ) - return - - if self.num_updates_list is None: - raise ValueError("`num_updates_list` is required for sparse updates") - if len(self.num_updates_list) == 0: - raise ValueError("`num_updates_list` cannot be empty for sparse updates") - if any(num_updates < 0 for num_updates in self.num_updates_list): - raise ValueError("Sparse `num_updates_list` entries must be non-negative") - - names = getattr(self, "names", None) - if names is not None and len(self.num_updates_list) != len(names): - raise ValueError( - f"`num_updates_list` should be of the same size as `names`: " - f"got {len(self.num_updates_list)} and {len(names)}" - ) - - -@dataclass -class SparseWeightPatch: - """A sparse in-place patch for one existing parameter.""" - - name: str - indices: torch.Tensor - values: torch.Tensor + pass # API-level request classes (accept dicts for backend-agnostic serialization) @@ -89,9 +55,15 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]): from a trainer to inference workers. This abstraction separates weight transfer transport logic from the worker - implementation, allowing different backends (NCCL, CUDA IPC[TODO], RDMA[TODO]) to be + implementation, allowing different backends (NCCL, CUDA IPC, RDMA[TODO]) to be plugged in. + Each engine owns its full weight-update lifecycle: `start_weight_update`, + `update_weights`, and `finish_weight_update`. Layerwise reloading (used by + checkpoint-format engines) is opted into per engine by running it inside + `start_weight_update`/`finish_weight_update`. Engines that apply weights in + place (e.g. sparse patches) leave those methods as no-ops. + Subclasses should define: init_info_cls: Type of backend-specific initialization info update_info_cls: Type of backend-specific update info @@ -104,7 +76,8 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]): def __init__( self, config: WeightTransferConfig, - parallel_config: ParallelConfig, + vllm_config: "VllmConfig", + device: torch.device, model: torch.nn.Module, ) -> None: """ @@ -112,11 +85,15 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]): Args: config: The configuration for the weight transfer engine - parallel_config: The configuration for the parallel setup + vllm_config: The full vLLM config (provides parallel/model config) + device: The device this worker's model lives on model: The local model instance which will receive the weights """ self.config = config - self.parallel_config = parallel_config + self.vllm_config = vllm_config + self.parallel_config: ParallelConfig = vllm_config.parallel_config + self.model_config = vllm_config.model_config + self.device = device self.model = model def parse_init_info(self, init_dict: dict[str, Any]) -> TInitInfo: @@ -171,32 +148,50 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]): raise NotImplementedError @abstractmethod - def receive_weights( - self, - update_info: TUpdateInfo, - load_weights: Callable[[list[tuple[str, torch.Tensor]]], None], - ) -> None: + def start_weight_update(self) -> None: """ - Receive weights from the trainer and load them incrementally. + Prepare the engine for a new weight update. + + Engines that receive weights in checkpoint format initialize layerwise reloading + here, else this is typically a no-op. + See: https://docs.vllm.ai/en/latest/training/layerwise/ for more details. + """ + raise NotImplementedError + + @abstractmethod + def finish_weight_update(self) -> None: + """ + Finalize the current weight update. + + Checkpoint-format engines finalize layerwise reloading here; engines + that apply weights in place leave this as a no-op. + """ + raise NotImplementedError + + def update_weights(self, update_info: dict[str, Any]) -> None: + """ + Receive one weight update chunk and load it into the model. + + Args: + update_info: Dictionary containing backend-specific update info + """ + typed_update_info = self.parse_update_info(update_info) + self.receive_weights(typed_update_info) + # NCCL broadcast / IPC paths may be asynchronous. Synchronize here so the + # next step uses the new weights. + torch.accelerator.synchronize() + + @abstractmethod + def receive_weights(self, update_info: TUpdateInfo) -> None: + """ + Receive weights from the trainer and load them into the model. Args: update_info: Backend-specific update info containing parameter metadata and any backend-specific data - load_weights: Callable that loads weights into the model. Called - incrementally for each weight to avoid OOM. """ raise NotImplementedError - def receive_sparse_weights( - self, - update_info: TUpdateInfo, - apply_patches: Callable[[list[SparseWeightPatch]], None], - ) -> None: - """Receive sparse weight patches from the trainer.""" - raise NotImplementedError( - f"{self.__class__.__name__} does not support sparse weight updates" - ) - @abstractmethod def shutdown(self) -> None: """ @@ -208,7 +203,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]): @staticmethod @abstractmethod def trainer_send_weights( - iterator: Iterator[tuple[str, torch.Tensor]], + iterator: Iterator[Any], trainer_args: dict[str, Any] | Any, ) -> None: """ @@ -218,8 +213,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]): to send weights to all inference workers. Args: - iterator: Iterator of model parameters. Returns (name, tensor) tuples. - The tensors should be on the appropriate device for the backend. + iterator: Iterator of backend-specific items to send. trainer_args: Dictionary containing backend-specific arguments needed to send weights. The structure depends on the backend: - NCCL: Contains 'group', 'src', 'packed', etc. @@ -231,11 +225,3 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]): >>> engine.trainer_send_weights(param_iter, trainer_args) """ raise NotImplementedError - - @staticmethod - def trainer_send_sparse_weights( - _iterator: Iterator[SparseWeightPatch], - _trainer_args: dict[str, Any] | Any, - ) -> None: - """Send sparse weight patches from trainer to inference workers.""" - raise NotImplementedError("Sparse weight updates are not supported") diff --git a/vllm/distributed/weight_transfer/factory.py b/vllm/distributed/weight_transfer/factory.py index 791401c7c2..a253363d73 100644 --- a/vllm/distributed/weight_transfer/factory.py +++ b/vllm/distributed/weight_transfer/factory.py @@ -12,7 +12,7 @@ from vllm.logger import init_logger if TYPE_CHECKING: import torch - from vllm.config.parallel import ParallelConfig + from vllm.config import VllmConfig from vllm.config.weight_transfer import WeightTransferConfig logger = init_logger(__name__) @@ -76,14 +76,16 @@ class WeightTransferEngineFactory: def create_engine( cls, config: "WeightTransferConfig", - parallel_config: "ParallelConfig", + vllm_config: "VllmConfig", + device: "torch.device", model: "torch.nn.Module", ) -> WeightTransferEngine: """Create a weight transfer engine instance. Args: config: Weight transfer configuration containing the backend name - parallel_config: Parallel configuration for the engine + vllm_config: The full vLLM config (provides parallel/model config) + device: The device this worker's model lives on model: The local model instance which will receive the weights Returns: @@ -106,7 +108,7 @@ class WeightTransferEngineFactory: engine_cls.__name__, ) - return engine_cls(config, parallel_config, model) + return engine_cls(config, vllm_config, device, model) # Register built-in weight transfer engines here. @@ -124,3 +126,9 @@ WeightTransferEngineFactory.register_engine( "vllm.distributed.weight_transfer.ipc_engine", "IPCWeightTransferEngine", ) + +WeightTransferEngineFactory.register_engine( + "sparse_nccl", + "vllm.distributed.weight_transfer.sparse_nccl_engine", + "SparseNCCLWeightTransferEngine", +) diff --git a/vllm/distributed/weight_transfer/ipc_engine.py b/vllm/distributed/weight_transfer/ipc_engine.py index a77aab751f..f1b6070893 100644 --- a/vllm/distributed/weight_transfer/ipc_engine.py +++ b/vllm/distributed/weight_transfer/ipc_engine.py @@ -5,7 +5,7 @@ import pickle from collections.abc import Callable, Iterator from dataclasses import asdict, dataclass -from typing import Any +from typing import TYPE_CHECKING, Any import pybase64 as base64 import ray @@ -14,13 +14,15 @@ import torch from torch.multiprocessing.reductions import rebuild_cuda_tensor, reduce_tensor from vllm import envs -from vllm.config.parallel import ParallelConfig from vllm.config.weight_transfer import WeightTransferConfig from vllm.distributed.weight_transfer.base import ( WeightTransferEngine, WeightTransferInitInfo, WeightTransferUpdateInfo, ) + +if TYPE_CHECKING: + from vllm.config import VllmConfig from vllm.distributed.weight_transfer.packed_tensor import ( DEFAULT_PACKED_BUFFER_SIZE_BYTES, packed_ipc_consumer, @@ -87,10 +89,6 @@ class IPCWeightTransferUpdateInfo(WeightTransferUpdateInfo): """Whether this update uses packed tensor format.""" def __post_init__(self): - super().__post_init__() - if self.update_kind != "dense": - raise NotImplementedError("IPC weight transfer only supports dense updates") - if self.ipc_handles_pickled is not None: if self.ipc_handles is not None: raise ValueError( @@ -152,7 +150,8 @@ class IPCWeightTransferEngine( def __init__( self, config: WeightTransferConfig, - parallel_config: ParallelConfig, + vllm_config: "VllmConfig", + device: torch.device, model: torch.nn.Module, ) -> None: """ @@ -160,10 +159,11 @@ class IPCWeightTransferEngine( Args: config: The configuration for the weight transfer engine - parallel_config: The configuration for the parallel setup + vllm_config: The full vLLM config + device: The device this worker's model lives on model: The local model instance which will receive the weights """ - super().__init__(config, parallel_config, model) + super().__init__(config, vllm_config, device, model) def parse_update_info( self, update_dict: dict[str, Any] @@ -206,22 +206,37 @@ class IPCWeightTransferEngine( """ pass - def receive_weights( - self, - update_info: IPCWeightTransferUpdateInfo, - load_weights: Callable[[list[tuple[str, torch.Tensor]]], None], - ) -> None: + def start_weight_update(self) -> None: + """Initialize layerwise reloading for the incoming checkpoint weights.""" + from vllm.model_executor.model_loader.reload import ( + initialize_layerwise_reload, + ) + + initialize_layerwise_reload(self.model) + + def finish_weight_update(self) -> None: + """Finalize layerwise reloading after all weights have been received.""" + from vllm.model_executor.model_loader.reload import ( + finalize_layerwise_reload, + ) + + finalize_layerwise_reload(self.model, self.model_config) + + def receive_weights(self, update_info: IPCWeightTransferUpdateInfo) -> None: """ - Receive weights from the trainer via CUDA IPC handles. + Receive weights from the trainer via CUDA IPC handles and load them. Args: update_info: IPC update info containing parameter names, dtypes, shapes, and IPC handles. Each IPC handle is a mapping between physical GPU UUID and the rebuild_cuda_tensor args tuple. - load_weights: Callable that loads weights into the model. Called - incrementally for each weight to avoid OOM. """ - device_index = torch.accelerator.current_device_index() + # Use the worker's assigned device rather than the ambient current + # device: the receive path is no longer wrapped in + # `with torch.device(self.device)` by the caller, so the current device + # is not guaranteed to match self.device. The IPC tensors must be + # rebuilt on the device the model lives on. + device_index = self.device.index if update_info.packed: assert update_info.tensor_sizes is not None @@ -234,7 +249,6 @@ class IPCWeightTransferEngine( tensor_sizes=update_info.tensor_sizes, device_index=device_index, ) - load_weights(weights) else: assert isinstance(update_info.ipc_handles, list) weights = [] @@ -260,7 +274,7 @@ class IPCWeightTransferEngine( weight = rebuild_cuda_tensor(*list_args) weights.append((name, weight)) - load_weights(weights) + self.model.load_weights(weights) def shutdown(self) -> None: pass diff --git a/vllm/distributed/weight_transfer/nccl_common.py b/vllm/distributed/weight_transfer/nccl_common.py new file mode 100644 index 0000000000..fd2f2a3514 --- /dev/null +++ b/vllm/distributed/weight_transfer/nccl_common.py @@ -0,0 +1,120 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Shared NCCL initialization helpers for weight transfer engines. + +The dense (`NCCLWeightTransferEngine`) and sparse +(`SparseNCCLWeightTransferEngine`) backends are independent engines that share +*only* their process-group initialization. That common logic lives here so the +sparse engine does not have to subclass the dense one. +""" + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +import torch + +if TYPE_CHECKING: + from vllm.config.parallel import ParallelConfig + from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator + +from vllm.distributed.weight_transfer.base import WeightTransferInitInfo + + +@dataclass +class NCCLWeightTransferInitInfo(WeightTransferInitInfo): + """Initialization info for NCCL-based weight transfer backends.""" + + master_address: str + master_port: int + rank_offset: int + world_size: int + + +def stateless_init_process_group( + master_address: str, + master_port: int, + rank: int, + world_size: int, + device, +) -> "PyNcclCommunicator": + """ + vLLM provides `StatelessProcessGroup` to create a process group + without considering the global process group in torch.distributed. + It is recommended to create `StatelessProcessGroup`, and then initialize + the data-plane communication (NCCL) between external (train processes) + and vLLM workers. + """ + from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator + from vllm.distributed.utils import StatelessProcessGroup + + pg = StatelessProcessGroup.create( + host=master_address, port=master_port, rank=rank, world_size=world_size + ) + return PyNcclCommunicator(pg, device=device) + + +def worker_init_process_group( + init_info: NCCLWeightTransferInitInfo, + parallel_config: "ParallelConfig", +) -> "PyNcclCommunicator": + """Create the trainer<->worker NCCL group on an inference worker. + + Computes a unique rank for this worker across all data-parallel groups and + joins the stateless process group with the trainer. + """ + # Calculate the global rank in the trainer-worker process group. + # Must account for data parallel to get unique ranks across all workers. + dp_rank = parallel_config.data_parallel_index + world_size_per_dp = parallel_config.world_size # TP * PP + rank_within_dp = parallel_config.rank + + # Unique rank across all DP groups + worker_rank = dp_rank * world_size_per_dp + rank_within_dp + rank = worker_rank + init_info.rank_offset + + device = torch.accelerator.current_device_index() + return stateless_init_process_group( + init_info.master_address, + init_info.master_port, + rank, + init_info.world_size, + device=device, + ) + + +def trainer_init( + init_info: NCCLWeightTransferInitInfo | dict, +) -> "PyNcclCommunicator": + """ + Initialize NCCL process group for trainer-side weight transfer. + + The trainer is always rank 0 in the process group. Uses the current + CUDA device (torch.accelerator.current_device_index()). + + Args: + init_info: Either an NCCLWeightTransferInitInfo object or a dict with keys: + - master_address: str + - master_port: int + - world_size: int + + Returns: + PyNcclCommunicator for weight transfer. + """ + if isinstance(init_info, dict): + master_address = init_info["master_address"] + master_port = init_info["master_port"] + world_size = init_info["world_size"] + else: + master_address = init_info.master_address + master_port = init_info.master_port + world_size = init_info.world_size + + # Trainer is always rank 0 + device = torch.accelerator.current_device_index() + return stateless_init_process_group( + master_address, + master_port, + 0, + world_size, + device, + ) diff --git a/vllm/distributed/weight_transfer/nccl_engine.py b/vllm/distributed/weight_transfer/nccl_engine.py index 674f5b524d..5838a8ba8a 100644 --- a/vllm/distributed/weight_transfer/nccl_engine.py +++ b/vllm/distributed/weight_transfer/nccl_engine.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""NCCL-based weight transfer engine.""" +"""NCCL-based (dense) weight transfer engine.""" from collections.abc import Callable, Iterator from dataclasses import dataclass @@ -9,31 +9,32 @@ from typing import TYPE_CHECKING, Any import torch if TYPE_CHECKING: + from vllm.config import VllmConfig from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator -from vllm.config.parallel import ParallelConfig from vllm.config.weight_transfer import WeightTransferConfig from vllm.distributed.weight_transfer.base import ( - SparseWeightPatch, WeightTransferEngine, - WeightTransferInitInfo, WeightTransferUpdateInfo, ) +from vllm.distributed.weight_transfer.nccl_common import ( + NCCLWeightTransferInitInfo, + trainer_init, + worker_init_process_group, +) from vllm.distributed.weight_transfer.packed_tensor import ( DEFAULT_PACKED_BUFFER_SIZE_BYTES, DEFAULT_PACKED_NUM_BUFFERS, packed_nccl_broadcast_consumer, ) - -@dataclass -class NCCLWeightTransferInitInfo(WeightTransferInitInfo): - """Initialization info for NCCL weight transfer backend.""" - - master_address: str - master_port: int - rank_offset: int - world_size: int +# Re-exported for backward compatibility; canonical home is nccl_common. +__all__ = [ + "NCCLWeightTransferInitInfo", + "NCCLTrainerSendWeightsArgs", + "NCCLWeightTransferUpdateInfo", + "NCCLWeightTransferEngine", +] @dataclass @@ -82,7 +83,6 @@ class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo): def __post_init__(self): """Validate that all lists have the same length.""" - super().__post_init__() num_params = len(self.names) if len(self.dtype_names) != num_params: raise ValueError( @@ -94,13 +94,6 @@ class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo): f"`shapes` should be of the same size as `names`: " f"got {len(self.shapes)} and {len(self.names)}" ) - if self.update_kind == "dense": - return - - if self.packed: - raise ValueError( - "`update_kind='sparse_flat'` cannot be combined with `packed=True`" - ) class NCCLWeightTransferEngine( @@ -109,8 +102,10 @@ class NCCLWeightTransferEngine( """ Weight transfer engine using NCCL for communication between trainer and workers. - This implementation uses NCCL broadcast operations to transfer weights from - the trainer (rank 0) to all inference workers in a process group. + This implementation uses NCCL broadcast operations to transfer dense + checkpoint-format weights from the trainer (rank 0) to all inference workers + in a process group. Received weights are loaded via the model's + `load_weights` using the layerwise reload lifecycle. """ # Define backend-specific dataclass types @@ -120,18 +115,11 @@ class NCCLWeightTransferEngine( def __init__( self, config: WeightTransferConfig, - parallel_config: ParallelConfig, + vllm_config: "VllmConfig", + device: torch.device, model: torch.nn.Module, ) -> None: - """ - Initialize the NCCL weight transfer engine. - - Args: - config: The configuration for the weight transfer engine - parallel_config: The configuration for the parallel setup - model: The local model instance which will receive the weights - """ - super().__init__(config, parallel_config, model) + super().__init__(config, vllm_config, device, model) self.model_update_group: PyNcclCommunicator | None = None def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None: @@ -142,35 +130,29 @@ class NCCLWeightTransferEngine( init_info: NCCL initialization info containing master address, port, rank offset, and world size """ - - # Calculate the global rank in the trainer-worker process group - # Must account for data parallel to get unique ranks across all workers - dp_rank = self.parallel_config.data_parallel_index - world_size_per_dp = self.parallel_config.world_size # TP * PP - rank_within_dp = self.parallel_config.rank - - # Unique rank across all DP groups - worker_rank = dp_rank * world_size_per_dp + rank_within_dp - rank = worker_rank + init_info.rank_offset - # Create stateless process group - device = torch.accelerator.current_device_index() - self.model_update_group = ( - NCCLWeightTransferEngine._stateless_init_process_group( - init_info.master_address, - init_info.master_port, - rank, - init_info.world_size, - device=device, - ) + self.model_update_group = worker_init_process_group( + init_info, self.parallel_config ) - def receive_weights( - self, - update_info: NCCLWeightTransferUpdateInfo, - load_weights: Callable[[list[tuple[str, torch.Tensor]]], None], - ) -> None: + def start_weight_update(self) -> None: + """Initialize layerwise reloading for the incoming checkpoint weights.""" + from vllm.model_executor.model_loader.reload import ( + initialize_layerwise_reload, + ) + + initialize_layerwise_reload(self.model) + + def finish_weight_update(self) -> None: + """Finalize layerwise reloading after all weights have been received.""" + from vllm.model_executor.model_loader.reload import ( + finalize_layerwise_reload, + ) + + finalize_layerwise_reload(self.model, self.model_config) + + def receive_weights(self, update_info: NCCLWeightTransferUpdateInfo) -> None: """ - Receive weights from trainer via NCCL broadcast and load them incrementally. + Receive weights from trainer via NCCL broadcast. If update_info.packed is True, uses packed tensor broadcasting for efficient transfer of multiple weights in batches. Otherwise, uses simple @@ -179,19 +161,12 @@ class NCCLWeightTransferEngine( Args: update_info: NCCL update info containing parameter names, dtypes, shapes, and packed flag - load_weights: Callable that loads weights into the model. Called - incrementally for each batch of weights to avoid OOM. """ if self.model_update_group is None: raise RuntimeError( "NCCL weight transfer not initialized. " "Call init_transfer_engine() first." ) - if update_info.update_kind != "dense": - raise ValueError( - "Sparse updates must use `receive_sparse_weights`, not " - "`receive_weights`" - ) if update_info.packed: # Build iterator of (name, (shape, dtype)) from update_info @@ -206,9 +181,10 @@ class NCCLWeightTransferEngine( iterator=state_dict_info_iterator(), group=self.model_update_group, src=0, - post_unpack_func=load_weights, + post_unpack_func=self.model.load_weights, buffer_size_bytes=update_info.packed_buffer_size_bytes, num_buffers=update_info.packed_num_buffers, + device=self.device, ) else: # Use simple one-by-one broadcasting @@ -216,49 +192,13 @@ class NCCLWeightTransferEngine( update_info.names, update_info.dtype_names, update_info.shapes ): dtype = getattr(torch, dtype_name) - weight = torch.empty(shape, dtype=dtype, device="cuda") + weight = torch.empty(shape, dtype=dtype, device=self.device) self.model_update_group.broadcast( weight, src=0, stream=torch.cuda.current_stream() ) - load_weights([(name, weight)]) + self.model.load_weights([(name, weight)]) del weight - def receive_sparse_weights( - self, - update_info: NCCLWeightTransferUpdateInfo, - apply_patches: Callable[[list[SparseWeightPatch]], None], - ) -> None: - """Receive sparse flat-index patches from trainer via NCCL.""" - if self.model_update_group is None: - raise RuntimeError( - "NCCL weight transfer not initialized. " - "Call init_transfer_engine() first." - ) - if update_info.update_kind != "sparse_flat": - raise ValueError("Sparse receive path requires `update_kind='sparse_flat'`") - assert update_info.num_updates_list is not None - - for name, dtype_name, num_updates in zip( - update_info.names, - update_info.dtype_names, - update_info.num_updates_list, - ): - dtype = getattr(torch, dtype_name) - device = torch.accelerator.current_device_index() - indices = torch.empty(num_updates, dtype=torch.int32, device=device) - values = torch.empty(num_updates, dtype=dtype, device=device) - self.model_update_group.broadcast( - indices, src=0, stream=torch.cuda.current_stream() - ) - self.model_update_group.broadcast( - values, src=0, stream=torch.cuda.current_stream() - ) - apply_patches( - [SparseWeightPatch(name=name, indices=indices, values=values)] - ) - del indices - del values - def shutdown(self) -> None: if self.model_update_group is not None: # Clean up the communicator by removing the reference @@ -269,7 +209,7 @@ class NCCLWeightTransferEngine( iterator: Iterator[tuple[str, torch.Tensor]], trainer_args: dict[str, Any] | NCCLTrainerSendWeightsArgs, ) -> None: - """Broadcast weights from trainer to vLLM workers. + """Broadcast dense weights from trainer to vLLM workers. Args: iterator: Iterator of model parameters. Returns (name, tensor) tuples @@ -322,94 +262,6 @@ class NCCLWeightTransferEngine( stream=args.stream or torch.cuda.current_stream(), ) - @staticmethod - def trainer_send_sparse_weights( - iterator: Iterator[SparseWeightPatch], - trainer_args: dict[str, Any] | NCCLTrainerSendWeightsArgs, - ) -> None: - """Broadcast sparse flat-index patches from trainer to vLLM workers.""" - if isinstance(trainer_args, dict): - args = NCCLTrainerSendWeightsArgs(**trainer_args) - else: - args = trainer_args - - if args.packed: - raise ValueError( - "Sparse NCCL updates cannot be combined with `packed=True`" - ) - - stream = args.stream or torch.cuda.current_stream() - for patch in iterator: - args.group.broadcast(patch.indices, src=args.src, stream=stream) - args.group.broadcast(patch.values, src=args.src, stream=stream) - - @staticmethod - def trainer_init( - init_info: NCCLWeightTransferInitInfo | dict, - ) -> "PyNcclCommunicator": - """ - Initialize NCCL process group for trainer-side weight transfer. - - The trainer is always rank 0 in the process group. Uses the current - CUDA device (torch.accelerator.current_device_index()). - - Args: - init_info: Either an NCCLWeightTransferInitInfo object or a dict with keys: - - master_address: str - - master_port: int - - world_size: int - - Returns: - PyNcclCommunicator for weight transfer. - - Example: - >>> from vllm.distributed.weight_transfer.nccl_engine import ( - ... NCCLWeightTransferEngine, - ... ) - >>> group = NCCLWeightTransferEngine.trainer_init( - ... dict( - ... master_address=master_address, - ... master_port=master_port, - ... world_size=world_size, - ... ), - ... ) - """ - if isinstance(init_info, dict): - master_address = init_info["master_address"] - master_port = init_info["master_port"] - world_size = init_info["world_size"] - else: - # NCCLWeightTransferInitInfo object - master_address = init_info.master_address - master_port = init_info.master_port - world_size = init_info.world_size - - # Trainer is always rank 0 - device = torch.accelerator.current_device_index() - return NCCLWeightTransferEngine._stateless_init_process_group( - master_address, - master_port, - 0, - world_size, - device, - ) - - @staticmethod - def _stateless_init_process_group( - master_address, master_port, rank, world_size, device - ): - """ - vLLM provides `StatelessProcessGroup` to create a process group - without considering the global process group in torch.distributed. - It is recommended to create `StatelessProcessGroup`, and then initialize - the data-plane communication (NCCL) between external (train processes) - and vLLM workers. - """ - from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator - from vllm.distributed.utils import StatelessProcessGroup - - pg = StatelessProcessGroup.create( - host=master_address, port=master_port, rank=rank, world_size=world_size - ) - pynccl = PyNcclCommunicator(pg, device=device) - return pynccl + # Trainer-side process-group setup. Delegates to the shared helper so the + # sparse engine can reuse the exact same rendezvous without subclassing. + trainer_init = staticmethod(trainer_init) diff --git a/vllm/distributed/weight_transfer/packed_tensor.py b/vllm/distributed/weight_transfer/packed_tensor.py index 1001eba2e8..ce42c691f4 100644 --- a/vllm/distributed/weight_transfer/packed_tensor.py +++ b/vllm/distributed/weight_transfer/packed_tensor.py @@ -186,6 +186,7 @@ def packed_nccl_broadcast_consumer( post_unpack_func: Callable[[list[tuple[str, torch.Tensor]]], None], buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES, num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS, + device: torch.device | str = "cuda", ) -> None: """Consume packed tensors and unpack them into a list of tensors. @@ -199,6 +200,8 @@ def packed_nccl_broadcast_consumer( Both producer and consumer must use the same value. num_buffers: Number of buffers for double/triple buffering. Both producer and consumer must use the same value. + device: Device for the receive buffers. Must match the device the NCCL + communicator was created on (the worker's assigned device). """ target_packed_tensor_size = buffer_size_bytes @@ -211,7 +214,7 @@ def packed_nccl_broadcast_consumer( ] packing_tensor_sizes: list[int] = [0 for _ in range(num_buffers)] packed_tensors: list[torch.Tensor] = [ - torch.empty(0, dtype=torch.uint8, device="cuda") for _ in range(num_buffers) + torch.empty(0, dtype=torch.uint8, device=device) for _ in range(num_buffers) ] while True: @@ -234,7 +237,7 @@ def packed_nccl_broadcast_consumer( break # Create a packed tensor and broadcast it packed_tensors[buffer_idx] = torch.empty( - packing_tensor_sizes[buffer_idx], dtype=torch.uint8, device="cuda" + packing_tensor_sizes[buffer_idx], dtype=torch.uint8, device=device ) group.broadcast(packed_tensors[buffer_idx], src=src) # Load the packed tensor into the model @@ -259,7 +262,7 @@ def packed_nccl_broadcast_consumer( packed_tensors[buffer_idx] = torch.empty( packing_tensor_sizes[buffer_idx], dtype=torch.uint8, - device="cuda", + device=device, ) group.broadcast(packed_tensors[buffer_idx], src=src) # Load the packed tensor into the model diff --git a/vllm/distributed/weight_transfer/sparse_nccl_engine.py b/vllm/distributed/weight_transfer/sparse_nccl_engine.py new file mode 100644 index 0000000000..669b066a8e --- /dev/null +++ b/vllm/distributed/weight_transfer/sparse_nccl_engine.py @@ -0,0 +1,223 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Sparse NCCL weight transfer engine. + +A standalone engine (not a subclass of `NCCLWeightTransferEngine`) for applying +sparse, flat-index weight patches in place. It shares only NCCL process-group +initialization with the dense engine (via `nccl_common`); the update path +applies index/value patches directly to existing model parameters and never runs +layerwise reload. + +MVP limitations: +* TP=1 and PP=1 only +* uses runtime/kernel-format parameter names +* not composable with checkpoint-format or packed updates +""" + +from collections.abc import Iterator +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +import torch + +if TYPE_CHECKING: + from vllm.config import VllmConfig + from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator + +from vllm.config.weight_transfer import WeightTransferConfig +from vllm.distributed.weight_transfer.base import ( + WeightTransferEngine, + WeightTransferUpdateInfo, +) +from vllm.distributed.weight_transfer.nccl_common import ( + NCCLWeightTransferInitInfo, + trainer_init, + worker_init_process_group, +) +from vllm.distributed.weight_transfer.nccl_engine import NCCLTrainerSendWeightsArgs + +__all__ = [ + "SparseWeightPatch", + "SparseNCCLWeightTransferUpdateInfo", + "SparseNCCLWeightTransferEngine", +] + + +@dataclass +class SparseWeightPatch: + """A sparse in-place patch for one existing parameter.""" + + name: str + indices: torch.Tensor + values: torch.Tensor + + +@dataclass +class SparseNCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo): + """Update info for the sparse NCCL weight transfer backend.""" + + names: list[str] + dtype_names: list[str] + shapes: list[list[int]] + num_updates_list: list[int] + """Number of sparse entries to receive for each parameter in ``names``.""" + + def __post_init__(self) -> None: + num_params = len(self.names) + if len(self.dtype_names) != num_params: + raise ValueError( + f"`dtype_names` should be of the same size as `names`: " + f"got {len(self.dtype_names)} and {len(self.names)}" + ) + if len(self.shapes) != num_params: + raise ValueError( + f"`shapes` should be of the same size as `names`: " + f"got {len(self.shapes)} and {len(self.names)}" + ) + if len(self.num_updates_list) == 0: + raise ValueError("`num_updates_list` cannot be empty for sparse updates") + if len(self.num_updates_list) != num_params: + raise ValueError( + f"`num_updates_list` should be of the same size as `names`: " + f"got {len(self.num_updates_list)} and {len(self.names)}" + ) + if any(num_updates < 0 for num_updates in self.num_updates_list): + raise ValueError("Sparse `num_updates_list` entries must be non-negative") + + +class SparseNCCLWeightTransferEngine( + WeightTransferEngine[NCCLWeightTransferInitInfo, SparseNCCLWeightTransferUpdateInfo] +): + """ + Sparse weight transfer engine using NCCL. + + Receives flat-index (indices, values) patches broadcast from the trainer + (rank 0) and applies them in place to existing model parameters. Weights are + applied directly without layerwise reload, so `start_weight_update` and + `finish_weight_update` are no-ops. + """ + + init_info_cls = NCCLWeightTransferInitInfo + update_info_cls = SparseNCCLWeightTransferUpdateInfo + + def __init__( + self, + config: WeightTransferConfig, + vllm_config: "VllmConfig", + device: torch.device, + model: torch.nn.Module, + ) -> None: + super().__init__(config, vllm_config, device, model) + self.model_update_group: PyNcclCommunicator | None = None + + def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None: + """Initialize the NCCL process group with the trainer.""" + self.model_update_group = worker_init_process_group( + init_info, self.parallel_config + ) + + def start_weight_update(self) -> None: + """No-op: sparse patches are applied in place, no layerwise reload.""" + if self.parallel_config.world_size != 1: + raise NotImplementedError( + "Sparse weight updates currently require TP=1 and PP=1" + ) + + def finish_weight_update(self) -> None: + """No-op: sparse patches are applied in place, no layerwise reload.""" + pass + + def receive_weights(self, update_info: SparseNCCLWeightTransferUpdateInfo) -> None: + """Receive sparse flat-index patches from the trainer and apply them.""" + if self.model_update_group is None: + raise RuntimeError( + "NCCL weight transfer not initialized. " + "Call init_transfer_engine() first." + ) + + # Use the worker's assigned device rather than the ambient current + # device: the receive path is no longer wrapped in + # `with torch.device(self.device)` by the caller, so the current device + # is not guaranteed to match self.device. The recv buffers must live on + # the same device as the NCCL communicator (created on self.device). + device = self.device + for name, dtype_name, num_updates in zip( + update_info.names, + update_info.dtype_names, + update_info.num_updates_list, + ): + dtype = getattr(torch, dtype_name) + indices = torch.empty(num_updates, dtype=torch.int32, device=device) + values = torch.empty(num_updates, dtype=dtype, device=device) + self.model_update_group.broadcast( + indices, src=0, stream=torch.cuda.current_stream() + ) + self.model_update_group.broadcast( + values, src=0, stream=torch.cuda.current_stream() + ) + self._apply_patch( + SparseWeightPatch(name=name, indices=indices, values=values) + ) + del indices + del values + + def _apply_patch(self, patch: SparseWeightPatch) -> None: + """Apply a single sparse flat-index patch to an existing model param.""" + param = self.model.get_parameter(patch.name) + if not param.data.is_contiguous(): + raise NotImplementedError( + "Sparse weight updates currently require contiguous params: " + f"{patch.name}" + ) + if patch.indices.dtype != torch.int32: + raise ValueError( + f"Sparse weight updates currently require int32 indices: {patch.name}" + ) + if patch.indices.ndim != 1 or patch.values.ndim != 1: + raise ValueError( + f"Sparse weight patches must be 1D flattened updates: {patch.name}" + ) + if patch.indices.numel() != patch.values.numel(): + raise ValueError( + f"`indices` and `values` must have matching lengths for {patch.name}" + ) + if patch.values.dtype != param.dtype: + raise ValueError( + f"Sparse values dtype {patch.values.dtype} does not match " + f"parameter dtype {param.dtype} for {patch.name}" + ) + + flat_param = param.data.view(-1) + flat_param.index_copy_( + 0, + patch.indices.to(device=flat_param.device, dtype=torch.long), + patch.values.to(device=flat_param.device), + ) + + def shutdown(self) -> None: + if self.model_update_group is not None: + self.model_update_group = None + + @staticmethod + def trainer_send_weights( + iterator: Iterator[SparseWeightPatch], + trainer_args: dict[str, Any] | NCCLTrainerSendWeightsArgs, + ) -> None: + """Broadcast sparse flat-index patches from trainer to vLLM workers.""" + if isinstance(trainer_args, dict): + args = NCCLTrainerSendWeightsArgs(**trainer_args) + else: + args = trainer_args + + if args.packed: + raise ValueError( + "Sparse NCCL updates cannot be combined with `packed=True`" + ) + + stream = args.stream or torch.cuda.current_stream() + for patch in iterator: + args.group.broadcast(patch.indices, src=args.src, stream=stream) + args.group.broadcast(patch.values, src=args.src, stream=stream) + + # Trainer-side process-group setup (shared with the dense engine). + trainer_init = staticmethod(trainer_init) diff --git a/vllm/engine/protocol.py b/vllm/engine/protocol.py index 3f83734a5b..7d5cc164f0 100644 --- a/vllm/engine/protocol.py +++ b/vllm/engine/protocol.py @@ -244,7 +244,7 @@ class EngineClient(ABC): """Initialize weight transfer for RL training.""" raise NotImplementedError - async def start_weight_update(self, is_checkpoint_format: bool = True) -> None: + async def start_weight_update(self) -> None: """Start a new weight update.""" raise NotImplementedError diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index 892e5035ab..a3ed94ee0a 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -873,12 +873,9 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin): "init_weight_transfer_engine", kwargs={"init_info": init_info_dict} ) - def start_weight_update(self, is_checkpoint_format: bool = True) -> None: + def start_weight_update(self) -> None: """Start a new weight update.""" - self.llm_engine.collective_rpc( - "start_weight_update", - kwargs={"is_checkpoint_format": is_checkpoint_format}, - ) + self.llm_engine.collective_rpc("start_weight_update") def update_weights(self, request: WeightTransferUpdateRequest | dict) -> None: """ diff --git a/vllm/entrypoints/serve/dev/rlhf/api_router.py b/vllm/entrypoints/serve/dev/rlhf/api_router.py index 6237de8776..310e4021eb 100644 --- a/vllm/entrypoints/serve/dev/rlhf/api_router.py +++ b/vllm/entrypoints/serve/dev/rlhf/api_router.py @@ -129,14 +129,7 @@ async def init_weight_transfer_engine(raw_request: Request): @router.post("/start_weight_update") async def start_weight_update(raw_request: Request): - try: - body = await raw_request.json() - except json.JSONDecodeError as e: - raise HTTPException(status_code=400, detail="Invalid JSON format") from e # noqa: B904 - is_checkpoint_format = body.get("is_checkpoint_format", True) - await engine_client(raw_request).start_weight_update( - is_checkpoint_format=is_checkpoint_format - ) + await engine_client(raw_request).start_weight_update() return JSONResponse(content={"message": "Weight update started"}) diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index 419e15163a..61f02092bd 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -1080,12 +1080,9 @@ class AsyncLLM(EngineClient): "init_weight_transfer_engine", kwargs={"init_info": init_info_dict} ) - async def start_weight_update(self, is_checkpoint_format: bool = True) -> None: + async def start_weight_update(self) -> None: """Start a new weight update.""" - await self.collective_rpc( - "start_weight_update", - kwargs={"is_checkpoint_format": is_checkpoint_format}, - ) + await self.collective_rpc("start_weight_update") async def update_weights(self, request: WeightTransferUpdateRequest) -> None: """ diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index f94d96c833..58509f72d1 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -377,12 +377,6 @@ class GPUModelRunner(LoRAModelRunnerMixin): GPUModelRunnerV1.reload_weights(self, *args, **kwargs) # type: ignore[arg-type] - def apply_sparse_weight_patches(self, *args, **kwargs) -> None: - # TODO: Use full version instead of import when fully migrated to v2 - from vllm.v1.worker.gpu_model_runner import GPUModelRunner as GPUModelRunnerV1 - - GPUModelRunnerV1.apply_sparse_weight_patches(self, *args, **kwargs) # type: ignore[arg-type] - def update_config(self, *args, **kwargs) -> None: # TODO(Wentao): Use full version instead of import when fully migrated to v2 from vllm.v1.worker.gpu_model_runner import GPUModelRunner as GPUModelRunnerV1 diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 84d989827c..37e0c9f80a 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -49,7 +49,6 @@ from vllm.distributed.parallel_state import ( is_global_first_rank, prepare_communication_buffer_for_model, ) -from vllm.distributed.weight_transfer.base import SparseWeightPatch from vllm.forward_context import ( BatchDescriptor, set_forward_context, @@ -3244,44 +3243,6 @@ class GPUModelRunner( return self.model.unwrap() return self.model - def apply_sparse_weight_patches(self, patches: Iterable[SparseWeightPatch]) -> None: - """Apply sparse flat-index patches directly to existing model params.""" - model = self.get_model() - for patch in patches: - param = model.get_parameter(patch.name) - if not param.data.is_contiguous(): - raise NotImplementedError( - "Sparse weight updates currently require contiguous params: " - f"{patch.name}" - ) - - if patch.indices.dtype != torch.int32: - raise ValueError( - "Sparse weight updates currently require int32 indices: " - f"{patch.name}" - ) - if patch.indices.ndim != 1 or patch.values.ndim != 1: - raise ValueError( - f"Sparse weight patches must be 1D flattened updates: {patch.name}" - ) - if patch.indices.numel() != patch.values.numel(): - raise ValueError( - "`indices` and `values` must have matching lengths for " - f"{patch.name}" - ) - if patch.values.dtype != param.dtype: - raise ValueError( - f"Sparse values dtype {patch.values.dtype} does not match " - f"parameter dtype {param.dtype} for {patch.name}" - ) - - flat_param = param.data.view(-1) - flat_param.index_copy_( - 0, - patch.indices.to(device=flat_param.device, dtype=torch.long), - patch.values.to(device=flat_param.device), - ) - def get_supported_generation_tasks(self) -> list[GenerationTask]: model = self.get_model() supported_tasks = list[GenerationTask]() diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 554ffbaea6..28efee3dee 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -155,7 +155,6 @@ class Worker(WorkerBase): # is available, since the engine needs a reference to the model. self.weight_transfer_engine: WeightTransferEngine | None = None self._weight_update_active = False - self._is_checkpoint_format = True # Torch/CUDA profiler. Enabled and configured through profiler_config. # Profiler wrapper is created lazily in profile() when start is called, @@ -416,7 +415,8 @@ class Worker(WorkerBase): if self.vllm_config.weight_transfer_config is not None: self.weight_transfer_engine = WeightTransferEngineFactory.create_engine( self.vllm_config.weight_transfer_config, - self.vllm_config.parallel_config, + self.vllm_config, + self.device, self.model_runner.get_model(), ) @@ -1161,16 +1161,16 @@ class Worker(WorkerBase): typed_init_info = self.weight_transfer_engine.parse_init_info(init_info) self.weight_transfer_engine.init_transfer_engine(typed_init_info) - def start_weight_update(self, is_checkpoint_format: bool = True) -> None: + def start_weight_update(self) -> None: """ Start a new weight update session. - Args: - is_checkpoint_format: Whether incoming weights are in checkpoint - format (need layerwise processing) or kernel format (direct - copy / sparse patch application). + Delegates engine-specific preparation (e.g. layerwise reload setup) to + the configured weight transfer engine. The worker only tracks that a + session is active. """ self._check_weight_transfer_engine() + assert self.weight_transfer_engine is not None if self._weight_update_active: raise RuntimeError( @@ -1178,16 +1178,7 @@ class Worker(WorkerBase): "active. Call finish_weight_update first." ) - if is_checkpoint_format: - from vllm.model_executor.model_loader.reload import ( - initialize_layerwise_reload, - ) - - model = self.model_runner.model - with torch.device(self.device): - initialize_layerwise_reload(model) - - self._is_checkpoint_format = is_checkpoint_format + self.weight_transfer_engine.start_weight_update() self._weight_update_active = True def update_weights(self, update_info: dict) -> None: @@ -1208,82 +1199,24 @@ class Worker(WorkerBase): "start_weight_update must be called before update_weights." ) - update_succeeded = False try: - # Parse dict into backend-specific typed dataclass - typed_update_info = self.weight_transfer_engine.parse_update_info( - update_info - ) - - with torch.device(self.device): - if self._is_checkpoint_format: - if typed_update_info.update_kind != "dense": - raise ValueError( - "Sparse weight updates require " - "`start_weight_update(is_checkpoint_format=False)`." - ) - - model = self.model_runner.model - - # Use layerwise reload pattern for checkpoint format weights - self.weight_transfer_engine.receive_weights( - typed_update_info, - load_weights=model.load_weights, - ) - elif typed_update_info.update_kind == "sparse_flat": - if self.parallel_config.world_size != 1: - raise NotImplementedError( - "Sparse weight updates currently require TP=1 and PP=1" - ) - self.weight_transfer_engine.receive_sparse_weights( - typed_update_info, - apply_patches=self.model_runner.apply_sparse_weight_patches, - ) - else: - model = self.model_runner.model - - # Weights are already in kernel format, copy directly. - def load_weights_direct( - weights: list[tuple[str, torch.Tensor]], - ) -> None: - for name, weight in weights: - param = model.get_parameter(name) - param.copy_(weight) - - self.weight_transfer_engine.receive_weights( - typed_update_info, - load_weights=load_weights_direct, - ) - - # NCCL broadcast/packed path are asynchronous. - # Sync here so the next step uses the new weights. - torch.accelerator.synchronize() - update_succeeded = True - finally: - if not update_succeeded: - self._weight_update_active = False - self._is_checkpoint_format = True + self.weight_transfer_engine.update_weights(update_info) + except BaseException: + self._weight_update_active = False + raise def finish_weight_update(self) -> None: """Finish the current weight update session.""" self._check_weight_transfer_engine() + assert self.weight_transfer_engine is not None if not self._weight_update_active: raise RuntimeError( "finish_weight_update called without a matching start_weight_update." ) - if self._is_checkpoint_format: - from vllm.model_executor.model_loader.reload import ( - finalize_layerwise_reload, - ) - - model = self.model_runner.model - with torch.device(self.device): - finalize_layerwise_reload(model, self.model_config) - + self.weight_transfer_engine.finish_weight_update() self._weight_update_active = False - self._is_checkpoint_format = True def shutdown(self) -> None: gc.unfreeze()