From 3cb83c95922f34d0a3dc5d7795ab7b4c6ef6e54b Mon Sep 17 00:00:00 2001 From: Sumanth R Hegde <39546518+SumanthRH@users.noreply.github.com> Date: Fri, 22 May 2026 17:52:15 -0700 Subject: [PATCH] Add `model` to `WeightTransferEngine.__init__` (#42922) Signed-off-by: SumanthRH Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- tests/distributed/test_weight_transfer.py | 52 ++++++++++++++----- vllm/distributed/weight_transfer/__init__.py | 2 + vllm/distributed/weight_transfer/base.py | 7 ++- vllm/distributed/weight_transfer/factory.py | 6 ++- .../distributed/weight_transfer/ipc_engine.py | 8 ++- .../weight_transfer/nccl_engine.py | 8 ++- vllm/v1/worker/gpu_worker.py | 24 +++++---- 7 files changed, 78 insertions(+), 29 deletions(-) diff --git a/tests/distributed/test_weight_transfer.py b/tests/distributed/test_weight_transfer.py index cb1f77f59c..295e812a12 100644 --- a/tests/distributed/test_weight_transfer.py +++ b/tests/distributed/test_weight_transfer.py @@ -100,7 +100,9 @@ class TestNCCLEngineParsing: """Test parsing valid init info dict.""" config = WeightTransferConfig(backend="nccl") parallel_config = create_mock_parallel_config() - engine = NCCLWeightTransferEngine(config, parallel_config) + engine = NCCLWeightTransferEngine( + config, parallel_config, MagicMock(spec=torch.nn.Module) + ) init_info = engine.parse_init_info( { @@ -121,7 +123,9 @@ class TestNCCLEngineParsing: """Test parsing init info with missing required field.""" config = WeightTransferConfig(backend="nccl") parallel_config = create_mock_parallel_config() - engine = NCCLWeightTransferEngine(config, parallel_config) + engine = NCCLWeightTransferEngine( + config, parallel_config, MagicMock(spec=torch.nn.Module) + ) with pytest.raises(ValueError, match="Invalid init_info"): engine.parse_init_info( @@ -135,7 +139,9 @@ class TestNCCLEngineParsing: """Test parsing valid update info dict.""" config = WeightTransferConfig(backend="nccl") parallel_config = create_mock_parallel_config() - engine = NCCLWeightTransferEngine(config, parallel_config) + engine = NCCLWeightTransferEngine( + config, parallel_config, MagicMock(spec=torch.nn.Module) + ) update_info = engine.parse_update_info( { @@ -161,14 +167,18 @@ class TestEngineRegistry: """Test factory creates NCCL engine.""" config = WeightTransferConfig(backend="nccl") parallel_config = create_mock_parallel_config() - engine = WeightTransferEngineFactory.create_engine(config, parallel_config) + engine = WeightTransferEngineFactory.create_engine( + config, parallel_config, MagicMock(spec=torch.nn.Module) + ) assert isinstance(engine, NCCLWeightTransferEngine) def test_create_engine_ipc(self): """Test factory creates IPC engine.""" config = WeightTransferConfig(backend="ipc") parallel_config = create_mock_parallel_config() - engine = WeightTransferEngineFactory.create_engine(config, parallel_config) + engine = WeightTransferEngineFactory.create_engine( + config, parallel_config, MagicMock(spec=torch.nn.Module) + ) assert isinstance(engine, IPCWeightTransferEngine) def test_create_engine_invalid_backend(self): @@ -176,7 +186,9 @@ class TestEngineRegistry: config = WeightTransferConfig(backend="invalid") parallel_config = create_mock_parallel_config() with pytest.raises(ValueError, match="Invalid weight transfer backend"): - WeightTransferEngineFactory.create_engine(config, parallel_config) + WeightTransferEngineFactory.create_engine( + config, parallel_config, MagicMock(spec=torch.nn.Module) + ) def test_register_duplicate_raises(self): """Test registering duplicate engine name raises.""" @@ -196,7 +208,9 @@ def test_nccl_receive_weights_without_init_raises(): config = WeightTransferConfig(backend="nccl") parallel_config = create_mock_parallel_config() - engine = NCCLWeightTransferEngine(config, parallel_config) + engine = NCCLWeightTransferEngine( + config, parallel_config, MagicMock(spec=torch.nn.Module) + ) update_info = NCCLWeightTransferUpdateInfo( names=["w"], @@ -273,7 +287,9 @@ def inference_receive_tensor( parallel_config.data_parallel_rank = 0 parallel_config.data_parallel_index = 0 - engine = NCCLWeightTransferEngine(config, parallel_config) + engine = NCCLWeightTransferEngine( + config, parallel_config, MagicMock(spec=torch.nn.Module) + ) # Initialize the engine (joins as rank 1) init_info = NCCLWeightTransferInitInfo( @@ -478,7 +494,9 @@ class TestIPCEngineParsing: config = WeightTransferConfig(backend="ipc") parallel_config = create_mock_parallel_config() - engine = IPCWeightTransferEngine(config, parallel_config) + engine = IPCWeightTransferEngine( + config, parallel_config, MagicMock(spec=torch.nn.Module) + ) # Create dummy IPC handles dummy_tensor1 = torch.ones(100, 100, device="cuda:0") @@ -512,7 +530,9 @@ class TestIPCEngineParsing: config = WeightTransferConfig(backend="ipc") parallel_config = create_mock_parallel_config() - engine = IPCWeightTransferEngine(config, parallel_config) + engine = IPCWeightTransferEngine( + config, parallel_config, MagicMock(spec=torch.nn.Module) + ) dummy_tensor1 = torch.ones(100, 100, device="cuda:0") dummy_tensor2 = torch.ones(50, device="cuda:0") @@ -545,7 +565,9 @@ class TestIPCEngineParsing: config = WeightTransferConfig(backend="ipc") parallel_config = create_mock_parallel_config() - engine = IPCWeightTransferEngine(config, parallel_config) + engine = IPCWeightTransferEngine( + config, parallel_config, MagicMock(spec=torch.nn.Module) + ) dummy_tensor = torch.ones(10, 10, device="cuda:0") _, ipc_handle = reduce_tensor(dummy_tensor) @@ -635,7 +657,9 @@ def inference_receive_ipc_tensor( parallel_config.data_parallel_rank = 0 parallel_config.data_parallel_index = 0 - engine = IPCWeightTransferEngine(config, parallel_config) + engine = IPCWeightTransferEngine( + config, parallel_config, MagicMock(spec=torch.nn.Module) + ) # Initialize the engine (no-op for IPC) init_info = IPCWeightTransferInitInfo() @@ -763,7 +787,9 @@ def test_ipc_receive_weights_missing_gpu_uuid_raises(): config = WeightTransferConfig(backend="ipc") parallel_config = create_mock_parallel_config() - engine = IPCWeightTransferEngine(config, parallel_config) + engine = IPCWeightTransferEngine( + config, parallel_config, MagicMock(spec=torch.nn.Module) + ) # Create IPC handle with wrong GPU UUID dummy_tensor = torch.ones(10, 10, device="cuda:0") diff --git a/vllm/distributed/weight_transfer/__init__.py b/vllm/distributed/weight_transfer/__init__.py index c96ad0e3bb..af3322e0cb 100644 --- a/vllm/distributed/weight_transfer/__init__.py +++ b/vllm/distributed/weight_transfer/__init__.py @@ -5,8 +5,10 @@ Weight transfer engines for syncing model weights from trainers to inference workers. """ +from vllm.distributed.weight_transfer.base import WeightTransferEngine from vllm.distributed.weight_transfer.factory import WeightTransferEngineFactory __all__ = [ + "WeightTransferEngine", "WeightTransferEngineFactory", ] diff --git a/vllm/distributed/weight_transfer/base.py b/vllm/distributed/weight_transfer/base.py index 743971156b..6e99adde1c 100644 --- a/vllm/distributed/weight_transfer/base.py +++ b/vllm/distributed/weight_transfer/base.py @@ -65,7 +65,10 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]): update_info_cls: type[TUpdateInfo] def __init__( - self, config: WeightTransferConfig, parallel_config: ParallelConfig + self, + config: WeightTransferConfig, + parallel_config: ParallelConfig, + model: torch.nn.Module, ) -> None: """ Initialize the weight transfer engine. @@ -73,9 +76,11 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]): Args: config: The configuration for the weight transfer engine parallel_config: The configuration for the parallel setup + model: The local model instance which will receive the weights """ self.config = config self.parallel_config = parallel_config + self.model = model def parse_init_info(self, init_dict: dict[str, Any]) -> TInitInfo: """ diff --git a/vllm/distributed/weight_transfer/factory.py b/vllm/distributed/weight_transfer/factory.py index f8e9c864fc..791401c7c2 100644 --- a/vllm/distributed/weight_transfer/factory.py +++ b/vllm/distributed/weight_transfer/factory.py @@ -10,6 +10,8 @@ from vllm.distributed.weight_transfer.base import WeightTransferEngine from vllm.logger import init_logger if TYPE_CHECKING: + import torch + from vllm.config.parallel import ParallelConfig from vllm.config.weight_transfer import WeightTransferConfig @@ -75,12 +77,14 @@ class WeightTransferEngineFactory: cls, config: "WeightTransferConfig", parallel_config: "ParallelConfig", + model: "torch.nn.Module", ) -> WeightTransferEngine: """Create a weight transfer engine instance. Args: config: Weight transfer configuration containing the backend name parallel_config: Parallel configuration for the engine + model: The local model instance which will receive the weights Returns: An initialized weight transfer engine instance @@ -102,7 +106,7 @@ class WeightTransferEngineFactory: engine_cls.__name__, ) - return engine_cls(config, parallel_config) + return engine_cls(config, parallel_config, model) # Register built-in weight transfer engines here. diff --git a/vllm/distributed/weight_transfer/ipc_engine.py b/vllm/distributed/weight_transfer/ipc_engine.py index b077747560..b138c7dd93 100644 --- a/vllm/distributed/weight_transfer/ipc_engine.py +++ b/vllm/distributed/weight_transfer/ipc_engine.py @@ -125,7 +125,10 @@ class IPCWeightTransferEngine( update_info_cls = IPCWeightTransferUpdateInfo def __init__( - self, config: WeightTransferConfig, parallel_config: ParallelConfig + self, + config: WeightTransferConfig, + parallel_config: ParallelConfig, + model: torch.nn.Module, ) -> None: """ Initialize the IPC weight transfer engine. @@ -133,8 +136,9 @@ class IPCWeightTransferEngine( Args: config: The configuration for the weight transfer engine parallel_config: The configuration for the parallel setup + model: The local model instance which will receive the weights """ - super().__init__(config, parallel_config) + super().__init__(config, parallel_config, model) def parse_update_info( self, update_dict: dict[str, Any] diff --git a/vllm/distributed/weight_transfer/nccl_engine.py b/vllm/distributed/weight_transfer/nccl_engine.py index 98d0a14b07..3b04a5f65b 100644 --- a/vllm/distributed/weight_transfer/nccl_engine.py +++ b/vllm/distributed/weight_transfer/nccl_engine.py @@ -109,7 +109,10 @@ class NCCLWeightTransferEngine( update_info_cls = NCCLWeightTransferUpdateInfo def __init__( - self, config: WeightTransferConfig, parallel_config: ParallelConfig + self, + config: WeightTransferConfig, + parallel_config: ParallelConfig, + model: torch.nn.Module, ) -> None: """ Initialize the NCCL weight transfer engine. @@ -117,8 +120,9 @@ class NCCLWeightTransferEngine( Args: config: The configuration for the weight transfer engine parallel_config: The configuration for the parallel setup + model: The local model instance which will receive the weights """ - super().__init__(config, parallel_config) + super().__init__(config, parallel_config, model) self.model_update_group: PyNcclCommunicator | None = None def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None: diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 0dcf1614f4..582c6a17cb 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -36,7 +36,10 @@ from vllm.distributed.parallel_state import ( get_pp_group, get_tp_group, ) -from vllm.distributed.weight_transfer import WeightTransferEngineFactory +from vllm.distributed.weight_transfer import ( + WeightTransferEngine, + WeightTransferEngineFactory, +) from vllm.logger import init_logger from vllm.lora.request import LoRARequest from vllm.model_executor.warmup.kernel_warmup import kernel_warmup @@ -131,15 +134,9 @@ class Worker(WorkerBase): # Buffers saved before sleep self._sleep_saved_buffers: dict[str, torch.Tensor] = {} - # Weight transfer engine (initialized on-demand) - self.weight_transfer_engine = ( - WeightTransferEngineFactory.create_engine( - self.vllm_config.weight_transfer_config, - self.vllm_config.parallel_config, - ) - if self.vllm_config.weight_transfer_config is not None - else None - ) + # Weight transfer engine is created in `load_model` once the model + # is available, since the engine needs a reference to the model. + self.weight_transfer_engine: WeightTransferEngine | None = None self._weight_update_active = False self._is_checkpoint_format = True @@ -344,6 +341,13 @@ class Worker(WorkerBase): ): self.model_runner.load_model(load_dummy_weights=load_dummy_weights) + if self.vllm_config.weight_transfer_config is not None: + self.weight_transfer_engine = WeightTransferEngineFactory.create_engine( + self.vllm_config.weight_transfer_config, + self.vllm_config.parallel_config, + self.model_runner.get_model(), + ) + def update_config(self, overrides: dict[str, Any]) -> None: self.model_runner.update_config(overrides)