Add model to WeightTransferEngine.__init__ (#42922)
Signed-off-by: SumanthRH <sumanthrh99@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
552bbe6f4e
commit
3cb83c9592
@@ -100,7 +100,9 @@ class TestNCCLEngineParsing:
|
||||
"""Test parsing valid init info dict."""
|
||||
config = WeightTransferConfig(backend="nccl")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = NCCLWeightTransferEngine(config, parallel_config)
|
||||
engine = NCCLWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
|
||||
init_info = engine.parse_init_info(
|
||||
{
|
||||
@@ -121,7 +123,9 @@ class TestNCCLEngineParsing:
|
||||
"""Test parsing init info with missing required field."""
|
||||
config = WeightTransferConfig(backend="nccl")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = NCCLWeightTransferEngine(config, parallel_config)
|
||||
engine = NCCLWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="Invalid init_info"):
|
||||
engine.parse_init_info(
|
||||
@@ -135,7 +139,9 @@ class TestNCCLEngineParsing:
|
||||
"""Test parsing valid update info dict."""
|
||||
config = WeightTransferConfig(backend="nccl")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = NCCLWeightTransferEngine(config, parallel_config)
|
||||
engine = NCCLWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
|
||||
update_info = engine.parse_update_info(
|
||||
{
|
||||
@@ -161,14 +167,18 @@ class TestEngineRegistry:
|
||||
"""Test factory creates NCCL engine."""
|
||||
config = WeightTransferConfig(backend="nccl")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = WeightTransferEngineFactory.create_engine(config, parallel_config)
|
||||
engine = WeightTransferEngineFactory.create_engine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
assert isinstance(engine, NCCLWeightTransferEngine)
|
||||
|
||||
def test_create_engine_ipc(self):
|
||||
"""Test factory creates IPC engine."""
|
||||
config = WeightTransferConfig(backend="ipc")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = WeightTransferEngineFactory.create_engine(config, parallel_config)
|
||||
engine = WeightTransferEngineFactory.create_engine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
assert isinstance(engine, IPCWeightTransferEngine)
|
||||
|
||||
def test_create_engine_invalid_backend(self):
|
||||
@@ -176,7 +186,9 @@ class TestEngineRegistry:
|
||||
config = WeightTransferConfig(backend="invalid")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
with pytest.raises(ValueError, match="Invalid weight transfer backend"):
|
||||
WeightTransferEngineFactory.create_engine(config, parallel_config)
|
||||
WeightTransferEngineFactory.create_engine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
|
||||
def test_register_duplicate_raises(self):
|
||||
"""Test registering duplicate engine name raises."""
|
||||
@@ -196,7 +208,9 @@ def test_nccl_receive_weights_without_init_raises():
|
||||
|
||||
config = WeightTransferConfig(backend="nccl")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = NCCLWeightTransferEngine(config, parallel_config)
|
||||
engine = NCCLWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
|
||||
update_info = NCCLWeightTransferUpdateInfo(
|
||||
names=["w"],
|
||||
@@ -273,7 +287,9 @@ def inference_receive_tensor(
|
||||
parallel_config.data_parallel_rank = 0
|
||||
parallel_config.data_parallel_index = 0
|
||||
|
||||
engine = NCCLWeightTransferEngine(config, parallel_config)
|
||||
engine = NCCLWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
|
||||
# Initialize the engine (joins as rank 1)
|
||||
init_info = NCCLWeightTransferInitInfo(
|
||||
@@ -478,7 +494,9 @@ class TestIPCEngineParsing:
|
||||
|
||||
config = WeightTransferConfig(backend="ipc")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = IPCWeightTransferEngine(config, parallel_config)
|
||||
engine = IPCWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
|
||||
# Create dummy IPC handles
|
||||
dummy_tensor1 = torch.ones(100, 100, device="cuda:0")
|
||||
@@ -512,7 +530,9 @@ class TestIPCEngineParsing:
|
||||
|
||||
config = WeightTransferConfig(backend="ipc")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = IPCWeightTransferEngine(config, parallel_config)
|
||||
engine = IPCWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
|
||||
dummy_tensor1 = torch.ones(100, 100, device="cuda:0")
|
||||
dummy_tensor2 = torch.ones(50, device="cuda:0")
|
||||
@@ -545,7 +565,9 @@ class TestIPCEngineParsing:
|
||||
|
||||
config = WeightTransferConfig(backend="ipc")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = IPCWeightTransferEngine(config, parallel_config)
|
||||
engine = IPCWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
|
||||
dummy_tensor = torch.ones(10, 10, device="cuda:0")
|
||||
_, ipc_handle = reduce_tensor(dummy_tensor)
|
||||
@@ -635,7 +657,9 @@ def inference_receive_ipc_tensor(
|
||||
parallel_config.data_parallel_rank = 0
|
||||
parallel_config.data_parallel_index = 0
|
||||
|
||||
engine = IPCWeightTransferEngine(config, parallel_config)
|
||||
engine = IPCWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
|
||||
# Initialize the engine (no-op for IPC)
|
||||
init_info = IPCWeightTransferInitInfo()
|
||||
@@ -763,7 +787,9 @@ def test_ipc_receive_weights_missing_gpu_uuid_raises():
|
||||
|
||||
config = WeightTransferConfig(backend="ipc")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = IPCWeightTransferEngine(config, parallel_config)
|
||||
engine = IPCWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
|
||||
# Create IPC handle with wrong GPU UUID
|
||||
dummy_tensor = torch.ones(10, 10, device="cuda:0")
|
||||
|
||||
@@ -5,8 +5,10 @@ Weight transfer engines for syncing model weights from trainers
|
||||
to inference workers.
|
||||
"""
|
||||
|
||||
from vllm.distributed.weight_transfer.base import WeightTransferEngine
|
||||
from vllm.distributed.weight_transfer.factory import WeightTransferEngineFactory
|
||||
|
||||
__all__ = [
|
||||
"WeightTransferEngine",
|
||||
"WeightTransferEngineFactory",
|
||||
]
|
||||
|
||||
@@ -65,7 +65,10 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
update_info_cls: type[TUpdateInfo]
|
||||
|
||||
def __init__(
|
||||
self, config: WeightTransferConfig, parallel_config: ParallelConfig
|
||||
self,
|
||||
config: WeightTransferConfig,
|
||||
parallel_config: ParallelConfig,
|
||||
model: torch.nn.Module,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize the weight transfer engine.
|
||||
@@ -73,9 +76,11 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
Args:
|
||||
config: The configuration for the weight transfer engine
|
||||
parallel_config: The configuration for the parallel setup
|
||||
model: The local model instance which will receive the weights
|
||||
"""
|
||||
self.config = config
|
||||
self.parallel_config = parallel_config
|
||||
self.model = model
|
||||
|
||||
def parse_init_info(self, init_dict: dict[str, Any]) -> TInitInfo:
|
||||
"""
|
||||
|
||||
@@ -10,6 +10,8 @@ from vllm.distributed.weight_transfer.base import WeightTransferEngine
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
|
||||
@@ -75,12 +77,14 @@ class WeightTransferEngineFactory:
|
||||
cls,
|
||||
config: "WeightTransferConfig",
|
||||
parallel_config: "ParallelConfig",
|
||||
model: "torch.nn.Module",
|
||||
) -> WeightTransferEngine:
|
||||
"""Create a weight transfer engine instance.
|
||||
|
||||
Args:
|
||||
config: Weight transfer configuration containing the backend name
|
||||
parallel_config: Parallel configuration for the engine
|
||||
model: The local model instance which will receive the weights
|
||||
|
||||
Returns:
|
||||
An initialized weight transfer engine instance
|
||||
@@ -102,7 +106,7 @@ class WeightTransferEngineFactory:
|
||||
engine_cls.__name__,
|
||||
)
|
||||
|
||||
return engine_cls(config, parallel_config)
|
||||
return engine_cls(config, parallel_config, model)
|
||||
|
||||
|
||||
# Register built-in weight transfer engines here.
|
||||
|
||||
@@ -125,7 +125,10 @@ class IPCWeightTransferEngine(
|
||||
update_info_cls = IPCWeightTransferUpdateInfo
|
||||
|
||||
def __init__(
|
||||
self, config: WeightTransferConfig, parallel_config: ParallelConfig
|
||||
self,
|
||||
config: WeightTransferConfig,
|
||||
parallel_config: ParallelConfig,
|
||||
model: torch.nn.Module,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize the IPC weight transfer engine.
|
||||
@@ -133,8 +136,9 @@ class IPCWeightTransferEngine(
|
||||
Args:
|
||||
config: The configuration for the weight transfer engine
|
||||
parallel_config: The configuration for the parallel setup
|
||||
model: The local model instance which will receive the weights
|
||||
"""
|
||||
super().__init__(config, parallel_config)
|
||||
super().__init__(config, parallel_config, model)
|
||||
|
||||
def parse_update_info(
|
||||
self, update_dict: dict[str, Any]
|
||||
|
||||
@@ -109,7 +109,10 @@ class NCCLWeightTransferEngine(
|
||||
update_info_cls = NCCLWeightTransferUpdateInfo
|
||||
|
||||
def __init__(
|
||||
self, config: WeightTransferConfig, parallel_config: ParallelConfig
|
||||
self,
|
||||
config: WeightTransferConfig,
|
||||
parallel_config: ParallelConfig,
|
||||
model: torch.nn.Module,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize the NCCL weight transfer engine.
|
||||
@@ -117,8 +120,9 @@ class NCCLWeightTransferEngine(
|
||||
Args:
|
||||
config: The configuration for the weight transfer engine
|
||||
parallel_config: The configuration for the parallel setup
|
||||
model: The local model instance which will receive the weights
|
||||
"""
|
||||
super().__init__(config, parallel_config)
|
||||
super().__init__(config, parallel_config, model)
|
||||
self.model_update_group: PyNcclCommunicator | None = None
|
||||
|
||||
def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None:
|
||||
|
||||
@@ -36,7 +36,10 @@ from vllm.distributed.parallel_state import (
|
||||
get_pp_group,
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.distributed.weight_transfer import WeightTransferEngineFactory
|
||||
from vllm.distributed.weight_transfer import (
|
||||
WeightTransferEngine,
|
||||
WeightTransferEngineFactory,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.model_executor.warmup.kernel_warmup import kernel_warmup
|
||||
@@ -131,15 +134,9 @@ class Worker(WorkerBase):
|
||||
# Buffers saved before sleep
|
||||
self._sleep_saved_buffers: dict[str, torch.Tensor] = {}
|
||||
|
||||
# Weight transfer engine (initialized on-demand)
|
||||
self.weight_transfer_engine = (
|
||||
WeightTransferEngineFactory.create_engine(
|
||||
self.vllm_config.weight_transfer_config,
|
||||
self.vllm_config.parallel_config,
|
||||
)
|
||||
if self.vllm_config.weight_transfer_config is not None
|
||||
else None
|
||||
)
|
||||
# Weight transfer engine is created in `load_model` once the model
|
||||
# is available, since the engine needs a reference to the model.
|
||||
self.weight_transfer_engine: WeightTransferEngine | None = None
|
||||
self._weight_update_active = False
|
||||
self._is_checkpoint_format = True
|
||||
|
||||
@@ -344,6 +341,13 @@ class Worker(WorkerBase):
|
||||
):
|
||||
self.model_runner.load_model(load_dummy_weights=load_dummy_weights)
|
||||
|
||||
if self.vllm_config.weight_transfer_config is not None:
|
||||
self.weight_transfer_engine = WeightTransferEngineFactory.create_engine(
|
||||
self.vllm_config.weight_transfer_config,
|
||||
self.vllm_config.parallel_config,
|
||||
self.model_runner.get_model(),
|
||||
)
|
||||
|
||||
def update_config(self, overrides: dict[str, Any]) -> None:
|
||||
self.model_runner.update_config(overrides)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user