[Feature] Add native ModelExpress weight transfer backend (#58399)
Signed-off-by: Hyunjae Woo <hwoo@nvidia.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
@@ -6,10 +6,14 @@ Unit tests for engine classes (parsing, validation, registry).
|
||||
Integration tests for NCCL and IPC weight transfer between processes using Ray.
|
||||
"""
|
||||
|
||||
import builtins
|
||||
import pickle
|
||||
import runpy
|
||||
import threading
|
||||
import time
|
||||
from contextlib import nullcontext
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pybase64 as base64
|
||||
@@ -297,6 +301,36 @@ class TestEngineRegistry:
|
||||
)
|
||||
assert isinstance(engine, SparseNCCLWeightTransferEngine)
|
||||
|
||||
def test_modelexpress_registration_is_native_and_lazy(self, monkeypatch):
|
||||
from vllm.distributed.weight_transfer import factory
|
||||
|
||||
imported = []
|
||||
module = MagicMock()
|
||||
module.ModelExpressWeightTransferEngine.__name__ = (
|
||||
"ModelExpressWeightTransferEngine"
|
||||
)
|
||||
|
||||
def import_module(name):
|
||||
imported.append(name)
|
||||
return module
|
||||
|
||||
monkeypatch.setattr(factory.importlib, "import_module", import_module)
|
||||
config = WeightTransferConfig(backend="modelexpress")
|
||||
vllm_config = create_mock_vllm_config()
|
||||
device = torch.device("cpu")
|
||||
model = MagicMock(spec=torch.nn.Module)
|
||||
|
||||
assert imported == []
|
||||
engine = WeightTransferEngineFactory.create_engine(
|
||||
config, vllm_config, device, model
|
||||
)
|
||||
|
||||
assert imported == ["vllm.distributed.weight_transfer.modelexpress_engine"]
|
||||
module.ModelExpressWeightTransferEngine.assert_called_once_with(
|
||||
config, vllm_config, device, model
|
||||
)
|
||||
assert engine is module.ModelExpressWeightTransferEngine.return_value
|
||||
|
||||
def test_create_engine_invalid_backend(self):
|
||||
config = WeightTransferConfig(backend="invalid")
|
||||
with pytest.raises(ValueError, match="Invalid weight transfer backend"):
|
||||
@@ -1976,3 +2010,322 @@ def test_sparse_nccl_trainer_non_sender_skips_client():
|
||||
engine.send_weights([_sparse_patch()])
|
||||
engine.send_weight_chunk([_sparse_patch()])
|
||||
assert client.order == []
|
||||
|
||||
|
||||
# --- Optional ModelExpress Client Lifecycle ---
|
||||
|
||||
|
||||
def _import_modelexpress_shim():
|
||||
path = (
|
||||
Path(__file__).parents[2]
|
||||
/ "vllm/distributed/weight_transfer/modelexpress_engine.py"
|
||||
)
|
||||
runpy.run_path(str(path), run_name="_test_modelexpress_shim")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"missing_module",
|
||||
[
|
||||
"modelexpress",
|
||||
"modelexpress_rl",
|
||||
"modelexpress_rl.inference",
|
||||
"modelexpress_rl.inference.engines",
|
||||
"modelexpress_rl.inference.engines.vllm",
|
||||
"modelexpress_rl.inference.engines.vllm.weight_transfer_engine",
|
||||
],
|
||||
)
|
||||
def test_modelexpress_missing_backend_explains_source_install(
|
||||
monkeypatch, missing_module
|
||||
):
|
||||
original_import = builtins.__import__
|
||||
error = ModuleNotFoundError(name=missing_module)
|
||||
|
||||
def import_without_backend(name, *args, **kwargs):
|
||||
if name == "modelexpress":
|
||||
raise error
|
||||
return original_import(name, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(builtins, "__import__", import_without_backend)
|
||||
with pytest.raises(ImportError, match="modelexpress_client/python") as exc_info:
|
||||
_import_modelexpress_shim()
|
||||
assert exc_info.value.__cause__ is error
|
||||
assert "uv pip install" in str(exc_info.value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("missing_module", ["grpc", "modelexpress.internal_dependency"])
|
||||
def test_modelexpress_preserves_transitive_import_errors(monkeypatch, missing_module):
|
||||
original_import = builtins.__import__
|
||||
error = ModuleNotFoundError(name=missing_module)
|
||||
|
||||
def import_without_dependency(name, *args, **kwargs):
|
||||
if name == "modelexpress":
|
||||
raise error
|
||||
return original_import(name, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(builtins, "__import__", import_without_dependency)
|
||||
with pytest.raises(ModuleNotFoundError) as exc_info:
|
||||
_import_modelexpress_shim()
|
||||
assert exc_info.value is error
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mx_client(monkeypatch):
|
||||
client_module = pytest.importorskip("modelexpress_rl.inference.client")
|
||||
client = MagicMock()
|
||||
monkeypatch.setattr(
|
||||
client_module.ModelExpressGeneratorClient,
|
||||
"initialize",
|
||||
MagicMock(return_value=client),
|
||||
)
|
||||
monkeypatch.setattr(torch.accelerator, "synchronize", MagicMock())
|
||||
return client
|
||||
|
||||
|
||||
def _make_modelexpress_engine():
|
||||
return WeightTransferEngineFactory.create_engine(
|
||||
WeightTransferConfig(backend="modelexpress"),
|
||||
SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(),
|
||||
model_config=SimpleNamespace(model="test/model"),
|
||||
),
|
||||
torch.device("cpu"),
|
||||
torch.nn.Linear(2, 2),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mx_worker(mx_client, monkeypatch):
|
||||
from vllm.v1.worker import gpu_worker
|
||||
|
||||
monkeypatch.setattr(gpu_worker, "set_current_vllm_config", lambda _: nullcontext())
|
||||
worker = object.__new__(gpu_worker.Worker)
|
||||
worker.weight_transfer_engine = _make_modelexpress_engine()
|
||||
worker.vllm_config = worker.weight_transfer_engine.vllm_config
|
||||
worker._weight_update_active = False
|
||||
worker._weight_update_is_draft = False
|
||||
worker.model_runner = MagicMock()
|
||||
return worker
|
||||
|
||||
|
||||
def test_modelexpress_worker_rejects_updates_before_initialization(
|
||||
mx_worker, mx_client
|
||||
):
|
||||
with pytest.raises(RuntimeError, match="not initialized"):
|
||||
mx_worker.start_weight_update()
|
||||
with pytest.raises(RuntimeError, match="start_weight_update must be called"):
|
||||
mx_worker.update_weights({"version_id": "version-a"})
|
||||
with pytest.raises(RuntimeError, match="without a matching"):
|
||||
mx_worker.finish_weight_update()
|
||||
mx_client.stage_weight.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"second_update, error_type",
|
||||
[
|
||||
({"version_id": "version-a"}, RuntimeError),
|
||||
({"version_id": "version-b"}, RuntimeError),
|
||||
({"version_id": ""}, ValueError),
|
||||
({"version_id": None}, ValueError),
|
||||
({}, ValueError),
|
||||
],
|
||||
)
|
||||
def test_modelexpress_worker_recovers_after_rejected_update(
|
||||
mx_worker, mx_client, second_update, error_type
|
||||
):
|
||||
"""A rejected payload must release A so a fresh session can actually install B."""
|
||||
mx_worker.weight_transfer_engine.init_transfer_engine(
|
||||
mx_worker.weight_transfer_engine.parse_init_info({})
|
||||
)
|
||||
staged_a = SimpleNamespace(version_id="version-a", metrics={}, release=MagicMock())
|
||||
staged_b = SimpleNamespace(version_id="version-b", metrics={}, release=MagicMock())
|
||||
mx_client.stage_weight.side_effect = [staged_a, staged_b]
|
||||
mx_client.apply_weight.return_value = {}
|
||||
mx_worker.start_weight_update()
|
||||
mx_worker.update_weights({"version_id": "version-a"})
|
||||
|
||||
with pytest.raises(error_type):
|
||||
mx_worker.update_weights(second_update)
|
||||
staged_a.release.assert_called_once_with()
|
||||
mx_client.apply_weight.assert_called_once_with(staged_a)
|
||||
|
||||
mx_worker.start_weight_update()
|
||||
mx_worker.update_weights({"version_id": "version-b"})
|
||||
mx_worker.finish_weight_update()
|
||||
|
||||
assert [
|
||||
call.kwargs["version"].version_id
|
||||
for call in mx_client.stage_weight.call_args_list
|
||||
] == ["version-a", "version-b"]
|
||||
mx_client.apply_weight.assert_called_with(staged_b)
|
||||
staged_b.release.assert_called_once_with()
|
||||
|
||||
|
||||
def test_modelexpress_worker_can_retry_failed_finish(mx_worker, mx_client):
|
||||
mx_worker.weight_transfer_engine.init_transfer_engine(
|
||||
mx_worker.weight_transfer_engine.parse_init_info({})
|
||||
)
|
||||
staged = mx_client.stage_weight.return_value
|
||||
staged.release.side_effect = [RuntimeError("release failed"), None]
|
||||
mx_worker.start_weight_update()
|
||||
mx_worker.update_weights({"version_id": "version-a"})
|
||||
|
||||
with pytest.raises(RuntimeError, match="release failed"):
|
||||
mx_worker.finish_weight_update()
|
||||
mx_worker.finish_weight_update()
|
||||
|
||||
assert staged.release.call_count == 2
|
||||
assert not mx_worker._weight_update_active
|
||||
mx_worker.start_weight_update()
|
||||
|
||||
|
||||
def test_native_engine_applies_and_releases_exact_versions(mx_client):
|
||||
from modelexpress_rl.inference.engines.vllm import (
|
||||
weight_transfer_engine as mx_engine,
|
||||
)
|
||||
|
||||
from vllm.distributed.weight_transfer import modelexpress_engine
|
||||
|
||||
engine = _make_modelexpress_engine()
|
||||
assert type(engine) is mx_engine.ModelExpressWeightTransferEngine
|
||||
assert (
|
||||
modelexpress_engine.ModelExpressWeightTransferInitInfo
|
||||
is mx_engine.ModelExpressWeightTransferInitInfo
|
||||
)
|
||||
assert (
|
||||
modelexpress_engine.ModelExpressWeightTransferUpdateInfo
|
||||
is mx_engine.ModelExpressWeightTransferUpdateInfo
|
||||
)
|
||||
assert not engine.supports_draft_weight_update
|
||||
engine.init_transfer_engine(engine.parse_init_info({}))
|
||||
|
||||
for version_id in ("version-a", "version-b"):
|
||||
staged = SimpleNamespace(version_id=version_id, metrics={}, release=MagicMock())
|
||||
mx_client.stage_weight.return_value = staged
|
||||
mx_client.apply_weight.return_value = {}
|
||||
engine.start_weight_update()
|
||||
engine.update_weights({"version_id": version_id})
|
||||
|
||||
assert (
|
||||
mx_client.stage_weight.call_args.kwargs["version"].version_id == version_id
|
||||
)
|
||||
mx_client.apply_weight.assert_called_with(staged)
|
||||
torch.accelerator.synchronize.assert_called()
|
||||
staged.release.assert_not_called()
|
||||
engine.finish_weight_update()
|
||||
staged.release.assert_called_once_with()
|
||||
|
||||
engine.shutdown()
|
||||
engine.shutdown()
|
||||
mx_client.close.assert_called_once_with()
|
||||
|
||||
|
||||
def test_vime_init_config_reaches_mx_client(mx_client):
|
||||
from modelexpress_rl.inference.client import ModelExpressGeneratorClient
|
||||
|
||||
engine = _make_modelexpress_engine()
|
||||
ModelExpressGeneratorClient.initialize.assert_not_called()
|
||||
engine.init_transfer_engine(
|
||||
engine.parse_init_info(
|
||||
{
|
||||
"model_name": "policy",
|
||||
"server_url": "mx:8001",
|
||||
"initial_serving_version_id": "serving-a",
|
||||
"object_storage_type": "S3",
|
||||
"initial_base_version_id": "base-a",
|
||||
"seed_checkpoint_path": "/models/launch",
|
||||
"refit_checkpoint_dir": "/mxdelta/receiver",
|
||||
"refit_checkpoint_max_size_gb": 200,
|
||||
"object_storage_endpoint_url": "http://minio:9000",
|
||||
"object_storage_region_name": "us-west-2",
|
||||
"registration_ttl_seconds": 90,
|
||||
"lease_ttl_seconds": 60,
|
||||
"max_transfer_attempts": 4,
|
||||
"max_replay_chain_length": 17,
|
||||
"rpc_timeout_seconds": 12.5,
|
||||
}
|
||||
)
|
||||
)
|
||||
config = ModelExpressGeneratorClient.initialize.call_args.args[0]
|
||||
assert config.engine_context.model is engine.model
|
||||
assert config.engine_context.vllm_config is engine.vllm_config
|
||||
assert config.model_name == "policy"
|
||||
assert config.server_url == "mx:8001"
|
||||
assert config.initial_serving_version_id == "serving-a"
|
||||
assert config.registration_ttl_seconds == 90
|
||||
assert config.lease_ttl_seconds == 60
|
||||
assert config.max_transfer_attempts == 4
|
||||
assert config.max_replay_chain_length == 17
|
||||
assert config.rpc_timeout_seconds == 12.5
|
||||
storage = config.object_storage
|
||||
assert storage.storage_type.value == "S3"
|
||||
assert storage.initial_base_version_id == "base-a"
|
||||
assert storage.seed_checkpoint_path == "/models/launch"
|
||||
assert storage.refit_checkpoint_dir == "/mxdelta/receiver"
|
||||
assert storage.refit_checkpoint_max_size_gb == 200
|
||||
assert storage.endpoint_url == "http://minio:9000"
|
||||
assert storage.region_name == "us-west-2"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"key",
|
||||
[
|
||||
"object_storage_type",
|
||||
"initial_base_version_id",
|
||||
"seed_checkpoint_path",
|
||||
"refit_checkpoint_dir",
|
||||
"object_storage_endpoint_url",
|
||||
"object_storage_region_name",
|
||||
],
|
||||
)
|
||||
def test_incomplete_storage_config_rejected_before_init(mx_client, key):
|
||||
from modelexpress_rl.inference.client import ModelExpressGeneratorClient
|
||||
|
||||
engine = _make_modelexpress_engine()
|
||||
with pytest.raises(ValueError, match="object storage requires"):
|
||||
engine.init_transfer_engine(engine.parse_init_info({key: "value"}))
|
||||
ModelExpressGeneratorClient.initialize.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version_id", ["", " ", None, 1])
|
||||
def test_invalid_version_rejected_before_staging(mx_client, version_id):
|
||||
engine = _make_modelexpress_engine()
|
||||
engine.init_transfer_engine(engine.parse_init_info({}))
|
||||
engine.start_weight_update()
|
||||
with pytest.raises(ValueError, match="version_id is required"):
|
||||
engine.update_weights({"version_id": version_id})
|
||||
mx_client.stage_weight.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["stage", "apply", "release"])
|
||||
def test_update_failure_preserves_error_and_releases_handle(mx_client, failure):
|
||||
engine = _make_modelexpress_engine()
|
||||
engine.init_transfer_engine(engine.parse_init_info({}))
|
||||
staged = SimpleNamespace(version_id="version-a", metrics={}, release=MagicMock())
|
||||
mx_client.stage_weight.return_value = staged
|
||||
if failure == "stage":
|
||||
mx_client.stage_weight.side_effect = RuntimeError("stage failed")
|
||||
else:
|
||||
mx_client.apply_weight.side_effect = RuntimeError("apply failed")
|
||||
if failure == "release":
|
||||
staged.release.side_effect = RuntimeError("release failed")
|
||||
engine.start_weight_update()
|
||||
with pytest.raises(RuntimeError, match="stage failed|apply failed"):
|
||||
engine.update_weights({"version_id": "version-a"})
|
||||
assert staged.release.call_count == (0 if failure == "stage" else 1)
|
||||
torch.accelerator.synchronize.assert_not_called()
|
||||
engine.shutdown()
|
||||
mx_client.close.assert_called_once_with()
|
||||
|
||||
|
||||
def test_plugin_does_not_replace_native_backend(mx_client):
|
||||
from modelexpress.engines.vllm.registration import (
|
||||
register_plugin_weight_transfer_engine,
|
||||
)
|
||||
from modelexpress_rl.inference.engines.vllm.weight_transfer_engine import (
|
||||
ModelExpressWeightTransferEngine,
|
||||
)
|
||||
|
||||
native_loader = WeightTransferEngineFactory._registry["modelexpress"]
|
||||
register_plugin_weight_transfer_engine()
|
||||
assert WeightTransferEngineFactory._registry["modelexpress"] is native_loader
|
||||
assert type(_make_modelexpress_engine()) is ModelExpressWeightTransferEngine
|
||||
|
||||
Reference in New Issue
Block a user