[RL] Release weight-transfer communicators without stopping inference (#61060)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: fh3 <a@b>
This commit is contained in:
@@ -218,6 +218,27 @@ class TestSparseNCCLWeightTransferUpdateInfoValidation:
|
||||
# --- Unit Tests: Engine Parsing ---
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"engine_type",
|
||||
[
|
||||
NCCLWeightTransferEngine,
|
||||
NCCLTrainerWeightTransferEngine,
|
||||
SparseNCCLWeightTransferEngine,
|
||||
SparseNCCLTrainerWeightTransferEngine,
|
||||
],
|
||||
)
|
||||
def test_nccl_shutdown_destroys_communicator_once(engine_type):
|
||||
engine = object.__new__(engine_type)
|
||||
group = MagicMock()
|
||||
engine.model_update_group = group
|
||||
|
||||
engine.shutdown()
|
||||
engine.shutdown()
|
||||
|
||||
group.destroy.assert_called_once_with()
|
||||
assert engine.model_update_group is None
|
||||
|
||||
|
||||
class TestNCCLEngineParsing:
|
||||
"""Test NCCLWeightTransferEngine parsing methods."""
|
||||
|
||||
@@ -1292,6 +1313,9 @@ class TestTrainerClients:
|
||||
handle.finish_weight_update.remote.assert_called_once_with()
|
||||
handle.update_weight_version.remote.assert_called_once_with("step-42")
|
||||
|
||||
client.shutdown_weight_transfer_engine()
|
||||
handle.shutdown_weight_transfer_engine.remote.assert_called_once_with()
|
||||
|
||||
def test_http_client_pickles_ipc_handles_for_json(self, monkeypatch):
|
||||
"""HTTP update_weights must encode raw ipc_handles as a base64 pickle."""
|
||||
captured = {}
|
||||
@@ -1322,6 +1346,7 @@ class TestTrainerClients:
|
||||
captured = {}
|
||||
|
||||
def fake_post(self, path, json=None):
|
||||
captured["path"] = path
|
||||
captured["json"] = json
|
||||
|
||||
monkeypatch.setattr(HTTPVLLMWeightSyncClient, "_post", fake_post)
|
||||
@@ -1333,6 +1358,9 @@ class TestTrainerClients:
|
||||
client.finish_weight_update("step-42")
|
||||
assert captured["json"] == {"weight_version": "step-42"}
|
||||
|
||||
client.shutdown_weight_transfer_engine()
|
||||
assert captured == {"path": "shutdown_weight_transfer_engine", "json": None}
|
||||
|
||||
|
||||
class TestModuleSource:
|
||||
"""`ModuleSource` metadata vs. materialized iteration (dense, no GPU)."""
|
||||
|
||||
@@ -46,7 +46,10 @@ from vllm.distributed.weight_transfer.m2n_source import (
|
||||
mesh_from_tensor,
|
||||
placements_from_tensor,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.m2n_trainer import M2NTrainerInitInfo
|
||||
from vllm.distributed.weight_transfer.m2n_trainer import (
|
||||
M2NTrainerInitInfo,
|
||||
M2NTrainerWeightTransferEngine,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
@@ -57,6 +60,27 @@ from vllm.platforms import current_platform
|
||||
VALID_UID_B64 = base64.b64encode(b"\x00" * 128).decode()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"engine_type,group_attribute",
|
||||
[
|
||||
(M2NWeightTransferEngine, "model_update_group"),
|
||||
(M2NTrainerWeightTransferEngine, "group"),
|
||||
],
|
||||
)
|
||||
def test_shutdown_destroys_communicator_once(engine_type, group_attribute):
|
||||
engine = object.__new__(engine_type)
|
||||
engine._handle = None
|
||||
engine._executor = None
|
||||
group = Mock()
|
||||
setattr(engine, group_attribute, group)
|
||||
|
||||
engine.shutdown()
|
||||
engine.shutdown()
|
||||
|
||||
group.destroy.assert_called_once_with()
|
||||
assert getattr(engine, group_attribute) is None
|
||||
|
||||
|
||||
class TestLayout:
|
||||
def test_replicated_splits_nothing(self):
|
||||
"""A replicated tensor imposes no divisibility constraint. The shape
|
||||
|
||||
@@ -89,6 +89,11 @@ def test_invalid_field_is_rejected_before_the_engine(client, path, body):
|
||||
("/update_weights", {"update_info": {"names": []}}, "update_weights"),
|
||||
("/update_weights", {"update_info": [{}, {}]}, "update_weights"),
|
||||
("/collective_rpc", {"method": "m"}, "collective_rpc"),
|
||||
(
|
||||
"/shutdown_weight_transfer_engine",
|
||||
{},
|
||||
"shutdown_weight_transfer_engine",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_valid_body_reaches_the_engine(client, path, body, engine_call):
|
||||
|
||||
@@ -169,6 +169,13 @@ def test_init_weight_transfer_engine_calls_engine(vllm_runner):
|
||||
assert called, "init_transfer_engine should have been called"
|
||||
assert param == "hello", f"Expected 'hello', got {param}"
|
||||
|
||||
llm.shutdown_weight_transfer_engine()
|
||||
assert all(
|
||||
llm.collective_rpc(
|
||||
lambda worker: worker.weight_transfer_engine.shutdown_called
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_update_weights_calls_engine(vllm_runner):
|
||||
|
||||
@@ -75,6 +75,7 @@ def test_record_releases_in_flight_and_observes_duration(registry, fail):
|
||||
"operation,route,body",
|
||||
[
|
||||
("init", api_router.init_weight_transfer_engine, {"init_info": {}}),
|
||||
("shutdown", api_router.shutdown_weight_transfer_engine, None),
|
||||
("start", api_router.start_weight_update, None),
|
||||
("start_draft", api_router.start_draft_weight_update, None),
|
||||
("update", api_router.update_weights, {"update_info": {}}),
|
||||
|
||||
@@ -24,6 +24,7 @@ class _RecordingEngine:
|
||||
self.raise_on_update = raise_on_update
|
||||
self.started = False
|
||||
self.finished = False
|
||||
self.closed = False
|
||||
self.reset_count = 0
|
||||
self.supports_draft_weight_update = False
|
||||
self.update_calls: list[dict] = []
|
||||
@@ -49,6 +50,9 @@ class _RecordingEngine:
|
||||
def reset_weight_update_target(self) -> None:
|
||||
self.reset_count += 1
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
class _RecordingModelRunner:
|
||||
def __init__(self) -> None:
|
||||
@@ -82,6 +86,25 @@ def test_reload_weights_sets_current_config():
|
||||
assert model_runner.seen_config is worker.vllm_config
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"enabled,active", [(True, False), (True, True), (False, False)]
|
||||
)
|
||||
def test_shutdown_weight_transfer_engine_preserves_inference(enabled, active):
|
||||
engine = _RecordingEngine() if enabled else None
|
||||
worker = _make_worker(engine)
|
||||
worker._weight_update_active = active
|
||||
model_runner = worker.model_runner
|
||||
if active:
|
||||
with pytest.raises(RuntimeError, match="during a weight update"):
|
||||
worker.shutdown_weight_transfer_engine()
|
||||
else:
|
||||
worker.shutdown_weight_transfer_engine()
|
||||
if engine is not None:
|
||||
assert engine.closed is not active
|
||||
assert worker.weight_transfer_engine is engine
|
||||
assert worker.model_runner is model_runner
|
||||
|
||||
|
||||
def test_reload_parameter_lookup_preserves_lora_module_names():
|
||||
base_layer = nn.Module()
|
||||
qweight = nn.Parameter(torch.ones(1))
|
||||
|
||||
Reference in New Issue
Block a user