[RL] Release weight-transfer communicators without stopping inference (#61060)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: fh3 <a@b>
This commit is contained in:
@@ -24,6 +24,7 @@ class _RecordingEngine:
|
||||
self.raise_on_update = raise_on_update
|
||||
self.started = False
|
||||
self.finished = False
|
||||
self.closed = False
|
||||
self.reset_count = 0
|
||||
self.supports_draft_weight_update = False
|
||||
self.update_calls: list[dict] = []
|
||||
@@ -49,6 +50,9 @@ class _RecordingEngine:
|
||||
def reset_weight_update_target(self) -> None:
|
||||
self.reset_count += 1
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
class _RecordingModelRunner:
|
||||
def __init__(self) -> None:
|
||||
@@ -82,6 +86,25 @@ def test_reload_weights_sets_current_config():
|
||||
assert model_runner.seen_config is worker.vllm_config
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"enabled,active", [(True, False), (True, True), (False, False)]
|
||||
)
|
||||
def test_shutdown_weight_transfer_engine_preserves_inference(enabled, active):
|
||||
engine = _RecordingEngine() if enabled else None
|
||||
worker = _make_worker(engine)
|
||||
worker._weight_update_active = active
|
||||
model_runner = worker.model_runner
|
||||
if active:
|
||||
with pytest.raises(RuntimeError, match="during a weight update"):
|
||||
worker.shutdown_weight_transfer_engine()
|
||||
else:
|
||||
worker.shutdown_weight_transfer_engine()
|
||||
if engine is not None:
|
||||
assert engine.closed is not active
|
||||
assert worker.weight_transfer_engine is engine
|
||||
assert worker.model_runner is model_runner
|
||||
|
||||
|
||||
def test_reload_parameter_lookup_preserves_lora_module_names():
|
||||
base_layer = nn.Module()
|
||||
qweight = nn.Parameter(torch.ones(1))
|
||||
|
||||
Reference in New Issue
Block a user