From c83935b3008553c72b1bad09f842b4c99b7fa47f Mon Sep 17 00:00:00 2001 From: viktorkou <105151911+viktorkou@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:04:35 +0300 Subject: [PATCH] [Bugfix][Model Loader] Support reload_weights with runai_streamer load format (#58124) Signed-off-by: Viktor Koukouliev Co-authored-by: Claude Fable 5.1 Co-authored-by: Isotr0py --- .../test_runai_model_streamer_loader.py | 93 +++++++++++++++++++ .../test_runai_model_streamer_s3.py | 71 ++++++++++++++ tests/v1/worker/test_gpu_model_runner.py | 30 ++++++ .../model_loader/runai_streamer_loader.py | 15 ++- vllm/v1/worker/gpu_model_runner.py | 6 +- 5 files changed, 208 insertions(+), 7 deletions(-) diff --git a/tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_loader.py b/tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_loader.py index 9164b8e4be..e0a083b3a8 100644 --- a/tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_loader.py +++ b/tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_loader.py @@ -6,11 +6,13 @@ import types from unittest.mock import patch import pytest +import torch from vllm import SamplingParams from vllm.config.load import LoadConfig from vllm.model_executor.model_loader import get_model_loader from vllm.model_executor.model_loader import runai_streamer_loader as rsl +from vllm.utils.mem_constants import GiB_bytes load_format = "runai_streamer" test_model = "openai-community/gpt2" @@ -122,3 +124,94 @@ def test_runai_invalid_extra_config_leaves_environ_untouched(): with pytest.raises(ValueError, match="memory_limit must be an integer >= -1"): _runai_loader({"concurrency": 16, "memory_limit": -5}) assert "RUNAI_STREAMER_CONCURRENCY" not in os.environ + + +@pytest.mark.parametrize( + "model_config,expected_source", + [ + # Object storage: model_weights holds the URI, model the pulled config dir. + ( + types.SimpleNamespace( + model="/tmp/pulled-config-files", + model_weights="s3://bucket/weights", + revision="myrev", + ), + "s3://bucket/weights", + ), + # HF repo or local path: model_weights is empty, model is the source. + ( + types.SimpleNamespace( + model="org/model", model_weights="", revision="myrev" + ), + "org/model", + ), + ], + ids=["object_storage", "hf_repo"], +) +def test_runai_get_all_weights_resolves_source(model_config, expected_source): + fake_self = types.SimpleNamespace( + _get_weights_iterator=lambda path, revision: iter( + [(f"{path}@{revision}", None)] + ) + ) + + weights = list( + rsl.RunaiModelStreamerLoader.get_all_weights( + fake_self, model_config, model=None + ) + ) + + assert weights == [(f"{expected_source}@myrev", None)] + + +@pytest.mark.parametrize( + "base_model,mul_model,add_model", + [ + ( + "Qwen/Qwen3-0.6B", + "inference-optimization/Qwen3-0.6B-debug-multiply", + "inference-optimization/Qwen3-0.6B-debug-add", + ), + ], +) +def test_runai_deep_sleep_reload_weights(base_model, mul_model, add_model, vllm_runner): + free, total = torch.accelerator.get_memory_info() + used_bytes_baseline = total - free + + def sleep_and_reload(llm, path): + # Level 2 discards the parameter memory, so after wake_up the weights + # can only come from the reload: a tensor the streamer fails to + # deliver changes the output instead of keeping a stale valid value. + llm.get_llm().sleep(level=2) + free, total = torch.accelerator.get_memory_info() + assert total - free - used_bytes_baseline < 3 * GiB_bytes + llm.get_llm().wake_up(tags=["weights"]) + llm.collective_rpc("reload_weights", kwargs={"weights_path": path}) + free, total = torch.accelerator.get_memory_info() + assert total - free - used_bytes_baseline < 4 * GiB_bytes + llm.get_llm().wake_up(tags=["kv_cache"]) + + with vllm_runner( + model_name=base_model, + load_format=load_format, + enable_sleep_mode=True, + enable_prefix_caching=False, + max_model_len=16, + max_num_seqs=1, + ) as llm: + base_output = llm.generate_greedy(["3 4 ="], max_tokens=4) + + sleep_and_reload(llm, mul_model) + mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0] + add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0] + assert mul_perp < add_perp + + sleep_and_reload(llm, add_model) + mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0] + add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0] + assert add_perp < mul_perp + + # Round trip to the original weights over discarded memory must + # reproduce the pre-sleep output. + sleep_and_reload(llm, base_model) + assert llm.generate_greedy(["3 4 ="], max_tokens=4) == base_output diff --git a/tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_s3.py b/tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_s3.py index d7abc80edb..a1440ae5d2 100644 --- a/tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_s3.py +++ b/tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_s3.py @@ -3,7 +3,9 @@ from pathlib import Path +import torch from runai_model_streamer.safetensors_streamer.streamer_mock import StreamerPatcher +from safetensors.torch import load_file, save_file from vllm.engine.arg_utils import EngineArgs from vllm.transformers_utils.repo_utils import hf_api @@ -50,3 +52,72 @@ def test_runai_model_loader_download_files_s3_mocked_with_patch( executor = RunaiDummyExecutor(vllm_config) executor.driver_worker.load_model() + + +def test_runai_reload_weights_path_from_mocked_s3(tmp_path: Path, monkeypatch): + # An engine started from object storage keeps the bucket URI in + # model_config.model_weights. A reload with weights_path must stream the + # new checkpoint, not re-stream the one the engine started from. + patcher = StreamerPatcher(str(tmp_path)) + monkeypatch.setattr( + "vllm.transformers_utils.runai_utils.runai_list_safetensors", + patcher.shim_list_safetensors, + ) + monkeypatch.setattr( + "vllm.transformers_utils.runai_utils.runai_pull_files", + patcher.shim_pull_files, + ) + monkeypatch.setattr( + "vllm.model_executor.model_loader.weight_utils.SafetensorsStreamer", + patcher.create_mock_streamer, + ) + + base_dir = tmp_path / "gpt2" + hf_api().snapshot_download( + repo_id=test_model, + local_dir=str(base_dir), + allow_patterns=["*.json", "*.txt", "model.safetensors"], + ) + # Second checkpoint under another prefix: the final layer norm scaled, so a + # reload that silently re-streams the original bucket is distinguishable. + tensors = load_file(base_dir / "model.safetensors") + original = tensors["ln_f.weight"].clone() + modified = original * 2 + tensors["ln_f.weight"] = modified + mod_dir = tmp_path / "gpt2-mod" + mod_dir.mkdir() + save_file(tensors, mod_dir / "model.safetensors", metadata={"format": "pt"}) + + vllm_config = EngineArgs( + model="s3://my-mock-bucket/gpt2/", + load_format=load_format, + tensor_parallel_size=1, + ).create_engine_config() + assert vllm_config.model_config.model_weights == "s3://my-mock-bucket/gpt2/" + + executor = RunaiDummyExecutor(vllm_config) + executor.driver_worker.load_model() + model = executor.driver_worker.model_runner.get_model() + (param,) = [p for n, p in model.named_parameters() if n.endswith("ln_f.weight")] + + def current() -> torch.Tensor: + return param.detach().float().cpu() + + torch.testing.assert_close(current(), original, rtol=1e-2, atol=1e-2) + + # No weights_path: re-stream the checkpoint the engine started from, i.e. + # the bucket kept in model_weights. Zero the live tensor first so a reload + # that did nothing is distinguishable from one that streamed the bucket. + param.data.zero_() + executor.driver_worker.reload_weights() + torch.testing.assert_close(current(), original, rtol=1e-2, atol=1e-2) + + # weights_path: the new prefix must win over the stale model_weights. + executor.driver_worker.reload_weights(weights_path="s3://my-mock-bucket/gpt2-mod/") + torch.testing.assert_close(current(), modified, rtol=1e-2, atol=1e-2) + + # After the override, a bare reload means the new checkpoint, not the + # one the engine was started with. + param.data.zero_() + executor.driver_worker.reload_weights() + torch.testing.assert_close(current(), modified, rtol=1e-2, atol=1e-2) diff --git a/tests/v1/worker/test_gpu_model_runner.py b/tests/v1/worker/test_gpu_model_runner.py index 5cf53899b8..deb4f7b421 100644 --- a/tests/v1/worker/test_gpu_model_runner.py +++ b/tests/v1/worker/test_gpu_model_runner.py @@ -1015,6 +1015,36 @@ def test_reload_weights_before_load_model(model_runner): model_runner.reload_weights() +def test_reload_weights_path_replaces_object_storage_source(monkeypatch): + # An engine started from an object-storage URI keeps that URI in + # model_config.model_weights, which the runai_streamer loader prefers over + # model_config.model. A new weights_path must replace it, or the loader + # silently re-streams the original checkpoint. + loader = Mock() + loader.get_all_weights.return_value = iter(()) + monkeypatch.setattr(gpu_model_runner_module, "get_model_loader", lambda _: loader) + monkeypatch.setattr(gpu_model_runner_module, "initialize_layerwise_reload", Mock()) + monkeypatch.setattr(gpu_model_runner_module, "finalize_layerwise_reload", Mock()) + + runner = Mock(lora_config=None) + runner.model_config = SimpleNamespace( + model="/tmp/pulled-config-files", + model_weights="s3://bucket/original", + revision="abc123", + quantization=None, + ) + runner.get_model.return_value.named_parameters.return_value = [] + runner.get_model.return_value.load_weights.return_value = None + + GPUModelRunner.reload_weights(runner, weights_path="org/new-model") + + loader.get_all_weights.assert_called_once_with( + runner.model_config, runner.get_model.return_value + ) + cfg = runner.model_config + assert (cfg.model, cfg.model_weights, cfg.revision) == ("org/new-model", "", None) + + def test_sample_passes_reordered_draft_probs_to_rejection_sampler(): runner = object.__new__(GPUModelRunner) runner.use_async_scheduling = False diff --git a/vllm/model_executor/model_loader/runai_streamer_loader.py b/vllm/model_executor/model_loader/runai_streamer_loader.py index 2f18a2e3ac..e001160062 100644 --- a/vllm/model_executor/model_loader/runai_streamer_loader.py +++ b/vllm/model_executor/model_loader/runai_streamer_loader.py @@ -134,11 +134,16 @@ class RunaiModelStreamerLoader(BaseModelLoader): """Download model if necessary.""" self._prepare_weights(model_config.model, model_config.revision) - def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None: - """Load weights into a model.""" + def get_all_weights( + self, + model_config: ModelConfig, + model: nn.Module, + ) -> Generator[tuple[str, torch.Tensor], None, None]: model_weights = model_config.model if model_weights_override := model_config.model_weights: model_weights = model_weights_override - model.load_weights( - self._get_weights_iterator(model_weights, model_config.revision) - ) + yield from self._get_weights_iterator(model_weights, model_config.revision) + + def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None: + """Load weights into a model.""" + model.load_weights(self.get_all_weights(model_config, model)) diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 76299308f8..0cc3fcc29a 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -5534,9 +5534,11 @@ class GPUModelRunner( ) if weights_path is not None: - # The revision belongs to the model we are reloading away from, - # so it must not be carried over to the new path. + # The revision and any object-storage `model_weights` source + # belong to the model we are reloading away from, so they must + # not be carried over to the new path. self.model_config.model = weights_path + self.model_config.model_weights = "" self.model_config.revision = None weights_iterator = model_loader.get_all_weights(self.model_config, model) weights_iterator = cast(