[Bugfix][Model Loader] Support reload_weights with runai_streamer load format (#58124)

Signed-off-by: Viktor Koukouliev <vkoukouliev@nvidia.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
viktorkou
2026-10-07 14:04:35 +00:00
committed by GitHub
co-authored by Claude Fable 5.1 Isotr0py
parent 5281e49908
commit c83935b300
5 changed files with 208 additions and 7 deletions
@@ -6,11 +6,13 @@ import types
from unittest.mock import patch
import pytest
import torch
from vllm import SamplingParams
from vllm.config.load import LoadConfig
from vllm.model_executor.model_loader import get_model_loader
from vllm.model_executor.model_loader import runai_streamer_loader as rsl
from vllm.utils.mem_constants import GiB_bytes
load_format = "runai_streamer"
test_model = "openai-community/gpt2"
@@ -122,3 +124,94 @@ def test_runai_invalid_extra_config_leaves_environ_untouched():
with pytest.raises(ValueError, match="memory_limit must be an integer >= -1"):
_runai_loader({"concurrency": 16, "memory_limit": -5})
assert "RUNAI_STREAMER_CONCURRENCY" not in os.environ
@pytest.mark.parametrize(
"model_config,expected_source",
[
# Object storage: model_weights holds the URI, model the pulled config dir.
(
types.SimpleNamespace(
model="/tmp/pulled-config-files",
model_weights="s3://bucket/weights",
revision="myrev",
),
"s3://bucket/weights",
),
# HF repo or local path: model_weights is empty, model is the source.
(
types.SimpleNamespace(
model="org/model", model_weights="", revision="myrev"
),
"org/model",
),
],
ids=["object_storage", "hf_repo"],
)
def test_runai_get_all_weights_resolves_source(model_config, expected_source):
fake_self = types.SimpleNamespace(
_get_weights_iterator=lambda path, revision: iter(
[(f"{path}@{revision}", None)]
)
)
weights = list(
rsl.RunaiModelStreamerLoader.get_all_weights(
fake_self, model_config, model=None
)
)
assert weights == [(f"{expected_source}@myrev", None)]
@pytest.mark.parametrize(
"base_model,mul_model,add_model",
[
(
"Qwen/Qwen3-0.6B",
"inference-optimization/Qwen3-0.6B-debug-multiply",
"inference-optimization/Qwen3-0.6B-debug-add",
),
],
)
def test_runai_deep_sleep_reload_weights(base_model, mul_model, add_model, vllm_runner):
free, total = torch.accelerator.get_memory_info()
used_bytes_baseline = total - free
def sleep_and_reload(llm, path):
# Level 2 discards the parameter memory, so after wake_up the weights
# can only come from the reload: a tensor the streamer fails to
# deliver changes the output instead of keeping a stale valid value.
llm.get_llm().sleep(level=2)
free, total = torch.accelerator.get_memory_info()
assert total - free - used_bytes_baseline < 3 * GiB_bytes
llm.get_llm().wake_up(tags=["weights"])
llm.collective_rpc("reload_weights", kwargs={"weights_path": path})
free, total = torch.accelerator.get_memory_info()
assert total - free - used_bytes_baseline < 4 * GiB_bytes
llm.get_llm().wake_up(tags=["kv_cache"])
with vllm_runner(
model_name=base_model,
load_format=load_format,
enable_sleep_mode=True,
enable_prefix_caching=False,
max_model_len=16,
max_num_seqs=1,
) as llm:
base_output = llm.generate_greedy(["3 4 ="], max_tokens=4)
sleep_and_reload(llm, mul_model)
mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
assert mul_perp < add_perp
sleep_and_reload(llm, add_model)
mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
assert add_perp < mul_perp
# Round trip to the original weights over discarded memory must
# reproduce the pre-sleep output.
sleep_and_reload(llm, base_model)
assert llm.generate_greedy(["3 4 ="], max_tokens=4) == base_output
@@ -3,7 +3,9 @@
from pathlib import Path
import torch
from runai_model_streamer.safetensors_streamer.streamer_mock import StreamerPatcher
from safetensors.torch import load_file, save_file
from vllm.engine.arg_utils import EngineArgs
from vllm.transformers_utils.repo_utils import hf_api
@@ -50,3 +52,72 @@ def test_runai_model_loader_download_files_s3_mocked_with_patch(
executor = RunaiDummyExecutor(vllm_config)
executor.driver_worker.load_model()
def test_runai_reload_weights_path_from_mocked_s3(tmp_path: Path, monkeypatch):
# An engine started from object storage keeps the bucket URI in
# model_config.model_weights. A reload with weights_path must stream the
# new checkpoint, not re-stream the one the engine started from.
patcher = StreamerPatcher(str(tmp_path))
monkeypatch.setattr(
"vllm.transformers_utils.runai_utils.runai_list_safetensors",
patcher.shim_list_safetensors,
)
monkeypatch.setattr(
"vllm.transformers_utils.runai_utils.runai_pull_files",
patcher.shim_pull_files,
)
monkeypatch.setattr(
"vllm.model_executor.model_loader.weight_utils.SafetensorsStreamer",
patcher.create_mock_streamer,
)
base_dir = tmp_path / "gpt2"
hf_api().snapshot_download(
repo_id=test_model,
local_dir=str(base_dir),
allow_patterns=["*.json", "*.txt", "model.safetensors"],
)
# Second checkpoint under another prefix: the final layer norm scaled, so a
# reload that silently re-streams the original bucket is distinguishable.
tensors = load_file(base_dir / "model.safetensors")
original = tensors["ln_f.weight"].clone()
modified = original * 2
tensors["ln_f.weight"] = modified
mod_dir = tmp_path / "gpt2-mod"
mod_dir.mkdir()
save_file(tensors, mod_dir / "model.safetensors", metadata={"format": "pt"})
vllm_config = EngineArgs(
model="s3://my-mock-bucket/gpt2/",
load_format=load_format,
tensor_parallel_size=1,
).create_engine_config()
assert vllm_config.model_config.model_weights == "s3://my-mock-bucket/gpt2/"
executor = RunaiDummyExecutor(vllm_config)
executor.driver_worker.load_model()
model = executor.driver_worker.model_runner.get_model()
(param,) = [p for n, p in model.named_parameters() if n.endswith("ln_f.weight")]
def current() -> torch.Tensor:
return param.detach().float().cpu()
torch.testing.assert_close(current(), original, rtol=1e-2, atol=1e-2)
# No weights_path: re-stream the checkpoint the engine started from, i.e.
# the bucket kept in model_weights. Zero the live tensor first so a reload
# that did nothing is distinguishable from one that streamed the bucket.
param.data.zero_()
executor.driver_worker.reload_weights()
torch.testing.assert_close(current(), original, rtol=1e-2, atol=1e-2)
# weights_path: the new prefix must win over the stale model_weights.
executor.driver_worker.reload_weights(weights_path="s3://my-mock-bucket/gpt2-mod/")
torch.testing.assert_close(current(), modified, rtol=1e-2, atol=1e-2)
# After the override, a bare reload means the new checkpoint, not the
# one the engine was started with.
param.data.zero_()
executor.driver_worker.reload_weights()
torch.testing.assert_close(current(), modified, rtol=1e-2, atol=1e-2)