[Bugfix][Model Loader] Support reload_weights with runai_streamer load format (#58124)
Signed-off-by: Viktor Koukouliev <vkoukouliev@nvidia.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
Isotr0py
parent
5281e49908
commit
c83935b300
+93
@@ -6,11 +6,13 @@ import types
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.model_executor.model_loader import get_model_loader
|
||||
from vllm.model_executor.model_loader import runai_streamer_loader as rsl
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
|
||||
load_format = "runai_streamer"
|
||||
test_model = "openai-community/gpt2"
|
||||
@@ -122,3 +124,94 @@ def test_runai_invalid_extra_config_leaves_environ_untouched():
|
||||
with pytest.raises(ValueError, match="memory_limit must be an integer >= -1"):
|
||||
_runai_loader({"concurrency": 16, "memory_limit": -5})
|
||||
assert "RUNAI_STREAMER_CONCURRENCY" not in os.environ
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_config,expected_source",
|
||||
[
|
||||
# Object storage: model_weights holds the URI, model the pulled config dir.
|
||||
(
|
||||
types.SimpleNamespace(
|
||||
model="/tmp/pulled-config-files",
|
||||
model_weights="s3://bucket/weights",
|
||||
revision="myrev",
|
||||
),
|
||||
"s3://bucket/weights",
|
||||
),
|
||||
# HF repo or local path: model_weights is empty, model is the source.
|
||||
(
|
||||
types.SimpleNamespace(
|
||||
model="org/model", model_weights="", revision="myrev"
|
||||
),
|
||||
"org/model",
|
||||
),
|
||||
],
|
||||
ids=["object_storage", "hf_repo"],
|
||||
)
|
||||
def test_runai_get_all_weights_resolves_source(model_config, expected_source):
|
||||
fake_self = types.SimpleNamespace(
|
||||
_get_weights_iterator=lambda path, revision: iter(
|
||||
[(f"{path}@{revision}", None)]
|
||||
)
|
||||
)
|
||||
|
||||
weights = list(
|
||||
rsl.RunaiModelStreamerLoader.get_all_weights(
|
||||
fake_self, model_config, model=None
|
||||
)
|
||||
)
|
||||
|
||||
assert weights == [(f"{expected_source}@myrev", None)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"base_model,mul_model,add_model",
|
||||
[
|
||||
(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"inference-optimization/Qwen3-0.6B-debug-multiply",
|
||||
"inference-optimization/Qwen3-0.6B-debug-add",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_runai_deep_sleep_reload_weights(base_model, mul_model, add_model, vllm_runner):
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
used_bytes_baseline = total - free
|
||||
|
||||
def sleep_and_reload(llm, path):
|
||||
# Level 2 discards the parameter memory, so after wake_up the weights
|
||||
# can only come from the reload: a tensor the streamer fails to
|
||||
# deliver changes the output instead of keeping a stale valid value.
|
||||
llm.get_llm().sleep(level=2)
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
assert total - free - used_bytes_baseline < 3 * GiB_bytes
|
||||
llm.get_llm().wake_up(tags=["weights"])
|
||||
llm.collective_rpc("reload_weights", kwargs={"weights_path": path})
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
assert total - free - used_bytes_baseline < 4 * GiB_bytes
|
||||
llm.get_llm().wake_up(tags=["kv_cache"])
|
||||
|
||||
with vllm_runner(
|
||||
model_name=base_model,
|
||||
load_format=load_format,
|
||||
enable_sleep_mode=True,
|
||||
enable_prefix_caching=False,
|
||||
max_model_len=16,
|
||||
max_num_seqs=1,
|
||||
) as llm:
|
||||
base_output = llm.generate_greedy(["3 4 ="], max_tokens=4)
|
||||
|
||||
sleep_and_reload(llm, mul_model)
|
||||
mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
|
||||
add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
|
||||
assert mul_perp < add_perp
|
||||
|
||||
sleep_and_reload(llm, add_model)
|
||||
mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
|
||||
add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
|
||||
assert add_perp < mul_perp
|
||||
|
||||
# Round trip to the original weights over discarded memory must
|
||||
# reproduce the pre-sleep output.
|
||||
sleep_and_reload(llm, base_model)
|
||||
assert llm.generate_greedy(["3 4 ="], max_tokens=4) == base_output
|
||||
|
||||
+71
@@ -3,7 +3,9 @@
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from runai_model_streamer.safetensors_streamer.streamer_mock import StreamerPatcher
|
||||
from safetensors.torch import load_file, save_file
|
||||
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
@@ -50,3 +52,72 @@ def test_runai_model_loader_download_files_s3_mocked_with_patch(
|
||||
|
||||
executor = RunaiDummyExecutor(vllm_config)
|
||||
executor.driver_worker.load_model()
|
||||
|
||||
|
||||
def test_runai_reload_weights_path_from_mocked_s3(tmp_path: Path, monkeypatch):
|
||||
# An engine started from object storage keeps the bucket URI in
|
||||
# model_config.model_weights. A reload with weights_path must stream the
|
||||
# new checkpoint, not re-stream the one the engine started from.
|
||||
patcher = StreamerPatcher(str(tmp_path))
|
||||
monkeypatch.setattr(
|
||||
"vllm.transformers_utils.runai_utils.runai_list_safetensors",
|
||||
patcher.shim_list_safetensors,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.transformers_utils.runai_utils.runai_pull_files",
|
||||
patcher.shim_pull_files,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.model_loader.weight_utils.SafetensorsStreamer",
|
||||
patcher.create_mock_streamer,
|
||||
)
|
||||
|
||||
base_dir = tmp_path / "gpt2"
|
||||
hf_api().snapshot_download(
|
||||
repo_id=test_model,
|
||||
local_dir=str(base_dir),
|
||||
allow_patterns=["*.json", "*.txt", "model.safetensors"],
|
||||
)
|
||||
# Second checkpoint under another prefix: the final layer norm scaled, so a
|
||||
# reload that silently re-streams the original bucket is distinguishable.
|
||||
tensors = load_file(base_dir / "model.safetensors")
|
||||
original = tensors["ln_f.weight"].clone()
|
||||
modified = original * 2
|
||||
tensors["ln_f.weight"] = modified
|
||||
mod_dir = tmp_path / "gpt2-mod"
|
||||
mod_dir.mkdir()
|
||||
save_file(tensors, mod_dir / "model.safetensors", metadata={"format": "pt"})
|
||||
|
||||
vllm_config = EngineArgs(
|
||||
model="s3://my-mock-bucket/gpt2/",
|
||||
load_format=load_format,
|
||||
tensor_parallel_size=1,
|
||||
).create_engine_config()
|
||||
assert vllm_config.model_config.model_weights == "s3://my-mock-bucket/gpt2/"
|
||||
|
||||
executor = RunaiDummyExecutor(vllm_config)
|
||||
executor.driver_worker.load_model()
|
||||
model = executor.driver_worker.model_runner.get_model()
|
||||
(param,) = [p for n, p in model.named_parameters() if n.endswith("ln_f.weight")]
|
||||
|
||||
def current() -> torch.Tensor:
|
||||
return param.detach().float().cpu()
|
||||
|
||||
torch.testing.assert_close(current(), original, rtol=1e-2, atol=1e-2)
|
||||
|
||||
# No weights_path: re-stream the checkpoint the engine started from, i.e.
|
||||
# the bucket kept in model_weights. Zero the live tensor first so a reload
|
||||
# that did nothing is distinguishable from one that streamed the bucket.
|
||||
param.data.zero_()
|
||||
executor.driver_worker.reload_weights()
|
||||
torch.testing.assert_close(current(), original, rtol=1e-2, atol=1e-2)
|
||||
|
||||
# weights_path: the new prefix must win over the stale model_weights.
|
||||
executor.driver_worker.reload_weights(weights_path="s3://my-mock-bucket/gpt2-mod/")
|
||||
torch.testing.assert_close(current(), modified, rtol=1e-2, atol=1e-2)
|
||||
|
||||
# After the override, a bare reload means the new checkpoint, not the
|
||||
# one the engine was started with.
|
||||
param.data.zero_()
|
||||
executor.driver_worker.reload_weights()
|
||||
torch.testing.assert_close(current(), modified, rtol=1e-2, atol=1e-2)
|
||||
|
||||
@@ -1015,6 +1015,36 @@ def test_reload_weights_before_load_model(model_runner):
|
||||
model_runner.reload_weights()
|
||||
|
||||
|
||||
def test_reload_weights_path_replaces_object_storage_source(monkeypatch):
|
||||
# An engine started from an object-storage URI keeps that URI in
|
||||
# model_config.model_weights, which the runai_streamer loader prefers over
|
||||
# model_config.model. A new weights_path must replace it, or the loader
|
||||
# silently re-streams the original checkpoint.
|
||||
loader = Mock()
|
||||
loader.get_all_weights.return_value = iter(())
|
||||
monkeypatch.setattr(gpu_model_runner_module, "get_model_loader", lambda _: loader)
|
||||
monkeypatch.setattr(gpu_model_runner_module, "initialize_layerwise_reload", Mock())
|
||||
monkeypatch.setattr(gpu_model_runner_module, "finalize_layerwise_reload", Mock())
|
||||
|
||||
runner = Mock(lora_config=None)
|
||||
runner.model_config = SimpleNamespace(
|
||||
model="/tmp/pulled-config-files",
|
||||
model_weights="s3://bucket/original",
|
||||
revision="abc123",
|
||||
quantization=None,
|
||||
)
|
||||
runner.get_model.return_value.named_parameters.return_value = []
|
||||
runner.get_model.return_value.load_weights.return_value = None
|
||||
|
||||
GPUModelRunner.reload_weights(runner, weights_path="org/new-model")
|
||||
|
||||
loader.get_all_weights.assert_called_once_with(
|
||||
runner.model_config, runner.get_model.return_value
|
||||
)
|
||||
cfg = runner.model_config
|
||||
assert (cfg.model, cfg.model_weights, cfg.revision) == ("org/new-model", "", None)
|
||||
|
||||
|
||||
def test_sample_passes_reordered_draft_probs_to_rejection_sampler():
|
||||
runner = object.__new__(GPUModelRunner)
|
||||
runner.use_async_scheduling = False
|
||||
|
||||
@@ -134,11 +134,16 @@ class RunaiModelStreamerLoader(BaseModelLoader):
|
||||
"""Download model if necessary."""
|
||||
self._prepare_weights(model_config.model, model_config.revision)
|
||||
|
||||
def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:
|
||||
"""Load weights into a model."""
|
||||
def get_all_weights(
|
||||
self,
|
||||
model_config: ModelConfig,
|
||||
model: nn.Module,
|
||||
) -> Generator[tuple[str, torch.Tensor], None, None]:
|
||||
model_weights = model_config.model
|
||||
if model_weights_override := model_config.model_weights:
|
||||
model_weights = model_weights_override
|
||||
model.load_weights(
|
||||
self._get_weights_iterator(model_weights, model_config.revision)
|
||||
)
|
||||
yield from self._get_weights_iterator(model_weights, model_config.revision)
|
||||
|
||||
def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:
|
||||
"""Load weights into a model."""
|
||||
model.load_weights(self.get_all_weights(model_config, model))
|
||||
|
||||
@@ -5534,9 +5534,11 @@ class GPUModelRunner(
|
||||
)
|
||||
|
||||
if weights_path is not None:
|
||||
# The revision belongs to the model we are reloading away from,
|
||||
# so it must not be carried over to the new path.
|
||||
# The revision and any object-storage `model_weights` source
|
||||
# belong to the model we are reloading away from, so they must
|
||||
# not be carried over to the new path.
|
||||
self.model_config.model = weights_path
|
||||
self.model_config.model_weights = ""
|
||||
self.model_config.revision = None
|
||||
weights_iterator = model_loader.get_all_weights(self.model_config, model)
|
||||
weights_iterator = cast(
|
||||
|
||||
Reference in New Issue
Block a user