[Feature][Model Loader] Apply model-owned MTP shard selection to RunAI ModelStreamer (#60759)

Signed-off-by: 浩韵 <zuohaiyu.zhy@alibaba-inc.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
Haiyu Zuo
2026-10-15 12:23:11 +08:00
committed by GitHub
co-authored by Kevin H. Luu
parent 833665cded
commit ff5a91a456
6 changed files with 724 additions and 13 deletions
@@ -1,13 +1,18 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import os
import types
from datetime import timedelta
from pathlib import Path
from unittest.mock import patch
import pytest
import torch
from safetensors.torch import load_file, save_file
from tests.utils import create_new_process_for_each_test
from vllm import SamplingParams
from vllm.config.load import LoadConfig
from vllm.model_executor.model_loader import get_model_loader
@@ -39,12 +44,130 @@ def test_get_model_loader_with_runai_flag():
assert model_loader.__class__.__name__ == "RunaiModelStreamerLoader"
def test_checkpoint_selection_hook_is_optional_for_model_detection():
from vllm.model_executor.models.interfaces import SupportsMultiModal
from vllm.model_executor.models.interfaces_base import (
VllmModel,
is_text_generation_model,
)
class ModelWithoutHook(torch.nn.Module):
def __init__(self, vllm_config, prefix=""):
super().__init__()
def embed_input_ids(self, input_ids):
return input_ids
def forward(self, input_ids, positions):
return input_ids
def compute_logits(self, hidden_states):
return hidden_states
assert is_text_generation_model(ModelWithoutHook)
assert is_text_generation_model(ModelWithoutHook(None))
class ModelWithDefaultRule(ModelWithoutHook, VllmModel):
pass
assert not ModelWithDefaultRule(None).is_unused_checkpoint_weight("any.weight")
# Structural models do not inherit the protocol's optional default hook.
for names in ([], ["language_model"]):
model = torch.nn.Module()
model.language_model = ModelWithoutHook(None)
model._language_model_names = names
assert SupportsMultiModal.get_language_model(model) is model.language_model
def test_runai_model_loader_download_files(vllm_runner):
with vllm_runner(test_model, load_format=load_format) as llm:
deserialized_outputs = llm.generate(prompts, sampling_params)
assert deserialized_outputs
# In-process MTP compilation must not retain GPU resources in the pytest worker.
@create_new_process_for_each_test("spawn")
def test_runai_mtp_real_checkpoint(vllm_runner, tmp_path, monkeypatch):
"""Skip main-only shards without changing a real MTP model's generation."""
from vllm.model_executor.models.qwen3_5_mtp import Qwen3_5MTP
from vllm.platforms import current_platform
from vllm.transformers_utils.repo_utils import hf_api
from vllm.v1.metrics.reader import Counter
if not current_platform.is_cuda_alike():
pytest.skip("MTP serving requires CUDA or ROCm")
checkpoint = tmp_path / "checkpoint"
hf_api().snapshot_download(
repo_id="Qwen/Qwen3.5-0.8B-Base", local_dir=str(checkpoint)
)
original_files = list(checkpoint.glob("*.safetensors"))
tensors = {}
for path in original_files:
tensors.update(load_file(path))
# Reshard real, unmodified weights so selection cannot pass vacuously on
# the published single mixed file. This predicate needs no model state.
selector = Qwen3_5MTP.__new__(Qwen3_5MTP)
main: dict[str, torch.Tensor] = {}
draft: dict[str, torch.Tensor] = {}
for name, tensor in tensors.items():
(main if selector.is_unused_checkpoint_weight(name) else draft)[name] = tensor
assert main and draft
files = [checkpoint / "main.safetensors", checkpoint / "draft.safetensors"]
for path, contents in zip(files, (main, draft)):
save_file(contents, path, metadata={"format": "pt"})
for path in original_files:
path.unlink()
(checkpoint / "model.safetensors.index.json").write_text(
json.dumps(
{
"weight_map": {
name: path.name
for path, data in zip(files, (main, draft))
for name in data
}
}
)
)
del tensors, main, draft
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
opened = []
real_stream = rsl.runai_safetensors_weights_iterator
def stream(paths, *args):
opened.append(list(paths))
yield from real_stream(paths, *args)
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream)
results = []
for load_format in ("auto", "runai_streamer"):
with vllm_runner(
str(checkpoint),
load_format=load_format,
block_size=None,
enable_chunked_prefill=True,
disable_log_stats=False,
limit_mm_per_prompt={"image": 0, "video": 0},
speculative_config={"method": "mtp", "num_speculative_tokens": 1},
) as runner:
outputs = runner.generate_greedy(prompts, max_tokens=32)
results.append(outputs)
accepted = next(
m.value
for m in runner.llm.get_metrics()
if isinstance(m, Counter)
and m.name == "vllm:spec_decode_num_accepted_tokens"
)
assert accepted > 0
assert len(opened) == 2
assert set(opened[0]) == {str(path) for path in files}
assert opened[1] == [str(files[1])]
matches = sum(a == b for a, b in zip(*results))
assert matches >= 3, "RunAI MTP generation diverged from the Default loader"
@pytest.mark.skip(
reason="Temporarily disabled due to GCS access issues. "
"TODO: Re-enable this test once the underlying issue is resolved."
@@ -150,7 +273,7 @@ def test_runai_invalid_extra_config_leaves_environ_untouched():
)
def test_runai_get_all_weights_resolves_source(model_config, expected_source):
fake_self = types.SimpleNamespace(
_get_weights_iterator=lambda path, revision: iter(
_get_weights_iterator=lambda path, revision, is_unused_weight=None: iter(
[(f"{path}@{revision}", None)]
)
)
@@ -164,6 +287,387 @@ def test_runai_get_all_weights_resolves_source(model_config, expected_source):
assert weights == [(f"{expected_source}@myrev", None)]
@pytest.mark.parametrize("selection", ["draft", "unadapted", "reject_all"])
def test_runai_selects_shards_before_streaming(tmp_path, monkeypatch, selection):
"""Unused shards must not reach RunAI; mixed shards retain their tensors."""
contents = [
{"main.weight": torch.tensor([1.0])},
{"main.bias": torch.tensor([2.0]), "draft.weight": torch.tensor([3.0])},
{"draft.bias": torch.tensor([4.0])},
]
files = []
for i, tensors in enumerate(contents):
path = str(tmp_path / f"part-{i}.safetensors")
save_file(tensors, path)
files.append(path)
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
opened = []
real_stream = rsl.runai_safetensors_weights_iterator
def stream(paths, *args):
opened.extend(paths)
yield from real_stream(paths, *args)
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream)
model = torch.nn.Module()
if selection == "draft":
model.is_unused_checkpoint_weight = lambda name: not name.startswith("draft.")
elif selection == "reject_all":
model.is_unused_checkpoint_weight = lambda name: True
config = types.SimpleNamespace(model=str(tmp_path), model_weights="", revision=None)
loaded = dict(_runai_loader({}).get_all_weights(config, model))
expected = files[1:] if selection == "draft" else files
assert opened == expected
assert set(loaded) == {
name
for tensors in (contents[1:] if selection == "draft" else contents)
for name in tensors
}
for name, tensor in {**contents[1], **contents[2]}.items():
torch.testing.assert_close(loaded[name], tensor)
def _check_distributed_selection(rank, rendezvous, scenario):
torch.distributed.init_process_group(
"gloo",
init_method=rendezvous,
rank=rank,
world_size=2,
timeout=timedelta(seconds=30),
)
root = "s3://bucket/weights"
if scenario == "different_roots":
root += str(rank)
files = [root + "/main.safetensors", root + "/draft.safetensors"]
def pull_files(source, destination, **kwargs):
if scenario == "metadata_failure" and rank == 1:
raise OSError("Index temporarily unavailable")
(Path(destination) / "model.safetensors.index.json").write_text(
json.dumps(
{
"weight_map": {
"main.weight": "main.safetensors",
"draft.weight": "draft.safetensors",
}
}
)
)
try:
with pytest.MonkeyPatch.context() as monkeypatch:
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
monkeypatch.setattr(
rsl,
"_get_remote_checkpoint_keys",
lambda paths: [
["main.weight"]
if path.endswith("main.safetensors")
else ["draft.weight"]
for path in paths
],
)
monkeypatch.setattr(
rsl,
"get_world_group",
lambda: types.SimpleNamespace(
world_size=2, cpu_group=torch.distributed.group.WORLD
),
raising=False,
)
monkeypatch.setattr(
rsl,
"get_pp_group",
lambda: types.SimpleNamespace(
world_size=2 if scenario == "pipeline_draft" else 1
),
raising=False,
)
if scenario == "pipeline_draft" and rank == 0:
# Earlier PP stages never invoke the draft loader.
torch.distributed.barrier()
return
# Exercise real selection and Gloo; observe the distributed stream input.
monkeypatch.setattr(
rsl,
"runai_safetensors_weights_iterator",
lambda paths, *args: iter((path, None) for path in paths),
)
model = torch.nn.Module()
if not (scenario == "missing_hook" and rank == 1):
needed = (
"main." if scenario == "different_roles" and rank == 1 else "draft."
)
model.is_unused_checkpoint_weight = lambda name: (
not name.startswith(needed)
)
config = types.SimpleNamespace(model=root, model_weights="", revision=None)
selected = [
path
for path, _ in _runai_loader({"distributed": True}).get_all_weights(
config, model
)
]
expected = files[1:] if scenario == "healthy" else files
assert selected == expected
if scenario == "pipeline_draft":
torch.distributed.barrier()
finally:
torch.distributed.destroy_process_group()
@pytest.mark.parametrize(
"scenario",
[
"healthy",
"metadata_failure",
"different_roles",
"missing_hook",
"different_roots",
"pipeline_draft",
],
)
def test_distributed_runai_preserves_every_ranks_required_shards(tmp_path, scenario):
"""All ranks must stream the same union even if one cannot filter safely."""
torch.multiprocessing.spawn(
_check_distributed_selection,
args=((tmp_path / "rendezvous").as_uri(), scenario),
nprocs=2,
)
@pytest.mark.parametrize("scheme", ["s3", "gs", "az"])
@pytest.mark.parametrize(
"index_kind",
[
"valid",
"missing",
"malformed",
"unavailable",
"unadapted",
"missing_shard",
"stale",
"header_unavailable",
],
)
def test_runai_remote_selection_uses_actual_weight_source(
tmp_path, monkeypatch, scheme, index_kind
):
root = f"{scheme}://bucket/weights"
files = [
root + "/main/part.safetensors",
root + "/draft/part.safetensors",
root + "/mixed.safetensors",
root + "/extra.safetensors",
]
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
pulled = []
def pull_files(source, destination, allow_pattern, ignore_pattern=None):
pulled.append((source.rstrip("/"), allow_pattern))
if index_kind == "unavailable":
# Storage SDKs do not necessarily derive errors from OSError.
raise Exception("Object metadata unavailable")
index = Path(destination) / "model.safetensors.index.json"
if index_kind in ("valid", "missing_shard", "stale", "header_unavailable"):
index.write_text(
json.dumps(
{
"weight_map": {
"main.weight": "main/part.safetensors",
"draft.weight": (
"absent.safetensors"
if index_kind == "missing_shard"
else "draft/part.safetensors"
),
"main.bias": "mixed.safetensors",
"draft.bias": "mixed.safetensors",
}
}
)
)
elif index_kind == "malformed":
index.write_text("not JSON")
monkeypatch.setattr(rsl, "runai_pull_files", pull_files, raising=False)
headers_checked = []
def checkpoint_keys(paths):
headers_checked.extend(paths)
if index_kind == "header_unavailable":
raise OSError("Header temporarily unavailable")
return [
["main.weight", "draft.unindexed_weight"]
if index_kind == "stale"
else ["main.weight"]
for _ in paths
]
monkeypatch.setattr(
rsl, "_get_remote_checkpoint_keys", checkpoint_keys, raising=False
)
opened = []
def stream(paths, *args):
opened.extend(paths)
return iter(())
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream)
model = torch.nn.Module()
if index_kind != "unadapted":
model.is_unused_checkpoint_weight = lambda name: not name.startswith("draft.")
# The config directory is not the checkpoint: using its index could drop
# tensors from an independent model_weights source.
(tmp_path / "model.safetensors.index.json").write_text("not the weights index")
config = types.SimpleNamespace(
model=str(tmp_path), model_weights=root, revision=None
)
list(_runai_loader({}).get_all_weights(config, model))
assert opened == (files[1:] if index_kind == "valid" else files)
assert headers_checked == (
files[:1] if index_kind in ("valid", "stale", "header_unavailable") else []
)
assert pulled == (
[]
if index_kind == "unadapted"
else [
(
root,
[
"weights/model.safetensors.index.json",
"model.safetensors.index.json",
],
)
]
)
@pytest.mark.parametrize(
"weight_map",
[None, {}, [], {"draft.weight": None}, {"draft.weight": "../part.safetensors"}],
)
def test_runai_invalid_weight_map_keeps_original_files(monkeypatch, weight_map):
files = ["s3://bucket/model/part.safetensors"]
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
def pull_files(source, directory, **kwargs):
(Path(directory) / "model.safetensors.index.json").write_text(
json.dumps({"weight_map": weight_map})
)
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
monkeypatch.setattr(
rsl, "runai_safetensors_weights_iterator", lambda paths, *args: iter(paths)
)
model = torch.nn.Module()
model.is_unused_checkpoint_weight = lambda name: True
config = types.SimpleNamespace(
model="s3://bucket/model", model_weights="", revision=None
)
assert list(_runai_loader({}).get_all_weights(config, model)) == files
def test_runai_model_rule_error_is_not_hidden_by_metadata_fallback(monkeypatch):
files = ["s3://bucket/model/part.safetensors"]
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
def pull_files(source, directory, **kwargs):
(Path(directory) / "model.safetensors.index.json").write_text(
json.dumps({"weight_map": {"draft.weight": "part.safetensors"}})
)
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
model = torch.nn.Module()
def invalid_rule(name):
raise ValueError("Invalid model rule")
model.is_unused_checkpoint_weight = invalid_rule
config = types.SimpleNamespace(
model="s3://bucket/model", model_weights="", revision=None
)
with pytest.raises(ValueError, match="Invalid model rule"):
list(_runai_loader({}).get_all_weights(config, model))
def test_runai_header_reader_uses_actual_tensor_names(tmp_path):
"""The RunAI header reader sees names even when an index omits them."""
path = tmp_path / "mixed.safetensors"
save_file(
{
"main.weight": torch.tensor([1.0]),
"draft.unindexed_weight": torch.tensor([2.0]),
},
path,
metadata={"format": "pt"},
)
(names,) = rsl._get_remote_checkpoint_keys([str(path)])
assert set(names) == {"main.weight", "draft.unindexed_weight"}
@pytest.mark.parametrize("stale", [False, True])
def test_runai_stale_index_keeps_actual_draft_weights(tmp_path, monkeypatch, stale):
"""Verify excluded headers before dropping a shard with real draft weights."""
root = "s3://bucket/checkpoint"
main_path = tmp_path / "main.safetensors"
draft_path = tmp_path / "draft.safetensors"
main_tensors = {"main.weight": torch.tensor([1.0])}
if stale:
main_tensors["draft.unindexed_weight"] = torch.tensor([2.0])
save_file(main_tensors, main_path)
save_file({"draft.weight": torch.tensor([3.0])}, draft_path)
paths = {
root + "/main.safetensors": str(main_path),
root + "/draft.safetensors": str(draft_path),
}
monkeypatch.setattr(rsl, "list_safetensors", lambda path: list(paths))
def pull_files(source, directory, **kwargs):
(Path(directory) / "model.safetensors.index.json").write_text(
json.dumps(
{
"weight_map": {
"main.weight": "main.safetensors",
"draft.weight": "draft.safetensors",
}
}
)
)
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
read_headers = rsl._get_remote_checkpoint_keys
monkeypatch.setattr(
rsl,
"_get_remote_checkpoint_keys",
lambda files: read_headers([paths[path] for path in files]),
)
stream = rsl.runai_safetensors_weights_iterator
def stream_local(files, *args):
for name, tensor in stream([paths[path] for path in files], *args):
yield name, tensor.clone()
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream_local)
model = torch.nn.Module()
model.is_unused_checkpoint_weight = lambda name: not name.startswith("draft.")
config = types.SimpleNamespace(model=root, model_weights="", revision=None)
loaded = dict(_runai_loader({}).get_all_weights(config, model))
assert set(loaded) == (
{"main.weight", "draft.unindexed_weight", "draft.weight"}
if stale
else {"draft.weight"}
)
torch.testing.assert_close(loaded["draft.weight"], torch.tensor([3.0]))
if stale:
torch.testing.assert_close(
loaded["draft.unindexed_weight"], torch.tensor([2.0])
)
@pytest.mark.parametrize(
"base_model,mul_model,add_model",
[