[UX] Skip checkpoint shards an MTP head does not load (#60152)
Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Robert Shaw
Claude Opus 5.5
parent
17c70c8953
commit
e572e02d21
@@ -4,11 +4,17 @@
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from safetensors.torch import save_file
|
||||
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.model_executor.model_loader import DefaultModelLoader
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
filter_duplicate_safetensors_files,
|
||||
filter_safetensors_files_by_weight_name,
|
||||
)
|
||||
|
||||
|
||||
@@ -77,3 +83,57 @@ def test_filter_duplicate_safetensors_files_all_exist():
|
||||
if __name__ == "__main__":
|
||||
test_filter_duplicate_safetensors_files_missing_weight()
|
||||
test_filter_duplicate_safetensors_files_all_exist()
|
||||
|
||||
|
||||
def _skip_non_mtp(name: str) -> bool:
|
||||
return not name.startswith("mtp.")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mtp_checkpoint(tmp_path):
|
||||
"""Three shards: target only, target + MTP, MTP only."""
|
||||
shards = {
|
||||
"model-00001-of-00003.safetensors": ["model.layers.0.weight"],
|
||||
"model-00002-of-00003.safetensors": ["model.layers.1.weight", "mtp.a.weight"],
|
||||
"model-00003-of-00003.safetensors": ["mtp.b.weight"],
|
||||
}
|
||||
for filename, names in shards.items():
|
||||
save_file({name: torch.zeros(1) for name in names}, tmp_path / filename)
|
||||
return tmp_path
|
||||
|
||||
|
||||
def test_filter_by_weight_name_drops_shards_without_wanted_weights(mtp_checkpoint):
|
||||
files = sorted(str(f) for f in mtp_checkpoint.glob("*.safetensors"))
|
||||
|
||||
assert filter_safetensors_files_by_weight_name(files, _skip_non_mtp) == files[1:]
|
||||
# A rule that rejects everything must not leave the loader without files.
|
||||
assert filter_safetensors_files_by_weight_name(files, lambda _: True) == files
|
||||
|
||||
|
||||
def test_loader_does_not_read_shards_the_model_skips(mtp_checkpoint, monkeypatch):
|
||||
"""Whole-file loaders such as InstantTensor only see the kept shards."""
|
||||
opened: list[list[str]] = []
|
||||
|
||||
def fake_iterator(hf_weights_files, *args, **kwargs):
|
||||
opened.append(sorted(os.path.basename(f) for f in hf_weights_files))
|
||||
return iter(())
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.model_loader.default_loader."
|
||||
"instanttensor_weights_iterator",
|
||||
fake_iterator,
|
||||
)
|
||||
|
||||
class Drafter(torch.nn.Module):
|
||||
def is_unused_checkpoint_weight(self, name: str) -> bool:
|
||||
return _skip_non_mtp(name)
|
||||
|
||||
loader = DefaultModelLoader(LoadConfig(load_format="instanttensor"))
|
||||
model_config = SimpleNamespace(model=str(mtp_checkpoint), revision=None)
|
||||
list(loader.get_all_weights(model_config, Drafter()))
|
||||
list(loader.get_all_weights(model_config, torch.nn.Module()))
|
||||
|
||||
assert opened == [
|
||||
["model-00002-of-00003.safetensors", "model-00003-of-00003.safetensors"],
|
||||
sorted(f.name for f in mtp_checkpoint.glob("*.safetensors")),
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user