[UX] Skip checkpoint shards an MTP head does not load (#60152)
Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Robert Shaw
Claude Opus 5.5
parent
17c70c8953
commit
e572e02d21
@@ -4,11 +4,17 @@
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from safetensors.torch import save_file
|
||||
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.model_executor.model_loader import DefaultModelLoader
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
filter_duplicate_safetensors_files,
|
||||
filter_safetensors_files_by_weight_name,
|
||||
)
|
||||
|
||||
|
||||
@@ -77,3 +83,57 @@ def test_filter_duplicate_safetensors_files_all_exist():
|
||||
if __name__ == "__main__":
|
||||
test_filter_duplicate_safetensors_files_missing_weight()
|
||||
test_filter_duplicate_safetensors_files_all_exist()
|
||||
|
||||
|
||||
def _skip_non_mtp(name: str) -> bool:
|
||||
return not name.startswith("mtp.")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mtp_checkpoint(tmp_path):
|
||||
"""Three shards: target only, target + MTP, MTP only."""
|
||||
shards = {
|
||||
"model-00001-of-00003.safetensors": ["model.layers.0.weight"],
|
||||
"model-00002-of-00003.safetensors": ["model.layers.1.weight", "mtp.a.weight"],
|
||||
"model-00003-of-00003.safetensors": ["mtp.b.weight"],
|
||||
}
|
||||
for filename, names in shards.items():
|
||||
save_file({name: torch.zeros(1) for name in names}, tmp_path / filename)
|
||||
return tmp_path
|
||||
|
||||
|
||||
def test_filter_by_weight_name_drops_shards_without_wanted_weights(mtp_checkpoint):
|
||||
files = sorted(str(f) for f in mtp_checkpoint.glob("*.safetensors"))
|
||||
|
||||
assert filter_safetensors_files_by_weight_name(files, _skip_non_mtp) == files[1:]
|
||||
# A rule that rejects everything must not leave the loader without files.
|
||||
assert filter_safetensors_files_by_weight_name(files, lambda _: True) == files
|
||||
|
||||
|
||||
def test_loader_does_not_read_shards_the_model_skips(mtp_checkpoint, monkeypatch):
|
||||
"""Whole-file loaders such as InstantTensor only see the kept shards."""
|
||||
opened: list[list[str]] = []
|
||||
|
||||
def fake_iterator(hf_weights_files, *args, **kwargs):
|
||||
opened.append(sorted(os.path.basename(f) for f in hf_weights_files))
|
||||
return iter(())
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.model_loader.default_loader."
|
||||
"instanttensor_weights_iterator",
|
||||
fake_iterator,
|
||||
)
|
||||
|
||||
class Drafter(torch.nn.Module):
|
||||
def is_unused_checkpoint_weight(self, name: str) -> bool:
|
||||
return _skip_non_mtp(name)
|
||||
|
||||
loader = DefaultModelLoader(LoadConfig(load_format="instanttensor"))
|
||||
model_config = SimpleNamespace(model=str(mtp_checkpoint), revision=None)
|
||||
list(loader.get_all_weights(model_config, Drafter()))
|
||||
list(loader.get_all_weights(model_config, torch.nn.Module()))
|
||||
|
||||
assert opened == [
|
||||
["model-00002-of-00003.safetensors", "model-00003-of-00003.safetensors"],
|
||||
sorted(f.name for f in mtp_checkpoint.glob("*.safetensors")),
|
||||
]
|
||||
|
||||
@@ -6,7 +6,7 @@ import dataclasses
|
||||
import glob
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Generator, Iterable
|
||||
from collections.abc import Callable, Generator, Iterable
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import torch
|
||||
@@ -28,6 +28,7 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
filter_duplicate_safetensors_files,
|
||||
filter_files_not_needed_for_inference,
|
||||
filter_mm_encoder_only_safetensors_files,
|
||||
filter_safetensors_files_by_weight_name,
|
||||
get_quant_config,
|
||||
instanttensor_weights_iterator,
|
||||
maybe_download_from_modelscope,
|
||||
@@ -75,6 +76,10 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
allow_patterns_overrides: list[str] | None = None
|
||||
"""If defined, weights will load exclusively using these patterns."""
|
||||
|
||||
is_unused_weight: Callable[[str], bool] | None = None
|
||||
"""If defined, safetensors shards holding only weights whose checkpoint
|
||||
name (before *prefix*) it accepts are not read."""
|
||||
|
||||
counter_before_loading_weights: float = 0.0
|
||||
counter_after_loading_weights: float = 0.0
|
||||
|
||||
@@ -283,6 +288,10 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
f"`{source.model_or_path}`; check language_model prefixes "
|
||||
f"{self._encoder_only_lm_prefixes}"
|
||||
)
|
||||
if source.is_unused_weight is not None and use_safetensors:
|
||||
hf_weights_files = filter_safetensors_files_by_weight_name(
|
||||
hf_weights_files, source.is_unused_weight
|
||||
)
|
||||
if self.load_config.load_format == "npcache":
|
||||
# Currently np_cache only support *.bin checkpoints
|
||||
assert use_safetensors is False
|
||||
@@ -359,6 +368,7 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
prefix="",
|
||||
fall_back_to_pt=getattr(model, "fall_back_to_pt_during_load", True),
|
||||
allow_patterns_overrides=getattr(model, "allow_patterns_overrides", None),
|
||||
is_unused_weight=getattr(model, "is_unused_checkpoint_weight", None),
|
||||
)
|
||||
yield from self._get_weights_iterator(primary_weights)
|
||||
|
||||
|
||||
@@ -785,6 +785,42 @@ def filter_mm_encoder_only_safetensors_files(
|
||||
return kept
|
||||
|
||||
|
||||
def filter_safetensors_files_by_weight_name(
|
||||
hf_weights_files: list[str],
|
||||
is_unused_weight: Callable[[str], bool],
|
||||
) -> list[str]:
|
||||
"""Drop safetensors shards in which `is_unused_weight` accepts every tensor.
|
||||
|
||||
Loaders that read whole files (InstantTensor, fastsafetensors, multi-thread,
|
||||
eager, prefetch) cannot skip single tensors, so dropping shards is their
|
||||
only way to avoid reading weights the model does not load.
|
||||
|
||||
Args:
|
||||
hf_weights_files: Safetensors shard paths.
|
||||
is_unused_weight: Returns True for checkpoint weight names the model does
|
||||
not load.
|
||||
|
||||
Returns:
|
||||
The shards holding at least one wanted tensor, or `hf_weights_files`
|
||||
unchanged when no shard does.
|
||||
|
||||
"""
|
||||
kept: list[str] = []
|
||||
for st_file in hf_weights_files:
|
||||
with safe_open(st_file, framework="pt") as f:
|
||||
if not all(is_unused_weight(name) for name in f.keys()): # noqa: SIM118
|
||||
kept.append(st_file)
|
||||
if not kept:
|
||||
return hf_weights_files
|
||||
if len(kept) < len(hf_weights_files):
|
||||
logger.info_once(
|
||||
"Skipped %d/%d safetensors shard(s) holding no weights the model loads",
|
||||
len(hf_weights_files) - len(kept),
|
||||
len(hf_weights_files),
|
||||
)
|
||||
return kept
|
||||
|
||||
|
||||
# explicitly use pure text format, with a newline at the end
|
||||
# this makes it impossible to see the animation in the progress bar
|
||||
# but will avoid messing up with ray or multiprocessing, which wraps
|
||||
|
||||
@@ -426,6 +426,11 @@ class NemotronHMTP(nn.Module, SupportsPP, SupportsQuant):
|
||||
)
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
def is_unused_checkpoint_weight(self, name: str) -> bool:
|
||||
"""Only MTP, embedding and LM head weights are loaded."""
|
||||
name = name.removeprefix("language_model.")
|
||||
return not name.startswith(("mtp.", "lm_head.")) and "embeddings" not in name
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load MTP weights with proper name remapping."""
|
||||
stacked_params_mapping = [
|
||||
@@ -456,17 +461,11 @@ class NemotronHMTP(nn.Module, SupportsPP, SupportsQuant):
|
||||
# MTP weights are nested in "language_model."
|
||||
# in Multimodal Nemotron-H checkpoints.
|
||||
name = name.removeprefix("language_model.")
|
||||
if self.is_unused_checkpoint_weight(name):
|
||||
continue
|
||||
is_lm_head_weight = name.startswith("lm_head.")
|
||||
if is_lm_head_weight:
|
||||
self.has_own_lm_head = True
|
||||
# Only process MTP and LM head weights -
|
||||
# skip all non-MTP and non-LM head weights
|
||||
if (
|
||||
not name.startswith("mtp.")
|
||||
and "embeddings" not in name
|
||||
and not is_lm_head_weight
|
||||
):
|
||||
continue
|
||||
# Skip rotary embeddings (computed, not loaded)
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
|
||||
@@ -300,16 +300,20 @@ class Qwen3_5MTP(LocalArgmaxMixin, nn.Module, SupportsMultiModal, SupportsPP):
|
||||
) -> torch.Tensor | None:
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
def is_unused_checkpoint_weight(self, name: str) -> bool:
|
||||
return not name.startswith("mtp.") and not any(
|
||||
key in name for key in ["embed_tokens", "lm_head"]
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
def remap_weight_names(weights):
|
||||
for name, weight in weights:
|
||||
if self.is_unused_checkpoint_weight(name):
|
||||
continue
|
||||
if name.startswith("mtp."):
|
||||
name = name.replace("mtp.", "model.")
|
||||
elif any(key in name for key in ["embed_tokens", "lm_head"]):
|
||||
if "embed_tokens" in name:
|
||||
name = name.replace("language_model.", "")
|
||||
else:
|
||||
continue
|
||||
elif "embed_tokens" in name:
|
||||
name = name.replace("language_model.", "")
|
||||
yield name, weight
|
||||
|
||||
loader = AutoWeightsLoader(self)
|
||||
|
||||
@@ -227,15 +227,18 @@ class Qwen3NextMTP(nn.Module, QwenNextMixtureOfExperts):
|
||||
) -> torch.Tensor | None:
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
shared_weight_names = ["embed_tokens", "lm_head"]
|
||||
def is_unused_checkpoint_weight(self, name: str) -> bool:
|
||||
return not name.startswith("mtp.") and not any(
|
||||
key in name for key in ["embed_tokens", "lm_head"]
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
def remap_weight_names(weights):
|
||||
for name, weight in weights:
|
||||
if self.is_unused_checkpoint_weight(name):
|
||||
continue
|
||||
if name.startswith("mtp."):
|
||||
name = name.replace("mtp.", "model.")
|
||||
elif not any(key in name for key in shared_weight_names):
|
||||
continue
|
||||
yield name, weight
|
||||
|
||||
loader = AutoWeightsLoader(self)
|
||||
|
||||
@@ -251,6 +251,12 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts, SupportsPP):
|
||||
name = name.replace(f"model.layers.{spec_layer}.", "model.")
|
||||
return name
|
||||
|
||||
def is_unused_checkpoint_weight(self, name: str) -> bool:
|
||||
return (
|
||||
get_spec_layer_idx_from_weight_name(self.config, name) is None
|
||||
and "embed_tokens" not in name
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
@@ -278,7 +284,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts, SupportsPP):
|
||||
loaded_params: set[str] = set()
|
||||
_pending_wk_fp8: dict = {}
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
if "rotary_emb.inv_freq" in name or self.is_unused_checkpoint_weight(name):
|
||||
continue
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
|
||||
if spec_layer is None:
|
||||
|
||||
@@ -356,6 +356,12 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts, SupportsPP):
|
||||
name = name.replace(f"model.layers.{spec_layer}.", "model.")
|
||||
return name
|
||||
|
||||
def is_unused_checkpoint_weight(self, name: str) -> bool:
|
||||
return (
|
||||
get_spec_layer_idx_from_weight_name(self.config, name) is None
|
||||
and "embed_tokens" not in name
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
@@ -378,7 +384,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts, SupportsPP):
|
||||
loaded_params: set[str] = set()
|
||||
_pending_wk_fp8: dict = {}
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
if "rotary_emb.inv_freq" in name or self.is_unused_checkpoint_weight(name):
|
||||
continue
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
|
||||
if spec_layer is None:
|
||||
|
||||
@@ -340,6 +340,26 @@ class DeepSeekV4MTP(nn.Module):
|
||||
) -> torch.Tensor | None:
|
||||
return self.model.compute_logits(hidden_states, spec_step_idx)
|
||||
|
||||
def _to_spec_layer_name(self, name: str) -> str:
|
||||
"""Remap V4 `mtp.{i}.*` names to `model.layers.{num_hidden_layers + i}.*`
|
||||
so that get_spec_layer_idx_from_weight_name can identify them."""
|
||||
mtp_layer_idx = 0
|
||||
for subname in name.split("."):
|
||||
try:
|
||||
# we use the first encountered integer
|
||||
mtp_layer_idx = int(subname)
|
||||
break
|
||||
except ValueError:
|
||||
continue
|
||||
return name.replace(
|
||||
f"mtp.{mtp_layer_idx}.",
|
||||
f"model.layers.{self.config.num_hidden_layers + mtp_layer_idx}.",
|
||||
)
|
||||
|
||||
def is_unused_checkpoint_weight(self, name: str) -> bool:
|
||||
name = self._to_spec_layer_name(name)
|
||||
return get_spec_layer_idx_from_weight_name(self.config, name) is None
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# Weight name remapping for checkpoint compatibility.
|
||||
# Maps checkpoint weight paths to model parameter paths.
|
||||
@@ -356,16 +376,6 @@ class DeepSeekV4MTP(nn.Module):
|
||||
name = name.replace(old_pattern, new_pattern)
|
||||
return name
|
||||
|
||||
def _find_mtp_layer_idx(name: str) -> int:
|
||||
subnames = name.split(".")
|
||||
for subname in subnames:
|
||||
try:
|
||||
# we return the first encountered integer
|
||||
return int(subname)
|
||||
except ValueError:
|
||||
continue
|
||||
return 0
|
||||
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("gate_up_proj", "w1", 0),
|
||||
@@ -409,15 +419,7 @@ class DeepSeekV4MTP(nn.Module):
|
||||
)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
mtp_layer_idx = _find_mtp_layer_idx(name)
|
||||
# V4 checkpoints store MTP weights as `mtp.{i}.*`; remap to
|
||||
# `model.layers.{num_hidden_layers + i}.*` so that
|
||||
# get_spec_layer_idx_from_weight_name can identify them.
|
||||
name = name.replace(
|
||||
f"mtp.{mtp_layer_idx}.",
|
||||
f"model.layers.{self.config.num_hidden_layers + mtp_layer_idx}.",
|
||||
)
|
||||
|
||||
name = self._to_spec_layer_name(name)
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
|
||||
if spec_layer is None:
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user