diff --git a/tests/model_executor/model_loader/test_filter_duplicate_safetensors.py b/tests/model_executor/model_loader/test_filter_duplicate_safetensors.py index babc6c061a..da04d0108f 100644 --- a/tests/model_executor/model_loader/test_filter_duplicate_safetensors.py +++ b/tests/model_executor/model_loader/test_filter_duplicate_safetensors.py @@ -4,11 +4,17 @@ import json import os import tempfile +from types import SimpleNamespace import pytest +import torch +from safetensors.torch import save_file +from vllm.config.load import LoadConfig +from vllm.model_executor.model_loader import DefaultModelLoader from vllm.model_executor.model_loader.weight_utils import ( filter_duplicate_safetensors_files, + filter_safetensors_files_by_weight_name, ) @@ -77,3 +83,57 @@ def test_filter_duplicate_safetensors_files_all_exist(): if __name__ == "__main__": test_filter_duplicate_safetensors_files_missing_weight() test_filter_duplicate_safetensors_files_all_exist() + + +def _skip_non_mtp(name: str) -> bool: + return not name.startswith("mtp.") + + +@pytest.fixture +def mtp_checkpoint(tmp_path): + """Three shards: target only, target + MTP, MTP only.""" + shards = { + "model-00001-of-00003.safetensors": ["model.layers.0.weight"], + "model-00002-of-00003.safetensors": ["model.layers.1.weight", "mtp.a.weight"], + "model-00003-of-00003.safetensors": ["mtp.b.weight"], + } + for filename, names in shards.items(): + save_file({name: torch.zeros(1) for name in names}, tmp_path / filename) + return tmp_path + + +def test_filter_by_weight_name_drops_shards_without_wanted_weights(mtp_checkpoint): + files = sorted(str(f) for f in mtp_checkpoint.glob("*.safetensors")) + + assert filter_safetensors_files_by_weight_name(files, _skip_non_mtp) == files[1:] + # A rule that rejects everything must not leave the loader without files. + assert filter_safetensors_files_by_weight_name(files, lambda _: True) == files + + +def test_loader_does_not_read_shards_the_model_skips(mtp_checkpoint, monkeypatch): + """Whole-file loaders such as InstantTensor only see the kept shards.""" + opened: list[list[str]] = [] + + def fake_iterator(hf_weights_files, *args, **kwargs): + opened.append(sorted(os.path.basename(f) for f in hf_weights_files)) + return iter(()) + + monkeypatch.setattr( + "vllm.model_executor.model_loader.default_loader." + "instanttensor_weights_iterator", + fake_iterator, + ) + + class Drafter(torch.nn.Module): + def is_unused_checkpoint_weight(self, name: str) -> bool: + return _skip_non_mtp(name) + + loader = DefaultModelLoader(LoadConfig(load_format="instanttensor")) + model_config = SimpleNamespace(model=str(mtp_checkpoint), revision=None) + list(loader.get_all_weights(model_config, Drafter())) + list(loader.get_all_weights(model_config, torch.nn.Module())) + + assert opened == [ + ["model-00002-of-00003.safetensors", "model-00003-of-00003.safetensors"], + sorted(f.name for f in mtp_checkpoint.glob("*.safetensors")), + ] diff --git a/vllm/model_executor/model_loader/default_loader.py b/vllm/model_executor/model_loader/default_loader.py index c3e09bccd4..9f1c25dd2e 100644 --- a/vllm/model_executor/model_loader/default_loader.py +++ b/vllm/model_executor/model_loader/default_loader.py @@ -6,7 +6,7 @@ import dataclasses import glob import os import time -from collections.abc import Generator, Iterable +from collections.abc import Callable, Generator, Iterable from typing import TYPE_CHECKING, cast import torch @@ -28,6 +28,7 @@ from vllm.model_executor.model_loader.weight_utils import ( filter_duplicate_safetensors_files, filter_files_not_needed_for_inference, filter_mm_encoder_only_safetensors_files, + filter_safetensors_files_by_weight_name, get_quant_config, instanttensor_weights_iterator, maybe_download_from_modelscope, @@ -75,6 +76,10 @@ class DefaultModelLoader(BaseModelLoader): allow_patterns_overrides: list[str] | None = None """If defined, weights will load exclusively using these patterns.""" + is_unused_weight: Callable[[str], bool] | None = None + """If defined, safetensors shards holding only weights whose checkpoint + name (before *prefix*) it accepts are not read.""" + counter_before_loading_weights: float = 0.0 counter_after_loading_weights: float = 0.0 @@ -283,6 +288,10 @@ class DefaultModelLoader(BaseModelLoader): f"`{source.model_or_path}`; check language_model prefixes " f"{self._encoder_only_lm_prefixes}" ) + if source.is_unused_weight is not None and use_safetensors: + hf_weights_files = filter_safetensors_files_by_weight_name( + hf_weights_files, source.is_unused_weight + ) if self.load_config.load_format == "npcache": # Currently np_cache only support *.bin checkpoints assert use_safetensors is False @@ -359,6 +368,7 @@ class DefaultModelLoader(BaseModelLoader): prefix="", fall_back_to_pt=getattr(model, "fall_back_to_pt_during_load", True), allow_patterns_overrides=getattr(model, "allow_patterns_overrides", None), + is_unused_weight=getattr(model, "is_unused_checkpoint_weight", None), ) yield from self._get_weights_iterator(primary_weights) diff --git a/vllm/model_executor/model_loader/weight_utils.py b/vllm/model_executor/model_loader/weight_utils.py index 12d557fa1c..c6a72c4c33 100644 --- a/vllm/model_executor/model_loader/weight_utils.py +++ b/vllm/model_executor/model_loader/weight_utils.py @@ -785,6 +785,42 @@ def filter_mm_encoder_only_safetensors_files( return kept +def filter_safetensors_files_by_weight_name( + hf_weights_files: list[str], + is_unused_weight: Callable[[str], bool], +) -> list[str]: + """Drop safetensors shards in which `is_unused_weight` accepts every tensor. + + Loaders that read whole files (InstantTensor, fastsafetensors, multi-thread, + eager, prefetch) cannot skip single tensors, so dropping shards is their + only way to avoid reading weights the model does not load. + + Args: + hf_weights_files: Safetensors shard paths. + is_unused_weight: Returns True for checkpoint weight names the model does + not load. + + Returns: + The shards holding at least one wanted tensor, or `hf_weights_files` + unchanged when no shard does. + + """ + kept: list[str] = [] + for st_file in hf_weights_files: + with safe_open(st_file, framework="pt") as f: + if not all(is_unused_weight(name) for name in f.keys()): # noqa: SIM118 + kept.append(st_file) + if not kept: + return hf_weights_files + if len(kept) < len(hf_weights_files): + logger.info_once( + "Skipped %d/%d safetensors shard(s) holding no weights the model loads", + len(hf_weights_files) - len(kept), + len(hf_weights_files), + ) + return kept + + # explicitly use pure text format, with a newline at the end # this makes it impossible to see the animation in the progress bar # but will avoid messing up with ray or multiprocessing, which wraps diff --git a/vllm/model_executor/models/nemotron_h_mtp.py b/vllm/model_executor/models/nemotron_h_mtp.py index 519f85a631..af93cc62ae 100644 --- a/vllm/model_executor/models/nemotron_h_mtp.py +++ b/vllm/model_executor/models/nemotron_h_mtp.py @@ -426,6 +426,11 @@ class NemotronHMTP(nn.Module, SupportsPP, SupportsQuant): ) return self.logits_processor(self.lm_head, hidden_states) + def is_unused_checkpoint_weight(self, name: str) -> bool: + """Only MTP, embedding and LM head weights are loaded.""" + name = name.removeprefix("language_model.") + return not name.startswith(("mtp.", "lm_head.")) and "embeddings" not in name + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: """Load MTP weights with proper name remapping.""" stacked_params_mapping = [ @@ -456,17 +461,11 @@ class NemotronHMTP(nn.Module, SupportsPP, SupportsQuant): # MTP weights are nested in "language_model." # in Multimodal Nemotron-H checkpoints. name = name.removeprefix("language_model.") + if self.is_unused_checkpoint_weight(name): + continue is_lm_head_weight = name.startswith("lm_head.") if is_lm_head_weight: self.has_own_lm_head = True - # Only process MTP and LM head weights - - # skip all non-MTP and non-LM head weights - if ( - not name.startswith("mtp.") - and "embeddings" not in name - and not is_lm_head_weight - ): - continue # Skip rotary embeddings (computed, not loaded) if "rotary_emb.inv_freq" in name: continue diff --git a/vllm/model_executor/models/qwen3_5_mtp.py b/vllm/model_executor/models/qwen3_5_mtp.py index c17d17014f..d7fb1d6e09 100644 --- a/vllm/model_executor/models/qwen3_5_mtp.py +++ b/vllm/model_executor/models/qwen3_5_mtp.py @@ -300,16 +300,20 @@ class Qwen3_5MTP(LocalArgmaxMixin, nn.Module, SupportsMultiModal, SupportsPP): ) -> torch.Tensor | None: return self.logits_processor(self.lm_head, hidden_states) + def is_unused_checkpoint_weight(self, name: str) -> bool: + return not name.startswith("mtp.") and not any( + key in name for key in ["embed_tokens", "lm_head"] + ) + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: def remap_weight_names(weights): for name, weight in weights: + if self.is_unused_checkpoint_weight(name): + continue if name.startswith("mtp."): name = name.replace("mtp.", "model.") - elif any(key in name for key in ["embed_tokens", "lm_head"]): - if "embed_tokens" in name: - name = name.replace("language_model.", "") - else: - continue + elif "embed_tokens" in name: + name = name.replace("language_model.", "") yield name, weight loader = AutoWeightsLoader(self) diff --git a/vllm/model_executor/models/qwen3_next_mtp.py b/vllm/model_executor/models/qwen3_next_mtp.py index e5e306bb86..ef08878eaa 100644 --- a/vllm/model_executor/models/qwen3_next_mtp.py +++ b/vllm/model_executor/models/qwen3_next_mtp.py @@ -227,15 +227,18 @@ class Qwen3NextMTP(nn.Module, QwenNextMixtureOfExperts): ) -> torch.Tensor | None: return self.logits_processor(self.lm_head, hidden_states) - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - shared_weight_names = ["embed_tokens", "lm_head"] + def is_unused_checkpoint_weight(self, name: str) -> bool: + return not name.startswith("mtp.") and not any( + key in name for key in ["embed_tokens", "lm_head"] + ) + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: def remap_weight_names(weights): for name, weight in weights: + if self.is_unused_checkpoint_weight(name): + continue if name.startswith("mtp."): name = name.replace("mtp.", "model.") - elif not any(key in name for key in shared_weight_names): - continue yield name, weight loader = AutoWeightsLoader(self) diff --git a/vllm/models/deepseek_v32/amd/mtp.py b/vllm/models/deepseek_v32/amd/mtp.py index 70ddeae374..6230348ef4 100644 --- a/vllm/models/deepseek_v32/amd/mtp.py +++ b/vllm/models/deepseek_v32/amd/mtp.py @@ -251,6 +251,12 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts, SupportsPP): name = name.replace(f"model.layers.{spec_layer}.", "model.") return name + def is_unused_checkpoint_weight(self, name: str) -> bool: + return ( + get_spec_layer_idx_from_weight_name(self.config, name) is None + and "embed_tokens" not in name + ) + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: stacked_params_mapping = [ ("gate_up_proj", "gate_proj", 0), @@ -278,7 +284,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts, SupportsPP): loaded_params: set[str] = set() _pending_wk_fp8: dict = {} for name, loaded_weight in weights: - if "rotary_emb.inv_freq" in name: + if "rotary_emb.inv_freq" in name or self.is_unused_checkpoint_weight(name): continue spec_layer = get_spec_layer_idx_from_weight_name(self.config, name) if spec_layer is None: diff --git a/vllm/models/deepseek_v32/nvidia/mtp.py b/vllm/models/deepseek_v32/nvidia/mtp.py index bdc6b95c15..d4dee7b9d4 100644 --- a/vllm/models/deepseek_v32/nvidia/mtp.py +++ b/vllm/models/deepseek_v32/nvidia/mtp.py @@ -356,6 +356,12 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts, SupportsPP): name = name.replace(f"model.layers.{spec_layer}.", "model.") return name + def is_unused_checkpoint_weight(self, name: str) -> bool: + return ( + get_spec_layer_idx_from_weight_name(self.config, name) is None + and "embed_tokens" not in name + ) + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: stacked_params_mapping = [ ("gate_up_proj", "gate_proj", 0), @@ -378,7 +384,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts, SupportsPP): loaded_params: set[str] = set() _pending_wk_fp8: dict = {} for name, loaded_weight in weights: - if "rotary_emb.inv_freq" in name: + if "rotary_emb.inv_freq" in name or self.is_unused_checkpoint_weight(name): continue spec_layer = get_spec_layer_idx_from_weight_name(self.config, name) if spec_layer is None: diff --git a/vllm/models/deepseek_v4/nvidia/mtp.py b/vllm/models/deepseek_v4/nvidia/mtp.py index 64547af038..c0f3471dd5 100644 --- a/vllm/models/deepseek_v4/nvidia/mtp.py +++ b/vllm/models/deepseek_v4/nvidia/mtp.py @@ -340,6 +340,26 @@ class DeepSeekV4MTP(nn.Module): ) -> torch.Tensor | None: return self.model.compute_logits(hidden_states, spec_step_idx) + def _to_spec_layer_name(self, name: str) -> str: + """Remap V4 `mtp.{i}.*` names to `model.layers.{num_hidden_layers + i}.*` + so that get_spec_layer_idx_from_weight_name can identify them.""" + mtp_layer_idx = 0 + for subname in name.split("."): + try: + # we use the first encountered integer + mtp_layer_idx = int(subname) + break + except ValueError: + continue + return name.replace( + f"mtp.{mtp_layer_idx}.", + f"model.layers.{self.config.num_hidden_layers + mtp_layer_idx}.", + ) + + def is_unused_checkpoint_weight(self, name: str) -> bool: + name = self._to_spec_layer_name(name) + return get_spec_layer_idx_from_weight_name(self.config, name) is None + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: # Weight name remapping for checkpoint compatibility. # Maps checkpoint weight paths to model parameter paths. @@ -356,16 +376,6 @@ class DeepSeekV4MTP(nn.Module): name = name.replace(old_pattern, new_pattern) return name - def _find_mtp_layer_idx(name: str) -> int: - subnames = name.split(".") - for subname in subnames: - try: - # we return the first encountered integer - return int(subname) - except ValueError: - continue - return 0 - stacked_params_mapping = [ # (param_name, shard_name, shard_id) ("gate_up_proj", "w1", 0), @@ -409,15 +419,7 @@ class DeepSeekV4MTP(nn.Module): ) for name, loaded_weight in weights: - mtp_layer_idx = _find_mtp_layer_idx(name) - # V4 checkpoints store MTP weights as `mtp.{i}.*`; remap to - # `model.layers.{num_hidden_layers + i}.*` so that - # get_spec_layer_idx_from_weight_name can identify them. - name = name.replace( - f"mtp.{mtp_layer_idx}.", - f"model.layers.{self.config.num_hidden_layers + mtp_layer_idx}.", - ) - + name = self._to_spec_layer_name(name) spec_layer = get_spec_layer_idx_from_weight_name(self.config, name) if spec_layer is None: continue