diff --git a/tests/model_executor/model_loader/test_mm_encoder_only_weight_filter.py b/tests/model_executor/model_loader/test_mm_encoder_only_weight_filter.py new file mode 100644 index 0000000000..210cd28d6f --- /dev/null +++ b/tests/model_executor/model_loader/test_mm_encoder_only_weight_filter.py @@ -0,0 +1,243 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for --mm-encoder-only safetensors shard filtering.""" + +import json +import os +import tempfile + +from transformers.utils import SAFE_WEIGHTS_INDEX_NAME + +from vllm.model_executor.model_loader.weight_utils import ( + filter_mm_encoder_only_safetensors_files, + resolve_mm_encoder_only_lm_prefixes, +) +from vllm.model_executor.models.utils import WeightsMapper + + +def _write_index(folder: str, weight_map: dict[str, str]) -> None: + with open(os.path.join(folder, SAFE_WEIGHTS_INDEX_NAME), "w") as f: + json.dump({"weight_map": weight_map}, f) + + +def _qwen25_mapper() -> WeightsMapper: + """Qwen2.5-VL dual layout mapper (nested + flat catch-all).""" + return WeightsMapper( + orig_to_new_prefix={ + "model.language_model.": "language_model.model.", + "model.visual.": "visual.", + "lm_head.": "language_model.lm_head.", + "model.": "language_model.model.", + } + ) + + +def _qwen3_mapper() -> WeightsMapper: + return WeightsMapper( + orig_to_new_prefix={ + "model.visual.": "visual.", + "lm_head.": "language_model.lm_head.", + "model.language_model.": "language_model.model.", + } + ) + + +def _molmo_mapper() -> WeightsMapper: + return WeightsMapper( + orig_to_new_prefix={ + "model.vision_backbone.": "vision_backbone.", + "model.transformer.blocks.": "model.layers.", + "model.transformer.ln_f.": "model.norm.", + "model.transformer.mlp.down_proj.": "lm_head.", + } + ) + + +def test_keeps_vision_shards_drops_language_only(): + with tempfile.TemporaryDirectory() as folder: + files = [ + os.path.join(folder, "model-00001-of-000003.safetensors"), + os.path.join(folder, "model-00002-of-000003.safetensors"), + os.path.join(folder, "model-00003-of-000003.safetensors"), + ] + for path in files: + open(path, "wb").close() + + _write_index( + folder, + { + "language_model.layers.0.weight": "model-00001-of-000003.safetensors", + "language_model.layers.1.weight": "model-00002-of-000003.safetensors", + "vision_tower.blocks.0.weight": "model-00003-of-000003.safetensors", + "mm_projector.weight": "model-00003-of-000003.safetensors", + }, + ) + + kept = filter_mm_encoder_only_safetensors_files( + files, + folder, + SAFE_WEIGHTS_INDEX_NAME, + ("language_model.",), + ) + assert kept == [files[2]] + + +def test_keeps_mixed_shard_with_vision_and_lm(): + with tempfile.TemporaryDirectory() as folder: + mixed = os.path.join(folder, "model-00001-of-000001.safetensors") + open(mixed, "wb").close() + _write_index( + folder, + { + "language_model.embed.weight": "model-00001-of-000001.safetensors", + "vision_tower.patch.weight": "model-00001-of-000001.safetensors", + }, + ) + + kept = filter_mm_encoder_only_safetensors_files( + [mixed], + folder, + SAFE_WEIGHTS_INDEX_NAME, + ("language_model.",), + ) + assert kept == [mixed] + + +def test_no_index_returns_unchanged(): + with tempfile.TemporaryDirectory() as folder: + files = [os.path.join(folder, "model.safetensors")] + open(files[0], "wb").close() + kept = filter_mm_encoder_only_safetensors_files( + files, + folder, + SAFE_WEIGHTS_INDEX_NAME, + ("language_model.",), + ) + assert kept == files + + +def test_resolve_returns_module_prefixes_without_hf_hardcode(): + """No global Qwen HF nest list — just vLLM module prefixes.""" + prefixes = resolve_mm_encoder_only_lm_prefixes(["language_model"]) + assert prefixes == ("language_model.",) + # With Qwen mapper, still module prefixes (keys classified via mapper). + assert resolve_mm_encoder_only_lm_prefixes( + ["language_model"], weights_mapper=_qwen25_mapper() + ) == ("language_model.",) + + +def test_resolve_empty_language_model_names_fail_closed(): + """Missing/empty ``_language_model_names`` → None (no language_model. default).""" + assert resolve_mm_encoder_only_lm_prefixes(None) is None + assert resolve_mm_encoder_only_lm_prefixes([]) is None + assert resolve_mm_encoder_only_lm_prefixes(()) is None + + +def test_resolve_skips_filter_for_shared_hf_root(): + """Shared HF root (Molmo-shaped mapper) → fail-closed, not magic name list.""" + assert ( + resolve_mm_encoder_only_lm_prefixes(["model"], weights_mapper=_molmo_mapper()) + is None + ) + # Without a mapper, identity layout: module prefix is returned as-is. + assert resolve_mm_encoder_only_lm_prefixes(["model"]) == ("model.",) + assert resolve_mm_encoder_only_lm_prefixes(["model."]) == ("model.",) + + +def test_resolve_keeps_llm_prefix_without_model_broadening(): + prefixes = resolve_mm_encoder_only_lm_prefixes(["llm"]) + assert prefixes == ("llm.",) + + +def test_qwen3_nested_index_skips_lm_keeps_visual(): + with tempfile.TemporaryDirectory() as folder: + files = [ + os.path.join(folder, "model-00001-of-000002.safetensors"), + os.path.join(folder, "model-00002-of-000002.safetensors"), + ] + for path in files: + open(path, "wb").close() + _write_index( + folder, + { + "model.language_model.layers.0.weight": ( + "model-00001-of-000002.safetensors" + ), + "lm_head.weight": "model-00001-of-000002.safetensors", + "model.visual.blocks.0.weight": "model-00002-of-000002.safetensors", + }, + ) + prefixes = resolve_mm_encoder_only_lm_prefixes( + ["language_model"], weights_mapper=_qwen3_mapper() + ) + assert prefixes is not None + kept = filter_mm_encoder_only_safetensors_files( + files, + folder, + SAFE_WEIGHTS_INDEX_NAME, + prefixes, + weights_mapper=_qwen3_mapper(), + ) + assert kept == [files[1]] + + +def test_qwen25_flat_index_skips_lm_keeps_visual(): + with tempfile.TemporaryDirectory() as folder: + files = [ + os.path.join(folder, "model-00001-of-000002.safetensors"), + os.path.join(folder, "model-00002-of-000002.safetensors"), + ] + for path in files: + open(path, "wb").close() + _write_index( + folder, + { + "model.layers.0.self_attn.q_proj.weight": ( + "model-00001-of-000002.safetensors" + ), + "model.embed_tokens.weight": "model-00001-of-000002.safetensors", + "model.norm.weight": "model-00001-of-000002.safetensors", + "lm_head.weight": "model-00001-of-000002.safetensors", + "visual.blocks.0.weight": "model-00002-of-000002.safetensors", + }, + ) + prefixes = resolve_mm_encoder_only_lm_prefixes( + ["language_model"], weights_mapper=_qwen25_mapper() + ) + assert prefixes is not None + kept = filter_mm_encoder_only_safetensors_files( + files, + folder, + SAFE_WEIGHTS_INDEX_NAME, + prefixes, + weights_mapper=_qwen25_mapper(), + ) + assert kept == [files[1]] + + +def test_molmo_shaped_index_not_filtered_when_deny_disabled(): + """Shared-root LM attr → resolve None; callers leave the file list alone.""" + assert ( + resolve_mm_encoder_only_lm_prefixes(["model"], weights_mapper=_molmo_mapper()) + is None + ) + with tempfile.TemporaryDirectory() as folder: + files = [ + os.path.join(folder, "vision.safetensors"), + os.path.join(folder, "lm.safetensors"), + ] + for path in files: + open(path, "wb").close() + _write_index( + folder, + { + "model.vision_backbone.patch.weight": "vision.safetensors", + "model.transformer.blocks.0.weight": "lm.safetensors", + }, + ) + # Document why raw HF startswith("model.") is unsafe without fail-closed: + # whole-shard deny would also drop vision. + if_applied_naively = filter_mm_encoder_only_safetensors_files( + files, folder, SAFE_WEIGHTS_INDEX_NAME, ("model.",) + ) + assert if_applied_naively == [] diff --git a/tests/model_executor/model_loader/test_registry.py b/tests/model_executor/model_loader/test_registry.py index 95b797bb51..a795497a11 100644 --- a/tests/model_executor/model_loader/test_registry.py +++ b/tests/model_executor/model_loader/test_registry.py @@ -79,7 +79,7 @@ def test_default_loader_hf_still_falls_back_to_pt(tmp_path): # Control: load_format="hf" still picks up .pt weights via fallback. (tmp_path / "model.pt").write_bytes(b"\x00\x00\x00\x00") loader = DefaultModelLoader(LoadConfig(load_format="hf")) - _, files, use_safetensors = loader._prepare_weights( + _, files, use_safetensors, _ = loader._prepare_weights( str(tmp_path), None, None, diff --git a/vllm/model_executor/model_loader/default_loader.py b/vllm/model_executor/model_loader/default_loader.py index 039a577f80..c3e09bccd4 100644 --- a/vllm/model_executor/model_loader/default_loader.py +++ b/vllm/model_executor/model_loader/default_loader.py @@ -1,11 +1,13 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from __future__ import annotations + import dataclasses import glob import os import time from collections.abc import Generator, Iterable -from typing import cast +from typing import TYPE_CHECKING, cast import torch from torch import nn @@ -25,6 +27,7 @@ from vllm.model_executor.model_loader.weight_utils import ( fastsafetensors_weights_iterator, filter_duplicate_safetensors_files, filter_files_not_needed_for_inference, + filter_mm_encoder_only_safetensors_files, get_quant_config, instanttensor_weights_iterator, maybe_download_from_modelscope, @@ -32,11 +35,15 @@ from vllm.model_executor.model_loader.weight_utils import ( multi_thread_safetensors_weights_iterator, np_cache_weights_iterator, pt_weights_iterator, + resolve_mm_encoder_only_lm_prefixes, safetensors_weights_iterator, ) from vllm.tracing import instrument from vllm.transformers_utils.repo_utils import list_filtered_repo_files +if TYPE_CHECKING: + from vllm.model_executor.models.utils import WeightsMapper + logger = init_logger(__name__) @@ -74,6 +81,9 @@ class DefaultModelLoader(BaseModelLoader): def __init__(self, load_config: LoadConfig): super().__init__(load_config) self.local_expert_ids: set[int] | None = None + # Set in load_weights when --mm-encoder-only; used to drop LM-only shards. + self._encoder_only_lm_prefixes: tuple[str, ...] | None = None + self._encoder_only_weights_mapper: WeightsMapper | None = None extra_config = load_config.model_loader_extra_config if not isinstance(extra_config, dict): @@ -132,7 +142,7 @@ class DefaultModelLoader(BaseModelLoader): revision: str | None, fall_back_to_pt: bool, allow_patterns_overrides: list[str] | None, - ) -> tuple[str, list[str], bool]: + ) -> tuple[str, list[str], bool, str]: """Prepare weights for the model. If the model is not local, it will be downloaded.""" @@ -239,20 +249,40 @@ class DefaultModelLoader(BaseModelLoader): f"Cannot find any model weights with `{model_name_or_path}`" ) - return hf_folder, hf_weights_files, use_safetensors + return hf_folder, hf_weights_files, use_safetensors, index_file def _get_weights_iterator( - self, source: "Source" + self, source: Source ) -> Generator[tuple[str, torch.Tensor], None, None]: """Get an iterator for the model weights based on the load format.""" extra_config = self.load_config.model_loader_extra_config - hf_folder, hf_weights_files, use_safetensors = self._prepare_weights( - source.model_or_path, - source.subfolder, - source.revision, - source.fall_back_to_pt, - source.allow_patterns_overrides, + hf_folder, hf_weights_files, use_safetensors, index_file = ( + self._prepare_weights( + source.model_or_path, + source.subfolder, + source.revision, + source.fall_back_to_pt, + source.allow_patterns_overrides, + ) ) + if ( + self._encoder_only_lm_prefixes is not None + and use_safetensors + and hf_weights_files + ): + hf_weights_files = filter_mm_encoder_only_safetensors_files( + hf_weights_files, + hf_folder, + index_file, + self._encoder_only_lm_prefixes, + weights_mapper=self._encoder_only_weights_mapper, + ) + if not hf_weights_files: + raise RuntimeError( + "mm-encoder-only shard filter removed all weight files for " + f"`{source.model_or_path}`; check language_model prefixes " + f"{self._encoder_only_lm_prefixes}" + ) if self.load_config.load_format == "npcache": # Currently np_cache only support *.bin checkpoints assert use_safetensors is False @@ -348,6 +378,29 @@ class DefaultModelLoader(BaseModelLoader): allow_patterns_overrides=None, ) + def _init_mm_encoder_only_weight_filter( + self, model: nn.Module, model_config: ModelConfig + ) -> None: + """Skip pure language-model safetensors shards under --mm-encoder-only.""" + mm_config = model_config.multimodal_config + if mm_config is None or not mm_config.mm_encoder_only: + self._encoder_only_lm_prefixes = None + self._encoder_only_weights_mapper = None + return + + # Derive from _language_model_names; fail-closed on shared HF roots + # (Molmo/Phi-4-MM/Muse). Qwen nested/flat keys classified via mapper. + weights_mapper = cast( + "WeightsMapper | None", getattr(model, "hf_to_vllm_mapper", None) + ) + self._encoder_only_lm_prefixes = resolve_mm_encoder_only_lm_prefixes( + getattr(model, "_language_model_names", None), + weights_mapper=weights_mapper, + ) + self._encoder_only_weights_mapper = ( + weights_mapper if self._encoder_only_lm_prefixes is not None else None + ) + def _init_ep_weight_filter(self, model_config: ModelConfig) -> None: """Compute local expert ids for EP weight filtering. @@ -423,6 +476,7 @@ class DefaultModelLoader(BaseModelLoader): self.load_config.safetensors_load_strategy = "torchao" self._init_ep_weight_filter(model_config) + self._init_mm_encoder_only_weight_filter(model, model_config) loaded_weights = model.load_weights(self.get_all_weights(model_config, model)) diff --git a/vllm/model_executor/model_loader/weight_utils.py b/vllm/model_executor/model_loader/weight_utils.py index 452c03a8ce..2fa79a4e1c 100644 --- a/vllm/model_executor/model_loader/weight_utils.py +++ b/vllm/model_executor/model_loader/weight_utils.py @@ -15,7 +15,7 @@ import time from collections.abc import Callable, Generator, Iterable from contextlib import contextmanager from pathlib import Path -from typing import IO, Any +from typing import IO, TYPE_CHECKING, Any import filelock import huggingface_hub.constants @@ -61,6 +61,9 @@ except ImportError: from vllm.model_executor.layers.quantization.torchao import torchao_version_at_least +if TYPE_CHECKING: + from vllm.model_executor.models.utils import WeightsMapper + logger = init_logger(__name__) # use system-level temp directory for file locks, so that multiple users @@ -605,6 +608,171 @@ def filter_files_not_needed_for_inference(hf_weights_files: list[str]) -> list[s return hf_weights_files +def _normalize_module_prefixes( + language_model_names: Iterable[str] | None, +) -> tuple[str, ...]: + names = tuple(language_model_names or ()) + return tuple(n if n.endswith(".") else f"{n}." for n in names) + + +def _mapped_weight_name(weights_mapper: "WeightsMapper | None", key: str) -> str | None: + """Apply ``WeightsMapper.map_name`` when present; else identity.""" + if weights_mapper is None: + return key + return weights_mapper.map_name(key) + + +def _hf_prefix_maps_to_lm( + new_name: str | None, lm_module_prefixes: tuple[str, ...] +) -> bool: + if not new_name: + return False + new_p = new_name if new_name.endswith(".") else f"{new_name}." + return any(new_p.startswith(p) or p.startswith(new_p) for p in lm_module_prefixes) + + +def _shared_hf_root_module_prefixes( + lm_module_prefixes: tuple[str, ...], + weights_mapper: "WeightsMapper | None", +) -> tuple[str, ...]: + """Module prefixes that are also an HF root for non-LM weights. + + E.g. Molmo/Phi-4-MM/Muse mark LM as ``model`` while vision HF keys stay + under ``model.vision_*`` / ``model.embed_tokens_extend.*``. Detected via + ``WeightsMapper.orig_to_new_prefix`` (no hard-coded name list). + """ + if weights_mapper is None: + return () + mapping = weights_mapper.orig_to_new_prefix + if not mapping: + return () + + non_lm_origs: list[str] = [] + for orig, new in mapping.items(): + orig_p = orig if orig.endswith(".") else f"{orig}." + if not _hf_prefix_maps_to_lm(new, lm_module_prefixes): + non_lm_origs.append(orig_p) + + if not non_lm_origs: + return () + + shared = [ + p + for p in lm_module_prefixes + if any(orig.startswith(p) for orig in non_lm_origs) + ] + return tuple(shared) + + +def resolve_mm_encoder_only_lm_prefixes( + language_model_names: Iterable[str] | None, + *, + weights_mapper: "WeightsMapper | None" = None, +) -> tuple[str, ...] | None: + """Resolve vLLM LM *module* prefixes for ``--mm-encoder-only`` shard skip. + + Prefixes come from ``_language_model_names``. Classification of HF index + keys is done later via optional ``weights_mapper`` (see + ``filter_mm_encoder_only_safetensors_files``), so this helper does **not** + hard-code Qwen/HF nest lists. + + Returns ``None`` (leave the safetensors file list unchanged) when + ``_language_model_names`` is empty/missing, or when a module prefix is a + shared HF checkpoint root for both LM and non-LM weights — fail-closed so + Molmo / Phi-4-MM / Muse cannot under-load the encoder. + """ + prefixes = _normalize_module_prefixes(language_model_names) + if not prefixes: + return None + shared = _shared_hf_root_module_prefixes(prefixes, weights_mapper) + if shared: + logger.warning_once( + "mm-encoder-only whole-shard filter not applied: LM module " + "prefix(es) %s share an HF checkpoint root with non-LM weights " + "(via WeightsMapper). Encoder-only load keeps the full " + "safetensors file list; finer HF prefixes or key-level " + "filtering can enable skip later.", + shared, + ) + return None + return prefixes + + +def filter_mm_encoder_only_safetensors_files( + hf_weights_files: list[str], + hf_folder: str, + index_file: str, + language_model_prefixes: Iterable[str], + *, + weights_mapper: "WeightsMapper | None" = None, +) -> list[str]: + """Drop safetensors shards that only contain language-model weights. + + Used with ``--mm-encoder-only`` so Encoder-only EPD instances avoid reading + pure LM shards from disk/DRAM. + + Each HF index key is classified by mapping through ``weights_mapper`` (when + provided) and testing the *vLLM* name against ``language_model_prefixes``. + Without a mapper, HF names are compared directly (identity checkpoint + layout, e.g. Kimi ``language_model.*``). + + A shard is kept if it contains any non-LM key. Without an index file, + returns ``hf_weights_files`` unchanged. + """ + prefixes = tuple(language_model_prefixes) + if not prefixes: + return hf_weights_files + + index_path = os.path.join(hf_folder, index_file) + if not os.path.isfile(index_path): + logger.warning_once( + "mm-encoder-only shard filter skipped: index file %s not found", + index_path, + ) + return hf_weights_files + + with open(index_path) as f: + weight_map: dict[str, str] = json.load(f)["weight_map"] + + keys_by_file: dict[str, list[str]] = {} + for weight_name, weight_file in weight_map.items(): + keys_by_file.setdefault(weight_file, []).append(weight_name) + + def _is_lm_key(key: str) -> bool: + mapped = _mapped_weight_name(weights_mapper, key) + if mapped is None: + return False + return any(mapped.startswith(p) for p in prefixes) + + def _is_lm_only(weight_file: str) -> bool: + keys = keys_by_file.get(weight_file) + if not keys: + # Not listed in the index (e.g. extra matched glob) — keep. + return False + return all(_is_lm_key(key) for key in keys) + + kept: list[str] = [] + skipped = 0 + for path in hf_weights_files: + rel = os.path.relpath(path, hf_folder) + # Index weight_map values are usually basenames; tolerate either form. + basename = os.path.basename(path) + if _is_lm_only(rel) or _is_lm_only(basename): + skipped += 1 + continue + kept.append(path) + + if skipped: + logger.info_once( + "mm-encoder-only: skipped %d/%d safetensors shard(s) that only " + "contain language-model weights (prefixes=%s)", + skipped, + skipped + len(kept), + prefixes, + ) + return kept + + # explicitly use pure text format, with a newline at the end # this makes it impossible to see the animation in the progress bar # but will avoid messing up with ray or multiprocessing, which wraps diff --git a/vllm/model_executor/models/utils.py b/vllm/model_executor/models/utils.py index 947a6f6276..0df12c81e5 100644 --- a/vllm/model_executor/models/utils.py +++ b/vllm/model_executor/models/utils.py @@ -78,11 +78,15 @@ class WeightsMapper: orig_to_new_suffix={**self.orig_to_new_suffix, **other.orig_to_new_suffix}, ) - def _map_name(self, key: str) -> str | None: - """Map a weight name (backward-compatible wrapper that discards shard_id).""" + def map_name(self, key: str) -> str | None: + """Map a weight name; returns ``None`` if the weight should be ignored.""" result = self._map_name_with_shard(key) return result[0] if result is not None else None + def _map_name(self, key: str) -> str | None: + """Backward-compatible alias for :meth:`map_name`.""" + return self.map_name(key) + def _map_name_with_shard(self, key: str) -> tuple[str, ShardIds | None] | None: """Map a weight name and extract any shard_id metadata.