[EPD][Model Loader] Skip language-model checkpoint shards for --mm-encoder-only (#58086)
Signed-off-by: grYe99 <guorongye99@gmail.com> Co-authored-by: grYe99 <guorongye99@gmail.com> Co-authored-by: Cursor Agent <cursoragent@cursor.com>
This commit is contained in:
co-authored by
grYe99
Cursor Agent
parent
c3f5270270
commit
1291bdb85e
@@ -1,11 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import glob
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Generator, Iterable
|
||||
from typing import cast
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
@@ -25,6 +27,7 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
fastsafetensors_weights_iterator,
|
||||
filter_duplicate_safetensors_files,
|
||||
filter_files_not_needed_for_inference,
|
||||
filter_mm_encoder_only_safetensors_files,
|
||||
get_quant_config,
|
||||
instanttensor_weights_iterator,
|
||||
maybe_download_from_modelscope,
|
||||
@@ -32,11 +35,15 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
multi_thread_safetensors_weights_iterator,
|
||||
np_cache_weights_iterator,
|
||||
pt_weights_iterator,
|
||||
resolve_mm_encoder_only_lm_prefixes,
|
||||
safetensors_weights_iterator,
|
||||
)
|
||||
from vllm.tracing import instrument
|
||||
from vllm.transformers_utils.repo_utils import list_filtered_repo_files
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -74,6 +81,9 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
def __init__(self, load_config: LoadConfig):
|
||||
super().__init__(load_config)
|
||||
self.local_expert_ids: set[int] | None = None
|
||||
# Set in load_weights when --mm-encoder-only; used to drop LM-only shards.
|
||||
self._encoder_only_lm_prefixes: tuple[str, ...] | None = None
|
||||
self._encoder_only_weights_mapper: WeightsMapper | None = None
|
||||
|
||||
extra_config = load_config.model_loader_extra_config
|
||||
if not isinstance(extra_config, dict):
|
||||
@@ -132,7 +142,7 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
revision: str | None,
|
||||
fall_back_to_pt: bool,
|
||||
allow_patterns_overrides: list[str] | None,
|
||||
) -> tuple[str, list[str], bool]:
|
||||
) -> tuple[str, list[str], bool, str]:
|
||||
"""Prepare weights for the model.
|
||||
|
||||
If the model is not local, it will be downloaded."""
|
||||
@@ -239,20 +249,40 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
f"Cannot find any model weights with `{model_name_or_path}`"
|
||||
)
|
||||
|
||||
return hf_folder, hf_weights_files, use_safetensors
|
||||
return hf_folder, hf_weights_files, use_safetensors, index_file
|
||||
|
||||
def _get_weights_iterator(
|
||||
self, source: "Source"
|
||||
self, source: Source
|
||||
) -> Generator[tuple[str, torch.Tensor], None, None]:
|
||||
"""Get an iterator for the model weights based on the load format."""
|
||||
extra_config = self.load_config.model_loader_extra_config
|
||||
hf_folder, hf_weights_files, use_safetensors = self._prepare_weights(
|
||||
source.model_or_path,
|
||||
source.subfolder,
|
||||
source.revision,
|
||||
source.fall_back_to_pt,
|
||||
source.allow_patterns_overrides,
|
||||
hf_folder, hf_weights_files, use_safetensors, index_file = (
|
||||
self._prepare_weights(
|
||||
source.model_or_path,
|
||||
source.subfolder,
|
||||
source.revision,
|
||||
source.fall_back_to_pt,
|
||||
source.allow_patterns_overrides,
|
||||
)
|
||||
)
|
||||
if (
|
||||
self._encoder_only_lm_prefixes is not None
|
||||
and use_safetensors
|
||||
and hf_weights_files
|
||||
):
|
||||
hf_weights_files = filter_mm_encoder_only_safetensors_files(
|
||||
hf_weights_files,
|
||||
hf_folder,
|
||||
index_file,
|
||||
self._encoder_only_lm_prefixes,
|
||||
weights_mapper=self._encoder_only_weights_mapper,
|
||||
)
|
||||
if not hf_weights_files:
|
||||
raise RuntimeError(
|
||||
"mm-encoder-only shard filter removed all weight files for "
|
||||
f"`{source.model_or_path}`; check language_model prefixes "
|
||||
f"{self._encoder_only_lm_prefixes}"
|
||||
)
|
||||
if self.load_config.load_format == "npcache":
|
||||
# Currently np_cache only support *.bin checkpoints
|
||||
assert use_safetensors is False
|
||||
@@ -348,6 +378,29 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
allow_patterns_overrides=None,
|
||||
)
|
||||
|
||||
def _init_mm_encoder_only_weight_filter(
|
||||
self, model: nn.Module, model_config: ModelConfig
|
||||
) -> None:
|
||||
"""Skip pure language-model safetensors shards under --mm-encoder-only."""
|
||||
mm_config = model_config.multimodal_config
|
||||
if mm_config is None or not mm_config.mm_encoder_only:
|
||||
self._encoder_only_lm_prefixes = None
|
||||
self._encoder_only_weights_mapper = None
|
||||
return
|
||||
|
||||
# Derive from _language_model_names; fail-closed on shared HF roots
|
||||
# (Molmo/Phi-4-MM/Muse). Qwen nested/flat keys classified via mapper.
|
||||
weights_mapper = cast(
|
||||
"WeightsMapper | None", getattr(model, "hf_to_vllm_mapper", None)
|
||||
)
|
||||
self._encoder_only_lm_prefixes = resolve_mm_encoder_only_lm_prefixes(
|
||||
getattr(model, "_language_model_names", None),
|
||||
weights_mapper=weights_mapper,
|
||||
)
|
||||
self._encoder_only_weights_mapper = (
|
||||
weights_mapper if self._encoder_only_lm_prefixes is not None else None
|
||||
)
|
||||
|
||||
def _init_ep_weight_filter(self, model_config: ModelConfig) -> None:
|
||||
"""Compute local expert ids for EP weight filtering.
|
||||
|
||||
@@ -423,6 +476,7 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
self.load_config.safetensors_load_strategy = "torchao"
|
||||
|
||||
self._init_ep_weight_filter(model_config)
|
||||
self._init_mm_encoder_only_weight_filter(model, model_config)
|
||||
|
||||
loaded_weights = model.load_weights(self.get_all_weights(model_config, model))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user