[EPD][Model Loader] Skip language-model checkpoint shards for --mm-encoder-only (#58086)
Signed-off-by: grYe99 <guorongye99@gmail.com> Co-authored-by: grYe99 <guorongye99@gmail.com> Co-authored-by: Cursor Agent <cursoragent@cursor.com>
This commit is contained in:
co-authored by
grYe99
Cursor Agent
parent
c3f5270270
commit
1291bdb85e
@@ -0,0 +1,243 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for --mm-encoder-only safetensors shard filtering."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
|
||||
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
filter_mm_encoder_only_safetensors_files,
|
||||
resolve_mm_encoder_only_lm_prefixes,
|
||||
)
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
|
||||
def _write_index(folder: str, weight_map: dict[str, str]) -> None:
|
||||
with open(os.path.join(folder, SAFE_WEIGHTS_INDEX_NAME), "w") as f:
|
||||
json.dump({"weight_map": weight_map}, f)
|
||||
|
||||
|
||||
def _qwen25_mapper() -> WeightsMapper:
|
||||
"""Qwen2.5-VL dual layout mapper (nested + flat catch-all)."""
|
||||
return WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
"model.language_model.": "language_model.model.",
|
||||
"model.visual.": "visual.",
|
||||
"lm_head.": "language_model.lm_head.",
|
||||
"model.": "language_model.model.",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _qwen3_mapper() -> WeightsMapper:
|
||||
return WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
"model.visual.": "visual.",
|
||||
"lm_head.": "language_model.lm_head.",
|
||||
"model.language_model.": "language_model.model.",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _molmo_mapper() -> WeightsMapper:
|
||||
return WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
"model.vision_backbone.": "vision_backbone.",
|
||||
"model.transformer.blocks.": "model.layers.",
|
||||
"model.transformer.ln_f.": "model.norm.",
|
||||
"model.transformer.mlp.down_proj.": "lm_head.",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_keeps_vision_shards_drops_language_only():
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
files = [
|
||||
os.path.join(folder, "model-00001-of-000003.safetensors"),
|
||||
os.path.join(folder, "model-00002-of-000003.safetensors"),
|
||||
os.path.join(folder, "model-00003-of-000003.safetensors"),
|
||||
]
|
||||
for path in files:
|
||||
open(path, "wb").close()
|
||||
|
||||
_write_index(
|
||||
folder,
|
||||
{
|
||||
"language_model.layers.0.weight": "model-00001-of-000003.safetensors",
|
||||
"language_model.layers.1.weight": "model-00002-of-000003.safetensors",
|
||||
"vision_tower.blocks.0.weight": "model-00003-of-000003.safetensors",
|
||||
"mm_projector.weight": "model-00003-of-000003.safetensors",
|
||||
},
|
||||
)
|
||||
|
||||
kept = filter_mm_encoder_only_safetensors_files(
|
||||
files,
|
||||
folder,
|
||||
SAFE_WEIGHTS_INDEX_NAME,
|
||||
("language_model.",),
|
||||
)
|
||||
assert kept == [files[2]]
|
||||
|
||||
|
||||
def test_keeps_mixed_shard_with_vision_and_lm():
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
mixed = os.path.join(folder, "model-00001-of-000001.safetensors")
|
||||
open(mixed, "wb").close()
|
||||
_write_index(
|
||||
folder,
|
||||
{
|
||||
"language_model.embed.weight": "model-00001-of-000001.safetensors",
|
||||
"vision_tower.patch.weight": "model-00001-of-000001.safetensors",
|
||||
},
|
||||
)
|
||||
|
||||
kept = filter_mm_encoder_only_safetensors_files(
|
||||
[mixed],
|
||||
folder,
|
||||
SAFE_WEIGHTS_INDEX_NAME,
|
||||
("language_model.",),
|
||||
)
|
||||
assert kept == [mixed]
|
||||
|
||||
|
||||
def test_no_index_returns_unchanged():
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
files = [os.path.join(folder, "model.safetensors")]
|
||||
open(files[0], "wb").close()
|
||||
kept = filter_mm_encoder_only_safetensors_files(
|
||||
files,
|
||||
folder,
|
||||
SAFE_WEIGHTS_INDEX_NAME,
|
||||
("language_model.",),
|
||||
)
|
||||
assert kept == files
|
||||
|
||||
|
||||
def test_resolve_returns_module_prefixes_without_hf_hardcode():
|
||||
"""No global Qwen HF nest list — just vLLM module prefixes."""
|
||||
prefixes = resolve_mm_encoder_only_lm_prefixes(["language_model"])
|
||||
assert prefixes == ("language_model.",)
|
||||
# With Qwen mapper, still module prefixes (keys classified via mapper).
|
||||
assert resolve_mm_encoder_only_lm_prefixes(
|
||||
["language_model"], weights_mapper=_qwen25_mapper()
|
||||
) == ("language_model.",)
|
||||
|
||||
|
||||
def test_resolve_empty_language_model_names_fail_closed():
|
||||
"""Missing/empty ``_language_model_names`` → None (no language_model. default)."""
|
||||
assert resolve_mm_encoder_only_lm_prefixes(None) is None
|
||||
assert resolve_mm_encoder_only_lm_prefixes([]) is None
|
||||
assert resolve_mm_encoder_only_lm_prefixes(()) is None
|
||||
|
||||
|
||||
def test_resolve_skips_filter_for_shared_hf_root():
|
||||
"""Shared HF root (Molmo-shaped mapper) → fail-closed, not magic name list."""
|
||||
assert (
|
||||
resolve_mm_encoder_only_lm_prefixes(["model"], weights_mapper=_molmo_mapper())
|
||||
is None
|
||||
)
|
||||
# Without a mapper, identity layout: module prefix is returned as-is.
|
||||
assert resolve_mm_encoder_only_lm_prefixes(["model"]) == ("model.",)
|
||||
assert resolve_mm_encoder_only_lm_prefixes(["model."]) == ("model.",)
|
||||
|
||||
|
||||
def test_resolve_keeps_llm_prefix_without_model_broadening():
|
||||
prefixes = resolve_mm_encoder_only_lm_prefixes(["llm"])
|
||||
assert prefixes == ("llm.",)
|
||||
|
||||
|
||||
def test_qwen3_nested_index_skips_lm_keeps_visual():
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
files = [
|
||||
os.path.join(folder, "model-00001-of-000002.safetensors"),
|
||||
os.path.join(folder, "model-00002-of-000002.safetensors"),
|
||||
]
|
||||
for path in files:
|
||||
open(path, "wb").close()
|
||||
_write_index(
|
||||
folder,
|
||||
{
|
||||
"model.language_model.layers.0.weight": (
|
||||
"model-00001-of-000002.safetensors"
|
||||
),
|
||||
"lm_head.weight": "model-00001-of-000002.safetensors",
|
||||
"model.visual.blocks.0.weight": "model-00002-of-000002.safetensors",
|
||||
},
|
||||
)
|
||||
prefixes = resolve_mm_encoder_only_lm_prefixes(
|
||||
["language_model"], weights_mapper=_qwen3_mapper()
|
||||
)
|
||||
assert prefixes is not None
|
||||
kept = filter_mm_encoder_only_safetensors_files(
|
||||
files,
|
||||
folder,
|
||||
SAFE_WEIGHTS_INDEX_NAME,
|
||||
prefixes,
|
||||
weights_mapper=_qwen3_mapper(),
|
||||
)
|
||||
assert kept == [files[1]]
|
||||
|
||||
|
||||
def test_qwen25_flat_index_skips_lm_keeps_visual():
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
files = [
|
||||
os.path.join(folder, "model-00001-of-000002.safetensors"),
|
||||
os.path.join(folder, "model-00002-of-000002.safetensors"),
|
||||
]
|
||||
for path in files:
|
||||
open(path, "wb").close()
|
||||
_write_index(
|
||||
folder,
|
||||
{
|
||||
"model.layers.0.self_attn.q_proj.weight": (
|
||||
"model-00001-of-000002.safetensors"
|
||||
),
|
||||
"model.embed_tokens.weight": "model-00001-of-000002.safetensors",
|
||||
"model.norm.weight": "model-00001-of-000002.safetensors",
|
||||
"lm_head.weight": "model-00001-of-000002.safetensors",
|
||||
"visual.blocks.0.weight": "model-00002-of-000002.safetensors",
|
||||
},
|
||||
)
|
||||
prefixes = resolve_mm_encoder_only_lm_prefixes(
|
||||
["language_model"], weights_mapper=_qwen25_mapper()
|
||||
)
|
||||
assert prefixes is not None
|
||||
kept = filter_mm_encoder_only_safetensors_files(
|
||||
files,
|
||||
folder,
|
||||
SAFE_WEIGHTS_INDEX_NAME,
|
||||
prefixes,
|
||||
weights_mapper=_qwen25_mapper(),
|
||||
)
|
||||
assert kept == [files[1]]
|
||||
|
||||
|
||||
def test_molmo_shaped_index_not_filtered_when_deny_disabled():
|
||||
"""Shared-root LM attr → resolve None; callers leave the file list alone."""
|
||||
assert (
|
||||
resolve_mm_encoder_only_lm_prefixes(["model"], weights_mapper=_molmo_mapper())
|
||||
is None
|
||||
)
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
files = [
|
||||
os.path.join(folder, "vision.safetensors"),
|
||||
os.path.join(folder, "lm.safetensors"),
|
||||
]
|
||||
for path in files:
|
||||
open(path, "wb").close()
|
||||
_write_index(
|
||||
folder,
|
||||
{
|
||||
"model.vision_backbone.patch.weight": "vision.safetensors",
|
||||
"model.transformer.blocks.0.weight": "lm.safetensors",
|
||||
},
|
||||
)
|
||||
# Document why raw HF startswith("model.") is unsafe without fail-closed:
|
||||
# whole-shard deny would also drop vision.
|
||||
if_applied_naively = filter_mm_encoder_only_safetensors_files(
|
||||
files, folder, SAFE_WEIGHTS_INDEX_NAME, ("model.",)
|
||||
)
|
||||
assert if_applied_naively == []
|
||||
@@ -79,7 +79,7 @@ def test_default_loader_hf_still_falls_back_to_pt(tmp_path):
|
||||
# Control: load_format="hf" still picks up .pt weights via fallback.
|
||||
(tmp_path / "model.pt").write_bytes(b"\x00\x00\x00\x00")
|
||||
loader = DefaultModelLoader(LoadConfig(load_format="hf"))
|
||||
_, files, use_safetensors = loader._prepare_weights(
|
||||
_, files, use_safetensors, _ = loader._prepare_weights(
|
||||
str(tmp_path),
|
||||
None,
|
||||
None,
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import glob
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Generator, Iterable
|
||||
from typing import cast
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
@@ -25,6 +27,7 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
fastsafetensors_weights_iterator,
|
||||
filter_duplicate_safetensors_files,
|
||||
filter_files_not_needed_for_inference,
|
||||
filter_mm_encoder_only_safetensors_files,
|
||||
get_quant_config,
|
||||
instanttensor_weights_iterator,
|
||||
maybe_download_from_modelscope,
|
||||
@@ -32,11 +35,15 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
multi_thread_safetensors_weights_iterator,
|
||||
np_cache_weights_iterator,
|
||||
pt_weights_iterator,
|
||||
resolve_mm_encoder_only_lm_prefixes,
|
||||
safetensors_weights_iterator,
|
||||
)
|
||||
from vllm.tracing import instrument
|
||||
from vllm.transformers_utils.repo_utils import list_filtered_repo_files
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -74,6 +81,9 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
def __init__(self, load_config: LoadConfig):
|
||||
super().__init__(load_config)
|
||||
self.local_expert_ids: set[int] | None = None
|
||||
# Set in load_weights when --mm-encoder-only; used to drop LM-only shards.
|
||||
self._encoder_only_lm_prefixes: tuple[str, ...] | None = None
|
||||
self._encoder_only_weights_mapper: WeightsMapper | None = None
|
||||
|
||||
extra_config = load_config.model_loader_extra_config
|
||||
if not isinstance(extra_config, dict):
|
||||
@@ -132,7 +142,7 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
revision: str | None,
|
||||
fall_back_to_pt: bool,
|
||||
allow_patterns_overrides: list[str] | None,
|
||||
) -> tuple[str, list[str], bool]:
|
||||
) -> tuple[str, list[str], bool, str]:
|
||||
"""Prepare weights for the model.
|
||||
|
||||
If the model is not local, it will be downloaded."""
|
||||
@@ -239,20 +249,40 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
f"Cannot find any model weights with `{model_name_or_path}`"
|
||||
)
|
||||
|
||||
return hf_folder, hf_weights_files, use_safetensors
|
||||
return hf_folder, hf_weights_files, use_safetensors, index_file
|
||||
|
||||
def _get_weights_iterator(
|
||||
self, source: "Source"
|
||||
self, source: Source
|
||||
) -> Generator[tuple[str, torch.Tensor], None, None]:
|
||||
"""Get an iterator for the model weights based on the load format."""
|
||||
extra_config = self.load_config.model_loader_extra_config
|
||||
hf_folder, hf_weights_files, use_safetensors = self._prepare_weights(
|
||||
source.model_or_path,
|
||||
source.subfolder,
|
||||
source.revision,
|
||||
source.fall_back_to_pt,
|
||||
source.allow_patterns_overrides,
|
||||
hf_folder, hf_weights_files, use_safetensors, index_file = (
|
||||
self._prepare_weights(
|
||||
source.model_or_path,
|
||||
source.subfolder,
|
||||
source.revision,
|
||||
source.fall_back_to_pt,
|
||||
source.allow_patterns_overrides,
|
||||
)
|
||||
)
|
||||
if (
|
||||
self._encoder_only_lm_prefixes is not None
|
||||
and use_safetensors
|
||||
and hf_weights_files
|
||||
):
|
||||
hf_weights_files = filter_mm_encoder_only_safetensors_files(
|
||||
hf_weights_files,
|
||||
hf_folder,
|
||||
index_file,
|
||||
self._encoder_only_lm_prefixes,
|
||||
weights_mapper=self._encoder_only_weights_mapper,
|
||||
)
|
||||
if not hf_weights_files:
|
||||
raise RuntimeError(
|
||||
"mm-encoder-only shard filter removed all weight files for "
|
||||
f"`{source.model_or_path}`; check language_model prefixes "
|
||||
f"{self._encoder_only_lm_prefixes}"
|
||||
)
|
||||
if self.load_config.load_format == "npcache":
|
||||
# Currently np_cache only support *.bin checkpoints
|
||||
assert use_safetensors is False
|
||||
@@ -348,6 +378,29 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
allow_patterns_overrides=None,
|
||||
)
|
||||
|
||||
def _init_mm_encoder_only_weight_filter(
|
||||
self, model: nn.Module, model_config: ModelConfig
|
||||
) -> None:
|
||||
"""Skip pure language-model safetensors shards under --mm-encoder-only."""
|
||||
mm_config = model_config.multimodal_config
|
||||
if mm_config is None or not mm_config.mm_encoder_only:
|
||||
self._encoder_only_lm_prefixes = None
|
||||
self._encoder_only_weights_mapper = None
|
||||
return
|
||||
|
||||
# Derive from _language_model_names; fail-closed on shared HF roots
|
||||
# (Molmo/Phi-4-MM/Muse). Qwen nested/flat keys classified via mapper.
|
||||
weights_mapper = cast(
|
||||
"WeightsMapper | None", getattr(model, "hf_to_vllm_mapper", None)
|
||||
)
|
||||
self._encoder_only_lm_prefixes = resolve_mm_encoder_only_lm_prefixes(
|
||||
getattr(model, "_language_model_names", None),
|
||||
weights_mapper=weights_mapper,
|
||||
)
|
||||
self._encoder_only_weights_mapper = (
|
||||
weights_mapper if self._encoder_only_lm_prefixes is not None else None
|
||||
)
|
||||
|
||||
def _init_ep_weight_filter(self, model_config: ModelConfig) -> None:
|
||||
"""Compute local expert ids for EP weight filtering.
|
||||
|
||||
@@ -423,6 +476,7 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
self.load_config.safetensors_load_strategy = "torchao"
|
||||
|
||||
self._init_ep_weight_filter(model_config)
|
||||
self._init_mm_encoder_only_weight_filter(model, model_config)
|
||||
|
||||
loaded_weights = model.load_weights(self.get_all_weights(model_config, model))
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ import time
|
||||
from collections.abc import Callable, Generator, Iterable
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import IO, Any
|
||||
from typing import IO, TYPE_CHECKING, Any
|
||||
|
||||
import filelock
|
||||
import huggingface_hub.constants
|
||||
@@ -61,6 +61,9 @@ except ImportError:
|
||||
|
||||
from vllm.model_executor.layers.quantization.torchao import torchao_version_at_least
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# use system-level temp directory for file locks, so that multiple users
|
||||
@@ -605,6 +608,171 @@ def filter_files_not_needed_for_inference(hf_weights_files: list[str]) -> list[s
|
||||
return hf_weights_files
|
||||
|
||||
|
||||
def _normalize_module_prefixes(
|
||||
language_model_names: Iterable[str] | None,
|
||||
) -> tuple[str, ...]:
|
||||
names = tuple(language_model_names or ())
|
||||
return tuple(n if n.endswith(".") else f"{n}." for n in names)
|
||||
|
||||
|
||||
def _mapped_weight_name(weights_mapper: "WeightsMapper | None", key: str) -> str | None:
|
||||
"""Apply ``WeightsMapper.map_name`` when present; else identity."""
|
||||
if weights_mapper is None:
|
||||
return key
|
||||
return weights_mapper.map_name(key)
|
||||
|
||||
|
||||
def _hf_prefix_maps_to_lm(
|
||||
new_name: str | None, lm_module_prefixes: tuple[str, ...]
|
||||
) -> bool:
|
||||
if not new_name:
|
||||
return False
|
||||
new_p = new_name if new_name.endswith(".") else f"{new_name}."
|
||||
return any(new_p.startswith(p) or p.startswith(new_p) for p in lm_module_prefixes)
|
||||
|
||||
|
||||
def _shared_hf_root_module_prefixes(
|
||||
lm_module_prefixes: tuple[str, ...],
|
||||
weights_mapper: "WeightsMapper | None",
|
||||
) -> tuple[str, ...]:
|
||||
"""Module prefixes that are also an HF root for non-LM weights.
|
||||
|
||||
E.g. Molmo/Phi-4-MM/Muse mark LM as ``model`` while vision HF keys stay
|
||||
under ``model.vision_*`` / ``model.embed_tokens_extend.*``. Detected via
|
||||
``WeightsMapper.orig_to_new_prefix`` (no hard-coded name list).
|
||||
"""
|
||||
if weights_mapper is None:
|
||||
return ()
|
||||
mapping = weights_mapper.orig_to_new_prefix
|
||||
if not mapping:
|
||||
return ()
|
||||
|
||||
non_lm_origs: list[str] = []
|
||||
for orig, new in mapping.items():
|
||||
orig_p = orig if orig.endswith(".") else f"{orig}."
|
||||
if not _hf_prefix_maps_to_lm(new, lm_module_prefixes):
|
||||
non_lm_origs.append(orig_p)
|
||||
|
||||
if not non_lm_origs:
|
||||
return ()
|
||||
|
||||
shared = [
|
||||
p
|
||||
for p in lm_module_prefixes
|
||||
if any(orig.startswith(p) for orig in non_lm_origs)
|
||||
]
|
||||
return tuple(shared)
|
||||
|
||||
|
||||
def resolve_mm_encoder_only_lm_prefixes(
|
||||
language_model_names: Iterable[str] | None,
|
||||
*,
|
||||
weights_mapper: "WeightsMapper | None" = None,
|
||||
) -> tuple[str, ...] | None:
|
||||
"""Resolve vLLM LM *module* prefixes for ``--mm-encoder-only`` shard skip.
|
||||
|
||||
Prefixes come from ``_language_model_names``. Classification of HF index
|
||||
keys is done later via optional ``weights_mapper`` (see
|
||||
``filter_mm_encoder_only_safetensors_files``), so this helper does **not**
|
||||
hard-code Qwen/HF nest lists.
|
||||
|
||||
Returns ``None`` (leave the safetensors file list unchanged) when
|
||||
``_language_model_names`` is empty/missing, or when a module prefix is a
|
||||
shared HF checkpoint root for both LM and non-LM weights — fail-closed so
|
||||
Molmo / Phi-4-MM / Muse cannot under-load the encoder.
|
||||
"""
|
||||
prefixes = _normalize_module_prefixes(language_model_names)
|
||||
if not prefixes:
|
||||
return None
|
||||
shared = _shared_hf_root_module_prefixes(prefixes, weights_mapper)
|
||||
if shared:
|
||||
logger.warning_once(
|
||||
"mm-encoder-only whole-shard filter not applied: LM module "
|
||||
"prefix(es) %s share an HF checkpoint root with non-LM weights "
|
||||
"(via WeightsMapper). Encoder-only load keeps the full "
|
||||
"safetensors file list; finer HF prefixes or key-level "
|
||||
"filtering can enable skip later.",
|
||||
shared,
|
||||
)
|
||||
return None
|
||||
return prefixes
|
||||
|
||||
|
||||
def filter_mm_encoder_only_safetensors_files(
|
||||
hf_weights_files: list[str],
|
||||
hf_folder: str,
|
||||
index_file: str,
|
||||
language_model_prefixes: Iterable[str],
|
||||
*,
|
||||
weights_mapper: "WeightsMapper | None" = None,
|
||||
) -> list[str]:
|
||||
"""Drop safetensors shards that only contain language-model weights.
|
||||
|
||||
Used with ``--mm-encoder-only`` so Encoder-only EPD instances avoid reading
|
||||
pure LM shards from disk/DRAM.
|
||||
|
||||
Each HF index key is classified by mapping through ``weights_mapper`` (when
|
||||
provided) and testing the *vLLM* name against ``language_model_prefixes``.
|
||||
Without a mapper, HF names are compared directly (identity checkpoint
|
||||
layout, e.g. Kimi ``language_model.*``).
|
||||
|
||||
A shard is kept if it contains any non-LM key. Without an index file,
|
||||
returns ``hf_weights_files`` unchanged.
|
||||
"""
|
||||
prefixes = tuple(language_model_prefixes)
|
||||
if not prefixes:
|
||||
return hf_weights_files
|
||||
|
||||
index_path = os.path.join(hf_folder, index_file)
|
||||
if not os.path.isfile(index_path):
|
||||
logger.warning_once(
|
||||
"mm-encoder-only shard filter skipped: index file %s not found",
|
||||
index_path,
|
||||
)
|
||||
return hf_weights_files
|
||||
|
||||
with open(index_path) as f:
|
||||
weight_map: dict[str, str] = json.load(f)["weight_map"]
|
||||
|
||||
keys_by_file: dict[str, list[str]] = {}
|
||||
for weight_name, weight_file in weight_map.items():
|
||||
keys_by_file.setdefault(weight_file, []).append(weight_name)
|
||||
|
||||
def _is_lm_key(key: str) -> bool:
|
||||
mapped = _mapped_weight_name(weights_mapper, key)
|
||||
if mapped is None:
|
||||
return False
|
||||
return any(mapped.startswith(p) for p in prefixes)
|
||||
|
||||
def _is_lm_only(weight_file: str) -> bool:
|
||||
keys = keys_by_file.get(weight_file)
|
||||
if not keys:
|
||||
# Not listed in the index (e.g. extra matched glob) — keep.
|
||||
return False
|
||||
return all(_is_lm_key(key) for key in keys)
|
||||
|
||||
kept: list[str] = []
|
||||
skipped = 0
|
||||
for path in hf_weights_files:
|
||||
rel = os.path.relpath(path, hf_folder)
|
||||
# Index weight_map values are usually basenames; tolerate either form.
|
||||
basename = os.path.basename(path)
|
||||
if _is_lm_only(rel) or _is_lm_only(basename):
|
||||
skipped += 1
|
||||
continue
|
||||
kept.append(path)
|
||||
|
||||
if skipped:
|
||||
logger.info_once(
|
||||
"mm-encoder-only: skipped %d/%d safetensors shard(s) that only "
|
||||
"contain language-model weights (prefixes=%s)",
|
||||
skipped,
|
||||
skipped + len(kept),
|
||||
prefixes,
|
||||
)
|
||||
return kept
|
||||
|
||||
|
||||
# explicitly use pure text format, with a newline at the end
|
||||
# this makes it impossible to see the animation in the progress bar
|
||||
# but will avoid messing up with ray or multiprocessing, which wraps
|
||||
|
||||
@@ -78,11 +78,15 @@ class WeightsMapper:
|
||||
orig_to_new_suffix={**self.orig_to_new_suffix, **other.orig_to_new_suffix},
|
||||
)
|
||||
|
||||
def _map_name(self, key: str) -> str | None:
|
||||
"""Map a weight name (backward-compatible wrapper that discards shard_id)."""
|
||||
def map_name(self, key: str) -> str | None:
|
||||
"""Map a weight name; returns ``None`` if the weight should be ignored."""
|
||||
result = self._map_name_with_shard(key)
|
||||
return result[0] if result is not None else None
|
||||
|
||||
def _map_name(self, key: str) -> str | None:
|
||||
"""Backward-compatible alias for :meth:`map_name`."""
|
||||
return self.map_name(key)
|
||||
|
||||
def _map_name_with_shard(self, key: str) -> tuple[str, ShardIds | None] | None:
|
||||
"""Map a weight name and extract any shard_id metadata.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user