[UX] Skip checkpoint shards an MTP head does not load (#60152)
Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Robert Shaw
Claude Opus 5.5
parent
17c70c8953
commit
e572e02d21
@@ -6,7 +6,7 @@ import dataclasses
|
||||
import glob
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Generator, Iterable
|
||||
from collections.abc import Callable, Generator, Iterable
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import torch
|
||||
@@ -28,6 +28,7 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
filter_duplicate_safetensors_files,
|
||||
filter_files_not_needed_for_inference,
|
||||
filter_mm_encoder_only_safetensors_files,
|
||||
filter_safetensors_files_by_weight_name,
|
||||
get_quant_config,
|
||||
instanttensor_weights_iterator,
|
||||
maybe_download_from_modelscope,
|
||||
@@ -75,6 +76,10 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
allow_patterns_overrides: list[str] | None = None
|
||||
"""If defined, weights will load exclusively using these patterns."""
|
||||
|
||||
is_unused_weight: Callable[[str], bool] | None = None
|
||||
"""If defined, safetensors shards holding only weights whose checkpoint
|
||||
name (before *prefix*) it accepts are not read."""
|
||||
|
||||
counter_before_loading_weights: float = 0.0
|
||||
counter_after_loading_weights: float = 0.0
|
||||
|
||||
@@ -283,6 +288,10 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
f"`{source.model_or_path}`; check language_model prefixes "
|
||||
f"{self._encoder_only_lm_prefixes}"
|
||||
)
|
||||
if source.is_unused_weight is not None and use_safetensors:
|
||||
hf_weights_files = filter_safetensors_files_by_weight_name(
|
||||
hf_weights_files, source.is_unused_weight
|
||||
)
|
||||
if self.load_config.load_format == "npcache":
|
||||
# Currently np_cache only support *.bin checkpoints
|
||||
assert use_safetensors is False
|
||||
@@ -359,6 +368,7 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
prefix="",
|
||||
fall_back_to_pt=getattr(model, "fall_back_to_pt_during_load", True),
|
||||
allow_patterns_overrides=getattr(model, "allow_patterns_overrides", None),
|
||||
is_unused_weight=getattr(model, "is_unused_checkpoint_weight", None),
|
||||
)
|
||||
yield from self._get_weights_iterator(primary_weights)
|
||||
|
||||
|
||||
@@ -785,6 +785,42 @@ def filter_mm_encoder_only_safetensors_files(
|
||||
return kept
|
||||
|
||||
|
||||
def filter_safetensors_files_by_weight_name(
|
||||
hf_weights_files: list[str],
|
||||
is_unused_weight: Callable[[str], bool],
|
||||
) -> list[str]:
|
||||
"""Drop safetensors shards in which `is_unused_weight` accepts every tensor.
|
||||
|
||||
Loaders that read whole files (InstantTensor, fastsafetensors, multi-thread,
|
||||
eager, prefetch) cannot skip single tensors, so dropping shards is their
|
||||
only way to avoid reading weights the model does not load.
|
||||
|
||||
Args:
|
||||
hf_weights_files: Safetensors shard paths.
|
||||
is_unused_weight: Returns True for checkpoint weight names the model does
|
||||
not load.
|
||||
|
||||
Returns:
|
||||
The shards holding at least one wanted tensor, or `hf_weights_files`
|
||||
unchanged when no shard does.
|
||||
|
||||
"""
|
||||
kept: list[str] = []
|
||||
for st_file in hf_weights_files:
|
||||
with safe_open(st_file, framework="pt") as f:
|
||||
if not all(is_unused_weight(name) for name in f.keys()): # noqa: SIM118
|
||||
kept.append(st_file)
|
||||
if not kept:
|
||||
return hf_weights_files
|
||||
if len(kept) < len(hf_weights_files):
|
||||
logger.info_once(
|
||||
"Skipped %d/%d safetensors shard(s) holding no weights the model loads",
|
||||
len(hf_weights_files) - len(kept),
|
||||
len(hf_weights_files),
|
||||
)
|
||||
return kept
|
||||
|
||||
|
||||
# explicitly use pure text format, with a newline at the end
|
||||
# this makes it impossible to see the animation in the progress bar
|
||||
# but will avoid messing up with ray or multiprocessing, which wraps
|
||||
|
||||
Reference in New Issue
Block a user