Convert formatting to use ruff instead of yapf + isort (#26247)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-05 07:06:22 -07:00
committed by GitHub
parent 17edd8a807
commit d6953beb91
1508 changed files with 115205 additions and 94107 deletions
+141 -139
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Utilities for downloading and initializing model weights."""
import concurrent.futures
import fnmatch
import glob
@@ -28,18 +29,18 @@ from vllm.config import ModelConfig
from vllm.config.load import LoadConfig
from vllm.distributed import get_tensor_model_parallel_rank
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization import (QuantizationConfig,
get_quantization_config)
from vllm.model_executor.layers.quantization import (
QuantizationConfig,
get_quantization_config,
)
from vllm.platforms import current_platform
from vllm.utils import PlaceholderModule
try:
from runai_model_streamer import SafetensorsStreamer
except ImportError:
runai_model_streamer = PlaceholderModule(
"runai_model_streamer") # type: ignore[assignment]
SafetensorsStreamer = runai_model_streamer.placeholder_attr(
"SafetensorsStreamer")
runai_model_streamer = PlaceholderModule("runai_model_streamer") # type: ignore[assignment]
SafetensorsStreamer = runai_model_streamer.placeholder_attr("SafetensorsStreamer")
try:
import gguf
@@ -50,8 +51,7 @@ try:
from fastsafetensors import SafeTensorsFileLoader, SingleGroup
except ImportError:
fastsafetensors = PlaceholderModule("fastsafetensors")
SafeTensorsFileLoader = fastsafetensors.placeholder_attr(
"SafeTensorsFileLoader")
SafeTensorsFileLoader = fastsafetensors.placeholder_attr("SafeTensorsFileLoader")
SingleGroup = fastsafetensors.placeholder_attr("SingleGroup")
logger = init_logger(__name__)
@@ -64,12 +64,12 @@ temp_dir = tempfile.gettempdir()
def enable_hf_transfer():
"""automatically activates hf_transfer
"""
"""automatically activates hf_transfer"""
if "HF_HUB_ENABLE_HF_TRANSFER" not in os.environ:
try:
# enable hf hub transfer if available
import hf_transfer # type: ignore # noqa
huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER = True
except ImportError:
pass
@@ -79,13 +79,11 @@ enable_hf_transfer()
class DisabledTqdm(tqdm):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs, disable=True)
def get_lock(model_name_or_path: Union[str, Path],
cache_dir: Optional[str] = None):
def get_lock(model_name_or_path: Union[str, Path], cache_dir: Optional[str] = None):
lock_dir = cache_dir or temp_dir
model_name_or_path = str(model_name_or_path)
os.makedirs(os.path.dirname(lock_dir), exist_ok=True)
@@ -94,15 +92,14 @@ def get_lock(model_name_or_path: Union[str, Path],
# add hash to avoid conflict with old users' lock files
lock_file_name = hash_name + model_name + ".lock"
# mode 0o666 is required for the filelock to be shared across users
lock = filelock.FileLock(os.path.join(lock_dir, lock_file_name),
mode=0o666)
lock = filelock.FileLock(os.path.join(lock_dir, lock_file_name), mode=0o666)
return lock
@contextmanager
def atomic_writer(filepath: Union[str, Path],
mode: str = 'w',
encoding: Optional[str] = None) -> Generator[IO]:
def atomic_writer(
filepath: Union[str, Path], mode: str = "w", encoding: Optional[str] = None
) -> Generator[IO]:
"""
Context manager that provides an atomic file writing routine.
@@ -133,8 +130,8 @@ def atomic_writer(filepath: Union[str, Path],
except Exception:
logger.exception(
"Error during atomic write. Original file '%s' not modified",
filepath)
"Error during atomic write. Original file '%s' not modified", filepath
)
raise
finally:
# Clean up the temporary file if it still exists.
@@ -143,16 +140,16 @@ def atomic_writer(filepath: Union[str, Path],
def maybe_download_from_modelscope(
model: str,
revision: Optional[str] = None,
download_dir: Optional[str] = None,
ignore_patterns: Optional[Union[str, list[str]]] = None,
allow_patterns: Optional[Union[list[str],
str]] = None) -> Optional[str]:
model: str,
revision: Optional[str] = None,
download_dir: Optional[str] = None,
ignore_patterns: Optional[Union[str, list[str]]] = None,
allow_patterns: Optional[Union[list[str], str]] = None,
) -> Optional[str]:
"""Download model from ModelScope hub if VLLM_USE_MODELSCOPE is True.
Returns the path to the downloaded model, or None if the model is not
downloaded from ModelScope."""
Returns the path to the downloaded model, or None if the model is not
downloaded from ModelScope."""
if envs.VLLM_USE_MODELSCOPE:
# download model from ModelScope hub,
# lazy import so that modelscope is not required for normal use.
@@ -226,9 +223,9 @@ def convert_bin_to_safetensor_file(
# TODO(woosuk): Move this to other place.
def get_quant_config(model_config: ModelConfig,
load_config: LoadConfig) -> QuantizationConfig:
def get_quant_config(
model_config: ModelConfig, load_config: LoadConfig
) -> QuantizationConfig:
quant_cls = get_quantization_config(model_config.quantization)
# GGUF doesn't have config file
@@ -236,16 +233,14 @@ def get_quant_config(model_config: ModelConfig,
return quant_cls()
# Read the quantization config from the HF model config, if available.
hf_quant_config = getattr(model_config.hf_config, "quantization_config",
None)
hf_quant_config = getattr(model_config.hf_config, "quantization_config", None)
# some vision model may keep quantization_config in their text_config
hf_text_config = getattr(model_config.hf_config, "text_config", None)
if hf_quant_config is None and hf_text_config is not None:
hf_quant_config = getattr(hf_text_config, "quantization_config", None)
if hf_quant_config is None:
# compressed-tensors uses a compressions_config
hf_quant_config = getattr(model_config.hf_config, "compression_config",
None)
hf_quant_config = getattr(model_config.hf_config, "compression_config", None)
if hf_quant_config is not None:
return quant_cls.from_config(hf_quant_config)
@@ -253,8 +248,7 @@ def get_quant_config(model_config: ModelConfig,
# if hf_quant_config is None, we will try to get config from
# hf_overrides
hf_overrides = model_config.hf_overrides
quantization_config_file = hf_overrides.get("quantization_config_file",
None)
quantization_config_file = hf_overrides.get("quantization_config_file", None)
if quantization_config_file is not None:
if hasattr(quant_cls, "from_config_file"):
return quant_cls.from_config_file(quantization_config_file)
@@ -262,9 +256,9 @@ def get_quant_config(model_config: ModelConfig,
raise NotImplementedError(
"from_config_file is specified in hf_override config, "
"but quant_cls.from_config_file is not implemented in "
f"{quant_cls}")
quantization_config_json = hf_overrides.get(
"quantization_config_dict_json", None)
f"{quant_cls}"
)
quantization_config_json = hf_overrides.get("quantization_config_dict_json", None)
if quantization_config_json is not None:
if hasattr(quant_cls, "from_config_dict_json"):
return quant_cls.from_config_dict_json(quantization_config_json)
@@ -272,17 +266,21 @@ def get_quant_config(model_config: ModelConfig,
raise NotImplementedError(
"from_config_dict_json is specified in hf_override config, "
"but quant_cls.from_config_dict_json is not implemented in "
f"{quant_cls}")
f"{quant_cls}"
)
# Inflight BNB quantization
if model_config.quantization == "bitsandbytes":
return quant_cls.from_config({})
model_name_or_path = maybe_download_from_modelscope(
model_config.model,
revision=model_config.revision,
download_dir=load_config.download_dir,
allow_patterns=["*.json"],
) or model_config.model
model_name_or_path = (
maybe_download_from_modelscope(
model_config.model,
revision=model_config.revision,
download_dir=load_config.download_dir,
allow_patterns=["*.json"],
)
or model_config.model
)
is_local = os.path.isdir(model_name_or_path)
if not is_local:
# Download the config files.
@@ -307,16 +305,15 @@ def get_quant_config(model_config: ModelConfig,
config_files = glob.glob(os.path.join(hf_folder, "*.json"))
quant_config_files = [
f for f in config_files if any(
f.endswith(x) for x in possible_config_filenames)
f for f in config_files if any(f.endswith(x) for x in possible_config_filenames)
]
if len(quant_config_files) == 0:
raise ValueError(
f"Cannot find the config file for {model_config.quantization}")
raise ValueError(f"Cannot find the config file for {model_config.quantization}")
if len(quant_config_files) > 1:
raise ValueError(
f"Found multiple config files for {model_config.quantization}: "
f"{quant_config_files}")
f"{quant_config_files}"
)
quant_config_file = quant_config_files[0]
with open(quant_config_file) as f:
@@ -330,7 +327,8 @@ def get_quant_config(model_config: ModelConfig,
else:
raise ValueError(
f"Unsupported quantization config"
f" found for {model_config.quantization} in {f}.")
f" found for {model_config.quantization} in {f}."
)
return quant_cls.from_config(config)
@@ -399,9 +397,7 @@ def download_weights_from_hf(
# so we only have to call snapshot_download once.
try:
fs = HfFileSystem()
file_list = fs.ls(model_name_or_path,
detail=False,
revision=revision)
file_list = fs.ls(model_name_or_path, detail=False, revision=revision)
# Use the first pattern found in the HF repo's files.
for pattern in allow_patterns:
@@ -413,7 +409,10 @@ def download_weights_from_hf(
logger.warning(
"Failed to get file list for '%s'. Trying each pattern in "
"allow_patterns individually until weights have been "
"downloaded. Error: %s", model_name_or_path, e)
"downloaded. Error: %s",
model_name_or_path,
e,
)
logger.info("Using model weights format %s", allow_patterns)
# Use file lock to prevent multiple processes from
@@ -436,8 +435,11 @@ def download_weights_from_hf(
break
time_taken = time.perf_counter() - start_time
if time_taken > 0.5:
logger.info("Time spent downloading weights for %s: %.6f seconds",
model_name_or_path, time_taken)
logger.info(
"Time spent downloading weights for %s: %.6f seconds",
model_name_or_path,
time_taken,
)
return hf_folder
@@ -481,9 +483,9 @@ def download_safetensors_index_file_from_hf(
# Passing both of these to the weight loader functionality breaks.
# So, we use the index_file to
# look up which safetensors files should be used.
def filter_duplicate_safetensors_files(hf_weights_files: list[str],
hf_folder: str,
index_file: str) -> list[str]:
def filter_duplicate_safetensors_files(
hf_weights_files: list[str], hf_folder: str, index_file: str
) -> list[str]:
# model.safetensors.index.json is a mapping from keys in the
# torch state_dict to safetensors file holding that weight.
index_file_name = os.path.join(hf_folder, index_file)
@@ -496,17 +498,13 @@ def filter_duplicate_safetensors_files(hf_weights_files: list[str],
weight_map = json.load(f)["weight_map"]
weight_files_in_index = set()
for weight_name in weight_map:
weight_files_in_index.add(
os.path.join(hf_folder, weight_map[weight_name]))
weight_files_in_index.add(os.path.join(hf_folder, weight_map[weight_name]))
# Filter out any fields that are not found in the index file.
hf_weights_files = [
f for f in hf_weights_files if f in weight_files_in_index
]
hf_weights_files = [f for f in hf_weights_files if f in weight_files_in_index]
return hf_weights_files
def filter_files_not_needed_for_inference(
hf_weights_files: list[str]) -> list[str]:
def filter_files_not_needed_for_inference(hf_weights_files: list[str]) -> list[str]:
"""
Exclude files that are not needed for inference.
@@ -520,8 +518,7 @@ def filter_files_not_needed_for_inference(
"scaler.pt",
]
hf_weights_files = [
f for f in hf_weights_files
if not any(f.endswith(x) for x in blacklist)
f for f in hf_weights_files if not any(f.endswith(x) for x in blacklist)
]
return hf_weights_files
@@ -534,8 +531,9 @@ _BAR_FORMAT = "{desc}: {percentage:3.0f}% Completed | {n_fmt}/{total_fmt} [{elap
def enable_tqdm(use_tqdm_on_load: bool):
return use_tqdm_on_load and (not torch.distributed.is_initialized()
or torch.distributed.get_rank() == 0)
return use_tqdm_on_load and (
not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0
)
def np_cache_weights_iterator(
@@ -560,14 +558,12 @@ def np_cache_weights_iterator(
if not os.path.exists(weight_names_file):
weight_names: list[str] = []
for bin_file in tqdm(
hf_weights_files,
desc="Loading np_cache checkpoint shards",
disable=not enable_tqdm(use_tqdm_on_load),
bar_format=_BAR_FORMAT,
hf_weights_files,
desc="Loading np_cache checkpoint shards",
disable=not enable_tqdm(use_tqdm_on_load),
bar_format=_BAR_FORMAT,
):
state = torch.load(bin_file,
map_location="cpu",
weights_only=True)
state = torch.load(bin_file, map_location="cpu", weights_only=True)
for name, param in state.items():
param_path = os.path.join(np_folder, name)
with open(param_path, "wb") as f:
@@ -597,10 +593,10 @@ def safetensors_weights_iterator(
loading_desc += " (eager)"
for st_file in tqdm(
hf_weights_files,
desc=loading_desc,
disable=not enable_tqdm(use_tqdm_on_load),
bar_format=_BAR_FORMAT,
hf_weights_files,
desc=loading_desc,
disable=not enable_tqdm(use_tqdm_on_load),
bar_format=_BAR_FORMAT,
):
if safetensors_load_strategy == "eager":
with open(st_file, "rb") as f:
@@ -624,12 +620,8 @@ def multi_thread_safetensors_weights_iterator(
result = load_file(st_file, device="cpu")
return result
with concurrent.futures.ThreadPoolExecutor(
max_workers=max_workers) as executor:
futures = [
executor.submit(_load_file, st_file)
for st_file in hf_weights_files
]
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = [executor.submit(_load_file, st_file) for st_file in hf_weights_files]
futures_iter = tqdm(
concurrent.futures.as_completed(futures),
total=len(hf_weights_files),
@@ -652,7 +644,8 @@ def runai_safetensors_weights_iterator(
streamer.stream_files(hf_weights_files)
total_tensors = sum(
len(tensors_meta)
for tensors_meta in streamer.files_to_tensors_metadata.values())
for tensors_meta in streamer.files_to_tensors_metadata.values()
)
tensor_iter = tqdm(
streamer.get_tensors(),
@@ -689,19 +682,19 @@ def fastsafetensors_weights_iterator(
else:
pg = SingleGroup()
device = torch.device(f'cuda:{pg.rank()}')
device = torch.device(f"cuda:{pg.rank()}")
weight_files_sub_lists = [
hf_weights_files[i:i + pg.size()]
hf_weights_files[i : i + pg.size()]
for i in range(0, len(hf_weights_files), pg.size())
]
nogds = False
for f_list in tqdm(
weight_files_sub_lists,
desc="Loading safetensors using Fastsafetensor loader",
disable=not enable_tqdm(use_tqdm_on_load),
bar_format=_BAR_FORMAT,
weight_files_sub_lists,
desc="Loading safetensors using Fastsafetensor loader",
disable=not enable_tqdm(use_tqdm_on_load),
bar_format=_BAR_FORMAT,
):
loader = _init_loader(pg, device, f_list, nogds=nogds)
try:
@@ -738,14 +731,14 @@ def pt_weights_iterator(
) -> Generator[tuple[str, torch.Tensor], None, None]:
"""Iterate over the weights in the model bin/pt files."""
for bin_file in tqdm(
hf_weights_files,
desc="Loading pt checkpoint shards",
disable=not enable_tqdm(use_tqdm_on_load),
bar_format=_BAR_FORMAT,
hf_weights_files,
desc="Loading pt checkpoint shards",
disable=not enable_tqdm(use_tqdm_on_load),
bar_format=_BAR_FORMAT,
):
state = torch.load(bin_file,
map_location=pt_load_map_location,
weights_only=True)
state = torch.load(
bin_file, map_location=pt_load_map_location, weights_only=True
)
yield from state.items()
del state
@@ -759,15 +752,13 @@ def multi_thread_pt_weights_iterator(
"""Multi-Thread iterate over the weights in the model bin/pt files."""
def _load_file(bin_file: str):
return torch.load(bin_file,
map_location=pt_load_map_location,
weights_only=True)
return torch.load(
bin_file, map_location=pt_load_map_location, weights_only=True
)
with concurrent.futures.ThreadPoolExecutor(
max_workers=max_workers) as executor:
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = [
executor.submit(_load_file, bin_file)
for bin_file in hf_weights_files
executor.submit(_load_file, bin_file) for bin_file in hf_weights_files
]
futures_iter = tqdm(
concurrent.futures.as_completed(futures),
@@ -784,7 +775,8 @@ def multi_thread_pt_weights_iterator(
def get_gguf_extra_tensor_names(
gguf_file: str, gguf_to_hf_name_map: dict[str, str]) -> list[str]:
gguf_file: str, gguf_to_hf_name_map: dict[str, str]
) -> list[str]:
reader = gguf.GGUFReader(gguf_file)
expected_gguf_keys = set(gguf_to_hf_name_map.keys())
exact_gguf_keys = set([tensor.name for tensor in reader.tensors])
@@ -793,14 +785,16 @@ def get_gguf_extra_tensor_names(
def get_gguf_weight_type_map(
gguf_file: str, gguf_to_hf_name_map: dict[str, str]) -> dict[str, str]:
gguf_file: str, gguf_to_hf_name_map: dict[str, str]
) -> dict[str, str]:
"""
Return GGUF mapped weight's name and its quant type
"""
reader = gguf.GGUFReader(gguf_file)
return {
gguf_to_hf_name_map[tensor.name]: tensor.tensor_type.name
for tensor in reader.tensors if tensor.name in gguf_to_hf_name_map
for tensor in reader.tensors
if tensor.name in gguf_to_hf_name_map
}
@@ -850,8 +844,7 @@ def convert_pyslice_to_tensor(x: Any) -> torch.Tensor:
return x
def default_weight_loader(param: torch.Tensor,
loaded_weight: torch.Tensor) -> None:
def default_weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
"""Default weight loader."""
try:
if param.numel() == 1 and loaded_weight.numel() == 1:
@@ -862,7 +855,8 @@ def default_weight_loader(param: torch.Tensor,
else:
assert param.size() == loaded_weight.size(), (
f"Attempted to load weight ({loaded_weight.size()}) "
f"into parameter ({param.size()})")
f"into parameter ({param.size()})"
)
param.data.copy_(loaded_weight)
except Exception:
@@ -871,8 +865,9 @@ def default_weight_loader(param: torch.Tensor,
raise
def row_parallel_weight_loader(param: torch.Tensor,
loaded_weight: torch.Tensor) -> None:
def row_parallel_weight_loader(
param: torch.Tensor, loaded_weight: torch.Tensor
) -> None:
"""Load weights that are row-parallelized."""
tp_rank = get_tensor_model_parallel_rank()
shard_dim = 0 if param.dim() != 1 else None
@@ -904,12 +899,11 @@ def sharded_weight_loader(shard_axis: int) -> LoaderFunction:
def composed_weight_loader(
loader: LoaderFunction, fn: Callable[[torch.Tensor],
torch.Tensor]) -> LoaderFunction:
loader: LoaderFunction, fn: Callable[[torch.Tensor], torch.Tensor]
) -> LoaderFunction:
"""Create a weight loader that post-processes the weights after loading"""
def composed_loader(param: torch.Tensor,
loaded_weight: torch.Tensor) -> None:
def composed_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
loader(param, loaded_weight)
param.data.copy_(fn(param))
return
@@ -945,13 +939,18 @@ def initialize_dummy_weights(
# from a CPU tensor.
# Note: We avoid using torch.rank_like as it doesn't currently
# support the generator argument.
param.copy_((high - low) *
torch.rand(param.shape,
generator=generator,
dtype=param.dtype,
layout=param.layout,
requires_grad=param.requires_grad,
device="cpu") + low)
param.copy_(
(high - low)
* torch.rand(
param.shape,
generator=generator,
dtype=param.dtype,
layout=param.layout,
requires_grad=param.requires_grad,
device="cpu",
)
+ low
)
torch._sync(param)
continue
@@ -961,8 +960,7 @@ def initialize_dummy_weights(
# uniform_ doesn't support < 16-bit datatypes (FP8)
dtype = param.data.dtype
tmp_param = param.data.to(torch.float16)
tmp_param = tmp_param.uniform_(low, high,
generator=generator).to(dtype)
tmp_param = tmp_param.uniform_(low, high, generator=generator).to(dtype)
param.data.copy_(tmp_param)
else:
param.uniform_(low, high, generator=generator)
@@ -991,7 +989,8 @@ def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> Optional[str]:
"This format is deprecated in favor of separate k_scale and "
"v_scale tensors and will be removed in a future release. "
"Functionally, we will remap kv_scale to k_scale and duplicate "
"k_scale to v_scale")
"k_scale to v_scale"
)
# NOTE: we remap the deprecated kv_scale to k_scale
remapped_name = name.replace(".kv_scale", ".attn.k_scale")
if remapped_name not in params_dict:
@@ -1005,23 +1004,26 @@ def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> Optional[str]:
if any("mla_attn" in key for key in params_dict):
attn_str = "mla_attn.mla_attn"
logger.debug_once(f"Found mla_attn with k_scale and v_scale in "
f"the checkpoint, using {attn_str} as attn_str")
logger.debug_once(
f"Found mla_attn with k_scale and v_scale in "
f"the checkpoint, using {attn_str} as attn_str"
)
else:
attn_str = "attn"
# Define scale name mapping patterns in order of precedence
scale_mapping_patterns = [
# ModelOpt format: .self_attn.{k,v}_proj.{k,v}_scale ->
# .self_attn.attn.{k,v}_scale
(r"\.self_attn\.([kv])_proj\.([kv])_scale$",
rf".self_attn.{attn_str}.\2_scale"),
(
r"\.self_attn\.([kv])_proj\.([kv])_scale$",
rf".self_attn.{attn_str}.\2_scale",
),
# QKV proj format: .self_attn.qkv_proj.{k,v}_scale ->
# .self_attn.attn.{k,v}_scale
(r"\.self_attn\.qkv_proj\.([kv])_scale$", r".self_attn.attn.\1_scale"),
# Qwen3 MoE format: .self_attn.qkqkv_proj.{k,v}_scale ->
# .self_attn.attn.{k,v}_scale
(r"\.self_attn\.qkqkv_proj\.([kv])_scale$", r".self_attn.attn.\1_scale"
),
(r"\.self_attn\.qkqkv_proj\.([kv])_scale$", r".self_attn.attn.\1_scale"),
# Default format: .{k,v}_scale -> .attn.{k,v}_scale
(r"\.([kv])_scale$", r".attn.\1_scale"),
]