Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Utilities for downloading and initializing model weights."""
|
||||
|
||||
import concurrent.futures
|
||||
import fnmatch
|
||||
import glob
|
||||
@@ -28,18 +29,18 @@ from vllm.config import ModelConfig
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.distributed import get_tensor_model_parallel_rank
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization import (QuantizationConfig,
|
||||
get_quantization_config)
|
||||
from vllm.model_executor.layers.quantization import (
|
||||
QuantizationConfig,
|
||||
get_quantization_config,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils import PlaceholderModule
|
||||
|
||||
try:
|
||||
from runai_model_streamer import SafetensorsStreamer
|
||||
except ImportError:
|
||||
runai_model_streamer = PlaceholderModule(
|
||||
"runai_model_streamer") # type: ignore[assignment]
|
||||
SafetensorsStreamer = runai_model_streamer.placeholder_attr(
|
||||
"SafetensorsStreamer")
|
||||
runai_model_streamer = PlaceholderModule("runai_model_streamer") # type: ignore[assignment]
|
||||
SafetensorsStreamer = runai_model_streamer.placeholder_attr("SafetensorsStreamer")
|
||||
|
||||
try:
|
||||
import gguf
|
||||
@@ -50,8 +51,7 @@ try:
|
||||
from fastsafetensors import SafeTensorsFileLoader, SingleGroup
|
||||
except ImportError:
|
||||
fastsafetensors = PlaceholderModule("fastsafetensors")
|
||||
SafeTensorsFileLoader = fastsafetensors.placeholder_attr(
|
||||
"SafeTensorsFileLoader")
|
||||
SafeTensorsFileLoader = fastsafetensors.placeholder_attr("SafeTensorsFileLoader")
|
||||
SingleGroup = fastsafetensors.placeholder_attr("SingleGroup")
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -64,12 +64,12 @@ temp_dir = tempfile.gettempdir()
|
||||
|
||||
|
||||
def enable_hf_transfer():
|
||||
"""automatically activates hf_transfer
|
||||
"""
|
||||
"""automatically activates hf_transfer"""
|
||||
if "HF_HUB_ENABLE_HF_TRANSFER" not in os.environ:
|
||||
try:
|
||||
# enable hf hub transfer if available
|
||||
import hf_transfer # type: ignore # noqa
|
||||
|
||||
huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER = True
|
||||
except ImportError:
|
||||
pass
|
||||
@@ -79,13 +79,11 @@ enable_hf_transfer()
|
||||
|
||||
|
||||
class DisabledTqdm(tqdm):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs, disable=True)
|
||||
|
||||
|
||||
def get_lock(model_name_or_path: Union[str, Path],
|
||||
cache_dir: Optional[str] = None):
|
||||
def get_lock(model_name_or_path: Union[str, Path], cache_dir: Optional[str] = None):
|
||||
lock_dir = cache_dir or temp_dir
|
||||
model_name_or_path = str(model_name_or_path)
|
||||
os.makedirs(os.path.dirname(lock_dir), exist_ok=True)
|
||||
@@ -94,15 +92,14 @@ def get_lock(model_name_or_path: Union[str, Path],
|
||||
# add hash to avoid conflict with old users' lock files
|
||||
lock_file_name = hash_name + model_name + ".lock"
|
||||
# mode 0o666 is required for the filelock to be shared across users
|
||||
lock = filelock.FileLock(os.path.join(lock_dir, lock_file_name),
|
||||
mode=0o666)
|
||||
lock = filelock.FileLock(os.path.join(lock_dir, lock_file_name), mode=0o666)
|
||||
return lock
|
||||
|
||||
|
||||
@contextmanager
|
||||
def atomic_writer(filepath: Union[str, Path],
|
||||
mode: str = 'w',
|
||||
encoding: Optional[str] = None) -> Generator[IO]:
|
||||
def atomic_writer(
|
||||
filepath: Union[str, Path], mode: str = "w", encoding: Optional[str] = None
|
||||
) -> Generator[IO]:
|
||||
"""
|
||||
Context manager that provides an atomic file writing routine.
|
||||
|
||||
@@ -133,8 +130,8 @@ def atomic_writer(filepath: Union[str, Path],
|
||||
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Error during atomic write. Original file '%s' not modified",
|
||||
filepath)
|
||||
"Error during atomic write. Original file '%s' not modified", filepath
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
# Clean up the temporary file if it still exists.
|
||||
@@ -143,16 +140,16 @@ def atomic_writer(filepath: Union[str, Path],
|
||||
|
||||
|
||||
def maybe_download_from_modelscope(
|
||||
model: str,
|
||||
revision: Optional[str] = None,
|
||||
download_dir: Optional[str] = None,
|
||||
ignore_patterns: Optional[Union[str, list[str]]] = None,
|
||||
allow_patterns: Optional[Union[list[str],
|
||||
str]] = None) -> Optional[str]:
|
||||
model: str,
|
||||
revision: Optional[str] = None,
|
||||
download_dir: Optional[str] = None,
|
||||
ignore_patterns: Optional[Union[str, list[str]]] = None,
|
||||
allow_patterns: Optional[Union[list[str], str]] = None,
|
||||
) -> Optional[str]:
|
||||
"""Download model from ModelScope hub if VLLM_USE_MODELSCOPE is True.
|
||||
|
||||
Returns the path to the downloaded model, or None if the model is not
|
||||
downloaded from ModelScope."""
|
||||
Returns the path to the downloaded model, or None if the model is not
|
||||
downloaded from ModelScope."""
|
||||
if envs.VLLM_USE_MODELSCOPE:
|
||||
# download model from ModelScope hub,
|
||||
# lazy import so that modelscope is not required for normal use.
|
||||
@@ -226,9 +223,9 @@ def convert_bin_to_safetensor_file(
|
||||
|
||||
|
||||
# TODO(woosuk): Move this to other place.
|
||||
def get_quant_config(model_config: ModelConfig,
|
||||
load_config: LoadConfig) -> QuantizationConfig:
|
||||
|
||||
def get_quant_config(
|
||||
model_config: ModelConfig, load_config: LoadConfig
|
||||
) -> QuantizationConfig:
|
||||
quant_cls = get_quantization_config(model_config.quantization)
|
||||
|
||||
# GGUF doesn't have config file
|
||||
@@ -236,16 +233,14 @@ def get_quant_config(model_config: ModelConfig,
|
||||
return quant_cls()
|
||||
|
||||
# Read the quantization config from the HF model config, if available.
|
||||
hf_quant_config = getattr(model_config.hf_config, "quantization_config",
|
||||
None)
|
||||
hf_quant_config = getattr(model_config.hf_config, "quantization_config", None)
|
||||
# some vision model may keep quantization_config in their text_config
|
||||
hf_text_config = getattr(model_config.hf_config, "text_config", None)
|
||||
if hf_quant_config is None and hf_text_config is not None:
|
||||
hf_quant_config = getattr(hf_text_config, "quantization_config", None)
|
||||
if hf_quant_config is None:
|
||||
# compressed-tensors uses a compressions_config
|
||||
hf_quant_config = getattr(model_config.hf_config, "compression_config",
|
||||
None)
|
||||
hf_quant_config = getattr(model_config.hf_config, "compression_config", None)
|
||||
|
||||
if hf_quant_config is not None:
|
||||
return quant_cls.from_config(hf_quant_config)
|
||||
@@ -253,8 +248,7 @@ def get_quant_config(model_config: ModelConfig,
|
||||
# if hf_quant_config is None, we will try to get config from
|
||||
# hf_overrides
|
||||
hf_overrides = model_config.hf_overrides
|
||||
quantization_config_file = hf_overrides.get("quantization_config_file",
|
||||
None)
|
||||
quantization_config_file = hf_overrides.get("quantization_config_file", None)
|
||||
if quantization_config_file is not None:
|
||||
if hasattr(quant_cls, "from_config_file"):
|
||||
return quant_cls.from_config_file(quantization_config_file)
|
||||
@@ -262,9 +256,9 @@ def get_quant_config(model_config: ModelConfig,
|
||||
raise NotImplementedError(
|
||||
"from_config_file is specified in hf_override config, "
|
||||
"but quant_cls.from_config_file is not implemented in "
|
||||
f"{quant_cls}")
|
||||
quantization_config_json = hf_overrides.get(
|
||||
"quantization_config_dict_json", None)
|
||||
f"{quant_cls}"
|
||||
)
|
||||
quantization_config_json = hf_overrides.get("quantization_config_dict_json", None)
|
||||
if quantization_config_json is not None:
|
||||
if hasattr(quant_cls, "from_config_dict_json"):
|
||||
return quant_cls.from_config_dict_json(quantization_config_json)
|
||||
@@ -272,17 +266,21 @@ def get_quant_config(model_config: ModelConfig,
|
||||
raise NotImplementedError(
|
||||
"from_config_dict_json is specified in hf_override config, "
|
||||
"but quant_cls.from_config_dict_json is not implemented in "
|
||||
f"{quant_cls}")
|
||||
f"{quant_cls}"
|
||||
)
|
||||
|
||||
# Inflight BNB quantization
|
||||
if model_config.quantization == "bitsandbytes":
|
||||
return quant_cls.from_config({})
|
||||
model_name_or_path = maybe_download_from_modelscope(
|
||||
model_config.model,
|
||||
revision=model_config.revision,
|
||||
download_dir=load_config.download_dir,
|
||||
allow_patterns=["*.json"],
|
||||
) or model_config.model
|
||||
model_name_or_path = (
|
||||
maybe_download_from_modelscope(
|
||||
model_config.model,
|
||||
revision=model_config.revision,
|
||||
download_dir=load_config.download_dir,
|
||||
allow_patterns=["*.json"],
|
||||
)
|
||||
or model_config.model
|
||||
)
|
||||
is_local = os.path.isdir(model_name_or_path)
|
||||
if not is_local:
|
||||
# Download the config files.
|
||||
@@ -307,16 +305,15 @@ def get_quant_config(model_config: ModelConfig,
|
||||
config_files = glob.glob(os.path.join(hf_folder, "*.json"))
|
||||
|
||||
quant_config_files = [
|
||||
f for f in config_files if any(
|
||||
f.endswith(x) for x in possible_config_filenames)
|
||||
f for f in config_files if any(f.endswith(x) for x in possible_config_filenames)
|
||||
]
|
||||
if len(quant_config_files) == 0:
|
||||
raise ValueError(
|
||||
f"Cannot find the config file for {model_config.quantization}")
|
||||
raise ValueError(f"Cannot find the config file for {model_config.quantization}")
|
||||
if len(quant_config_files) > 1:
|
||||
raise ValueError(
|
||||
f"Found multiple config files for {model_config.quantization}: "
|
||||
f"{quant_config_files}")
|
||||
f"{quant_config_files}"
|
||||
)
|
||||
|
||||
quant_config_file = quant_config_files[0]
|
||||
with open(quant_config_file) as f:
|
||||
@@ -330,7 +327,8 @@ def get_quant_config(model_config: ModelConfig,
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported quantization config"
|
||||
f" found for {model_config.quantization} in {f}.")
|
||||
f" found for {model_config.quantization} in {f}."
|
||||
)
|
||||
|
||||
return quant_cls.from_config(config)
|
||||
|
||||
@@ -399,9 +397,7 @@ def download_weights_from_hf(
|
||||
# so we only have to call snapshot_download once.
|
||||
try:
|
||||
fs = HfFileSystem()
|
||||
file_list = fs.ls(model_name_or_path,
|
||||
detail=False,
|
||||
revision=revision)
|
||||
file_list = fs.ls(model_name_or_path, detail=False, revision=revision)
|
||||
|
||||
# Use the first pattern found in the HF repo's files.
|
||||
for pattern in allow_patterns:
|
||||
@@ -413,7 +409,10 @@ def download_weights_from_hf(
|
||||
logger.warning(
|
||||
"Failed to get file list for '%s'. Trying each pattern in "
|
||||
"allow_patterns individually until weights have been "
|
||||
"downloaded. Error: %s", model_name_or_path, e)
|
||||
"downloaded. Error: %s",
|
||||
model_name_or_path,
|
||||
e,
|
||||
)
|
||||
|
||||
logger.info("Using model weights format %s", allow_patterns)
|
||||
# Use file lock to prevent multiple processes from
|
||||
@@ -436,8 +435,11 @@ def download_weights_from_hf(
|
||||
break
|
||||
time_taken = time.perf_counter() - start_time
|
||||
if time_taken > 0.5:
|
||||
logger.info("Time spent downloading weights for %s: %.6f seconds",
|
||||
model_name_or_path, time_taken)
|
||||
logger.info(
|
||||
"Time spent downloading weights for %s: %.6f seconds",
|
||||
model_name_or_path,
|
||||
time_taken,
|
||||
)
|
||||
return hf_folder
|
||||
|
||||
|
||||
@@ -481,9 +483,9 @@ def download_safetensors_index_file_from_hf(
|
||||
# Passing both of these to the weight loader functionality breaks.
|
||||
# So, we use the index_file to
|
||||
# look up which safetensors files should be used.
|
||||
def filter_duplicate_safetensors_files(hf_weights_files: list[str],
|
||||
hf_folder: str,
|
||||
index_file: str) -> list[str]:
|
||||
def filter_duplicate_safetensors_files(
|
||||
hf_weights_files: list[str], hf_folder: str, index_file: str
|
||||
) -> list[str]:
|
||||
# model.safetensors.index.json is a mapping from keys in the
|
||||
# torch state_dict to safetensors file holding that weight.
|
||||
index_file_name = os.path.join(hf_folder, index_file)
|
||||
@@ -496,17 +498,13 @@ def filter_duplicate_safetensors_files(hf_weights_files: list[str],
|
||||
weight_map = json.load(f)["weight_map"]
|
||||
weight_files_in_index = set()
|
||||
for weight_name in weight_map:
|
||||
weight_files_in_index.add(
|
||||
os.path.join(hf_folder, weight_map[weight_name]))
|
||||
weight_files_in_index.add(os.path.join(hf_folder, weight_map[weight_name]))
|
||||
# Filter out any fields that are not found in the index file.
|
||||
hf_weights_files = [
|
||||
f for f in hf_weights_files if f in weight_files_in_index
|
||||
]
|
||||
hf_weights_files = [f for f in hf_weights_files if f in weight_files_in_index]
|
||||
return hf_weights_files
|
||||
|
||||
|
||||
def filter_files_not_needed_for_inference(
|
||||
hf_weights_files: list[str]) -> list[str]:
|
||||
def filter_files_not_needed_for_inference(hf_weights_files: list[str]) -> list[str]:
|
||||
"""
|
||||
Exclude files that are not needed for inference.
|
||||
|
||||
@@ -520,8 +518,7 @@ def filter_files_not_needed_for_inference(
|
||||
"scaler.pt",
|
||||
]
|
||||
hf_weights_files = [
|
||||
f for f in hf_weights_files
|
||||
if not any(f.endswith(x) for x in blacklist)
|
||||
f for f in hf_weights_files if not any(f.endswith(x) for x in blacklist)
|
||||
]
|
||||
return hf_weights_files
|
||||
|
||||
@@ -534,8 +531,9 @@ _BAR_FORMAT = "{desc}: {percentage:3.0f}% Completed | {n_fmt}/{total_fmt} [{elap
|
||||
|
||||
|
||||
def enable_tqdm(use_tqdm_on_load: bool):
|
||||
return use_tqdm_on_load and (not torch.distributed.is_initialized()
|
||||
or torch.distributed.get_rank() == 0)
|
||||
return use_tqdm_on_load and (
|
||||
not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0
|
||||
)
|
||||
|
||||
|
||||
def np_cache_weights_iterator(
|
||||
@@ -560,14 +558,12 @@ def np_cache_weights_iterator(
|
||||
if not os.path.exists(weight_names_file):
|
||||
weight_names: list[str] = []
|
||||
for bin_file in tqdm(
|
||||
hf_weights_files,
|
||||
desc="Loading np_cache checkpoint shards",
|
||||
disable=not enable_tqdm(use_tqdm_on_load),
|
||||
bar_format=_BAR_FORMAT,
|
||||
hf_weights_files,
|
||||
desc="Loading np_cache checkpoint shards",
|
||||
disable=not enable_tqdm(use_tqdm_on_load),
|
||||
bar_format=_BAR_FORMAT,
|
||||
):
|
||||
state = torch.load(bin_file,
|
||||
map_location="cpu",
|
||||
weights_only=True)
|
||||
state = torch.load(bin_file, map_location="cpu", weights_only=True)
|
||||
for name, param in state.items():
|
||||
param_path = os.path.join(np_folder, name)
|
||||
with open(param_path, "wb") as f:
|
||||
@@ -597,10 +593,10 @@ def safetensors_weights_iterator(
|
||||
loading_desc += " (eager)"
|
||||
|
||||
for st_file in tqdm(
|
||||
hf_weights_files,
|
||||
desc=loading_desc,
|
||||
disable=not enable_tqdm(use_tqdm_on_load),
|
||||
bar_format=_BAR_FORMAT,
|
||||
hf_weights_files,
|
||||
desc=loading_desc,
|
||||
disable=not enable_tqdm(use_tqdm_on_load),
|
||||
bar_format=_BAR_FORMAT,
|
||||
):
|
||||
if safetensors_load_strategy == "eager":
|
||||
with open(st_file, "rb") as f:
|
||||
@@ -624,12 +620,8 @@ def multi_thread_safetensors_weights_iterator(
|
||||
result = load_file(st_file, device="cpu")
|
||||
return result
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=max_workers) as executor:
|
||||
futures = [
|
||||
executor.submit(_load_file, st_file)
|
||||
for st_file in hf_weights_files
|
||||
]
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
futures = [executor.submit(_load_file, st_file) for st_file in hf_weights_files]
|
||||
futures_iter = tqdm(
|
||||
concurrent.futures.as_completed(futures),
|
||||
total=len(hf_weights_files),
|
||||
@@ -652,7 +644,8 @@ def runai_safetensors_weights_iterator(
|
||||
streamer.stream_files(hf_weights_files)
|
||||
total_tensors = sum(
|
||||
len(tensors_meta)
|
||||
for tensors_meta in streamer.files_to_tensors_metadata.values())
|
||||
for tensors_meta in streamer.files_to_tensors_metadata.values()
|
||||
)
|
||||
|
||||
tensor_iter = tqdm(
|
||||
streamer.get_tensors(),
|
||||
@@ -689,19 +682,19 @@ def fastsafetensors_weights_iterator(
|
||||
else:
|
||||
pg = SingleGroup()
|
||||
|
||||
device = torch.device(f'cuda:{pg.rank()}')
|
||||
device = torch.device(f"cuda:{pg.rank()}")
|
||||
weight_files_sub_lists = [
|
||||
hf_weights_files[i:i + pg.size()]
|
||||
hf_weights_files[i : i + pg.size()]
|
||||
for i in range(0, len(hf_weights_files), pg.size())
|
||||
]
|
||||
|
||||
nogds = False
|
||||
|
||||
for f_list in tqdm(
|
||||
weight_files_sub_lists,
|
||||
desc="Loading safetensors using Fastsafetensor loader",
|
||||
disable=not enable_tqdm(use_tqdm_on_load),
|
||||
bar_format=_BAR_FORMAT,
|
||||
weight_files_sub_lists,
|
||||
desc="Loading safetensors using Fastsafetensor loader",
|
||||
disable=not enable_tqdm(use_tqdm_on_load),
|
||||
bar_format=_BAR_FORMAT,
|
||||
):
|
||||
loader = _init_loader(pg, device, f_list, nogds=nogds)
|
||||
try:
|
||||
@@ -738,14 +731,14 @@ def pt_weights_iterator(
|
||||
) -> Generator[tuple[str, torch.Tensor], None, None]:
|
||||
"""Iterate over the weights in the model bin/pt files."""
|
||||
for bin_file in tqdm(
|
||||
hf_weights_files,
|
||||
desc="Loading pt checkpoint shards",
|
||||
disable=not enable_tqdm(use_tqdm_on_load),
|
||||
bar_format=_BAR_FORMAT,
|
||||
hf_weights_files,
|
||||
desc="Loading pt checkpoint shards",
|
||||
disable=not enable_tqdm(use_tqdm_on_load),
|
||||
bar_format=_BAR_FORMAT,
|
||||
):
|
||||
state = torch.load(bin_file,
|
||||
map_location=pt_load_map_location,
|
||||
weights_only=True)
|
||||
state = torch.load(
|
||||
bin_file, map_location=pt_load_map_location, weights_only=True
|
||||
)
|
||||
yield from state.items()
|
||||
del state
|
||||
|
||||
@@ -759,15 +752,13 @@ def multi_thread_pt_weights_iterator(
|
||||
"""Multi-Thread iterate over the weights in the model bin/pt files."""
|
||||
|
||||
def _load_file(bin_file: str):
|
||||
return torch.load(bin_file,
|
||||
map_location=pt_load_map_location,
|
||||
weights_only=True)
|
||||
return torch.load(
|
||||
bin_file, map_location=pt_load_map_location, weights_only=True
|
||||
)
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=max_workers) as executor:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
futures = [
|
||||
executor.submit(_load_file, bin_file)
|
||||
for bin_file in hf_weights_files
|
||||
executor.submit(_load_file, bin_file) for bin_file in hf_weights_files
|
||||
]
|
||||
futures_iter = tqdm(
|
||||
concurrent.futures.as_completed(futures),
|
||||
@@ -784,7 +775,8 @@ def multi_thread_pt_weights_iterator(
|
||||
|
||||
|
||||
def get_gguf_extra_tensor_names(
|
||||
gguf_file: str, gguf_to_hf_name_map: dict[str, str]) -> list[str]:
|
||||
gguf_file: str, gguf_to_hf_name_map: dict[str, str]
|
||||
) -> list[str]:
|
||||
reader = gguf.GGUFReader(gguf_file)
|
||||
expected_gguf_keys = set(gguf_to_hf_name_map.keys())
|
||||
exact_gguf_keys = set([tensor.name for tensor in reader.tensors])
|
||||
@@ -793,14 +785,16 @@ def get_gguf_extra_tensor_names(
|
||||
|
||||
|
||||
def get_gguf_weight_type_map(
|
||||
gguf_file: str, gguf_to_hf_name_map: dict[str, str]) -> dict[str, str]:
|
||||
gguf_file: str, gguf_to_hf_name_map: dict[str, str]
|
||||
) -> dict[str, str]:
|
||||
"""
|
||||
Return GGUF mapped weight's name and its quant type
|
||||
"""
|
||||
reader = gguf.GGUFReader(gguf_file)
|
||||
return {
|
||||
gguf_to_hf_name_map[tensor.name]: tensor.tensor_type.name
|
||||
for tensor in reader.tensors if tensor.name in gguf_to_hf_name_map
|
||||
for tensor in reader.tensors
|
||||
if tensor.name in gguf_to_hf_name_map
|
||||
}
|
||||
|
||||
|
||||
@@ -850,8 +844,7 @@ def convert_pyslice_to_tensor(x: Any) -> torch.Tensor:
|
||||
return x
|
||||
|
||||
|
||||
def default_weight_loader(param: torch.Tensor,
|
||||
loaded_weight: torch.Tensor) -> None:
|
||||
def default_weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
|
||||
"""Default weight loader."""
|
||||
try:
|
||||
if param.numel() == 1 and loaded_weight.numel() == 1:
|
||||
@@ -862,7 +855,8 @@ def default_weight_loader(param: torch.Tensor,
|
||||
else:
|
||||
assert param.size() == loaded_weight.size(), (
|
||||
f"Attempted to load weight ({loaded_weight.size()}) "
|
||||
f"into parameter ({param.size()})")
|
||||
f"into parameter ({param.size()})"
|
||||
)
|
||||
|
||||
param.data.copy_(loaded_weight)
|
||||
except Exception:
|
||||
@@ -871,8 +865,9 @@ def default_weight_loader(param: torch.Tensor,
|
||||
raise
|
||||
|
||||
|
||||
def row_parallel_weight_loader(param: torch.Tensor,
|
||||
loaded_weight: torch.Tensor) -> None:
|
||||
def row_parallel_weight_loader(
|
||||
param: torch.Tensor, loaded_weight: torch.Tensor
|
||||
) -> None:
|
||||
"""Load weights that are row-parallelized."""
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
shard_dim = 0 if param.dim() != 1 else None
|
||||
@@ -904,12 +899,11 @@ def sharded_weight_loader(shard_axis: int) -> LoaderFunction:
|
||||
|
||||
|
||||
def composed_weight_loader(
|
||||
loader: LoaderFunction, fn: Callable[[torch.Tensor],
|
||||
torch.Tensor]) -> LoaderFunction:
|
||||
loader: LoaderFunction, fn: Callable[[torch.Tensor], torch.Tensor]
|
||||
) -> LoaderFunction:
|
||||
"""Create a weight loader that post-processes the weights after loading"""
|
||||
|
||||
def composed_loader(param: torch.Tensor,
|
||||
loaded_weight: torch.Tensor) -> None:
|
||||
def composed_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
|
||||
loader(param, loaded_weight)
|
||||
param.data.copy_(fn(param))
|
||||
return
|
||||
@@ -945,13 +939,18 @@ def initialize_dummy_weights(
|
||||
# from a CPU tensor.
|
||||
# Note: We avoid using torch.rank_like as it doesn't currently
|
||||
# support the generator argument.
|
||||
param.copy_((high - low) *
|
||||
torch.rand(param.shape,
|
||||
generator=generator,
|
||||
dtype=param.dtype,
|
||||
layout=param.layout,
|
||||
requires_grad=param.requires_grad,
|
||||
device="cpu") + low)
|
||||
param.copy_(
|
||||
(high - low)
|
||||
* torch.rand(
|
||||
param.shape,
|
||||
generator=generator,
|
||||
dtype=param.dtype,
|
||||
layout=param.layout,
|
||||
requires_grad=param.requires_grad,
|
||||
device="cpu",
|
||||
)
|
||||
+ low
|
||||
)
|
||||
torch._sync(param)
|
||||
continue
|
||||
|
||||
@@ -961,8 +960,7 @@ def initialize_dummy_weights(
|
||||
# uniform_ doesn't support < 16-bit datatypes (FP8)
|
||||
dtype = param.data.dtype
|
||||
tmp_param = param.data.to(torch.float16)
|
||||
tmp_param = tmp_param.uniform_(low, high,
|
||||
generator=generator).to(dtype)
|
||||
tmp_param = tmp_param.uniform_(low, high, generator=generator).to(dtype)
|
||||
param.data.copy_(tmp_param)
|
||||
else:
|
||||
param.uniform_(low, high, generator=generator)
|
||||
@@ -991,7 +989,8 @@ def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> Optional[str]:
|
||||
"This format is deprecated in favor of separate k_scale and "
|
||||
"v_scale tensors and will be removed in a future release. "
|
||||
"Functionally, we will remap kv_scale to k_scale and duplicate "
|
||||
"k_scale to v_scale")
|
||||
"k_scale to v_scale"
|
||||
)
|
||||
# NOTE: we remap the deprecated kv_scale to k_scale
|
||||
remapped_name = name.replace(".kv_scale", ".attn.k_scale")
|
||||
if remapped_name not in params_dict:
|
||||
@@ -1005,23 +1004,26 @@ def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> Optional[str]:
|
||||
|
||||
if any("mla_attn" in key for key in params_dict):
|
||||
attn_str = "mla_attn.mla_attn"
|
||||
logger.debug_once(f"Found mla_attn with k_scale and v_scale in "
|
||||
f"the checkpoint, using {attn_str} as attn_str")
|
||||
logger.debug_once(
|
||||
f"Found mla_attn with k_scale and v_scale in "
|
||||
f"the checkpoint, using {attn_str} as attn_str"
|
||||
)
|
||||
else:
|
||||
attn_str = "attn"
|
||||
# Define scale name mapping patterns in order of precedence
|
||||
scale_mapping_patterns = [
|
||||
# ModelOpt format: .self_attn.{k,v}_proj.{k,v}_scale ->
|
||||
# .self_attn.attn.{k,v}_scale
|
||||
(r"\.self_attn\.([kv])_proj\.([kv])_scale$",
|
||||
rf".self_attn.{attn_str}.\2_scale"),
|
||||
(
|
||||
r"\.self_attn\.([kv])_proj\.([kv])_scale$",
|
||||
rf".self_attn.{attn_str}.\2_scale",
|
||||
),
|
||||
# QKV proj format: .self_attn.qkv_proj.{k,v}_scale ->
|
||||
# .self_attn.attn.{k,v}_scale
|
||||
(r"\.self_attn\.qkv_proj\.([kv])_scale$", r".self_attn.attn.\1_scale"),
|
||||
# Qwen3 MoE format: .self_attn.qkqkv_proj.{k,v}_scale ->
|
||||
# .self_attn.attn.{k,v}_scale
|
||||
(r"\.self_attn\.qkqkv_proj\.([kv])_scale$", r".self_attn.attn.\1_scale"
|
||||
),
|
||||
(r"\.self_attn\.qkqkv_proj\.([kv])_scale$", r".self_attn.attn.\1_scale"),
|
||||
# Default format: .{k,v}_scale -> .attn.{k,v}_scale
|
||||
(r"\.([kv])_scale$", r".attn.\1_scale"),
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user