Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -94,6 +94,7 @@ def register_model_loader(load_format: str):
|
||||
>>> load_config = LoadConfig(load_format="my_loader")
|
||||
>>> type(get_model_loader(load_config))
|
||||
<class 'MyModelLoader'>
|
||||
|
||||
""" # noqa: E501
|
||||
|
||||
def _wrapper(model_loader_cls):
|
||||
|
||||
@@ -36,7 +36,8 @@ class BaseModelLoader(ABC):
|
||||
@abstractmethod
|
||||
def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:
|
||||
"""Load weights into a model. This standalone API allows
|
||||
inplace weights loading for an already-initialized model"""
|
||||
inplace weights loading for an already-initialized model
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def create_model(
|
||||
|
||||
@@ -36,6 +36,7 @@ class CheckpointWeightPatch(NamedTuple):
|
||||
``indices`` for a sparse patch.
|
||||
indices: Flat indices into the full checkpoint tensor described by
|
||||
``shape``. ``None`` makes ``values`` a dense replacement.
|
||||
|
||||
"""
|
||||
|
||||
name: str
|
||||
@@ -176,8 +177,8 @@ def load_checkpoint_weight_patches(
|
||||
|
||||
Returns:
|
||||
Union of the weight names reported by all ``model.load_weights`` calls.
|
||||
"""
|
||||
|
||||
"""
|
||||
if max_chunk_bytes <= 0:
|
||||
raise ValueError("max_chunk_bytes must be positive")
|
||||
|
||||
|
||||
@@ -135,7 +135,8 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
) -> tuple[str, list[str], bool]:
|
||||
"""Prepare weights for the model.
|
||||
|
||||
If the model is not local, it will be downloaded."""
|
||||
If the model is not local, it will be downloaded.
|
||||
"""
|
||||
model_name_or_path = (
|
||||
maybe_download_from_modelscope(model_name_or_path, revision)
|
||||
or model_name_or_path
|
||||
|
||||
@@ -23,7 +23,8 @@ def parse_expert_id(weight_name: str) -> int | None:
|
||||
|
||||
Returns ``None`` for dense weights (attention, layernorm, embedding),
|
||||
shared experts, and 3D fused-expert tensors where all experts are stored
|
||||
in a single tensor without a numeric expert id in the name."""
|
||||
in a single tensor without a numeric expert id in the name.
|
||||
"""
|
||||
m = _EXPERT_ID_RE.search(weight_name)
|
||||
return int(m.group(1)) if m else None
|
||||
|
||||
@@ -43,8 +44,12 @@ def compute_local_expert_ids(
|
||||
:func:`vllm.model_executor.layers.fused_moe.layer.determine_expert_map`.
|
||||
|
||||
Args:
|
||||
num_experts: Total number of experts in the layer.
|
||||
ep_size: Expert parallel world size.
|
||||
ep_rank: Rank whose local expert ids are computed.
|
||||
placement: ``"linear"`` for contiguous assignment,
|
||||
``"round_robin"`` for interleaved assignment.
|
||||
|
||||
"""
|
||||
if ep_size <= 1:
|
||||
return None
|
||||
@@ -66,7 +71,8 @@ def should_skip_weight(
|
||||
local_expert_ids: set[int] | None,
|
||||
) -> bool:
|
||||
"""Return ``True`` if *weight_name* is an expert weight that does not
|
||||
belong to the local rank and should be skipped during loading."""
|
||||
belong to the local rank and should be skipped during loading.
|
||||
"""
|
||||
if local_expert_ids is None:
|
||||
return False
|
||||
eid = parse_expert_id(weight_name)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Layerwise weight reloading utilities for vLLM.
|
||||
"""Layerwise weight reloading utilities for vLLM.
|
||||
|
||||
This module provides functionality to reload model weights layer-by-layer,
|
||||
which is useful for weight updates without full model reconstruction.
|
||||
|
||||
@@ -54,8 +54,7 @@ LOADING_LAYERS: WeakSet[torch.nn.Module] = WeakSet()
|
||||
|
||||
|
||||
def get_layerwise_info(layer: torch.nn.Module) -> LayerReloadingInfo:
|
||||
"""
|
||||
Get information related to restoring and layerwise processing. If no previous
|
||||
"""Get information related to restoring and layerwise processing. If no previous
|
||||
information existed, a new entry is constructed
|
||||
"""
|
||||
if layer not in LAYERWISE_INFO:
|
||||
@@ -68,8 +67,7 @@ def get_layerwise_info(layer: torch.nn.Module) -> LayerReloadingInfo:
|
||||
|
||||
|
||||
def record_metadata_for_reloading(model: torch.nn.Module):
|
||||
"""
|
||||
Record layer metadata needed for later reloading.
|
||||
"""Record layer metadata needed for later reloading.
|
||||
|
||||
Stores parameter and buffer metadata as meta tensors for restoration.
|
||||
Must be called before `initialize_layerwise_reload`.
|
||||
@@ -82,8 +80,7 @@ def record_metadata_for_reloading(model: torch.nn.Module):
|
||||
|
||||
@torch.no_grad()
|
||||
def initialize_layerwise_reload(model: torch.nn.Module):
|
||||
"""
|
||||
Set up layerwise weight loading with deferred processing.
|
||||
"""Set up layerwise weight loading with deferred processing.
|
||||
|
||||
Must be called after `record_metadata_for_reloading`. This function:
|
||||
1. Saves current kernel tensors for later copying
|
||||
@@ -120,13 +117,13 @@ def initialize_layerwise_reload(model: torch.nn.Module):
|
||||
|
||||
|
||||
def initialize_online_processing(layer: torch.nn.Module):
|
||||
"""
|
||||
Wrap a layer's weight loaders with online processing loaders.
|
||||
"""Wrap a layer's weight loaders with online processing loaders.
|
||||
Called by either `initialize_layerwise_reload` or an online quantization scheme,
|
||||
prevents double wrapping in the case of online quantization + reloading
|
||||
|
||||
Args:
|
||||
layer: layer whose parameter weight loaders will be wrapped
|
||||
|
||||
"""
|
||||
info = get_layerwise_info(layer)
|
||||
|
||||
@@ -226,8 +223,7 @@ def make_online_process_loader(layer: torch.nn.Module, param_name: str) -> Calla
|
||||
|
||||
|
||||
def finalize_layerwise_processing(model: torch.nn.Module, model_config: ModelConfig):
|
||||
"""
|
||||
Apply processing to any layers which were not layerwise processed during loading.
|
||||
"""Apply processing to any layers which were not layerwise processed during loading.
|
||||
This includes attention layers and layers which have weight elements which are not
|
||||
loaded (due to padding).
|
||||
|
||||
@@ -237,6 +233,7 @@ def finalize_layerwise_processing(model: torch.nn.Module, model_config: ModelCon
|
||||
Args:
|
||||
model: model to finalize processing for
|
||||
model_config: config needed for applying processing to attention layers
|
||||
|
||||
"""
|
||||
if hasattr(model, "_original_do_torchao_reload"):
|
||||
model._do_torchao_reload = model._original_do_torchao_reload
|
||||
@@ -311,7 +308,8 @@ def _reload_attention_scales(layer: torch.nn.Module, info: LayerReloadingInfo) -
|
||||
|
||||
Assumes dtype/shapes of attention tensors do not change during
|
||||
processing, since we use .data.copy_() to preserve kernel tensor
|
||||
references."""
|
||||
references.
|
||||
"""
|
||||
quant_method = getattr(layer, "quant_method", None)
|
||||
if quant_method is not None:
|
||||
# Re-create scale Parameters with sentinel values so unloaded scales
|
||||
@@ -330,8 +328,7 @@ def _reload_attention_scales(layer: torch.nn.Module, info: LayerReloadingInfo) -
|
||||
|
||||
|
||||
def _layerwise_process(layer: torch.nn.Module, info: LayerReloadingInfo):
|
||||
"""
|
||||
Finalize layer loading after all weights have been buffered.
|
||||
"""Finalize layer loading after all weights have been buffered.
|
||||
|
||||
This function:
|
||||
1. Materializes the layer onto the target device
|
||||
@@ -377,7 +374,7 @@ def _layerwise_process(layer: torch.nn.Module, info: LayerReloadingInfo):
|
||||
|
||||
|
||||
def _get_original_loader(tensor: torch.Tensor) -> Callable:
|
||||
"""Return the weight loader with any layerwise wrappers removed"""
|
||||
"""Return the weight loader with any layerwise wrappers removed."""
|
||||
loader = _get_weight_loader(tensor)
|
||||
while loader.__name__ == "online_process_loader":
|
||||
loader = loader.__wrapped__ # type: ignore[union-attr]
|
||||
@@ -391,7 +388,8 @@ def _get_weight_loader(tensor: torch.Tensor):
|
||||
|
||||
def _copy_and_restore_kernel_tensors(layer: torch.nn.Module, info: LayerReloadingInfo):
|
||||
"""Copy processed values into original kernel tensor storage and restore
|
||||
kernel tensor references on the layer. Preserves cudagraph references."""
|
||||
kernel tensor references on the layer. Preserves cudagraph references.
|
||||
"""
|
||||
assert info.kernel_tensors is not None
|
||||
parameters, buffers = info.kernel_tensors
|
||||
non_persistent = info.kernel_non_persistent_buffers
|
||||
|
||||
@@ -47,8 +47,7 @@ def to_meta_tensor(tensor: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
|
||||
def materialize_meta_tensor(meta_tensor: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Materialize a meta tensor into an actual tensor on the current device.
|
||||
"""Materialize a meta tensor into an actual tensor on the current device.
|
||||
Should be called within the torch device context for the given rank.
|
||||
"""
|
||||
tensor = torch.empty_strided(
|
||||
@@ -119,7 +118,7 @@ def capture_layer_to_meta(layer: torch.nn.Module) -> LayerTensors:
|
||||
|
||||
|
||||
def restore_layer_on_meta(layer: torch.nn.Module, info: LayerReloadingInfo):
|
||||
"""Restore a layer to model format with tensors on the meta device"""
|
||||
"""Restore a layer to model format with tensors on the meta device."""
|
||||
if layer.__class__.__name__ in SKIP_MODULES:
|
||||
return
|
||||
|
||||
@@ -155,8 +154,7 @@ def materialize_layer(layer: torch.nn.Module, info: LayerReloadingInfo):
|
||||
|
||||
|
||||
class CopyCounter(TorchDispatchMode):
|
||||
"""
|
||||
Tracks total number of elements modified with `copy_`.
|
||||
"""Tracks total number of elements modified with `copy_`.
|
||||
|
||||
Useful for keeping track of weight loading where underlying weights can be
|
||||
arbitrarily transformed (such as with `narrow`) before calling copy.
|
||||
@@ -182,8 +180,7 @@ class CopyCounter(TorchDispatchMode):
|
||||
def get_numel_loaded(
|
||||
weight_loader: Callable, args: inspect.BoundArguments
|
||||
) -> tuple[int, object]:
|
||||
"""
|
||||
Determine how many elements would be loaded by a weight loader call.
|
||||
"""Determine how many elements would be loaded by a weight loader call.
|
||||
|
||||
Args:
|
||||
weight_loader: used to load weights
|
||||
@@ -192,6 +189,7 @@ def get_numel_loaded(
|
||||
Returns:
|
||||
number of elements loaded by the weight loader, the return value of the
|
||||
weight loader
|
||||
|
||||
"""
|
||||
with CopyCounter() as counter:
|
||||
return_value = weight_loader(*args.args, **args.kwargs)
|
||||
|
||||
@@ -11,8 +11,7 @@ layer_ref_sentinel = object()
|
||||
|
||||
|
||||
def sanitize_layer_refs(tensor: torch.Tensor, layer: torch.nn.Module) -> torch.Tensor:
|
||||
"""
|
||||
Removes references to layer held by tensor attributes. Specifically, removes the
|
||||
"""Removes references to layer held by tensor attributes. Specifically, removes the
|
||||
`__self__` attribute of weight loader methods attached to the tensor.
|
||||
|
||||
Used by `capture_layer_to_meta` to avoid circular references to layers in
|
||||
@@ -26,6 +25,7 @@ def sanitize_layer_refs(tensor: torch.Tensor, layer: torch.nn.Module) -> torch.T
|
||||
|
||||
Returns:
|
||||
sanitized tensor
|
||||
|
||||
"""
|
||||
for key, value in tensor.__dict__.items():
|
||||
if isinstance(value, MethodType) and value.__self__ is layer:
|
||||
@@ -35,8 +35,7 @@ def sanitize_layer_refs(tensor: torch.Tensor, layer: torch.nn.Module) -> torch.T
|
||||
|
||||
|
||||
def restore_layer_refs(tensor: torch.Tensor, layer: torch.nn.Module) -> torch.Tensor:
|
||||
"""
|
||||
Restores references to layer held by tensor attributes.
|
||||
"""Restores references to layer held by tensor attributes.
|
||||
|
||||
Used by `restore_layer_on_meta` to add back layer references, allowing for proper
|
||||
weight loading.
|
||||
@@ -47,6 +46,7 @@ def restore_layer_refs(tensor: torch.Tensor, layer: torch.nn.Module) -> torch.Te
|
||||
|
||||
Returns:
|
||||
sanitized tensor
|
||||
|
||||
"""
|
||||
for key, value in tensor.__dict__.items():
|
||||
if isinstance(value, MethodType) and value.__self__ is layer_ref_sentinel:
|
||||
|
||||
@@ -27,8 +27,7 @@ def set_torchao_reload_attrs(model: torch.nn.Module, model_config: ModelConfig):
|
||||
|
||||
|
||||
def support_quantized_model_reload_from_hp_weights(original_load_weights: FunctionType):
|
||||
"""
|
||||
Decorator for `load_weights` method for AutoWeightsLoader.load_weights to support
|
||||
"""Decorator for `load_weights` method for AutoWeightsLoader.load_weights to support
|
||||
reloading high precision (bfloat16/float16/float32) weight for an already quantized
|
||||
model, this involves restoring the weights to a high precision weights and
|
||||
then online quantize the weights.
|
||||
|
||||
@@ -46,14 +46,14 @@ def get_layer_size(layer: torch.nn.Module) -> int:
|
||||
|
||||
|
||||
def has_device_tensors(bound_args: BoundArguments) -> bool:
|
||||
"""
|
||||
Return True if the loaded weights exist on an accelerator device
|
||||
"""Return True if the loaded weights exist on an accelerator device.
|
||||
|
||||
Args:
|
||||
bound_args: args to load weights
|
||||
|
||||
Returns:
|
||||
True if weights are on accelerator device
|
||||
|
||||
"""
|
||||
return any(
|
||||
isinstance(value, torch.Tensor) and value.device.type not in ("meta", "cpu")
|
||||
@@ -62,14 +62,14 @@ def has_device_tensors(bound_args: BoundArguments) -> bool:
|
||||
|
||||
|
||||
def get_info_size(info: LayerReloadingInfo) -> int:
|
||||
"""
|
||||
Calculate the number of bytes used by loaded weights for a given layer
|
||||
"""Calculate the number of bytes used by loaded weights for a given layer.
|
||||
|
||||
Args:
|
||||
info: layerwise info to get size of
|
||||
|
||||
Returns:
|
||||
number of bytes used by loaded weights
|
||||
|
||||
"""
|
||||
return sum(
|
||||
value.nbytes
|
||||
|
||||
@@ -19,8 +19,7 @@ from vllm.transformers_utils.runai_utils import is_runai_obj_uri, list_safetenso
|
||||
|
||||
|
||||
class RunaiModelStreamerLoader(BaseModelLoader):
|
||||
"""
|
||||
Model loader that can load safetensors
|
||||
"""Model loader that can load safetensors
|
||||
files from local FS, S3, GCS, or Azure Blob Storage.
|
||||
"""
|
||||
|
||||
@@ -82,8 +81,8 @@ class RunaiModelStreamerLoader(BaseModelLoader):
|
||||
) -> list[str]:
|
||||
"""Prepare weights for the model.
|
||||
|
||||
If the model is not local, it will be downloaded."""
|
||||
|
||||
If the model is not local, it will be downloaded.
|
||||
"""
|
||||
is_object_storage_path = is_runai_obj_uri(model_name_or_path)
|
||||
is_local = os.path.isdir(model_name_or_path)
|
||||
safetensors_pattern = "*.safetensors"
|
||||
@@ -127,7 +126,7 @@ class RunaiModelStreamerLoader(BaseModelLoader):
|
||||
)
|
||||
|
||||
def download_model(self, model_config: ModelConfig) -> None:
|
||||
"""Download model if necessary"""
|
||||
"""Download model if necessary."""
|
||||
self._prepare_weights(model_config.model, model_config.revision)
|
||||
|
||||
def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:
|
||||
|
||||
@@ -27,8 +27,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ShardedStateLoader(BaseModelLoader):
|
||||
"""
|
||||
Model loader that directly loads each worker's model state dict, which
|
||||
"""Model loader that directly loads each worker's model state dict, which
|
||||
enables a fast load path for large tensor-parallel models where each worker
|
||||
only needs to read its own shard rather than the entire checkpoint. See
|
||||
`examples/features/sharded_state/save_sharded_state_offline.py` for creating
|
||||
@@ -57,8 +56,7 @@ class ShardedStateLoader(BaseModelLoader):
|
||||
def _filter_subtensors(
|
||||
tensors: dict[str, torch.Tensor],
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""
|
||||
Filter out all tensors that share the same memory or a subset of the
|
||||
"""Filter out all tensors that share the same memory or a subset of the
|
||||
memory of another tensor.
|
||||
"""
|
||||
same_storage_groups: dict[Any, list[tuple[str, torch.Tensor]]] = (
|
||||
|
||||
@@ -403,8 +403,7 @@ class TensorizerArgs:
|
||||
|
||||
@staticmethod
|
||||
def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||
"""Tensorizer CLI arguments"""
|
||||
|
||||
"""Tensorizer CLI arguments."""
|
||||
# Tensorizer options arg group
|
||||
group = parser.add_argument_group(
|
||||
"tensorizer options",
|
||||
@@ -498,7 +497,8 @@ def _check_tensors_on_meta_device(model: nn.Module) -> None:
|
||||
|
||||
def _resize_lora_embeddings(model: nn.Module):
|
||||
"""Modify LoRA embedding layers to use bigger tensors
|
||||
to allow for adapter added tokens."""
|
||||
to allow for adapter added tokens.
|
||||
"""
|
||||
for child in model.modules():
|
||||
if (
|
||||
isinstance(child, VocabParallelEmbedding)
|
||||
@@ -593,8 +593,7 @@ def tensorizer_weights_iterator(
|
||||
|
||||
|
||||
def is_vllm_tensorized(tensorizer_config: "TensorizerConfig") -> bool:
|
||||
"""
|
||||
Infer if the model is a vLLM model by checking the weights for
|
||||
"""Infer if the model is a vLLM model by checking the weights for
|
||||
a vLLM tensorized marker.
|
||||
|
||||
Args:
|
||||
@@ -603,6 +602,7 @@ def is_vllm_tensorized(tensorizer_config: "TensorizerConfig") -> bool:
|
||||
|
||||
Returns:
|
||||
bool: True if the model is a vLLM model, False otherwise.
|
||||
|
||||
"""
|
||||
tensorizer_args = tensorizer_config._construct_tensorizer_args()
|
||||
deserializer = TensorDeserializer(
|
||||
@@ -711,7 +711,7 @@ def tensorize_vllm_model(
|
||||
tensorizer_config: TensorizerConfig,
|
||||
generate_keyfile: bool = True,
|
||||
):
|
||||
"""Utility to load a model and then serialize it with Tensorizer
|
||||
"""Utility to load a model and then serialize it with Tensorizer.
|
||||
|
||||
Intended to be used separately from running a vLLM server since it
|
||||
creates its own Engine instance.
|
||||
@@ -773,8 +773,7 @@ def tensorize_vllm_model(
|
||||
|
||||
|
||||
def tensorize_lora_adapter(lora_path: str, tensorizer_config: TensorizerConfig):
|
||||
"""
|
||||
Uses tensorizer to serialize a LoRA adapter. Assumes that the files
|
||||
"""Uses tensorizer to serialize a LoRA adapter. Assumes that the files
|
||||
needed to load a LoRA adapter are a safetensors-format file called
|
||||
adapter_model.safetensors and a json config file called adapter_config.json.
|
||||
|
||||
|
||||
@@ -105,7 +105,8 @@ class TensorizerLoader(BaseModelLoader):
|
||||
|
||||
Expects a vLLM-tensorized model. See the
|
||||
examples/features/tensorize_vllm_model.py example script
|
||||
for serializing vLLM models."""
|
||||
for serializing vLLM models.
|
||||
"""
|
||||
if is_vllm_tensorized(self.tensorizer_config):
|
||||
tensorizer_config = self._patch_tensorizer_config(model_config)
|
||||
deserialize_tensorizer_model(model, tensorizer_config)
|
||||
|
||||
@@ -278,8 +278,7 @@ def get_architecture_class_name(model_config: ModelConfig) -> str:
|
||||
def configure_quant_config(
|
||||
quant_config: QuantizationConfig, model_class: type[nn.Module]
|
||||
):
|
||||
"""
|
||||
Pass packed_modules_mapping by reference to quant_config so that
|
||||
"""Pass packed_modules_mapping by reference to quant_config so that
|
||||
quant_config can properly match fused modules
|
||||
|
||||
Note that model attributes are passed by reference to quant_config,
|
||||
|
||||
@@ -105,6 +105,7 @@ def export_entries(
|
||||
A ``(entries, aliases)`` pair where ``entries`` maps a canonical name to
|
||||
its ``TensorEntry`` and ``aliases`` maps each duplicate name to its
|
||||
canonical name.
|
||||
|
||||
"""
|
||||
entries: dict[str, TensorEntry] = {}
|
||||
aliases: dict[str, str] = {}
|
||||
@@ -206,6 +207,7 @@ class WeightCacheDaemon:
|
||||
Args:
|
||||
ready_callback: Invoked once the socket is bound and listening,
|
||||
so the launcher can report overall readiness.
|
||||
|
||||
"""
|
||||
socket_path = self._socket_path
|
||||
ensure_private_socket_dir(
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""IPC model loader: maps post-quantized weights from a local weight cache
|
||||
daemon via CUDA IPC instead of loading from disk."""
|
||||
daemon via CUDA IPC instead of loading from disk.
|
||||
"""
|
||||
|
||||
import dataclasses
|
||||
import socket
|
||||
|
||||
@@ -74,6 +74,7 @@ def check_ipc_platform_support() -> None:
|
||||
Raises:
|
||||
UnsupportedPlatformForIPCError: If the current platform is not
|
||||
CUDA/ROCm.
|
||||
|
||||
"""
|
||||
if current_platform.is_cuda_alike():
|
||||
return
|
||||
@@ -94,6 +95,7 @@ def check_ipc_quant_support(model: torch.nn.Module) -> None:
|
||||
Raises:
|
||||
UnsupportedQuantForIPCError: If any quant method does not declare
|
||||
``supports_pre_processed_weights``.
|
||||
|
||||
"""
|
||||
for name, module in model.named_modules():
|
||||
quant_method = getattr(module, "quant_method", None)
|
||||
|
||||
@@ -78,7 +78,8 @@ def maybe_retie_word_embeddings(model: nn.Module, model_config: ModelConfig) ->
|
||||
|
||||
Checkpoints produced by quantization or fine-tuning tooling often keep a redundant
|
||||
copy of the tied `lm_head`. Sharing the storage again reclaims the memory it would
|
||||
otherwise cost."""
|
||||
otherwise cost.
|
||||
"""
|
||||
if not model_config.word_embeddings_untied_by_checkpoint:
|
||||
return
|
||||
if (untied := _get_untied_lm_head(model)) is None:
|
||||
|
||||
@@ -72,7 +72,7 @@ temp_dir = tempfile.gettempdir()
|
||||
|
||||
|
||||
def enable_xet_high_performance():
|
||||
"""automatically activates xet high performance mode"""
|
||||
"""Automatically activates xet high performance mode"""
|
||||
if "HF_XET_HIGH_PERFORMANCE" not in os.environ:
|
||||
huggingface_hub.constants.HF_XET_HIGH_PERFORMANCE = True
|
||||
|
||||
@@ -103,8 +103,7 @@ def get_lock(model_name_or_path: str | Path, cache_dir: str | None = None):
|
||||
def atomic_writer(
|
||||
filepath: str | Path, mode: str = "w", encoding: str | None = None
|
||||
) -> Generator[IO]:
|
||||
"""
|
||||
Context manager that provides an atomic file writing routine.
|
||||
"""Context manager that provides an atomic file writing routine.
|
||||
|
||||
The context manager writes to a temporary file and, if successful,
|
||||
atomically replaces the original file.
|
||||
@@ -116,6 +115,7 @@ def atomic_writer(
|
||||
|
||||
Yields:
|
||||
file object: A handle to the temporary file.
|
||||
|
||||
"""
|
||||
# Create a temporary file in the same directory as the target file
|
||||
# to ensure it's on the same filesystem for an atomic replace.
|
||||
@@ -144,7 +144,8 @@ def atomic_writer(
|
||||
|
||||
def _natural_sort_key(filepath: str) -> list:
|
||||
"""Natural sort key for filenames with numeric components, such as
|
||||
model-00001-of-00005.safetensors -> ['model-', 1, '-of-', 5, '.safetensors']"""
|
||||
model-00001-of-00005.safetensors -> ['model-', 1, '-of-', 5, '.safetensors']
|
||||
"""
|
||||
return [
|
||||
int(s) if s.isdigit() else s
|
||||
for s in re.split(r"(\d+)", os.path.basename(filepath))
|
||||
@@ -161,7 +162,8 @@ def maybe_download_from_modelscope(
|
||||
"""Download model from ModelScope hub if VLLM_USE_MODELSCOPE is True.
|
||||
|
||||
Returns the path to the downloaded model, or None if the model is not
|
||||
downloaded from ModelScope."""
|
||||
downloaded from ModelScope.
|
||||
"""
|
||||
if envs.VLLM_USE_MODELSCOPE:
|
||||
# download model from ModelScope hub,
|
||||
# lazy import so that modelscope is not required for normal use.
|
||||
@@ -482,6 +484,7 @@ def download_weights_from_hf(
|
||||
|
||||
Returns:
|
||||
str: The path to the downloaded model weights.
|
||||
|
||||
"""
|
||||
assert len(allow_patterns) > 0
|
||||
local_only = huggingface_hub.constants.HF_HUB_OFFLINE
|
||||
@@ -580,6 +583,7 @@ def download_safetensors_index_file_from_hf(
|
||||
subfolder (Optional[str]): The subfolder within the model repository
|
||||
to download weights from.
|
||||
revision (Optional[str]): The revision of the model.
|
||||
|
||||
"""
|
||||
# Use file lock to prevent multiple processes from
|
||||
# downloading the same model weights at the same time.
|
||||
@@ -637,8 +641,7 @@ def filter_duplicate_safetensors_files(
|
||||
|
||||
|
||||
def filter_files_not_needed_for_inference(hf_weights_files: list[str]) -> list[str]:
|
||||
"""
|
||||
Exclude files that are not needed for inference.
|
||||
"""Exclude files that are not needed for inference.
|
||||
|
||||
See https://github.com/huggingface/transformers/blob/v4.34.0/src/transformers/trainer.py#L227-L233
|
||||
"""
|
||||
@@ -1145,7 +1148,8 @@ def instanttensor_weights_iterator(
|
||||
use_tqdm_on_load: bool,
|
||||
) -> Generator[tuple[str, torch.Tensor], None, None]:
|
||||
"""Iterate over the weights in the model safetensor files
|
||||
using instanttensor library."""
|
||||
using instanttensor library.
|
||||
"""
|
||||
try:
|
||||
import instanttensor
|
||||
except ImportError as e:
|
||||
@@ -1407,6 +1411,7 @@ def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> str | None:
|
||||
str: The remapped parameter name if successful, or the original name
|
||||
if no remapping is needed.
|
||||
None: If the remapped name is not found in params_dict.
|
||||
|
||||
"""
|
||||
# Already in vLLM's expected form (e.g. weights pre-renamed by a
|
||||
# `WeightsMapper` from the quant config). Skip the regex remap, which
|
||||
@@ -1506,8 +1511,7 @@ def maybe_remap_moe_expert_param_name(
|
||||
name: str,
|
||||
params_dict: dict[str, torch.nn.Parameter],
|
||||
) -> str:
|
||||
"""
|
||||
Remap MoE expert parameter names to account for routed_experts hierarchy.
|
||||
"""Remap MoE expert parameter names to account for routed_experts hierarchy.
|
||||
|
||||
This handles the transition from the old FusedMoE structure where weights
|
||||
were directly in the experts module, to the new MoERunner → RoutedExperts
|
||||
@@ -1529,6 +1533,7 @@ def maybe_remap_moe_expert_param_name(
|
||||
Returns:
|
||||
Remapped parameter name if routed_experts hierarchy exists,
|
||||
otherwise the original name
|
||||
|
||||
"""
|
||||
# Only remap if this looks like an expert parameter
|
||||
if ".experts." not in name:
|
||||
@@ -1582,8 +1587,7 @@ def remap_moe_expert_weights(
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
params_dict: dict[str, torch.nn.Parameter],
|
||||
) -> Generator[tuple[str, torch.Tensor], None, None]:
|
||||
"""
|
||||
Wrapper generator that remaps MoE expert parameter names for backward compatibility.
|
||||
"""Remap MoE expert parameter names for backward compatibility.
|
||||
|
||||
This allows models with custom weight loading to automatically handle both old
|
||||
and new checkpoint formats without needing model-specific remapping code.
|
||||
@@ -1601,6 +1605,7 @@ def remap_moe_expert_weights(
|
||||
|
||||
Yields:
|
||||
(remapped_name, tensor) tuples
|
||||
|
||||
"""
|
||||
for name, weight in weights:
|
||||
remapped_name = maybe_remap_moe_expert_param_name(name, params_dict)
|
||||
|
||||
Reference in New Issue
Block a user