Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Layerwise weight reloading utilities for vLLM.
|
||||
"""Layerwise weight reloading utilities for vLLM.
|
||||
|
||||
This module provides functionality to reload model weights layer-by-layer,
|
||||
which is useful for weight updates without full model reconstruction.
|
||||
|
||||
@@ -54,8 +54,7 @@ LOADING_LAYERS: WeakSet[torch.nn.Module] = WeakSet()
|
||||
|
||||
|
||||
def get_layerwise_info(layer: torch.nn.Module) -> LayerReloadingInfo:
|
||||
"""
|
||||
Get information related to restoring and layerwise processing. If no previous
|
||||
"""Get information related to restoring and layerwise processing. If no previous
|
||||
information existed, a new entry is constructed
|
||||
"""
|
||||
if layer not in LAYERWISE_INFO:
|
||||
@@ -68,8 +67,7 @@ def get_layerwise_info(layer: torch.nn.Module) -> LayerReloadingInfo:
|
||||
|
||||
|
||||
def record_metadata_for_reloading(model: torch.nn.Module):
|
||||
"""
|
||||
Record layer metadata needed for later reloading.
|
||||
"""Record layer metadata needed for later reloading.
|
||||
|
||||
Stores parameter and buffer metadata as meta tensors for restoration.
|
||||
Must be called before `initialize_layerwise_reload`.
|
||||
@@ -82,8 +80,7 @@ def record_metadata_for_reloading(model: torch.nn.Module):
|
||||
|
||||
@torch.no_grad()
|
||||
def initialize_layerwise_reload(model: torch.nn.Module):
|
||||
"""
|
||||
Set up layerwise weight loading with deferred processing.
|
||||
"""Set up layerwise weight loading with deferred processing.
|
||||
|
||||
Must be called after `record_metadata_for_reloading`. This function:
|
||||
1. Saves current kernel tensors for later copying
|
||||
@@ -120,13 +117,13 @@ def initialize_layerwise_reload(model: torch.nn.Module):
|
||||
|
||||
|
||||
def initialize_online_processing(layer: torch.nn.Module):
|
||||
"""
|
||||
Wrap a layer's weight loaders with online processing loaders.
|
||||
"""Wrap a layer's weight loaders with online processing loaders.
|
||||
Called by either `initialize_layerwise_reload` or an online quantization scheme,
|
||||
prevents double wrapping in the case of online quantization + reloading
|
||||
|
||||
Args:
|
||||
layer: layer whose parameter weight loaders will be wrapped
|
||||
|
||||
"""
|
||||
info = get_layerwise_info(layer)
|
||||
|
||||
@@ -226,8 +223,7 @@ def make_online_process_loader(layer: torch.nn.Module, param_name: str) -> Calla
|
||||
|
||||
|
||||
def finalize_layerwise_processing(model: torch.nn.Module, model_config: ModelConfig):
|
||||
"""
|
||||
Apply processing to any layers which were not layerwise processed during loading.
|
||||
"""Apply processing to any layers which were not layerwise processed during loading.
|
||||
This includes attention layers and layers which have weight elements which are not
|
||||
loaded (due to padding).
|
||||
|
||||
@@ -237,6 +233,7 @@ def finalize_layerwise_processing(model: torch.nn.Module, model_config: ModelCon
|
||||
Args:
|
||||
model: model to finalize processing for
|
||||
model_config: config needed for applying processing to attention layers
|
||||
|
||||
"""
|
||||
if hasattr(model, "_original_do_torchao_reload"):
|
||||
model._do_torchao_reload = model._original_do_torchao_reload
|
||||
@@ -311,7 +308,8 @@ def _reload_attention_scales(layer: torch.nn.Module, info: LayerReloadingInfo) -
|
||||
|
||||
Assumes dtype/shapes of attention tensors do not change during
|
||||
processing, since we use .data.copy_() to preserve kernel tensor
|
||||
references."""
|
||||
references.
|
||||
"""
|
||||
quant_method = getattr(layer, "quant_method", None)
|
||||
if quant_method is not None:
|
||||
# Re-create scale Parameters with sentinel values so unloaded scales
|
||||
@@ -330,8 +328,7 @@ def _reload_attention_scales(layer: torch.nn.Module, info: LayerReloadingInfo) -
|
||||
|
||||
|
||||
def _layerwise_process(layer: torch.nn.Module, info: LayerReloadingInfo):
|
||||
"""
|
||||
Finalize layer loading after all weights have been buffered.
|
||||
"""Finalize layer loading after all weights have been buffered.
|
||||
|
||||
This function:
|
||||
1. Materializes the layer onto the target device
|
||||
@@ -377,7 +374,7 @@ def _layerwise_process(layer: torch.nn.Module, info: LayerReloadingInfo):
|
||||
|
||||
|
||||
def _get_original_loader(tensor: torch.Tensor) -> Callable:
|
||||
"""Return the weight loader with any layerwise wrappers removed"""
|
||||
"""Return the weight loader with any layerwise wrappers removed."""
|
||||
loader = _get_weight_loader(tensor)
|
||||
while loader.__name__ == "online_process_loader":
|
||||
loader = loader.__wrapped__ # type: ignore[union-attr]
|
||||
@@ -391,7 +388,8 @@ def _get_weight_loader(tensor: torch.Tensor):
|
||||
|
||||
def _copy_and_restore_kernel_tensors(layer: torch.nn.Module, info: LayerReloadingInfo):
|
||||
"""Copy processed values into original kernel tensor storage and restore
|
||||
kernel tensor references on the layer. Preserves cudagraph references."""
|
||||
kernel tensor references on the layer. Preserves cudagraph references.
|
||||
"""
|
||||
assert info.kernel_tensors is not None
|
||||
parameters, buffers = info.kernel_tensors
|
||||
non_persistent = info.kernel_non_persistent_buffers
|
||||
|
||||
@@ -47,8 +47,7 @@ def to_meta_tensor(tensor: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
|
||||
def materialize_meta_tensor(meta_tensor: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Materialize a meta tensor into an actual tensor on the current device.
|
||||
"""Materialize a meta tensor into an actual tensor on the current device.
|
||||
Should be called within the torch device context for the given rank.
|
||||
"""
|
||||
tensor = torch.empty_strided(
|
||||
@@ -119,7 +118,7 @@ def capture_layer_to_meta(layer: torch.nn.Module) -> LayerTensors:
|
||||
|
||||
|
||||
def restore_layer_on_meta(layer: torch.nn.Module, info: LayerReloadingInfo):
|
||||
"""Restore a layer to model format with tensors on the meta device"""
|
||||
"""Restore a layer to model format with tensors on the meta device."""
|
||||
if layer.__class__.__name__ in SKIP_MODULES:
|
||||
return
|
||||
|
||||
@@ -155,8 +154,7 @@ def materialize_layer(layer: torch.nn.Module, info: LayerReloadingInfo):
|
||||
|
||||
|
||||
class CopyCounter(TorchDispatchMode):
|
||||
"""
|
||||
Tracks total number of elements modified with `copy_`.
|
||||
"""Tracks total number of elements modified with `copy_`.
|
||||
|
||||
Useful for keeping track of weight loading where underlying weights can be
|
||||
arbitrarily transformed (such as with `narrow`) before calling copy.
|
||||
@@ -182,8 +180,7 @@ class CopyCounter(TorchDispatchMode):
|
||||
def get_numel_loaded(
|
||||
weight_loader: Callable, args: inspect.BoundArguments
|
||||
) -> tuple[int, object]:
|
||||
"""
|
||||
Determine how many elements would be loaded by a weight loader call.
|
||||
"""Determine how many elements would be loaded by a weight loader call.
|
||||
|
||||
Args:
|
||||
weight_loader: used to load weights
|
||||
@@ -192,6 +189,7 @@ def get_numel_loaded(
|
||||
Returns:
|
||||
number of elements loaded by the weight loader, the return value of the
|
||||
weight loader
|
||||
|
||||
"""
|
||||
with CopyCounter() as counter:
|
||||
return_value = weight_loader(*args.args, **args.kwargs)
|
||||
|
||||
@@ -11,8 +11,7 @@ layer_ref_sentinel = object()
|
||||
|
||||
|
||||
def sanitize_layer_refs(tensor: torch.Tensor, layer: torch.nn.Module) -> torch.Tensor:
|
||||
"""
|
||||
Removes references to layer held by tensor attributes. Specifically, removes the
|
||||
"""Removes references to layer held by tensor attributes. Specifically, removes the
|
||||
`__self__` attribute of weight loader methods attached to the tensor.
|
||||
|
||||
Used by `capture_layer_to_meta` to avoid circular references to layers in
|
||||
@@ -26,6 +25,7 @@ def sanitize_layer_refs(tensor: torch.Tensor, layer: torch.nn.Module) -> torch.T
|
||||
|
||||
Returns:
|
||||
sanitized tensor
|
||||
|
||||
"""
|
||||
for key, value in tensor.__dict__.items():
|
||||
if isinstance(value, MethodType) and value.__self__ is layer:
|
||||
@@ -35,8 +35,7 @@ def sanitize_layer_refs(tensor: torch.Tensor, layer: torch.nn.Module) -> torch.T
|
||||
|
||||
|
||||
def restore_layer_refs(tensor: torch.Tensor, layer: torch.nn.Module) -> torch.Tensor:
|
||||
"""
|
||||
Restores references to layer held by tensor attributes.
|
||||
"""Restores references to layer held by tensor attributes.
|
||||
|
||||
Used by `restore_layer_on_meta` to add back layer references, allowing for proper
|
||||
weight loading.
|
||||
@@ -47,6 +46,7 @@ def restore_layer_refs(tensor: torch.Tensor, layer: torch.nn.Module) -> torch.Te
|
||||
|
||||
Returns:
|
||||
sanitized tensor
|
||||
|
||||
"""
|
||||
for key, value in tensor.__dict__.items():
|
||||
if isinstance(value, MethodType) and value.__self__ is layer_ref_sentinel:
|
||||
|
||||
@@ -27,8 +27,7 @@ def set_torchao_reload_attrs(model: torch.nn.Module, model_config: ModelConfig):
|
||||
|
||||
|
||||
def support_quantized_model_reload_from_hp_weights(original_load_weights: FunctionType):
|
||||
"""
|
||||
Decorator for `load_weights` method for AutoWeightsLoader.load_weights to support
|
||||
"""Decorator for `load_weights` method for AutoWeightsLoader.load_weights to support
|
||||
reloading high precision (bfloat16/float16/float32) weight for an already quantized
|
||||
model, this involves restoring the weights to a high precision weights and
|
||||
then online quantize the weights.
|
||||
|
||||
@@ -46,14 +46,14 @@ def get_layer_size(layer: torch.nn.Module) -> int:
|
||||
|
||||
|
||||
def has_device_tensors(bound_args: BoundArguments) -> bool:
|
||||
"""
|
||||
Return True if the loaded weights exist on an accelerator device
|
||||
"""Return True if the loaded weights exist on an accelerator device.
|
||||
|
||||
Args:
|
||||
bound_args: args to load weights
|
||||
|
||||
Returns:
|
||||
True if weights are on accelerator device
|
||||
|
||||
"""
|
||||
return any(
|
||||
isinstance(value, torch.Tensor) and value.device.type not in ("meta", "cpu")
|
||||
@@ -62,14 +62,14 @@ def has_device_tensors(bound_args: BoundArguments) -> bool:
|
||||
|
||||
|
||||
def get_info_size(info: LayerReloadingInfo) -> int:
|
||||
"""
|
||||
Calculate the number of bytes used by loaded weights for a given layer
|
||||
"""Calculate the number of bytes used by loaded weights for a given layer.
|
||||
|
||||
Args:
|
||||
info: layerwise info to get size of
|
||||
|
||||
Returns:
|
||||
number of bytes used by loaded weights
|
||||
|
||||
"""
|
||||
return sum(
|
||||
value.nbytes
|
||||
|
||||
Reference in New Issue
Block a user