[Docs] [QeRL] Layerwise Reloading Documentation (#40317)
Signed-off-by: Kyle Sayers <kylesayrs@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
co-authored by
Flora Feng
parent
7fd05e05ae
commit
a085b5257d
@@ -1,14 +1,18 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from inspect import BoundArguments
|
||||
|
||||
import torch
|
||||
|
||||
from .types import LayerTensors
|
||||
from .types import LayerReloadingInfo, LayerTensors
|
||||
|
||||
__all__ = [
|
||||
"get_layer_tensors",
|
||||
"get_layer_params_buffers",
|
||||
"get_layer_size",
|
||||
"has_device_tensors",
|
||||
"get_info_size",
|
||||
]
|
||||
|
||||
|
||||
@@ -39,3 +43,31 @@ def get_layer_size(layer: torch.nn.Module) -> int:
|
||||
for name, tensor in get_layer_tensors(layer).items()
|
||||
if name not in SKIP_TENSORS
|
||||
)
|
||||
|
||||
|
||||
def has_device_tensors(bound_args: BoundArguments) -> bool:
|
||||
"""
|
||||
Return True if the loaded weights exist on an accelerator device
|
||||
|
||||
:param bound_args: args to load weights
|
||||
:return: True if weights are on accelerator device
|
||||
"""
|
||||
return any(
|
||||
isinstance(value, torch.Tensor) and value.device.type not in ("meta", "cpu")
|
||||
for value in bound_args.arguments.values()
|
||||
)
|
||||
|
||||
|
||||
def get_info_size(info: LayerReloadingInfo) -> int:
|
||||
"""
|
||||
Calculate the number of bytes used by loaded weights for a given layer
|
||||
|
||||
:param info: layerwise info to get size of
|
||||
:return: number of bytes used by loaded weights
|
||||
"""
|
||||
return sum(
|
||||
value.nbytes
|
||||
for _, args in info.loaded_weights
|
||||
for value in args.arguments.values()
|
||||
if isinstance(value, torch.Tensor) and value.device.type not in ("meta", "cpu")
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user