[Docs] [QeRL] Layerwise Reloading Documentation (#40317)

Signed-off-by: Kyle Sayers <kylesayrs@gmail.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Kyle Sayers
2026-04-28 21:06:38 -07:00
committed by GitHub
co-authored by Flora Feng
parent 7fd05e05ae
commit a085b5257d
6 changed files with 214 additions and 6 deletions
@@ -1,14 +1,18 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from inspect import BoundArguments
import torch
from .types import LayerTensors
from .types import LayerReloadingInfo, LayerTensors
__all__ = [
"get_layer_tensors",
"get_layer_params_buffers",
"get_layer_size",
"has_device_tensors",
"get_info_size",
]
@@ -39,3 +43,31 @@ def get_layer_size(layer: torch.nn.Module) -> int:
for name, tensor in get_layer_tensors(layer).items()
if name not in SKIP_TENSORS
)
def has_device_tensors(bound_args: BoundArguments) -> bool:
"""
Return True if the loaded weights exist on an accelerator device
:param bound_args: args to load weights
:return: True if weights are on accelerator device
"""
return any(
isinstance(value, torch.Tensor) and value.device.type not in ("meta", "cpu")
for value in bound_args.arguments.values()
)
def get_info_size(info: LayerReloadingInfo) -> int:
"""
Calculate the number of bytes used by loaded weights for a given layer
:param info: layerwise info to get size of
:return: number of bytes used by loaded weights
"""
return sum(
value.nbytes
for _, args in info.loaded_weights
for value in args.arguments.values()
if isinstance(value, torch.Tensor) and value.device.type not in ("meta", "cpu")
)