[Docs] [QeRL] Layerwise Reloading Documentation (#40317)

Signed-off-by: Kyle Sayers <kylesayrs@gmail.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Kyle Sayers
2026-04-28 21:06:38 -07:00
committed by GitHub
co-authored by Flora Feng
parent 7fd05e05ae
commit a085b5257d
6 changed files with 214 additions and 6 deletions
@@ -3,7 +3,7 @@
import inspect
from collections.abc import Callable
from functools import wraps
from weakref import WeakKeyDictionary
from weakref import WeakKeyDictionary, WeakSet
import torch
@@ -21,7 +21,13 @@ from .meta import (
restore_layer_on_meta,
)
from .types import LayerReloadingInfo
from .utils import get_layer_params_buffers, get_layer_size, get_layer_tensors
from .utils import (
get_info_size,
get_layer_params_buffers,
get_layer_size,
get_layer_tensors,
has_device_tensors,
)
logger = init_logger(__name__)
@@ -43,6 +49,9 @@ LAYERWISE_INFO: WeakKeyDictionary[torch.nn.Module, LayerReloadingInfo] = (
WeakKeyDictionary()
)
# Global set used to track loading for logging purposes only
LOADING_LAYERS: WeakSet[torch.nn.Module] = WeakSet()
def get_layerwise_info(layer: torch.nn.Module) -> LayerReloadingInfo:
"""
@@ -174,11 +183,30 @@ def make_online_process_loader(layer: torch.nn.Module, param_name: str) -> Calla
info.load_numel_total,
)
# Do not online process attention layers, must wait until finalize
if isinstance(layer, (Attention, MLAAttention)):
return ret
# Log warnings allocating excessive buffers on device
if has_device_tensors(bound_args):
LOADING_LAYERS.add(layer)
if len(LOADING_LAYERS) >= 2:
names = sorted([layer.__class__.__name__ for layer in LOADING_LAYERS])
mem_used = sum(
get_info_size(LAYERWISE_INFO[layer]) for layer in LOADING_LAYERS
)
logger.warning_once(
"Allocating %.1f MB of device memory to buffers to load %s layers. "
"This extra memory usage can be avoided by ordering weights "
"by their parent layer when reloading.",
mem_used / 1e6,
str(list(names)),
)
# Process and copy when all weights are loaded
if info.load_numel >= info.load_numel_total and not isinstance( # type: ignore[operator]
layer, (Attention, MLAAttention)
):
if info.load_numel >= info.load_numel_total: # type: ignore[operator]
_layerwise_process(layer, info)
LOADING_LAYERS.discard(layer)
return ret
@@ -240,6 +268,8 @@ def finalize_layerwise_processing(model: torch.nn.Module, model_config: ModelCon
_finalize_attention_layer(layer, info, model_config)
info.reset()
LOADING_LAYERS.clear()
def finalize_layerwise_reload(*args, **kwargs):
finalize_layerwise_processing(*args, **kwargs)
@@ -1,14 +1,18 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from inspect import BoundArguments
import torch
from .types import LayerTensors
from .types import LayerReloadingInfo, LayerTensors
__all__ = [
"get_layer_tensors",
"get_layer_params_buffers",
"get_layer_size",
"has_device_tensors",
"get_info_size",
]
@@ -39,3 +43,31 @@ def get_layer_size(layer: torch.nn.Module) -> int:
for name, tensor in get_layer_tensors(layer).items()
if name not in SKIP_TENSORS
)
def has_device_tensors(bound_args: BoundArguments) -> bool:
"""
Return True if the loaded weights exist on an accelerator device
:param bound_args: args to load weights
:return: True if weights are on accelerator device
"""
return any(
isinstance(value, torch.Tensor) and value.device.type not in ("meta", "cpu")
for value in bound_args.arguments.values()
)
def get_info_size(info: LayerReloadingInfo) -> int:
"""
Calculate the number of bytes used by loaded weights for a given layer
:param info: layerwise info to get size of
:return: number of bytes used by loaded weights
"""
return sum(
value.nbytes
for _, args in info.loaded_weights
for value in args.arguments.values()
if isinstance(value, torch.Tensor) and value.device.type not in ("meta", "cpu")
)