[Docs] [QeRL] Layerwise Reloading Documentation (#40317)
Signed-off-by: Kyle Sayers <kylesayrs@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
co-authored by
Flora Feng
parent
7fd05e05ae
commit
a085b5257d
@@ -3,7 +3,7 @@
|
||||
import inspect
|
||||
from collections.abc import Callable
|
||||
from functools import wraps
|
||||
from weakref import WeakKeyDictionary
|
||||
from weakref import WeakKeyDictionary, WeakSet
|
||||
|
||||
import torch
|
||||
|
||||
@@ -21,7 +21,13 @@ from .meta import (
|
||||
restore_layer_on_meta,
|
||||
)
|
||||
from .types import LayerReloadingInfo
|
||||
from .utils import get_layer_params_buffers, get_layer_size, get_layer_tensors
|
||||
from .utils import (
|
||||
get_info_size,
|
||||
get_layer_params_buffers,
|
||||
get_layer_size,
|
||||
get_layer_tensors,
|
||||
has_device_tensors,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -43,6 +49,9 @@ LAYERWISE_INFO: WeakKeyDictionary[torch.nn.Module, LayerReloadingInfo] = (
|
||||
WeakKeyDictionary()
|
||||
)
|
||||
|
||||
# Global set used to track loading for logging purposes only
|
||||
LOADING_LAYERS: WeakSet[torch.nn.Module] = WeakSet()
|
||||
|
||||
|
||||
def get_layerwise_info(layer: torch.nn.Module) -> LayerReloadingInfo:
|
||||
"""
|
||||
@@ -174,11 +183,30 @@ def make_online_process_loader(layer: torch.nn.Module, param_name: str) -> Calla
|
||||
info.load_numel_total,
|
||||
)
|
||||
|
||||
# Do not online process attention layers, must wait until finalize
|
||||
if isinstance(layer, (Attention, MLAAttention)):
|
||||
return ret
|
||||
|
||||
# Log warnings allocating excessive buffers on device
|
||||
if has_device_tensors(bound_args):
|
||||
LOADING_LAYERS.add(layer)
|
||||
if len(LOADING_LAYERS) >= 2:
|
||||
names = sorted([layer.__class__.__name__ for layer in LOADING_LAYERS])
|
||||
mem_used = sum(
|
||||
get_info_size(LAYERWISE_INFO[layer]) for layer in LOADING_LAYERS
|
||||
)
|
||||
logger.warning_once(
|
||||
"Allocating %.1f MB of device memory to buffers to load %s layers. "
|
||||
"This extra memory usage can be avoided by ordering weights "
|
||||
"by their parent layer when reloading.",
|
||||
mem_used / 1e6,
|
||||
str(list(names)),
|
||||
)
|
||||
|
||||
# Process and copy when all weights are loaded
|
||||
if info.load_numel >= info.load_numel_total and not isinstance( # type: ignore[operator]
|
||||
layer, (Attention, MLAAttention)
|
||||
):
|
||||
if info.load_numel >= info.load_numel_total: # type: ignore[operator]
|
||||
_layerwise_process(layer, info)
|
||||
LOADING_LAYERS.discard(layer)
|
||||
|
||||
return ret
|
||||
|
||||
@@ -240,6 +268,8 @@ def finalize_layerwise_processing(model: torch.nn.Module, model_config: ModelCon
|
||||
_finalize_attention_layer(layer, info, model_config)
|
||||
info.reset()
|
||||
|
||||
LOADING_LAYERS.clear()
|
||||
|
||||
|
||||
def finalize_layerwise_reload(*args, **kwargs):
|
||||
finalize_layerwise_processing(*args, **kwargs)
|
||||
|
||||
@@ -1,14 +1,18 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from inspect import BoundArguments
|
||||
|
||||
import torch
|
||||
|
||||
from .types import LayerTensors
|
||||
from .types import LayerReloadingInfo, LayerTensors
|
||||
|
||||
__all__ = [
|
||||
"get_layer_tensors",
|
||||
"get_layer_params_buffers",
|
||||
"get_layer_size",
|
||||
"has_device_tensors",
|
||||
"get_info_size",
|
||||
]
|
||||
|
||||
|
||||
@@ -39,3 +43,31 @@ def get_layer_size(layer: torch.nn.Module) -> int:
|
||||
for name, tensor in get_layer_tensors(layer).items()
|
||||
if name not in SKIP_TENSORS
|
||||
)
|
||||
|
||||
|
||||
def has_device_tensors(bound_args: BoundArguments) -> bool:
|
||||
"""
|
||||
Return True if the loaded weights exist on an accelerator device
|
||||
|
||||
:param bound_args: args to load weights
|
||||
:return: True if weights are on accelerator device
|
||||
"""
|
||||
return any(
|
||||
isinstance(value, torch.Tensor) and value.device.type not in ("meta", "cpu")
|
||||
for value in bound_args.arguments.values()
|
||||
)
|
||||
|
||||
|
||||
def get_info_size(info: LayerReloadingInfo) -> int:
|
||||
"""
|
||||
Calculate the number of bytes used by loaded weights for a given layer
|
||||
|
||||
:param info: layerwise info to get size of
|
||||
:return: number of bytes used by loaded weights
|
||||
"""
|
||||
return sum(
|
||||
value.nbytes
|
||||
for _, args in info.loaded_weights
|
||||
for value in args.arguments.values()
|
||||
if isinstance(value, torch.Tensor) and value.device.type not in ("meta", "cpu")
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user