[Model] Extend collect_children and no_init_weights contexts (#32757)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -2,13 +2,15 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import itertools
|
||||
from collections.abc import Iterable, Mapping
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Literal, Protocol, overload
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.func import functional_call
|
||||
from torch.nn.modules.module import register_module_module_registration_hook
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
@@ -24,11 +26,7 @@ from vllm.model_executor.model_loader.online_quantization import (
|
||||
support_quantized_model_reload_from_hp_weights,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
LMMissingLayer,
|
||||
TowerMissingLayer,
|
||||
supports_any_eagle,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import supports_any_eagle
|
||||
from vllm.multimodal import NestedTensors
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.math_utils import cdiv
|
||||
@@ -214,8 +212,8 @@ class AutoWeightsLoader:
|
||||
continue
|
||||
|
||||
raise ValueError(
|
||||
f"Attempted to load nested weight '{weight_qualname}' "
|
||||
f"into a single parameter '{base_prefix}'"
|
||||
f"Attempted to load nested weight {weight_qualname!r} "
|
||||
f"into a single parameter {base_prefix!r}"
|
||||
)
|
||||
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
@@ -254,7 +252,7 @@ class AutoWeightsLoader:
|
||||
module: nn.Module,
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> Iterable[str]:
|
||||
if isinstance(module, (LMMissingLayer, TowerMissingLayer, PPMissingLayer)):
|
||||
if isinstance(module, (StageMissingLayer, PPMissingLayer)):
|
||||
return
|
||||
|
||||
# Avoid infinite recursion since this function is typically
|
||||
@@ -316,9 +314,14 @@ class AutoWeightsLoader:
|
||||
|
||||
continue
|
||||
|
||||
desc_param_keys = {
|
||||
base_prefix + k for k, _ in module.named_parameters(recurse=True)
|
||||
}
|
||||
msg = (
|
||||
f"There is no module or parameter named '{prefix}' "
|
||||
f"in {type(self.module).__name__}"
|
||||
f"There is no module or parameter named {prefix!r} "
|
||||
f"in {self.module._get_name()}. "
|
||||
f"The available parameters belonging to {base_prefix} "
|
||||
f"({module._get_name()}) are: {desc_param_keys}"
|
||||
)
|
||||
raise ValueError(msg)
|
||||
|
||||
@@ -496,6 +499,100 @@ def isin_list(
|
||||
return torch.isin(elements, test_elements)
|
||||
|
||||
|
||||
class StageMissingLayer(nn.Module):
|
||||
def __init__(self, stage_name: str, module: nn.Module | None = None) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.stage_name = stage_name
|
||||
|
||||
# Don't register this as a child module in order to
|
||||
# avoid missing keys when loading weights
|
||||
self.__dict__["module"] = module
|
||||
|
||||
def __getattr__(self, name: str):
|
||||
return getattr(self.__dict__["module"], name)
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
raise RuntimeError(f"{self} should not be called")
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
return f"stage_name={self.stage_name!r}"
|
||||
|
||||
|
||||
@contextmanager
|
||||
def collect_children(
|
||||
module: nn.Module,
|
||||
*,
|
||||
targets: type[nn.Module] | tuple[type[nn.Module], ...] | None = None,
|
||||
):
|
||||
"""
|
||||
Within this context, collect all direct child assignments to `module`,
|
||||
returning a list of children names that is internally updated until the
|
||||
context is exited.
|
||||
|
||||
If `targets` is set, instead collect descendents of `module`
|
||||
that are an instance of `targets`, even if they aren't direct children.
|
||||
"""
|
||||
children_names = list[str]()
|
||||
|
||||
if targets is None:
|
||||
|
||||
def hook(module_: nn.Module, name: str, submodule: nn.Module):
|
||||
if module_ is module:
|
||||
children_names.append(name)
|
||||
|
||||
with register_module_module_registration_hook(hook):
|
||||
yield children_names
|
||||
else:
|
||||
yield children_names
|
||||
|
||||
for name, module_ in module.named_modules():
|
||||
if isinstance(module_, targets):
|
||||
children_names.append(name)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def no_init_weights(
|
||||
module: nn.Module,
|
||||
placeholder: Callable[[nn.Module], nn.Module],
|
||||
*,
|
||||
targets: type[nn.Module] | tuple[type[nn.Module], ...] | None = None,
|
||||
):
|
||||
"""
|
||||
Within this context, prevent weight initialization from using device memory and
|
||||
replace direct child assignments to `module` with the result of `placeholder()`.
|
||||
|
||||
If `targets` is set, instead prevent weight initialization and
|
||||
replace assignments where the child is an instance of `targets`,
|
||||
even if they aren't direct children of `module`.
|
||||
"""
|
||||
if targets is None:
|
||||
|
||||
def hook(module_: nn.Module, name: str, submodule: nn.Module):
|
||||
if module_ is module:
|
||||
return placeholder(submodule)
|
||||
|
||||
return submodule
|
||||
|
||||
with register_module_module_registration_hook(hook), torch.device("meta"):
|
||||
yield
|
||||
else:
|
||||
|
||||
def hook(module_: nn.Module, name: str, submodule: nn.Module):
|
||||
if isinstance(module_, targets):
|
||||
submodule.to("meta") # Free memory
|
||||
if isinstance(submodule, targets):
|
||||
submodule.to("meta") # Free memory
|
||||
return placeholder(submodule)
|
||||
|
||||
return submodule
|
||||
|
||||
# Not all descendents are targeted, so we can't use a blanket
|
||||
# `torch.device("meta")` context
|
||||
with register_module_module_registration_hook(hook):
|
||||
yield
|
||||
|
||||
|
||||
class LayerFn(Protocol):
|
||||
def __call__(self, prefix: str) -> torch.nn.Module: ...
|
||||
|
||||
@@ -627,7 +724,7 @@ def get_pp_missing_layer_names(model: torch.nn.Module) -> list[str]:
|
||||
|
||||
missing_layer_names = []
|
||||
for name, module in model.named_modules():
|
||||
if isinstance(module, PPMissingLayer):
|
||||
if isinstance(module, (StageMissingLayer, PPMissingLayer)):
|
||||
# NOTE: the trailing dot is used to match the prefix of the layer.
|
||||
# without the dot, we could match a layer that is not missing,
|
||||
# e.g., 'encoder.layer.1' would match 'encoder.layer.11'
|
||||
@@ -639,7 +736,7 @@ def get_pp_missing_layer_names(model: torch.nn.Module) -> list[str]:
|
||||
|
||||
def is_pp_missing_parameter(name: str, model: torch.nn.Module) -> bool:
|
||||
"""Check if a parameter is missing in a pipeline parallel model."""
|
||||
if isinstance(model, PPMissingLayer):
|
||||
if isinstance(model, (StageMissingLayer, PPMissingLayer)):
|
||||
return True
|
||||
|
||||
return any(
|
||||
|
||||
Reference in New Issue
Block a user