[Model] Extend collect_children and no_init_weights contexts (#32757)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-01-22 08:20:27 +00:00
committed by GitHub
parent 1bf1a34b19
commit 2b8a38b6d6
20 changed files with 444 additions and 257 deletions
+110 -13
View File
@@ -2,13 +2,15 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import itertools
from collections.abc import Iterable, Mapping
from collections.abc import Callable, Iterable, Mapping
from contextlib import contextmanager
from dataclasses import dataclass, field
from typing import Any, Literal, Protocol, overload
import torch
import torch.nn as nn
from torch.func import functional_call
from torch.nn.modules.module import register_module_module_registration_hook
from transformers import PretrainedConfig
from vllm.config import VllmConfig
@@ -24,11 +26,7 @@ from vllm.model_executor.model_loader.online_quantization import (
support_quantized_model_reload_from_hp_weights,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.interfaces import (
LMMissingLayer,
TowerMissingLayer,
supports_any_eagle,
)
from vllm.model_executor.models.interfaces import supports_any_eagle
from vllm.multimodal import NestedTensors
from vllm.sequence import IntermediateTensors
from vllm.utils.math_utils import cdiv
@@ -214,8 +212,8 @@ class AutoWeightsLoader:
continue
raise ValueError(
f"Attempted to load nested weight '{weight_qualname}' "
f"into a single parameter '{base_prefix}'"
f"Attempted to load nested weight {weight_qualname!r} "
f"into a single parameter {base_prefix!r}"
)
weight_loader = getattr(param, "weight_loader", default_weight_loader)
@@ -254,7 +252,7 @@ class AutoWeightsLoader:
module: nn.Module,
weights: Iterable[tuple[str, torch.Tensor]],
) -> Iterable[str]:
if isinstance(module, (LMMissingLayer, TowerMissingLayer, PPMissingLayer)):
if isinstance(module, (StageMissingLayer, PPMissingLayer)):
return
# Avoid infinite recursion since this function is typically
@@ -316,9 +314,14 @@ class AutoWeightsLoader:
continue
desc_param_keys = {
base_prefix + k for k, _ in module.named_parameters(recurse=True)
}
msg = (
f"There is no module or parameter named '{prefix}' "
f"in {type(self.module).__name__}"
f"There is no module or parameter named {prefix!r} "
f"in {self.module._get_name()}. "
f"The available parameters belonging to {base_prefix} "
f"({module._get_name()}) are: {desc_param_keys}"
)
raise ValueError(msg)
@@ -496,6 +499,100 @@ def isin_list(
return torch.isin(elements, test_elements)
class StageMissingLayer(nn.Module):
def __init__(self, stage_name: str, module: nn.Module | None = None) -> None:
super().__init__()
self.stage_name = stage_name
# Don't register this as a child module in order to
# avoid missing keys when loading weights
self.__dict__["module"] = module
def __getattr__(self, name: str):
return getattr(self.__dict__["module"], name)
def __call__(self, *args, **kwargs):
raise RuntimeError(f"{self} should not be called")
def extra_repr(self) -> str:
return f"stage_name={self.stage_name!r}"
@contextmanager
def collect_children(
module: nn.Module,
*,
targets: type[nn.Module] | tuple[type[nn.Module], ...] | None = None,
):
"""
Within this context, collect all direct child assignments to `module`,
returning a list of children names that is internally updated until the
context is exited.
If `targets` is set, instead collect descendents of `module`
that are an instance of `targets`, even if they aren't direct children.
"""
children_names = list[str]()
if targets is None:
def hook(module_: nn.Module, name: str, submodule: nn.Module):
if module_ is module:
children_names.append(name)
with register_module_module_registration_hook(hook):
yield children_names
else:
yield children_names
for name, module_ in module.named_modules():
if isinstance(module_, targets):
children_names.append(name)
@contextmanager
def no_init_weights(
module: nn.Module,
placeholder: Callable[[nn.Module], nn.Module],
*,
targets: type[nn.Module] | tuple[type[nn.Module], ...] | None = None,
):
"""
Within this context, prevent weight initialization from using device memory and
replace direct child assignments to `module` with the result of `placeholder()`.
If `targets` is set, instead prevent weight initialization and
replace assignments where the child is an instance of `targets`,
even if they aren't direct children of `module`.
"""
if targets is None:
def hook(module_: nn.Module, name: str, submodule: nn.Module):
if module_ is module:
return placeholder(submodule)
return submodule
with register_module_module_registration_hook(hook), torch.device("meta"):
yield
else:
def hook(module_: nn.Module, name: str, submodule: nn.Module):
if isinstance(module_, targets):
submodule.to("meta") # Free memory
if isinstance(submodule, targets):
submodule.to("meta") # Free memory
return placeholder(submodule)
return submodule
# Not all descendents are targeted, so we can't use a blanket
# `torch.device("meta")` context
with register_module_module_registration_hook(hook):
yield
class LayerFn(Protocol):
def __call__(self, prefix: str) -> torch.nn.Module: ...
@@ -627,7 +724,7 @@ def get_pp_missing_layer_names(model: torch.nn.Module) -> list[str]:
missing_layer_names = []
for name, module in model.named_modules():
if isinstance(module, PPMissingLayer):
if isinstance(module, (StageMissingLayer, PPMissingLayer)):
# NOTE: the trailing dot is used to match the prefix of the layer.
# without the dot, we could match a layer that is not missing,
# e.g., 'encoder.layer.1' would match 'encoder.layer.11'
@@ -639,7 +736,7 @@ def get_pp_missing_layer_names(model: torch.nn.Module) -> list[str]:
def is_pp_missing_parameter(name: str, model: torch.nn.Module) -> bool:
"""Check if a parameter is missing in a pipeline parallel model."""
if isinstance(model, PPMissingLayer):
if isinstance(model, (StageMissingLayer, PPMissingLayer)):
return True
return any(