[Model] Extend collect_children and no_init_weights contexts (#32757)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-01-22 08:20:27 +00:00
committed by GitHub
parent 1bf1a34b19
commit 2b8a38b6d6
20 changed files with 444 additions and 257 deletions
+106 -65
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable, Iterable, Mapping, MutableSequence
from contextlib import contextmanager, nullcontext
from contextlib import ExitStack, contextmanager, nullcontext
from typing import (
TYPE_CHECKING,
ClassVar,
@@ -25,6 +25,7 @@ from vllm.inputs import TokensPrompt
from vllm.inputs.data import PromptType
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.utils.collection_utils import common_prefix
from vllm.utils.func_utils import supports_kw
from .interfaces_base import VllmModel, is_pooling_model
@@ -70,46 +71,8 @@ def _require_is_multimodal(is_multimodal: Tensor | None) -> Tensor:
return is_multimodal
class LMMissingLayer(nn.Module):
def make_empty_intermediate_tensors(self, *args, **kwargs):
raise RuntimeError("This module should not be called in MM encoder-only mode")
def __call__(self, *args, **kwargs):
raise RuntimeError("This module should not be called in MM encoder-only mode")
class TowerMissingLayer(nn.Module):
def __init__(self, modalities: set[str] | str) -> None:
if isinstance(modalities, str):
modalities = {modalities}
super().__init__()
self.modalities = modalities
def __call__(self, *args, **kwargs):
raise RuntimeError(
f"This module should not be called when the following "
f"modalities are disabled: {self.modalities}"
)
@contextmanager
def _no_init_weights(module: nn.Module, placeholder: Callable[[], nn.Module]):
"""
Within this context, prevent weight initialization from using device memory and
replace direct child assignments to `module` with the result of `placeholder()`.
"""
def callback(module_, name, submodule):
if module_ is module:
return placeholder()
return submodule
with torch.nn.modules.module.register_module_module_registration_hook(callback): # noqa: E501,SIM117
with torch.device("meta"):
yield
# Cache results of `SupportsMultiModal.get_language_model`
_language_model_by_module = dict[nn.Module, VllmModel]()
@runtime_checkable
@@ -187,31 +150,61 @@ class SupportsMultiModal(Protocol):
Returns:
torch.nn.Module: The core language model component.
"""
# Cached
if self in _language_model_by_module:
return _language_model_by_module[self]
if self._language_model_names:
return getattr(self, self._language_model_names[0])
mod = self
for attr in common_prefix(
[name.split(".") for name in self._language_model_names]
):
if attr:
mod = getattr(mod, attr)
if mod is not self and hasattr(mod, "embed_input_ids"):
_language_model_by_module[self] = mod
return mod
# Fallback
for mod in self.children():
if hasattr(mod, "embed_input_ids"):
_language_model_by_module[self] = mod
return mod
raise NotImplementedError(
f"No language model found in {type(self).__name__}! "
"You should initialize it inside `_mark_language_model`."
"You should initialize it via `_mark_language_model`."
)
@contextmanager
def _mark_language_model(self, vllm_config: VllmConfig):
def _mark_language_model(
self,
vllm_config: VllmConfig,
*,
targets: type[nn.Module] | tuple[type[nn.Module], ...] | None = None,
):
"""
Mark each child module that was assigned to this model
during this context as a language model component.
Mark each child module that was assigned to this model during this context
as a language model component.
Language model components are automatically skipped in `--mm-encoder-only`
mode.
If `targets` is set, instead include descendants that are an instance
of `targets`, even if they aren't direct children.
"""
from .utils import StageMissingLayer, collect_children, no_init_weights
mm_config = vllm_config.model_config.multimodal_config
children_names = list[str]()
def callback(module_, name, submodule):
if module_ is self:
children_names.append(name)
with torch.nn.modules.module.register_module_module_registration_hook(callback): # noqa: E501,SIM117
with collect_children(self, targets=targets) as children_names: # noqa: SIM117
with (
_no_init_weights(self, LMMissingLayer)
no_init_weights(
self,
lambda mod: StageMissingLayer("language_model", mod),
targets=targets,
)
if mm_config.mm_encoder_only
else nullcontext()
):
@@ -220,25 +213,42 @@ class SupportsMultiModal(Protocol):
self._language_model_names = children_names
@contextmanager
def _mark_tower_model(self, vllm_config: VllmConfig, modalities: set[str] | str):
def _mark_tower_model(
self,
vllm_config: VllmConfig,
modalities: set[str] | str,
*,
targets: type[nn.Module] | tuple[type[nn.Module], ...] | None = None,
):
"""
Mark each child module that was assigned to this model
during this context as a tower model component.
Mark each child module that was assigned to this model during this context
as a tower model component.
Tower model components are automatically skipped when `--limit-mm-per-prompt`
is set to zero for all of their modalities.
If `targets` is set, instead include descendants that are an instance
of `targets`, even if they aren't direct children.
"""
from .utils import StageMissingLayer, collect_children, no_init_weights
if isinstance(modalities, str):
modalities = {modalities}
if modalities == {"image", "video"}:
stage_name = "vision_tower"
else:
stage_name = "_".join([*modalities, "tower"])
mm_config = vllm_config.model_config.multimodal_config
children_names = list[str]()
def callback(module_, name, submodule):
if module_ is self:
children_names.append(name)
with torch.nn.modules.module.register_module_module_registration_hook(callback): # noqa: E501,SIM117
with collect_children(self, targets=targets) as children_names: # noqa: SIM117
with (
_no_init_weights(self, lambda: TowerMissingLayer(modalities))
no_init_weights(
self,
lambda mod: StageMissingLayer(stage_name, mod),
targets=targets,
)
if all(mm_config.get_limit_per_prompt(m) == 0 for m in modalities)
else nullcontext()
):
@@ -246,6 +256,37 @@ class SupportsMultiModal(Protocol):
self._tower_model_names = children_names
@contextmanager
def _mark_composite_model(
self,
vllm_config: VllmConfig,
*,
language_targets: type[nn.Module] | tuple[type[nn.Module], ...],
tower_targets: dict[str, type[nn.Module] | tuple[type[nn.Module], ...]],
):
"""
Composite wrapper over `_mark_language_model` and
`_mark_tower_model` by modality.
"""
with ExitStack() as stack:
stack.enter_context(
self._mark_language_model(
vllm_config,
targets=language_targets,
)
)
for modality, modality_targets in tower_targets.items():
stack.enter_context(
self._mark_tower_model(
vllm_config,
modality,
targets=modality_targets,
)
)
yield
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
"""
Implement this function to enable LoRA support