[Offloader] Offload submodules that make_layers never reaches (#53120)

Signed-off-by: ray24777 <103923677+ray24777@users.noreply.github.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
ray24777
2026-08-26 18:37:00 +08:00
committed by GitHub
co-authored by Claude Opus 5 Isotr0py mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 7a9993878c
commit 61d4f56635
5 changed files with 113 additions and 4 deletions
+15
View File
@@ -11,6 +11,7 @@ from collections.abc import (
)
from contextlib import ExitStack, contextmanager, nullcontext
from dataclasses import dataclass
from operator import attrgetter
from typing import (
TYPE_CHECKING,
Any,
@@ -321,7 +322,13 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
If `targets` is set, instead include descendants that are an instance
of `targets`, even if they aren't direct children.
Marked components are also routed through the active offloader (when
it supports tower offloading), since `make_layers` only ever sees the
decoder layer stack.
"""
from vllm.model_executor.offloader import get_offloader
from .utils import StageMissingLayer, collect_children, no_init_weights
if isinstance(modalities, str):
@@ -348,6 +355,14 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
self._tower_model_names = children_names
# Towers are constructed directly, so `make_layers` never routes them
# through the offloader. Do it here, at the same construction stage,
# so offloaded tower weights are never allocated on the device.
offloader = get_offloader()
if offloader.supports_tower_offload:
for name in children_names:
offloader.wrap_modules(iter([attrgetter(name)(self)]), prefix=name)
@contextmanager
def _mark_composite_model(
self,