[Offloader] Offload submodules that make_layers never reaches (#53120)
Signed-off-by: ray24777 <103923677+ray24777@users.noreply.github.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5
Isotr0py
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
7a9993878c
commit
61d4f56635
@@ -11,6 +11,7 @@ from collections.abc import (
|
||||
)
|
||||
from contextlib import ExitStack, contextmanager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
from operator import attrgetter
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
@@ -321,7 +322,13 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
|
||||
If `targets` is set, instead include descendants that are an instance
|
||||
of `targets`, even if they aren't direct children.
|
||||
|
||||
Marked components are also routed through the active offloader (when
|
||||
it supports tower offloading), since `make_layers` only ever sees the
|
||||
decoder layer stack.
|
||||
"""
|
||||
from vllm.model_executor.offloader import get_offloader
|
||||
|
||||
from .utils import StageMissingLayer, collect_children, no_init_weights
|
||||
|
||||
if isinstance(modalities, str):
|
||||
@@ -348,6 +355,14 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
|
||||
self._tower_model_names = children_names
|
||||
|
||||
# Towers are constructed directly, so `make_layers` never routes them
|
||||
# through the offloader. Do it here, at the same construction stage,
|
||||
# so offloaded tower weights are never allocated on the device.
|
||||
offloader = get_offloader()
|
||||
if offloader.supports_tower_offload:
|
||||
for name in children_names:
|
||||
offloader.wrap_modules(iter([attrgetter(name)(self)]), prefix=name)
|
||||
|
||||
@contextmanager
|
||||
def _mark_composite_model(
|
||||
self,
|
||||
|
||||
@@ -51,15 +51,29 @@ class BaseOffloader(ABC):
|
||||
inference. Different strategies trade memory for compute/transfer time.
|
||||
"""
|
||||
|
||||
supports_tower_offload: bool = False
|
||||
"""Whether `wrap_modules` also accepts modules routed by
|
||||
`SupportsMultiModal._mark_tower_model`, outside the `make_layers` call.
|
||||
|
||||
Offloaders whose `wrap_modules` may only be called on the decoder layer
|
||||
stack (e.g. `PrefetchOffloader`, which schedules prefetches over a
|
||||
circular layer stack) must keep this `False`.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def wrap_modules(
|
||||
self,
|
||||
modules_generator: Generator[nn.Module, None, None],
|
||||
prefix: str = "",
|
||||
) -> list[nn.Module]:
|
||||
"""Wrap modules with offloading logic.
|
||||
|
||||
Args:
|
||||
modules_generator: Generator yielding modules to potentially offload.
|
||||
prefix: Name prefix prepended to parameter names before matching
|
||||
them against the offloading parameter set. Used when the
|
||||
modules are not the full model, so that name segments stay
|
||||
fully qualified (e.g. `visual` for a tower module).
|
||||
|
||||
Returns:
|
||||
List of modules, potentially with offloading hooks installed.
|
||||
@@ -99,6 +113,7 @@ class NoopOffloader(BaseOffloader):
|
||||
def wrap_modules(
|
||||
self,
|
||||
modules_generator: Generator[nn.Module, None, None],
|
||||
prefix: str = "",
|
||||
) -> list[nn.Module]:
|
||||
"""Return modules unchanged."""
|
||||
return list(modules_generator)
|
||||
|
||||
@@ -163,12 +163,16 @@ class PrefetchOffloader(BaseOffloader):
|
||||
def wrap_modules(
|
||||
self,
|
||||
modules_generator: Generator[nn.Module, None, None],
|
||||
prefix: str = "",
|
||||
) -> list[nn.Module]:
|
||||
"""Wrap modules with prefetch offloading logic."""
|
||||
assert len(self.module_offloaders) == 0, (
|
||||
"wrap_modules should only be called once"
|
||||
)
|
||||
|
||||
if prefix:
|
||||
prefix = f"{prefix}."
|
||||
|
||||
all_modules = []
|
||||
offload_modules = []
|
||||
|
||||
@@ -182,7 +186,9 @@ class PrefetchOffloader(BaseOffloader):
|
||||
whitelist = [
|
||||
name
|
||||
for name, _ in module.named_parameters()
|
||||
if any(f".{p}." in f".{name}." for p in self.offload_params)
|
||||
if any(
|
||||
f".{p}." in f".{prefix}{name}." for p in self.offload_params
|
||||
)
|
||||
]
|
||||
else:
|
||||
whitelist = [name for name, _ in module.named_parameters()]
|
||||
|
||||
@@ -34,6 +34,8 @@ class UVAOffloader(BaseOffloader):
|
||||
offload. If empty, all parameters are eligible up to the byte limit.
|
||||
"""
|
||||
|
||||
supports_tower_offload = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
cpu_offload_max_bytes: int,
|
||||
@@ -51,9 +53,14 @@ class UVAOffloader(BaseOffloader):
|
||||
def wrap_modules(
|
||||
self,
|
||||
modules_generator: Generator[nn.Module, None, None],
|
||||
prefix: str = "",
|
||||
) -> list[nn.Module]:
|
||||
"""Wrap modules with UVA offloading."""
|
||||
modules = [self._maybe_offload_to_cpu(module) for module in modules_generator]
|
||||
if prefix:
|
||||
prefix = f"{prefix}."
|
||||
modules = [
|
||||
self._maybe_offload_to_cpu(module, prefix) for module in modules_generator
|
||||
]
|
||||
if self.cpu_offload_bytes > 0:
|
||||
logger.info(
|
||||
"Total CPU offloaded parameters: %s",
|
||||
@@ -61,7 +68,7 @@ class UVAOffloader(BaseOffloader):
|
||||
)
|
||||
return modules
|
||||
|
||||
def _maybe_offload_to_cpu(self, module: nn.Module) -> nn.Module:
|
||||
def _maybe_offload_to_cpu(self, module: nn.Module, prefix: str = "") -> nn.Module:
|
||||
"""Offload module parameters to CPU using UVA if budget allows."""
|
||||
if (params := next(module.parameters(), None)) is None:
|
||||
return module
|
||||
@@ -83,13 +90,20 @@ class UVAOffloader(BaseOffloader):
|
||||
# one module might have some parameters offloaded and some not
|
||||
break
|
||||
|
||||
# Skip parameters an earlier wrap_modules call already offloaded.
|
||||
# The UVA path leaves p.device as the accelerator (a view of CPU
|
||||
# memory), so the marker is the only way to recognize those.
|
||||
if p.device.type == "cpu" or getattr(p, "_vllm_is_uva_offloaded", False):
|
||||
continue
|
||||
|
||||
if self.cpu_offload_params:
|
||||
# Check if parameter belongs to the offloading set
|
||||
# Add dots here to ensure we match full segments only
|
||||
# e.g., "experts.w2_weight" matches "mlp.experts.w2_weight"
|
||||
# but not "mlp.experts.w2_weight_scale"
|
||||
should_offload = any(
|
||||
f".{param}." in f".{name}." for param in self.cpu_offload_params
|
||||
f".{param}." in f".{prefix}{name}."
|
||||
for param in self.cpu_offload_params
|
||||
)
|
||||
if not should_offload:
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user