[Offloader] Offload submodules that make_layers never reaches (#53120)

Signed-off-by: ray24777 <103923677+ray24777@users.noreply.github.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
ray24777
2026-08-26 18:37:00 +08:00
committed by GitHub
co-authored by Claude Opus 5 Isotr0py mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 7a9993878c
commit 61d4f56635
5 changed files with 113 additions and 4 deletions
+15
View File
@@ -51,15 +51,29 @@ class BaseOffloader(ABC):
inference. Different strategies trade memory for compute/transfer time.
"""
supports_tower_offload: bool = False
"""Whether `wrap_modules` also accepts modules routed by
`SupportsMultiModal._mark_tower_model`, outside the `make_layers` call.
Offloaders whose `wrap_modules` may only be called on the decoder layer
stack (e.g. `PrefetchOffloader`, which schedules prefetches over a
circular layer stack) must keep this `False`.
"""
@abstractmethod
def wrap_modules(
self,
modules_generator: Generator[nn.Module, None, None],
prefix: str = "",
) -> list[nn.Module]:
"""Wrap modules with offloading logic.
Args:
modules_generator: Generator yielding modules to potentially offload.
prefix: Name prefix prepended to parameter names before matching
them against the offloading parameter set. Used when the
modules are not the full model, so that name segments stay
fully qualified (e.g. `visual` for a tower module).
Returns:
List of modules, potentially with offloading hooks installed.
@@ -99,6 +113,7 @@ class NoopOffloader(BaseOffloader):
def wrap_modules(
self,
modules_generator: Generator[nn.Module, None, None],
prefix: str = "",
) -> list[nn.Module]:
"""Return modules unchanged."""
return list(modules_generator)
+7 -1
View File
@@ -163,12 +163,16 @@ class PrefetchOffloader(BaseOffloader):
def wrap_modules(
self,
modules_generator: Generator[nn.Module, None, None],
prefix: str = "",
) -> list[nn.Module]:
"""Wrap modules with prefetch offloading logic."""
assert len(self.module_offloaders) == 0, (
"wrap_modules should only be called once"
)
if prefix:
prefix = f"{prefix}."
all_modules = []
offload_modules = []
@@ -182,7 +186,9 @@ class PrefetchOffloader(BaseOffloader):
whitelist = [
name
for name, _ in module.named_parameters()
if any(f".{p}." in f".{name}." for p in self.offload_params)
if any(
f".{p}." in f".{prefix}{name}." for p in self.offload_params
)
]
else:
whitelist = [name for name, _ in module.named_parameters()]
+17 -3
View File
@@ -34,6 +34,8 @@ class UVAOffloader(BaseOffloader):
offload. If empty, all parameters are eligible up to the byte limit.
"""
supports_tower_offload = True
def __init__(
self,
cpu_offload_max_bytes: int,
@@ -51,9 +53,14 @@ class UVAOffloader(BaseOffloader):
def wrap_modules(
self,
modules_generator: Generator[nn.Module, None, None],
prefix: str = "",
) -> list[nn.Module]:
"""Wrap modules with UVA offloading."""
modules = [self._maybe_offload_to_cpu(module) for module in modules_generator]
if prefix:
prefix = f"{prefix}."
modules = [
self._maybe_offload_to_cpu(module, prefix) for module in modules_generator
]
if self.cpu_offload_bytes > 0:
logger.info(
"Total CPU offloaded parameters: %s",
@@ -61,7 +68,7 @@ class UVAOffloader(BaseOffloader):
)
return modules
def _maybe_offload_to_cpu(self, module: nn.Module) -> nn.Module:
def _maybe_offload_to_cpu(self, module: nn.Module, prefix: str = "") -> nn.Module:
"""Offload module parameters to CPU using UVA if budget allows."""
if (params := next(module.parameters(), None)) is None:
return module
@@ -83,13 +90,20 @@ class UVAOffloader(BaseOffloader):
# one module might have some parameters offloaded and some not
break
# Skip parameters an earlier wrap_modules call already offloaded.
# The UVA path leaves p.device as the accelerator (a view of CPU
# memory), so the marker is the only way to recognize those.
if p.device.type == "cpu" or getattr(p, "_vllm_is_uva_offloaded", False):
continue
if self.cpu_offload_params:
# Check if parameter belongs to the offloading set
# Add dots here to ensure we match full segments only
# e.g., "experts.w2_weight" matches "mlp.experts.w2_weight"
# but not "mlp.experts.w2_weight_scale"
should_offload = any(
f".{param}." in f".{name}." for param in self.cpu_offload_params
f".{param}." in f".{prefix}{name}."
for param in self.cpu_offload_params
)
if not should_offload:
continue