[Offloader] Offload submodules that make_layers never reaches (#53120)

Signed-off-by: ray24777 <103923677+ray24777@users.noreply.github.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
ray24777
2026-08-26 18:37:00 +08:00
committed by GitHub
co-authored by Claude Opus 5 Isotr0py mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 7a9993878c
commit 61d4f56635
5 changed files with 113 additions and 4 deletions
+17 -3
View File
@@ -34,6 +34,8 @@ class UVAOffloader(BaseOffloader):
offload. If empty, all parameters are eligible up to the byte limit.
"""
supports_tower_offload = True
def __init__(
self,
cpu_offload_max_bytes: int,
@@ -51,9 +53,14 @@ class UVAOffloader(BaseOffloader):
def wrap_modules(
self,
modules_generator: Generator[nn.Module, None, None],
prefix: str = "",
) -> list[nn.Module]:
"""Wrap modules with UVA offloading."""
modules = [self._maybe_offload_to_cpu(module) for module in modules_generator]
if prefix:
prefix = f"{prefix}."
modules = [
self._maybe_offload_to_cpu(module, prefix) for module in modules_generator
]
if self.cpu_offload_bytes > 0:
logger.info(
"Total CPU offloaded parameters: %s",
@@ -61,7 +68,7 @@ class UVAOffloader(BaseOffloader):
)
return modules
def _maybe_offload_to_cpu(self, module: nn.Module) -> nn.Module:
def _maybe_offload_to_cpu(self, module: nn.Module, prefix: str = "") -> nn.Module:
"""Offload module parameters to CPU using UVA if budget allows."""
if (params := next(module.parameters(), None)) is None:
return module
@@ -83,13 +90,20 @@ class UVAOffloader(BaseOffloader):
# one module might have some parameters offloaded and some not
break
# Skip parameters an earlier wrap_modules call already offloaded.
# The UVA path leaves p.device as the accelerator (a view of CPU
# memory), so the marker is the only way to recognize those.
if p.device.type == "cpu" or getattr(p, "_vllm_is_uva_offloaded", False):
continue
if self.cpu_offload_params:
# Check if parameter belongs to the offloading set
# Add dots here to ensure we match full segments only
# e.g., "experts.w2_weight" matches "mlp.experts.w2_weight"
# but not "mlp.experts.w2_weight_scale"
should_offload = any(
f".{param}." in f".{name}." for param in self.cpu_offload_params
f".{param}." in f".{prefix}{name}."
for param in self.cpu_offload_params
)
if not should_offload:
continue