[Offloader] Offload submodules that make_layers never reaches (#53120)
Signed-off-by: ray24777 <103923677+ray24777@users.noreply.github.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5
Isotr0py
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
7a9993878c
commit
61d4f56635
@@ -34,6 +34,8 @@ class UVAOffloader(BaseOffloader):
|
||||
offload. If empty, all parameters are eligible up to the byte limit.
|
||||
"""
|
||||
|
||||
supports_tower_offload = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
cpu_offload_max_bytes: int,
|
||||
@@ -51,9 +53,14 @@ class UVAOffloader(BaseOffloader):
|
||||
def wrap_modules(
|
||||
self,
|
||||
modules_generator: Generator[nn.Module, None, None],
|
||||
prefix: str = "",
|
||||
) -> list[nn.Module]:
|
||||
"""Wrap modules with UVA offloading."""
|
||||
modules = [self._maybe_offload_to_cpu(module) for module in modules_generator]
|
||||
if prefix:
|
||||
prefix = f"{prefix}."
|
||||
modules = [
|
||||
self._maybe_offload_to_cpu(module, prefix) for module in modules_generator
|
||||
]
|
||||
if self.cpu_offload_bytes > 0:
|
||||
logger.info(
|
||||
"Total CPU offloaded parameters: %s",
|
||||
@@ -61,7 +68,7 @@ class UVAOffloader(BaseOffloader):
|
||||
)
|
||||
return modules
|
||||
|
||||
def _maybe_offload_to_cpu(self, module: nn.Module) -> nn.Module:
|
||||
def _maybe_offload_to_cpu(self, module: nn.Module, prefix: str = "") -> nn.Module:
|
||||
"""Offload module parameters to CPU using UVA if budget allows."""
|
||||
if (params := next(module.parameters(), None)) is None:
|
||||
return module
|
||||
@@ -83,13 +90,20 @@ class UVAOffloader(BaseOffloader):
|
||||
# one module might have some parameters offloaded and some not
|
||||
break
|
||||
|
||||
# Skip parameters an earlier wrap_modules call already offloaded.
|
||||
# The UVA path leaves p.device as the accelerator (a view of CPU
|
||||
# memory), so the marker is the only way to recognize those.
|
||||
if p.device.type == "cpu" or getattr(p, "_vllm_is_uva_offloaded", False):
|
||||
continue
|
||||
|
||||
if self.cpu_offload_params:
|
||||
# Check if parameter belongs to the offloading set
|
||||
# Add dots here to ensure we match full segments only
|
||||
# e.g., "experts.w2_weight" matches "mlp.experts.w2_weight"
|
||||
# but not "mlp.experts.w2_weight_scale"
|
||||
should_offload = any(
|
||||
f".{param}." in f".{name}." for param in self.cpu_offload_params
|
||||
f".{param}." in f".{prefix}{name}."
|
||||
for param in self.cpu_offload_params
|
||||
)
|
||||
if not should_offload:
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user