[RL][Sleep] Retain frozen weights across level-2 sleep (#57891)
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
aoshen02
Codex
parent
c4d424c2e2
commit
639461eda4
@@ -8,6 +8,7 @@ import time
|
||||
from collections.abc import Callable
|
||||
from contextlib import AbstractContextManager, contextmanager, nullcontext
|
||||
from datetime import timedelta
|
||||
from fnmatch import filter as fnmatch_filter
|
||||
from types import NoneType
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
@@ -208,6 +209,7 @@ class Worker(WorkerBase):
|
||||
self.worker_sentinel = WorkerSentinel(worker=self)
|
||||
# Buffers saved before sleep
|
||||
self._sleep_saved_buffers: dict[str, torch.Tensor] = {}
|
||||
self._sleep_saved_parameters: dict[str, torch.Tensor] = {}
|
||||
self._sleep_saved_draft_buffers: dict[str, torch.Tensor] = {}
|
||||
|
||||
# Weight transfer engine is created in `load_model` once the model
|
||||
@@ -243,6 +245,30 @@ class Worker(WorkerBase):
|
||||
)
|
||||
return self._sleep_mode_backend
|
||||
|
||||
def _save_sleep_parameters(self, model: nn.Module) -> None:
|
||||
patterns = self.model_config.sleep_preserve_parameter_names
|
||||
if not patterns:
|
||||
self._sleep_saved_parameters = {}
|
||||
return
|
||||
parameters = dict(model.named_parameters())
|
||||
names: set[str] = set()
|
||||
for pattern in patterns:
|
||||
matches = fnmatch_filter(parameters, pattern)
|
||||
if not matches:
|
||||
raise ValueError(f"No parameter matches sleep retention: {pattern}")
|
||||
names.update(matches)
|
||||
self._sleep_saved_parameters = {
|
||||
name: param.detach().to("cpu")
|
||||
for name, param in parameters.items()
|
||||
if name in names and not param.is_cpu
|
||||
}
|
||||
|
||||
@torch.no_grad()
|
||||
def _restore_sleep_parameters(self, model: nn.Module) -> None:
|
||||
for name, value in self._sleep_saved_parameters.items():
|
||||
model.get_parameter(name).copy_(value)
|
||||
self._sleep_saved_parameters.clear()
|
||||
|
||||
def sleep(self, level: int = 1) -> None:
|
||||
torch.accelerator.synchronize()
|
||||
free_bytes_before_sleep = torch.accelerator.get_memory_info()[0]
|
||||
@@ -250,6 +276,7 @@ class Worker(WorkerBase):
|
||||
# Save the buffers before level 2 sleep
|
||||
if level == 2:
|
||||
model = self.model_runner.model
|
||||
self._save_sleep_parameters(model)
|
||||
self._sleep_saved_buffers = {
|
||||
name: buffer.cpu().clone() for name, buffer in model.named_buffers()
|
||||
}
|
||||
@@ -287,6 +314,8 @@ class Worker(WorkerBase):
|
||||
|
||||
# Restore the buffers after level 2 sleep
|
||||
wake_weights = tags is None or "weights" in tags
|
||||
if wake_weights and self._sleep_saved_parameters:
|
||||
self._restore_sleep_parameters(self.model_runner.model)
|
||||
if wake_weights and len(self._sleep_saved_buffers):
|
||||
model = self.model_runner.model
|
||||
for name, buffer in model.named_buffers():
|
||||
|
||||
Reference in New Issue
Block a user