[RL][Sleep] Retain frozen weights across level-2 sleep (#57891)

Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
aoshen02
2026-09-22 12:36:55 +08:00
committed by GitHub
co-authored by aoshen02 Codex
parent c4d424c2e2
commit 639461eda4
5 changed files with 56 additions and 0 deletions
+29
View File
@@ -8,6 +8,7 @@ import time
from collections.abc import Callable
from contextlib import AbstractContextManager, contextmanager, nullcontext
from datetime import timedelta
from fnmatch import filter as fnmatch_filter
from types import NoneType
from typing import TYPE_CHECKING, Any, cast
@@ -208,6 +209,7 @@ class Worker(WorkerBase):
self.worker_sentinel = WorkerSentinel(worker=self)
# Buffers saved before sleep
self._sleep_saved_buffers: dict[str, torch.Tensor] = {}
self._sleep_saved_parameters: dict[str, torch.Tensor] = {}
self._sleep_saved_draft_buffers: dict[str, torch.Tensor] = {}
# Weight transfer engine is created in `load_model` once the model
@@ -243,6 +245,30 @@ class Worker(WorkerBase):
)
return self._sleep_mode_backend
def _save_sleep_parameters(self, model: nn.Module) -> None:
patterns = self.model_config.sleep_preserve_parameter_names
if not patterns:
self._sleep_saved_parameters = {}
return
parameters = dict(model.named_parameters())
names: set[str] = set()
for pattern in patterns:
matches = fnmatch_filter(parameters, pattern)
if not matches:
raise ValueError(f"No parameter matches sleep retention: {pattern}")
names.update(matches)
self._sleep_saved_parameters = {
name: param.detach().to("cpu")
for name, param in parameters.items()
if name in names and not param.is_cpu
}
@torch.no_grad()
def _restore_sleep_parameters(self, model: nn.Module) -> None:
for name, value in self._sleep_saved_parameters.items():
model.get_parameter(name).copy_(value)
self._sleep_saved_parameters.clear()
def sleep(self, level: int = 1) -> None:
torch.accelerator.synchronize()
free_bytes_before_sleep = torch.accelerator.get_memory_info()[0]
@@ -250,6 +276,7 @@ class Worker(WorkerBase):
# Save the buffers before level 2 sleep
if level == 2:
model = self.model_runner.model
self._save_sleep_parameters(model)
self._sleep_saved_buffers = {
name: buffer.cpu().clone() for name, buffer in model.named_buffers()
}
@@ -287,6 +314,8 @@ class Worker(WorkerBase):
# Restore the buffers after level 2 sleep
wake_weights = tags is None or "weights" in tags
if wake_weights and self._sleep_saved_parameters:
self._restore_sleep_parameters(self.model_runner.model)
if wake_weights and len(self._sleep_saved_buffers):
model = self.model_runner.model
for name, buffer in model.named_buffers():