[Bugfix][DP] Synchronize the device on pause completion (#52914)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
aoshen02
2026-08-26 09:50:33 -07:00
committed by GitHub
parent 8d301f075b
commit 17da48596c
5 changed files with 164 additions and 8 deletions
+2
View File
@@ -259,6 +259,8 @@ class Worker(WorkerBase):
if tags is None or "kv_cache" in tags:
self.model_runner.post_kv_cache_wake_up()
self.synchronize_device()
def checkpoint_prepare(self) -> None:
checkpoint_prepare_distributed_state()
+6
View File
@@ -125,6 +125,12 @@ class WorkerBase:
"""Basic health check (override for device-specific checks)."""
return
def synchronize_device(self) -> None:
"""Block until in-flight device work completes; backends outside
``torch.accelerator`` must override with their own wait."""
if torch.accelerator.is_available():
torch.accelerator.synchronize()
def init_device(self) -> None:
"""Initialize device state, such as loading the model or other on-device
memory allocations.