[Bugfix][DP] Synchronize the device on pause completion (#52914)
Signed-off-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
@@ -259,6 +259,8 @@ class Worker(WorkerBase):
|
||||
if tags is None or "kv_cache" in tags:
|
||||
self.model_runner.post_kv_cache_wake_up()
|
||||
|
||||
self.synchronize_device()
|
||||
|
||||
def checkpoint_prepare(self) -> None:
|
||||
checkpoint_prepare_distributed_state()
|
||||
|
||||
|
||||
@@ -125,6 +125,12 @@ class WorkerBase:
|
||||
"""Basic health check (override for device-specific checks)."""
|
||||
return
|
||||
|
||||
def synchronize_device(self) -> None:
|
||||
"""Block until in-flight device work completes; backends outside
|
||||
``torch.accelerator`` must override with their own wait."""
|
||||
if torch.accelerator.is_available():
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
def init_device(self) -> None:
|
||||
"""Initialize device state, such as loading the model or other on-device
|
||||
memory allocations.
|
||||
|
||||
Reference in New Issue
Block a user