[MRV2] Release weight offloader on shutdown (#57834)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -129,14 +129,14 @@ def get_offloader() -> BaseOffloader:
|
|||||||
return _instance
|
return _instance
|
||||||
|
|
||||||
|
|
||||||
def set_offloader(instance: BaseOffloader) -> None:
|
def set_offloader(instance: BaseOffloader | None) -> None:
|
||||||
"""Set the global offloader instance."""
|
"""Set or reset the global offloader instance."""
|
||||||
global _instance
|
global _instance
|
||||||
_instance = instance
|
_instance = NoopOffloader() if instance is None else instance
|
||||||
if isinstance(instance, NoopOffloader):
|
if isinstance(_instance, NoopOffloader):
|
||||||
logger.debug_once("Offloader set to NoopOffloader (no offloading).")
|
logger.debug_once("Offloader set to NoopOffloader (no offloading).")
|
||||||
else:
|
else:
|
||||||
logger.info_once("Offloader set to %s", type(instance).__name__)
|
logger.info_once("Offloader set to %s", type(_instance).__name__)
|
||||||
|
|
||||||
|
|
||||||
def create_offloader(offload_config: "OffloadConfig") -> BaseOffloader:
|
def create_offloader(offload_config: "OffloadConfig") -> BaseOffloader:
|
||||||
|
|||||||
@@ -2248,6 +2248,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
torch.accelerator.synchronize()
|
torch.accelerator.synchronize()
|
||||||
if self.aux_output_connector is not None:
|
if self.aux_output_connector is not None:
|
||||||
self.aux_output_connector.close()
|
self.aux_output_connector.close()
|
||||||
|
set_offloader(None)
|
||||||
self.cudagraph_manager = None
|
self.cudagraph_manager = None
|
||||||
self.fast_prefill = None
|
self.fast_prefill = None
|
||||||
self.pooling_runner = None
|
self.pooling_runner = None
|
||||||
|
|||||||
Reference in New Issue
Block a user