[MRV2] Support attention-free models (#52374)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -372,7 +372,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
self.model_memory_usage = m.consumed_memory
|
||||
logger.info(
|
||||
"Model loading took %s GiB and %.6f seconds",
|
||||
"Model loading took %s GiB memory and %.6f seconds",
|
||||
format_gib(m.consumed_memory),
|
||||
time_after_load - time_before_load,
|
||||
)
|
||||
@@ -1889,6 +1889,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
"""Release GPU tensors (model weights, KV caches, workspace) so that
|
||||
memory is reclaimable when running in the same process."""
|
||||
torch.accelerator.synchronize()
|
||||
self.cudagraph_manager = None
|
||||
if hasattr(self, "kv_caches"):
|
||||
self.kv_caches.clear()
|
||||
if hasattr(self, "attn_groups"):
|
||||
|
||||
Reference in New Issue
Block a user