[Bugfix] Fix level-2 sleep/wake/reload with enable_lora=True (#39935)
Signed-off-by: Silen Naihin <silen.naihin@gmail.com> Co-authored-by: Dakai An <dakaian108@gmail.com> Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
co-authored by
Dakai An
Jiangyun Zhu
parent
8cfa01cdd2
commit
b50fdebce0
@@ -118,6 +118,18 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
|
||||
else:
|
||||
self.sharded_to_full_mapping_gpu = None
|
||||
|
||||
def reset_sharded_to_full_mapping(self) -> None:
|
||||
"""Restore the TP logits mapping after its GPU memory is reused."""
|
||||
mapping_gpu = self.sharded_to_full_mapping_gpu
|
||||
if mapping_gpu is not None:
|
||||
mapping_gpu.copy_(
|
||||
torch.tensor(
|
||||
self.sharded_to_full_mapping,
|
||||
device=mapping_gpu.device,
|
||||
dtype=mapping_gpu.dtype,
|
||||
)
|
||||
)
|
||||
|
||||
def reset_lora(self, index: int):
|
||||
self.lora_a_stacked[index] = 0
|
||||
self.lora_b_stacked[index] = 0
|
||||
|
||||
Reference in New Issue
Block a user