[Bugfix][LoRA] Use stored rsLoRA scaling factor in MoE expert packing (#55548)

Signed-off-by: Kushal <72650064+kushaldabbe@users.noreply.github.com>
This commit is contained in:
Kushal
2026-09-11 03:52:33 +00:00
committed by GitHub
parent ce08bb5b34
commit b5d4186300
2 changed files with 56 additions and 3 deletions
+4 -3
View File
@@ -201,8 +201,9 @@ class PackedLoRALayerWeights(LoRALayerWeights):
w1_lora_b = torch.stack(w1_lora_b_lst, dim=0) # (num_experts,output_size,rank)
w2_lora_b = torch.stack(w2_lora_b_lst, dim=0)
# All w1, w2, w3 have the same scaling factor.
scaling = lora_alpha / rank
# All w1, w2, w3 have the same scaling factor. Use the per-adapter
# scaling (e.g. alpha/sqrt(r) for rsLoRA) instead of alpha/rank.
scaling = first_lora.scaling
last_scaling = scaling
if is_non_gated_moe:
@@ -250,7 +251,7 @@ class PackedLoRALayerWeights(LoRALayerWeights):
assert w1_lora is not None and w2_lora is not None and w3_lora is not None
rank = w1_lora.rank
lora_alpha = w1_lora.lora_alpha
scaling = lora_alpha / rank
scaling = w1_lora.scaling
return cls(
module_name,
rank,