[Bugfix][LoRA] Fix GPT-OSS gate/up interleaving with mixed MoE LoRA formats (#60812)
Signed-off-by: linitra24 <renshuang.zhou@daocloud.io>
This commit is contained in:
@@ -3,6 +3,8 @@
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.model_executor.layers.fused_moe.experts.lora_context import MoELoRAContext
|
||||
|
||||
|
||||
@@ -22,6 +24,7 @@ class LoRAExpertsMixin:
|
||||
state is on lora_context or passed as arguments.
|
||||
"""
|
||||
|
||||
moe_config: FusedMoEConfig
|
||||
_lora_context: MoELoRAContext | None = None
|
||||
|
||||
def set_lora_context(self, ctx: MoELoRAContext) -> None:
|
||||
@@ -70,8 +73,13 @@ class LoRAExpertsMixin:
|
||||
# concat swap afterwards.
|
||||
w13_lora_a_stacked = w13_lora_a_stacked[::-1]
|
||||
w13_lora_b_stacked = w13_lora_b_stacked[::-1]
|
||||
return lora_context.punica_wrapper.add_lora_w13(
|
||||
y,
|
||||
interleave_w13 = (
|
||||
lora_context.w13_num_slices == 2
|
||||
and self.moe_config.activation == MoEActivation.SWIGLUOAI
|
||||
)
|
||||
lora_output = torch.zeros_like(y) if interleave_w13 else y
|
||||
result = lora_context.punica_wrapper.add_lora_w13(
|
||||
lora_output,
|
||||
x,
|
||||
w13_lora_a_stacked,
|
||||
w13_lora_b_stacked,
|
||||
@@ -89,9 +97,18 @@ class LoRAExpertsMixin:
|
||||
lora_context.w13_num_slices,
|
||||
lora_context.fully_sharded,
|
||||
lora_context.use_tuned_config,
|
||||
add_inputs=add_inputs,
|
||||
add_inputs=add_inputs or interleave_w13,
|
||||
token_lora_mapping=lora_context.local_token_lora_mapping,
|
||||
)
|
||||
if interleave_w13:
|
||||
# Map gate/up slices onto even/odd output channels without a copy.
|
||||
gate_up_delta = lora_output.unflatten(-1, (2, -1))
|
||||
gate_up_output = y.unflatten(-1, (-1, 2)).transpose(-1, -2)
|
||||
if add_inputs:
|
||||
gate_up_output.add_(gate_up_delta)
|
||||
else:
|
||||
gate_up_output.copy_(gate_up_delta)
|
||||
return result
|
||||
|
||||
def apply_w2_lora(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user