[Bugfix][LoRA] Fix GPT-OSS gate/up interleaving with mixed MoE LoRA formats (#60812)

Signed-off-by: linitra24 <renshuang.zhou@daocloud.io>
This commit is contained in:
linitra24
2026-10-10 12:52:51 +08:00
committed by GitHub
parent 5ca26880ad
commit 364bc81a9e
2 changed files with 56 additions and 5 deletions
@@ -3,6 +3,8 @@
import torch
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.fused_moe.experts.lora_context import MoELoRAContext
@@ -22,6 +24,7 @@ class LoRAExpertsMixin:
state is on lora_context or passed as arguments.
"""
moe_config: FusedMoEConfig
_lora_context: MoELoRAContext | None = None
def set_lora_context(self, ctx: MoELoRAContext) -> None:
@@ -70,8 +73,13 @@ class LoRAExpertsMixin:
# concat swap afterwards.
w13_lora_a_stacked = w13_lora_a_stacked[::-1]
w13_lora_b_stacked = w13_lora_b_stacked[::-1]
return lora_context.punica_wrapper.add_lora_w13(
y,
interleave_w13 = (
lora_context.w13_num_slices == 2
and self.moe_config.activation == MoEActivation.SWIGLUOAI
)
lora_output = torch.zeros_like(y) if interleave_w13 else y
result = lora_context.punica_wrapper.add_lora_w13(
lora_output,
x,
w13_lora_a_stacked,
w13_lora_b_stacked,
@@ -89,9 +97,18 @@ class LoRAExpertsMixin:
lora_context.w13_num_slices,
lora_context.fully_sharded,
lora_context.use_tuned_config,
add_inputs=add_inputs,
add_inputs=add_inputs or interleave_w13,
token_lora_mapping=lora_context.local_token_lora_mapping,
)
if interleave_w13:
# Map gate/up slices onto even/odd output channels without a copy.
gate_up_delta = lora_output.unflatten(-1, (2, -1))
gate_up_output = y.unflatten(-1, (-1, 2)).transpose(-1, -2)
if add_inputs:
gate_up_output.add_(gate_up_delta)
else:
gate_up_output.copy_(gate_up_delta)
return result
def apply_w2_lora(
self,