[Bugfix] Use FusedMoEWithLoRA for shared MoE LoRAs on 3D models (#60098)
Signed-off-by: ARAVINDHAN T <arvindhant01@gmail.com> Co-authored-by: linitra24 <renshuang.zhou@daocloud.io>
This commit is contained in:
co-authored by
linitra24
parent
032d39e9b4
commit
e58f8dde7f
+3
-2
@@ -81,8 +81,9 @@ class LoRAConfig:
|
|||||||
"""If True, load MoE expert adapters in the "shared-outer" layout, where the
|
"""If True, load MoE expert adapters in the "shared-outer" layout, where the
|
||||||
gate/up (`w1`/`w3`) lora_A and the down (`w2`) lora_B are shared across all
|
gate/up (`w1`/`w3`) lora_A and the down (`w2`) lora_B are shared across all
|
||||||
experts (stored once with expert-dim 1) instead of per-expert. The shared
|
experts (stored once with expert-dim 1) instead of per-expert. The shared
|
||||||
factors are broadcast to the expert count at kernel time. Only meaningful for
|
factors are broadcast to the expert count at kernel time. 3D-weight MoE
|
||||||
MoE models whose adapters use this layout; ignored otherwise."""
|
models use `FusedMoEWithLoRA` for this layout. Only meaningful for MoE
|
||||||
|
models whose adapters use this layout; ignored otherwise."""
|
||||||
max_lora_cls_labels: int | None = Field(default=None, ge=1)
|
max_lora_cls_labels: int | None = Field(default=None, ge=1)
|
||||||
"""Maximum output size for LoRA classification heads. Defaults to the
|
"""Maximum output size for LoRA classification heads. Defaults to the
|
||||||
base classification head size."""
|
base classification head size."""
|
||||||
|
|||||||
@@ -144,20 +144,18 @@ class LoRAModelManager:
|
|||||||
else set()
|
else set()
|
||||||
)
|
)
|
||||||
|
|
||||||
# When the engine is started with enable_mixed_moe_lora_format=True
|
# Mixed-format and shared-outer adapters both use FusedMoEWithLoRA.
|
||||||
# we force the universal 2D wrapper (FusedMoEWithLoRA) regardless of
|
# FusedMoE3DWithLoRA only implements the fused gate/up PEFT pair.
|
||||||
# the model's 3D flag, so 2D and 3D adapters can coexist.
|
|
||||||
self._enable_mixed_moe_lora_format = (
|
self._enable_mixed_moe_lora_format = (
|
||||||
is_moe and lora_config.enable_mixed_moe_lora_format
|
is_moe and lora_config.enable_mixed_moe_lora_format
|
||||||
)
|
)
|
||||||
|
self._enable_moe_shared_loras = is_moe and lora_config.enable_moe_shared_loras
|
||||||
self._is_3d_moe_model = (
|
self._is_3d_moe_model = (
|
||||||
self._is_moe
|
self._is_moe
|
||||||
and self.model.is_3d_moe_weight
|
and self.model.is_3d_moe_weight
|
||||||
and not self._enable_mixed_moe_lora_format
|
and not self._enable_mixed_moe_lora_format
|
||||||
|
and not self._enable_moe_shared_loras
|
||||||
)
|
)
|
||||||
# Shared MoE adapters: w13 lora_A / w2 lora_B shared across experts,
|
|
||||||
# stored as pre-stacked experts.w{1,2,3} tensors (startup opt-in).
|
|
||||||
self._enable_moe_shared_loras = is_moe and lora_config.enable_moe_shared_loras
|
|
||||||
self.packed_modules_mapping = process_packed_modules_mapping(
|
self.packed_modules_mapping = process_packed_modules_mapping(
|
||||||
self.model,
|
self.model,
|
||||||
force_2d_moe=self._enable_mixed_moe_lora_format,
|
force_2d_moe=self._enable_mixed_moe_lora_format,
|
||||||
|
|||||||
Reference in New Issue
Block a user