[LoRA] feat: Support LoRA for DeepSeek V4 (#53361)

Signed-off-by: Hollow Man <hollowman@opensuse.org>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
ℍ𝕠𝕝𝕝𝕠𝕨 𝕄𝕒𝕟
2026-08-24 03:57:57 +00:00
committed by GitHub
co-authored by Jee Jee Li
parent e8888b2d68
commit 702e1d7186
3 changed files with 70 additions and 5 deletions
@@ -1002,6 +1002,8 @@ class RoutedExperts(PluggableLayer):
See `build_expert_params_mapping` for the returned tuple format.
"""
has_base_layer = any(".base_layer." in n for n, _ in model.named_parameters())
prefix = "base_layer." if has_base_layer else ""
# These loaders index ``params_dict[full_name]``, so both sides get it.
return RoutedExperts.build_expert_params_mapping(
ckpt_gate_proj_name,
ckpt_down_proj_name,
@@ -1009,7 +1011,8 @@ class RoutedExperts(PluggableLayer):
num_experts,
num_redundant_experts,
routed_experts_prefix,
"base_layer." if has_base_layer else "",
lora_base_layer_prefix=prefix,
lora_base_layer_prefix_on_param_name=prefix,
)
@staticmethod
@@ -1021,6 +1024,7 @@ class RoutedExperts(PluggableLayer):
num_redundant_experts: int = 0,
routed_experts_prefix: str = "routed_experts",
lora_base_layer_prefix: str = "",
lora_base_layer_prefix_on_param_name: str = "",
include_fused: bool = False,
) -> list[tuple[str, str, int, str]]:
"""
@@ -1035,7 +1039,14 @@ class RoutedExperts(PluggableLayer):
ckpt_up_proj_name: Name of up projection in checkpoint
num_experts: Number of logical (non-redundant) experts
num_redundant_experts: Number of redundant experts
lora_base_layer_prefix: Prefix to add if this layer is a LoRA base layer
lora_base_layer_prefix: LoRA ``base_layer.`` prefix for the
``weight_name`` (checkpoint) side
lora_base_layer_prefix_on_param_name: same, for the ``param_name``
side. Independent because ``get_expert_mapping`` resolves
``param_name`` via ``getattr`` against this layer's bare
``w13_weight``/``w2_weight`` (no prefix), while
``make_expert_params_mapping`` indexes the model-wide
``params_dict`` (prefix included).
include_fused: Prepend the fused pre-fused-checkpoint entries
Returns:
@@ -1061,8 +1072,10 @@ class RoutedExperts(PluggableLayer):
if routed_experts_prefix != "":
routed_experts_prefix = f"{routed_experts_prefix}."
w13 = f"experts.{lora_base_layer_prefix}{routed_experts_prefix}w13_"
w2 = f"experts.{lora_base_layer_prefix}{routed_experts_prefix}w2_"
w13 = (
f"experts.{lora_base_layer_prefix_on_param_name}{routed_experts_prefix}w13_"
)
w2 = f"experts.{lora_base_layer_prefix_on_param_name}{routed_experts_prefix}w2_"
fused_mapping = []
if include_fused: