[LoRA] feat: Support LoRA for DeepSeek V4 (#53361)
Signed-off-by: Hollow Man <hollowman@opensuse.org> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
@@ -15,6 +15,24 @@ if TYPE_CHECKING:
|
||||
|
||||
|
||||
class BaseLayerWithLoRA(nn.Module):
|
||||
def __getattr__(self, name):
|
||||
d = self.__dict__
|
||||
if name in d.get("_parameters", ()):
|
||||
return d["_parameters"][name]
|
||||
if name in d.get("_buffers", ()):
|
||||
return d["_buffers"][name]
|
||||
if name in d.get("_modules", ()):
|
||||
return d["_modules"][name]
|
||||
# Forward public misses to ``base_layer``; private names are framework
|
||||
# bookkeeping and must stay local.
|
||||
if not name.startswith("_"):
|
||||
base_layer = d.get("_modules", {}).get("base_layer")
|
||||
if base_layer is not None:
|
||||
return getattr(base_layer, name)
|
||||
raise AttributeError(
|
||||
f"{type(self).__name__!r} object has no attribute {name!r}"
|
||||
)
|
||||
|
||||
def load_weights(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[str]:
|
||||
|
||||
@@ -1002,6 +1002,8 @@ class RoutedExperts(PluggableLayer):
|
||||
See `build_expert_params_mapping` for the returned tuple format.
|
||||
"""
|
||||
has_base_layer = any(".base_layer." in n for n, _ in model.named_parameters())
|
||||
prefix = "base_layer." if has_base_layer else ""
|
||||
# These loaders index ``params_dict[full_name]``, so both sides get it.
|
||||
return RoutedExperts.build_expert_params_mapping(
|
||||
ckpt_gate_proj_name,
|
||||
ckpt_down_proj_name,
|
||||
@@ -1009,7 +1011,8 @@ class RoutedExperts(PluggableLayer):
|
||||
num_experts,
|
||||
num_redundant_experts,
|
||||
routed_experts_prefix,
|
||||
"base_layer." if has_base_layer else "",
|
||||
lora_base_layer_prefix=prefix,
|
||||
lora_base_layer_prefix_on_param_name=prefix,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -1021,6 +1024,7 @@ class RoutedExperts(PluggableLayer):
|
||||
num_redundant_experts: int = 0,
|
||||
routed_experts_prefix: str = "routed_experts",
|
||||
lora_base_layer_prefix: str = "",
|
||||
lora_base_layer_prefix_on_param_name: str = "",
|
||||
include_fused: bool = False,
|
||||
) -> list[tuple[str, str, int, str]]:
|
||||
"""
|
||||
@@ -1035,7 +1039,14 @@ class RoutedExperts(PluggableLayer):
|
||||
ckpt_up_proj_name: Name of up projection in checkpoint
|
||||
num_experts: Number of logical (non-redundant) experts
|
||||
num_redundant_experts: Number of redundant experts
|
||||
lora_base_layer_prefix: Prefix to add if this layer is a LoRA base layer
|
||||
lora_base_layer_prefix: LoRA ``base_layer.`` prefix for the
|
||||
``weight_name`` (checkpoint) side
|
||||
lora_base_layer_prefix_on_param_name: same, for the ``param_name``
|
||||
side. Independent because ``get_expert_mapping`` resolves
|
||||
``param_name`` via ``getattr`` against this layer's bare
|
||||
``w13_weight``/``w2_weight`` (no prefix), while
|
||||
``make_expert_params_mapping`` indexes the model-wide
|
||||
``params_dict`` (prefix included).
|
||||
include_fused: Prepend the fused pre-fused-checkpoint entries
|
||||
|
||||
Returns:
|
||||
@@ -1061,8 +1072,10 @@ class RoutedExperts(PluggableLayer):
|
||||
if routed_experts_prefix != "":
|
||||
routed_experts_prefix = f"{routed_experts_prefix}."
|
||||
|
||||
w13 = f"experts.{lora_base_layer_prefix}{routed_experts_prefix}w13_"
|
||||
w2 = f"experts.{lora_base_layer_prefix}{routed_experts_prefix}w2_"
|
||||
w13 = (
|
||||
f"experts.{lora_base_layer_prefix_on_param_name}{routed_experts_prefix}w13_"
|
||||
)
|
||||
w2 = f"experts.{lora_base_layer_prefix_on_param_name}{routed_experts_prefix}w2_"
|
||||
|
||||
fused_mapping = []
|
||||
if include_fused:
|
||||
|
||||
@@ -55,6 +55,7 @@ from vllm.model_executor.models.interfaces import (
|
||||
EagleModelMixin,
|
||||
MixtureOfExperts,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
SupportsPP,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
@@ -1468,6 +1469,11 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
break
|
||||
if name not in params_dict:
|
||||
head, _, leaf = name.rpartition(".")
|
||||
suffixed = f"{head}.base_layer.{leaf}"
|
||||
if suffixed in params_dict:
|
||||
name = suffixed
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
@@ -1522,6 +1528,13 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
else:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
# Non-LoRA params on a LoRA-wrapped module live at
|
||||
# ``<head>.base_layer.<leaf>``; the checkpoint is plain.
|
||||
if name not in params_dict:
|
||||
head, _, leaf = name.rpartition(".")
|
||||
suffixed = f"{head}.base_layer.{leaf}"
|
||||
if suffixed in params_dict:
|
||||
name = suffixed
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
@@ -1598,6 +1611,10 @@ def _make_deepseek_v4_weights_mapper(expert_dtype: str) -> WeightsMapper:
|
||||
# shared experts use Fp8LinearMethod's block scales, which
|
||||
# register as ``weight_scale_inv``.
|
||||
scale_regex = {
|
||||
# ``.base_layer.``-namespace variant (LoRA-wrapped experts).
|
||||
re.compile(
|
||||
r"(\.experts\.\d+\.w[123]\.base_layer)\.scale$"
|
||||
): r"\1.weight_scale",
|
||||
re.compile(r"(\.experts\.\d+\.w[123])\.scale$"): r"\1.weight_scale",
|
||||
re.compile(r"\.scale$"): ".weight_scale_inv",
|
||||
}
|
||||
@@ -1606,6 +1623,10 @@ def _make_deepseek_v4_weights_mapper(expert_dtype: str) -> WeightsMapper:
|
||||
# scales as ``w{13,2}_weight_scale_inv``. Map all ``.scale`` keys
|
||||
# there.
|
||||
scale_regex = {
|
||||
# ``.base_layer.``-namespace variant of the above.
|
||||
re.compile(
|
||||
r"(\.experts\.\d+\.w[123]\.base_layer)\.scale$"
|
||||
): r"\1.weight_scale_inv",
|
||||
re.compile(r"\.scale$"): ".weight_scale_inv",
|
||||
}
|
||||
return WeightsMapper(
|
||||
@@ -1667,7 +1688,11 @@ class DeepseekV4MixtureOfExperts(MixtureOfExperts):
|
||||
|
||||
|
||||
class DeepseekV4ForCausalLM(
|
||||
nn.Module, SupportsPP, SupportsEagle3, DeepseekV4MixtureOfExperts
|
||||
nn.Module,
|
||||
SupportsPP,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
DeepseekV4MixtureOfExperts,
|
||||
):
|
||||
model_cls = DeepseekV4Model
|
||||
|
||||
@@ -1675,6 +1700,15 @@ class DeepseekV4ForCausalLM(
|
||||
# Overridden per-instance in __init__ when expert_dtype != "fp4".
|
||||
hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper("fp4")
|
||||
|
||||
packed_modules_mapping = {
|
||||
"gate_up_proj": ["w1", "w3"],
|
||||
"fused_wqa_wkv": ["wq_a", "wkv"],
|
||||
"fused_wkv_wgate": ["wkv", "wgate"],
|
||||
}
|
||||
|
||||
# The MTP draft head is not LoRA-adapted.
|
||||
lora_skip_prefixes = ["mtp."]
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user