[Bugfix] LoRA for DeepSeek V3.2 (#35077)

Signed-off-by: Hollow Man <hollowman@opensuse.org>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
ℍ𝕠𝕝𝕝𝕠𝕨 𝕄𝕒𝕟
2026-04-22 19:33:50 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Jee Jee Li
parent 04eac6ba24
commit a250f1bd5f
13 changed files with 614 additions and 37 deletions
+42 -10
View File
@@ -40,11 +40,19 @@ def _mcp_apply(x, bias, layer: "ColumnParallelLinearWithLoRA"):
# Since communication is needed, the buffer is directly initialized as a
# tensor rather than a tuple of tensor.
buffers = torch.zeros(
(layer.n_slices, x.shape[0], layer.lora_a_stacked[0].shape[2]),
local_lora_rank = layer.lora_a_stacked[0].shape[2]
buffer_shape = (layer.n_slices, x.shape[0], local_lora_rank)
# Under torch.compile, the local-rank-1 fully-sharded path can otherwise
# get lowered to a reinterpret view with a non-canonical layout. The
# Triton shrink op mutates this buffer in place and expects the standard
# contiguous [slice, token, rank] stride contract.
buffers = torch.empty_strided(
buffer_shape,
(x.shape[0] * local_lora_rank, local_lora_rank, 1),
dtype=torch.float32,
device=x.device,
)
buffers.zero_()
shrunk_buffers: torch.Tensor | None = layer.punica_wrapper.add_shrink(
buffers, x, layer.lora_a_stacked, 1.0
@@ -86,7 +94,7 @@ class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
# The base_layer type is ColumnParallelLinear or
# MergedColumnParallelLinear, their weight sharding logic is
# inconsistent when TP is greater than 1.
self.is_merged_col_linear = type(base_layer) is MergedColumnParallelLinear
self.is_merged_col_linear = isinstance(base_layer, MergedColumnParallelLinear)
self.output_size = self.base_layer.output_size_per_partition
# There is only one LoRA layer
self.n_slices = 1
@@ -158,7 +166,7 @@ class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
) -> bool:
if type(source_layer) is maybe_get_oot_by_class(ColumnParallelLinear):
return True
if type(source_layer) is maybe_get_oot_by_class(MergedColumnParallelLinear):
if isinstance(source_layer, maybe_get_oot_by_class(MergedColumnParallelLinear)):
if len(packed_modules_list) != 1:
return False
# Exclude layers with 3+ output sizes - those are handled by
@@ -275,19 +283,41 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
index, 0, : lora_b_i.shape[0], : lora_b_i.shape[1]
].copy_(lora_b_i, non_blocking=True)
def apply(self, x: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor:
merged_cls = maybe_get_oot_by_class(MergedColumnParallelLinear)
# Effectively unsharded subclasses can safely reuse their custom
# forward() implementation before applying the LoRA delta.
if (
self.tp_size == 1
and type(self.base_layer) is not merged_cls
and type(self.base_layer).forward is not merged_cls.forward
):
return self._apply_base_forward(x)
return _mcp_apply(x, bias, self)
@classmethod
@_not_fully_sharded_can_replace
def can_replace_layer(
cls,
source_layer: nn.Module,
lora_config: LoRAConfig,
packed_modules_list: list,
model_config: PretrainedConfig | None = None,
decorate: bool = True,
) -> bool:
return (
type(source_layer) is MergedColumnParallelLinear
and len(packed_modules_list) == 2
)
merged_cls = maybe_get_oot_by_class(MergedColumnParallelLinear)
if not isinstance(source_layer, merged_cls) or len(packed_modules_list) != 2:
return False
tp_size = getattr(source_layer, "tp_size", 1)
if type(source_layer) is merged_cls:
if not decorate:
return True
return not lora_config.fully_sharded_loras or tp_size == 1
# Only support effectively unsharded subclasses here. Sharded
# subclasses may have custom communication semantics that the generic
# merged-column LoRA path does not know how to preserve.
return tp_size == 1
class QKVParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
@@ -607,7 +637,9 @@ class MergedColumnParallelLinearVariableSliceWithLoRA(
) -> bool:
# Support MergedColumnParallelLinear with 3 or more slices
# (2 slices are handled by MergedColumnParallelLinearWithLoRA)
if type(source_layer) is not maybe_get_oot_by_class(MergedColumnParallelLinear):
if not isinstance(
source_layer, maybe_get_oot_by_class(MergedColumnParallelLinear)
):
return False
# If packed_modules_list has 3+ items, use this class