[Bugfix] LoRA for DeepSeek V3.2 (#35077)

Signed-off-by: Hollow Man <hollowman@opensuse.org>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
ℍ𝕠𝕝𝕝𝕠𝕨 𝕄𝕒𝕟
2026-04-22 19:33:50 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Jee Jee Li
parent 04eac6ba24
commit a250f1bd5f
13 changed files with 614 additions and 37 deletions
+150
View File
@@ -13,6 +13,7 @@ from vllm.config.lora import LoRAConfig
from vllm.lora.layers import (
ColumnParallelLinearWithLoRA,
MergedColumnParallelLinearWithLoRA,
ReplicatedLinearWithLoRA,
RowParallelLinearWithLoRA,
)
from vllm.lora.lora_model import LoRAModel
@@ -26,6 +27,7 @@ from vllm.lora.model_manager import (
from vllm.lora.peft_helper import PEFTHelper
from vllm.lora.request import LoRARequest
from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager, WorkerLoRAManager
from vllm.model_executor.layers.fused_moe import GateLinear
from vllm.platforms import current_platform
from .utils import create_peft_lora
@@ -132,6 +134,135 @@ def test_replace_submodules(default_vllm_config, dist_init, dummy_model):
assert isinstance(model.get_submodule("layer1.dense2"), RowParallelLinearWithLoRA)
def test_wrap_replicated_linear_subclasses(default_vllm_config, dist_init, dummy_model):
from vllm.model_executor.layers.linear import ReplicatedLinear
class CustomReplicatedLinear(ReplicatedLinear):
pass
model = dummy_model
model.add_module("custom_gate", CustomReplicatedLinear(10, 10, bias=False))
manager = LoRAModelManager(
model,
1,
1,
1,
LoRAConfig(
max_lora_rank=8, max_cpu_loras=8, max_loras=8, lora_dtype=DEFAULT_DTYPE
),
torch.device(DEVICES[0]),
)
assert isinstance(
manager.model.get_submodule("custom_gate"), ReplicatedLinearWithLoRA
)
def test_wrap_gate_linear(default_vllm_config, dist_init, dummy_model):
model = dummy_model
model.add_module("router_gate", GateLinear(10, 4, bias=False))
manager = LoRAModelManager(
model,
1,
1,
1,
LoRAConfig(
max_lora_rank=8, max_cpu_loras=8, max_loras=8, lora_dtype=DEFAULT_DTYPE
),
torch.device(DEVICES[0]),
)
assert isinstance(
manager.model.get_submodule("router_gate"), ReplicatedLinearWithLoRA
)
def test_skip_unsupported_matched_modules(default_vllm_config, dist_init, dummy_model):
class UnsupportedContainer(nn.Module):
def __init__(self):
super().__init__()
# This name matches a supported target suffix ("dense1"),
# but nn.Linear is not currently a LoRA-wrappable layer type.
self.dense1 = nn.Linear(10, 10, bias=False)
model = dummy_model
model.add_module("unsupported", UnsupportedContainer())
manager = LoRAModelManager(
model,
1,
1,
1,
LoRAConfig(
max_lora_rank=8, max_cpu_loras=8, max_loras=8, lora_dtype=DEFAULT_DTYPE
),
torch.device(DEVICES[0]),
)
# Should not crash and should keep unsupported matched modules unchanged.
assert isinstance(manager.model.get_submodule("unsupported.dense1"), nn.Linear)
assert "unsupported.dense1" not in manager.modules
def test_target_modules_fail_closed_on_unsupported_matched_modules(
default_vllm_config, dist_init, dummy_model
):
class UnsupportedContainer(nn.Module):
def __init__(self):
super().__init__()
self.dense1 = nn.Linear(10, 10, bias=False)
model = dummy_model
model.add_module("unsupported", UnsupportedContainer())
with pytest.raises(ValueError, match="unsupported.dense1"):
LoRAModelManager(
model,
1,
1,
1,
LoRAConfig(
max_lora_rank=8,
max_cpu_loras=8,
max_loras=8,
lora_dtype=DEFAULT_DTYPE,
target_modules=["dense1"],
),
torch.device(DEVICES[0]),
)
def test_get_dummy_lora_warmup_rank_for_fully_sharded_moe():
manager = LoRAModelManager.__new__(LoRAModelManager)
manager.lora_config = LoRAConfig(
max_lora_rank=64,
max_cpu_loras=1,
max_loras=1,
lora_dtype=DEFAULT_DTYPE,
fully_sharded_loras=True,
)
class DummyModule:
def __init__(self, tp_size: int, fully_sharded: bool):
self.tp_size = tp_size
self.fully_sharded = fully_sharded
manager.modules = {
"model.layers.0.self_attn.q_proj": DummyModule(
tp_size=32,
fully_sharded=True,
),
"model.layers.0.mlp.experts": DummyModule(
tp_size=32,
fully_sharded=True,
),
}
assert manager.get_dummy_lora_warmup_rank(8) == 32
@pytest.mark.parametrize("device", DEVICES)
def test_lora_model_manager(default_vllm_config, dist_init, dummy_model, device):
model = dummy_model
@@ -795,6 +926,25 @@ def test_target_modules_none_uses_all(
)
@pytest.mark.parametrize("device", DEVICES)
def test_target_modules_match_packed_runtime_modules(
default_vllm_config, dist_init, dummy_model_gate_up, device
):
"""Packed runtime modules should be selected by their adapter-visible names."""
_test_target_modules(
dummy_model_gate_up,
["gate_proj"],
device,
expected_lora=[("gate_up_proj", MergedColumnParallelLinearWithLoRA)],
expected_no_lora=[
("dense1", ColumnParallelLinearWithLoRA),
("dense2", RowParallelLinearWithLoRA),
("layer1.dense1", ColumnParallelLinearWithLoRA),
("layer1.dense2", RowParallelLinearWithLoRA),
],
)
@pytest.mark.parametrize("device", DEVICES)
def test_load_adapter_warns_on_unsupported_modules(
default_vllm_config, dist_init, dummy_model_gate_up, device, tmp_path