[Bugfix] LoRA for DeepSeek V3.2 (#35077)

Signed-off-by: Hollow Man <hollowman@opensuse.org>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
ℍ𝕠𝕝𝕝𝕠𝕨 𝕄𝕒𝕟
2026-04-22 19:33:50 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Jee Jee Li
parent 04eac6ba24
commit a250f1bd5f
13 changed files with 614 additions and 37 deletions
+7 -12
View File
@@ -1620,18 +1620,13 @@ def _parallel_worker(
else:
print("F", end="")
finally:
# Note: for some reason DeepEP buffers don't seem to be
# entirely reusable on B200. In order to work around this
# we clear the all2all manager's cache after each testpoint.
cap = current_platform.get_device_capability()
if (
cap is not None
and cap.major == 10
and (
test_config.backend == "deepep_low_latency"
or test_config.backend == "deepep_high_throughput"
)
):
# DeepEP managers are not reliably reusable across many subtests in
# a single worker process. Tear them down after each DeepEP case so
# later subtests do not inherit stale communication state.
if test_config.backend in {
"deepep_low_latency",
"deepep_high_throughput",
}:
torch.accelerator.synchronize()
all2all_manager = get_ep_group().device_communicator.all2all_manager
if all2all_manager is not None:
+270 -2
View File
@@ -44,6 +44,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
get_masked_input_and_mask,
)
from vllm.model_executor.models.deepseek_v2 import DeepSeekV2FusedQkvAProjLinear
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
@@ -1422,7 +1423,107 @@ def test_variable_slice_lora_class_selection(default_vllm_config, dist_init):
f"for 2 packed modules, got {type(selected_layer_merged).__name__}"
)
# Case 5: Plain ColumnParallelLinear (not merged) - common in many models
fully_sharded_tp_lora_config = LoRAConfig(
max_loras=8,
max_lora_rank=16,
lora_dtype=torch.float16,
fully_sharded_loras=True,
)
fully_sharded_tp_layer = MergedColumnParallelLinear(
4096, [2048, 2048], bias=False, params_dtype=torch.float16
)
fully_sharded_tp_layer.tp_size = 2
assert not MergedColumnParallelLinearWithLoRA.can_replace_layer(
source_layer=fully_sharded_tp_layer,
lora_config=fully_sharded_tp_lora_config,
packed_modules_list=packed_modules_two,
), "Generic merged wrapper should reject fully sharded TP layers"
assert MergedColumnParallelLinearWithShardedLoRA.can_replace_layer(
source_layer=fully_sharded_tp_layer,
lora_config=fully_sharded_tp_lora_config,
packed_modules_list=packed_modules_two,
), "Sharded merged wrapper should remain eligible for fully sharded TP layers"
selected_fully_sharded_tp_layer = from_layer(
fully_sharded_tp_layer,
max_loras=8,
lora_config=fully_sharded_tp_lora_config,
packed_modules_list=packed_modules_two,
)
assert isinstance(
selected_fully_sharded_tp_layer,
MergedColumnParallelLinearWithShardedLoRA,
), (
"from_layer should select MergedColumnParallelLinearWithShardedLoRA "
"for fully sharded TP merged layers, got "
f"{type(selected_fully_sharded_tp_layer).__name__}"
)
# Case 5: DeepSeek's fused_qkv_a_proj should reuse the generic merged
# wrapper while preserving its custom base forward path.
deepseek_fused_layer = DeepSeekV2FusedQkvAProjLinear(
4096, [2048, 2048], prefix="model.layers.0.self_attn.fused_qkv_a_proj"
)
selected_deepseek_layer = from_layer(
deepseek_fused_layer,
max_loras=8,
lora_config=lora_config,
packed_modules_list=packed_modules_two,
)
assert isinstance(selected_deepseek_layer, MergedColumnParallelLinearWithLoRA), (
"from_layer should select MergedColumnParallelLinearWithLoRA "
f"for DeepSeek fused_qkv_a_proj, got {type(selected_deepseek_layer).__name__}"
)
fully_sharded_lora_config = LoRAConfig(
max_loras=8,
max_lora_rank=16,
lora_dtype=torch.float16,
fully_sharded_loras=True,
)
selected_fully_sharded_deepseek_layer = from_layer(
deepseek_fused_layer,
max_loras=8,
lora_config=fully_sharded_lora_config,
packed_modules_list=packed_modules_two,
)
assert isinstance(
selected_fully_sharded_deepseek_layer,
MergedColumnParallelLinearWithLoRA,
), (
"from_layer should keep using MergedColumnParallelLinearWithLoRA "
"for fused_qkv_a_proj when the base layer is effectively unsharded, got "
f"{type(selected_fully_sharded_deepseek_layer).__name__}"
)
# Case 6: Generic subclass of MergedColumnParallelLinear with 2 packed
# modules should still use the generic merged wrapper.
class CustomMergedColumnParallelLinear(MergedColumnParallelLinear):
pass
custom_merged_layer = CustomMergedColumnParallelLinear(
4096, [2048, 2048], bias=False, params_dtype=torch.float16
)
assert MergedColumnParallelLinearWithLoRA.can_replace_layer(
source_layer=custom_merged_layer,
lora_config=lora_config,
packed_modules_list=packed_modules_two,
), "MergedColumnParallelLinearWithLoRA should handle subclasses"
selected_custom_layer = from_layer(
custom_merged_layer,
max_loras=8,
lora_config=lora_config,
packed_modules_list=packed_modules_two,
)
assert isinstance(selected_custom_layer, MergedColumnParallelLinearWithLoRA), (
f"from_layer should select MergedColumnParallelLinearWithLoRA "
f"for subclassed merged layers, got {type(selected_custom_layer).__name__}"
)
# Case 7: Plain ColumnParallelLinear (not merged) - common in many models
# -> ColumnParallelLinearWithLoRA should be selected
plain_column_parallel = ColumnParallelLinear(
4096, 4096, bias=False, params_dtype=torch.float16
@@ -1455,7 +1556,7 @@ def test_variable_slice_lora_class_selection(default_vllm_config, dist_init):
f"for plain ColumnParallelLinear, got {type(selected_plain).__name__}"
)
# Case 6: MergedColumnParallelLinear with exactly 2 output sizes
# Case 8: MergedColumnParallelLinear with exactly 2 output sizes
# and empty packed_modules_list
# -> ColumnParallelLinearWithLoRA should NOT match (packed_modules_list != 1)
# -> MergedColumnParallelLinearVariableSliceWithLoRA should NOT match (< 3 slices)
@@ -1473,3 +1574,170 @@ def test_variable_slice_lora_class_selection(default_vllm_config, dist_init):
"MergedColumnParallelLinearVariableSliceWithLoRA "
"should NOT handle 2 slices even with empty packed_modules_list"
)
@pytest.mark.parametrize(
"wrapper_cls",
[ColumnParallelLinearWithLoRA, ColumnParallelLinearWithShardedLoRA],
)
def test_get_and_maybe_dequant_weights_accepts_lora_wrappers(dist_init, wrapper_cls):
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_and_maybe_dequant_weights,
)
linear = ColumnParallelLinear(4096, 4096, bias=False, params_dtype=torch.float16)
lora_linear = wrapper_cls(linear)
# Should work with LoRA wrappers and return [out, in] weights.
dequant_weight = get_and_maybe_dequant_weights(lora_linear, out_dtype=torch.float16)
assert dequant_weight.shape == linear.weight.shape
@torch.inference_mode()
@pytest.mark.parametrize("device", DEVICES)
@pytest.mark.parametrize("stage", STAGES)
@pytest.mark.parametrize("fully_sharded", [False, True])
def test_deepseek_fused_qkv_a_proj_lora_preserves_base_forward(
default_vllm_config, dist_init, device, stage, fully_sharded
):
if current_platform.is_cuda_alike():
torch.accelerator.set_device_index(device)
torch.set_default_device(device)
dtype = torch.float16 if current_platform.is_cuda_alike() else torch.float32
max_loras = 8
lora_config = LoRAConfig(
max_loras=max_loras,
max_lora_rank=8,
lora_dtype=dtype,
fully_sharded_loras=fully_sharded,
)
punica_wrapper = get_punica_wrapper(8192, 256, device, lora_config=lora_config)
assert check_punica_wrapper(punica_wrapper)
class OffsetDeepSeekFusedQkvAProjLinear(DeepSeekV2FusedQkvAProjLinear):
def forward(self, input_):
output, output_bias = super().forward(input_)
return output + 1, output_bias
layer = OffsetDeepSeekFusedQkvAProjLinear(
32, [16, 16], prefix="model.layers.0.self_attn.fused_qkv_a_proj"
)
layer.weight.data = torch.rand_like(layer.weight.data, dtype=dtype)
lora_layer = MergedColumnParallelLinearWithLoRA(layer)
lora_layer.create_lora_weights(max_loras, lora_config)
lora_layer.set_mapping(punica_wrapper)
id_to_index = get_random_id_to_index(1, max_loras, log=False)
active_slot = next(i for i, lora_id in enumerate(id_to_index) if lora_id == 1)
lora_a = [
torch.rand(8, 32, dtype=dtype, device=device),
torch.rand(8, 32, dtype=dtype, device=device),
]
lora_b = [
torch.rand(16, 8, dtype=dtype, device=device),
torch.rand(16, 8, dtype=dtype, device=device),
]
lora_layer.set_lora(active_slot, lora_a=lora_a, lora_b=lora_b)
inputs, index_mapping, prompt_mapping = create_random_inputs(
active_lora_ids=[1],
num_inputs=4,
input_size=(1, 32),
input_range=(0, 1),
input_type=dtype,
device=device,
)
lora_mapping = LoRAMapping(index_mapping, prompt_mapping, is_prefill=stage)
punica_wrapper.update_metadata(lora_mapping, id_to_index, max_loras, 512)
lora_result = lora_layer(torch.cat(inputs))[0]
expected_results = []
for input_ in inputs:
result = layer(input_)[0]
result[:, :16] += input_ @ lora_a[0].T @ lora_b[0].T
result[:, 16:] += input_ @ lora_a[1].T @ lora_b[1].T
expected_results.append(result)
rtol, atol = TOLERANCES[lora_result.dtype]
torch.testing.assert_close(
lora_result, torch.cat(expected_results), rtol=rtol, atol=atol
)
merged_layer = OffsetDeepSeekFusedQkvAProjLinear(
32, [16, 16], prefix="model.layers.0.self_attn.fused_qkv_a_proj"
)
merged_layer.weight.data = layer.weight.data.clone()
merged_layer.weight.data[:16].add_(lora_b[0] @ lora_a[0])
merged_layer.weight.data[16:].add_(lora_b[1] @ lora_a[1])
merged_result = merged_layer(torch.cat(inputs))[0]
torch.testing.assert_close(lora_result, merged_result, rtol=rtol, atol=atol)
@torch.inference_mode()
@pytest.mark.parametrize("device", DEVICES)
@pytest.mark.parametrize("stage", STAGES)
def test_replicated_lora_preserves_base_forward_for_subclasses(
default_vllm_config, dist_init, device, stage
):
if current_platform.is_cuda_alike():
torch.accelerator.set_device_index(device)
torch.set_default_device(device)
dtype = torch.float16 if current_platform.is_cuda_alike() else torch.float32
max_loras = 8
lora_config = LoRAConfig(max_loras=max_loras, max_lora_rank=8, lora_dtype=dtype)
punica_wrapper = get_punica_wrapper(8192, 256, device, lora_config=lora_config)
assert check_punica_wrapper(punica_wrapper)
class OffsetReplicatedLinear(ReplicatedLinear):
def forward(self, input_):
output, output_bias = super().forward(input_)
return output + 1, output_bias
layer = OffsetReplicatedLinear(32, 16, bias=False, params_dtype=dtype)
layer.weight.data = torch.rand_like(layer.weight.data, dtype=dtype)
lora_layer = ReplicatedLinearWithLoRA(layer)
lora_layer.create_lora_weights(max_loras, lora_config)
lora_layer.set_mapping(punica_wrapper)
id_to_index = get_random_id_to_index(1, max_loras, log=False)
active_slot = next(i for i, lora_id in enumerate(id_to_index) if lora_id == 1)
lora_a = torch.rand(8, 32, dtype=dtype, device=device)
lora_b = torch.rand(16, 8, dtype=dtype, device=device)
lora_layer.set_lora(active_slot, lora_a=lora_a, lora_b=lora_b)
inputs, index_mapping, prompt_mapping = create_random_inputs(
active_lora_ids=[1],
num_inputs=4,
input_size=(1, 32),
input_range=(0, 1),
input_type=dtype,
device=device,
)
lora_mapping = LoRAMapping(index_mapping, prompt_mapping, is_prefill=stage)
punica_wrapper.update_metadata(lora_mapping, id_to_index, max_loras, 512)
lora_result = lora_layer(torch.cat(inputs))[0]
expected_results = []
for input_ in inputs:
result = layer(input_)[0]
result += input_ @ lora_a.T @ lora_b.T
expected_results.append(result)
rtol, atol = TOLERANCES[lora_result.dtype]
torch.testing.assert_close(
lora_result, torch.cat(expected_results), rtol=rtol, atol=atol
)
merged_layer = OffsetReplicatedLinear(32, 16, bias=False, params_dtype=dtype)
merged_layer.weight.data = layer.weight.data.clone()
merged_layer.weight.data.add_(lora_b @ lora_a)
merged_result = merged_layer(torch.cat(inputs))[0]
torch.testing.assert_close(lora_result, merged_result, rtol=rtol, atol=atol)
+150
View File
@@ -13,6 +13,7 @@ from vllm.config.lora import LoRAConfig
from vllm.lora.layers import (
ColumnParallelLinearWithLoRA,
MergedColumnParallelLinearWithLoRA,
ReplicatedLinearWithLoRA,
RowParallelLinearWithLoRA,
)
from vllm.lora.lora_model import LoRAModel
@@ -26,6 +27,7 @@ from vllm.lora.model_manager import (
from vllm.lora.peft_helper import PEFTHelper
from vllm.lora.request import LoRARequest
from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager, WorkerLoRAManager
from vllm.model_executor.layers.fused_moe import GateLinear
from vllm.platforms import current_platform
from .utils import create_peft_lora
@@ -132,6 +134,135 @@ def test_replace_submodules(default_vllm_config, dist_init, dummy_model):
assert isinstance(model.get_submodule("layer1.dense2"), RowParallelLinearWithLoRA)
def test_wrap_replicated_linear_subclasses(default_vllm_config, dist_init, dummy_model):
from vllm.model_executor.layers.linear import ReplicatedLinear
class CustomReplicatedLinear(ReplicatedLinear):
pass
model = dummy_model
model.add_module("custom_gate", CustomReplicatedLinear(10, 10, bias=False))
manager = LoRAModelManager(
model,
1,
1,
1,
LoRAConfig(
max_lora_rank=8, max_cpu_loras=8, max_loras=8, lora_dtype=DEFAULT_DTYPE
),
torch.device(DEVICES[0]),
)
assert isinstance(
manager.model.get_submodule("custom_gate"), ReplicatedLinearWithLoRA
)
def test_wrap_gate_linear(default_vllm_config, dist_init, dummy_model):
model = dummy_model
model.add_module("router_gate", GateLinear(10, 4, bias=False))
manager = LoRAModelManager(
model,
1,
1,
1,
LoRAConfig(
max_lora_rank=8, max_cpu_loras=8, max_loras=8, lora_dtype=DEFAULT_DTYPE
),
torch.device(DEVICES[0]),
)
assert isinstance(
manager.model.get_submodule("router_gate"), ReplicatedLinearWithLoRA
)
def test_skip_unsupported_matched_modules(default_vllm_config, dist_init, dummy_model):
class UnsupportedContainer(nn.Module):
def __init__(self):
super().__init__()
# This name matches a supported target suffix ("dense1"),
# but nn.Linear is not currently a LoRA-wrappable layer type.
self.dense1 = nn.Linear(10, 10, bias=False)
model = dummy_model
model.add_module("unsupported", UnsupportedContainer())
manager = LoRAModelManager(
model,
1,
1,
1,
LoRAConfig(
max_lora_rank=8, max_cpu_loras=8, max_loras=8, lora_dtype=DEFAULT_DTYPE
),
torch.device(DEVICES[0]),
)
# Should not crash and should keep unsupported matched modules unchanged.
assert isinstance(manager.model.get_submodule("unsupported.dense1"), nn.Linear)
assert "unsupported.dense1" not in manager.modules
def test_target_modules_fail_closed_on_unsupported_matched_modules(
default_vllm_config, dist_init, dummy_model
):
class UnsupportedContainer(nn.Module):
def __init__(self):
super().__init__()
self.dense1 = nn.Linear(10, 10, bias=False)
model = dummy_model
model.add_module("unsupported", UnsupportedContainer())
with pytest.raises(ValueError, match="unsupported.dense1"):
LoRAModelManager(
model,
1,
1,
1,
LoRAConfig(
max_lora_rank=8,
max_cpu_loras=8,
max_loras=8,
lora_dtype=DEFAULT_DTYPE,
target_modules=["dense1"],
),
torch.device(DEVICES[0]),
)
def test_get_dummy_lora_warmup_rank_for_fully_sharded_moe():
manager = LoRAModelManager.__new__(LoRAModelManager)
manager.lora_config = LoRAConfig(
max_lora_rank=64,
max_cpu_loras=1,
max_loras=1,
lora_dtype=DEFAULT_DTYPE,
fully_sharded_loras=True,
)
class DummyModule:
def __init__(self, tp_size: int, fully_sharded: bool):
self.tp_size = tp_size
self.fully_sharded = fully_sharded
manager.modules = {
"model.layers.0.self_attn.q_proj": DummyModule(
tp_size=32,
fully_sharded=True,
),
"model.layers.0.mlp.experts": DummyModule(
tp_size=32,
fully_sharded=True,
),
}
assert manager.get_dummy_lora_warmup_rank(8) == 32
@pytest.mark.parametrize("device", DEVICES)
def test_lora_model_manager(default_vllm_config, dist_init, dummy_model, device):
model = dummy_model
@@ -795,6 +926,25 @@ def test_target_modules_none_uses_all(
)
@pytest.mark.parametrize("device", DEVICES)
def test_target_modules_match_packed_runtime_modules(
default_vllm_config, dist_init, dummy_model_gate_up, device
):
"""Packed runtime modules should be selected by their adapter-visible names."""
_test_target_modules(
dummy_model_gate_up,
["gate_proj"],
device,
expected_lora=[("gate_up_proj", MergedColumnParallelLinearWithLoRA)],
expected_no_lora=[
("dense1", ColumnParallelLinearWithLoRA),
("dense2", RowParallelLinearWithLoRA),
("layer1.dense1", ColumnParallelLinearWithLoRA),
("layer1.dense2", RowParallelLinearWithLoRA),
],
)
@pytest.mark.parametrize("device", DEVICES)
def test_load_adapter_warns_on_unsupported_modules(
default_vllm_config, dist_init, dummy_model_gate_up, device, tmp_path
+21
View File
@@ -58,3 +58,24 @@ class TestIsInTargetModules:
def test_exact_name_no_match(self):
assert not is_in_target_modules("dense3", ["dense1", "dense2"])
def test_packed_parent_matches_child_target_modules(self):
assert is_in_target_modules(
"model.layers.0.mlp.gate_up_proj",
["gate_proj", "up_proj"],
{"gate_up_proj": ["gate_proj", "up_proj"]},
)
def test_packed_child_matches_parent_target_modules(self):
assert is_in_target_modules(
"model.layers.0.mlp.gate_proj",
["gate_up_proj"],
{"gate_up_proj": ["gate_proj", "up_proj"]},
)
def test_fused_parent_matches_child_target_modules(self):
assert is_in_target_modules(
"model.layers.0.self_attn.fused_qkv_a_proj",
["q_a_proj", "kv_a_proj_with_mqa"],
{"fused_qkv_a_proj": ["q_a_proj", "kv_a_proj_with_mqa"]},
)