[Bugfix][LoRA] Apply PEFT rank_pattern / alpha_pattern scaling per module (#59801)

Signed-off-by: suraj <surajsharan85@gmail.com>
Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
Co-authored-by: linitra24 <renshuang.zhou@daocloud.io>
This commit is contained in:
Suraj Sharan
2026-10-12 21:49:30 +08:00
committed by GitHub
co-authored by Vadim Gimpelson linitra24
parent 3facf822b7
commit 4c36d21280
5 changed files with 249 additions and 15 deletions
+57
View File
@@ -4,6 +4,7 @@ import pytest
import torch
from vllm.lora.lora_weights import LoRALayerWeights, PackedLoRALayerWeights
from vllm.lora.peft_helper import PEFTHelper
RANK, ALPHA = 16, 32
# PEFTHelper stores alpha/sqrt(r) for use_rslora adapters.
@@ -50,3 +51,59 @@ def test_non_gated_moe_keeps_w3_unscaled():
)
assert packed.scaling[:2] == pytest.approx([RSLORA_SCALING] * 2)
assert packed.scaling[2] == 1.0
@pytest.mark.parametrize(
"packer,stacked",
[
(PackedLoRALayerWeights.pack_moe, False),
(PackedLoRALayerWeights.pack_moe_stacked, True),
],
)
def test_moe_packing_keeps_per_projection_scaling(packer, stacked: bool):
# e.g. rank_pattern / alpha_pattern giving gate, down and up different scalings
loras = moe_loras(1.0, stacked)
for lora, scaling in zip(loras, (2.0, 4.0, 8.0)):
lora.scaling = scaling
packed = packer(loras, "experts")
assert packed.scaling == pytest.approx([2.0, 4.0, 8.0])
def test_moe_packing_keeps_per_expert_scaling():
"""Experts of one projection can have different scaling, e.g. from
alpha_pattern."""
loras = moe_loras(RSLORA_SCALING) + moe_loras(RSLORA_SCALING)
loras[4].scaling = 2 * RSLORA_SCALING # w2 of expert 1
expected = [lora.lora_b * lora.scaling for lora in loras]
packed = PackedLoRALayerWeights.pack_moe(loras, "experts").optimize()
for i, lora_b in enumerate(packed.lora_b):
torch.testing.assert_close(lora_b, torch.stack(expected[i::3]))
def test_moe_packing_rejects_per_expert_rank():
"""Experts of one projection must share a rank to be stacked."""
loras = moe_loras(1.0) + moe_loras(1.0)
loras[4].lora_a = torch.randn(RANK // 2, 4)
loras[4].lora_b = torch.randn(6, RANK // 2)
with pytest.raises(ValueError, match="different LoRA ranks"):
PackedLoRALayerWeights.pack_moe(loras, "experts")
def test_from_config_uses_checkpoint_module_name():
peft_helper = PEFTHelper(
r=RANK,
lora_alpha=ALPHA,
target_modules=["q_proj"],
rank_pattern={r"^model\.layers\.0\.self_attn\.q_proj": 4},
)
# the pattern refers to the checkpoint name, not the name after weights mapping
lora = LoRALayerWeights.from_config(
"language_model.model.layers.0.self_attn.q_proj",
peft_helper,
"model.layers.0.self_attn.q_proj",
)
assert lora.rank == 4
assert lora.scaling == pytest.approx(ALPHA / 4)
lora = LoRALayerWeights.from_config("model.layers.1.self_attn.q_proj", peft_helper)
assert lora.rank == RANK
assert lora.scaling == pytest.approx(ALPHA / RANK)
+105
View File
@@ -22,6 +22,11 @@ ERROR_CASES = [
{"modules_to_save": ["lm_head"]},
"Unsupported modules_to_save",
),
(
"test_rank_pattern",
{"r": 8, "rank_pattern": {"q_proj": 32}},
"LoRA rank 32 is greater than max_lora_rank",
),
("test_rank_zero", {"r": 0}, "must be a positive integer"),
("test_rank_negative", {"r": -8}, "must be a positive integer"),
("test_lora_bias", {"lora_bias": True}, "does not support LoRA bias"),
@@ -153,3 +158,103 @@ def test_peft_helper_init_lora_weights_supported(init_lora_weights):
"use_bdlora": None,
}
).validate_legal(lora_config)
@pytest.mark.parametrize(
"config,module_name,expected",
[
# rank_pattern only: alpha / r_m
({"rank_pattern": {"q_proj": 4}}, "model.layers.0.self_attn.q_proj", (4, 8.0)),
({"rank_pattern": {"q_proj": 4}}, "model.layers.0.self_attn.v_proj", (16, 2.0)),
# alpha_pattern only: alpha_m / r
(
{"alpha_pattern": {"v_proj": 64}},
"model.layers.0.self_attn.v_proj",
(16, 4.0),
),
# keys only match at a "." boundary, like in PEFT
({"rank_pattern": {"proj": 4}}, "model.layers.0.self_attn.q_proj", (16, 2.0)),
# fused MoE experts from PEFT `target_parameters`: gate_up_proj is stored
# as `experts.base_layer`, down_proj as `experts`
(
{"rank_pattern": {"experts.gate_up_proj": 2}},
"model.layers.0.mlp.experts.base_layer",
(2, 16.0),
),
(
{"rank_pattern": {"experts.down_proj": 2}},
"model.layers.0.mlp.experts",
(2, 16.0),
),
(
{"rank_pattern": {"experts.gate_up_proj": 2}},
"model.layers.0.mlp.experts",
(16, 2.0),
),
],
)
def test_peft_helper_rank_and_alpha_pattern(config, module_name, expected):
peft_helper = PEFTHelper(
r=16, lora_alpha=32, target_modules=["q_proj", "v_proj"], **config
)
rank, scaling = peft_helper.get_rank_and_scaling(module_name)
assert rank == expected[0]
assert scaling == pytest.approx(expected[1])
def test_peft_helper_pattern_rslora():
peft_helper = PEFTHelper(
r=16,
lora_alpha=32,
target_modules=["q_proj"],
use_rslora=True,
rank_pattern={"q_proj": 4},
)
assert peft_helper.get_rank_and_scaling("model.q_proj") == (4, 32 / math.sqrt(4))
def test_peft_helper_dynamic_rank_conversion():
"""PEFT's `save_as_lora` with a dynamic rank writes r=1, lora_alpha=1 and the
same per-module values in rank_pattern and alpha_pattern, so every module
keeps a scaling of 1."""
peft_helper = PEFTHelper(
r=1,
lora_alpha=1,
target_modules=["q_proj", "v_proj"],
rank_pattern={"layers.0.self_attn.q_proj": 32, "layers.0.self_attn.v_proj": 3},
alpha_pattern={"layers.0.self_attn.q_proj": 32, "layers.0.self_attn.v_proj": 3},
)
assert peft_helper.get_rank_and_scaling("model.layers.0.self_attn.q_proj") == (
32,
1.0,
)
assert peft_helper.get_rank_and_scaling("model.layers.0.self_attn.v_proj") == (
3,
1.0,
)
with pytest.raises(ValueError, match="LoRA rank 32 is greater than max_lora_rank"):
peft_helper.validate_legal(LoRAConfig(max_lora_rank=16))
def test_peft_helper_null_patterns():
peft_helper = PEFTHelper.from_dict(
{
"r": 8,
"lora_alpha": 16,
"target_modules": ["q_proj"],
"rank_pattern": None,
"alpha_pattern": None,
}
)
assert peft_helper.get_rank_and_scaling("model.q_proj") == (8, 2.0)
@pytest.mark.parametrize("bad_rank", [0, -4, 2.5])
def test_peft_helper_invalid_rank_pattern(bad_rank):
with pytest.raises(ValueError, match="rank_pattern"):
PEFTHelper(
r=8,
lora_alpha=16,
target_modules=["q_proj"],
rank_pattern={"q_proj": bad_rank},
)