[Bugfix][LoRA] Apply PEFT rank_pattern / alpha_pattern scaling per module (#59801)
Signed-off-by: suraj <surajsharan85@gmail.com> Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> Co-authored-by: linitra24 <renshuang.zhou@daocloud.io>
This commit is contained in:
co-authored by
Vadim Gimpelson
linitra24
parent
3facf822b7
commit
4c36d21280
@@ -4,6 +4,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.lora.lora_weights import LoRALayerWeights, PackedLoRALayerWeights
|
||||
from vllm.lora.peft_helper import PEFTHelper
|
||||
|
||||
RANK, ALPHA = 16, 32
|
||||
# PEFTHelper stores alpha/sqrt(r) for use_rslora adapters.
|
||||
@@ -50,3 +51,59 @@ def test_non_gated_moe_keeps_w3_unscaled():
|
||||
)
|
||||
assert packed.scaling[:2] == pytest.approx([RSLORA_SCALING] * 2)
|
||||
assert packed.scaling[2] == 1.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"packer,stacked",
|
||||
[
|
||||
(PackedLoRALayerWeights.pack_moe, False),
|
||||
(PackedLoRALayerWeights.pack_moe_stacked, True),
|
||||
],
|
||||
)
|
||||
def test_moe_packing_keeps_per_projection_scaling(packer, stacked: bool):
|
||||
# e.g. rank_pattern / alpha_pattern giving gate, down and up different scalings
|
||||
loras = moe_loras(1.0, stacked)
|
||||
for lora, scaling in zip(loras, (2.0, 4.0, 8.0)):
|
||||
lora.scaling = scaling
|
||||
packed = packer(loras, "experts")
|
||||
assert packed.scaling == pytest.approx([2.0, 4.0, 8.0])
|
||||
|
||||
|
||||
def test_moe_packing_keeps_per_expert_scaling():
|
||||
"""Experts of one projection can have different scaling, e.g. from
|
||||
alpha_pattern."""
|
||||
loras = moe_loras(RSLORA_SCALING) + moe_loras(RSLORA_SCALING)
|
||||
loras[4].scaling = 2 * RSLORA_SCALING # w2 of expert 1
|
||||
expected = [lora.lora_b * lora.scaling for lora in loras]
|
||||
packed = PackedLoRALayerWeights.pack_moe(loras, "experts").optimize()
|
||||
for i, lora_b in enumerate(packed.lora_b):
|
||||
torch.testing.assert_close(lora_b, torch.stack(expected[i::3]))
|
||||
|
||||
|
||||
def test_moe_packing_rejects_per_expert_rank():
|
||||
"""Experts of one projection must share a rank to be stacked."""
|
||||
loras = moe_loras(1.0) + moe_loras(1.0)
|
||||
loras[4].lora_a = torch.randn(RANK // 2, 4)
|
||||
loras[4].lora_b = torch.randn(6, RANK // 2)
|
||||
with pytest.raises(ValueError, match="different LoRA ranks"):
|
||||
PackedLoRALayerWeights.pack_moe(loras, "experts")
|
||||
|
||||
|
||||
def test_from_config_uses_checkpoint_module_name():
|
||||
peft_helper = PEFTHelper(
|
||||
r=RANK,
|
||||
lora_alpha=ALPHA,
|
||||
target_modules=["q_proj"],
|
||||
rank_pattern={r"^model\.layers\.0\.self_attn\.q_proj": 4},
|
||||
)
|
||||
# the pattern refers to the checkpoint name, not the name after weights mapping
|
||||
lora = LoRALayerWeights.from_config(
|
||||
"language_model.model.layers.0.self_attn.q_proj",
|
||||
peft_helper,
|
||||
"model.layers.0.self_attn.q_proj",
|
||||
)
|
||||
assert lora.rank == 4
|
||||
assert lora.scaling == pytest.approx(ALPHA / 4)
|
||||
lora = LoRALayerWeights.from_config("model.layers.1.self_attn.q_proj", peft_helper)
|
||||
assert lora.rank == RANK
|
||||
assert lora.scaling == pytest.approx(ALPHA / RANK)
|
||||
|
||||
@@ -22,6 +22,11 @@ ERROR_CASES = [
|
||||
{"modules_to_save": ["lm_head"]},
|
||||
"Unsupported modules_to_save",
|
||||
),
|
||||
(
|
||||
"test_rank_pattern",
|
||||
{"r": 8, "rank_pattern": {"q_proj": 32}},
|
||||
"LoRA rank 32 is greater than max_lora_rank",
|
||||
),
|
||||
("test_rank_zero", {"r": 0}, "must be a positive integer"),
|
||||
("test_rank_negative", {"r": -8}, "must be a positive integer"),
|
||||
("test_lora_bias", {"lora_bias": True}, "does not support LoRA bias"),
|
||||
@@ -153,3 +158,103 @@ def test_peft_helper_init_lora_weights_supported(init_lora_weights):
|
||||
"use_bdlora": None,
|
||||
}
|
||||
).validate_legal(lora_config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"config,module_name,expected",
|
||||
[
|
||||
# rank_pattern only: alpha / r_m
|
||||
({"rank_pattern": {"q_proj": 4}}, "model.layers.0.self_attn.q_proj", (4, 8.0)),
|
||||
({"rank_pattern": {"q_proj": 4}}, "model.layers.0.self_attn.v_proj", (16, 2.0)),
|
||||
# alpha_pattern only: alpha_m / r
|
||||
(
|
||||
{"alpha_pattern": {"v_proj": 64}},
|
||||
"model.layers.0.self_attn.v_proj",
|
||||
(16, 4.0),
|
||||
),
|
||||
# keys only match at a "." boundary, like in PEFT
|
||||
({"rank_pattern": {"proj": 4}}, "model.layers.0.self_attn.q_proj", (16, 2.0)),
|
||||
# fused MoE experts from PEFT `target_parameters`: gate_up_proj is stored
|
||||
# as `experts.base_layer`, down_proj as `experts`
|
||||
(
|
||||
{"rank_pattern": {"experts.gate_up_proj": 2}},
|
||||
"model.layers.0.mlp.experts.base_layer",
|
||||
(2, 16.0),
|
||||
),
|
||||
(
|
||||
{"rank_pattern": {"experts.down_proj": 2}},
|
||||
"model.layers.0.mlp.experts",
|
||||
(2, 16.0),
|
||||
),
|
||||
(
|
||||
{"rank_pattern": {"experts.gate_up_proj": 2}},
|
||||
"model.layers.0.mlp.experts",
|
||||
(16, 2.0),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_peft_helper_rank_and_alpha_pattern(config, module_name, expected):
|
||||
peft_helper = PEFTHelper(
|
||||
r=16, lora_alpha=32, target_modules=["q_proj", "v_proj"], **config
|
||||
)
|
||||
rank, scaling = peft_helper.get_rank_and_scaling(module_name)
|
||||
assert rank == expected[0]
|
||||
assert scaling == pytest.approx(expected[1])
|
||||
|
||||
|
||||
def test_peft_helper_pattern_rslora():
|
||||
peft_helper = PEFTHelper(
|
||||
r=16,
|
||||
lora_alpha=32,
|
||||
target_modules=["q_proj"],
|
||||
use_rslora=True,
|
||||
rank_pattern={"q_proj": 4},
|
||||
)
|
||||
assert peft_helper.get_rank_and_scaling("model.q_proj") == (4, 32 / math.sqrt(4))
|
||||
|
||||
|
||||
def test_peft_helper_dynamic_rank_conversion():
|
||||
"""PEFT's `save_as_lora` with a dynamic rank writes r=1, lora_alpha=1 and the
|
||||
same per-module values in rank_pattern and alpha_pattern, so every module
|
||||
keeps a scaling of 1."""
|
||||
peft_helper = PEFTHelper(
|
||||
r=1,
|
||||
lora_alpha=1,
|
||||
target_modules=["q_proj", "v_proj"],
|
||||
rank_pattern={"layers.0.self_attn.q_proj": 32, "layers.0.self_attn.v_proj": 3},
|
||||
alpha_pattern={"layers.0.self_attn.q_proj": 32, "layers.0.self_attn.v_proj": 3},
|
||||
)
|
||||
assert peft_helper.get_rank_and_scaling("model.layers.0.self_attn.q_proj") == (
|
||||
32,
|
||||
1.0,
|
||||
)
|
||||
assert peft_helper.get_rank_and_scaling("model.layers.0.self_attn.v_proj") == (
|
||||
3,
|
||||
1.0,
|
||||
)
|
||||
with pytest.raises(ValueError, match="LoRA rank 32 is greater than max_lora_rank"):
|
||||
peft_helper.validate_legal(LoRAConfig(max_lora_rank=16))
|
||||
|
||||
|
||||
def test_peft_helper_null_patterns():
|
||||
peft_helper = PEFTHelper.from_dict(
|
||||
{
|
||||
"r": 8,
|
||||
"lora_alpha": 16,
|
||||
"target_modules": ["q_proj"],
|
||||
"rank_pattern": None,
|
||||
"alpha_pattern": None,
|
||||
}
|
||||
)
|
||||
assert peft_helper.get_rank_and_scaling("model.q_proj") == (8, 2.0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bad_rank", [0, -4, 2.5])
|
||||
def test_peft_helper_invalid_rank_pattern(bad_rank):
|
||||
with pytest.raises(ValueError, match="rank_pattern"):
|
||||
PEFTHelper(
|
||||
r=8,
|
||||
lora_alpha=16,
|
||||
target_modules=["q_proj"],
|
||||
rank_pattern={"q_proj": bad_rank},
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user