[Bugfix][LoRA] Reject PEFT adapter features vLLM does not implement (#60225)
Signed-off-by: suraj <surajsharan85@gmail.com> Co-authored-by: linitra24 <renshuang.zhou@daocloud.io>
This commit is contained in:
co-authored by
linitra24
parent
c17fadcfdc
commit
1ee69dde9c
@@ -23,6 +23,7 @@ BADREQUEST_CASES = [
|
||||
"is greater than max_lora_rank",
|
||||
),
|
||||
("test_dora", {"use_dora": True}, "does not yet support DoRA"),
|
||||
("test_pissa", {"init_lora_weights": "pissa"}, "init_lora_weights='pissa'"),
|
||||
(
|
||||
"test_modules_to_save",
|
||||
{"modules_to_save": ["lm_head"]},
|
||||
|
||||
@@ -24,6 +24,25 @@ ERROR_CASES = [
|
||||
),
|
||||
("test_rank_zero", {"r": 0}, "must be a positive integer"),
|
||||
("test_rank_negative", {"r": -8}, "must be a positive integer"),
|
||||
("test_lora_bias", {"lora_bias": True}, "does not support LoRA bias"),
|
||||
("test_pissa", {"init_lora_weights": "pissa"}, "init_lora_weights='pissa'"),
|
||||
(
|
||||
"test_pissa_niter",
|
||||
{"init_lora_weights": "pissa_niter_4"},
|
||||
"init_lora_weights='pissa_niter_4'",
|
||||
),
|
||||
("test_olora", {"init_lora_weights": "olora"}, "init_lora_weights='olora'"),
|
||||
("test_corda", {"init_lora_weights": "corda"}, "init_lora_weights='corda'"),
|
||||
("test_loftq", {"init_lora_weights": "loftq"}, "init_lora_weights='loftq'"),
|
||||
(
|
||||
"test_unknown_init",
|
||||
{"init_lora_weights": "future_init"},
|
||||
"init_lora_weights='future_init'",
|
||||
),
|
||||
("test_alora", {"alora_invocation_tokens": [1, 2]}, "Activated LoRA"),
|
||||
("test_layer_replication", {"layer_replication": [[0, 2]]}, "layer_replication"),
|
||||
("test_bdlora", {"use_bdlora": {"nblocks": 2}}, "BD-LoRA"),
|
||||
("test_qalora", {"use_qalora": True}, "QALoRA"),
|
||||
]
|
||||
|
||||
|
||||
@@ -114,3 +133,23 @@ def test_peft_helper_invalid_rank_direct(bad_rank: int):
|
||||
"""
|
||||
with pytest.raises(ValueError, match="must be a positive integer"):
|
||||
PEFTHelper(r=bad_rank, lora_alpha=16, target_modules=["q_proj"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"init_lora_weights",
|
||||
[True, False, "gaussian", "eva", "orthogonal", "mica", "lora_ga"],
|
||||
)
|
||||
def test_peft_helper_init_lora_weights_supported(init_lora_weights):
|
||||
"""A saved adapter with these inits loads in PEFT as a plain LoRA."""
|
||||
lora_config = LoRAConfig(max_lora_rank=16, max_cpu_loras=3, max_loras=2)
|
||||
PEFTHelper.from_dict(
|
||||
{
|
||||
"r": 8,
|
||||
"lora_alpha": 16,
|
||||
"target_modules": ["q_proj"],
|
||||
"init_lora_weights": init_lora_weights,
|
||||
"alora_invocation_tokens": None,
|
||||
"layer_replication": None,
|
||||
"use_bdlora": None,
|
||||
}
|
||||
).validate_legal(lora_config)
|
||||
|
||||
@@ -106,6 +106,8 @@ def test_parse_fine_tuned_lora_name_invalid():
|
||||
fixture = {
|
||||
"base_model.weight",
|
||||
"base_model.model.weight",
|
||||
"base_model.model.model.layers.9.mlp.down_proj.lora_A.bias",
|
||||
"base_model.model.model.layers.9.mlp.down_proj.lora_B.bias",
|
||||
}
|
||||
for name in fixture:
|
||||
with pytest.raises(ValueError, match="unsupported LoRA weight"):
|
||||
|
||||
Reference in New Issue
Block a user