[Misc][LoRA] Support Rank Stabilized LoRA (RSLoRA) (#6909)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
co-authored by
Jee Jee Li
parent
74fa1d123c
commit
82c49d3260
@@ -1,4 +1,5 @@
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from typing import Dict, List
|
||||
|
||||
@@ -50,6 +51,18 @@ def test_peft_helper(sql_lora_files):
|
||||
"embed_tokens",
|
||||
"lm_head",
|
||||
]
|
||||
scaling = peft_helper.lora_alpha / peft_helper.r
|
||||
assert abs(peft_helper.vllm_lora_scaling_factor - scaling) < 1e-3
|
||||
|
||||
# test RSLoRA
|
||||
config = dict(r=8,
|
||||
lora_alpha=16,
|
||||
target_modules=["gate_proj"],
|
||||
use_rslora=True)
|
||||
peft_helper = PEFTHelper.from_dict(config)
|
||||
|
||||
scaling = peft_helper.lora_alpha / math.sqrt(peft_helper.r)
|
||||
assert abs(peft_helper.vllm_lora_scaling_factor - scaling) < 1e-3
|
||||
|
||||
expected_error = "vLLM only supports modules_to_save being None."
|
||||
with pytest.raises(ValueError, match=expected_error):
|
||||
@@ -60,13 +73,6 @@ def test_peft_helper(sql_lora_files):
|
||||
modules_to_save=["lm_head"],
|
||||
)
|
||||
PEFTHelper.from_dict(config)
|
||||
expected_error = "vLLM does not yet support RSLoRA."
|
||||
with pytest.raises(ValueError, match=expected_error):
|
||||
config = dict(r=8,
|
||||
lora_alpha=16,
|
||||
target_modules=["gate_proj"],
|
||||
use_rslora=True)
|
||||
PEFTHelper.from_dict(config)
|
||||
|
||||
expected_error = "vLLM does not yet support DoRA."
|
||||
with pytest.raises(ValueError, match=expected_error):
|
||||
|
||||
Reference in New Issue
Block a user