[LoRA] Cleanup LoRA unused code (#29611)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Cyrus Leung
parent
4a80ad0a25
commit
39e63dec7c
+8
-19
@@ -154,23 +154,6 @@ def dummy_model_gate_up() -> nn.Module:
|
||||
return model
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def llama_2_7b_base_huggingface_id():
|
||||
# used as a base model for testing with sql lora adapter
|
||||
return "meta-llama/Llama-2-7b-hf"
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def sql_lora_huggingface_id():
|
||||
# huggingface repo id is used to test lora runtime downloading.
|
||||
return "yard1/llama-2-7b-sql-lora-test"
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def sql_lora_files(sql_lora_huggingface_id):
|
||||
return snapshot_download(repo_id=sql_lora_huggingface_id)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def mixtral_lora_files():
|
||||
# Note: this module has incorrect adapter_config.json to test
|
||||
@@ -256,8 +239,14 @@ def qwen3_lora_files():
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def llama32_lora_files():
|
||||
return snapshot_download(repo_id="jeeejeee/llama32-3b-text2sql-spider")
|
||||
def llama32_lora_huggingface_id():
|
||||
# huggingface repo id is used to test lora runtime downloading.
|
||||
return "jeeejeee/llama32-3b-text2sql-spider"
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def llama32_lora_files(llama32_lora_huggingface_id):
|
||||
return snapshot_download(repo_id=llama32_lora_huggingface_id)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
||||
@@ -26,8 +26,7 @@ def test_load_checkpoints(
|
||||
chatglm3_lora_files,
|
||||
):
|
||||
packed_modules_mapping = BaiChuanBaseForCausalLM.packed_modules_mapping
|
||||
embedding_modules = BaiChuanBaseForCausalLM.embedding_modules
|
||||
embed_padding_modules = BaiChuanBaseForCausalLM.embedding_padding_modules
|
||||
|
||||
expected_lora_lst: list[str] = []
|
||||
for module in BAICHUAN_LORA_MODULES:
|
||||
if module in packed_modules_mapping:
|
||||
@@ -47,8 +46,7 @@ def test_load_checkpoints(
|
||||
peft_helper=peft_helper,
|
||||
lora_model_id=1,
|
||||
device="cpu",
|
||||
embedding_modules=embedding_modules,
|
||||
embedding_padding_modules=embed_padding_modules,
|
||||
model_vocab_size=64000,
|
||||
)
|
||||
elif lora_name == "baichuan7B-zero":
|
||||
# Test that the target_modules contain prefix
|
||||
@@ -63,8 +61,7 @@ def test_load_checkpoints(
|
||||
peft_helper=peft_helper,
|
||||
lora_model_id=1,
|
||||
device="cpu",
|
||||
embedding_modules=embedding_modules,
|
||||
embedding_padding_modules=embed_padding_modules,
|
||||
model_vocab_size=64000,
|
||||
)
|
||||
elif lora_name == "baichuan7B-zero-regex":
|
||||
# Test that the `target_modules` in the form of regular expressions,
|
||||
@@ -78,8 +75,7 @@ def test_load_checkpoints(
|
||||
peft_helper=peft_helper,
|
||||
lora_model_id=1,
|
||||
device="cpu",
|
||||
embedding_modules=embedding_modules,
|
||||
embedding_padding_modules=embed_padding_modules,
|
||||
model_vocab_size=64000,
|
||||
)
|
||||
else:
|
||||
# For the baichuan7B model, load chatglm3-6b's LoRA,
|
||||
@@ -95,15 +91,13 @@ def test_load_checkpoints(
|
||||
peft_helper=peft_helper,
|
||||
lora_model_id=1,
|
||||
device="cpu",
|
||||
embedding_modules=embedding_modules,
|
||||
embedding_padding_modules=embed_padding_modules,
|
||||
model_vocab_size=64000,
|
||||
)
|
||||
|
||||
|
||||
def test_lora_weights_mapping(baichuan_lora_files):
|
||||
packed_modules_mapping = BaiChuanBaseForCausalLM.packed_modules_mapping
|
||||
embedding_modules = BaiChuanBaseForCausalLM.embedding_modules
|
||||
embed_padding_modules = BaiChuanBaseForCausalLM.embedding_padding_modules
|
||||
|
||||
expected_lora_lst: list[str] = []
|
||||
for module in BAICHUAN_LORA_MODULES:
|
||||
if module in packed_modules_mapping:
|
||||
@@ -128,8 +122,7 @@ def test_lora_weights_mapping(baichuan_lora_files):
|
||||
peft_helper=peft_helper,
|
||||
lora_model_id=1,
|
||||
device="cpu",
|
||||
embedding_modules=embedding_modules,
|
||||
embedding_padding_modules=embed_padding_modules,
|
||||
model_vocab_size=64000,
|
||||
weights_mapper=hf_to_vllm_mapper,
|
||||
)
|
||||
for name in lora_model.loras:
|
||||
|
||||
@@ -6,10 +6,10 @@ import pytest
|
||||
from vllm.lora.models import LoRAModel
|
||||
from vllm.lora.peft_helper import PEFTHelper
|
||||
from vllm.lora.utils import get_adapter_absolute_path
|
||||
from vllm.model_executor.models.llama import LlamaForCausalLM
|
||||
from vllm.model_executor.models.qwen3 import Qwen3ForCausalLM
|
||||
|
||||
# Provide absolute path and huggingface lora ids
|
||||
lora_fixture_name = ["sql_lora_files", "sql_lora_huggingface_id"]
|
||||
lora_fixture_name = ["llama32_lora_files", "llama32_lora_huggingface_id"]
|
||||
LLAMA_LORA_MODULES = [
|
||||
"qkv_proj",
|
||||
"o_proj",
|
||||
@@ -23,9 +23,8 @@ LLAMA_LORA_MODULES = [
|
||||
@pytest.mark.parametrize("lora_fixture_name", lora_fixture_name)
|
||||
def test_load_checkpoints_from_huggingface(lora_fixture_name, request):
|
||||
lora_name = request.getfixturevalue(lora_fixture_name)
|
||||
packed_modules_mapping = LlamaForCausalLM.packed_modules_mapping
|
||||
embedding_modules = LlamaForCausalLM.embedding_modules
|
||||
embed_padding_modules = LlamaForCausalLM.embedding_padding_modules
|
||||
packed_modules_mapping = Qwen3ForCausalLM.packed_modules_mapping
|
||||
|
||||
expected_lora_lst: list[str] = []
|
||||
for module in LLAMA_LORA_MODULES:
|
||||
if module in packed_modules_mapping:
|
||||
@@ -43,8 +42,6 @@ def test_load_checkpoints_from_huggingface(lora_fixture_name, request):
|
||||
peft_helper=peft_helper,
|
||||
lora_model_id=1,
|
||||
device="cpu",
|
||||
embedding_modules=embedding_modules,
|
||||
embedding_padding_modules=embed_padding_modules,
|
||||
)
|
||||
|
||||
# Assertions to ensure the model is loaded correctly
|
||||
|
||||
@@ -34,7 +34,6 @@ EMBEDDING_MODULES = {
|
||||
"lm_head": "output_embeddings",
|
||||
}
|
||||
|
||||
EMBEDDING_PADDING_MODULES = ["lm_head"]
|
||||
|
||||
DEVICES = (
|
||||
[f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)]
|
||||
@@ -46,24 +45,22 @@ DEFAULT_DTYPE = torch.get_default_dtype()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", DEVICES)
|
||||
def test_from_lora_tensors(sql_lora_files, device):
|
||||
tensors = load_file(os.path.join(sql_lora_files, "adapter_model.safetensors"))
|
||||
def test_from_lora_tensors(qwen3_lora_files, device):
|
||||
tensors = load_file(os.path.join(qwen3_lora_files, "adapter_model.safetensors"))
|
||||
|
||||
peft_helper = PEFTHelper.from_local_dir(
|
||||
sql_lora_files, max_position_embeddings=4096
|
||||
qwen3_lora_files, max_position_embeddings=4096
|
||||
)
|
||||
lora_model = LoRAModel.from_lora_tensors(
|
||||
1,
|
||||
tensors,
|
||||
peft_helper=peft_helper,
|
||||
device=device,
|
||||
embedding_modules=EMBEDDING_MODULES,
|
||||
embedding_padding_modules=EMBEDDING_PADDING_MODULES,
|
||||
)
|
||||
for module_name, lora in lora_model.loras.items():
|
||||
assert lora.module_name == module_name
|
||||
assert lora.rank == 8
|
||||
assert lora.lora_alpha == 16
|
||||
assert lora.lora_alpha == 32
|
||||
assert lora.lora_a is not None
|
||||
assert lora.lora_b is not None
|
||||
assert lora.lora_a.device == torch.device(device)
|
||||
@@ -430,7 +427,7 @@ def test_lru_cache_worker_adapter_manager(dist_init, dummy_model, device, tmp_pa
|
||||
vllm_config.scheduler_config.max_num_seqs = 4
|
||||
vllm_config.scheduler_config.max_num_batched_tokens = 2
|
||||
worker_adapter_manager = LRUCacheWorkerLoRAManager(
|
||||
vllm_config, device, EMBEDDING_MODULES, EMBEDDING_PADDING_MODULES
|
||||
vllm_config, device, EMBEDDING_MODULES
|
||||
)
|
||||
|
||||
worker_adapter_manager.max_num_seqs = 4
|
||||
@@ -533,9 +530,7 @@ def test_worker_adapter_manager(dist_init, dummy_model_gate_up, device, tmp_path
|
||||
vllm_config.scheduler_config.max_num_seqs = 4
|
||||
vllm_config.scheduler_config.max_num_batched_tokens = 2
|
||||
|
||||
worker_adapter_manager = WorkerLoRAManager(
|
||||
vllm_config, device, EMBEDDING_MODULES, EMBEDDING_PADDING_MODULES
|
||||
)
|
||||
worker_adapter_manager = WorkerLoRAManager(vllm_config, device, EMBEDDING_MODULES)
|
||||
worker_adapter_manager.vocab_size = dummy_model_gate_up.unpadded_vocab_size
|
||||
worker_adapter_manager.create_lora_manager(dummy_model_gate_up)
|
||||
|
||||
|
||||
@@ -40,7 +40,10 @@ EXPECTED_BASE_MODEL_OUTPUT = [
|
||||
|
||||
|
||||
def generate_and_test(
|
||||
llm: vllm.LLM, lora_path: str, lora_id: list[int | None] | int | None
|
||||
llm: vllm.LLM,
|
||||
lora_path: str,
|
||||
lora_id: list[int | None] | int | None,
|
||||
compare_lower: bool = False,
|
||||
) -> None:
|
||||
prompts = [
|
||||
PROMPT_TEMPLATE.format(context="How many candidates are there?"),
|
||||
@@ -74,12 +77,18 @@ def generate_and_test(
|
||||
|
||||
for i in range(len(EXPECTED_LORA_OUTPUT)):
|
||||
req_lora_id = lora_id[i] if isinstance(lora_id, list) else lora_id
|
||||
generated_text = generated_texts[i]
|
||||
expected_output = (
|
||||
EXPECTED_LORA_OUTPUT[i]
|
||||
if req_lora_id is not None
|
||||
else EXPECTED_BASE_MODEL_OUTPUT[i]
|
||||
)
|
||||
assert generated_texts[i].startswith(expected_output)
|
||||
|
||||
if compare_lower:
|
||||
generated_text = generated_text.lower()
|
||||
expected_output = expected_output.lower()
|
||||
|
||||
assert generated_text.startswith(expected_output)
|
||||
|
||||
|
||||
def test_olmoe_lora(olmoe_lora_files):
|
||||
@@ -146,6 +155,9 @@ def test_olmoe_lora_tp4(olmoe_lora_files, fully_sharded_loras):
|
||||
tensor_parallel_size=4,
|
||||
fully_sharded_loras=fully_sharded_loras,
|
||||
)
|
||||
|
||||
generate_and_test(llm, olmoe_lora_files, lora_id=1)
|
||||
generate_and_test(llm, olmoe_lora_files, lora_id=2)
|
||||
generate_and_test(
|
||||
llm, olmoe_lora_files, lora_id=1, compare_lower=fully_sharded_loras
|
||||
)
|
||||
generate_and_test(
|
||||
llm, olmoe_lora_files, lora_id=2, compare_lower=fully_sharded_loras
|
||||
)
|
||||
|
||||
@@ -25,31 +25,33 @@ ERROR_CASES = [
|
||||
]
|
||||
|
||||
|
||||
def test_peft_helper_pass(sql_lora_files, tmp_path):
|
||||
def test_peft_helper_pass(llama32_lora_files, tmp_path):
|
||||
peft_helper = PEFTHelper.from_local_dir(
|
||||
sql_lora_files, max_position_embeddings=4096
|
||||
llama32_lora_files, max_position_embeddings=4096
|
||||
)
|
||||
lora_config = LoRAConfig(max_lora_rank=16, max_cpu_loras=3, max_loras=2)
|
||||
peft_helper.validate_legal(lora_config)
|
||||
assert peft_helper.r == 8
|
||||
assert peft_helper.lora_alpha == 16
|
||||
assert peft_helper.target_modules == [
|
||||
"q_proj",
|
||||
"v_proj",
|
||||
"k_proj",
|
||||
"o_proj",
|
||||
"gate_proj",
|
||||
"up_proj",
|
||||
assert peft_helper.lora_alpha == 32
|
||||
target_modules = sorted(peft_helper.target_modules)
|
||||
|
||||
assert target_modules == [
|
||||
"down_proj",
|
||||
"embed_tokens",
|
||||
"gate_proj",
|
||||
"k_proj",
|
||||
"lm_head",
|
||||
"o_proj",
|
||||
"q_proj",
|
||||
"up_proj",
|
||||
"v_proj",
|
||||
]
|
||||
assert peft_helper.vllm_max_position_embeddings == 4096
|
||||
|
||||
# test RSLoRA
|
||||
rslora_config = dict(use_rslora=True)
|
||||
test_dir = tmp_path / "test_rslora"
|
||||
shutil.copytree(sql_lora_files, test_dir)
|
||||
shutil.copytree(llama32_lora_files, test_dir)
|
||||
|
||||
# Load and modify configuration
|
||||
config_path = test_dir / "adapter_config.json"
|
||||
@@ -70,14 +72,14 @@ def test_peft_helper_pass(sql_lora_files, tmp_path):
|
||||
|
||||
@pytest.mark.parametrize("test_name,config_change,expected_error", ERROR_CASES)
|
||||
def test_peft_helper_error(
|
||||
sql_lora_files,
|
||||
llama32_lora_files,
|
||||
tmp_path,
|
||||
test_name: str,
|
||||
config_change: dict,
|
||||
expected_error: str,
|
||||
):
|
||||
test_dir = tmp_path / test_name
|
||||
shutil.copytree(sql_lora_files, test_dir)
|
||||
shutil.copytree(llama32_lora_files, test_dir)
|
||||
|
||||
# Load and modify configuration
|
||||
config_path = test_dir / "adapter_config.json"
|
||||
|
||||
Reference in New Issue
Block a user