[LoRA] Remove tensorizer (#60024)
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
@@ -13,13 +13,11 @@ from tests.utils import RemoteOpenAIServer
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.model_executor.model_loader.tensorizer import (
|
||||
TensorizerConfig,
|
||||
tensorize_lora_adapter,
|
||||
tensorize_vllm_model,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
MODEL_NAME = "unsloth/llama-3.2-1b-Instruct"
|
||||
LORA_PATH = "davzoku/finqa_adapter_1b"
|
||||
|
||||
|
||||
def _cleanup():
|
||||
@@ -44,11 +42,10 @@ def model_uri(tmp_dir):
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def tensorize_model_and_lora(tmp_dir, model_uri):
|
||||
tensorizer_config = TensorizerConfig(tensorizer_uri=model_uri, lora_dir=tmp_dir)
|
||||
def tensorize_model(model_uri):
|
||||
tensorizer_config = TensorizerConfig(tensorizer_uri=model_uri)
|
||||
args = EngineArgs(model=MODEL_NAME)
|
||||
|
||||
tensorize_lora_adapter(LORA_PATH, tensorizer_config)
|
||||
tensorize_vllm_model(args, tensorizer_config)
|
||||
|
||||
# Manually invoke a _cleanup() here, as the cleanup()
|
||||
@@ -59,7 +56,7 @@ def tensorize_model_and_lora(tmp_dir, model_uri):
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(model_uri, tensorize_model_and_lora):
|
||||
def server(model_uri, tensorize_model):
|
||||
# In this case, model_uri is a directory with a model.tensors
|
||||
# file and all necessary model artifacts, particularly a
|
||||
# HF `config.json` file. In this case, Tensorizer can infer the
|
||||
@@ -72,7 +69,6 @@ def server(model_uri, tensorize_model_and_lora):
|
||||
"tensorizer",
|
||||
"--served-model-name",
|
||||
MODEL_NAME,
|
||||
"--enable-lora",
|
||||
]
|
||||
if current_platform.is_rocm():
|
||||
args += ["--attention-backend", "TRITON_ATTN"]
|
||||
|
||||
+9
-124
@@ -1,18 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
import vllm
|
||||
import vllm.config
|
||||
from vllm import LLM
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..utils import VLLM_PATH, create_new_process_for_each_test, multi_gpu_test
|
||||
from ..utils import create_new_process_for_each_test, multi_gpu_test
|
||||
|
||||
PROMPT_TEMPLATE = """<|eot_id|><|start_header_id|>user<|end_header_id|>
|
||||
I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.
|
||||
@@ -40,7 +36,6 @@ def do_sample(
|
||||
llm: vllm.LLM,
|
||||
lora_path: str,
|
||||
lora_id: int,
|
||||
tensorizer_config_dict: dict | None = None,
|
||||
) -> list[str]:
|
||||
prompts = [
|
||||
PROMPT_TEMPLATE.format(context="How many candidates are there?"),
|
||||
@@ -56,27 +51,11 @@ def do_sample(
|
||||
sampling_params = vllm.SamplingParams(
|
||||
temperature=0, max_tokens=64, stop=["<|im_end|>"]
|
||||
)
|
||||
if tensorizer_config_dict is not None:
|
||||
outputs = llm.generate(
|
||||
prompts,
|
||||
sampling_params,
|
||||
lora_request=LoRARequest(
|
||||
str(lora_id),
|
||||
lora_id,
|
||||
lora_path,
|
||||
tensorizer_config_dict=tensorizer_config_dict,
|
||||
)
|
||||
if lora_id
|
||||
else None,
|
||||
)
|
||||
else:
|
||||
outputs = llm.generate(
|
||||
prompts,
|
||||
sampling_params,
|
||||
lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
|
||||
if lora_id
|
||||
else None,
|
||||
)
|
||||
outputs = llm.generate(
|
||||
prompts,
|
||||
sampling_params,
|
||||
lora_request=LoRARequest(str(lora_id), lora_id, lora_path) if lora_id else None,
|
||||
)
|
||||
lora_request = LoRARequest(str(lora_id), lora_id, lora_path) if lora_id else None
|
||||
generated_texts: list[str] = []
|
||||
for output in outputs:
|
||||
@@ -94,31 +73,13 @@ def do_sample(
|
||||
return generated_texts
|
||||
|
||||
|
||||
def generate_and_test(
|
||||
llm, llama32_lora_files, tensorizer_config_dict: dict | None = None
|
||||
):
|
||||
def generate_and_test(llm, llama32_lora_files):
|
||||
print("lora adapter created")
|
||||
print("lora 1")
|
||||
assert (
|
||||
do_sample(
|
||||
llm,
|
||||
llama32_lora_files,
|
||||
tensorizer_config_dict=tensorizer_config_dict,
|
||||
lora_id=1,
|
||||
)
|
||||
== EXPECTED_LORA_OUTPUT
|
||||
)
|
||||
assert do_sample(llm, llama32_lora_files, lora_id=1) == EXPECTED_LORA_OUTPUT
|
||||
|
||||
print("lora 2")
|
||||
assert (
|
||||
do_sample(
|
||||
llm,
|
||||
llama32_lora_files,
|
||||
tensorizer_config_dict=tensorizer_config_dict,
|
||||
lora_id=2,
|
||||
)
|
||||
== EXPECTED_LORA_OUTPUT
|
||||
)
|
||||
assert do_sample(llm, llama32_lora_files, lora_id=2) == EXPECTED_LORA_OUTPUT
|
||||
|
||||
print("removing lora")
|
||||
|
||||
@@ -168,79 +129,3 @@ def test_llama_lora_tp4_fully_sharded_loras(llama32_lora_files):
|
||||
fully_sharded_loras=True,
|
||||
)
|
||||
generate_and_test(llm, llama32_lora_files)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_tp2_serialize_and_deserialize_lora(
|
||||
tmp_path,
|
||||
llama32_lora_files,
|
||||
):
|
||||
# Run the tensorizing of the LoRA adapter and the model in a subprocess
|
||||
# to guarantee cleanup
|
||||
|
||||
tp_size = 2
|
||||
model_name = "model-rank-%03d.tensors"
|
||||
|
||||
model_ref = MODEL_PATH
|
||||
lora_path = llama32_lora_files
|
||||
suffix = "test"
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
f"{VLLM_PATH}/examples/features/tensorize_vllm_model.py",
|
||||
"--model",
|
||||
MODEL_PATH,
|
||||
"--lora-path",
|
||||
lora_path,
|
||||
"--tensor-parallel-size",
|
||||
str(tp_size),
|
||||
"serialize",
|
||||
"--serialized-directory",
|
||||
str(tmp_path),
|
||||
"--suffix",
|
||||
suffix,
|
||||
"--serialization-kwargs",
|
||||
'{"limit_cpu_concurrency": 4}',
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print("Tensorizing failed.")
|
||||
print("STDOUT:\n", e.stdout)
|
||||
print("STDERR:\n", e.stderr)
|
||||
raise
|
||||
|
||||
print("STDOUT:\n", result.stdout)
|
||||
|
||||
model_uri = tmp_path / "vllm" / model_ref / suffix / model_name
|
||||
tensorizer_config = TensorizerConfig(tensorizer_uri=str(model_uri))
|
||||
|
||||
loaded_llm = LLM(
|
||||
model=model_ref,
|
||||
load_format="tensorizer",
|
||||
enable_lora=True,
|
||||
enforce_eager=True,
|
||||
model_loader_extra_config=tensorizer_config,
|
||||
max_num_seqs=7,
|
||||
max_model_len=1024,
|
||||
tensor_parallel_size=2,
|
||||
max_loras=2,
|
||||
# Leave headroom for LoRA adapter loading and Triton JIT
|
||||
# compilation, which can allocate GPU memory concurrently
|
||||
# during the first inference step.
|
||||
gpu_memory_utilization=0.85,
|
||||
)
|
||||
|
||||
tc_as_dict = tensorizer_config.to_serializable()
|
||||
|
||||
print("lora adapter created")
|
||||
print("lora 1")
|
||||
assert (
|
||||
do_sample(
|
||||
loaded_llm, llama32_lora_files, tensorizer_config_dict=tc_as_dict, lora_id=1
|
||||
)
|
||||
== EXPECTED_LORA_OUTPUT
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user