[LoRA] Remove tensorizer (#60024)

Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
Jee Jee Li
2026-10-05 07:01:56 -07:00
committed by GitHub
parent d3547f9d03
commit f9c9e8ac24
9 changed files with 30 additions and 375 deletions
@@ -13,13 +13,11 @@ from tests.utils import RemoteOpenAIServer
from vllm.engine.arg_utils import EngineArgs
from vllm.model_executor.model_loader.tensorizer import (
TensorizerConfig,
tensorize_lora_adapter,
tensorize_vllm_model,
)
from vllm.platforms import current_platform
MODEL_NAME = "unsloth/llama-3.2-1b-Instruct"
LORA_PATH = "davzoku/finqa_adapter_1b"
def _cleanup():
@@ -44,11 +42,10 @@ def model_uri(tmp_dir):
@pytest.fixture(scope="module")
def tensorize_model_and_lora(tmp_dir, model_uri):
tensorizer_config = TensorizerConfig(tensorizer_uri=model_uri, lora_dir=tmp_dir)
def tensorize_model(model_uri):
tensorizer_config = TensorizerConfig(tensorizer_uri=model_uri)
args = EngineArgs(model=MODEL_NAME)
tensorize_lora_adapter(LORA_PATH, tensorizer_config)
tensorize_vllm_model(args, tensorizer_config)
# Manually invoke a _cleanup() here, as the cleanup()
@@ -59,7 +56,7 @@ def tensorize_model_and_lora(tmp_dir, model_uri):
@pytest.fixture(scope="module")
def server(model_uri, tensorize_model_and_lora):
def server(model_uri, tensorize_model):
# In this case, model_uri is a directory with a model.tensors
# file and all necessary model artifacts, particularly a
# HF `config.json` file. In this case, Tensorizer can infer the
@@ -72,7 +69,6 @@ def server(model_uri, tensorize_model_and_lora):
"tensorizer",
"--served-model-name",
MODEL_NAME,
"--enable-lora",
]
if current_platform.is_rocm():
args += ["--attention-backend", "TRITON_ATTN"]
+9 -124
View File
@@ -1,18 +1,14 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import subprocess
import sys
import pytest
import vllm
import vllm.config
from vllm import LLM
from vllm.lora.request import LoRARequest
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
from vllm.platforms import current_platform
from ..utils import VLLM_PATH, create_new_process_for_each_test, multi_gpu_test
from ..utils import create_new_process_for_each_test, multi_gpu_test
PROMPT_TEMPLATE = """<|eot_id|><|start_header_id|>user<|end_header_id|>
I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.
@@ -40,7 +36,6 @@ def do_sample(
llm: vllm.LLM,
lora_path: str,
lora_id: int,
tensorizer_config_dict: dict | None = None,
) -> list[str]:
prompts = [
PROMPT_TEMPLATE.format(context="How many candidates are there?"),
@@ -56,27 +51,11 @@ def do_sample(
sampling_params = vllm.SamplingParams(
temperature=0, max_tokens=64, stop=["<|im_end|>"]
)
if tensorizer_config_dict is not None:
outputs = llm.generate(
prompts,
sampling_params,
lora_request=LoRARequest(
str(lora_id),
lora_id,
lora_path,
tensorizer_config_dict=tensorizer_config_dict,
)
if lora_id
else None,
)
else:
outputs = llm.generate(
prompts,
sampling_params,
lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
if lora_id
else None,
)
outputs = llm.generate(
prompts,
sampling_params,
lora_request=LoRARequest(str(lora_id), lora_id, lora_path) if lora_id else None,
)
lora_request = LoRARequest(str(lora_id), lora_id, lora_path) if lora_id else None
generated_texts: list[str] = []
for output in outputs:
@@ -94,31 +73,13 @@ def do_sample(
return generated_texts
def generate_and_test(
llm, llama32_lora_files, tensorizer_config_dict: dict | None = None
):
def generate_and_test(llm, llama32_lora_files):
print("lora adapter created")
print("lora 1")
assert (
do_sample(
llm,
llama32_lora_files,
tensorizer_config_dict=tensorizer_config_dict,
lora_id=1,
)
== EXPECTED_LORA_OUTPUT
)
assert do_sample(llm, llama32_lora_files, lora_id=1) == EXPECTED_LORA_OUTPUT
print("lora 2")
assert (
do_sample(
llm,
llama32_lora_files,
tensorizer_config_dict=tensorizer_config_dict,
lora_id=2,
)
== EXPECTED_LORA_OUTPUT
)
assert do_sample(llm, llama32_lora_files, lora_id=2) == EXPECTED_LORA_OUTPUT
print("removing lora")
@@ -168,79 +129,3 @@ def test_llama_lora_tp4_fully_sharded_loras(llama32_lora_files):
fully_sharded_loras=True,
)
generate_and_test(llm, llama32_lora_files)
@multi_gpu_test(num_gpus=2)
def test_tp2_serialize_and_deserialize_lora(
tmp_path,
llama32_lora_files,
):
# Run the tensorizing of the LoRA adapter and the model in a subprocess
# to guarantee cleanup
tp_size = 2
model_name = "model-rank-%03d.tensors"
model_ref = MODEL_PATH
lora_path = llama32_lora_files
suffix = "test"
try:
result = subprocess.run(
[
sys.executable,
f"{VLLM_PATH}/examples/features/tensorize_vllm_model.py",
"--model",
MODEL_PATH,
"--lora-path",
lora_path,
"--tensor-parallel-size",
str(tp_size),
"serialize",
"--serialized-directory",
str(tmp_path),
"--suffix",
suffix,
"--serialization-kwargs",
'{"limit_cpu_concurrency": 4}',
],
check=True,
capture_output=True,
text=True,
)
except subprocess.CalledProcessError as e:
print("Tensorizing failed.")
print("STDOUT:\n", e.stdout)
print("STDERR:\n", e.stderr)
raise
print("STDOUT:\n", result.stdout)
model_uri = tmp_path / "vllm" / model_ref / suffix / model_name
tensorizer_config = TensorizerConfig(tensorizer_uri=str(model_uri))
loaded_llm = LLM(
model=model_ref,
load_format="tensorizer",
enable_lora=True,
enforce_eager=True,
model_loader_extra_config=tensorizer_config,
max_num_seqs=7,
max_model_len=1024,
tensor_parallel_size=2,
max_loras=2,
# Leave headroom for LoRA adapter loading and Triton JIT
# compilation, which can allocate GPU memory concurrently
# during the first inference step.
gpu_memory_utilization=0.85,
)
tc_as_dict = tensorizer_config.to_serializable()
print("lora adapter created")
print("lora 1")
assert (
do_sample(
loaded_llm, llama32_lora_files, tensorizer_config_dict=tc_as_dict, lora_id=1
)
== EXPECTED_LORA_OUTPUT
)