[Model Runner V2][Bugfix] Fix MRV2 LoRA warmup (#35536)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
co-authored by
Nick Hill
Woosuk Kwon
parent
cd9078fe59
commit
eacff17c8d
@@ -6,6 +6,8 @@ This script contains:
|
||||
2. test multi loras request
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import multi_gpu_test
|
||||
@@ -39,6 +41,18 @@ def format_chatml_messages(
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def set_mrv2_env():
|
||||
original = os.environ.get("VLLM_USE_V2_MODEL_RUNNER", "0")
|
||||
os.environ["VLLM_USE_V2_MODEL_RUNNER"] = "1"
|
||||
yield
|
||||
|
||||
if original is None:
|
||||
os.environ.pop("VLLM_USE_V2_MODEL_RUNNER", None)
|
||||
else:
|
||||
os.environ["VLLM_USE_V2_MODEL_RUNNER"] = original
|
||||
|
||||
|
||||
def make_add_lora_request(name: str, path: str):
|
||||
global INCREASE_LORA_ID, LORA_NAME_ID_MAP
|
||||
|
||||
@@ -61,7 +75,6 @@ def test_multi_loras_with_tp_sync():
|
||||
max_lora_rank=LORA_RANK,
|
||||
max_model_len=512,
|
||||
gpu_memory_utilization=0.5,
|
||||
enforce_eager=True,
|
||||
tensor_parallel_size=2, # ensure tp >= 2
|
||||
max_cpu_loras=4, # ensure max_cpu_loras >= 2
|
||||
)
|
||||
@@ -167,7 +180,6 @@ def test_multiple_lora_requests():
|
||||
max_lora_rank=LORA_RANK,
|
||||
max_model_len=512,
|
||||
gpu_memory_utilization=0.5,
|
||||
enforce_eager=True,
|
||||
)
|
||||
PROMPTS = ["Hello, my name is"] * 2
|
||||
LORA_NAME = "Alice"
|
||||
@@ -203,7 +215,6 @@ def test_load_inplace_offline_reload(
|
||||
max_lora_rank=LORA_RANK,
|
||||
max_model_len=512,
|
||||
gpu_memory_utilization=0.5,
|
||||
enforce_eager=True,
|
||||
)
|
||||
adapter_id = 1
|
||||
messages = format_chatml_messages(
|
||||
@@ -254,7 +265,6 @@ def test_load_inplace_false_no_reload(
|
||||
max_lora_rank=LORA_RANK,
|
||||
max_model_len=512,
|
||||
gpu_memory_utilization=0.5,
|
||||
enforce_eager=True,
|
||||
)
|
||||
adapter_id = 2
|
||||
messages = format_chatml_messages(
|
||||
|
||||
Reference in New Issue
Block a user