[Core] Remove tokenizer group in vLLM (#24078)
Signed-off-by: Zhuohan Li <zhuohan123@gmail.com>
This commit is contained in:
@@ -13,14 +13,6 @@ from ..utils import VLLM_PATH, create_new_process_for_each_test, multi_gpu_test
|
||||
|
||||
MODEL_PATH = "meta-llama/Llama-2-7b-hf"
|
||||
|
||||
EXPECTED_NO_LORA_OUTPUT = [
|
||||
"\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_75 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_76 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_77 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_78 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user]", # noqa: E501
|
||||
" Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? ", # noqa: E501
|
||||
"\n\n answer: 1\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_96 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_97 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_98 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one m", # noqa: E501
|
||||
" Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. ", # noqa: E501
|
||||
" Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_60 (pick INTEGER, former_wnba_team VARCHAR)\n\n question: What pick was a player that previously played for the Minnesota Lynx? ", # noqa: E501
|
||||
"\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE", # noqa: E501
|
||||
]
|
||||
EXPECTED_LORA_OUTPUT = [
|
||||
" SELECT icao FROM table_name_74 WHERE airport = 'lilongwe international airport' ", # noqa: E501
|
||||
" SELECT nationality FROM table_name_11 WHERE elector = 'anchero pantaleone' ", # noqa: E501
|
||||
@@ -79,23 +71,12 @@ def generate_and_test(llm,
|
||||
sql_lora_files,
|
||||
tensorizer_config_dict: Union[dict, None] = None):
|
||||
print("lora adapter created")
|
||||
assert do_sample(llm,
|
||||
sql_lora_files,
|
||||
tensorizer_config_dict=tensorizer_config_dict,
|
||||
lora_id=0) == EXPECTED_NO_LORA_OUTPUT
|
||||
|
||||
print("lora 1")
|
||||
assert do_sample(llm,
|
||||
sql_lora_files,
|
||||
tensorizer_config_dict=tensorizer_config_dict,
|
||||
lora_id=1) == EXPECTED_LORA_OUTPUT
|
||||
|
||||
print("no lora")
|
||||
assert do_sample(llm,
|
||||
sql_lora_files,
|
||||
tensorizer_config_dict=tensorizer_config_dict,
|
||||
lora_id=0) == EXPECTED_NO_LORA_OUTPUT
|
||||
|
||||
print("lora 2")
|
||||
assert do_sample(llm,
|
||||
sql_lora_files,
|
||||
@@ -110,6 +91,7 @@ def test_llama_lora(sql_lora_files):
|
||||
|
||||
llm = vllm.LLM(
|
||||
MODEL_PATH,
|
||||
tokenizer=sql_lora_files,
|
||||
enable_lora=True,
|
||||
# also test odd max_num_seqs
|
||||
max_num_seqs=13,
|
||||
@@ -123,6 +105,7 @@ def test_llama_lora_tp4(sql_lora_files):
|
||||
|
||||
llm = vllm.LLM(
|
||||
MODEL_PATH,
|
||||
tokenizer=sql_lora_files,
|
||||
enable_lora=True,
|
||||
max_num_seqs=16,
|
||||
max_loras=4,
|
||||
@@ -137,6 +120,7 @@ def test_llama_lora_tp4_fully_sharded_loras(sql_lora_files):
|
||||
|
||||
llm = vllm.LLM(
|
||||
MODEL_PATH,
|
||||
tokenizer=sql_lora_files,
|
||||
enable_lora=True,
|
||||
max_num_seqs=16,
|
||||
max_loras=4,
|
||||
@@ -184,6 +168,7 @@ def test_tp2_serialize_and_deserialize_lora(tmp_path, sql_lora_files,
|
||||
tensorizer_config = TensorizerConfig(tensorizer_uri=str(model_uri))
|
||||
|
||||
loaded_llm = LLM(model=model_ref,
|
||||
tokenizer=sql_lora_files,
|
||||
load_format="tensorizer",
|
||||
enable_lora=True,
|
||||
enforce_eager=True,
|
||||
@@ -195,11 +180,6 @@ def test_tp2_serialize_and_deserialize_lora(tmp_path, sql_lora_files,
|
||||
tc_as_dict = tensorizer_config.to_serializable()
|
||||
|
||||
print("lora adapter created")
|
||||
assert do_sample(loaded_llm,
|
||||
sql_lora_files,
|
||||
tensorizer_config_dict=tc_as_dict,
|
||||
lora_id=0) == EXPECTED_NO_LORA_OUTPUT
|
||||
|
||||
print("lora 1")
|
||||
assert do_sample(loaded_llm,
|
||||
sql_lora_files,
|
||||
|
||||
@@ -1,135 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import CacheConfig, DeviceConfig, ModelConfig, VllmConfig
|
||||
from vllm.config.lora import LoRAConfig
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
|
||||
from vllm.v1.engine.processor import Processor
|
||||
|
||||
|
||||
def test_allowed_token_ids_with_lora_vocab(llama_2_7b_base_huggingface_id,
|
||||
sql_lora_files):
|
||||
"""
|
||||
Test that we properly resolve the range of allowed token ids for lora
|
||||
adapters that define additional tokens.
|
||||
"""
|
||||
|
||||
# Set up a base model compatible with the sql_lora_files adapter and
|
||||
# a known number of tokens in the base model.
|
||||
model_config = ModelConfig(
|
||||
model=llama_2_7b_base_huggingface_id,
|
||||
tokenizer=llama_2_7b_base_huggingface_id,
|
||||
tokenizer_mode="auto",
|
||||
)
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
model_config=model_config,
|
||||
cache_config=CacheConfig(),
|
||||
device_config=DeviceConfig(),
|
||||
lora_config=LoRAConfig(),
|
||||
)
|
||||
|
||||
tokenizer = init_tokenizer_from_configs(
|
||||
model_config=vllm_config.model_config,
|
||||
scheduler_config=vllm_config.scheduler_config,
|
||||
lora_config=vllm_config.lora_config)
|
||||
processor = Processor(vllm_config, tokenizer)
|
||||
|
||||
lora_request = LoRARequest("1", 1, str(sql_lora_files))
|
||||
request_id = "1"
|
||||
prompt = "a prompt"
|
||||
|
||||
# tokens added in the lora adapter should not raise an error
|
||||
lora_token_ids = [32000, 32001, 32002, 32003]
|
||||
processor.process_inputs(
|
||||
request_id,
|
||||
prompt,
|
||||
params=SamplingParams(allowed_token_ids=lora_token_ids),
|
||||
lora_request=lora_request)
|
||||
|
||||
# tokens in the base model should not raise an error
|
||||
base_token_ids = [1000, 1001, 1002, 1003]
|
||||
processor.process_inputs(
|
||||
request_id,
|
||||
prompt,
|
||||
params=SamplingParams(allowed_token_ids=base_token_ids),
|
||||
lora_request=lora_request)
|
||||
|
||||
# tokens not in the lora adapter should raise an error
|
||||
invalid_token_ids = [35000, 35001, 35002, 35003]
|
||||
with pytest.raises(ValueError):
|
||||
processor.process_inputs(
|
||||
request_id,
|
||||
prompt,
|
||||
params=SamplingParams(allowed_token_ids=invalid_token_ids),
|
||||
lora_request=lora_request)
|
||||
|
||||
# tokens in the lora adapter with no lora request should raise an error
|
||||
with pytest.raises(ValueError):
|
||||
processor.process_inputs(
|
||||
request_id,
|
||||
prompt,
|
||||
params=SamplingParams(allowed_token_ids=lora_token_ids),
|
||||
)
|
||||
|
||||
|
||||
def test_allowed_token_ids_with_lora_adapter_no_vocab(
|
||||
qwen25vl_base_huggingface_id, qwen25vl_lora_files):
|
||||
"""
|
||||
Test that we properly resolve the range of allowed token ids for lora
|
||||
adapters that do not define additional tokens.
|
||||
"""
|
||||
|
||||
# Set up a base model compatible with the qwen25vl_lora_files adapter and
|
||||
# a known number of tokens in the base model.
|
||||
model_config = ModelConfig(
|
||||
model=qwen25vl_base_huggingface_id,
|
||||
tokenizer=qwen25vl_base_huggingface_id,
|
||||
tokenizer_mode="auto",
|
||||
)
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
model_config=model_config,
|
||||
cache_config=CacheConfig(),
|
||||
device_config=DeviceConfig(),
|
||||
lora_config=LoRAConfig(),
|
||||
)
|
||||
|
||||
tokenizer = init_tokenizer_from_configs(
|
||||
model_config=vllm_config.model_config,
|
||||
scheduler_config=vllm_config.scheduler_config,
|
||||
lora_config=vllm_config.lora_config)
|
||||
processor = Processor(vllm_config, tokenizer)
|
||||
|
||||
lora_request = LoRARequest("1", 1, str(qwen25vl_lora_files))
|
||||
request_id = "1"
|
||||
prompt = "a prompt"
|
||||
|
||||
# tokens in the base model should not raise an error
|
||||
base_token_ids = [1000, 1001, 1002, 1003]
|
||||
processor.process_inputs(
|
||||
request_id,
|
||||
prompt,
|
||||
params=SamplingParams(allowed_token_ids=base_token_ids),
|
||||
lora_request=lora_request)
|
||||
|
||||
# tokens in the base model with no lora request should not raise an error
|
||||
base_token_ids = [1000, 1001, 1002, 1003]
|
||||
processor.process_inputs(
|
||||
request_id,
|
||||
prompt,
|
||||
params=SamplingParams(allowed_token_ids=base_token_ids),
|
||||
)
|
||||
|
||||
# tokens not in the base model should raise an error
|
||||
invalid_token_ids = [200000, 200001, 200002, 200003]
|
||||
with pytest.raises(ValueError):
|
||||
processor.process_inputs(
|
||||
request_id,
|
||||
prompt,
|
||||
params=SamplingParams(allowed_token_ids=invalid_token_ids),
|
||||
lora_request=lora_request)
|
||||
@@ -82,31 +82,20 @@ def test_quant_model_lora(tinyllama_lora_files, model):
|
||||
gpu_memory_utilization=0.2, #avoid OOM
|
||||
quantization=model.quantization,
|
||||
trust_remote_code=True,
|
||||
enable_chunked_prefill=True)
|
||||
enable_chunked_prefill=True,
|
||||
tokenizer=tinyllama_lora_files)
|
||||
|
||||
if model.quantization is None:
|
||||
expected_no_lora_output = [
|
||||
"Here are some examples of orange-brown colors",
|
||||
"I'm sorry, I don't have"
|
||||
]
|
||||
expected_lora_output = [
|
||||
"#ff8050",
|
||||
"#ff8080",
|
||||
]
|
||||
elif model.quantization == "awq":
|
||||
expected_no_lora_output = [
|
||||
"I'm sorry, I don't understand",
|
||||
"I'm sorry, I don't understand",
|
||||
]
|
||||
expected_lora_output = [
|
||||
"#f07700: A v",
|
||||
"#f00000: A v",
|
||||
]
|
||||
elif model.quantization == "gptq":
|
||||
expected_no_lora_output = [
|
||||
"I'm sorry, I don't have",
|
||||
"I'm sorry, I don't have",
|
||||
]
|
||||
expected_lora_output = [
|
||||
"#f08800: This is",
|
||||
"#f07788 \n#",
|
||||
@@ -117,7 +106,6 @@ def test_quant_model_lora(tinyllama_lora_files, model):
|
||||
# Assert that the outputs changed.
|
||||
if (model.quantization == "gptq"
|
||||
and expected_output is expected_lora_output):
|
||||
assert output != expected_no_lora_output
|
||||
for i, o in enumerate(output):
|
||||
assert o.startswith(
|
||||
'#'), f"Expected example {i} to start with # but got {o}"
|
||||
@@ -127,12 +115,6 @@ def test_quant_model_lora(tinyllama_lora_files, model):
|
||||
max_tokens = 10
|
||||
|
||||
print("lora adapter created")
|
||||
output = do_sample(llm,
|
||||
tinyllama_lora_files,
|
||||
lora_id=0,
|
||||
max_tokens=max_tokens)
|
||||
expect_match(output, expected_no_lora_output)
|
||||
|
||||
print("lora 1")
|
||||
output = do_sample(llm,
|
||||
tinyllama_lora_files,
|
||||
@@ -140,13 +122,6 @@ def test_quant_model_lora(tinyllama_lora_files, model):
|
||||
max_tokens=max_tokens)
|
||||
expect_match(output, expected_lora_output)
|
||||
|
||||
print("no lora")
|
||||
output = do_sample(llm,
|
||||
tinyllama_lora_files,
|
||||
lora_id=0,
|
||||
max_tokens=max_tokens)
|
||||
expect_match(output, expected_no_lora_output)
|
||||
|
||||
print("lora 2")
|
||||
output = do_sample(llm,
|
||||
tinyllama_lora_files,
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.transformers_utils.tokenizer import get_lora_tokenizer
|
||||
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("tokenizer_group_type", [None, "ray"])
|
||||
async def test_tokenizer_group_lora(sql_lora_files, tokenizer_group_type):
|
||||
reference_tokenizer = AutoTokenizer.from_pretrained(sql_lora_files)
|
||||
tokenizer_group = TokenizerGroup(
|
||||
tokenizer_id="gpt2",
|
||||
enable_lora=True,
|
||||
max_num_seqs=1,
|
||||
max_loras=1,
|
||||
max_input_length=None,
|
||||
)
|
||||
lora_request = LoRARequest("1", 1, sql_lora_files)
|
||||
assert reference_tokenizer.encode("prompt") == tokenizer_group.encode(
|
||||
prompt="prompt", lora_request=lora_request)
|
||||
assert reference_tokenizer.encode(
|
||||
"prompt") == await tokenizer_group.encode_async(
|
||||
prompt="prompt", lora_request=lora_request)
|
||||
assert isinstance(tokenizer_group.get_lora_tokenizer(None),
|
||||
PreTrainedTokenizerBase)
|
||||
assert tokenizer_group.get_lora_tokenizer(
|
||||
None) == await tokenizer_group.get_lora_tokenizer_async(None)
|
||||
|
||||
assert isinstance(tokenizer_group.get_lora_tokenizer(lora_request),
|
||||
PreTrainedTokenizerBase)
|
||||
assert tokenizer_group.get_lora_tokenizer(
|
||||
lora_request) != tokenizer_group.get_lora_tokenizer(None)
|
||||
assert tokenizer_group.get_lora_tokenizer(
|
||||
lora_request) == await tokenizer_group.get_lora_tokenizer_async(
|
||||
lora_request)
|
||||
|
||||
|
||||
def test_get_lora_tokenizer(sql_lora_files, tmp_path):
|
||||
lora_request = None
|
||||
tokenizer = get_lora_tokenizer(lora_request)
|
||||
assert not tokenizer
|
||||
|
||||
lora_request = LoRARequest("1", 1, sql_lora_files)
|
||||
tokenizer = get_lora_tokenizer(lora_request)
|
||||
assert tokenizer.get_added_vocab()
|
||||
|
||||
lora_request = LoRARequest("1", 1, str(tmp_path))
|
||||
tokenizer = get_lora_tokenizer(lora_request)
|
||||
assert not tokenizer
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enable_lora", [True, False])
|
||||
@pytest.mark.parametrize("max_num_seqs", [1, 2])
|
||||
@pytest.mark.parametrize("max_loras", [1, 2])
|
||||
def test_lora_tokenizers(enable_lora, max_num_seqs, max_loras):
|
||||
tokenizer_group = TokenizerGroup(
|
||||
tokenizer_id="gpt2",
|
||||
enable_lora=enable_lora,
|
||||
max_num_seqs=max_num_seqs,
|
||||
max_loras=max_loras,
|
||||
max_input_length=None,
|
||||
)
|
||||
if enable_lora:
|
||||
assert tokenizer_group.lora_tokenizers.capacity == max(
|
||||
max_num_seqs, max_loras)
|
||||
else:
|
||||
assert tokenizer_group.lora_tokenizers.capacity == 0
|
||||
Reference in New Issue
Block a user