[Quant][Kernel] Remove GPTQ Group/Dynamic Activation Ordering (#54809)
Signed-off-by: Roderick-Wu <roderickwu2003@gmail.com> Signed-off-by: Roderick Wu <roderickwu2003@gmail.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -24,42 +24,51 @@ MODELS: list[ModelWithQuantization]
|
||||
if current_platform.is_rocm():
|
||||
MODELS = [
|
||||
ModelWithQuantization(
|
||||
model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ", quantization="gptq"
|
||||
model_path="JunHowie/Qwen3-0.6B-GPTQ-Int4",
|
||||
quantization="gptq",
|
||||
),
|
||||
]
|
||||
else:
|
||||
MODELS = [
|
||||
ModelWithQuantization(
|
||||
model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ", quantization="awq"
|
||||
model_path="Orion-zhen/Qwen3-0.6B-AWQ", quantization="awq"
|
||||
),
|
||||
ModelWithQuantization(
|
||||
model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ", quantization="gptq"
|
||||
model_path="JunHowie/Qwen3-0.6B-GPTQ-Int4",
|
||||
quantization="gptq",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def do_sample(
|
||||
llm: vllm.LLM, lora_path: str, lora_id: int, max_tokens: int = 256
|
||||
llm: vllm.LLM, lora_path: str | None, lora_id: int, max_tokens: int = 256
|
||||
) -> list[str]:
|
||||
raw_prompts = [
|
||||
"Give me an orange-ish brown color",
|
||||
"Give me a neon pink color",
|
||||
messages = [
|
||||
[
|
||||
{
|
||||
"role": "system",
|
||||
"content": "Follow the instructions to make animal noises",
|
||||
},
|
||||
{"role": "user", "content": "Make your favorite animal noise."},
|
||||
],
|
||||
[
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a cat. Reply only with your sound.",
|
||||
},
|
||||
{"role": "user", "content": "What do you say?"},
|
||||
],
|
||||
]
|
||||
|
||||
def format_prompt_tuples(prompt):
|
||||
return f"<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n"
|
||||
|
||||
prompts = [format_prompt_tuples(p) for p in raw_prompts]
|
||||
|
||||
sampling_params = vllm.SamplingParams(
|
||||
temperature=0, max_tokens=max_tokens, stop=["<|im_end|>"]
|
||||
)
|
||||
outputs = llm.generate(
|
||||
prompts,
|
||||
sampling_params = vllm.SamplingParams(temperature=0, max_tokens=max_tokens)
|
||||
outputs = llm.chat(
|
||||
messages,
|
||||
sampling_params,
|
||||
lora_request=LoRARequest(str(lora_id), lora_id, lora_path) if lora_id else None,
|
||||
chat_template_kwargs={"enable_thinking": False},
|
||||
lora_request=(
|
||||
LoRARequest(str(lora_id), lora_id, lora_path) if lora_path else None
|
||||
),
|
||||
use_tqdm=False,
|
||||
)
|
||||
# Print the outputs.
|
||||
generated_texts: list[str] = []
|
||||
for output in outputs:
|
||||
prompt = output.prompt
|
||||
@@ -70,7 +79,7 @@ def do_sample(
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
def test_quant_model_lora(tinyllama_lora_files, model):
|
||||
def test_quant_model_lora(qwen3_meowing_lora_files, qwen3_woofing_lora_files, model):
|
||||
llm = vllm.LLM(
|
||||
model=model.model_path,
|
||||
enable_lora=True,
|
||||
@@ -79,75 +88,59 @@ def test_quant_model_lora(tinyllama_lora_files, model):
|
||||
max_model_len=400,
|
||||
gpu_memory_utilization=0.2, # avoid OOM
|
||||
quantization=model.quantization,
|
||||
trust_remote_code=True,
|
||||
enable_chunked_prefill=True,
|
||||
tokenizer=tinyllama_lora_files,
|
||||
)
|
||||
|
||||
if model.quantization is None:
|
||||
expected_lora_output = [
|
||||
"#ff8050",
|
||||
"#ff8080",
|
||||
]
|
||||
elif model.quantization == "awq":
|
||||
expected_lora_output = [
|
||||
"#f07700: A v",
|
||||
"#f00000: A v",
|
||||
]
|
||||
elif model.quantization == "gptq":
|
||||
expected_lora_output = [
|
||||
"#f08800: This is",
|
||||
"#f07788 \n#",
|
||||
]
|
||||
|
||||
def expect_match(output, expected_output):
|
||||
# HACK: GPTQ lora outputs are just incredibly unstable.
|
||||
# Assert that the outputs changed.
|
||||
if model.quantization == "gptq" and expected_output is expected_lora_output:
|
||||
for i, o in enumerate(output):
|
||||
assert o.startswith("#"), (
|
||||
f"Expected example {i} to start with # but got {o}"
|
||||
)
|
||||
return
|
||||
assert output == expected_output
|
||||
|
||||
loras = [
|
||||
(
|
||||
qwen3_meowing_lora_files,
|
||||
1,
|
||||
["Meow Meow Meow Meow Meow"] * 2,
|
||||
),
|
||||
(
|
||||
qwen3_woofing_lora_files,
|
||||
2,
|
||||
["Woof Woof Woof Woof Woof"] * 2,
|
||||
),
|
||||
]
|
||||
max_tokens = 10
|
||||
|
||||
print("lora adapter created")
|
||||
print("lora 1")
|
||||
output = do_sample(llm, tinyllama_lora_files, lora_id=1, max_tokens=max_tokens)
|
||||
expect_match(output, expected_lora_output)
|
||||
|
||||
print("lora 2")
|
||||
output = do_sample(llm, tinyllama_lora_files, lora_id=2, max_tokens=max_tokens)
|
||||
expect_match(output, expected_lora_output)
|
||||
|
||||
print("removing lora")
|
||||
|
||||
del llm
|
||||
cleanup_dist_env_and_memory()
|
||||
try:
|
||||
base_output = do_sample(llm, None, lora_id=0, max_tokens=max_tokens)
|
||||
for lora_path, lora_id, expected_output in loras:
|
||||
output = do_sample(
|
||||
llm,
|
||||
lora_path,
|
||||
lora_id=lora_id,
|
||||
max_tokens=max_tokens,
|
||||
)
|
||||
assert output == expected_output
|
||||
assert output != base_output
|
||||
finally:
|
||||
del llm
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
def test_quant_model_tp_equality(tinyllama_lora_files, num_gpus_available, model):
|
||||
def test_quant_model_tp_equality(qwen3_meowing_lora_files, num_gpus_available, model):
|
||||
if num_gpus_available < 2:
|
||||
pytest.skip(f"Not enough GPUs for tensor parallelism {2}")
|
||||
if model.quantization == "gptq":
|
||||
pytest.skip("GPTQ lora outputs are just incredibly unstable")
|
||||
llm_tp1 = vllm.LLM(
|
||||
model=model.model_path,
|
||||
enable_lora=True,
|
||||
max_num_seqs=16,
|
||||
max_loras=4,
|
||||
max_model_len=400,
|
||||
gpu_memory_utilization=0.2, # avoid OOM
|
||||
quantization=model.quantization,
|
||||
trust_remote_code=True,
|
||||
enable_chunked_prefill=True,
|
||||
)
|
||||
output_tp1 = do_sample(llm_tp1, tinyllama_lora_files, lora_id=1)
|
||||
|
||||
del llm_tp1
|
||||
cleanup_dist_env_and_memory()
|
||||
try:
|
||||
output_tp1 = do_sample(
|
||||
llm_tp1, qwen3_meowing_lora_files, lora_id=1, max_tokens=10
|
||||
)
|
||||
finally:
|
||||
del llm_tp1
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
llm_tp2 = vllm.LLM(
|
||||
model=model.model_path,
|
||||
@@ -155,13 +148,17 @@ def test_quant_model_tp_equality(tinyllama_lora_files, num_gpus_available, model
|
||||
max_num_seqs=16,
|
||||
max_loras=4,
|
||||
tensor_parallel_size=2,
|
||||
max_model_len=400,
|
||||
gpu_memory_utilization=0.2, # avoid OOM
|
||||
quantization=model.quantization,
|
||||
enable_chunked_prefill=True,
|
||||
)
|
||||
output_tp2 = do_sample(llm_tp2, tinyllama_lora_files, lora_id=1)
|
||||
|
||||
del llm_tp2
|
||||
cleanup_dist_env_and_memory()
|
||||
try:
|
||||
output_tp2 = do_sample(
|
||||
llm_tp2, qwen3_meowing_lora_files, lora_id=1, max_tokens=10
|
||||
)
|
||||
finally:
|
||||
del llm_tp2
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
assert output_tp1 == output_tp2
|
||||
|
||||
Reference in New Issue
Block a user