[Quant][Kernel] Remove GPTQ Group/Dynamic Activation Ordering (#54809)

Signed-off-by: Roderick-Wu <roderickwu2003@gmail.com>
Signed-off-by: Roderick Wu <roderickwu2003@gmail.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Roderick Wu
2026-09-08 12:45:12 -07:00
committed by GitHub
co-authored by mgoin
parent 1a522b6949
commit 2c9d68f80b
99 changed files with 1182 additions and 3917 deletions
+73 -76
View File
@@ -24,42 +24,51 @@ MODELS: list[ModelWithQuantization]
if current_platform.is_rocm():
MODELS = [
ModelWithQuantization(
model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ", quantization="gptq"
model_path="JunHowie/Qwen3-0.6B-GPTQ-Int4",
quantization="gptq",
),
]
else:
MODELS = [
ModelWithQuantization(
model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ", quantization="awq"
model_path="Orion-zhen/Qwen3-0.6B-AWQ", quantization="awq"
),
ModelWithQuantization(
model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ", quantization="gptq"
model_path="JunHowie/Qwen3-0.6B-GPTQ-Int4",
quantization="gptq",
),
]
def do_sample(
llm: vllm.LLM, lora_path: str, lora_id: int, max_tokens: int = 256
llm: vllm.LLM, lora_path: str | None, lora_id: int, max_tokens: int = 256
) -> list[str]:
raw_prompts = [
"Give me an orange-ish brown color",
"Give me a neon pink color",
messages = [
[
{
"role": "system",
"content": "Follow the instructions to make animal noises",
},
{"role": "user", "content": "Make your favorite animal noise."},
],
[
{
"role": "system",
"content": "You are a cat. Reply only with your sound.",
},
{"role": "user", "content": "What do you say?"},
],
]
def format_prompt_tuples(prompt):
return f"<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n"
prompts = [format_prompt_tuples(p) for p in raw_prompts]
sampling_params = vllm.SamplingParams(
temperature=0, max_tokens=max_tokens, stop=["<|im_end|>"]
)
outputs = llm.generate(
prompts,
sampling_params = vllm.SamplingParams(temperature=0, max_tokens=max_tokens)
outputs = llm.chat(
messages,
sampling_params,
lora_request=LoRARequest(str(lora_id), lora_id, lora_path) if lora_id else None,
chat_template_kwargs={"enable_thinking": False},
lora_request=(
LoRARequest(str(lora_id), lora_id, lora_path) if lora_path else None
),
use_tqdm=False,
)
# Print the outputs.
generated_texts: list[str] = []
for output in outputs:
prompt = output.prompt
@@ -70,7 +79,7 @@ def do_sample(
@pytest.mark.parametrize("model", MODELS)
def test_quant_model_lora(tinyllama_lora_files, model):
def test_quant_model_lora(qwen3_meowing_lora_files, qwen3_woofing_lora_files, model):
llm = vllm.LLM(
model=model.model_path,
enable_lora=True,
@@ -79,75 +88,59 @@ def test_quant_model_lora(tinyllama_lora_files, model):
max_model_len=400,
gpu_memory_utilization=0.2, # avoid OOM
quantization=model.quantization,
trust_remote_code=True,
enable_chunked_prefill=True,
tokenizer=tinyllama_lora_files,
)
if model.quantization is None:
expected_lora_output = [
"#ff8050",
"#ff8080",
]
elif model.quantization == "awq":
expected_lora_output = [
"#f07700: A v",
"#f00000: A v",
]
elif model.quantization == "gptq":
expected_lora_output = [
"#f08800: This is",
"#f07788 \n#",
]
def expect_match(output, expected_output):
# HACK: GPTQ lora outputs are just incredibly unstable.
# Assert that the outputs changed.
if model.quantization == "gptq" and expected_output is expected_lora_output:
for i, o in enumerate(output):
assert o.startswith("#"), (
f"Expected example {i} to start with # but got {o}"
)
return
assert output == expected_output
loras = [
(
qwen3_meowing_lora_files,
1,
["Meow Meow Meow Meow Meow"] * 2,
),
(
qwen3_woofing_lora_files,
2,
["Woof Woof Woof Woof Woof"] * 2,
),
]
max_tokens = 10
print("lora adapter created")
print("lora 1")
output = do_sample(llm, tinyllama_lora_files, lora_id=1, max_tokens=max_tokens)
expect_match(output, expected_lora_output)
print("lora 2")
output = do_sample(llm, tinyllama_lora_files, lora_id=2, max_tokens=max_tokens)
expect_match(output, expected_lora_output)
print("removing lora")
del llm
cleanup_dist_env_and_memory()
try:
base_output = do_sample(llm, None, lora_id=0, max_tokens=max_tokens)
for lora_path, lora_id, expected_output in loras:
output = do_sample(
llm,
lora_path,
lora_id=lora_id,
max_tokens=max_tokens,
)
assert output == expected_output
assert output != base_output
finally:
del llm
cleanup_dist_env_and_memory()
@pytest.mark.parametrize("model", MODELS)
def test_quant_model_tp_equality(tinyllama_lora_files, num_gpus_available, model):
def test_quant_model_tp_equality(qwen3_meowing_lora_files, num_gpus_available, model):
if num_gpus_available < 2:
pytest.skip(f"Not enough GPUs for tensor parallelism {2}")
if model.quantization == "gptq":
pytest.skip("GPTQ lora outputs are just incredibly unstable")
llm_tp1 = vllm.LLM(
model=model.model_path,
enable_lora=True,
max_num_seqs=16,
max_loras=4,
max_model_len=400,
gpu_memory_utilization=0.2, # avoid OOM
quantization=model.quantization,
trust_remote_code=True,
enable_chunked_prefill=True,
)
output_tp1 = do_sample(llm_tp1, tinyllama_lora_files, lora_id=1)
del llm_tp1
cleanup_dist_env_and_memory()
try:
output_tp1 = do_sample(
llm_tp1, qwen3_meowing_lora_files, lora_id=1, max_tokens=10
)
finally:
del llm_tp1
cleanup_dist_env_and_memory()
llm_tp2 = vllm.LLM(
model=model.model_path,
@@ -155,13 +148,17 @@ def test_quant_model_tp_equality(tinyllama_lora_files, num_gpus_available, model
max_num_seqs=16,
max_loras=4,
tensor_parallel_size=2,
max_model_len=400,
gpu_memory_utilization=0.2, # avoid OOM
quantization=model.quantization,
enable_chunked_prefill=True,
)
output_tp2 = do_sample(llm_tp2, tinyllama_lora_files, lora_id=1)
del llm_tp2
cleanup_dist_env_and_memory()
try:
output_tp2 = do_sample(
llm_tp2, qwen3_meowing_lora_files, lora_id=1, max_tokens=10
)
finally:
del llm_tp2
cleanup_dist_env_and_memory()
assert output_tp1 == output_tp2