fix(quantization): Fix AWQ dequantize on Intel XPU and refactor AutoAWQ config (#42727)
Signed-off-by: Alex <alex.tech.lab@outlook.com> Signed-off-by: AlexHuang <jihuihuang@tencent.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude
Kunshang Ji
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
351c72d6e5
commit
8d4f54966c
@@ -43,16 +43,18 @@ MODEL_ARG_EXPTYPES = [
|
||||
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "gptq", "auto_gptq"),
|
||||
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "awq", "ERROR"),
|
||||
# AUTOAWQ
|
||||
# AutoAWQConfig.override_quantization_method() returns "auto_awq" for AWQ models
|
||||
# when user_quant is None, "awq", "awq_marlin", "marlin", or "auto_awq"
|
||||
(
|
||||
"TheBloke/OpenHermes-2.5-Mistral-7B-AWQ",
|
||||
None,
|
||||
"awq_marlin" if current_platform.is_cuda_alike() else "awq",
|
||||
"auto_awq",
|
||||
),
|
||||
("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "awq", "awq"),
|
||||
("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "awq", "auto_awq"),
|
||||
(
|
||||
"TheBloke/OpenHermes-2.5-Mistral-7B-AWQ",
|
||||
"marlin",
|
||||
"awq_marlin" if current_platform.is_cuda_alike() else "ERROR",
|
||||
"auto_awq" if current_platform.is_cuda_alike() else "ERROR",
|
||||
),
|
||||
("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "gptq", "ERROR"),
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user