[Kernel] Initial Activation Quantization Support (#4525)

Co-authored-by: Varun Sundar Rabindranath <varunsundar08@gmail.com>
Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
This commit is contained in:
Dipika Sikka
2024-05-23 21:29:18 +00:00
committed by GitHub
co-authored by Varun Sundar Rabindranath Varun Sundar Rabindranath
parent 5eda2ea02a
commit a1242324c9
17 changed files with 683 additions and 94 deletions
@@ -120,6 +120,13 @@ def get_quant_config(model_config: ModelConfig,
# Read the quantization config from the HF model config, if available.
hf_quant_config = getattr(model_config.hf_config, "quantization_config",
None)
if hf_quant_config is None:
compression_config = getattr(model_config.hf_config,
"compression_config", None)
if compression_config is not None:
hf_quant_config = compression_config.get("quantization_config",
None)
if hf_quant_config is not None:
return quant_cls.from_config(hf_quant_config)
model_name_or_path = model_config.model