Use quantization_config in hf config (#1695)

This commit is contained in:
Woosuk Kwon
2023-11-17 16:23:49 -08:00
committed by GitHub
parent e87557b069
commit bb00f66e19
3 changed files with 34 additions and 10 deletions
+1
View File
@@ -66,6 +66,7 @@ def get_model(model_config: ModelConfig) -> nn.Module:
if model_config.quantization is not None:
quant_config = get_quant_config(model_config.quantization,
model_config.model,
model_config.hf_config,
model_config.download_dir)
capability = torch.cuda.get_device_capability()
capability = capability[0] * 10 + capability[1]